3 namespace dokuwiki\Search
;
5 use dokuwiki\Utf8\Asian
;
6 use dokuwiki\Utf8\Clean
;
7 use dokuwiki\Utf8\PhpString
;
8 use dokuwiki\Extension\Event
;
11 * Class that encapsulates operations on the indexer database.
13 * @author Tom N Harris <tnharris@whoopdedo.org>
18 * @var array $pidCache Cache for getPID()
20 protected $pidCache = [];
23 * Adds the contents of a page to the fulltext index
25 * The added text replaces previous words for the same page.
26 * An empty value erases the page.
28 * @param string $page a page name
29 * @param string $text the body of the page
30 * @return string|boolean the function completed successfully
32 * @author Tom N Harris <tnharris@whoopdedo.org>
33 * @author Andreas Gohr <andi@splitbrain.org>
35 public function addPageWords($page, $text)
40 // load known documents
41 $pid = $this->getPIDNoLock($page);
48 // get word usage in page
49 $words = $this->getPageWords($text);
50 if ($words === false) {
56 foreach (array_keys($words) as $wlen) {
57 $index = $this->getIndex('i', $wlen);
58 foreach ($words[$wlen] as $wid => $freq) {
59 $idx = ($wid < count($index)) ?
$index[$wid] : '';
60 $index[$wid] = $this->updateTuple($idx, $pid, $freq);
61 $pagewords[] = "$wlen*$wid";
63 if (!$this->saveIndex('i', $wlen, $index)) {
70 // Remove obsolete index entries
71 $pageword_idx = $this->getIndexKey('pageword', '', $pid);
72 if ($pageword_idx !== '') {
73 $oldwords = explode(':', $pageword_idx);
74 $delwords = array_diff($oldwords, $pagewords);
76 foreach ($delwords as $word) {
78 [$wlen, $wid] = explode('*', $word);
80 $upwords[$wlen][] = $wid;
83 foreach ($upwords as $wlen => $widx) {
84 $index = $this->getIndex('i', $wlen);
85 foreach ($widx as $wid) {
86 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
88 $this->saveIndex('i', $wlen, $index);
91 // Save the reverse index
92 $pageword_idx = implode(':', $pagewords);
93 if (!$this->saveIndexKey('pageword', '', $pid, $pageword_idx)) {
103 * Split the words in a page and add them to the index.
105 * @param string $text content of the page
106 * @return array list of word IDs and number of times used
108 * @author Andreas Gohr <andi@splitbrain.org>
109 * @author Christopher Smith <chris@jalakai.co.uk>
110 * @author Tom N Harris <tnharris@whoopdedo.org>
112 protected function getPageWords($text)
115 $tokens = $this->tokenizer($text);
116 $tokens = array_count_values($tokens); // count the frequency of each token
119 foreach ($tokens as $w => $c) {
121 if (isset($words[$l])) {
122 $words[$l][$w] = $c +
($words[$l][$w] ??
0);
124 $words[$l] = [$w => $c];
128 // arrive here with $words = array(wordlen => array(word => frequency))
129 $index = []; //resulting index
130 foreach (array_keys($words) as $wlen) {
131 $word_idx = $this->getIndex('w', $wlen);
132 $word_idx_modified = false;
133 foreach ($words[$wlen] as $word => $freq) {
134 $word = (string)$word;
135 $wid = array_search($word, $word_idx, true);
136 if ($wid === false) {
137 $wid = count($word_idx);
139 $word_idx_modified = true;
141 if (!isset($index[$wlen]))
143 $index[$wlen][$wid] = $freq;
145 // save back the word index
146 if ($word_idx_modified && !$this->saveIndex('w', $wlen, $word_idx))
154 * Add/update keys to/of the metadata index.
156 * Adding new keys does not remove other keys for the page.
157 * An empty value will erase the key.
158 * The $key parameter can be an array to add multiple keys. $value will
159 * not be used if $key is an array.
161 * @param string $page a page name
162 * @param mixed $key a key string or array of key=>value pairs
163 * @param mixed $value the value or list of values
164 * @return boolean|string the function completed successfully
166 * @author Tom N Harris <tnharris@whoopdedo.org>
167 * @author Michael Hamann <michael@content-space.de>
169 public function addMetaKeys($page, $key, $value = null)
171 if (!is_array($key)) {
172 $key = [$key => $value];
173 } elseif (!is_null($value)) {
174 // $key is array, but $value is not null
175 trigger_error("array passed to addMetaKeys but value is not null", E_USER_WARNING
);
181 // load known documents
182 $pid = $this->getPIDNoLock($page);
183 if ($pid === false) {
188 // Special handling for titles so the index file is simpler
189 if (isset($key['title'])) {
190 $value = $key['title'];
191 if (is_array($value)) {
194 $this->saveIndexKey('title', '', $pid, $value);
195 unset($key['title']);
198 foreach ($key as $name => $values) {
199 $metaname = idx_cleanName($name);
200 $this->addIndexKey('metadata', '', $metaname);
201 $metaidx = $this->getIndex($metaname . '_i', '');
202 $metawords = $this->getIndex($metaname . '_w', '');
205 if (!is_array($values)) $values = [$values];
207 $val_idx = $this->getIndexKey($metaname . '_p', '', $pid);
208 if ($val_idx !== '') {
209 $val_idx = explode(':', $val_idx);
210 // -1 means remove, 0 keep, 1 add
211 $val_idx = array_combine($val_idx, array_fill(0, count($val_idx), -1));
216 foreach ($values as $val) {
219 $id = array_search($val, $metawords, true);
221 // didn't find $val, so we'll add it to the end of metawords and create a placeholder in metaidx
222 $id = count($metawords);
223 $metawords[$id] = $val;
227 // test if value is already in the index
228 if (isset($val_idx[$id]) && $val_idx[$id] <= 0) {
230 } else { // else add it
237 $this->saveIndex($metaname . '_w', '', $metawords);
239 $vals_changed = false;
240 foreach ($val_idx as $id => $action) {
242 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 0);
243 $vals_changed = true;
244 unset($val_idx[$id]);
245 } elseif ($action == 1) {
246 $metaidx[$id] = $this->updateTuple($metaidx[$id], $pid, 1);
247 $vals_changed = true;
252 $this->saveIndex($metaname . '_i', '', $metaidx);
253 $val_idx = implode(':', array_keys($val_idx));
254 $this->saveIndexKey($metaname . '_p', '', $pid, $val_idx);
266 * Rename a page in the search index without changing the indexed content. This function doesn't check if the
267 * old or new name exists in the filesystem. It returns an error if the old page isn't in the page list of the
268 * indexer and it deletes all previously indexed content of the new page.
270 * @param string $oldpage The old page name
271 * @param string $newpage The new page name
272 * @return string|bool If the page was successfully renamed, can be a message in the case of an error
274 public function renamePage($oldpage, $newpage)
276 if (!$this->lock()) return 'locked';
278 $pages = $this->getPages();
280 $id = array_search($oldpage, $pages, true);
283 return 'page is not in index';
286 $new_id = array_search($newpage, $pages, true);
287 if ($new_id !== false) {
288 // make sure the page is not in the index anymore
289 if (!$this->deletePageNoLock($newpage)) {
293 $pages[$new_id] = 'deleted:' . time() . random_int(0, 9999);
296 $pages[$id] = $newpage;
299 if (!$this->saveIndex('page', '', $pages)) {
304 // reset the pid cache
305 $this->pidCache
= [];
312 * Renames a meta value in the index. This doesn't change the meta value in the pages, it assumes that all pages
315 * @param string $key The metadata key of which a value shall be changed
316 * @param string $oldvalue The old value that shall be renamed
317 * @param string $newvalue The new value to which the old value shall be renamed, if exists values will be merged
318 * @return bool|string If renaming the value has been successful, false or error message on error.
320 public function renameMetaValue($key, $oldvalue, $newvalue)
322 if (!$this->lock()) return 'locked';
324 // change the relation references index
325 $metavalues = $this->getIndex($key, '_w');
326 $oldid = array_search($oldvalue, $metavalues, true);
327 if ($oldid !== false) {
328 $newid = array_search($newvalue, $metavalues, true);
329 if ($newid !== false) {
333 // okay, now we have two entries for the same value. we need to merge them.
334 $indexline = $this->getIndexKey($key . '_i', '', $oldid);
335 if ($indexline != '') {
336 $newindexline = $this->getIndexKey($key . '_i', '', $newid);
337 $pagekeys = $this->getIndex($key . '_p', '');
338 $parts = explode(':', $indexline);
339 foreach ($parts as $part) {
340 [$id, $count] = explode('*', $part);
341 $newindexline = $this->updateTuple($newindexline, $id, $count);
343 $keyline = explode(':', $pagekeys[$id]);
344 // remove old meta value
345 $keyline = array_diff($keyline, [$oldid]);
346 // add new meta value when not already present
347 if (!in_array($newid, $keyline)) {
350 $pagekeys[$id] = implode(':', $keyline);
352 $this->saveIndex($key . '_p', '', $pagekeys);
354 $this->saveIndexKey($key . '_i', '', $oldid, '');
355 $this->saveIndexKey($key . '_i', '', $newid, $newindexline);
358 $metavalues[$oldid] = $newvalue;
359 if (!$this->saveIndex($key . '_w', '', $metavalues)) {
371 * Remove a page from the index
373 * Erases entries in all known indexes.
375 * @param string $page a page name
376 * @return string|boolean the function completed successfully
378 * @author Tom N Harris <tnharris@whoopdedo.org>
380 public function deletePage($page)
385 $result = $this->deletePageNoLock($page);
393 * Remove a page from the index without locking the index, only use this function if the index is already locked
395 * Erases entries in all known indexes.
397 * @param string $page a page name
398 * @return boolean the function completed successfully
400 * @author Tom N Harris <tnharris@whoopdedo.org>
402 protected function deletePageNoLock($page)
404 // load known documents
405 $pid = $this->getPIDNoLock($page);
406 if ($pid === false) {
410 // Remove obsolete index entries
411 $pageword_idx = $this->getIndexKey('pageword', '', $pid);
412 if ($pageword_idx !== '') {
413 $delwords = explode(':', $pageword_idx);
415 foreach ($delwords as $word) {
417 [$wlen, $wid] = explode('*', $word);
419 $upwords[$wlen][] = $wid;
422 foreach ($upwords as $wlen => $widx) {
423 $index = $this->getIndex('i', $wlen);
424 foreach ($widx as $wid) {
425 $index[$wid] = $this->updateTuple($index[$wid], $pid, 0);
427 $this->saveIndex('i', $wlen, $index);
430 // Save the reverse index
431 if (!$this->saveIndexKey('pageword', '', $pid, "")) {
435 $this->saveIndexKey('title', '', $pid, "");
436 $keyidx = $this->getIndex('metadata', '');
437 foreach ($keyidx as $metaname) {
438 $val_idx = explode(':', $this->getIndexKey($metaname . '_p', '', $pid));
439 $meta_idx = $this->getIndex($metaname . '_i', '');
440 foreach ($val_idx as $id) {
441 if ($id === '') continue;
442 $meta_idx[$id] = $this->updateTuple($meta_idx[$id], $pid, 0);
444 $this->saveIndex($metaname . '_i', '', $meta_idx);
445 $this->saveIndexKey($metaname . '_p', '', $pid, '');
452 * Clear the whole index
454 * @return bool If the index has been cleared successfully
456 public function clear()
460 if (!$this->lock()) return false;
462 @unlink
($conf['indexdir'] . '/page.idx');
463 @unlink
($conf['indexdir'] . '/title.idx');
464 @unlink
($conf['indexdir'] . '/pageword.idx');
465 @unlink
($conf['indexdir'] . '/metadata.idx');
466 $dir = @opendir
($conf['indexdir']);
467 if ($dir !== false) {
468 while (($f = readdir($dir)) !== false) {
470 str_ends_with($f, '.idx') &&
471 (str_starts_with($f, 'i') ||
472 str_starts_with($f, 'w') ||
473 str_ends_with($f, '_w.idx') ||
474 str_ends_with($f, '_i.idx') ||
475 str_ends_with($f, '_p.idx'))
477 @unlink
($conf['indexdir'] . "/$f");
480 @unlink
($conf['indexdir'] . '/lengths.idx');
482 // clear the pid cache
483 $this->pidCache
= [];
490 * Split the text into words for fulltext search
492 * TODO: does this also need &$stopwords ?
494 * @triggers INDEXER_TEXT_PREPARE
495 * This event allows plugins to modify the text before it gets tokenized.
496 * Plugins intercepting this event should also intercept INDEX_VERSION_GET
498 * @param string $text plain text
499 * @param boolean $wc are wildcards allowed?
500 * @return array list of words in the text
502 * @author Tom N Harris <tnharris@whoopdedo.org>
503 * @author Andreas Gohr <andi@splitbrain.org>
505 public function tokenizer($text, $wc = false)
507 $wc = ($wc) ?
'' : '\*';
508 $stopwords =& idx_get_stopwords();
510 // prepare the text to be tokenized
511 $evt = new Event('INDEXER_TEXT_PREPARE', $text);
512 if ($evt->advise_before(true)) {
513 if (preg_match('/[^0-9A-Za-z ]/u', $text)) {
514 $text = Asian
::separateAsianWords($text);
517 $evt->advise_after();
522 ["\r" => ' ', "\n" => ' ', "\t" => ' ', "\xC2\xAD" => '']
524 if (preg_match('/[^0-9A-Za-z ]/u', $text))
525 $text = Clean
::stripspecials($text, ' ', '\._\-:' . $wc);
527 $wordlist = explode(' ', $text);
528 foreach ($wordlist as $i => $word) {
529 $wordlist[$i] = (preg_match('/[^0-9A-Za-z]/u', $word)) ?
530 PhpString
::strtolower($word) : strtolower($word);
533 foreach ($wordlist as $i => $word) {
535 (!is_numeric($word) && strlen($word) < IDX_MINWORDLENGTH
)
536 ||
in_array($word, $stopwords, true)
538 unset($wordlist[$i]);
540 return array_values($wordlist);
544 * Get the numeric PID of a page
546 * @param string $page The page to get the PID for
547 * @return bool|int The page id on success, false on error
549 public function getPID($page)
551 // return PID without locking when it is in the cache
552 if (isset($this->pidCache
[$page])) return $this->pidCache
[$page];
557 // load known documents
558 $pid = $this->getPIDNoLock($page);
559 if ($pid === false) {
569 * Get the numeric PID of a page without locking the index.
570 * Only use this function when the index is already locked.
572 * @param string $page The page to get the PID for
573 * @return bool|int The page id on success, false on error
575 protected function getPIDNoLock($page)
577 // avoid expensive addIndexKey operation for the most recently requested pages by using a cache
578 if (isset($this->pidCache
[$page])) return $this->pidCache
[$page];
579 $pid = $this->addIndexKey('page', '', $page);
580 // limit cache to 10 entries by discarding the oldest element as in DokuWiki usually only the most recently
581 // added item will be requested again
582 if (count($this->pidCache
) > 10) array_shift($this->pidCache
);
583 $this->pidCache
[$page] = $pid;
588 * Get the page id of a numeric PID
590 * @param int $pid The PID to get the page id for
591 * @return string The page id
593 public function getPageFromPID($pid)
595 return $this->getIndexKey('page', '', $pid);
599 * Find pages in the fulltext index containing the words,
601 * The search words must be pre-tokenized, meaning only letters and
602 * numbers with an optional wildcard
604 * The returned array will have the original tokens as key. The values
605 * in the returned list is an array with the page names as keys and the
606 * number of times that token appears on the page as value.
608 * @param array $tokens list of words to search for
609 * @return array list of page names with usage counts
611 * @author Tom N Harris <tnharris@whoopdedo.org>
612 * @author Andreas Gohr <andi@splitbrain.org>
614 public function lookup(&$tokens)
617 $wids = $this->getIndexWords($tokens, $result);
618 if (empty($wids)) return [];
619 // load known words and documents
620 $page_idx = $this->getIndex('page', '');
622 foreach (array_keys($wids) as $wlen) {
623 $wids[$wlen] = array_unique($wids[$wlen]);
624 $index = $this->getIndex('i', $wlen);
625 foreach ($wids[$wlen] as $ixid) {
626 if ($ixid < count($index))
627 $docs["$wlen*$ixid"] = $this->parseTuples($page_idx, $index[$ixid]);
630 // merge found pages into final result array
632 foreach ($result as $word => $res) {
634 foreach ($res as $wid) {
635 // handle the case when ($ixid < count($index)) has been false
636 // and thus $docs[$wid] hasn't been set.
637 if (!isset($docs[$wid])) continue;
638 $hits = &$docs[$wid];
639 foreach ($hits as $hitkey => $hitcnt) {
640 // make sure the document still exists
641 if (!page_exists($hitkey, '', false)) continue;
642 if (!isset($final[$word][$hitkey]))
643 $final[$word][$hitkey] = $hitcnt;
644 else $final[$word][$hitkey] +
= $hitcnt;
652 * Find pages containing a metadata key.
654 * The metadata values are compared as case-sensitive strings. Pass a
655 * callback function that returns true or false to use a different
656 * comparison function. The function will be called with the $value being
657 * searched for as the first argument, and the word in the index as the
658 * second argument. The function preg_match can be used directly if the
659 * values are regexes.
661 * @param string $key name of the metadata key to look for
662 * @param string $value search term to look for, must be a string or array of strings
663 * @param callback $func comparison function
664 * @return array lists with page names, keys are query values if $value is array
666 * @author Tom N Harris <tnharris@whoopdedo.org>
667 * @author Michael Hamann <michael@content-space.de>
669 public function lookupKey($key, &$value, $func = null)
671 if (!is_array($value))
672 $value_array = [$value];
673 else $value_array =& $value;
675 // the matching ids for the provided value(s)
678 $metaname = idx_cleanName($key);
680 // get all words in order to search the matching ids
681 if ($key == 'title') {
682 $words = $this->getIndex('title', '');
684 $words = $this->getIndex($metaname . '_w', '');
687 if (!is_null($func)) {
688 foreach ($value_array as $val) {
689 foreach ($words as $i => $word) {
690 if (call_user_func_array($func, [$val, $word]))
691 $value_ids[$i][] = $val;
695 foreach ($value_array as $val) {
699 // check for wildcards
700 if (str_starts_with($xval, '*')) {
701 $xval = substr($xval, 1);
704 if (str_ends_with($xval, '*')) {
705 $xval = substr($xval, 0, -1);
708 if (!$caret ||
!$dollar) {
709 $re = $caret . preg_quote($xval, '/') . $dollar;
710 foreach (array_keys(preg_grep('/' . $re . '/', $words)) as $i)
711 $value_ids[$i][] = $val;
712 } elseif (($i = array_search($val, $words, true)) !== false) {
713 $value_ids[$i][] = $val;
718 unset($words); // free the used memory
720 // initialize the result so it won't be null
722 foreach ($value_array as $val) {
726 $page_idx = $this->getIndex('page', '');
728 // Special handling for titles
729 if ($key == 'title') {
730 foreach ($value_ids as $pid => $val_list) {
731 $page = $page_idx[$pid];
732 foreach ($val_list as $val) {
733 $result[$val][] = $page;
737 // load all lines and pages so the used lines can be taken and matched with the pages
738 $lines = $this->getIndex($metaname . '_i', '');
740 foreach ($value_ids as $value_id => $val_list) {
741 // parse the tuples of the form page_id*1:page2_id*1 and so on, return value
742 // is an array with page_id => 1, page2_id => 1 etc. so take the keys only
743 $pages = array_keys($this->parseTuples($page_idx, $lines[$value_id]));
744 foreach ($val_list as $val) {
745 $result[$val] = [...$result[$val], ...$pages];
749 if (!is_array($value)) $result = $result[$value];
754 * Find the index ID of each search term.
756 * The query terms should only contain valid characters, with a '*' at
757 * either the beginning or end of the word (or both).
758 * The $result parameter can be used to merge the index locations with
759 * the appropriate query term.
761 * @param array $words The query terms.
762 * @param array $result Set to word => array("length*id" ...)
763 * @return array Set to length => array(id ...)
765 * @author Tom N Harris <tnharris@whoopdedo.org>
767 protected function getIndexWords(&$words, &$result)
772 foreach ($words as $word) {
777 $wlen = wordlen($word);
779 // check for wildcards
780 if (str_starts_with($xword, '*')) {
781 $xword = substr($xword, 1);
785 if (str_ends_with($xword, '*')) {
786 $xword = substr($xword, 0, -1);
790 if ($wlen < IDX_MINWORDLENGTH
&& $caret && $dollar && !is_numeric($xword))
792 if (!isset($tokens[$xword]))
793 $tokenlength[$wlen][] = $xword;
794 if (!$caret ||
!$dollar) {
795 $re = $caret . preg_quote($xword, '/') . $dollar;
796 $tokens[$xword][] = [$word, '/' . $re . '/'];
797 if (!isset($tokenwild[$xword]))
798 $tokenwild[$xword] = $wlen;
800 $tokens[$xword][] = [$word, null];
804 // $tokens = array( base word => array( [ query term , regexp ] ... ) ... )
805 // $tokenlength = array( base word length => base word ... )
806 // $tokenwild = array( base word => base word length ... )
807 $length_filter = $tokenwild === [] ?
$tokenlength : min(array_keys($tokenlength));
808 $indexes_known = $this->indexLengths($length_filter);
809 if ($tokenwild !== []) sort($indexes_known);
812 foreach ($indexes_known as $ixlen) {
813 $word_idx = $this->getIndex('w', $ixlen);
814 // handle exact search
815 if (isset($tokenlength[$ixlen])) {
816 foreach ($tokenlength[$ixlen] as $xword) {
817 $wid = array_search($xword, $word_idx, true);
818 if ($wid !== false) {
819 $wids[$ixlen][] = $wid;
820 foreach ($tokens[$xword] as $w)
821 $result[$w[0]][] = "$ixlen*$wid";
825 // handle wildcard search
826 foreach ($tokenwild as $xword => $wlen) {
827 if ($wlen >= $ixlen) break;
828 foreach ($tokens[$xword] as $w) {
829 if (is_null($w[1])) continue;
830 foreach (array_keys(preg_grep($w[1], $word_idx)) as $wid) {
831 $wids[$ixlen][] = $wid;
832 $result[$w[0]][] = "$ixlen*$wid";
841 * Return a list of all pages
842 * Warning: pages may not exist!
844 * @param string $key list only pages containing the metadata key (optional)
845 * @return array list of page names
847 * @author Tom N Harris <tnharris@whoopdedo.org>
849 public function getPages($key = null)
851 $page_idx = $this->getIndex('page', '');
852 if (is_null($key)) return $page_idx;
854 $metaname = idx_cleanName($key);
856 // Special handling for titles
857 if ($key == 'title') {
858 $title_idx = $this->getIndex('title', '');
859 array_splice($page_idx, count($title_idx));
860 foreach ($title_idx as $i => $title)
861 if ($title === "") unset($page_idx[$i]);
862 return array_values($page_idx);
866 $lines = $this->getIndex($metaname . '_i', '');
867 foreach ($lines as $line) {
868 $pages = array_merge($pages, $this->parseTuples($page_idx, $line));
870 return array_keys($pages);
874 * Return a list of words sorted by number of times used
876 * @param int $min bottom frequency threshold
877 * @param int $max upper frequency limit. No limit if $max<$min
878 * @param int $minlen minimum length of words to count
879 * @param string $key metadata key to list. Uses the fulltext index if not given
880 * @return array list of words as the keys and frequency as values
882 * @author Tom N Harris <tnharris@whoopdedo.org>
884 public function histogram($min = 1, $max = 0, $minlen = 3, $key = null)
893 if ($key == 'title') {
894 $index = $this->getIndex('title', '');
895 $index = array_count_values($index);
896 foreach ($index as $val => $cnt) {
897 if ($cnt >= $min && (!$max ||
$cnt <= $max) && strlen($val) >= $minlen)
898 $result[$val] = $cnt;
900 } elseif (!is_null($key)) {
901 $metaname = idx_cleanName($key);
902 $index = $this->getIndex($metaname . '_i', '');
904 foreach ($index as $wid => $line) {
905 $freq = $this->countTuples($line);
906 if ($freq >= $min && (!$max ||
$freq <= $max))
907 $val_idx[$wid] = $freq;
909 if (!empty($val_idx)) {
910 $words = $this->getIndex($metaname . '_w', '');
911 foreach ($val_idx as $wid => $freq) {
912 if (strlen($words[$wid]) >= $minlen)
913 $result[$words[$wid]] = $freq;
917 $lengths = idx_listIndexLengths();
918 foreach ($lengths as $length) {
919 if ($length < $minlen) continue;
920 $index = $this->getIndex('i', $length);
922 foreach ($index as $wid => $line) {
923 $freq = $this->countTuples($line);
924 if ($freq >= $min && (!$max ||
$freq <= $max)) {
926 $words = $this->getIndex('w', $length);
927 $result[$words[$wid]] = $freq;
940 * @author Tom N Harris <tnharris@whoopdedo.org>
942 * @return bool|string
944 protected function lock()
949 $lock = $conf['lockdir'] . '/_indexer.lock';
950 while (!@mkdir
($lock)) {
952 if (is_dir($lock) && time() - @filemtime
($lock) > 60 * 5) {
953 // looks like a stale lock - remove it
954 if (!@rmdir
($lock)) {
955 $status = "removing the stale lock failed";
958 $status = "stale lock removed";
960 } elseif ($run++
== 1000) {
961 // we waited 5 seconds for that lock
965 if ($conf['dperm']) {
966 chmod($lock, $conf['dperm']);
972 * Release the indexer lock.
974 * @author Tom N Harris <tnharris@whoopdedo.org>
978 protected function unlock()
981 @rmdir
($conf['lockdir'] . '/_indexer.lock');
986 * Retrieve the entire index.
988 * The $suffix argument is for an index that is split into
989 * multiple parts. Different index files should use different
992 * @param string $idx name of the index
993 * @param string $suffix subpart identifier
994 * @return array list of lines without CR or LF
996 * @author Tom N Harris <tnharris@whoopdedo.org>
998 protected function getIndex($idx, $suffix)
1001 $fn = $conf['indexdir'] . '/' . $idx . $suffix . '.idx';
1002 if (!file_exists($fn)) return [];
1003 return file($fn, FILE_IGNORE_NEW_LINES
);
1007 * Replace the contents of the index with an array.
1009 * @param string $idx name of the index
1010 * @param string $suffix subpart identifier
1011 * @param array $lines list of lines without LF
1012 * @return bool If saving succeeded
1014 * @author Tom N Harris <tnharris@whoopdedo.org>
1016 protected function saveIndex($idx, $suffix, &$lines)
1019 $fn = $conf['indexdir'] . '/' . $idx . $suffix;
1020 $fh = @fopen
($fn . '.tmp', 'w');
1021 if (!$fh) return false;
1022 fwrite($fh, implode("\n", $lines));
1027 chmod($fn . '.tmp', $conf['fperm']);
1028 io_rename($fn . '.tmp', $fn . '.idx');
1033 * Retrieve a line from the index.
1035 * @param string $idx name of the index
1036 * @param string $suffix subpart identifier
1037 * @param int $id the line number
1038 * @return string a line with trailing whitespace removed
1040 * @author Tom N Harris <tnharris@whoopdedo.org>
1042 protected function getIndexKey($idx, $suffix, $id)
1045 $fn = $conf['indexdir'] . '/' . $idx . $suffix . '.idx';
1046 if (!file_exists($fn)) return '';
1047 $fh = @fopen
($fn, 'r');
1048 if (!$fh) return '';
1050 while (($line = fgets($fh)) !== false) {
1051 if (++
$ln == $id) break;
1054 return rtrim((string)$line);
1058 * Write a line into the index.
1060 * @param string $idx name of the index
1061 * @param string $suffix subpart identifier
1062 * @param int $id the line number
1063 * @param string $line line to write
1064 * @return bool If saving succeeded
1066 * @author Tom N Harris <tnharris@whoopdedo.org>
1068 protected function saveIndexKey($idx, $suffix, $id, $line)
1071 if (!str_ends_with($line, "\n"))
1073 $fn = $conf['indexdir'] . '/' . $idx . $suffix;
1074 $fh = @fopen
($fn . '.tmp', 'w');
1075 if (!$fh) return false;
1076 $ih = @fopen
($fn . '.idx', 'r');
1079 while (($curline = fgets($ih)) !== false) {
1080 fwrite($fh, (++
$ln == $id) ?
$line : $curline);
1096 chmod($fn . '.tmp', $conf['fperm']);
1097 io_rename($fn . '.tmp', $fn . '.idx');
1102 * Retrieve or insert a value in the index.
1104 * @param string $idx name of the index
1105 * @param string $suffix subpart identifier
1106 * @param string $value line to find in the index
1107 * @return int|bool line number of the value in the index or false if writing the index failed
1109 * @author Tom N Harris <tnharris@whoopdedo.org>
1111 protected function addIndexKey($idx, $suffix, $value)
1113 $index = $this->getIndex($idx, $suffix);
1114 $id = array_search($value, $index, true);
1115 if ($id === false) {
1116 $id = count($index);
1117 $index[$id] = $value;
1118 if (!$this->saveIndex($idx, $suffix, $index)) {
1119 trigger_error("Failed to write $idx index", E_USER_ERROR
);
1127 * Get the list of lengths indexed in the wiki.
1129 * Read the index directory or a cache file and returns
1130 * a sorted array of lengths of the words used in the wiki.
1132 * @author YoBoY <yoboy.leguesh@gmail.com>
1136 protected function listIndexLengths()
1138 return idx_listIndexLengths();
1142 * Get the word lengths that have been indexed.
1144 * Reads the index directory and returns an array of lengths
1145 * that there are indices for.
1147 * @author YoBoY <yoboy.leguesh@gmail.com>
1149 * @param array|int $filter
1152 protected function indexLengths($filter)
1156 if (is_array($filter)) {
1157 // testing if index files exist only
1158 $path = $conf['indexdir'] . "/i";
1159 foreach (array_keys($filter) as $key) {
1160 if (file_exists($path . $key . '.idx'))
1164 $lengths = idx_listIndexLengths();
1165 foreach ($lengths as $length) {
1166 // keep all the values equal or superior
1167 if ((int)$length >= (int)$filter)
1175 * Insert or replace a tuple in a line.
1177 * @author Tom N Harris <tnharris@whoopdedo.org>
1179 * @param string $line
1180 * @param string|int $id
1184 protected function updateTuple($line, $id, $count)
1187 $line = preg_replace('/(^|:)' . preg_quote($id, '/') . '\*\d*/', '', $line);
1189 $line = trim($line, ':');
1192 return "$id*$count:" . $line;
1194 return "$id*$count";
1201 * Split a line into an array of tuples.
1203 * @author Tom N Harris <tnharris@whoopdedo.org>
1204 * @author Andreas Gohr <andi@splitbrain.org>
1206 * @param array $keys
1207 * @param string $line
1210 protected function parseTuples(&$keys, $line)
1213 if ($line == '') return $result;
1214 $parts = explode(':', $line);
1215 foreach ($parts as $tuple) {
1216 if ($tuple === '') continue;
1217 [$key, $cnt] = explode('*', $tuple);
1218 if (!$cnt) continue;
1219 if (isset($keys[$key])) {
1221 if ($key === false ||
is_null($key)) continue;
1223 $result[$key] = $cnt;
1229 * Sum the counts in a list of tuples.
1231 * @author Tom N Harris <tnharris@whoopdedo.org>
1233 * @param string $line
1236 protected function countTuples($line)
1239 $parts = explode(':', $line);
1240 foreach ($parts as $tuple) {
1241 if ($tuple === '') continue;
1242 [/* pid */, $cnt] = explode('*', $tuple);