inc/indexer.php

   1 <?php
   2 /**
   3  * Common DokuWiki functions
   4  *
   5  * @license    GPL 2 (http://www.gnu.org/licenses/gpl.html)
   6  * @author     Andreas Gohr <andi@splitbrain.org>
   7  */
   8
   9 if(!defined('DOKU_INC')) die('meh.');
  10 require_once(DOKU_INC.'inc/io.php');
  11 require_once(DOKU_INC.'inc/utf8.php');
  12 require_once(DOKU_INC.'inc/parserutils.php');
  13
  14 // set the minimum token length to use in the index (note, this doesn't apply to numeric tokens)
  15 if (!defined('IDX_MINWORDLENGTH')) define('IDX_MINWORDLENGTH',2);
  16
  17 // Asian characters are handled as words. The following regexp defines the
  18 // Unicode-Ranges for Asian characters
  19 // Ranges taken from http://en.wikipedia.org/wiki/Unicode_block
  20 // I'm no language expert. If you think some ranges are wrongly chosen or
  21 // a range is missing, please contact me
  22 define('IDX_ASIAN1','[\x{0E00}-\x{0E7F}]'); // Thai
  23 define('IDX_ASIAN2','['.
  24                    '\x{2E80}-\x{3040}'.  // CJK -> Hangul
  25                    '\x{309D}-\x{30A0}'.
  26                    '\x{30FD}-\x{31EF}\x{3200}-\x{D7AF}'.
  27                    '\x{F900}-\x{FAFF}'.  // CJK Compatibility Ideographs
  28                    '\x{FE30}-\x{FE4F}'.  // CJK Compatibility Forms
  29                    ']');
  30 define('IDX_ASIAN3','['.                // Hiragana/Katakana (can be two characters)
  31                    '\x{3042}\x{3044}\x{3046}\x{3048}'.
  32                    '\x{304A}-\x{3062}\x{3064}-\x{3082}'.
  33                    '\x{3084}\x{3086}\x{3088}-\x{308D}'.
  34                    '\x{308F}-\x{3094}'.
  35                    '\x{30A2}\x{30A4}\x{30A6}\x{30A8}'.
  36                    '\x{30AA}-\x{30C2}\x{30C4}-\x{30E2}'.
  37                    '\x{30E4}\x{30E6}\x{30E8}-\x{30ED}'.
  38                    '\x{30EF}-\x{30F4}\x{30F7}-\x{30FA}'.
  39                    ']['.
  40                    '\x{3041}\x{3043}\x{3045}\x{3047}\x{3049}'.
  41                    '\x{3063}\x{3083}\x{3085}\x{3087}\x{308E}\x{3095}-\x{309C}'.
  42                    '\x{30A1}\x{30A3}\x{30A5}\x{30A7}\x{30A9}'.
  43                    '\x{30C3}\x{30E3}\x{30E5}\x{30E7}\x{30EE}\x{30F5}\x{30F6}\x{30FB}\x{30FC}'.
  44                    '\x{31F0}-\x{31FF}'.
  45                    ']?');
  46 define('IDX_ASIAN', '(?:'.IDX_ASIAN1.'|'.IDX_ASIAN2.'|'.IDX_ASIAN3.')');
  47
  48 /**
  49  * Measure the length of a string.
  50  * Differs from strlen in handling of asian characters.
  51  *
  52  * @author Tom N Harris <tnharris@whoopdedo.org>
  53  */
  54 function wordlen($w){
  55     $l = strlen($w);
  56     // If left alone, all chinese "words" will get put into w3.idx
  57     // So the "length" of a "word" is faked
  58     if(preg_match('/'.IDX_ASIAN2.'/u',$w))
  59         $l += ord($w) - 0xE1;  // Lead bytes from 0xE2-0xEF
  60     return $l;
  61 }
  62
  63 /**
  64  * Write a list of strings to an index file.
  65  *
  66  * @author Tom N Harris <tnharris@whoopdedo.org>
  67  */
  68 function idx_saveIndex($pre, $wlen, &$idx){
  69     global $conf;
  70     $fn = $conf['indexdir'].'/'.$pre.$wlen;
  71     $fh = @fopen($fn.'.tmp','w');
  72     if(!$fh) return false;
  73     foreach ($idx as $line) {
  74         fwrite($fh,$line);
  75     }
  76     fclose($fh);
  77     if(isset($conf['fperm'])) chmod($fn.'.tmp', $conf['fperm']);
  78     io_rename($fn.'.tmp', $fn.'.idx');
  79     return true;
  80 }
  81
  82 /**
  83  * Append a given line to an index file.
  84  *
  85  * @author Andreas Gohr <andi@splitbrain.org>
  86  */
  87 function idx_appendIndex($pre, $wlen, $line){
  88     global $conf;
  89     $fn = $conf['indexdir'].'/'.$pre.$wlen;
  90     $fh = @fopen($fn.'.idx','a');
  91     if(!$fh) return false;
  92     fwrite($fh,$line);
  93     fclose($fh);
  94     return true;
  95 }
  96
  97 /**
  98  * Read the list of words in an index (if it exists).
  99  *
 100  * @author Tom N Harris <tnharris@whoopdedo.org>
 101  */
 102 function idx_getIndex($pre, $wlen){
 103     global $conf;
 104     $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
 105     if(!@file_exists($fn)) return array();
 106     return file($fn);
 107 }
 108
 109 /**
 110  * Create an empty index file if it doesn't exist yet.
 111  *
 112  * FIXME: This function isn't currently used. It will probably be removed soon.
 113  *
 114  * @author Tom N Harris <tnharris@whoopdedo.org>
 115  */
 116 function idx_touchIndex($pre, $wlen){
 117     global $conf;
 118     $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
 119     if(!@file_exists($fn)){
 120         touch($fn);
 121         if($conf['fperm']) chmod($fn, $conf['fperm']);
 122     }
 123 }
 124
 125 /**
 126  * Read a line ending with \n.
 127  * Returns false on EOF.
 128  *
 129  * @author Tom N Harris <tnharris@whoopdedo.org>
 130  */
 131 function _freadline($fh) {
 132     if (feof($fh)) return false;
 133     $ln = '';
 134     while (($buf = fgets($fh,4096)) !== false) {
 135         $ln .= $buf;
 136         if (substr($buf,-1) == "\n") break;
 137     }
 138     if ($ln === '') return false;
 139     if (substr($ln,-1) != "\n") $ln .= "\n";
 140     return $ln;
 141 }
 142
 143 /**
 144  * Write a line to an index file.
 145  *
 146  * @author Tom N Harris <tnharris@whoopdedo.org>
 147  */
 148 function idx_saveIndexLine($pre, $wlen, $idx, $line){
 149     global $conf;
 150     if(substr($line,-1) != "\n") $line .= "\n";
 151     $fn = $conf['indexdir'].'/'.$pre.$wlen;
 152     $fh = @fopen($fn.'.tmp','w');
 153     if(!$fh) return false;
 154     $ih = @fopen($fn.'.idx','r');
 155     if ($ih) {
 156         $ln = -1;
 157         while (($curline = _freadline($ih)) !== false) {
 158             if (++$ln == $idx) {
 159                 fwrite($fh, $line);
 160             } else {
 161                 fwrite($fh, $curline);
 162             }
 163         }
 164         if ($idx > $ln) {
 165             fwrite($fh,$line);
 166         }
 167         fclose($ih);
 168     } else {
 169         fwrite($fh,$line);
 170     }
 171     fclose($fh);
 172     if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']);
 173     io_rename($fn.'.tmp', $fn.'.idx');
 174     return true;
 175 }
 176
 177 /**
 178  * Read a single line from an index (if it exists).
 179  *
 180  * @author Tom N Harris <tnharris@whoopdedo.org>
 181  */
 182 function idx_getIndexLine($pre, $wlen, $idx){
 183     global $conf;
 184     $fn = $conf['indexdir'].'/'.$pre.$wlen.'.idx';
 185     if(!@file_exists($fn)) return '';
 186     $fh = @fopen($fn,'r');
 187     if(!$fh) return '';
 188     $ln = -1;
 189     while (($line = _freadline($fh)) !== false) {
 190         if (++$ln == $idx) break;
 191     }
 192     fclose($fh);
 193     return "$line";
 194 }
 195
 196 /**
 197  * Split a page into words
 198  *
 199  * Returns an array of word counts, false if an error occurred.
 200  * Array is keyed on the word length, then the word index.
 201  *
 202  * @author Andreas Gohr <andi@splitbrain.org>
 203  * @author Christopher Smith <chris@jalakai.co.uk>
 204  */
 205 function idx_getPageWords($page){
 206     global $conf;
 207     $swfile   = DOKU_INC.'inc/lang/'.$conf['lang'].'/stopwords.txt';
 208     if(@file_exists($swfile)){
 209         $stopwords = file($swfile);
 210     }else{
 211         $stopwords = array();
 212     }
 213
 214     $body = '';
 215     $data = array($page, $body);
 216     $evt = new Doku_Event('INDEXER_PAGE_ADD', $data);
 217     if ($evt->advise_before()) $data[1] .= rawWiki($page);
 218     $evt->advise_after();
 219     unset($evt);
 220
 221     list($page,$body) = $data;
 222
 223     $body   = strtr($body, "\r\n\t", '   ');
 224     $tokens = explode(' ', $body);
 225     $tokens = array_count_values($tokens);   // count the frequency of each token
 226
 227     // ensure the deaccented or romanised page names of internal links are added to the token array
 228     // (this is necessary for the backlink function -- there maybe a better way!)
 229     if ($conf['deaccent']) {
 230         $links = p_get_metadata($page,'relation references');
 231
 232         if (!empty($links)) {
 233             $tmp = join(' ',array_keys($links));                // make a single string
 234             $tmp = strtr($tmp, ':', ' ');                       // replace namespace separator with a space
 235             $link_tokens = array_unique(explode(' ', $tmp));    // break into tokens
 236
 237             foreach ($link_tokens as $link_token) {
 238                 if (isset($tokens[$link_token])) continue;
 239                 $tokens[$link_token] = 1;
 240             }
 241         }
 242     }
 243
 244     $words = array();
 245     foreach ($tokens as $word => $count) {
 246         $arr = idx_tokenizer($word,$stopwords);
 247         $arr = array_count_values($arr);
 248         foreach ($arr as $w => $c) {
 249             $l = wordlen($w);
 250             if(isset($words[$l])){
 251                 $words[$l][$w] = $c * $count + (isset($words[$l][$w]) ? $words[$l][$w] : 0);
 252             }else{
 253                 $words[$l] = array($w => $c * $count);
 254             }
 255         }
 256     }
 257
 258     // arrive here with $words = array(wordlen => array(word => frequency))
 259
 260     $index = array(); //resulting index
 261     foreach (array_keys($words) as $wlen){
 262         $word_idx = idx_getIndex('w',$wlen);
 263         foreach ($words[$wlen] as $word => $freq) {
 264             $wid = array_search("$word\n",$word_idx);
 265             if(!is_int($wid)){
 266                 $wid = count($word_idx);
 267                 $word_idx[] = "$word\n";
 268             }
 269             if(!isset($index[$wlen]))
 270                 $index[$wlen] = array();
 271             $index[$wlen][$wid] = $freq;
 272         }
 273
 274         // save back word index
 275         if(!idx_saveIndex('w',$wlen,$word_idx)){
 276             trigger_error("Failed to write word index", E_USER_ERROR);
 277             return false;
 278         }
 279     }
 280
 281     return $index;
 282 }
 283
 284 /**
 285  * Adds/updates the search for the given page
 286  *
 287  * This is the core function of the indexer which does most
 288  * of the work. This function needs to be called with proper
 289  * locking!
 290  *
 291  * @author Andreas Gohr <andi@splitbrain.org>
 292  */
 293 function idx_addPage($page){
 294     global $conf;
 295
 296     // load known documents
 297     $page_idx = idx_getIndex('page','');
 298
 299     // get page id (this is the linenumber in page.idx)
 300     $pid = array_search("$page\n",$page_idx);
 301     if(!is_int($pid)){
 302         $pid = count($page_idx);
 303         // page was new - write back
 304         if (!idx_appendIndex('page','',"$page\n")){
 305             trigger_error("Failed to write page index", E_USER_ERROR);
 306             return false;
 307         }
 308     }
 309     unset($page_idx); // free memory
 310
 311     $pagewords = array();
 312     // get word usage in page
 313     $words = idx_getPageWords($page);
 314     if($words === false) return false;
 315
 316     if(!empty($words)) {
 317         foreach(array_keys($words) as $wlen){
 318             $index = idx_getIndex('i',$wlen);
 319             foreach($words[$wlen] as $wid => $freq){
 320                 if($wid<count($index)){
 321                     $index[$wid] = idx_updateIndexLine($index[$wid],$pid,$freq);
 322                 }else{
 323                     // New words **should** have been added in increasing order
 324                     // starting with the first unassigned index.
 325                     // If someone can show how this isn't true, then I'll need to sort
 326                     // or do something special.
 327                     $index[$wid] = idx_updateIndexLine('',$pid,$freq);
 328                 }
 329                 $pagewords[] = "$wlen*$wid";
 330             }
 331             // save back word index
 332             if(!idx_saveIndex('i',$wlen,$index)){
 333                 trigger_error("Failed to write index", E_USER_ERROR);
 334                 return false;
 335             }
 336         }
 337     }
 338
 339     // Remove obsolete index entries
 340     $pageword_idx = trim(idx_getIndexLine('pageword','',$pid));
 341     if ($pageword_idx !== '') {
 342         $oldwords = explode(':',$pageword_idx);
 343         $delwords = array_diff($oldwords, $pagewords);
 344         $upwords = array();
 345         foreach ($delwords as $word) {
 346             if($word=='') continue;
 347             list($wlen,$wid) = explode('*',$word);
 348             $wid = (int)$wid;
 349             $upwords[$wlen][] = $wid;
 350         }
 351         foreach ($upwords as $wlen => $widx) {
 352             $index = idx_getIndex('i',$wlen);
 353             foreach ($widx as $wid) {
 354                 $index[$wid] = idx_updateIndexLine($index[$wid],$pid,0);
 355             }
 356             idx_saveIndex('i',$wlen,$index);
 357         }
 358     }
 359     // Save the reverse index
 360     $pageword_idx = join(':',$pagewords)."\n";
 361     if(!idx_saveIndexLine('pageword','',$pid,$pageword_idx)){
 362         trigger_error("Failed to write word index", E_USER_ERROR);
 363         return false;
 364     }
 365
 366     return true;
 367 }
 368
 369 /**
 370  * Write a new index line to the filehandle
 371  *
 372  * This function writes an line for the index file to the
 373  * given filehandle. It removes the given document from
 374  * the given line and readds it when $count is >0.
 375  *
 376  * @deprecated - see idx_updateIndexLine
 377  * @author Andreas Gohr <andi@splitbrain.org>
 378  */
 379 function idx_writeIndexLine($fh,$line,$pid,$count){
 380     fwrite($fh,idx_updateIndexLine($line,$pid,$count));
 381 }
 382
 383 /**
 384  * Modify an index line with new information
 385  *
 386  * This returns a line of the index. It removes the
 387  * given document from the line and readds it if
 388  * $count is >0.
 389  *
 390  * @author Tom N Harris <tnharris@whoopdedo.org>
 391  * @author Andreas Gohr <andi@splitbrain.org>
 392  */
 393 function idx_updateIndexLine($line,$pid,$count){
 394     $line = trim($line);
 395     $updated = array();
 396     if($line != ''){
 397         $parts = explode(':',$line);
 398         // remove doc from given line
 399         foreach($parts as $part){
 400             if($part == '') continue;
 401             list($doc,$cnt) = explode('*',$part);
 402             if($doc != $pid){
 403                 $updated[] = $part;
 404             }
 405         }
 406     }
 407
 408     // add doc
 409     if ($count){
 410         $updated[] = "$pid*$count";
 411     }
 412
 413     return join(':',$updated)."\n";
 414 }
 415
 416 /**
 417  * Get the word lengths that have been indexed.
 418  *
 419  * Reads the index directory and returns an array of lengths
 420  * that there are indices for.
 421  *
 422  * @author Tom N Harris <tnharris@whoopdedo.org>
 423  */
 424 function idx_indexLengths(&$filter){
 425     global $conf;
 426     $dir = @opendir($conf['indexdir']);
 427     if($dir===false)
 428         return array();
 429     $idx = array();
 430     if(is_array($filter)){
 431         while (($f = readdir($dir)) !== false) {
 432             if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){
 433                 $i = substr($f,1,-4);
 434                 if (is_numeric($i) && isset($filter[(int)$i]))
 435                     $idx[] = (int)$i;
 436             }
 437         }
 438     }else{
 439         // Exact match first.
 440         if(@file_exists($conf['indexdir']."/i$filter.idx"))
 441             $idx[] = $filter;
 442         while (($f = readdir($dir)) !== false) {
 443             if (substr($f,0,1) == 'i' && substr($f,-4) == '.idx'){
 444                 $i = substr($f,1,-4);
 445                 if (is_numeric($i) && $i > $filter)
 446                     $idx[] = (int)$i;
 447             }
 448         }
 449     }
 450     closedir($dir);
 451     return $idx;
 452 }
 453
 454 /**
 455  * Find the the index number of each search term.
 456  *
 457  * This will group together words that appear in the same index.
 458  * So it should perform better, because it only opens each index once.
 459  * Actually, it's not that great. (in my experience) Probably because of the disk cache.
 460  * And the sorted function does more work, making it slightly slower in some cases.
 461  *
 462  * @param array    $words   The query terms. Words should only contain valid characters,
 463  *                          with a '*' at either the beginning or end of the word (or both)
 464  * @param arrayref $result  Set to word => array("length*id" ...), use this to merge the
 465  *                          index locations with the appropriate query term.
 466  * @return array            Set to length => array(id ...)
 467  *
 468  * @author Tom N Harris <tnharris@whoopdedo.org>
 469  */
 470 function idx_getIndexWordsSorted($words,&$result){
 471     // parse and sort tokens
 472     $tokens = array();
 473     $tokenlength = array();
 474     $tokenwild = array();
 475     foreach($words as $word){
 476         $result[$word] = array();
 477         $wild = 0;
 478         $xword = $word;
 479         $wlen = wordlen($word);
 480
 481         // check for wildcards
 482         if(substr($xword,0,1) == '*'){
 483             $xword = substr($xword,1);
 484             $wild |= 1;
 485             $wlen -= 1;
 486         }
 487         if(substr($xword,-1,1) == '*'){
 488             $xword = substr($xword,0,-1);
 489             $wild |= 2;
 490             $wlen -= 1;
 491         }
 492         if ($wlen < IDX_MINWORDLENGTH && $wild == 0 && !is_numeric($xword)) continue;
 493         if(!isset($tokens[$xword])){
 494             $tokenlength[$wlen][] = $xword;
 495         }
 496         if($wild){
 497             $ptn = preg_quote($xword,'/');
 498             if(($wild&1) == 0) $ptn = '^'.$ptn;
 499             if(($wild&2) == 0) $ptn = $ptn.'$';
 500             $tokens[$xword][] = array($word, '/'.$ptn.'/');
 501             if(!isset($tokenwild[$xword])) $tokenwild[$xword] = $wlen;
 502         }else
 503             $tokens[$xword][] = array($word, null);
 504     }
 505     asort($tokenwild);
 506     // $tokens = array( base word => array( [ query word , grep pattern ] ... ) ... )
 507     // $tokenlength = array( base word length => base word ... )
 508     // $tokenwild = array( base word => base word length ... )
 509
 510     $length_filter = empty($tokenwild) ? $tokenlength : min(array_keys($tokenlength));
 511     $indexes_known = idx_indexLengths($length_filter);
 512     if(!empty($tokenwild)) sort($indexes_known);
 513     // get word IDs
 514     $wids = array();
 515     foreach($indexes_known as $ixlen){
 516         $word_idx = idx_getIndex('w',$ixlen);
 517         // handle exact search
 518         if(isset($tokenlength[$ixlen])){
 519             foreach($tokenlength[$ixlen] as $xword){
 520                 $wid = array_search("$xword\n",$word_idx);
 521                 if(is_int($wid)){
 522                     $wids[$ixlen][] = $wid;
 523                     foreach($tokens[$xword] as $w)
 524                         $result[$w[0]][] = "$ixlen*$wid";
 525                 }
 526             }
 527         }
 528         // handle wildcard search
 529         foreach($tokenwild as $xword => $wlen){
 530             if($wlen >= $ixlen) break;
 531             foreach($tokens[$xword] as $w){
 532                 if(is_null($w[1])) continue;
 533                 foreach(array_keys(preg_grep($w[1],$word_idx)) as $wid){
 534                     $wids[$ixlen][] = $wid;
 535                     $result[$w[0]][] = "$ixlen*$wid";
 536                 }
 537             }
 538         }
 539     }
 540     return $wids;
 541 }
 542
 543 /**
 544  * Lookup words in index
 545  *
 546  * Takes an array of word and will return a list of matching
 547  * documents for each one.
 548  *
 549  * Important: No ACL checking is done here! All results are
 550  *            returned, regardless of permissions
 551  *
 552  * @author Andreas Gohr <andi@splitbrain.org>
 553  */
 554 function idx_lookup($words){
 555     global $conf;
 556
 557     $result = array();
 558
 559     $wids = idx_getIndexWordsSorted($words, $result);
 560     if(empty($wids)) return array();
 561
 562     // load known words and documents
 563     $page_idx = idx_getIndex('page','');
 564
 565     $docs = array();                          // hold docs found
 566     foreach(array_keys($wids) as $wlen){
 567         $wids[$wlen] = array_unique($wids[$wlen]);
 568         $index = idx_getIndex('i',$wlen);
 569         foreach($wids[$wlen] as $ixid){
 570             if($ixid < count($index))
 571                 $docs["$wlen*$ixid"] = idx_parseIndexLine($page_idx,$index[$ixid]);
 572         }
 573     }
 574
 575     // merge found pages into final result array
 576     $final = array();
 577     foreach($result as $word => $res){
 578         $final[$word] = array();
 579         foreach($res as $wid){
 580             $hits = &$docs[$wid];
 581             foreach ($hits as $hitkey => $hitcnt) {
 582                 if (!isset($final[$word][$hitkey])) {
 583                     $final[$word][$hitkey] = $hitcnt;
 584                 } else {
 585                     $final[$word][$hitkey] += $hitcnt;
 586                 }
 587             }
 588         }
 589     }
 590     return $final;
 591 }
 592
 593 /**
 594  * Returns a list of documents and counts from a index line
 595  *
 596  * It omits docs with a count of 0 and pages that no longer
 597  * exist.
 598  *
 599  * @param  array  $page_idx The list of known pages
 600  * @param  string $line     A line from the main index
 601  * @author Andreas Gohr <andi@splitbrain.org>
 602  */
 603 function idx_parseIndexLine(&$page_idx,$line){
 604     $result = array();
 605
 606     $line = trim($line);
 607     if($line == '') return $result;
 608
 609     $parts = explode(':',$line);
 610     foreach($parts as $part){
 611         if($part == '') continue;
 612         list($doc,$cnt) = explode('*',$part);
 613         if(!$cnt) continue;
 614         $doc = trim($page_idx[$doc]);
 615         if(!$doc) continue;
 616         // make sure the document still exists
 617         if(!page_exists($doc,'',false)) continue;
 618
 619         $result[$doc] = $cnt;
 620     }
 621     return $result;
 622 }
 623
 624 /**
 625  * Tokenizes a string into an array of search words
 626  *
 627  * Uses the same algorithm as idx_getPageWords()
 628  *
 629  * @param string   $string     the query as given by the user
 630  * @param arrayref $stopwords  array of stopwords
 631  * @param boolean  $wc         are wildcards allowed?
 632  */
 633 function idx_tokenizer($string,&$stopwords,$wc=false){
 634     $words = array();
 635     $wc = ($wc) ? '' : $wc = '\*';
 636
 637     if(preg_match('/[^0-9A-Za-z]/u', $string)){
 638         // handle asian chars as single words (may fail on older PHP version)
 639         $asia = @preg_replace('/('.IDX_ASIAN.')/u',' \1 ',$string);
 640         if(!is_null($asia)) $string = $asia; //recover from regexp failure
 641
 642         $arr = explode(' ', utf8_stripspecials($string,' ','\._\-:'.$wc));
 643         foreach ($arr as $w) {
 644             if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) continue;
 645             $w = utf8_strtolower($w);
 646             if($stopwords && is_int(array_search("$w\n",$stopwords))) continue;
 647             $words[] = $w;
 648         }
 649     }else{
 650         $w = $string;
 651         if (!is_numeric($w) && strlen($w) < IDX_MINWORDLENGTH) return $words;
 652         $w = strtolower($w);
 653         if(is_int(array_search("$w\n",$stopwords))) return $words;
 654         $words[] = $w;
 655     }
 656
 657     return $words;
 658 }
 659
 660 /**
 661  * Create a pagewords index from the existing index.
 662  *
 663  * @author Tom N Harris <tnharris@whoopdedo.org>
 664  */
 665 function idx_upgradePageWords(){
 666     global $conf;
 667     $page_idx = idx_getIndex('page','');
 668     if (empty($page_idx)) return;
 669     $pagewords = array();
 670     $len = count($page_idx);
 671     for ($n=0;$n<$len;$n++){
 672         $pagewords[] = array();
 673     }
 674     unset($page_idx);
 675
 676     $n=0;
 677     foreach (idx_indexLengths($n) as $wlen) {
 678         $lines = idx_getIndex('i',$wlen);
 679         $len = count($lines);
 680         for ($wid=0;$wid<$len;$wid++) {
 681             $wkey = "$wlen*$wid";
 682             foreach (explode(':',trim($lines[$wid])) as $part) {
 683                 if($part == '') continue;
 684                 list($doc,$cnt) = explode('*',$part);
 685                 $pagewords[(int)$doc][] = $wkey;
 686             }
 687         }
 688     }
 689
 690     $fn = $conf['indexdir'].'/pageword';
 691     $fh = @fopen($fn.'.tmp','w');
 692     if (!$fh){
 693         trigger_error("Failed to write word index", E_USER_ERROR);
 694         return false;
 695     }
 696     foreach ($pagewords as $line){
 697         fwrite($fh, join(':',$line)."\n");
 698     }
 699     fclose($fh);
 700     if($conf['fperm']) chmod($fn.'.tmp', $conf['fperm']);
 701     io_rename($fn.'.tmp', $fn.'.idx');
 702     return true;
 703 }
 704
 705 //Setup VIM: ex: et ts=4 enc=utf-8 :