?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
$config to phpbb::$config
[phpbb.git] / phpBB / includes / search / fulltext_native.php
blobe10b37f79c6aaa6aed1d890c0af39ee7201a23b7
1 <?php
2 /**
3 *
4 * @package search
5 * @version $Id$
6 * @copyright (c) 2005 phpBB Group
7 * @license http://opensource.org/licenses/gpl-license.php GNU Public License
8 *
9 */
10
11 /**
12 * @ignore
13 */
14 if (!defined('IN_PHPBB'))
15 {
16 exit;
17 }
18
19 /**
20 * @ignore
21 */
22 include_once(PHPBB_ROOT_PATH . 'includes/search/search.' . PHP_EXT);
23
24 /**
25 * fulltext_native
26 * phpBB's own db driven fulltext search, version 2
27 * @package search
28 */
29 class fulltext_native extends search_backend
30 {
31 private $stats = array();
32 public $word_length = array();
33 public $search_query;
34 public $common_words = array();
35
36 private $must_contain_ids = array();
37 private $must_not_contain_ids = array();
38 private $must_exclude_one_ids = array();
39
40 /**
41 * Initialises the fulltext_native search backend with min/max word length and makes sure the UTF-8 normalizer is loaded.
42 *
43 * @param boolean|string &$error is passed by reference and should either be set to false on success or an error message on failure.
44 *
45 * @access public
46 */
47 function __construct(&$error)
48 {
49 $this->word_length = array('min' => phpbb::$config['fulltext_native_min_chars'], 'max' => phpbb::$config['fulltext_native_max_chars']);
50
51 /**
52 * Load the UTF tools
53 */
54 if (!class_exists('utf_normalizer'))
55 {
56 include(PHPBB_ROOT_PATH . 'includes/utf/utf_normalizer.' . PHP_EXT);
57 }
58
59
60 $error = false;
61 }
62
63 /**
64 * This function fills $this->search_query with the cleaned user search query.
65 *
66 * If $terms is 'any' then the words will be extracted from the search query
67 * and combined with | inside brackets. They will afterwards be treated like
68 * an standard search query.
69 *
70 * Then it analyses the query and fills the internal arrays $must_not_contain_ids,
71 * $must_contain_ids and $must_exclude_one_ids which are later used by keyword_search().
72 *
73 * @param string $keywords contains the search query string as entered by the user
74 * @param string $terms is either 'all' (use search query as entered, default words to 'must be contained in post')
75 * or 'any' (find all posts containing at least one of the given words)
76 * @return boolean false if no valid keywords were found and otherwise true
77 *
78 * @access public
79 */
80 public function split_keywords($keywords, $terms)
81 {
82 global $db, $user;
83
84 $keywords = trim($this->cleanup($keywords, '+-|()*'));
85
86 // allow word|word|word without brackets
87 if ((strpos($keywords, ' ') === false) && (strpos($keywords, '|') !== false) && (strpos($keywords, '(') === false))
88 {
89 $keywords = '(' . $keywords . ')';
90 }
91
92 $open_bracket = $space = false;
93 for ($i = 0, $n = strlen($keywords); $i < $n; $i++)
94 {
95 if ($open_bracket !== false)
96 {
97 switch ($keywords[$i])
98 {
99 case ')':
100 if ($open_bracket + 1 == $i)
101 {
102 $keywords[$i - 1] = '|';
103 $keywords[$i] = '|';
104 }
105 $open_bracket = false;
106 break;
107 case '(':
108 $keywords[$i] = '|';
109 break;
110 case '+':
111 case '-':
112 case ' ':
113 $keywords[$i] = '|';
114 break;
115 }
116 }
117 else
118 {
119 switch ($keywords[$i])
120 {
121 case ')':
122 $keywords[$i] = ' ';
123 break;
124 case '(':
125 $open_bracket = $i;
126 $space = false;
127 break;
128 case '|':
129 $keywords[$i] = ' ';
130 break;
131 case '-':
132 case '+':
133 $space = $keywords[$i];
134 break;
135 case ' ':
136 if ($space !== false)
137 {
138 $keywords[$i] = $space;
139 }
140 break;
141 default:
142 $space = false;
143 }
144 }
145 }
146
147 if ($open_bracket)
148 {
149 $keywords .= ')';
150 }
151
152 $match = array(
153 '# +#',
154 '#\|\|+#',
155 '#(\+|\-)(?:\+|\-)+#',
156 '#\(\|#',
157 '#\|\)#',
158 );
159 $replace = array(
160 ' ',
161 '|',
162 '$1',
163 '(',
164 ')',
165 );
166
167 $keywords = preg_replace($match, $replace, $keywords);
168
169 // $keywords input format: each word separated by a space, words in a bracket are not separated
170
171 // the user wants to search for any word, convert the search query
172 if ($terms == 'any')
173 {
174 $words = array();
175
176 preg_match_all('#([^\\s+\\-|()]+)(?:$|[\\s+\\-|()])#u', $keywords, $words);
177 if (sizeof($words[1]))
178 {
179 $keywords = '(' . implode('|', $words[1]) . ')';
180 }
181 }
182
183 // set the search_query which is shown to the user
184 $this->search_query = $keywords;
185
186 $exact_words = array();
187 preg_match_all('#([^\\s+\\-|*()]+)(?:$|[\\s+\\-|()])#u', $keywords, $exact_words);
188 $exact_words = $exact_words[1];
189
190 $common_ids = $words = array();
191
192 if (sizeof($exact_words))
193 {
194 $sql = 'SELECT word_id, word_text, word_common
195 FROM ' . SEARCH_WORDLIST_TABLE . '
196 WHERE ' . $db->sql_in_set('word_text', $exact_words);
197 $result = $db->sql_query($sql);
198
199 // store an array of words and ids, remove common words
200 while ($row = $db->sql_fetchrow($result))
201 {
202 if ($row['word_common'])
203 {
204 $this->common_words[] = $row['word_text'];
205 $common_ids[$row['word_text']] = (int) $row['word_id'];
206 continue;
207 }
208
209 $words[$row['word_text']] = (int) $row['word_id'];
210 }
211 $db->sql_freeresult($result);
212 }
213 unset($exact_words);
214
215 // now analyse the search query, first split it using the spaces
216 $query = explode(' ', $keywords);
217
218 $this->must_contain_ids = array();
219 $this->must_not_contain_ids = array();
220 $this->must_exclude_one_ids = array();
221
222 $mode = '';
223 $ignore_no_id = true;
224
225 foreach ($query as $word)
226 {
227 if (empty($word))
228 {
229 continue;
230 }
231
232 // words which should not be included
233 if ($word[0] == '-')
234 {
235 $word = substr($word, 1);
236
237 // a group of which at least one may not be in the resulting posts
238 if ($word[0] == '(')
239 {
240 $word = array_unique(explode('|', substr($word, 1, -1)));
241 $mode = 'must_exclude_one';
242 }
243 // one word which should not be in the resulting posts
244 else
245 {
246 $mode = 'must_not_contain';
247 }
248 $ignore_no_id = true;
249 }
250 // words which have to be included
251 else
252 {
253 // no prefix is the same as a +prefix
254 if ($word[0] == '+')
255 {
256 $word = substr($word, 1);
257 }
258
259 // a group of words of which at least one word should be in every resulting post
260 if ($word[0] == '(')
261 {
262 $word = array_unique(explode('|', substr($word, 1, -1)));
263 }
264 $ignore_no_id = false;
265 $mode = 'must_contain';
266 }
267
268 if (empty($word))
269 {
270 continue;
271 }
272
273 // if this is an array of words then retrieve an id for each
274 if (is_array($word))
275 {
276 $non_common_words = array();
277 $id_words = array();
278 foreach ($word as $i => $word_part)
279 {
280 if (strpos($word_part, '*') !== false)
281 {
282 $id_words[] = '\'' . $db->sql_escape(str_replace('*', '%', $word_part)) . '\'';
283 $non_common_words[] = $word_part;
284 }
285 else if (isset($words[$word_part]))
286 {
287 $id_words[] = $words[$word_part];
288 $non_common_words[] = $word_part;
289 }
290 else
291 {
292 $len = utf8_strlen($word_part);
293 if ($len < $this->word_length['min'] || $len > $this->word_length['max'])
294 {
295 $this->common_words[] = $word_part;
296 }
297 }
298 }
299 if (sizeof($id_words))
300 {
301 sort($id_words);
302 if (sizeof($id_words) > 1)
303 {
304 $this->{$mode . '_ids'}[] = $id_words;
305 }
306 else
307 {
308 $mode = ($mode == 'must_exclude_one') ? 'must_not_contain' : $mode;
309 $this->{$mode . '_ids'}[] = $id_words[0];
310 }
311 }
312 // throw an error if we shall not ignore unexistant words
313 else if (!$ignore_no_id && sizeof($non_common_words))
314 {
315 trigger_error(sprintf($user->lang['WORDS_IN_NO_POST'], implode(', ', $non_common_words)));
316 }
317 unset($non_common_words);
318 }
319 // else we only need one id
320 else if (($wildcard = strpos($word, '*') !== false) || isset($words[$word]))
321 {
322 if ($wildcard)
323 {
324 $len = utf8_strlen(str_replace('*', '', $word));
325 if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])
326 {
327 $this->{$mode . '_ids'}[] = '\'' . $db->sql_escape(str_replace('*', '%', $word)) . '\'';
328 }
329 else
330 {
331 $this->common_words[] = $word;
332 }
333 }
334 else
335 {
336 $this->{$mode . '_ids'}[] = $words[$word];
337 }
338 }
339 // throw an error if we shall not ignore unexistant words
340 else if (!$ignore_no_id)
341 {
342 if (!isset($common_ids[$word]))
343 {
344 $len = utf8_strlen($word);
345 if ($len >= $this->word_length['min'] && $len <= $this->word_length['max'])
346 {
347 trigger_error(sprintf($user->lang['WORD_IN_NO_POST'], $word));
348 }
349 else
350 {
351 $this->common_words[] = $word;
352 }
353 }
354 }
355 else
356 {
357 $len = utf8_strlen($word);
358 if ($len < $this->word_length['min'] || $len > $this->word_length['max'])
359 {
360 $this->common_words[] = $word;
361 }
362 }
363 }
364
365 // we can't search for negatives only
366 if (!sizeof($this->must_contain_ids))
367 {
368 return false;
369 }
370
371 sort($this->must_contain_ids);
372 sort($this->must_not_contain_ids);
373 sort($this->must_exclude_one_ids);
374
375 if (!empty($this->search_query))
376 {
377 return true;
378 }
379 return false;
380 }
381
382 /**
383 * Performs a search on keywords depending on display specific params. You have to run split_keywords() first.
384 *
385 * @param string $type contains either posts or topics depending on what should be searched for
386 * @param string &$fields contains either titleonly (topic titles should be searched), msgonly (only message bodies should be searched), firstpost (only subject and body of the first post should be searched) or all (all post bodies and subjects should be searched)
387 * @param string &$terms is either 'all' (use query as entered, words without prefix should default to "have to be in field") or 'any' (ignore search query parts and just return all posts that contain any of the specified words)
388 * @param array &$sort_by_sql contains SQL code for the ORDER BY part of a query
389 * @param string &$sort_key is the key of $sort_by_sql for the selected sorting
390 * @param string &$sort_dir is either a or d representing ASC and DESC
391 * @param string &$sort_days specifies the maximum amount of days a post may be old
392 * @param array &$ex_fid_ary specifies an array of forum ids which should not be searched
393 * @param array &$m_approve_fid_ary specifies an array of forum ids in which the searcher is allowed to view unapproved posts
394 * @param int &$topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
395 * @param array &$author_ary an array of author ids if the author should be ignored during the search the array is empty
396 * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
397 * @param int $start indicates the first index of the page
398 * @param int $per_page number of ids each page is supposed to contain
399 * @return boolean|int total number of results
400 *
401 * @access public
402 */
403 public function keyword_search($type, &$fields, &$terms, &$sort_by_sql, &$sort_key, &$sort_dir, &$sort_days, &$ex_fid_ary, &$m_approve_fid_ary, &$topic_id, &$author_ary, &$id_ary, $start, $per_page)
404 {
405 global $db;
406
407 // No keywords? No posts.
408 if (empty($this->search_query))
409 {
410 return false;
411 }
412
413 // generate a search_key from all the options to identify the results
414 $search_key = md5(implode('#', array(
415 serialize($this->must_contain_ids),
416 serialize($this->must_not_contain_ids),
417 serialize($this->must_exclude_one_ids),
418 $type,
419 $fields,
420 $terms,
421 $sort_days,
422 $sort_key,
423 $topic_id,
424 implode(',', $ex_fid_ary),
425 implode(',', $m_approve_fid_ary),
426 implode(',', $author_ary)
427 )));
428
429 // try reading the results from cache
430 $total_results = 0;
431 if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::SEARCH_RESULT_IN_CACHE)
432 {
433 return $total_results;
434 }
435
436 $id_ary = array();
437
438 $sql_where = array();
439 $group_by = false;
440 $m_num = 0;
441 $w_num = 0;
442
443 $sql_array = array(
444 'SELECT' => ($type == 'posts') ? 'p.post_id' : 'p.topic_id',
445 'FROM' => array(
446 SEARCH_WORDMATCH_TABLE => array(),
447 SEARCH_WORDLIST_TABLE => array(),
448 ),
449 'LEFT_JOIN' => array(array(
450 'FROM' => array(POSTS_TABLE => 'p'),
451 'ON' => 'm0.post_id = p.post_id',
452 )),
453 );
454
455 $title_match = '';
456 $group_by = true;
457 // Build some display specific sql strings
458 switch ($fields)
459 {
460 case 'titleonly':
461 $title_match = 'title_match = 1';
462 $group_by = false;
463 // no break
464 case 'firstpost':
465 $sql_array['FROM'][TOPICS_TABLE] = 't';
466 $sql_where[] = 'p.post_id = t.topic_first_post_id';
467 break;
468
469 case 'msgonly':
470 $title_match = 'title_match = 0';
471 $group_by = false;
472 break;
473 }
474
475 if ($type == 'topics')
476 {
477 if (!isset($sql_array['FROM'][TOPICS_TABLE]))
478 {
479 $sql_array['FROM'][TOPICS_TABLE] = 't';
480 $sql_where[] = 'p.topic_id = t.topic_id';
481 }
482 $group_by = true;
483 }
484
485 /**
486 * @todo Add a query optimizer (handle stuff like "+(4|3) +4")
487 */
488
489 foreach ($this->must_contain_ids as $subquery)
490 {
491 if (is_array($subquery))
492 {
493 $group_by = true;
494
495 $word_id_sql = array();
496 $word_ids = array();
497 foreach ($subquery as $id)
498 {
499 if (is_string($id))
500 {
501 $sql_array['LEFT_JOIN'][] = array(
502 'FROM' => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),
503 'ON' => "w$w_num.word_text LIKE $id"
504 );
505 $word_ids[] = "w$w_num.word_id";
506
507 $w_num++;
508 }
509 else
510 {
511 $word_ids[] = $id;
512 }
513 }
514
515 $sql_where[] = $db->sql_in_set("m$m_num.word_id", $word_ids);
516
517 unset($word_id_sql);
518 unset($word_ids);
519 }
520 else if (is_string($subquery))
521 {
522 $sql_array['FROM'][SEARCH_WORDLIST_TABLE][] = 'w' . $w_num;
523
524 $sql_where[] = "w$w_num.word_text LIKE $subquery";
525 $sql_where[] = "m$m_num.word_id = w$w_num.word_id";
526
527 $group_by = true;
528 $w_num++;
529 }
530 else
531 {
532 $sql_where[] = "m$m_num.word_id = $subquery";
533 }
534
535 $sql_array['FROM'][SEARCH_WORDMATCH_TABLE][] = 'm' . $m_num;
536
537 if ($title_match)
538 {
539 $sql_where[] = "m$m_num.$title_match";
540 }
541
542 if ($m_num != 0)
543 {
544 $sql_where[] = "m$m_num.post_id = m0.post_id";
545 }
546 $m_num++;
547 }
548
549 foreach ($this->must_not_contain_ids as $key => $subquery)
550 {
551 if (is_string($subquery))
552 {
553 $sql_array['LEFT_JOIN'][] = array(
554 'FROM' => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),
555 'ON' => "w$w_num.word_text LIKE $subquery"
556 );
557
558 $this->must_not_contain_ids[$key] = "w$w_num.word_id";
559
560 $group_by = true;
561 $w_num++;
562 }
563 }
564
565 if (sizeof($this->must_not_contain_ids))
566 {
567 $sql_array['LEFT_JOIN'][] = array(
568 'FROM' => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),
569 'ON' => $db->sql_in_set("m$m_num.word_id", $this->must_not_contain_ids) . (($title_match) ? " AND m$m_num.$title_match" : '') . " AND m$m_num.post_id = m0.post_id"
570 );
571
572 $sql_where[] = "m$m_num.word_id IS NULL";
573 $m_num++;
574 }
575
576 foreach ($this->must_exclude_one_ids as $ids)
577 {
578 $is_null_joins = array();
579 foreach ($ids as $id)
580 {
581 if (is_string($id))
582 {
583 $sql_array['LEFT_JOIN'][] = array(
584 'FROM' => array(SEARCH_WORDLIST_TABLE => 'w' . $w_num),
585 'ON' => "w$w_num.word_text LIKE $id"
586 );
587 $id = "w$w_num.word_id";
588
589 $group_by = true;
590 $w_num++;
591 }
592
593 $sql_array['LEFT_JOIN'][] = array(
594 'FROM' => array(SEARCH_WORDMATCH_TABLE => 'm' . $m_num),
595 'ON' => "m$m_num.word_id = $id AND m$m_num.post_id = m0.post_id" . (($title_match) ? " AND m$m_num.$title_match" : '')
596 );
597 $is_null_joins[] = "m$m_num.word_id IS NULL";
598
599 $m_num++;
600 }
601 $sql_where[] = '(' . implode(' OR ', $is_null_joins) . ')';
602 }
603
604 if (!sizeof($m_approve_fid_ary))
605 {
606 $sql_where[] = 'p.post_approved = 1';
607 }
608 else if ($m_approve_fid_ary !== array(-1))
609 {
610 $sql_where[] = '(p.post_approved = 1 OR ' . $db->sql_in_set('p.forum_id', $m_approve_fid_ary, true) . ')';
611 }
612
613 if ($topic_id)
614 {
615 $sql_where[] = 'p.topic_id = ' . $topic_id;
616 }
617
618 if (sizeof($author_ary))
619 {
620 $sql_where[] = $db->sql_in_set('p.poster_id', $author_ary);
621 }
622
623 if (sizeof($ex_fid_ary))
624 {
625 $sql_where[] = $db->sql_in_set('p.forum_id', $ex_fid_ary, true);
626 }
627
628 if ($sort_days)
629 {
630 $sql_where[] = 'p.post_time >= ' . (time() - ($sort_days * 86400));
631 }
632
633 $sql_array['WHERE'] = implode(' AND ', $sql_where);
634
635 $is_mysql = false;
636 // if the total result count is not cached yet, retrieve it from the db
637 if (!$total_results)
638 {
639 $sql = '';
640 $sql_array_count = $sql_array;
641
642 if ($db->dbms_type === 'mysql')
643 {
644 $sql_array['SELECT'] = 'SQL_CALC_FOUND_ROWS ' . $sql_array['SELECT'];
645 $is_mysql = true;
646 }
647 else
648 {
649 if (!$db->count_distinct)
650 {
651 $sql_array_count['SELECT'] = ($type == 'posts') ? 'DISTINCT p.post_id' : 'DISTINCT p.topic_id';
652 $sql = 'SELECT COUNT(' . (($type == 'posts') ? 'post_id' : 'topic_id') . ') as total_results
653 FROM (' . $db->sql_build_query('SELECT', $sql_array_count) . ')';
654 }
655
656 $sql_array_count['SELECT'] = ($type == 'posts') ? 'COUNT(DISTINCT p.post_id) AS total_results' : 'COUNT(DISTINCT p.topic_id) AS total_results';
657 $sql = (!$sql) ? $db->sql_build_query('SELECT', $sql_array_count) : $sql;
658
659 $result = $db->sql_query($sql);
660 $total_results = (int) $db->sql_fetchfield('total_results');
661 $db->sql_freeresult($result);
662
663 if (!$total_results)
664 {
665 return false;
666 }
667 }
668
669 unset($sql_array_count, $sql);
670 }
671
672 // Build sql strings for sorting
673 $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC');
674
675 switch ($sql_sort[0])
676 {
677 case 'u':
678 $sql_array['FROM'][USERS_TABLE] = 'u';
679 $sql_where[] = 'u.user_id = p.poster_id ';
680 break;
681
682 case 't':
683 if (!isset($sql_array['FROM'][TOPICS_TABLE]))
684 {
685 $sql_array['FROM'][TOPICS_TABLE] = 't';
686 $sql_where[] = 'p.topic_id = t.topic_id';
687 }
688 break;
689
690 case 'f':
691 $sql_array['FROM'][FORUMS_TABLE] = 'f';
692 $sql_where[] = 'f.forum_id = p.forum_id';
693 break;
694 }
695
696 $sql_array['WHERE'] = implode(' AND ', $sql_where);
697 $sql_array['GROUP_BY'] = ($group_by) ? (($type == 'posts') ? 'p.post_id' : 'p.topic_id') . ', ' . $sort_by_sql[$sort_key] : '';
698 $sql_array['ORDER_BY'] = $sql_sort;
699
700 unset($sql_where, $sql_sort, $group_by);
701
702 $sql = $db->sql_build_query('SELECT', $sql_array);
703 $result = $db->sql_query_limit($sql, phpbb::$config['search_block_size'], $start);
704
705 while ($row = $db->sql_fetchrow($result))
706 {
707 $id_ary[] = $row[(($type == 'posts') ? 'post_id' : 'topic_id')];
708 }
709 $db->sql_freeresult($result);
710
711 if (!sizeof($id_ary))
712 {
713 return false;
714 }
715
716 // if we use mysql and the total result count is not cached yet, retrieve it from the db
717 if (!$total_results && $is_mysql)
718 {
719 $sql = 'SELECT FOUND_ROWS() as total_results';
720 $result = $db->sql_query($sql);
721 $total_results = (int) $db->sql_fetchfield('total_results');
722 $db->sql_freeresult($result);
723
724 if (!$total_results)
725 {
726 return false;
727 }
728 }
729
730 // store the ids, from start on then delete anything that isn't on the current page because we only need ids for one page
731 $this->save_ids($search_key, $this->search_query, $author_ary, $total_results, $id_ary, $start, $sort_dir);
732 $id_ary = array_slice($id_ary, 0, (int) $per_page);
733
734 return $total_results;
735 }
736
737 /**
738 * Performs a search on an author's posts without caring about message contents. Depends on display specific params
739 *
740 * @param string $type contains either posts or topics depending on what should be searched for
741 * @param boolean $firstpost_only if true, only topic starting posts will be considered
742 * @param array &$sort_by_sql contains SQL code for the ORDER BY part of a query
743 * @param string &$sort_key is the key of $sort_by_sql for the selected sorting
744 * @param string &$sort_dir is either a or d representing ASC and DESC
745 * @param string &$sort_days specifies the maximum amount of days a post may be old
746 * @param array &$ex_fid_ary specifies an array of forum ids which should not be searched
747 * @param array &$m_approve_fid_ary specifies an array of forum ids in which the searcher is allowed to view unapproved posts
748 * @param int &$topic_id is set to 0 or a topic id, if it is not 0 then only posts in this topic should be searched
749 * @param array &$author_ary an array of author ids
750 * @param array &$id_ary passed by reference, to be filled with ids for the page specified by $start and $per_page, should be ordered
751 * @param int $start indicates the first index of the page
752 * @param int $per_page number of ids each page is supposed to contain
753 * @return boolean|int total number of results
754 *
755 * @access public
756 */
757 public function author_search($type, $firstpost_only, &$sort_by_sql, &$sort_key, &$sort_dir, &$sort_days, &$ex_fid_ary, &$m_approve_fid_ary, &$topic_id, &$author_ary, &$id_ary, $start, $per_page)
758 {
759 global $db;
760
761 // No author? No posts.
762 if (!sizeof($author_ary))
763 {
764 return 0;
765 }
766
767 // generate a search_key from all the options to identify the results
768 $search_key = md5(implode('#', array(
769 '',
770 $type,
771 ($firstpost_only) ? 'firstpost' : '',
772 '',
773 '',
774 $sort_days,
775 $sort_key,
776 $topic_id,
777 implode(',', $ex_fid_ary),
778 implode(',', $m_approve_fid_ary),
779 implode(',', $author_ary)
780 )));
781
782 // try reading the results from cache
783 $total_results = 0;
784 if ($this->obtain_ids($search_key, $total_results, $id_ary, $start, $per_page, $sort_dir) == self::SEARCH_RESULT_IN_CACHE)
785 {
786 return $total_results;
787 }
788
789 $id_ary = array();
790
791 // Create some display specific sql strings
792 $sql_author = $db->sql_in_set('p.poster_id', $author_ary);
793 $sql_fora = (sizeof($ex_fid_ary)) ? ' AND ' . $db->sql_in_set('p.forum_id', $ex_fid_ary, true) : '';
794 $sql_time = ($sort_days) ? ' AND p.post_time >= ' . (time() - ($sort_days * 86400)) : '';
795 $sql_topic_id = ($topic_id) ? ' AND p.topic_id = ' . (int) $topic_id : '';
796 $sql_firstpost = ($firstpost_only) ? ' AND p.post_id = t.topic_first_post_id' : '';
797
798 // Build sql strings for sorting
799 $sql_sort = $sort_by_sql[$sort_key] . (($sort_dir == 'a') ? ' ASC' : ' DESC');
800 $sql_sort_table = $sql_sort_join = '';
801 switch ($sql_sort[0])
802 {
803 case 'u':
804 $sql_sort_table = USERS_TABLE . ' u, ';
805 $sql_sort_join = ' AND u.user_id = p.poster_id ';
806 break;
807
808 case 't':
809 $sql_sort_table = ($type == 'posts' && !$firstpost_only) ? TOPICS_TABLE . ' t, ' : '';
810 $sql_sort_join = ($type == 'posts' && !$firstpost_only) ? ' AND t.topic_id = p.topic_id ' : '';
811 break;
812
813 case 'f':
814 $sql_sort_table = FORUMS_TABLE . ' f, ';
815 $sql_sort_join = ' AND f.forum_id = p.forum_id ';
816 break;
817 }
818
819 if (!sizeof($m_approve_fid_ary))
820 {
821 $m_approve_fid_sql = ' AND p.post_approved = 1';
822 }
823 else if ($m_approve_fid_ary == array(-1))
824 {
825 $m_approve_fid_sql = '';
826 }
827 else
828 {
829 $m_approve_fid_sql = ' AND (p.post_approved = 1 OR ' . $db->sql_in_set('p.forum_id', $m_approve_fid_ary, true) . ')';
830 }
831
832 $select = ($type == 'posts') ? 'p.post_id' : 't.topic_id';
833 $is_mysql = false;
834
835 // If the cache was completely empty count the results
836 if (!$total_results)
837 {
838 if ($db->dbms_type === 'mysql')
839 {
840 $select = 'SQL_CALC_FOUND_ROWS ' . $select;
841 $is_mysql = true;
842 }
843 else
844 {
845 if ($type == 'posts')
846 {
847 $sql = 'SELECT COUNT(p.post_id) as total_results
848 FROM ' . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t ' : ' ') . "
849 WHERE $sql_author
850 $sql_topic_id
851 $sql_firstpost
852 $m_approve_fid_sql
853 $sql_fora
854 $sql_time";
855 }
856 else
857 {
858 if ($db->count_distinct)
859 {
860 $sql = 'SELECT COUNT(DISTINCT t.topic_id) as total_results';
861 }
862 else
863 {
864 $sql = 'SELECT COUNT(topic_id) as total_results
865 FROM (SELECT DISTINCT t.topic_id';
866 }
867
868 $sql .= ' FROM ' . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p
869 WHERE $sql_author
870 $sql_topic_id
871 $sql_firstpost
872 $m_approve_fid_sql
873 $sql_fora
874 AND t.topic_id = p.topic_id
875 $sql_time" . (($db->count_distinct) ? '' : ')');
876 }
877 $result = $db->sql_query($sql);
878
879 $total_results = (int) $db->sql_fetchfield('total_results');
880 $db->sql_freeresult($result);
881
882 if (!$total_results)
883 {
884 return false;
885 }
886 }
887 }
888
889 // Build the query for really selecting the post_ids
890 if ($type == 'posts')
891 {
892 $sql = "SELECT $select
893 FROM " . $sql_sort_table . POSTS_TABLE . ' p' . (($firstpost_only) ? ', ' . TOPICS_TABLE . ' t' : '') . "
894 WHERE $sql_author
895 $sql_topic_id
896 $sql_firstpost
897 $m_approve_fid_sql
898 $sql_fora
899 $sql_sort_join
900 $sql_time
901 ORDER BY $sql_sort";
902 $field = 'post_id';
903 }
904 else
905 {
906 $sql = "SELECT $select
907 FROM " . $sql_sort_table . TOPICS_TABLE . ' t, ' . POSTS_TABLE . " p
908 WHERE $sql_author
909 $sql_topic_id
910 $sql_firstpost
911 $m_approve_fid_sql
912 $sql_fora
913 AND t.topic_id = p.topic_id
914 $sql_sort_join
915 $sql_time
916 GROUP BY t.topic_id, " . $sort_by_sql[$sort_key] . '
917 ORDER BY ' . $sql_sort;
918 $field = 'topic_id';
919 }
920
921 // Only read one block of posts from the db and then cache it
922 $result = $db->sql_query_limit($sql, phpbb::$config['search_block_size'], $start);
923
924 while ($row = $db->sql_fetchrow($result))
925 {
926 $id_ary[] = $row[$field];
927 }
928 $db->sql_freeresult($result);
929
930 if (!$total_results && $is_mysql)
931 {
932 $sql = 'SELECT FOUND_ROWS() as total_results';
933 $result = $db->sql_query($sql);
934 $total_results = (int) $db->sql_fetchfield('total_results');
935 $db->sql_freeresult($result);
936
937 if (!$total_results)
938 {
939 return false;
940 }
941 }
942
943 if (sizeof($id_ary))
944 {
945 $this->save_ids($search_key, '', $author_ary, $total_results, $id_ary, $start, $sort_dir);
946 $id_ary = array_slice($id_ary, 0, $per_page);
947
948 return $total_results;
949 }
950 return false;
951 }
952
953 /**
954 * Split a text into words of a given length
955 *
956 * The text is converted to UTF-8, cleaned up, and split. Then, words that
957 * conform to the defined length range are returned in an array.
958 *
959 * NOTE: duplicates are NOT removed from the return array
960 *
961 * @param string $text Text to split, encoded in UTF-8
962 * @return array Array of UTF-8 words
963 *
964 * @access private
965 */
966 private function split_message($text)
967 {
968 global $user;
969
970 $match = $words = array();
971
972 /**
973 * Taken from the original code
974 */
975 // Do not index code
976 $match[] = '#\[code(?:=.*?)?(\:?[0-9a-z]{5,})\].*?\[\/code(\:?[0-9a-z]{5,})\]#is';
977 // BBcode
978 $match[] = '#\[\/?[a-z0-9\*\+\-]+(?:=.*?)?(?::[a-z])?(\:?[0-9a-z]{5,})\]#';
979
980 $min = $this->word_length['min'];
981 $max = $this->word_length['max'];
982
983 $isset_min = $min - 1;
984
985 /**
986 * Clean up the string, remove HTML tags, remove BBCodes
987 */
988 $word = strtok($this->cleanup(preg_replace($match, ' ', strip_tags($text)), -1), ' ');
989
990 while (strlen($word))
991 {
992 if (strlen($word) > 255 || strlen($word) <= $isset_min)
993 {
994 /**
995 * Words longer than 255 bytes are ignored. This will have to be
996 * changed whenever we change the length of search_wordlist.word_text
997 *
998 * Words shorter than $isset_min bytes are ignored, too
999 */
1000 $word = strtok(' ');
1001 continue;
1002 }
1003
1004 $len = utf8_strlen($word);
1005
1006 /**
1007 * Test whether the word is too short to be indexed.
1008 *
1009 * Note that this limit does NOT apply to CJK and Hangul
1010 */
1011 if ($len < $min)
1012 {
1013 /**
1014 * Note: this could be optimized. If the codepoint is lower than Hangul's range
1015 * we know that it will also be lower than CJK ranges
1016 */
1017 if ((strncmp($word, utf_normalizer::UTF8_HANGUL_FIRST, 3) < 0 || strncmp($word, utf_normalizer::UTF8_HANGUL_LAST, 3) > 0)
1018 && (strncmp($word, utf_normalizer::UTF8_CJK_FIRST, 3) < 0 || strncmp($word, utf_normalizer::UTF8_CJK_LAST, 3) > 0)
1019 && (strncmp($word, utf_normalizer::UTF8_CJK_B_FIRST, 4) < 0 || strncmp($word, utf_normalizer::UTF8_CJK_B_LAST, 4) > 0))
1020 {
1021 $word = strtok(' ');
1022 continue;
1023 }
1024 }
1025
1026 $words[] = $word;
1027 $word = strtok(' ');
1028 }
1029
1030 return $words;
1031 }
1032
1033 /**
1034 * Updates wordlist and wordmatch tables when a message is posted or changed
1035 *
1036 * @param string $mode Contains the post mode: edit, post, reply, quote
1037 * @param int $post_id The id of the post which is modified/created
1038 * @param string &$message New or updated post content
1039 * @param string &$subject New or updated post subject
1040 * @param int $poster_id Post author's user id
1041 * @param int $forum_id The id of the forum in which the post is located
1042 *
1043 * @access public
1044 */
1045 public function index($mode, $post_id, &$message, &$subject, $poster_id, $forum_id)
1046 {
1047 global $db, $user;
1048
1049 if (!phpbb::$config['fulltext_native_load_upd'])
1050 {
1051 /**
1052 * The search indexer is disabled, return
1053 */
1054 return;
1055 }
1056
1057 // Split old and new post/subject to obtain array of 'words'
1058 $split_text = $this->split_message($message);
1059 $split_title = $this->split_message($subject);
1060
1061 $cur_words = array('post' => array(), 'title' => array());
1062
1063 $words = array();
1064 if ($mode == 'edit')
1065 {
1066 $words['add']['post'] = array();
1067 $words['add']['title'] = array();
1068 $words['del']['post'] = array();
1069 $words['del']['title'] = array();
1070
1071 $sql = 'SELECT w.word_id, w.word_text, m.title_match
1072 FROM ' . SEARCH_WORDLIST_TABLE . ' w, ' . SEARCH_WORDMATCH_TABLE . " m
1073 WHERE m.post_id = $post_id
1074 AND w.word_id = m.word_id";
1075 $result = $db->sql_query($sql);
1076
1077 while ($row = $db->sql_fetchrow($result))
1078 {
1079 $which = ($row['title_match']) ? 'title' : 'post';
1080 $cur_words[$which][$row['word_text']] = $row['word_id'];
1081 }
1082 $db->sql_freeresult($result);
1083
1084 $words['add']['post'] = array_diff($split_text, array_keys($cur_words['post']));
1085 $words['add']['title'] = array_diff($split_title, array_keys($cur_words['title']));
1086 $words['del']['post'] = array_diff(array_keys($cur_words['post']), $split_text);
1087 $words['del']['title'] = array_diff(array_keys($cur_words['title']), $split_title);
1088 }
1089 else
1090 {
1091 $words['add']['post'] = $split_text;
1092 $words['add']['title'] = $split_title;
1093 $words['del']['post'] = array();
1094 $words['del']['title'] = array();
1095 }
1096 unset($split_text);
1097 unset($split_title);
1098
1099 // Get unique words from the above arrays
1100 $unique_add_words = array_unique(array_merge($words['add']['post'], $words['add']['title']));
1101
1102 // We now have unique arrays of all words to be added and removed and
1103 // individual arrays of added and removed words for text and title. What
1104 // we need to do now is add the new words (if they don't already exist)
1105 // and then add (or remove) matches between the words and this post
1106 if (sizeof($unique_add_words))
1107 {
1108 $sql = 'SELECT word_id, word_text
1109 FROM ' . SEARCH_WORDLIST_TABLE . '
1110 WHERE ' . $db->sql_in_set('word_text', $unique_add_words);
1111 $result = $db->sql_query($sql);
1112
1113 $word_ids = array();
1114 while ($row = $db->sql_fetchrow($result))
1115 {
1116 $word_ids[$row['word_text']] = $row['word_id'];
1117 }
1118 $db->sql_freeresult($result);
1119 $new_words = array_diff($unique_add_words, array_keys($word_ids));
1120
1121 $db->sql_transaction('begin');
1122 if (sizeof($new_words))
1123 {
1124 $sql_ary = array();
1125
1126 foreach ($new_words as $word)
1127 {
1128 $sql_ary[] = array('word_text' => (string) $word, 'word_count' => 0);
1129 }
1130 $db->sql_return_on_error(true);
1131 $db->sql_multi_insert(SEARCH_WORDLIST_TABLE, $sql_ary);
1132 $db->sql_return_on_error(false);
1133 }
1134 unset($new_words, $sql_ary);
1135 }
1136 else
1137 {
1138 $db->sql_transaction('begin');
1139 }
1140
1141 // now update the search match table, remove links to removed words and add links to new words
1142 foreach ($words['del'] as $word_in => $word_ary)
1143 {
1144 $title_match = ($word_in == 'title') ? 1 : 0;
1145
1146 if (sizeof($word_ary))
1147 {
1148 $sql_in = array();
1149 foreach ($word_ary as $word)
1150 {
1151 $sql_in[] = $cur_words[$word_in][$word];
1152 }
1153
1154 $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . '
1155 WHERE ' . $db->sql_in_set('word_id', $sql_in) . '
1156 AND post_id = ' . intval($post_id) . "
1157 AND title_match = $title_match";
1158 $db->sql_query($sql);
1159
1160 $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1161 SET word_count = word_count - 1
1162 WHERE ' . $db->sql_in_set('word_id', $sql_in) . '
1163 AND word_count > 0';
1164 $db->sql_query($sql);
1165
1166 unset($sql_in);
1167 }
1168 }
1169
1170 $db->sql_return_on_error(true);
1171 foreach ($words['add'] as $word_in => $word_ary)
1172 {
1173 $title_match = ($word_in == 'title') ? 1 : 0;
1174
1175 if (sizeof($word_ary))
1176 {
1177 $sql = 'INSERT INTO ' . SEARCH_WORDMATCH_TABLE . ' (post_id, word_id, title_match)
1178 SELECT ' . (int) $post_id . ', word_id, ' . (int) $title_match . '
1179 FROM ' . SEARCH_WORDLIST_TABLE . '
1180 WHERE ' . $db->sql_in_set('word_text', $word_ary);
1181 $db->sql_query($sql);
1182
1183 $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1184 SET word_count = word_count + 1
1185 WHERE ' . $db->sql_in_set('word_text', $word_ary);
1186 $db->sql_query($sql);
1187 }
1188 }
1189 $db->sql_return_on_error(false);
1190
1191 $db->sql_transaction('commit');
1192
1193 // destroy cached search results containing any of the words removed or added
1194 $this->destroy_cache(array_unique(array_merge($words['add']['post'], $words['add']['title'], $words['del']['post'], $words['del']['title'])), array($poster_id));
1195
1196 unset($unique_add_words);
1197 unset($words);
1198 unset($cur_words);
1199 }
1200
1201 /**
1202 * Removes entries from the wordmatch table for the specified post_ids
1203 */
1204 public function index_remove($post_ids, $author_ids, $forum_ids)
1205 {
1206 global $db;
1207
1208 if (sizeof($post_ids))
1209 {
1210 $sql = 'SELECT w.word_id, w.word_text, m.title_match
1211 FROM ' . SEARCH_WORDMATCH_TABLE . ' m, ' . SEARCH_WORDLIST_TABLE . ' w
1212 WHERE ' . $db->sql_in_set('m.post_id', $post_ids) . '
1213 AND w.word_id = m.word_id';
1214 $result = $db->sql_query($sql);
1215
1216 $message_word_ids = $title_word_ids = $word_texts = array();
1217 while ($row = $db->sql_fetchrow($result))
1218 {
1219 if ($row['title_match'])
1220 {
1221 $title_word_ids[] = $row['word_id'];
1222 }
1223 else
1224 {
1225 $message_word_ids[] = $row['word_id'];
1226 }
1227 $word_texts[] = $row['word_text'];
1228 }
1229 $db->sql_freeresult($result);
1230
1231 if (sizeof($title_word_ids))
1232 {
1233 $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1234 SET word_count = word_count - 1
1235 WHERE ' . $db->sql_in_set('word_id', $title_word_ids) . '
1236 AND word_count > 0';
1237 $db->sql_query($sql);
1238 }
1239
1240 if (sizeof($message_word_ids))
1241 {
1242 $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1243 SET word_count = word_count - 1
1244 WHERE ' . $db->sql_in_set('word_id', $message_word_ids) . '
1245 AND word_count > 0';
1246 $db->sql_query($sql);
1247 }
1248
1249 unset($title_word_ids);
1250 unset($message_word_ids);
1251
1252 $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . '
1253 WHERE ' . $db->sql_in_set('post_id', $post_ids);
1254 $db->sql_query($sql);
1255 }
1256
1257 $this->destroy_cache(array_unique($word_texts), $author_ids);
1258 }
1259
1260 /**
1261 * Tidy up indexes: Tag 'common words' and remove
1262 * words no longer referenced in the match table
1263 */
1264 public function tidy()
1265 {
1266 global $db;
1267
1268 // Is the fulltext indexer disabled? If yes then we need not
1269 // carry on ... it's okay ... I know when I'm not wanted boo hoo
1270 if (!phpbb::$config['fulltext_native_load_upd'])
1271 {
1272 set_config('search_last_gc', time(), true);
1273 return;
1274 }
1275
1276 $destroy_cache_words = array();
1277
1278 // Remove common words
1279 if (phpbb::$config['num_posts'] >= 100 && phpbb::$config['fulltext_native_common_thres'])
1280 {
1281 $common_threshold = ((double) phpbb::$config['fulltext_native_common_thres']) / 100.0;
1282 // First, get the IDs of common words
1283 $sql = 'SELECT word_id, word_text
1284 FROM ' . SEARCH_WORDLIST_TABLE . '
1285 WHERE word_count > ' . floor(phpbb::$config['num_posts'] * $common_threshold) . '
1286 OR word_common = 1';
1287 $result = $db->sql_query($sql);
1288
1289 $sql_in = array();
1290 while ($row = $db->sql_fetchrow($result))
1291 {
1292 $sql_in[] = $row['word_id'];
1293 $destroy_cache_words[] = $row['word_text'];
1294 }
1295 $db->sql_freeresult($result);
1296
1297 if (sizeof($sql_in))
1298 {
1299 // Flag the words
1300 $sql = 'UPDATE ' . SEARCH_WORDLIST_TABLE . '
1301 SET word_common = 1
1302 WHERE ' . $db->sql_in_set('word_id', $sql_in);
1303 $db->sql_query($sql);
1304
1305 // by setting search_last_gc to the new time here we make sure that if a user reloads because the
1306 // following query takes too long, he won't run into it again
1307 set_config('search_last_gc', time(), true);
1308
1309 // Delete the matches
1310 $sql = 'DELETE FROM ' . SEARCH_WORDMATCH_TABLE . '
1311 WHERE ' . $db->sql_in_set('word_id', $sql_in);
1312 $db->sql_query($sql);
1313 }
1314 unset($sql_in);
1315 }
1316
1317 if (sizeof($destroy_cache_words))
1318 {
1319 // destroy cached search results containing any of the words that are now common or were removed
1320 $this->destroy_cache(array_unique($destroy_cache_words));
1321 }
1322
1323 set_config('search_last_gc', time(), true);
1324 }
1325
1326 /**
1327 * Deletes all words from the index
1328 */
1329 public function delete_index($acp_module, $u_action)
1330 {
1331 global $db;
1332
1333 if ($db->truncate)
1334 {
1335 $db->sql_query('TRUNCATE TABLE ' . SEARCH_WORDLIST_TABLE);
1336 $db->sql_query('TRUNCATE TABLE ' . SEARCH_WORDMATCH_TABLE);
1337 $db->sql_query('TRUNCATE TABLE ' . SEARCH_RESULTS_TABLE);
1338 }
1339 else
1340 {
1341 $db->sql_query('DELETE FROM ' . SEARCH_WORDLIST_TABLE);
1342 $db->sql_query('DELETE FROM ' . SEARCH_WORDMATCH_TABLE);
1343 $db->sql_query('DELETE FROM ' . SEARCH_RESULTS_TABLE);
1344 }
1345 }
1346
1347 /**
1348 * Returns true if both FULLTEXT indexes exist
1349 */
1350 public function index_created()
1351 {
1352 if (!sizeof($this->stats))
1353 {
1354 $this->get_stats();
1355 }
1356
1357 return ($this->stats['total_words'] && $this->stats['total_matches']) ? true : false;
1358 }
1359
1360 /**
1361 * Returns an associative array containing information about the indexes
1362 */
1363 public function index_stats()
1364 {
1365 global $user;
1366
1367 if (!sizeof($this->stats))
1368 {
1369 $this->get_stats();
1370 }
1371
1372 return array(
1373 $user->lang['TOTAL_WORDS'] => $this->stats['total_words'],
1374 $user->lang['TOTAL_MATCHES'] => $this->stats['total_matches']);
1375 }
1376
1377 private function get_stats()
1378 {
1379 global $db;
1380
1381 $sql = 'SELECT COUNT(*) as total_words
1382 FROM ' . SEARCH_WORDLIST_TABLE;
1383 $result = $db->sql_query($sql);
1384 $this->stats['total_words'] = (int) $db->sql_fetchfield('total_words');
1385 $db->sql_freeresult($result);
1386
1387 $sql = 'SELECT COUNT(*) as total_matches
1388 FROM ' . SEARCH_WORDMATCH_TABLE;
1389 $result = $db->sql_query($sql);
1390 $this->stats['total_matches'] = (int) $db->sql_fetchfield('total_matches');
1391 $db->sql_freeresult($result);
1392 }
1393
1394 /**
1395 * Clean up a text to remove non-alphanumeric characters
1396 *
1397 * This method receives a UTF-8 string, normalizes and validates it, replaces all
1398 * non-alphanumeric characters with strings then returns the result.
1399 *
1400 * Any number of "allowed chars" can be passed as a UTF-8 string in NFC.
1401 *
1402 * @param string $text Text to split, in UTF-8 (not normalized or sanitized)
1403 * @param string $allowed_chars String of special chars to allow
1404 * @return string Cleaned up text, only alphanumeric chars are left
1405 *
1406 * @todo normalizer::cleanup being able to be used?
1407 */
1408 private function cleanup($text, $allowed_chars = null)
1409 {
1410 static $conv = array(), $conv_loaded = array();
1411 $words = $allow = array();
1412
1413 $utf_len_mask = array(
1414 "\xC0" => 2,
1415 "\xD0" => 2,
1416 "\xE0" => 3,
1417 "\xF0" => 4
1418 );
1419
1420 /**
1421 * Replace HTML entities and NCRs
1422 */
1423 $text = htmlspecialchars_decode(utf8_decode_ncr($text), ENT_QUOTES);
1424
1425 /**
1426 * Load the UTF-8 normalizer
1427 *
1428 * If we use it more widely, an instance of that class should be held in a
1429 * a global variable instead
1430 */
1431 utf_normalizer::nfc($text);
1432
1433 /**
1434 * The first thing we do is:
1435 *
1436 * - convert ASCII-7 letters to lowercase
1437 * - remove the ASCII-7 non-alpha characters
1438 * - remove the bytes that should not appear in a valid UTF-8 string: 0xC0,
1439 * 0xC1 and 0xF5-0xFF
1440 *
1441 * @todo in theory, the third one is already taken care of during normalization and those chars should have been replaced by Unicode replacement chars
1442 */
1443 $sb_match = "ISTCPAMELRDOJBNHFGVWUQKYXZ\r\n\t!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\x7f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x0B\x0C\x0E\x0F\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1A\x1B\x1C\x1D\x1E\x1F\xC0\xC1\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF";
1444 $sb_replace = 'istcpamelrdojbnhfgvwuqkyxz ';
1445
1446 /**
1447 * This is the list of legal ASCII chars, it is automatically extended
1448 * with ASCII chars from $allowed_chars
1449 */
1450 $legal_ascii = ' eaisntroludcpmghbfvq10xy2j9kw354867z';
1451
1452 /**
1453 * Prepare an array containing the extra chars to allow
1454 */
1455 if (isset($allowed_chars[0]))
1456 {
1457 $pos = 0;
1458 $len = strlen($allowed_chars);
1459 do
1460 {
1461 $c = $allowed_chars[$pos];
1462
1463 if ($c < "\x80")
1464 {
1465 /**
1466 * ASCII char
1467 */
1468 $sb_pos = strpos($sb_match, $c);
1469 if (is_int($sb_pos))
1470 {
1471 /**
1472 * Remove the char from $sb_match and its corresponding
1473 * replacement in $sb_replace
1474 */
1475 $sb_match = substr($sb_match, 0, $sb_pos) . substr($sb_match, $sb_pos + 1);
1476 $sb_replace = substr($sb_replace, 0, $sb_pos) . substr($sb_replace, $sb_pos + 1);
1477 $legal_ascii .= $c;
1478 }
1479
1480 ++$pos;
1481 }
1482 else
1483 {
1484 /**
1485 * UTF-8 char
1486 */
1487 $utf_len = $utf_len_mask[$c & "\xF0"];
1488 $allow[substr($allowed_chars, $pos, $utf_len)] = 1;
1489 $pos += $utf_len;
1490 }
1491 }
1492 while ($pos < $len);
1493 }
1494
1495 $text = strtr($text, $sb_match, $sb_replace);
1496 $ret = '';
1497
1498 $pos = 0;
1499 $len = strlen($text);
1500
1501 do
1502 {
1503 /**
1504 * Do all consecutive ASCII chars at once
1505 */
1506 if ($spn = strspn($text, $legal_ascii, $pos))
1507 {
1508 $ret .= substr($text, $pos, $spn);
1509 $pos += $spn;
1510 }
1511
1512 if ($pos >= $len)
1513 {
1514 return $ret;
1515 }
1516
1517 /**
1518 * Capture the UTF char
1519 */
1520 $utf_len = $utf_len_mask[$text[$pos] & "\xF0"];
1521 $utf_char = substr($text, $pos, $utf_len);
1522 $pos += $utf_len;
1523
1524 if (($utf_char >= UTF8_HANGUL_FIRST && $utf_char <= UTF8_HANGUL_LAST)
1525 || ($utf_char >= UTF8_CJK_FIRST && $utf_char <= UTF8_CJK_LAST)
1526 || ($utf_char >= UTF8_CJK_B_FIRST && $utf_char <= UTF8_CJK_B_LAST))
1527 {
1528 /**
1529 * All characters within these ranges are valid
1530 *
1531 * We separate them with a space in order to index each character
1532 * individually
1533 */
1534 $ret .= ' ' . $utf_char . ' ';
1535 continue;
1536 }
1537
1538 if (isset($allow[$utf_char]))
1539 {
1540 /**
1541 * The char is explicitly allowed
1542 */
1543 $ret .= $utf_char;
1544 continue;
1545 }
1546
1547 if (isset($conv[$utf_char]))
1548 {
1549 /**
1550 * The char is mapped to something, maybe to itself actually
1551 */
1552 $ret .= $conv[$utf_char];
1553 continue;
1554 }
1555
1556 /**
1557 * The char isn't mapped, but did we load its conversion table?
1558 *
1559 * The search indexer table is split into blocks. The block number of
1560 * each char is equal to its codepoint right-shifted for 11 bits. It
1561 * means that out of the 11, 16 or 21 meaningful bits of a 2-, 3- or
1562 * 4- byte sequence we only keep the leftmost 0, 5 or 10 bits. Thus,
1563 * all UTF chars encoded in 2 bytes are in the same first block.
1564 */
1565 if (isset($utf_char[2]))
1566 {
1567 if (isset($utf_char[3]))
1568 {
1569 /**
1570 * 1111 0nnn 10nn nnnn 10nx xxxx 10xx xxxx
1571 * 0000 0111 0011 1111 0010 0000
1572 */
1573 $idx = ((ord($utf_char[0]) & 0x07) << 7) | ((ord($utf_char[1]) & 0x3F) << 1) | ((ord($utf_char[2]) & 0x20) >> 5);
1574 }
1575 else
1576 {
1577 /**
1578 * 1110 nnnn 10nx xxxx 10xx xxxx
1579 * 0000 0111 0010 0000
1580 */
1581 $idx = ((ord($utf_char[0]) & 0x07) << 1) | ((ord($utf_char[1]) & 0x20) >> 5);
1582 }
1583 }
1584 else
1585 {
1586 /**
1587 * 110x xxxx 10xx xxxx
1588 * 0000 0000 0000 0000
1589 */
1590 $idx = 0;
1591 }
1592
1593 /**
1594 * Check if the required conv table has been loaded already
1595 */
1596 if (!isset($conv_loaded[$idx]))
1597 {
1598 $conv_loaded[$idx] = 1;
1599 $file = PHPBB_ROOT_PATH . 'includes/utf/data/search_indexer_' . $idx . '.' . PHP_EXT;
1600
1601 if (file_exists($file))
1602 {
1603 $conv += include($file);
1604 }
1605 }
1606
1607 if (isset($conv[$utf_char]))
1608 {
1609 $ret .= $conv[$utf_char];
1610 }
1611 else
1612 {
1613 /**
1614 * We add an entry to the conversion table so that we
1615 * don't have to convert to codepoint and perform the checks
1616 * that are above this block
1617 */
1618 $conv[$utf_char] = ' ';
1619 $ret .= ' ';
1620 }
1621 }
1622 while (1);
1623
1624 return $ret;
1625 }
1626
1627 /**
1628 * Returns a list of options for the ACP to display
1629 */
1630 public function acp()
1631 {
1632 global $user;
1633
1634
1635 /**
1636 * if we need any options, copied from fulltext_native for now, will have to be adjusted or removed
1637 */
1638
1639 $tpl = '
1640 <dl>
1641 <dt><label for="fulltext_native_load_upd">' . $user->lang['YES_SEARCH_UPDATE'] . ':</label><br /><span>' . $user->lang['YES_SEARCH_UPDATE_EXPLAIN'] . '</span></dt>
1642 <dd><label><input type="radio" id="fulltext_native_load_upd" name="config[fulltext_native_load_upd]" value="1"' . ((phpbb::$config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $user->lang['YES'] . '</label><label><input type="radio" name="config[fulltext_native_load_upd]" value="0"' . ((!phpbb::$config['fulltext_native_load_upd']) ? ' checked="checked"' : '') . ' class="radio" /> ' . $user->lang['NO'] . '</label></dd>
1643 </dl>
1644 <dl>
1645 <dt><label for="fulltext_native_min_chars">' . $user->lang['MIN_SEARCH_CHARS'] . ':</label><br /><span>' . $user->lang['MIN_SEARCH_CHARS_EXPLAIN'] . '</span></dt>
1646 <dd><input id="fulltext_native_min_chars" type="text" size="3" maxlength="3" name="config[fulltext_native_min_chars]" value="' . (int) phpbb::$config['fulltext_native_min_chars'] . '" /></dd>
1647 </dl>
1648 <dl>
1649 <dt><label for="fulltext_native_max_chars">' . $user->lang['MAX_SEARCH_CHARS'] . ':</label><br /><span>' . $user->lang['MAX_SEARCH_CHARS_EXPLAIN'] . '</span></dt>
1650 <dd><input id="fulltext_native_max_chars" type="text" size="3" maxlength="3" name="config[fulltext_native_max_chars]" value="' . (int) phpbb::$config['fulltext_native_max_chars'] . '" /></dd>
1651 </dl>
1652 <dl>
1653 <dt><label for="fulltext_native_common_thres">' . $user->lang['COMMON_WORD_THRESHOLD'] . ':</label><br /><span>' . $user->lang['COMMON_WORD_THRESHOLD_EXPLAIN'] . '</span></dt>
1654 <dd><input id="fulltext_native_common_thres" type="text" size="3" maxlength="3" name="config[fulltext_native_common_thres]" value="' . (int) phpbb::$config['fulltext_native_common_thres'] . '" /> %</dd>
1655 </dl>
1656 ';
1657
1658 // These are fields required in the config table
1659 return array(
1660 'tpl' => $tpl,
1661 'config' => array('fulltext_native_load_upd' => 'bool', 'fulltext_native_min_chars' => 'integer:0:255', 'fulltext_native_max_chars' => 'integer:0:255', 'fulltext_native_common_thres' => 'double:0:100')
1662 );
1663 }
1664 }
1665
1666 ?>
phpbb is a php-based BBS (buletin board system)