1 /* omega.cc: Main module for omega (example CGI frontend for Xapian)
3 * Copyright 1999,2000,2001 BrightStation PLC
4 * Copyright 2001 James Aylett
5 * Copyright 2001,2002 Ananova Ltd
6 * Copyright 2002,2003,2004,2006,2007,2008,2009,2010,2011,2014,2015,2016 Olly Betts
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License as
10 * published by the Free Software Foundation; either version 2 of the
11 * License, or (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301
26 // If we're building against git after the expand API changed but before the
27 // version gets bumped to 1.3.2, we'll get a deprecation warning from
28 // get_eset() unless we suppress such warnings here.
29 #define XAPIAN_DEPRECATED(D) D
39 #include "safefcntl.h"
40 #include "safeunistd.h"
47 #include "stringutils.h"
52 static const char DEFAULT_STEM_LANGUAGE
[] = "english";
54 // A character which doesn't require URL encoding, and isn't likely to appear
56 const char filter_sep
= '~';
58 // What we used for filter_sep in Omega < 1.3.4.
59 const char filter_sep_old
= '-';
61 Xapian::Enquire
* enquire
;
65 map
<string
, string
> option
;
67 string date_start
, date_end
, date_span
;
68 Xapian::valueno date_value_slot
= Xapian::BAD_VALUENO
;
70 bool set_content_type
= false;
72 bool suppress_http_headers
= false;
76 string filters
, old_filters
;
78 Xapian::docid topdoc
= 0;
79 Xapian::docid hits_per_page
= 0;
80 Xapian::docid min_hits
= 0;
85 Xapian::MultiValueKeyMaker
* sort_keymaker
= NULL
;
86 Xapian::valueno sort_key
= Xapian::BAD_VALUENO
; // Don't sort.
87 bool reverse_sort
= true;
88 bool sort_after
= false;
89 Xapian::Enquire::docid_order docid_order
= Xapian::Enquire::ASCENDING
;
91 Xapian::valueno collapse_key
= 0;
92 bool collapse
= false;
95 map_dbname_to_dir(const string
&database_name
)
97 return database_dir
+ database_name
;
100 int main(int argc
, char *argv
[])
107 option
["flag_default"] = "true";
109 // set default thousands and decimal separators: e.g. "16,729 hits" "1.4K"
110 option
["decimal"] = ".";
111 option
["thousand"] = ",";
113 // set the default stemming language
114 option
["stemmer"] = DEFAULT_STEM_LANGUAGE
;
116 // FIXME: set cout to linebuffered not stdout. Or just flush regularly...
117 // setvbuf(stdout, NULL, _IOLBF, 0);
119 const char * method
= getenv("REQUEST_METHOD");
120 if (method
== NULL
) {
121 if (argc
> 1 && (argv
[1][0] != '-' || strchr(argv
[1], '='))) {
122 // omega 'P=information retrieval' DB=papers
123 // check for a leading '-' on the first arg so "omega --version",
124 // "omega --help", and similar take the next branch
125 decode_argv(argv
+ 1);
127 // Seems we're running from the command line so give version
128 // and allow a query to be entered for testing
129 cout
<< PROGRAM_NAME
" - " PACKAGE
" " VERSION
"\n";
130 if (argc
> 1) exit(0);
131 cout
<< "Enter NAME=VALUE lines, end with blank line\n";
142 // get database(s) to search
144 set
<string
> seen
; // only add a repeated db once
145 g
= cgi_params
.equal_range("DB");
146 for (MCI i
= g
.first
; i
!= g
.second
; ++i
) {
147 const string
& v
= i
->second
;
152 string
s(v
, p
, q
- p
);
153 if (!s
.empty() && seen
.find(s
) == seen
.end()) {
154 // Translate DB parameter to path of database directory
155 if (!dbname
.empty()) dbname
+= '/';
157 db
.add_database(Xapian::Database(map_dbname_to_dir(s
)));
160 if (q
== string::npos
) break;
165 if (dbname
.empty()) {
167 db
.add_database(Xapian::Database(map_dbname_to_dir(dbname
)));
169 enquire
= new Xapian::Enquire(db
);
170 } catch (const Xapian::Error
&) {
175 val
= cgi_params
.find("HITSPERPAGE");
176 if (val
!= cgi_params
.end()) hits_per_page
= atol(val
->second
.c_str());
177 if (hits_per_page
== 0) {
179 } else if (hits_per_page
> 1000) {
180 hits_per_page
= 1000;
183 val
= cgi_params
.find("DEFAULTOP");
184 if (val
!= cgi_params
.end()) {
185 const string
& v
= val
->second
;
186 if (v
== "OR" || v
== "or")
187 default_op
= Xapian::Query::OP_OR
;
190 val
= cgi_params
.find("FMT");
191 if (val
!= cgi_params
.end()) {
192 const string
& v
= val
->second
;
193 if (!v
.empty()) fmtname
= v
;
196 fmtname
= default_template
;
198 val
= cgi_params
.find("MORELIKE");
199 if (enquire
&& val
!= cgi_params
.end()) {
200 const string
& v
= val
->second
;
201 Xapian::docid docid
= atol(v
.c_str());
203 // Assume it's MORELIKE=Quid1138 and that Quid1138 is a UID
204 // from an external source - we just find the correspond docid
205 Xapian::PostingIterator p
= db
.postlist_begin(v
);
206 if (p
!= db
.postlist_end(v
)) docid
= *p
;
210 Xapian::RSet tmprset
;
211 tmprset
.add_document(docid
);
213 OmegaExpandDecider
decider(db
);
214 set_expansion_scheme(*enquire
, option
);
215 Xapian::ESet
eset(enquire
->get_eset(40, tmprset
, &decider
));
216 string morelike_query
;
217 for (auto&& term
: eset
) {
218 if (!morelike_query
.empty()) morelike_query
+= ' ';
219 morelike_query
+= pretty_term(term
);
221 set_probabilistic_query(string(), morelike_query
);
224 // add expand/topterms terms if appropriate
226 if (cgi_params
.find("ADD") != cgi_params
.end()) {
227 g
= cgi_params
.equal_range("X");
228 for (MCI i
= g
.first
; i
!= g
.second
; ++i
) {
229 const string
& v
= i
->second
;
231 if (!expand_terms
.empty())
238 // collect the unprefixed prob fields
239 g
= cgi_params
.equal_range("P");
240 for (MCI i
= g
.first
; i
!= g
.second
; ++i
) {
241 const string
& v
= i
->second
;
243 // If there are expand terms, append them to the first
244 // non-empty P parameter.
245 if (!expand_terms
.empty()) {
249 set_probabilistic_query(string(), q
);
250 expand_terms
= string();
252 set_probabilistic_query(string(), v
);
257 if (!expand_terms
.empty()) {
258 set_probabilistic_query(string(), expand_terms
);
262 g
.first
= cgi_params
.lower_bound("P.");
263 g
.second
= cgi_params
.lower_bound("P/"); // '/' is '.' + 1.
264 for (MCI i
= g
.first
; i
!= g
.second
; ++i
) {
265 const string
& v
= i
->second
;
267 string
pfx(i
->first
, 2, string::npos
);
268 set_probabilistic_query(pfx
, v
);
272 // set any boolean filters
273 g
= cgi_params
.equal_range("B");
274 if (g
.first
!= g
.second
) {
275 vector
<string
> filter_v
;
276 for (MCI i
= g
.first
; i
!= g
.second
; ++i
) {
277 const string
& v
= i
->second
;
278 // we'll definitely get empty B fields from "-ALL-" options
279 if (!v
.empty() && C_isalnum(v
[0])) {
281 filter_v
.push_back(v
);
284 sort(filter_v
.begin(), filter_v
.end());
285 vector
<string
>::const_iterator j
;
286 for (j
= filter_v
.begin(); j
!= filter_v
.end(); ++j
) {
287 const string
& bterm
= *j
;
288 string::size_type e
= bterm
.find(filter_sep
);
289 if (usual(e
== string::npos
)) {
292 // If a filter contains filter_sep then double it to escape.
293 // Each filter must start with an alnum (checked above) and
294 // the value after the last filter is the default op, which
295 // is encoded as a non-alnum so filter_sep followed by
296 // something other than filter_sep must be separating filters.
297 string::size_type b
= 0;
298 while (e
!= string::npos
) {
299 filters
.append(bterm
, b
, e
+ 1 - b
);
301 e
= bterm
.find(filter_sep
, b
+ 1);
303 filters
.append(bterm
, b
, string::npos
);
305 filters
+= filter_sep
;
306 old_filters
+= bterm
;
307 old_filters
+= filter_sep_old
;
311 // set any negated boolean filters
312 g
= cgi_params
.equal_range("N");
313 if (g
.first
!= g
.second
) {
314 vector
<string
> filter_v
;
315 for (MCI i
= g
.first
; i
!= g
.second
; ++i
) {
316 const string
& v
= i
->second
;
317 // we'll definitely get empty N fields from "-ALL-" options
318 if (!v
.empty() && C_isalnum(v
[0])) {
320 filter_v
.push_back(v
);
323 sort(filter_v
.begin(), filter_v
.end());
324 vector
<string
>::const_iterator j
;
325 for (j
= filter_v
.begin(); j
!= filter_v
.end(); ++j
) {
326 const string
& nterm
= *j
;
327 string::size_type e
= nterm
.find(filter_sep
);
329 if (usual(e
== string::npos
)) {
332 // If a filter contains filter_sep then double it to escape.
333 // Each filter must start with an alnum (checked above) and
334 // the value after the last filter is the default op, which
335 // is encoded as a non-alnum so filter_sep followed by
336 // something other than filter_sep must be separating filters.
337 string::size_type b
= 0;
338 while (e
!= string::npos
) {
339 filters
.append(nterm
, b
, e
+ 1 - b
);
341 e
= nterm
.find(filter_sep
, b
+ 1);
343 filters
.append(nterm
, b
, string::npos
);
345 filters
+= filter_sep
;
346 // old_filters predates 'N' terms, so if there are 'N' terms this
347 // is definitely a different query.
352 // date range filters
353 val
= cgi_params
.find("START");
354 if (val
!= cgi_params
.end()) date_start
= val
->second
;
355 val
= cgi_params
.find("END");
356 if (val
!= cgi_params
.end()) date_end
= val
->second
;
357 val
= cgi_params
.find("SPAN");
358 if (val
!= cgi_params
.end()) date_span
= val
->second
;
359 val
= cgi_params
.find("DATEVALUE");
360 if (val
!= cgi_params
.end()) date_value_slot
= string_to_int(val
->second
);
362 // If more default_op values are supported, encode them as non-alnums
363 // other than filter_sep or '!'.
364 filters
+= (default_op
== Xapian::Query::OP_AND
? '.' : '-');
365 filters
+= date_start
;
366 filters
+= filter_sep
;
368 filters
+= filter_sep
;
369 filters
+= date_span
;
370 if (date_value_slot
!= Xapian::BAD_VALUENO
) {
371 // This means we'll force the first page when reloading or changing
372 // page starting from existing URLs upon upgrade to 1.4.1, but the
373 // exact same existing URL could be for a search without the date
374 // filter where we want to force the first page, so there's an inherent
375 // ambiguity there. Forcing first page in this case seems the least
376 // problematic side-effect.
377 filters
+= filter_sep
;
378 filters
+= str(date_value_slot
);
381 if (!old_filters
.empty()) {
382 old_filters
+= date_start
;
383 old_filters
+= filter_sep_old
;
384 old_filters
+= date_end
;
385 old_filters
+= filter_sep_old
;
386 old_filters
+= date_span
;
387 old_filters
+= (default_op
== Xapian::Query::OP_AND
? 'A' : 'O');
390 // Percentage relevance cut-off
391 val
= cgi_params
.find("THRESHOLD");
392 if (val
!= cgi_params
.end()) {
393 threshold
= atoi(val
->second
.c_str());
394 if (threshold
< 0) threshold
= 0;
395 if (threshold
> 100) threshold
= 100;
399 val
= cgi_params
.find("COLLAPSE");
400 if (val
!= cgi_params
.end()) {
401 const string
& v
= val
->second
;
403 collapse_key
= atoi(v
.c_str());
405 filters
+= filter_sep
;
406 filters
+= str(collapse_key
);
407 if (!old_filters
.empty()) {
408 old_filters
+= filter_sep_old
;
409 old_filters
+= str(collapse_key
);
413 if (!collapse
&& date_value_slot
!= Xapian::BAD_VALUENO
) {
414 // We need to either omit filter_sep for both or neither, or else the
415 // encoding is ambiguous.
416 filters
+= filter_sep
;
420 val
= cgi_params
.find("DOCIDORDER");
421 if (val
!= cgi_params
.end()) {
422 const string
& v
= val
->second
;
426 docid_order
= Xapian::Enquire::DESCENDING
;
428 if (!old_filters
.empty()) old_filters
+= 'D';
429 } else if (ch
!= 'A') {
430 docid_order
= Xapian::Enquire::DONT_CARE
;
432 // This is a bug (should add nothing here and 'X' in the (ch !=
433 // 'A') case, but the current "DONT_CARE" implementation
434 // actually always results in ascending docid order so it's not
435 // worth breaking compatibility to fix - let's just do it next
436 // time we change the encoding $filters uses.
438 if (!old_filters
.empty()) old_filters
+= 'X';
444 val
= cgi_params
.find("SORT");
445 if (val
!= cgi_params
.end()) {
446 const char * base
= val
->second
.c_str();
447 const char * p
= base
;
449 bool rev
= (*p
!= '+');
450 if (*p
== '-' || *p
== '+') {
451 // old_filters predates support for direction in SORT, so if
452 // there's a direction specified this is definitely a different
457 if (!C_isdigit(*p
)) {
463 Xapian::valueno slot
= strtoul(p
, &q
, 10);
470 if (sort_key
!= Xapian::BAD_VALUENO
) {
471 // Multiple sort keys specified, so we need a KeyMaker.
474 if (reverse_sort
) filters
+= '-';
475 filters
+= str(sort_key
);
477 sort_keymaker
= new Xapian::MultiValueKeyMaker
;
478 sort_keymaker
->add_value(sort_key
, !reverse_sort
);
479 sort_key
= Xapian::BAD_VALUENO
;
481 // old_filters predates multiple sort keys, so if there are
482 // multiple sort keys this is definitely a different query.
487 filters
+= (rev
? '-' : '+');
488 filters
+= str(slot
);
489 sort_keymaker
->add_value(slot
, !rev
);
494 while (C_isspace(*p
) || *p
== ',') ++p
;
497 val
= cgi_params
.find("SORTREVERSE");
498 if (val
!= cgi_params
.end() && atoi(val
->second
.c_str()) != 0) {
499 reverse_sort
= !reverse_sort
;
502 val
= cgi_params
.find("SORTAFTER");
503 if (val
!= cgi_params
.end()) {
504 sort_after
= (atoi(val
->second
.c_str()) != 0);
507 // Add the sorting related options to filters too.
509 // Note: old_filters really does encode a reversed sort as 'F' and a
510 // non-reversed sort as 'R' or 'r'.
512 // filters has them the other way around for sanity (except in
513 // development snapshot 1.3.4, which was when the new filter encoding
515 if (!sort_keymaker
) filters
+= str(sort_key
);
516 if (!old_filters
.empty()) old_filters
+= str(sort_key
);
520 if (!old_filters
.empty()) old_filters
+= 'F';
523 if (!old_filters
.empty()) old_filters
+= 'R';
528 if (!old_filters
.empty()) old_filters
+= 'r';
533 if (old_filters
.empty()) old_filters
= filters
;
535 // min_hits (fill mset past topdoc+(hits_per_page+1) to
536 // topdoc+max(hits_per_page+1,min_hits)
537 val
= cgi_params
.find("MINHITS");
538 if (val
!= cgi_params
.end()) {
539 min_hits
= atol(val
->second
.c_str());
543 } catch (const Xapian::Error
&e
) {
544 if (!set_content_type
&& !suppress_http_headers
)
545 cout
<< "Content-Type: text/html\n\n";
546 cout
<< "Exception: " << html_escape(e
.get_msg()) << endl
;
547 } catch (const std::exception
&e
) {
548 if (!set_content_type
&& !suppress_http_headers
)
549 cout
<< "Content-Type: text/html\n\n";
550 cout
<< "Exception: std::exception " << html_escape(e
.what()) << endl
;
551 } catch (const string
&s
) {
552 if (!set_content_type
&& !suppress_http_headers
)
553 cout
<< "Content-Type: text/html\n\n";
554 cout
<< "Exception: " << html_escape(s
) << endl
;
555 } catch (const char *s
) {
556 if (!set_content_type
&& !suppress_http_headers
)
557 cout
<< "Content-Type: text/html\n\n";
558 cout
<< "Exception: " << html_escape(s
) << endl
;
560 if (!set_content_type
&& !suppress_http_headers
)
561 cout
<< "Content-Type: text/html\n\n";
562 cout
<< "Caught unknown exception" << endl
;