modules/search-engine.js

   1 /**
   2  * (C) Copyright 2008 Jeremy Maitin-Shepard
   3  *
   4  * Use, modification, and distribution are subject to the terms specified in the
   5  * COPYING file.
   6 **/
   7
   8 in_module(null);
   9
  10 require("utils.js");
  11
  12 var search_engines = new string_hashmap();
  13
  14 function search_engine_parse_error(msg) {
  15     var e = new Error(msg);
  16     e.__proto__ = search_engine_parse_error.prototype;
  17     return e;
  18 }
  19 search_engine_parse_error.prototype.__proto__ = Error.prototype;
  20
  21 function search_engine() {
  22     this.urls = new string_hashmap();
  23 }
  24
  25 function search_engine_url(type, method, template) {
  26     if (!method || !type || !template)
  27         throw search_engine_parse_error("Missing method, type, or template for search engine URL");
  28     method = method.toUpperCase();
  29     type = type.toUpperCase();
  30     if (method != "GET" && method != "POST")
  31         throw search_engine_parse_error("Invalid method");
  32     var template_uri = make_uri(template);
  33     switch (template_uri.scheme) {
  34     case "http":
  35     case "https":
  36         break;
  37     default:
  38         throw search_engine_parse_error("URL template has invalid scheme.");
  39         break;
  40     }
  41     this.type = type;
  42     this.method = method;
  43     this.template = template;
  44     this.params =  [];
  45 }
  46 search_engine_url.prototype.add_param = function search_engine_url__add_param(name, value) {
  47     this.params.push({name: name, value: value});
  48 }
  49
  50 function load_search_engines_in_directory(dir) {
  51     var files = null;
  52     try {
  53         files = dir.directoryEntries.QueryInterface(Ci.nsIDirectoryEnumerator);
  54
  55         while (files.hasMoreElements()) {
  56             var file = files.nextFile;
  57
  58             if (!file.isFile())
  59                 continue;
  60
  61             try {
  62                 load_search_engine_from_file(file);
  63             } catch (e) {
  64                 dumpln("WARNING: Failed to load search engine from file: " + file.path);
  65                 dump_error(e);
  66             }
  67         }
  68     } catch (e) {
  69         // FIXME: maybe have a better error message
  70         dump_error(e);
  71     } finally {
  72         if (files)
  73             files.close();
  74     }
  75 }
  76
  77 function load_search_engine_from_file(file) {
  78     var file_istream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(Ci.nsIFileInputStream);
  79     file_istream.init(file, MODE_RDONLY, 0644, false);
  80     var dom_parser = Cc["@mozilla.org/xmlextras/domparser;1"].createInstance(Ci.nsIDOMParser);
  81     var doc = dom_parser.parseFromStream(file_istream, "UTF-8", file.fileSize, "text/xml");
  82
  83     var eng = parse_search_engine_from_dom_node(doc.documentElement);
  84     search_engines.put(file.leafName, eng);
  85 }
  86
  87 // Supported OpenSearch parameters
  88 // http://www.opensearch.org/Specifications/OpenSearch/1.1#OpenSearch_URL_template_syntax
  89 const OPENSEARCH_PARAM_USER_DEFINED    = /\{searchTerms\??\}/g;
  90 const OPENSEARCH_PARAM_INPUT_ENCODING  = /\{inputEncoding\??\}/g;
  91 const OPENSEARCH_PARAM_LANGUAGE        = /\{language\??\}/g;
  92 const OPENSEARCH_PARAM_OUTPUT_ENCODING = /\{outputEncoding\??\}/g;
  93
  94 // Default values
  95 const OPENSEARCH_PARAM_LANGUAGE_DEF         = "*";
  96 const OPENSEARCH_PARAM_OUTPUT_ENCODING_DEF  = "UTF-8";
  97 const OPENSEARCH_PARAM_INPUT_ENCODING_DEF   = "UTF-8";
  98
  99 // "Unsupported" OpenSearch parameters. For example, we don't support
 100 // page-based results, so if the engine requires that we send the "page index"
 101 // parameter, we'll always send "1".
 102 const OPENSEARCH_PARAM_COUNT        = /\{count\??\}/g;
 103 const OPENSEARCH_PARAM_START_INDEX  = /\{startIndex\??\}/g;
 104 const OPENSEARCH_PARAM_START_PAGE   = /\{startPage\??\}/g;
 105
 106 // Default values
 107 const OPENSEARCH_PARAM_COUNT_DEF        = "20"; // 20 results
 108 const OPENSEARCH_PARAM_START_INDEX_DEF  = "1";  // start at 1st result
 109 const OPENSEARCH_PARAM_START_PAGE_DEF   = "1";  // 1st page
 110
 111 // Optional parameter
 112 const OPENSEARCH_PARAM_OPTIONAL     = /\{(?:\w+:)?\w+\?\}/g;
 113
 114 // A array of arrays containing parameters that we don't fully support, and
 115 // their default values. We will only send values for these parameters if
 116 // required, since our values are just really arbitrary "guesses" that should
 117 // give us the output we want.
 118 var OPENSEARCH_UNSUPPORTED_PARAMS = [
 119   [OPENSEARCH_PARAM_COUNT, OPENSEARCH_PARAM_COUNT_DEF],
 120   [OPENSEARCH_PARAM_START_INDEX, OPENSEARCH_PARAM_START_INDEX_DEF],
 121   [OPENSEARCH_PARAM_START_PAGE, OPENSEARCH_PARAM_START_PAGE_DEF]
 122 ];
 123
 124
 125 function parse_search_engine_from_dom_node(node) {
 126     var eng = new search_engine();
 127     eng.query_charset = OPENSEARCH_PARAM_INPUT_ENCODING_DEF;
 128
 129     for each (let child in node.childNodes) {
 130         switch (child.localName) {
 131         case "ShortName":
 132             eng.name = child.textContent;
 133             break;
 134         case "Description":
 135             eng.description = child.textContent;
 136             break;
 137         case "Url":
 138             try {
 139                 let type = child.getAttribute("type");
 140                 let method = child.getAttribute("method") || "GET";
 141                 let template = child.getAttribute("template");
 142
 143                 let engine_url = new search_engine_url(type, method, template);
 144                 for each (let p in child.childNodes) {
 145                     if (p.localName == "Param") {
 146                         let name = p.getAttribute("name");
 147                         let value = p.getAttribute("value");
 148                         if (name && value)
 149                             engine_url.add_param(name, value);
 150                     }
 151                 }
 152                 eng.urls.put(type, engine_url);
 153             } catch (e) {
 154                 // Skip this element if parsing fails
 155             }
 156             break;
 157         case "InputEncoding":
 158             eng.query_charset = child.textContent.toUpperCase();
 159             break;
 160         }
 161     }
 162     return eng;
 163 }
 164
 165 search_engine.prototype.supports_response_type = function (type) {
 166     return this.urls.contains(type);
 167 };
 168
 169 /**
 170  * Returns null if the result mime_type isn't supported.  The string
 171  * search_terms will be escaped by this function.
 172  */
 173 search_engine.prototype.get_query_load_spec = function search_engine__get_query_load_spec(search_terms, type) {
 174     if (type == null)
 175         type = "text/html";
 176     var url = this.urls.get(type);
 177     if (!url)
 178         return null;
 179     search_terms = encodeURIComponent(search_terms);
 180     var eng = this;
 181
 182     function substitute(value) {
 183         // Insert the OpenSearch parameters we're confident about
 184         value = value.replace(OPENSEARCH_PARAM_USER_DEFINED, search_terms);
 185         value = value.replace(OPENSEARCH_PARAM_INPUT_ENCODING, eng.query_charset);
 186         value = value.replace(OPENSEARCH_PARAM_LANGUAGE,
 187                               get_locale() || OPENSEARCH_PARAM_LANGUAGE_DEF);
 188         value = value.replace(OPENSEARCH_PARAM_OUTPUT_ENCODING,
 189                               OPENSEARCH_PARAM_OUTPUT_ENCODING_DEF);
 190
 191         // Replace any optional parameters
 192         value = value.replace(OPENSEARCH_PARAM_OPTIONAL, "");
 193
 194         // Insert any remaining required params with our default values
 195         for (let i = 0; i < OPENSEARCH_UNSUPPORTED_PARAMS.length; ++i) {
 196             value = value.replace(OPENSEARCH_UNSUPPORTED_PARAMS[i][0],
 197                                   OPENSEARCH_UNSUPPORTED_PARAMS[i][1]);
 198         }
 199
 200         return value;
 201     }
 202
 203     var url_string = substitute(url.template);
 204
 205     var data = url.params.map(function (p) (p.name + "=" + substitute(p.value))).join("&");
 206
 207     if (url.method == "GET") {
 208         if (data.length > 0) {
 209             if (url_string.indexOf("?") == -1)
 210                 url_string += "?";
 211             else
 212                 url_string += "&";
 213             url_string += data;
 214         }
 215         return load_spec({uri: url_string});
 216     } else {
 217         return load_spec({uri: url_string, raw_post_data: data,
 218                                request_mime_type: "application/x-www-form-urlencoded"});
 219     }
 220 };
 221
 222 search_engine.prototype.__defineGetter__("completer", function () {
 223     const response_type_json = "application/x-suggestions+json";
 224     const response_type_xml = "application/x-suggestions+xml";
 225     const json = ("@mozilla.org/dom/json;1" in Cc) &&
 226         Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
 227     var eng = this;
 228     if (this.supports_response_type(response_type_xml)) {
 229         return function (input, pos, conservative) {
 230             if (pos == 0 && conservative)
 231                 yield co_return(undefined);
 232             let str = input.substring(0,pos);
 233             try {
 234                 let lspec = eng.get_query_load_spec(str, response_type_xml);
 235                 let result = yield send_http_request(lspec);
 236                 let doc = result.responseXML;
 237                 let data = [];
 238                 if (doc) {
 239                     let elems = doc.getElementsByTagName("CompleteSuggestion");
 240                     for (let i = 0; i < elems.length; ++i) {
 241                         let node = elems[i];
 242                         let name = node.firstChild.getAttribute("data");
 243                         let desc = node.lastChild.getAttribute("int");
 244                         if (name && desc)
 245                             data.push([name,desc]);
 246                     }
 247                     delete doc;
 248                     delete elem;
 249                     delete result;
 250                     delete lspec;
 251                     let c = { count: data.length,
 252                               get_string: function (i) data[i][0],
 253                               get_description: function (i) data[i][1] + " results",
 254                               get_input_state: function (i) [data[i][0]]
 255                             };
 256                     yield co_return(c);
 257                 }
 258             } catch (e) {
 259                 yield co_return(null);
 260             }
 261         };
 262     } else if (json && this.supports_response_type(response_type_json)) {
 263         return function (input, pos, conservative) {
 264             if (pos == 0 && conservative)
 265                 yield co_return(undefined);
 266             let str = input.substring(0,pos);
 267             try {
 268                 let lspec = eng.get_query_load_spec(str, response_type_json);
 269                 let result = yield send_http_request(lspec);
 270                 let data = json.decode(result.responseText);
 271                 delete result;
 272                 delete lspec;
 273
 274                 if (!(data instanceof Array &&
 275                       data.length >= 2 &&
 276                       typeof(data[0]) == "string" &&
 277                       data[0] == str &&
 278                       data[1] instanceof Array &&
 279                       (data[2] == null || (data[2] instanceof Array))))
 280                     yield co_return(null);
 281                 if (data[2] && data[2].length != data[1].length)
 282                     data[2] = null;
 283                 let c = { count: data[1].length,
 284                           get_string: function (i) String(data[1][i]),
 285                           get_description: (data[2] != null ? (function (i) String(data[2][i])) : null),
 286                           get_input_state: function (i) [String(data[1][i])]
 287                         };
 288                 yield co_return(c);
 289             } catch (e) {
 290                 yield co_return(null);
 291             }
 292         };
 293     } else {
 294         return null;
 295     }
 296 });
 297
 298 /**
 299  * Guess the url of a home page to correspond with the search engine.
 300  * Take the text/html url for the search engine, trim off the path and
 301  * any "search." prefix on the domain.
 302  * This works for all the provided search engines.
 303  */
 304 function search_engine_get_homepage(search_engine) {
 305     var url = search_engine.urls.get("text/html");
 306     if (!url)
 307         return null;
 308     url = url_path_trim(url.template);
 309     url = url.replace("//search.", "//");
 310     return url;
 311 }
 312
 313 // Load search engines from default directories
 314 {
 315     let dir = file_locator_service.get("CurProcD", Ci.nsIFile);
 316     dir.append("search-engines");
 317     if (dir.exists() && dir.isDirectory())
 318         load_search_engines_in_directory(dir);
 319
 320     dir = file_locator_service.get("ProfD", Ci.nsIFile);
 321     dir.append("search-engines");
 322     if (dir.exists() && dir.isDirectory())
 323         load_search_engines_in_directory(dir);
 324 }
 325
 326 define_keywords("$alternative");
 327 function define_search_engine_webjump(search_engine_name, key) {
 328     keywords(arguments);
 329     var eng = search_engines.get(search_engine_name);
 330     let alternative = arguments.$alternative;
 331
 332     if (key == null)
 333         key = search_engine_name;
 334     if (alternative == null)
 335         alternative = search_engine_get_homepage(eng);
 336
 337     define_webjump(key,
 338                    function (arg) {
 339                        return eng.get_query_load_spec(arg);
 340                    },
 341                    $alternative = alternative,
 342                    $description = eng.description,
 343                    $completer = eng.completer);
 344 }
 345
 346 define_search_engine_webjump("google.xml", "google");
 347 define_search_engine_webjump("mozilla-bugzilla.xml", "bugzilla");
 348 define_search_engine_webjump("wikipedia.xml", "wikipedia");
 349 define_search_engine_webjump("wiktionary.xml", "wiktionary");
 350 define_search_engine_webjump("answers.xml", "answers");
 351 define_search_engine_webjump("yahoo.xml", "yahoo");
 352 define_search_engine_webjump("creativecommons.xml", "creativecommons");
 353 define_search_engine_webjump("eBay.xml", "ebay");
 354
 355 provide("search-engine");