modules/search-engine.js

   1 require("utils.js");
   2
   3 var search_engines = new string_hashmap();
   4
   5 function search_engine_parse_error(msg) {
   6     var e = new Error(msg);
   7     e.__proto__ = search_engine_parse_error.prototype;
   8     return e;
   9 }
  10 search_engine_parse_error.prototype.__proto__ = Error.prototype;
  11
  12 function search_engine() {
  13     this.urls = new string_hashmap();
  14 }
  15
  16 function search_engine_url(type, method, template) {
  17     if (!method || !type || !template)
  18         throw search_engine_parse_error("Missing method, type, or template for search engine URL");
  19     method = method.toUpperCase();
  20     type = type.toUpperCase();
  21     if (method != "GET" && method != "POST")
  22         throw search_engine_parse_error("Invalid method");
  23     var template_uri = make_uri(template);
  24     switch (template_uri.scheme) {
  25     case "http":
  26     case "https":
  27         break;
  28     default:
  29         throw search_engine_parse_error("URL template has invalid scheme.");
  30         break;
  31     }
  32     this.type = type;
  33     this.method = method;
  34     this.template = template;
  35     this.params =  [];
  36 }
  37 search_engine_url.prototype.add_param = function search_engine_url__add_param(name, value) {
  38     this.params.push({name: name, value: value});
  39 }
  40
  41 function load_search_engines_in_directory(dir) {
  42     var files = null;
  43     try {
  44         files = dir.directoryEntries.QueryInterface(Ci.nsIDirectoryEnumerator);
  45
  46         while (files.hasMoreElements()) {
  47             var file = files.nextFile;
  48
  49             if (!file.isFile())
  50                 continue;
  51
  52             try {
  53                 load_search_engine_from_file(file);
  54             } catch (e) {
  55                 dumpln("WARNING: Failed to load search engine from file: " + file.path);
  56                 dump_error(e);
  57             }
  58         }
  59     } catch (e) {
  60         // FIXME: maybe have a better error message
  61         dump_error(e);
  62     } finally {
  63         if (files)
  64             files.close();
  65     }
  66 }
  67
  68 function load_search_engine_from_file(file) {
  69     var file_istream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(Ci.nsIFileInputStream);
  70     file_istream.init(file, MODE_RDONLY, 0644, false);
  71     var dom_parser = Cc["@mozilla.org/xmlextras/domparser;1"].createInstance(Ci.nsIDOMParser);
  72     var doc = dom_parser.parseFromStream(file_istream, "UTF-8", file.fileSize, "text/xml");
  73
  74     var eng = parse_search_engine_from_dom_node(doc.documentElement);
  75     search_engines.put(file.leafName, eng);
  76 }
  77
  78 // Supported OpenSearch parameters
  79 // http://www.opensearch.org/Specifications/OpenSearch/1.1#OpenSearch_URL_template_syntax
  80 const OPENSEARCH_PARAM_USER_DEFINED    = /\{searchTerms\??\}/g;
  81 const OPENSEARCH_PARAM_INPUT_ENCODING  = /\{inputEncoding\??\}/g;
  82 const OPENSEARCH_PARAM_LANGUAGE        = /\{language\??\}/g;
  83 const OPENSEARCH_PARAM_OUTPUT_ENCODING = /\{outputEncoding\??\}/g;
  84
  85 // Default values
  86 const OPENSEARCH_PARAM_LANGUAGE_DEF         = "*";
  87 const OPENSEARCH_PARAM_OUTPUT_ENCODING_DEF  = "UTF-8";
  88 const OPENSEARCH_PARAM_INPUT_ENCODING_DEF   = "UTF-8";
  89
  90 // "Unsupported" OpenSearch parameters. For example, we don't support
  91 // page-based results, so if the engine requires that we send the "page index"
  92 // parameter, we'll always send "1".
  93 const OPENSEARCH_PARAM_COUNT        = /\{count\??\}/g;
  94 const OPENSEARCH_PARAM_START_INDEX  = /\{startIndex\??\}/g;
  95 const OPENSEARCH_PARAM_START_PAGE   = /\{startPage\??\}/g;
  96
  97 // Default values
  98 const OPENSEARCH_PARAM_COUNT_DEF        = "20"; // 20 results
  99 const OPENSEARCH_PARAM_START_INDEX_DEF  = "1";  // start at 1st result
 100 const OPENSEARCH_PARAM_START_PAGE_DEF   = "1";  // 1st page
 101
 102 // Optional parameter
 103 const OPENSEARCH_PARAM_OPTIONAL     = /\{(?:\w+:)?\w+\?\}/g;
 104
 105 // A array of arrays containing parameters that we don't fully support, and
 106 // their default values. We will only send values for these parameters if
 107 // required, since our values are just really arbitrary "guesses" that should
 108 // give us the output we want.
 109 var OPENSEARCH_UNSUPPORTED_PARAMS = [
 110   [OPENSEARCH_PARAM_COUNT, OPENSEARCH_PARAM_COUNT_DEF],
 111   [OPENSEARCH_PARAM_START_INDEX, OPENSEARCH_PARAM_START_INDEX_DEF],
 112   [OPENSEARCH_PARAM_START_PAGE, OPENSEARCH_PARAM_START_PAGE_DEF],
 113 ];
 114
 115
 116 function parse_search_engine_from_dom_node(node) {
 117     var eng = new search_engine();
 118     eng.query_charset = OPENSEARCH_PARAM_INPUT_ENCODING_DEF;
 119
 120     for each (let child in node.childNodes) {
 121         switch (child.localName) {
 122         case "ShortName":
 123             eng.name = child.textContent;
 124             break;
 125         case "Description":
 126             eng.description = child.textContent;
 127             break;
 128         case "Url":
 129             try {
 130                 let type = child.getAttribute("type");
 131                 let method = child.getAttribute("method") || "GET";
 132                 let template = child.getAttribute("template");
 133
 134                 let engine_url = new search_engine_url(type, method, template);
 135                 for each (let p in child.childNodes) {
 136                     if (p.localName == "Param") {
 137                         let name = p.getAttribute("name");
 138                         let value = p.getAttribute("value");
 139                         if (name && value)
 140                             engine_url.add_param(name, value);
 141                     }
 142                 }
 143                 eng.urls.put(type, engine_url);
 144             } catch (e) {
 145                 // Skip this element if parsing fails
 146             }
 147             break;
 148         case "InputEncoding":
 149             eng.query_charset = child.textContent.toUpperCase();
 150             break;
 151         }
 152     }
 153     return eng;
 154 }
 155
 156 search_engine.prototype.supports_response_type = function (type) {
 157     return this.urls.contains(type);
 158 };
 159
 160 /**
 161  * Returns null if the result mime_type isn't supported.  The string
 162  * search_terms will be escaped by this function.
 163  */
 164 search_engine.prototype.get_query_load_spec = function search_engine__get_query_load_spec(search_terms, type) {
 165     if (type == null)
 166         type = "text/html";
 167     var url = this.urls.get(type);
 168     if (!url)
 169         return null;
 170     search_terms = encodeURIComponent(search_terms);
 171     var eng = this;
 172
 173     function substitute(value) {
 174         // Insert the OpenSearch parameters we're confident about
 175         value = value.replace(OPENSEARCH_PARAM_USER_DEFINED, search_terms);
 176         value = value.replace(OPENSEARCH_PARAM_INPUT_ENCODING, eng.query_charset);
 177         value = value.replace(OPENSEARCH_PARAM_LANGUAGE,
 178                               get_locale() || OPENSEARCH_PARAM_LANGUAGE_DEF);
 179         value = value.replace(OPENSEARCH_PARAM_OUTPUT_ENCODING,
 180                               OPENSEARCH_PARAM_OUTPUT_ENCODING_DEF);
 181
 182         // Replace any optional parameters
 183         value = value.replace(OPENSEARCH_PARAM_OPTIONAL, "");
 184
 185         // Insert any remaining required params with our default values
 186         for (let i = 0; i < OPENSEARCH_UNSUPPORTED_PARAMS.length; ++i) {
 187             value = value.replace(OPENSEARCH_UNSUPPORTED_PARAMS[i][0],
 188                                   OPENSEARCH_UNSUPPORTED_PARAMS[i][1]);
 189         }
 190
 191         return value;
 192     }
 193
 194     var url_string = substitute(url.template);
 195
 196     var data = url.params.map(function (p) (p.name + "=" + substitute(p.value))).join("&");
 197
 198     if (url.method == "GET") {
 199         if (data.length > 0) {
 200             if (url_string.indexOf("?") == -1)
 201                 url_string += "?";
 202             else
 203                 url_string += "&";
 204             url_string += data;
 205         }
 206         return load_spec({uri: url_string});
 207     } else {
 208         return load_spec({uri: url_string, raw_post_data: data,
 209                           request_mime_type: "application/x-www-form-urlencoded"});
 210     }
 211 }
 212
 213 search_engine.prototype.__defineGetter__("completer", function () {
 214     const response_type_json = "application/x-suggestions+json";
 215     const response_type_xml = "application/x-suggestions+xml";
 216
 217     var eng = this;
 218     if (this.supports_response_type(response_type_xml)) {
 219         return function (input, pos, conservative) {
 220             if (pos == 0 && conservative)
 221                 yield co_return(undefined);
 222             let str = input.substring(0,pos);
 223             try {
 224                 let lspec = eng.get_query_load_spec(str, response_type_xml,
 225                                                     $override_mime_type = "application/xml");
 226                 let result = yield send_http_request(lspec);
 227                 let doc = result.responseXML;
 228                 let data = [];
 229                 if (doc) {
 230                     let elems = doc.getElementsByTagName("CompleteSuggestion");
 231                     for (let i = 0; i < elems.length; ++i) {
 232                         let node = elems[i];
 233                         let name = node.firstChild.getAttribute("data");
 234                         let desc = node.lastChild.getAttribute("int");
 235                         if (name && desc)
 236                             data.push([name,desc]);
 237                     }
 238                     delete doc;
 239                     delete elem;
 240                     delete result;
 241                     delete lspec;
 242                     let c = { count: data.length,
 243                               get_string: function (i) data[i][0],
 244                               get_description: function (i) data[i][1] + " results",
 245                               get_input_state: function (i) [data[i][0]]
 246                             };
 247                     yield co_return(c);
 248                 }
 249             } catch (e) {
 250                 yield co_return(null);
 251             }
 252         };
 253     } else if (JSON && this.supports_response_type(response_type_json)) {
 254         return function (input, pos, conservative) {
 255             if (pos == 0 && conservative)
 256                 yield co_return(undefined);
 257             let str = input.substring(0,pos);
 258             try {
 259                 let lspec = eng.get_query_load_spec(str, response_type_json);
 260                 let result = yield send_http_request(lspec);
 261                 let data = JSON.decode(result.responseText);
 262                 delete result;
 263                 delete lspec;
 264
 265                 if (!(data instanceof Array &&
 266                       data.length >= 2 &&
 267                       typeof(data[0]) == "string" &&
 268                       data[0] == str &&
 269                       data[1] instanceof Array &&
 270                       (data[2] == null || (data[2] instanceof Array))))
 271                     yield co_return(null);
 272                 if (data[2].length != data[1].length)
 273                     data[2] = null;
 274                 let c = { count: data[1].length,
 275                           get_string: function (i) String(data[1][i]),
 276                           get_description: (data[2] != null) && (function (i) String(data[2][i])),
 277                           get_input_state: function (i) [String(data[1][i])]
 278                         };
 279                 yield co_return(c);
 280             } catch (e) {
 281                 yield co_return(null);
 282             }
 283         };
 284     } else {
 285         return null;
 286     }
 287 });
 288
 289 // Load search engines from default directories
 290 {
 291     let dir = file_locator.get("CurProcD", Ci.nsIFile);
 292     dir.append("search-engines");
 293     if (dir.exists() && dir.isDirectory())
 294         load_search_engines_in_directory(dir);
 295
 296     dir = file_locator.get("ProfD", Ci.nsIFile);
 297     dir.append("search-engines");
 298     if (dir.exists() && dir.isDirectory())
 299         load_search_engines_in_directory(dir);
 300 }
 301
 302 function define_search_engine_webjump(search_engine_name, key) {
 303     var eng = search_engines.get(search_engine_name);
 304
 305     if (key == null)
 306         key = search_engine_name;
 307
 308     define_webjump(key,
 309                    function (arg) {
 310                        return eng.get_query_load_spec(arg);
 311                    },
 312                    $description = eng.description,
 313                    $completer = eng.completer);
 314 }
 315
 316 define_search_engine_webjump("google.xml", "google");
 317 define_search_engine_webjump("mozilla-bugzilla.xml", "bugzilla");
 318 define_search_engine_webjump("amazondotcom.xml", "amazon");
 319 define_search_engine_webjump("wikipedia.xml", "wikipedia");
 320 define_search_engine_webjump("answers.xml", "answers");
 321 define_search_engine_webjump("yahoo.xml", "yahoo");
 322 define_search_engine_webjump("creativecommons.xml", "creativecommons");
 323 define_search_engine_webjump("eBay.xml", "ebay");