index.py

   1 # Copyright (C) 2011, Anders F Bjorklund
   2 # Copyright (C) 2013, Thomas Leonard
   3 # See the COPYING file for details, or visit http://0install.net.
   4 #
   5 # This version for 0mirror is based on original code for 0install:
   6 # http://thread.gmane.org/gmane.comp.file-systems.zero-install.devel/3847
   7
   8 import os
   9 import logging
  10
  11 from whoosh.index import create_in, open_dir
  12 from whoosh import fields
  13 from whoosh.analysis import StemmingAnalyzer
  14
  15 from zeroinstall.injector.namespaces import XMLNS_IFACE
  16
  17 sa = StemmingAnalyzer()
  18 schema = fields.Schema(
  19                 uri             = fields.ID(unique=True, stored=True),
  20                 name            = fields.TEXT(stored=True,field_boost=50.0),
  21                 summary         = fields.TEXT(stored=True, field_boost=5.0),
  22                 description     = fields.TEXT(analyzer=sa),
  23                 category        = fields.KEYWORD(stored=True),
  24                 homepage        = fields.STORED
  25         )
  26
  27 class Indexer:
  28         def __init__(self, config, index_dir):
  29                 self.config = config
  30                 self.index_dir = index_dir
  31
  32                 if not os.path.exists(index_dir):
  33                         os.makedirs(index_dir)
  34                         index = create_in(index_dir, schema)
  35                 else:
  36                         index = open_dir(index_dir)
  37
  38                 self.writer = index.writer()
  39
  40         def update(self, url):
  41                 feed = self.config.iface_cache.get_feed(url)
  42                 if not feed:
  43                         logging.warning("%s not cached", url)
  44                         return
  45
  46                 if feed.feed_for:
  47                         self.writer.delete_by_term('uri', unicode(url))
  48                         return          # Skip sub-feeds
  49
  50                 name = feed.get_name()
  51                 summary = feed.summary
  52                 description = feed.description
  53                 category = None
  54                 for meta in feed.get_metadata(XMLNS_IFACE, 'category'):
  55                         category = meta.content
  56                         break
  57                 homepage = None
  58                 for meta in feed.get_metadata(XMLNS_IFACE, 'homepage'):
  59                         homepage = meta.content
  60                         break
  61
  62                 print "Indexing", url
  63                 self.writer.update_document(uri=unicode(url), name=unicode(name),
  64                                        summary=unicode(summary), description=unicode(description),
  65                                        category=category, homepage=homepage)
  66
  67                 #print "%-32s\"%s\"" % ("%s (%s)" % (name, category), summary)
  68
  69         def commit(self):
  70                 self.writer.commit()