commands/parse.py

   1 from __future__ import with_statement
   2
   3 import os, re
   4
   5 from django.conf import settings
   6
   7 os.environ['DJANGO_SETTINGS_MODULE'] = 'ebuildfnd.settings'
   8
   9 from layman.debug import Message
  10 from manage_layman import LaymanManager
  11
  12 from ebuildfnd.ebuildfind.models import Ebuild, Overlay
  13
  14 from whoosh_manager import WhooshEbuildManager
  15
  16 OVERLAYS_BASE = "/home/timemachine/www/ebuildfnd/ebuildfind/commands/var/"
  17 OUT = Message("parser")
  18
  19 class EbuildParser:
  20     def __init__(self, overlay, category, app, file_path):
  21         path, filename = os.path.split(os.path.realpath(file_path))
  22
  23         #OUT.warn("%s\n%s\n%s\n%s" % (overlay, category, app, file_path))
  24         #OUT.warn("****************************************************")
  25
  26         name_version, ext = os.path.splitext(filename)
  27         n = len(app)
  28
  29         version = name_version[n+1:]
  30
  31         self.name = app
  32         self.version = version
  33         self.category = category
  34         self.overlay = overlay
  35
  36         with open(file_path) as file:
  37             f = file.read()
  38             regex1 = re.compile(r'([a-z_]+\s*\(\))\s*{.*}', re.DOTALL)
  39             f = re.sub(regex1, r'', f)
  40             regex2 = re.compile(r'(for.*done)', re.DOTALL)
  41             f = re.sub(regex2, r'', f)
  42             data = dict(re.findall("(DESCRIPTION|HOMEPAGE|KEYWORDS|LICENSE|IUSE)=\"(.+)\"", f))
  43
  44         def notnull(d, key):
  45             try:
  46                 return d[key]
  47             except:
  48                 return ""
  49
  50         self.description = notnull(data, "DESCRIPTION")
  51         self.description = self.description.replace("\n"," ")
  52         self.homepage = notnull(data, "HOMEPAGE")
  53         self.keywords = notnull(data, "KEYWORDS")
  54         self.license = notnull(data, "LICENSE")
  55         self.iuse = notnull(data, "IUSE")
  56
  57     def __repr__(self):
  58         output = "%s/%s [%s] @ %s" % (self.category, self.name, self.version, self.overlay)
  59         output += " " + self.description
  60         output += " " + self.homepage
  61         output += " " + self.keywords
  62         output += " " + self.license
  63         output += " " + self.iuse
  64         return output
  65
  66 def exclude_directory(path, dir):
  67     exclude_dir = ["eclass", "profiles", "README-TODO", "Documentation", "sets", "index"]
  68     return os.path.isdir(path) and dir not in exclude_dir and not dir.startswith(".")
  69
  70 def ParseEbuilds():
  71     i = 0
  72     Ebuild.objects.all().delete()
  73     overlays = os.listdir(OVERLAYS_BASE)
  74
  75     whoosh = WhooshEbuildManager(True)
  76
  77     for overlay in overlays:
  78         path_overlay = os.path.join(OVERLAYS_BASE, overlay)
  79
  80         if exclude_directory(path_overlay, overlay):
  81             print overlay
  82             overlay_name = overlay
  83             overlay = Overlay.objects.get(name=overlay)
  84
  85             categories = os.listdir(path_overlay)
  86
  87             for category in categories:
  88                 path_overlay_category = os.path.join(path_overlay, category)
  89
  90                 if exclude_directory(path_overlay_category, category):
  91                     apps = os.listdir(path_overlay_category)
  92                     for app in apps:
  93                         path_overlay_category_app = os.path.join(path_overlay_category, app)
  94                         if exclude_directory(path_overlay_category_app, app):
  95                             ebuilds = os.listdir(path_overlay_category_app)
  96
  97                             for ebuild in ebuilds:
  98                                 if ebuild.endswith(".ebuild"):
  99                                     if exclude_directory(path_overlay_category_app, ebuild):
 100                                         #OUT.warn(ebuild)
 101
 102                                         path_ebuild = os.path.join(path_overlay_category_app, ebuild)
 103
 104                                         e = EbuildParser(overlay_name, category, app, path_ebuild)
 105
 106
 107                                         ebuild = Ebuild()
 108                                         ebuild.name = unicode(e.name)
 109                                         ebuild.category = unicode(e.category)
 110                                         ebuild.version = unicode(e.version)
 111                                         ebuild.description = unicode(e.description, errors="replace")
 112                                         ebuild.keywords = unicode(e.keywords)
 113                                         ebuild.license = unicode(e.license)
 114                                         ebuild.iuse = unicode(e.iuse)
 115                                         ebuild.homepage = unicode(e.homepage)
 116                                         ebuild.overlay = overlay
 117                                         #ebuild.save()
 118
 119
 120                                         whoosh.Update(ebuild)
 121
 122                                         i +=1
 123     print i
 124
 125 def SyncOverlays():
 126     h = LaymanManager()
 127     h.AddAll()
 128
 129 def ParseOverlays():
 130     h = LaymanManager()
 131     overlays = h.List()
 132
 133     for name, overlay in overlays.items() :
 134         """ check if new overlay is ready """
 135         o = Overlay.objects.all().filter(name=name)
 136
 137         if not o:
 138             o = Overlay()
 139             o.name = name
 140             o.description = overlay["description"]
 141             o.link = overlay["link"]
 142             o.save()
 143
 144 def main():
 145     #SyncOverlays()
 146     ParseOverlays()
 147     ParseEbuilds()
 148
 149 if __name__ == "__main__":
 150     main()