www/pt2html.py

   1 import sys, os, os.path, cgi, io, codecs, glob, re, warnings
   2 import keyword, token, tokenize
   3 import xml.dom.minidom
   4 from zope.pagetemplate.pagetemplate import PageTemplate
   5 from PIL import Image
   6
   7 sys.path[:0]=[".."]
   8 import pyx
   9
  10 def PageTemplateFile(filename):
  11     pt = PageTemplate()
  12     pt.write(codecs.open(filename, "r", encoding="utf-8").read())
  13     return pt
  14
  15 _KEYWORD = token.NT_OFFSET
  16
  17 tokclasses = {token.NUMBER: 'number',
  18               token.OP: 'op',
  19               token.STRING: 'string',
  20               tokenize.COMMENT: 'comment',
  21               token.NAME: 'name',
  22               _KEYWORD: 'keyword'}
  23
  24 class MakeHtml:
  25
  26     def fromPython(self, input):
  27         input = io.StringIO(input)
  28         self.output = io.StringIO()
  29         self.col = 0
  30         self.tokclass = None
  31         self.output.write("<pre id=python>")
  32         for token in tokenize.generate_tokens(input.readline):
  33             self.tokeneater(*token)
  34         if self.tokclass is not None:
  35             self.output.write('</span>')
  36         self.output.write("</pre>\n")
  37         return self.output.getvalue()
  38
  39     def tokeneater(self, toktype, toktext, xxx_todo_changeme, xxx_todo_changeme1, line):
  40         (srow, scol) = xxx_todo_changeme
  41         (erow, ecol) = xxx_todo_changeme1
  42         if toktype == token.ERRORTOKEN:
  43             raise RuntimeError("ErrorToken occured")
  44         if toktype in [token.NEWLINE, tokenize.NL]:
  45             self.output.write('\n')
  46             self.col = 0
  47         else:
  48             # map token type to a color group
  49             if token.LPAR <= toktype and toktype <= token.OP:
  50                 toktype = token.OP
  51             elif toktype == token.NAME and keyword.iskeyword(toktext):
  52                 toktype = _KEYWORD
  53
  54             # restore whitespace
  55             assert scol >= self.col
  56             self.output.write(" "*(scol-self.col))
  57
  58             try:
  59                 tokclass = tokclasses[toktype]
  60             except KeyError:
  61                 tokclass = None
  62             if self.tokclass is not None and tokclass != self.tokclass:
  63                 self.output.write('</span>')
  64             if tokclass is not None and tokclass != self.tokclass:
  65                 self.output.write('<span class="%s">' % tokclass)
  66             self.output.write(cgi.escape(toktext))
  67             self.tokclass = tokclass
  68
  69             # calculate new column position
  70             self.col = scol + len(toktext)
  71             newline = toktext.rfind("\n")
  72             if newline != -1:
  73                 self.col = len(toktext) - newline - 1
  74
  75     emptypattern = re.compile(r"\s*$")
  76     parpattern = re.compile(r"([ \t]*\n)*(?P<data>(\S[^\n]*\n)+)(([ \t]*\n)+.*)?")
  77     parmakeline = re.compile(r"\s*[\r\n]\s*")
  78     parmakeinline = re.compile(r"`(.*?)`")
  79     parmakebold = re.compile(r"'''(.*?)'''")
  80     parmakeitalic = re.compile(r"''(.*?)''")
  81     parmakeref = re.compile(r"\[([^]]*)\s([^\s]*)\]")
  82     codepattern = re.compile(r"([ \t]*\n)*(?P<data>(([ \t]+[^\n]*)?\n)+)")
  83     indentpattern = re.compile(r"([ \t]*\n)*(?P<indent>[ \t]+)")
  84
  85     def fromText(self, input, bend=""):
  86         title = None
  87         pos = 0
  88         output = io.StringIO()
  89         while not self.emptypattern.match(input, pos):
  90             par = self.parpattern.match(input, pos)
  91             if par:
  92                 pos = par.end("data")
  93                 par = par.group("data").strip()
  94                 par = par.replace("__version__", pyx.__version__)
  95                 par = par.replace("&", "&amp;")
  96                 par = par.replace("<", "&lt;")
  97                 par = par.replace(">", "&gt;")
  98                 par = par.replace("\"", "&quot;")
  99                 par = self.parmakeline.subn(" ", par)[0]
 100                 par = self.parmakeinline.subn(r"<code>\1</code>", par)[0]
 101                 par = self.parmakebold.subn(r"<strong>\1</strong>", par)[0]
 102                 par = self.parmakeitalic.subn(r"<em>\1</em>", par)[0]
 103                 par = self.parmakeref.subn(r'<a href="\2">\1</a>', par)[0]
 104                 if not title:
 105                     title = par
 106                 else:
 107                     bends = 0
 108                     while par.startswith("!"):
 109                         if not bend:
 110                             warnings.warn("ignore bend sign")
 111                             break
 112                         bends += 1
 113                         par = par[1:]
 114                     output.write("%s<p>%s</p>\n" % (bend*bends, par))
 115             else:
 116                 code = self.codepattern.match(input, pos)
 117                 if not code:
 118                     raise RuntimeError("couldn't parse text file")
 119                 pos = code.end("data")
 120                 code = code.group("data")
 121                 indent = self.indentpattern.match(code).group("indent")
 122                 code = re.subn(r"\s*[\r\n]%s" % indent, "\n", code.strip())[0]
 123                 if len(indent.expandtabs()) >= 4:
 124                     code = self.fromPython(code + "\n")
 125                 else:
 126                     code = "<pre>%s</pre>" % code
 127                 output.write("<div class=\"codeindent\">%s</div>\n" % code)
 128         text = output.getvalue()
 129         shorttext = text.split("...")[0]
 130         text = text.replace("...", "", 1)
 131         return title, shorttext, text
 132
 133 makehtml = MakeHtml()
 134
 135
 136 class example:
 137
 138     def __init__(self, basesrcdir, dir, basename):
 139         self.basename = basename
 140         if dir:
 141             name = os.path.join(dir, basename)
 142         else:
 143             name = basename
 144         relname = os.path.join(basesrcdir, name)
 145         self.filename = "%s.py" % name
 146         self.code = makehtml.fromPython(codecs.open("%s.py" % relname, encoding="utf-8").read())
 147         self.html = "%s.html" % basename
 148         self.png = "%s.png" % basename
 149         self.width, self.height = Image.open("%s.png" % relname).size
 150         self.thumbpng = "%s_thumb.png" % basename
 151         self.thumbwidth, self.thumbheight = Image.open("%s_thumb.png" % relname).size
 152         self.downloads = []
 153         for suffix in ["py", "ipynb", "dat", "jpg", "eps", "pdf", "svg"]:
 154             try:
 155                 filesize = "%.1f KB" % (os.path.getsize("%s.%s" % (relname, suffix)) / 1024.0)
 156             except OSError:
 157                 pass
 158             else:
 159                 self.downloads.append({"filename": "%s.%s" % (basename, suffix),
 160                                        "suffixname": ".%s" % suffix,
 161                                        "filesize": filesize,
 162                                        "iconname": "%s.png" % suffix})
 163         if os.path.exists("%s.txt" % relname):
 164             self.title, self.shorttext, self.text = makehtml.fromText(codecs.open("%s.txt" % relname, encoding="utf-8").read(), bend="<div class=\"examplebend\"><img src=\"../../bend.png\" width=22 height=31></div>\n")
 165         else:
 166             self.title = basename
 167             self.shorttext = self.text = None
 168
 169
 170 def mkrellink(linkname, options):
 171     # returns a string containing the relative url for linkname (an absolute url)
 172     pagename = options["pagename"]
 173     while linkname.find("/") != -1 and pagename.find("/") != -1:
 174         linknamefirst, linknameother = linkname.split("/", 1)
 175         pagenamefirst, pagenameother = pagename.split("/", 1)
 176         if linknamefirst == pagenamefirst:
 177             linkname = linknameother
 178             pagename = pagenameother
 179         else:
 180             break
 181     for i in pagename.split("/")[:-1]:
 182         linkname = "../" + linkname
 183     return linkname
 184
 185 maintemplate = PageTemplateFile("maintemplate.pt")
 186
 187 latestnews = 2
 188 newsdom = xml.dom.minidom.parse("news.pt")
 189 news = "".join(["%s%s" % (dt.toxml(), dd.toxml())
 190                 for dt, dd in zip(newsdom.getElementsByTagName("dt")[:latestnews],
 191                                   newsdom.getElementsByTagName("dd")[:latestnews])])
 192
 193 for ptname in glob.glob("*.pt"):
 194     if ptname not in ["maintemplate.pt", "exampleindex.pt", "examples.pt", "example.pt"]:
 195         htmlname = "%s.html" % ptname[:-3]
 196         template = PageTemplateFile(ptname)
 197         content = template(pagename=htmlname,
 198                            maintemplate=maintemplate,
 199                            mkrellink=mkrellink,
 200                            version=pyx.__version__,
 201                            news=news)
 202         codecs.open("build/%s" % htmlname, "w", encoding="utf-8").write(content)
 203
 204
 205 def processexamples(basedir):
 206     exampleindextemplate = PageTemplateFile("exampleindex.pt")
 207     examplestemplate = PageTemplateFile("examples.pt")
 208     exampletemplate = PageTemplateFile("example.pt")
 209
 210     exampledirs = [None]
 211     examplepages = []
 212     for dir in open(os.path.join("..", basedir, "INDEX")).readlines():
 213         dir = dir.strip()
 214         if dir.endswith("/"):
 215             exampledirs.append(dir)
 216             dir = dir.rstrip("/")
 217             try:
 218                 title = open(os.path.join("..", basedir, dir, "README")).readline().strip()
 219             except IOError:
 220                 title = dir
 221             examplepages.append({"dir": dir, "title": title})
 222
 223     prev = None
 224     for dirindex, dir in enumerate(exampledirs):
 225         if dir:
 226             srcdir = os.path.join("..", basedir, dir)
 227             destdir = os.path.join(basedir, dir)
 228             bend = "<div class=\"examplebend\"><img src=\"../../bend.png\" width=22 height=31></div>\n"
 229         else:
 230             srcdir = os.path.join("..", basedir)
 231             destdir = basedir
 232             bend = "<div class=\"examplebend\"><img src=\"../bend.png\" width=22 height=31></div>\n"
 233         try:
 234             nextdir = exampledirs[dirindex + 1]
 235             nextdir = os.path.join(basedir, nextdir)
 236         except IndexError:
 237             nextdir = None
 238         try:
 239             title, shorttext, text = makehtml.fromText(open(os.path.join(srcdir, "README")).read(), bend=bend)
 240         except IOError:
 241             title = dir
 242             text = ""
 243         examples = [example(os.path.join("..", basedir), dir, item.strip())
 244                     for item in open(os.path.join(srcdir, "INDEX")).readlines()
 245                     if item[-2] != "/"]
 246         htmlname = os.path.join(destdir, "index.html")
 247         if dir:
 248             template = examplestemplate
 249             next = os.path.join(destdir, examples[0].html)
 250         else:
 251             template = exampleindextemplate
 252             next = os.path.join(nextdir, "index.html")
 253         content = template(pagename=htmlname,
 254                            maintemplate=maintemplate,
 255                            version=pyx.__version__,
 256                            dir=dir,
 257                            title=title,
 258                            text=text,
 259                            examples=examples,
 260                            subpages=examplepages,
 261                            mkrellink=mkrellink,
 262                            prev=prev,
 263                            next=next)
 264         codecs.open("build/%s" % htmlname, "w", encoding="utf-8").write(content)
 265         prev = os.path.join(destdir, "index.html")
 266         if dir:
 267             for exampleindex, aexample in enumerate(examples):
 268                 try:
 269                     next = os.path.join(destdir, examples[exampleindex+1].html)
 270                 except (TypeError, IndexError):
 271                     if nextdir:
 272                         next = os.path.join(nextdir, "index.html")
 273                     else:
 274                         next = None
 275                 htmlname = os.path.join(destdir, "%s.html" % aexample.basename)
 276                 content = exampletemplate(pagename=htmlname,
 277                                           maintemplate=maintemplate,
 278                                           version=pyx.__version__,
 279                                           dir=dir,
 280                                           example=aexample,
 281                                           subpages=examplepages,
 282                                           mkrellink=mkrellink,
 283                                           prev=prev,
 284                                           next=next)
 285                 codecs.open("build/%s" % htmlname, "w", encoding="utf-8").write(content)
 286                 prev = os.path.join(destdir, aexample.html)
 287
 288
 289 processexamples("examples")