bin/convwatch.py

   1 # -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
   2 #
   3 # This file is part of the LibreOffice project.
   4 #
   5 # This Source Code Form is subject to the terms of the Mozilla Public
   6 # License, v. 2.0. If a copy of the MPL was not distributed with this
   7 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   8 #
   9
  10 import getopt
  11 import os
  12 import subprocess
  13 import sys
  14 import time
  15 import uuid
  16 import datetime
  17 import traceback
  18 import threading
  19 try:
  20     from urllib.parse import quote
  21 except ImportError:
  22     from urllib import quote
  23
  24 try:
  25     import pyuno
  26     import uno
  27     import unohelper
  28 except ImportError:
  29     print("pyuno not found: try to set PYTHONPATH and URE_BOOTSTRAP variables")
  30     print("PYTHONPATH=/installation/opt/program")
  31     print("URE_BOOTSTRAP=file:///installation/opt/program/fundamentalrc")
  32     raise
  33
  34 try:
  35     from com.sun.star.document import XDocumentEventListener
  36 except ImportError:
  37     print("UNO API class not found: try to set URE_BOOTSTRAP variable")
  38     print("URE_BOOTSTRAP=file:///installation/opt/program/fundamentalrc")
  39     raise
  40
  41 ### utilities ###
  42
  43 def log(*args):
  44     print(*args, flush=True)
  45
  46 def partition(list, pred):
  47     left = []
  48     right = []
  49     for e in list:
  50         if pred(e):
  51             left.append(e)
  52         else:
  53             right.append(e)
  54     return (left, right)
  55
  56 def filelist(dir, suffix):
  57     if len(dir) == 0:
  58         raise Exception("filelist: empty directory")
  59     if not(dir[-1] == "/"):
  60         dir += "/"
  61     files = [dir + f for f in os.listdir(dir)]
  62 #    log(files)
  63     return [f for f in files
  64                     if os.path.isfile(f) and os.path.splitext(f)[1] == suffix]
  65
  66 def getFiles(dirs, suffix):
  67     files = []
  68     for dir in dirs:
  69         files += filelist(dir, suffix)
  70     return files
  71
  72 ### UNO utilities ###
  73
  74 class OfficeConnection:
  75     def __init__(self, args):
  76         self.args = args
  77         self.soffice = None
  78         self.socket = None
  79         self.xContext = None
  80     def setUp(self):
  81         (method, sep, rest) = self.args["--soffice"].partition(":")
  82         if sep != ":":
  83             raise Exception("soffice parameter does not specify method")
  84         if method == "path":
  85                 self.socket = "pipe,name=pytest" + str(uuid.uuid1())
  86                 try:
  87                     userdir = self.args["--userdir"]
  88                 except KeyError:
  89                     raise Exception("'path' method requires --userdir")
  90                 if not(userdir.startswith("file://")):
  91                     raise Exception("--userdir must be file URL")
  92                 self.soffice = self.bootstrap(rest, userdir, self.socket)
  93         elif method == "connect":
  94                 self.socket = rest
  95         else:
  96             raise Exception("unsupported connection method: " + method)
  97         self.xContext = self.connect(self.socket)
  98
  99     def bootstrap(self, soffice, userdir, socket):
 100         argv = [ soffice, "--accept=" + socket + ";urp",
 101                 "-env:UserInstallation=" + userdir,
 102                 "--quickstart=no",
 103                 "--norestore", "--nologo", "--headless" ]
 104         if "--valgrind" in self.args:
 105             argv.append("--valgrind")
 106         return subprocess.Popen(argv)
 107
 108     def connect(self, socket):
 109         xLocalContext = uno.getComponentContext()
 110         xUnoResolver = xLocalContext.ServiceManager.createInstanceWithContext(
 111                 "com.sun.star.bridge.UnoUrlResolver", xLocalContext)
 112         url = "uno:" + socket + ";urp;StarOffice.ComponentContext"
 113         log("OfficeConnection: connecting to: " + url)
 114         while True:
 115             try:
 116                 xContext = xUnoResolver.resolve(url)
 117                 return xContext
 118 #            except com.sun.star.connection.NoConnectException
 119             except pyuno.getClass("com.sun.star.connection.NoConnectException"):
 120                 log("NoConnectException: sleeping...")
 121                 time.sleep(1)
 122
 123     def tearDown(self):
 124         if self.soffice:
 125             if self.xContext:
 126                 try:
 127                     log("tearDown: calling terminate()...")
 128                     xMgr = self.xContext.ServiceManager
 129                     xDesktop = xMgr.createInstanceWithContext(
 130                             "com.sun.star.frame.Desktop", self.xContext)
 131                     xDesktop.terminate()
 132                     log("...done")
 133 #                except com.sun.star.lang.DisposedException:
 134                 except pyuno.getClass("com.sun.star.beans.UnknownPropertyException"):
 135                     log("caught UnknownPropertyException")
 136                     pass # ignore, also means disposed
 137                 except pyuno.getClass("com.sun.star.lang.DisposedException"):
 138                     log("caught DisposedException")
 139                     pass # ignore
 140             else:
 141                 self.soffice.terminate()
 142             ret = self.soffice.wait()
 143             self.xContext = None
 144             self.socket = None
 145             self.soffice = None
 146             if ret != 0:
 147                 raise Exception("Exit status indicates failure: " + str(ret))
 148 #            return ret
 149
 150 class WatchDog(threading.Thread):
 151     def __init__(self, connection):
 152         threading.Thread.__init__(self, name="WatchDog " + connection.socket)
 153         self.connection = connection
 154     def run(self):
 155         try:
 156             if self.connection.soffice: # not possible for "connect"
 157                 self.connection.soffice.wait(timeout=120) # 2 minutes?
 158         except subprocess.TimeoutExpired:
 159             log("WatchDog: TIMEOUT -> killing soffice")
 160             self.connection.soffice.terminate() # actually killing oosplash...
 161             self.connection.xContext = None
 162             log("WatchDog: killed soffice")
 163
 164 class PerTestConnection:
 165     def __init__(self, args):
 166         self.args = args
 167         self.connection = None
 168         self.watchdog = None
 169     def getContext(self):
 170         return self.connection.xContext
 171     def setUp(self):
 172         assert(not(self.connection))
 173     def preTest(self):
 174         conn = OfficeConnection(self.args)
 175         conn.setUp()
 176         self.connection = conn
 177         self.watchdog = WatchDog(self.connection)
 178         self.watchdog.start()
 179     def postTest(self):
 180         if self.connection:
 181             try:
 182                 self.connection.tearDown()
 183             finally:
 184                 self.connection = None
 185                 self.watchdog.join()
 186     def tearDown(self):
 187         assert(not(self.connection))
 188
 189 class PersistentConnection:
 190     def __init__(self, args):
 191         self.args = args
 192         self.connection = None
 193     def getContext(self):
 194         return self.connection.xContext
 195     def setUp(self):
 196         conn = OfficeConnection(self.args)
 197         conn.setUp()
 198         self.connection = conn
 199     def preTest(self):
 200         assert(self.connection)
 201     def postTest(self):
 202         assert(self.connection)
 203     def tearDown(self):
 204         if self.connection:
 205             try:
 206                 self.connection.tearDown()
 207             finally:
 208                 self.connection = None
 209
 210 def simpleInvoke(connection, test):
 211     try:
 212         connection.preTest()
 213         test.run(connection.getContext())
 214     finally:
 215         connection.postTest()
 216
 217 def retryInvoke(connection, test):
 218     tries = 5
 219     while tries > 0:
 220         try:
 221             tries -= 1
 222             try:
 223                 connection.preTest()
 224                 test.run(connection.getContext())
 225                 return
 226             finally:
 227                 connection.postTest()
 228         except KeyboardInterrupt:
 229             raise # Ctrl+C should work
 230         except:
 231             log("retryInvoke: caught exception")
 232     raise Exception("FAILED retryInvoke")
 233
 234 def runConnectionTests(connection, invoker, tests):
 235     try:
 236         connection.setUp()
 237         failed = []
 238         for test in tests:
 239             try:
 240                 invoker(connection, test)
 241             except KeyboardInterrupt:
 242                 raise # Ctrl+C should work
 243             except:
 244                 failed.append(test.file)
 245                 estr = traceback.format_exc()
 246                 log("... FAILED with exception:\n" + estr)
 247         return failed
 248     finally:
 249         connection.tearDown()
 250
 251 class EventListener(XDocumentEventListener,unohelper.Base):
 252     def __init__(self):
 253         self.layoutFinished = False
 254     def documentEventOccured(self, event):
 255 #        log(str(event.EventName))
 256         if event.EventName == "OnLayoutFinished":
 257             self.layoutFinished = True
 258     def disposing(event):
 259         pass
 260
 261 def mkPropertyValue(name, value):
 262     return uno.createUnoStruct("com.sun.star.beans.PropertyValue",
 263             name, 0, value, 0)
 264
 265 ### tests ###
 266
 267 def loadFromURL(xContext, url):
 268     xDesktop = xContext.ServiceManager.createInstanceWithContext(
 269             "com.sun.star.frame.Desktop", xContext)
 270     props = [("Hidden", True), ("ReadOnly", True)] # FilterName?
 271     loadProps = tuple([mkPropertyValue(name, value) for (name, value) in props])
 272     xListener = EventListener()
 273     xGEB = xContext.getValueByName(
 274         "/singletons/com.sun.star.frame.theGlobalEventBroadcaster")
 275     xGEB.addDocumentEventListener(xListener)
 276     xDoc = None
 277     try:
 278         xDoc = xDesktop.loadComponentFromURL(url, "_blank", 0, loadProps)
 279         if xDoc is None:
 280             raise Exception("No document loaded?")
 281         time_ = 0
 282         while time_ < 30:
 283             if xListener.layoutFinished:
 284                 return xDoc
 285             log("delaying...")
 286             time_ += 1
 287             time.sleep(1)
 288         log("timeout: no OnLayoutFinished received")
 289         return xDoc
 290     except:
 291         if xDoc:
 292             log("CLOSING")
 293             xDoc.close(True)
 294         raise
 295     finally:
 296         if xListener:
 297             xGEB.removeDocumentEventListener(xListener)
 298
 299 def printDoc(xContext, xDoc, url):
 300     props = [ mkPropertyValue("FileName", url) ]
 301 # xDoc.print(props)
 302     uno.invoke(xDoc, "print", (tuple(props),)) # damn, that's a keyword!
 303     busy = True
 304     while busy:
 305         log("printing...")
 306         time.sleep(1)
 307         prt = xDoc.getPrinter()
 308         for value in prt:
 309             if value.Name == "IsBusy":
 310                 busy = value.Value
 311     log("...done printing")
 312
 313 class LoadPrintFileTest:
 314     def __init__(self, file, prtsuffix):
 315         self.file = file
 316         self.prtsuffix = prtsuffix
 317     def run(self, xContext):
 318         start = datetime.datetime.now()
 319         log("Time: " + str(start) + " Loading document: " + self.file)
 320         xDoc = None
 321         try:
 322             url = "file://" + quote(self.file)
 323             xDoc = loadFromURL(xContext, url)
 324             printDoc(xContext, xDoc, url + self.prtsuffix)
 325         finally:
 326             if xDoc:
 327                 xDoc.close(True)
 328             end = datetime.datetime.now()
 329             log("...done with: " + self.file + " in: " + str(end - start))
 330
 331 def runLoadPrintFileTests(opts, dirs, suffix, reference):
 332     if reference:
 333         prtsuffix = ".pdf.reference"
 334     else:
 335         prtsuffix = ".pdf"
 336     files = getFiles(dirs, suffix)
 337     tests = (LoadPrintFileTest(file, prtsuffix) for file in files)
 338 #    connection = PersistentConnection(opts)
 339     connection = PerTestConnection(opts)
 340     failed = runConnectionTests(connection, simpleInvoke, tests)
 341     print("all printed: FAILURES: " + str(len(failed)))
 342     for fail in failed:
 343         print(fail)
 344     return failed
 345
 346 def mkImages(file, resolution):
 347     argv = [ "gs", "-r" + resolution, "-sOutputFile=" + file + ".%04d.jpeg",
 348              "-dNOPROMPT", "-dNOPAUSE", "-dBATCH", "-sDEVICE=jpeg", file ]
 349     ret = subprocess.check_call(argv)
 350
 351 def mkAllImages(dirs, suffix, resolution, reference, failed):
 352     if reference:
 353         prtsuffix = ".pdf.reference"
 354     else:
 355         prtsuffix = ".pdf"
 356     for dir in dirs:
 357         files = filelist(dir, suffix)
 358         log(files)
 359         for f in files:
 360             if f in failed:
 361                 log("Skipping failed: " + f)
 362             else:
 363                 mkImages(f + prtsuffix, resolution)
 364
 365 def identify(imagefile):
 366     argv = ["identify", "-format", "%k", imagefile]
 367     process = subprocess.Popen(argv, stdout=subprocess.PIPE)
 368     result, _ = process.communicate()
 369     if process.wait() != 0:
 370         raise Exception("identify failed")
 371     if result.partition(b"\n")[0] != b"1":
 372         log("identify result: " + result.decode('utf-8'))
 373         log("DIFFERENCE in " + imagefile)
 374
 375 def compose(refimagefile, imagefile, diffimagefile):
 376     argv = [ "composite", "-compose", "difference",
 377             refimagefile, imagefile, diffimagefile ]
 378     subprocess.check_call(argv)
 379
 380 def compareImages(file):
 381     allimages = [f for f in filelist(os.path.dirname(file), ".jpeg")
 382                    if f.startswith(file)]
 383 #    refimages = [f for f in filelist(os.path.dirname(file), ".jpeg")
 384 #                   if f.startswith(file + ".reference")]
 385 #    log("compareImages: allimages:" + str(allimages))
 386     (refimages, images) = partition(sorted(allimages),
 387             lambda f: f.startswith(file + ".pdf.reference"))
 388 #    log("compareImages: images" + str(images))
 389     for (image, refimage) in zip(images, refimages):
 390         compose(image, refimage, image + ".diff")
 391         identify(image + ".diff")
 392     if (len(images) != len(refimages)):
 393         log("DIFFERENT NUMBER OF IMAGES FOR: " + file)
 394
 395 def compareAllImages(dirs, suffix):
 396     log("compareAllImages...")
 397     for dir in dirs:
 398         files = filelist(dir, suffix)
 399 #        log("compareAllImages:" + str(files))
 400         for f in files:
 401             compareImages(f)
 402     log("...compareAllImages done")
 403
 404
 405 def parseArgs(argv):
 406     (optlist,args) = getopt.getopt(argv[1:], "hr",
 407             ["help", "soffice=", "userdir=", "reference", "valgrind"])
 408 #    print optlist
 409     return (dict(optlist), args)
 410
 411 def usage():
 412     message = """usage: {program} [option]... [directory]..."
 413  -h | --help:      print usage information
 414  -r | --reference: generate new reference files (otherwise: compare)
 415  --soffice=method:location
 416                    specify soffice instance to connect to
 417                    supported methods: 'path', 'connect'
 418  --userdir=URL     specify user installation directory for 'path' method
 419  --valgrind        pass --valgrind to soffice for 'path' method"""
 420     print(message.format(program = os.path.basename(sys.argv[0])))
 421
 422 def checkTools():
 423     try:
 424         subprocess.check_output(["gs", "--version"])
 425     except:
 426         print("Cannot execute 'gs'. Please install ghostscript.")
 427         sys.exit(1)
 428     try:
 429         subprocess.check_output(["composite", "-version"])
 430         subprocess.check_output(["identify", "-version"])
 431     except:
 432         print("Cannot execute 'composite' or 'identify'.")
 433         print("Please install ImageMagick.")
 434         sys.exit(1)
 435
 436 if __name__ == "__main__":
 437     checkTools()
 438     (opts,args) = parseArgs(sys.argv)
 439     if len(args) == 0:
 440         usage()
 441         sys.exit(1)
 442     if "-h" in opts or "--help" in opts:
 443         usage()
 444         sys.exit()
 445     elif "--soffice" in opts:
 446         reference = "-r" in opts or "--reference" in opts
 447         failed = runLoadPrintFileTests(opts, args, ".odt", reference)
 448         mkAllImages(args, ".odt", "200", reference, failed)
 449         if not(reference):
 450             compareAllImages(args, ".odt")
 451     else:
 452         usage()
 453         sys.exit(1)
 454
 455 # vim: set shiftwidth=4 softtabstop=4 expandtab: