pygnulib/GLFileSystem.py

   1 #!/usr/bin/python
   2 # encoding: UTF-8
   3
   4 #===============================================================================
   5 # Define global imports
   6 #===============================================================================
   7 import os
   8 import re
   9 import sys
  10 import codecs
  11 import shutil
  12 import filecmp
  13 import subprocess as sp
  14 from . import constants
  15 from .GLError import GLError
  16 from .GLConfig import GLConfig
  17
  18
  19 #===============================================================================
  20 # Define module information
  21 #===============================================================================
  22 __author__ = constants.__author__
  23 __license__ = constants.__license__
  24 __copyright__ = constants.__copyright__
  25
  26
  27 #===============================================================================
  28 # Define global constants
  29 #===============================================================================
  30 PYTHON3 = constants.PYTHON3
  31 NoneType = type(None)
  32 APP = constants.APP
  33 DIRS = constants.DIRS
  34 ENCS = constants.ENCS
  35 UTILS = constants.UTILS
  36 MODES = constants.MODES
  37 TESTS = constants.TESTS
  38 compiler = constants.compiler
  39 joinpath = constants.joinpath
  40 cleaner = constants.cleaner
  41 string = constants.string
  42 isabs = os.path.isabs
  43 isdir = os.path.isdir
  44 isfile = os.path.isfile
  45 normpath = os.path.normpath
  46 relpath = os.path.relpath
  47
  48
  49 #===============================================================================
  50 # Define GLFileSystem class
  51 #===============================================================================
  52 class GLFileSystem(object):
  53     '''GLFileSystem class is used to create virtual filesystem, which is based on
  54     the gnulib directory and directory specified by localdir argument. Its main
  55     method lookup(file) is used to find file in these directories or combine it
  56     using Linux 'patch' utility.'''
  57
  58     def __init__(self, config):
  59         '''Create new GLFileSystem instance. The only argument is localdir,
  60         which can be an empty string too.'''
  61         if type(config) is not GLConfig:
  62             raise(TypeError('config must be a GLConfig, not %s' %
  63                             type(config).__name__))
  64         self.config = config
  65
  66     def __repr__(self):
  67         '''x.__repr__ <==> repr(x)'''
  68         result = '<pygnulib.GLFileSystem %s>' % hex(id(self))
  69         return(result)
  70
  71     def lookup(self, name):
  72         '''GLFileSystem.lookup(name) -> tuple
  73
  74         Lookup a file in gnulib and localdir directories or combine it using Linux
  75         'patch' utility. If file was found, method returns string, else it raises
  76         GLError telling that file was not found. Function also returns flag which
  77         indicates whether file is a temporary file.
  78         GLConfig: localdir.'''
  79         if type(name) is bytes or type(name) is string:
  80             if type(name) is bytes:
  81                 name = name.decode(ENCS['default'])
  82         else:  # if name has not bytes or string type
  83             raise(TypeError(
  84                 'name must be a string, not %s' % type(module).__name__))
  85         # If name exists in localdir, then we use it
  86         path_gnulib = joinpath(DIRS['root'], name)
  87         path_local = joinpath(self.config['localdir'], name)
  88         path_diff = joinpath(self.config['localdir'], '%s.diff' % name)
  89         path_temp = joinpath(self.config['tempdir'], name)
  90         try:  # Try to create directories
  91             os.makedirs(os.path.dirname(path_temp))
  92         except OSError as error:
  93             pass  # Skip errors if directory exists
  94         if isfile(path_temp):
  95             os.remove(path_temp)
  96         if self.config['localdir'] and isfile(path_local):
  97             result = (path_local, False)
  98         else:  # if path_local does not exist
  99             if isfile(path_gnulib):
 100                 if self.config['localdir'] and isfile(path_diff):
 101                     shutil.copy(path_gnulib, path_temp)
 102                     command = 'patch -s "%s" < "%s" >&2' % (path_temp, path_diff)
 103                     try:  # Try to apply patch
 104                         sp.check_call(command, shell=True)
 105                     except sp.CalledProcessError as error:
 106                         raise(GLError(2, name))
 107                     result = (path_temp, True)
 108                 else:  # if path_diff does not exist
 109                     result = (path_gnulib, False)
 110             else:  # if path_gnulib does not exist
 111                 raise(GLError(1, name))
 112         return(result)
 113
 114
 115 #===============================================================================
 116 # Define GLFileAssistant class
 117 #===============================================================================
 118 class GLFileAssistant(object):
 119     '''GLFileAssistant is used to help with file processing.'''
 120
 121     def __init__(self, config, transformers=dict()):
 122         '''Create GLFileAssistant instance.'''
 123         if type(config) is not GLConfig:
 124             raise(TypeError('config must be a GLConfig, not %s' %
 125                             type(config).__name__))
 126         if type(transformers) is not dict:
 127             raise(TypeError('transformers must be a dict, not %s' %
 128                             type(transformers).__name__))
 129         for key in ['lib', 'aux', 'main', 'tests']:
 130             if key not in transformers:
 131                 transformers[key] = 's,x,x,'
 132             else:  # if key in transformers
 133                 value = transformers[key]
 134                 if type(value) is bytes or type(value) is string:
 135                     if type(value) is bytes:
 136                         transformers[key] = value.decode(ENCS['default'])
 137                 else:  # if value has not bytes or string type
 138                     raise(TypeError('transformers[%s] must be a string, not %s' %
 139                                     (key, type(value).__name__)))
 140         self.original = None
 141         self.rewritten = None
 142         self.added = list()
 143         self.makefile = list()
 144         self.config = config
 145         self.transformers = transformers
 146         self.filesystem = GLFileSystem(self.config)
 147
 148     def __repr__(self):
 149         '''x.__repr__() <==> repr(x)'''
 150         result = '<pygnulib.GLFileAssistant %s>' % hex(id(self))
 151         return(result)
 152
 153     def tmpfilename(self, path):
 154         '''GLFileAssistant.tmpfilename() -> string
 155
 156         Return the name of a temporary file (file is relative to destdir).'''
 157         if type(path) is bytes or type(path) is string:
 158             if type(path) is bytes:
 159                 path = path.decode(ENCS['default'])
 160         else:  # if path has not bytes or string type
 161             raise(TypeError(
 162                 'path must be a string, not %s' % (type(path).__name__)))
 163         if not self.config['dryrun']:
 164             # Put the new contents of $file in a file in the same directory (needed
 165             # to guarantee that an 'mv' to "$destdir/$file" works).
 166             result = joinpath(self.config['destdir'], '%s.tmp' % path)
 167             dirname = os.path.dirname(result)
 168             if dirname and not isdir(dirname):
 169                 os.makedirs(dirname)
 170         else:  # if self.config['dryrun']
 171             # Put the new contents of $file in a file in a temporary directory
 172             # (because the directory of "$file" might not exist).
 173             tempdir = self.config['tempdir']
 174             result = joinpath(tempdir, '%s.tmp' % os.path.basename(path))
 175             dirname = os.path.dirname(result)
 176             if not isdir(dirname):
 177                 os.makedirs(dirname)
 178         if type(result) is bytes:
 179             result = bytes.decode(ENCS['default'])
 180         return(result)
 181
 182     def setOriginal(self, original):
 183         '''GLFileAssistant.setOriginal(original)
 184
 185         Set the name of the original file which will be used.'''
 186         if type(original) is bytes or type(original) is string:
 187             if type(original) is bytes:
 188                 original = original.decode(ENCS['default'])
 189         else:  # if original has not bytes or string type
 190             raise(TypeError(
 191                 'original must be a string, not %s' % (type(original).__name__)))
 192         self.original = original
 193
 194     def setRewritten(self, rewritten):
 195         '''GLFileAssistant.setRewritten(rewritten)
 196
 197         Set the name of the rewritten file which will be used.'''
 198         if type(rewritten) is bytes or type(rewritten) is string:
 199             if type(rewritten) is bytes:
 200                 rewritten = rewritten.decode(ENCS['default'])
 201         else:  # if rewritten has not bytes or string type
 202             raise(TypeError(
 203                 'rewritten must be a string, not %s' % type(rewritten).__name__))
 204         self.rewritten = rewritten
 205
 206     def addFile(self, file):
 207         '''GLFileAssistant.addFile(file)
 208
 209         Add file to the list of added files.'''
 210         if file not in self.added:
 211             self.added += [file]
 212
 213     def removeFile(self, file):
 214         '''GLFileAssistant.removeFile(file)
 215
 216         Remove file from the list of added files.'''
 217         if file in self.added:
 218             self.added.pop(file)
 219
 220     def getFiles(self):
 221         '''Return list of the added files.'''
 222         return(list(self.added))
 223
 224     def add(self, lookedup, tmpflag, tmpfile):
 225         '''GLFileAssistant.add(lookedup, tmpflag, tmpfile)
 226
 227         This method copies a file from gnulib into the destination directory.
 228         The destination is known to exist. If tmpflag is True, then lookedup file
 229         is a temporary one.'''
 230         original = self.original
 231         rewritten = self.rewritten
 232         destdir = self.config['destdir']
 233         symbolic = self.config['symbolic']
 234         lsymbolic = self.config['lsymbolic']
 235         if original == None:
 236             raise(TypeError('original must be set before applying the method'))
 237         elif rewritten == None:
 238             raise(TypeError('rewritten must be set before applying the method'))
 239         if not self.config['dryrun']:
 240             print('Copying file %s' % rewritten)
 241             loriginal = joinpath(self.config['localdir'], original)
 242             if (symbolic or (lsymbolic and lookedup == loriginal)) \
 243                     and not tmpflag and filecmp.cmp(lookedup, tmpfile):
 244                 constants.link_if_changed(
 245                     lookedup, joinpath(destdir, rewritten))
 246             else:  # if any of these conditions is not met
 247                 try:  # Try to move file
 248                     shutil.move(tmpfile, joinpath(destdir, rewritten))
 249                 except Exception as error:
 250                     raise(GLError(17, original))
 251         else:  # if self.config['dryrun']
 252             print('Copy file %s' % rewritten)
 253
 254     def update(self, lookedup, tmpflag, tmpfile, already_present):
 255         '''GLFileAssistant.update(lookedup, tmpflag, tmpfile, already_present)
 256
 257         This method copies a file from gnulib into the destination directory.
 258         The destination is known to exist. If tmpflag is True, then lookedup file
 259         is a temporary one.'''
 260         original = self.original
 261         rewritten = self.rewritten
 262         destdir = self.config['destdir']
 263         symbolic = self.config['symbolic']
 264         lsymbolic = self.config['lsymbolic']
 265         if original == None:
 266             raise(TypeError('original must be set before applying the method'))
 267         elif rewritten == None:
 268             raise(TypeError('rewritten must be set before applying the method'))
 269         if type(lookedup) is bytes or type(lookedup) is string:
 270             if type(lookedup) is bytes:
 271                 lookedup = lookedup.decode(ENCS['default'])
 272         else:  # if lookedup has not bytes or string type
 273             raise(TypeError('lookedup must be a string, not %s' %
 274                             type(lookedup).__name__))
 275         if type(already_present) is not bool:
 276             raise(TypeError('already_present must be a bool, not %s' %
 277                             type(already_present).__name__))
 278         basename = rewritten
 279         backupname = string('%s~' % basename)
 280         basepath = joinpath(destdir, basename)
 281         backuppath = joinpath(destdir, backupname)
 282         if not filecmp.cmp(basepath, tmpfile):
 283             if not self.config['dryrun']:
 284                 if already_present:
 285                     print('Updating file %s (backup in %s)' %
 286                           (basename, backupname))
 287                 else:  # if not already_present
 288                     message = 'Replacing file '
 289                     message += '%s (non-gnulib code backed up in ' % basename
 290                     message += '%s) !!' % backupname
 291                     print(message)
 292                 if isfile(backuppath):
 293                     os.remove(backuppath)
 294                 try:  # Try to replace the given file
 295                     shutil.move(basepath, backuppath)
 296                 except Exception as error:
 297                     raise(GLError(17, original))
 298                 loriginal = joinpath(self.config['localdir'], original)
 299                 if (symbolic or (lsymbolic and lookedup == loriginal)) \
 300                         and not tmpflag and filecmp.cmp(lookedup, tmpfile):
 301                     constants.link_if_changed(lookedup, basepath)
 302                 else:  # if any of these conditions is not met
 303                     try:  # Try to move file
 304                         if os.path.exists(basepath):
 305                             os.remove(basepath)
 306                         shutil.copy(tmpfile, rewritten)
 307                     except Exception as error:
 308                         raise(GLError(17, original))
 309             else:  # if self.config['dryrun']
 310                 if already_present:
 311                     print('Update file %s (backup in %s)' %
 312                           (rewritten, backup))
 313                 else:  # if not already_present
 314                     print('Replace file %s (backup in %s)' %
 315                           (rewritten, backup))
 316
 317     def add_or_update(self, already_present):
 318         '''GLFileAssistant.add_or_update(already_present)
 319
 320         This method handles a file that ought to be present afterwards.'''
 321         original = self.original
 322         rewritten = self.rewritten
 323         if original == None:
 324             raise(TypeError('original must be set before applying the method'))
 325         elif rewritten == None:
 326             raise(TypeError('rewritten must be set before applying the method'))
 327         if type(already_present) is not bool:
 328             raise(TypeError('already_present must be a bool, not %s' %
 329                             type(already_present).__name__))
 330         xoriginal = original
 331         if original.startswith('tests=lib/'):
 332             xoriginal = constants.substart('tests=lib/', 'lib/', original)
 333         lookedup, tmpflag = self.filesystem.lookup(xoriginal)
 334         tmpfile = self.tmpfilename(rewritten)
 335         sed_transform_lib_file = self.transformers.get('lib', '')
 336         sed_transform_build_aux_file = self.transformers.get('aux', '')
 337         sed_transform_main_lib_file = self.transformers.get('main', '')
 338         sed_transform_testsrelated_lib_file = self.transformers.get(
 339             'tests', '')
 340         try:  # Try to copy lookedup file to tmpfile
 341             shutil.copy(lookedup, tmpfile)
 342         except Exception as error:
 343             raise(GLError(15, lookedup))
 344         # Don't process binary files with sed.
 345         if not (original.endswith(".class") or original.endswith(".mo")):
 346             transformer = string()
 347             if original.startswith('lib/'):
 348                 if sed_transform_main_lib_file:
 349                     transformer = sed_transform_main_lib_file
 350             elif original.startswith('build-aux/'):
 351                 if sed_transform_build_aux_file:
 352                     transformer = sed_transform_build_aux_file
 353             elif original.startswith('tests=lib/'):
 354                 if sed_transform_testsrelated_lib_file:
 355                     transformer = sed_transform_testsrelated_lib_file
 356             if transformer:
 357                 args = ['sed', '-e', transformer]
 358                 stdin = codecs.open(lookedup, 'rb', 'UTF-8')
 359                 try:  # Try to transform file
 360                     data = sp.check_output(args, stdin=stdin, shell=False)
 361                     data = data.decode("UTF-8")
 362                 except Exception as error:
 363                     raise(GLError(16, lookedup))
 364                 with codecs.open(tmpfile, 'wb', 'UTF-8') as file:
 365                     file.write(data)
 366         path = joinpath(self.config['destdir'], rewritten)
 367         if isfile(path):
 368             self.update(lookedup, tmpflag, tmpfile, already_present)
 369             os.remove(tmpfile)
 370         else:  # if not isfile(path)
 371             self.add(lookedup, tmpflag, tmpfile)
 372             self.addFile(rewritten)
 373
 374     def super_update(self, basename, tmpfile):
 375         '''GLFileAssistant.super_update(basename, tmpfile) -> tuple
 376
 377         Move tmpfile to destdir/basename path, making a backup of it.
 378         Returns tuple, which contains basename, backupname and status.
 379           0: tmpfile is the same as destfile;
 380           1: tmpfile was used to update destfile;
 381           2: destfile was created, because it didn't exist.'''
 382         backupname = '%s~' % basename
 383         basepath = joinpath(self.config['destdir'], basename)
 384         backuppath = joinpath(self.config['destdir'], backupname)
 385         if isfile(basepath):
 386             if filecmp.cmp(basepath, tmpfile):
 387                 result_flag = 0
 388             else:  # if not filecmp.cmp(basepath, tmpfile)
 389                 result_flag = 1
 390                 if not self.config['dryrun']:
 391                     if isfile(backuppath):
 392                         os.remove(backuppath)
 393                     shutil.move(basepath, backuppath)
 394                     shutil.move(tmpfile, basepath)
 395                 else:  # if self.config['dryrun']
 396                     os.remove(tmpfile)
 397         else:  # if not isfile(basepath)
 398             result_flag = 2
 399             if not self.config['dryrun']:
 400                 if isfile(basepath):
 401                     os.remove(basepath)
 402                 shutil.move(tmpfile, basepath)
 403             else:  # if self.config['dryrun']
 404                 os.remove(tmpfile)
 405         result = tuple([basename, backupname, result_flag])
 406         return(result)