xorn/src/gaf/clib.py

   1 # gaf - Python library for manipulating gEDA files
   2 # Copyright (C) 1998-2010 Ales Hvezda
   3 # Copyright (C) 1998-2010 gEDA Contributors (see ChangeLog for details)
   4 # Copyright (C) 2013-2020 Roland Lutz
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 2 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program; if not, write to the Free Software Foundation,
  18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  19
  20 ## \namespace gaf.clib
  21 ## The component library system
  22 #
  23 # The <b>component library</b> is made up of a number of <b>component
  24 # sources</b>, each of which in turn makes available a number of
  25 # component <b>symbols</b>.  Each source is represented by a Python
  26 # object which implements the methods \c list() to retrieve a list of
  27 # symbol names, and \c get(symbol) to retrieve the symbol data for a
  28 # given symbol name.
  29 #
  30 # There are two predefined source types: a DirectorySource represents
  31 # a directory on disk containing symbol files, and a CommandSource
  32 # represents a command in the system \c PATH which can generate gEDA
  33 # symbol data (e.g. from a database).
  34 #
  35 # Each symbol is identified by its name, which is stored in the saved
  36 # schematic file.  The name must be valid for storage in a gEDA
  37 # schematic file as the "basename" of a "component" object.  For
  38 # symbols from directory sources, the filename of the symbol is taken
  39 # as the symbol name.  For a command source, the name may be any
  40 # permissible string.  Guidelines to follow:
  41 #
  42 #   -# Do not begin a symbol name with "EMBEDDED"
  43 #   -# Do not use whitespace, or any of the characters "/", ":", "!",
  44 #      "*", or "?".
  45 #   -# Try to use unique names.
  46 #
  47 # The component database may be queried using \ref search.  A revision
  48 # object containing the symbol data may be obtained using \ref
  49 # get_symbol.  If the source of a symbol isn't known, the symbol data
  50 # may be requested using the convenience function \ref lookup_symbol.
  51
  52 import collections, fnmatch, os, shlex, stat, subprocess, sys
  53 from gettext import gettext as _
  54 import gaf.read
  55 import gaf.ref
  56 import xorn.proxy
  57 import xorn.storage
  58
  59 ## Decide based on filename whether a file in a directory source is
  60 ## considered a symbol.
  61
  62 def sym_filename_filter(basename):
  63     return basename.lower().endswith('.sym') or \
  64            basename.lower().endswith('.sym.xml')
  65
  66 ## Named tuple class for storing data about a particular component source.
  67
  68 Source = collections.namedtuple('Source', ['callback', 'symbols', 'name'])
  69
  70 ## List of source triples for all known component sources.
  71
  72 _sources = []
  73
  74 ## Cache for search results of \ref search.
  75 #
  76 # The key of the hashtable is a pair <tt>(pattern, glob)</tt>
  77 # describing the search that was carried out, and the value is a list
  78 # of pairs <tt>(source, symbol)</tt>.
  79
  80 _search_cache = {}
  81
  82 ## Symbol data cache.
  83 #
  84 # The key of the hashtable is a pair <tt>(source, symbol)</tt>, and
  85 # the value is the data.
  86
  87 _symbol_cache = {}
  88
  89 ## Whether to load pixmaps referenced by symbols.
  90 #
  91 # This should be set before reading any symbols.  Otherwise, cached
  92 # symbols loaded with the wrong \a load_pixmaps flag may be returned.
  93
  94 load_pixmaps = False
  95
  96
  97 ## Raised on symbol lookup if the symbol isn't found in the library.
  98
  99 class NotFoundError(Exception):
 100     pass
 101
 102 ## Raised on symbol lookup if there are multiple matching symbols.
 103
 104 class DuplicateError(Exception):
 105     pass
 106
 107
 108 ## Source object representing a directory of symbol files.
 109 #
 110 # This class allows a directory which contains one or more symbol
 111 # files in gEDA format to be used as a component source.  Only files
 112 # ending in ".sym" (case insensitive) are considered to be symbol
 113 # files.  Symbol files with filenames starting with a period "." are
 114 # ignored.
 115
 116 class DirectorySource:
 117     def __init__(self, directory, recursive):
 118         ## Path to directory
 119         self.directory = directory
 120         ## Whether to recurse into subdirectories.
 121         self.recursive = recursive
 122
 123     ## Scan the directory for symbols.
 124
 125     def list(self):
 126         if not self.recursive:
 127             # skip subdirectories and anything else that isn't a
 128             # regular file (this is what libgeda does)
 129             entries = (entry for entry in os.listdir(self.directory)
 130                        if stat.S_ISREG(
 131                          os.stat(os.path.join(self.directory, entry)).st_mode))
 132         else:
 133             entries = (entry for dirpath, dirnames, filenames
 134                           in sorted(os.walk(self.directory))
 135                        for entry in sorted(filenames))
 136
 137         return (entry for entry in entries
 138                 # skip hidden files ("." and ".." are excluded by
 139                 # os.walk but not by os.listdir)
 140                 if entry[0] != '.'
 141                 # skip filenames which don't have the right suffix
 142                 and sym_filename_filter(entry))
 143
 144     ## Get symbol data for a given symbol name.
 145
 146     def get(self, symbol):
 147         if not self.recursive:
 148             path = os.path.join(self.directory, symbol)
 149             if not os.path.isfile(path):  # resolves symlinks
 150                 path = None
 151         else:
 152             path = None
 153             for dirpath, dirnames, filenames in \
 154                     sorted(os.walk(self.directory)):
 155                 if symbol in filenames:
 156                     path = os.path.join(dirpath, symbol)
 157                     break
 158
 159         if path is not None:
 160             return gaf.read.read(path, load_pixmaps = load_pixmaps)
 161
 162         raise ValueError, 'symbol "%s" not found in library' % symbol
 163
 164
 165 ## Source object representing a pair of symbol-generating commands.
 166 #
 167 # This class allows a program or pair of programs in the system search
 168 # path which can generate symbols to be used as a component source.
 169 #
 170 # \a list_cmd and \a get_cmd should be pre-tokenized invocations
 171 # consisting of an executable name followed by any arguments required.
 172 # Executables are resolved using the current \c PATH.
 173 #
 174 # The list command will be executed with no additional arguments, and
 175 # should output a list of available symbol names on the stdandard
 176 # output.  The get command will have a symbol name appended to it as
 177 # the final argument, and should output gEDA symbol data on standard
 178 # output.
 179 #
 180 # If the command cannot successfully complete, it should exit with
 181 # non-zero exit status.  Anything it has output on stdout will be
 182 # ignored, so stderr should be used for diagnostics.
 183
 184 class CommandSource:
 185     def __init__(self, list_cmd, get_cmd):
 186         ## Command and arguments for listing available symbols
 187         self.list_cmd = list_cmd
 188         ## Command and arguments for retrieving symbol data
 189         self.get_cmd = get_cmd
 190
 191     ## Poll the library command for symbols.
 192     #
 193     # Runs the library command, requesting a list of available
 194     # symbols, and returns the new list.
 195
 196     def list(self):
 197         # Run the command to get the list of symbols
 198         lines = _run_source_command(shlex.split(self.list_cmd),
 199                                     lambda f: f.readlines())
 200
 201         for line in lines:
 202             if not line or line[-1] != '\n':
 203                 raise ValueError, "Missing newline at end of command output"
 204             if line == '\n':
 205                 raise ValueError, "Command printed an empty line"
 206             if line[0] == '.':
 207                 raise ValueError, "Command printed line starting with '.'"
 208
 209         return (line[:-1] for line in lines)
 210
 211     ## Get symbol data for a given symbol name.
 212
 213     def get(self, symbol):
 214         return _run_source_command(
 215             shlex.split(self.get_cmd) + [symbol],
 216             lambda f: gaf.read.read_file(
 217                 f, '<pipe>', load_pixmaps = load_pixmaps))
 218
 219
 220 ## Execute a library command.
 221 #
 222 # Executes the command given by the first item of the argument
 223 # sequence \a args, using the system search path to find the program
 224 # to execute.  Calls the function \a callback with a file-like object
 225 # representing the pipe connected to the command's standard output as
 226 # a single argument.  Once \a callback finishes, discards all data
 227 # left on the pipe and waits for the program to terminate.
 228 #
 229 # \returns the value returned by \a callback
 230 #
 231 # \throws ValueError if the command returns a non-zero exit status or
 232 #                    is terminated by a signal
 233
 234 def _run_source_command(args, callback):
 235     p = subprocess.Popen(
 236         args, bufsize = 4096, executable = executable,
 237         stdout = subprocess.PIPE, close_fds = True) # cwd = virtual_cwd
 238
 239     try:
 240         return callback(p.stdout)
 241     finally:
 242         try:
 243             p.read()  # avoid deadlock
 244         finally:
 245             p.wait()
 246             if p.returncode < 0:
 247                 raise ValueError, "Library command failed [%s]: " \
 248                     "Uncaught signal %i" % (args[0], -p.returncode)
 249             if p.returncode != 0:
 250                 raise ValueError, "Library command failed [%s]: "\
 251                     "returned exit status %d" % (args[0], p.returncode)
 252
 253 ## Update list of symbols available from a component source.
 254 #
 255 # Calls \c source.callback.list() and performs type and uniqueness
 256 # checks on the returned list.  If no errors are found, replaces the
 257 # list of available symbols with the returned list.
 258 #
 259 # \throws TypeError if \a symbols is not iterable
 260 # \throws TypeError if \a symbols yields an item that isn't a string
 261 # \throws ValueError if \a symbols yields a duplicate item
 262
 263 def _update_symbol_list(source):
 264     try:
 265         symbols = list(source.callback.list())
 266     except TypeError:
 267         raise TypeError, "Failed to scan library [%s]: " \
 268             "Python function returned non-list" % source.name
 269
 270     found = set()
 271     duplicate = set()
 272     for symbol in symbols:
 273         if not isinstance(symbol, str) and \
 274            not isinstance(symbol, unicode):
 275             raise TypeError, "Non-string symbol name " \
 276                 "while scanning library [%s]" % source.name
 277         if symbol in found:
 278             duplicate.add(symbol)
 279         found.add(symbol)
 280
 281     if duplicate:
 282         if source.name:
 283             sys.stderr.write(_("Library \"%s\" contains symbols with "
 284                                "conflicting names:\n") % source.name)
 285         else:
 286             sys.stderr.write(_("Library contains symbols with "
 287                                "conflicting names:\n"))
 288         for symbol in sorted(duplicate):
 289             sys.stderr.write(_("\t%s\n") % symbol)
 290
 291     symbols.sort()
 292     source.symbols[:] = symbols
 293
 294
 295 ## Add a component source to the library.
 296 #
 297 # \a callback must implement two methods: \c callback.list() to return
 298 # a list of symbol names, and \c callback.get(symbol) to return the
 299 # symbol data for a given a symbol name.
 300 #
 301 # \param callback  source object which implements \c list and \c get
 302 # \param name      unique descriptive name for the component source
 303 #
 304 # \throws ValueError if another source with this name already exists
 305
 306 def add_source(callback, name):
 307     if name is None:
 308         raise ValueError, "Cannot add source: name not specified"
 309     for source in _sources:
 310         if source.name == name:
 311             raise ValueError, "There is already a source called '%s'" % name
 312
 313     # Sources added later get scanned earlier
 314     source = Source(callback, [], name)
 315     _update_symbol_list(source)
 316     _sources.insert(0, source)
 317
 318     _search_cache.clear()
 319     _symbol_cache.clear()
 320
 321 ## Get a component source by name.
 322 #
 323 # Iterates through the known component sources, checking if there is a
 324 # source with the given \a name.
 325 #
 326 # \throws ValueError if no matching source was found
 327
 328 def lookup_source(name):
 329     for source in _sources:
 330         if source.name == name:
 331             return source
 332
 333     raise ValueError
 334
 335 ## Make sure a source name is unique.
 336 #
 337 # Checks if a source with the given \a name already exists.  If it
 338 # does, appends a number in angular brackets to the source name making
 339 # it unique.  If \a name is not already in use, returns it as is.
 340
 341 def uniquify_source_name(name):
 342     i = 0
 343     newname = name
 344
 345     while True:
 346         try:
 347             lookup_source(newname)
 348         except ValueError:
 349             break
 350         i += 1
 351         newname = '%s<%i>' % (name, i)
 352
 353     return newname
 354
 355 ## Rescan all available component libraries.
 356 #
 357 # Resets the list of symbols available from each source, and
 358 # repopulates it from scratch.  Useful e.g. for checking for new
 359 # symbols.
 360
 361 def refresh():
 362     for source in _sources:
 363         _update_symbol_list(source)
 364
 365     _search_cache.clear()
 366     _symbol_cache.clear()
 367
 368 ## Remove all component library sources.
 369
 370 def reset():
 371     del _sources[:]
 372     _search_cache.clear()
 373     _symbol_cache.clear()
 374
 375
 376 ## Get symbol object for a given source object and symbol name.
 377 #
 378 # Returns a gaf.ref.Symbol object containing the symbol called
 379 # \a symbol from the component source \a source.
 380 #
 381 # \throws ValueError if the source object's \c get function doesn't
 382 #                    return a xorn.storage.Revision or
 383 #                    xorn.proxy.RevisionProxy instance
 384
 385 def get_symbol(source, symbol):
 386     assert source is not None
 387     assert symbol is not None
 388
 389     # First, try the cache.
 390     try:
 391         return _symbol_cache[id(source), symbol]
 392     except KeyError:
 393         pass
 394
 395     # If the symbol wasn't found in the cache, get it directly.
 396     data = source.callback.get(symbol)
 397     if isinstance(data, xorn.proxy.RevisionProxy):
 398         data = data.rev
 399     if not isinstance(data, xorn.storage.Revision):
 400         raise ValueError, "Failed to load symbol data [%s] " \
 401             "from source [%s]" % (symbol, source.name)
 402     data.finalize()
 403
 404     symbol_ = gaf.ref.Symbol(symbol, data, False)
 405
 406     # Cache the symbol data
 407     _symbol_cache[id(source), symbol] = symbol_
 408
 409     return symbol_
 410
 411 ## Invalidate cached data about a symbol.
 412
 413 def invalidate_symbol_data(source, symbol):
 414     try:
 415         del _symbol_cache[id(source), symbol]
 416     except KeyError:
 417         pass
 418
 419
 420 ## Find all symbols matching a pattern.
 421 #
 422 # Searches the library, returning all symbols whose names match \a
 423 # pattern.  If \a glob is \c True, then \a pattern is assumed to be a
 424 # glob pattern; otherwise, only exact matches are returned.
 425 #
 426 # \returns a list of pairs <tt>(source, symbol)</tt>
 427 #
 428 # \throws DuplicateError if one library contains multiple matching symbols
 429
 430 def search(pattern, glob = False):
 431     # Check to see if the query is already in the cache
 432     try:
 433         return _search_cache[pattern, glob]
 434     except KeyError:
 435         pass
 436
 437     result = []
 438     for source in _sources:
 439         if glob:
 440             for symbol in fnmatch.filter(source.symbols, pattern):
 441                 result.append((source, symbol))
 442         elif pattern in source.symbols:
 443             if source.symbols.count(pattern) > 1:
 444                 raise DuplicateError, \
 445                     "More than one component found with name [%s]" % pattern
 446             result.append((source, pattern))
 447
 448     _search_cache[pattern, glob] = result
 449     return result[:]
 450
 451 ## Get source for a given symbol name.
 452 #
 453 # Returns the source of the first symbol found with the given \a name.
 454 # If more than one matching symbol is found, emits a warning to stderr.
 455 #
 456 # \throws NotFoundError if the symbol was not found
 457 # \throws DuplicateError if one library contains multiple matching symbols
 458
 459 def lookup_symbol_source(name):
 460     symlist = search(name)
 461
 462     if not symlist:
 463         raise NotFoundError, "Component [%s] was not found in the " \
 464             "component library" % name
 465
 466     if len(symlist) > 1:
 467         sys.stderr.write(_("More than one component found "
 468                            "with name [%s]\n")% name)
 469
 470     source, symbol = symlist[0]
 471     assert symbol == name
 472     return source
 473
 474 ## Get symbol object for a given symbol name.
 475 #
 476 # Returns the gaf.ref.Symbol object for the first symbol found
 477 # with the given name.  This is a helper function for the schematic
 478 # load system, as it will always want to load symbols given only their
 479 # name.
 480 #
 481 # \throws NotFoundError  if the component was not found
 482 # \throws DuplicateError if one library contains multiple matching symbols
 483 # \throws ValueError     if the source object's \c get function doesn't
 484 #                        return a xorn.storage.Revision or
 485 #                        xorn.proxy.RevisionProxy instance
 486
 487 def lookup_symbol(name):
 488     return get_symbol(lookup_symbol_source(name), name)
 489
 490
 491 ## Return a list of symbols used in a revision.
 492 #
 493 # The list is free of duplicates and preserves the order of the
 494 # symbols as they appear first in the file.  Each symbol is
 495 # represented by its actual gaf.ref.Symbol object.
 496
 497 def used_symbols0(rev):
 498     symbols = []
 499     for ob in rev.all_objects():
 500         data = ob.data()
 501         if isinstance(data, xorn.storage.Component) \
 502                and data.symbol not in symbols:
 503             symbols.append(data.symbol)
 504     return symbols
 505
 506 ## Return a list of symbols used in a revision.
 507 #
 508 # Scan a revision looking for symbols, look them up in the library,
 509 # and return them as a list.  Each symbol is represented by a pair
 510 # <tt>(source, symbol)</tt>.  The returned list only contains symbols
 511 # that were found in the library and is sorted in alphabetical order.
 512 #
 513 # \bug Only includes components which are not embedded, but they
 514 #      should (probably) also appear in the list.
 515
 516 def used_symbols1(rev):
 517     result = []
 518
 519     for ob in rev.all_objects():
 520         data = ob.data()
 521         if not isinstance(data, xorn.storage.Component):
 522             continue
 523
 524         # ignore embedded symbols
 525         if data.symbol.embedded:
 526             continue
 527
 528         # Since we're not looking at embedded symbols, the first
 529         # component with the given name will be the one we need.
 530         # N.b. we don't use lookup_symbol_source() because it's
 531         # spammeh.
 532         symlist = search(data.symbol.basename)
 533         if not symlist:
 534             continue
 535         result.append(symlist[0])
 536
 537     result.sort(key = lambda (source, symbol): symbol)
 538     return result