unistr/u{8,16,32}-uctomb: Avoid possible trouble with huge strings.
[gnulib.git] / pygnulib / GLModuleSystem.py
blob73c207d995addb7a4d9a8eb8a33983e6589080ae
1 #!/usr/bin/python
2 # encoding: UTF-8
4 #===============================================================================
5 # Define global imports
6 #===============================================================================
7 import os
8 import re
9 import sys
10 import codecs
11 import hashlib
12 import subprocess as sp
13 from . import constants
14 from .GLError import GLError
15 from .GLConfig import GLConfig
16 from .GLFileSystem import GLFileSystem
19 #===============================================================================
20 # Define module information
21 #===============================================================================
22 __author__ = constants.__author__
23 __license__ = constants.__license__
24 __copyright__ = constants.__copyright__
27 #===============================================================================
28 # Define global constants
29 #===============================================================================
30 PYTHON3 = constants.PYTHON3
31 NoneType = type(None)
32 APP = constants.APP
33 DIRS = constants.DIRS
34 ENCS = constants.ENCS
35 UTILS = constants.UTILS
36 MODES = constants.MODES
37 TESTS = constants.TESTS
38 compiler = constants.compiler
39 joinpath = constants.joinpath
40 cleaner = constants.cleaner
41 string = constants.string
42 isabs = os.path.isabs
43 isdir = os.path.isdir
44 isfile = os.path.isfile
45 normpath = os.path.normpath
46 relpath = os.path.relpath
47 filter_filelist = constants.filter_filelist
50 #===============================================================================
51 # Define GLModuleSystem class
52 #===============================================================================
53 class GLModuleSystem(object):
54 '''GLModuleSystem is used to operate with module system using dynamic
55 searching and patching.'''
57 def __init__(self, config):
58 '''GLModuleSystem.__init__(config) -> GLModuleSystem
60 Create new GLModuleSystem instance. Some functions use GLFileSystem class
61 to look up a file in localdir or gnulib directories, or combine it through
62 'patch' utility.'''
63 self.args = dict()
64 if type(config) is not GLConfig:
65 raise(TypeError('config must be a GLConfig, not %s' %
66 type(config).__name__))
67 self.config = config
68 self.filesystem = GLFileSystem(self.config)
70 def __repr__(self):
71 '''x.__repr__ <==> repr(x)'''
72 result = '<pygnulib.GLModuleSystem %s>' % hex(id(self))
73 return(result)
75 def exists(self, module):
76 '''GLModuleSystem.exists(module) -> bool
78 Check whether the given module exists.
79 GLConfig: localdir.'''
80 if type(module) is bytes or string:
81 if type(module) is bytes:
82 module = module.decode(ENCS['default'])
83 else: # if module has not bytes or string type
84 raise(TypeError(
85 'module must be a string, not %s' % type(module).__name__))
86 result = bool()
87 badnames = ['ChangeLog', 'COPYING', 'README', 'TEMPLATE',
88 'TEMPLATE-EXTENDED', 'TEMPLATE-TESTS']
89 if isfile(joinpath(DIRS['modules'], module)) or \
90 all([ # Begin all(iterable) function
91 self.config['localdir'],
92 isdir(joinpath(self.config['localdir'], 'modules')),
93 isfile(
94 joinpath(self.config['localdir'], 'modules', module))
95 ]): # Close all(iterable) function
96 if module not in badnames:
97 result = True
98 return(result)
100 def find(self, module):
101 '''GLModuleSystem.find(module) -> GLModule
103 Find the given module.'''
104 if type(module) is bytes or string:
105 if type(module) is bytes:
106 module = module.decode(ENCS['default'])
107 else: # if module has not bytes or string type
108 raise(TypeError(
109 'module must be a string, not %s' % type(module).__name__))
110 if self.exists(module):
111 path, istemp = self.filesystem.lookup(joinpath('modules', module))
112 result = GLModule(self.config, path, istemp)
113 return(result)
114 else: # if not self.exists(module)
115 if self.config['errors']:
116 raise(GLError(3, module))
117 else: # if not self.config['errors']
118 sys.stderr.write('gnulib-tool: warning: ')
119 sys.stderr.write('file %s does not exist\n' % str(module))
121 def list(self):
122 '''GLModuleSystem.list() -> list
124 Return the available module names as tuple. We could use a combination
125 of os.walk() function and re module. However, it takes too much time to
126 complete, so this version uses subprocess to run shell commands.'''
127 result = string()
128 listing = list()
129 localdir = self.config['localdir']
130 find_args = ['find', 'modules', '-type', 'f', '-print']
131 sed_args = \
133 'sed',
134 '-e', r's,^modules/,,',
135 '-e', r'/^ChangeLog$/d',
136 '-e', r'/\/ChangeLog$/d',
137 '-e', r'/^COPYING$/d',
138 '-e', r'/\/COPYING$/d',
139 '-e', r'/^README$/d',
140 '-e', r'/\/README$/d',
141 '-e', r'/^TEMPLATE$/d',
142 '-e', r'/^TEMPLATE-EXTENDED$/d',
143 '-e', r'/^TEMPLATE-TESTS$/d',
144 '-e', r'/^\..*/d',
145 '-e', r'/~$/d',
146 '-e', r'/-tests$/d',
149 # Read modules from gnulib root directory.
150 os.chdir(constants.DIRS['root'])
151 find = sp.Popen(find_args, stdout=sp.PIPE)
152 result += find.stdout.read().decode("UTF-8")
154 # Read modules from local directory.
155 if localdir and isdir(joinpath(localdir, 'modules')):
156 os.chdir(localdir)
157 find = sp.Popen(find_args, stdout=sp.PIPE)
158 result += find.stdout.read().decode("UTF-8")
159 sed_args += ['-e', r's,\.diff$,,']
161 # Save the list of the modules to file.
162 os.chdir(DIRS['cwd'])
163 path = joinpath(self.config['tempdir'], 'list')
164 with codecs.open(path, 'wb', 'UTF-8') as file:
165 file.write(result)
167 # Filter the list of the modules.
168 stdin = codecs.open(path, 'rb', 'UTF-8')
169 sed = sp.Popen(sed_args, stdin=stdin, stdout=sp.PIPE)
170 result = sed.stdout.read().decode("UTF-8")
171 stdin.close()
172 os.remove(path)
173 listing = [line for line in result.split('\n') if line.strip()]
174 listing = sorted(set(listing))
175 return(listing)
178 #===============================================================================
179 # Define GLModule class
180 #===============================================================================
181 class GLModule(object):
182 '''GLModule is used to create a module object from the file with the given
183 path. GLModule can get all information about module, get its dependencies,
184 files, etc.'''
186 def __init__(self, config, module, patched=False):
187 '''GLModule.__init__(config, module[, patched]) -> GLModule
189 Create new GLModule instance. Arguments are module and patched, where
190 module is a string representing the path to the module and patched is a
191 bool indicating that module was created after applying patch.'''
192 self.args = dict()
193 self.cache = dict()
194 self.content = string()
195 if type(config) is not GLConfig:
196 raise(TypeError('config must be a GLConfig, not %s' %
197 type(config).__name__))
198 if type(module) is bytes or type(module) is string:
199 if type(module) is bytes:
200 module = module.decode(ENCS['default'])
201 else: # if module has not bytes or string type
202 raise(TypeError('module must be a string, not %s' %
203 type(module).__name__))
204 if type(patched) is not bool:
205 raise(TypeError('patched must be a bool, not %s' %
206 type(module).__name__))
207 self.module = module
208 self.patched = patched
209 self.config = config
210 self.filesystem = GLFileSystem(self.config)
211 self.modulesystem = GLModuleSystem(self.config)
212 with codecs.open(module, 'rb', 'UTF-8') as file:
213 self.content = file.read()
214 self.regex = '(?:Description:|Comment:|Status:|Notice:|Applicability:|\
215 Files:|Depends-on:|configure\\.ac-early:|configure\\.ac:|Makefile\\.am:|\
216 Include:|Link:|License:|Maintainer:)'
218 def __eq__(self, module):
219 '''x.__eq__(y) <==> x==y'''
220 result = bool()
221 if type(module) is GLModule:
222 if self.module == module.module:
223 result = True
224 return(result)
226 def __ne__(self, module):
227 '''x.__ne__(y) <==> x!=y'''
228 result = bool()
229 if type(module) is GLModule:
230 if self.module != module.module:
231 result = True
232 return(result)
234 def __ge__(self, module):
235 '''x.__ge__(y) <==> x>=y'''
236 result = bool()
237 if type(module) is GLModule:
238 if self.module >= module.module:
239 result = True
240 return(result)
242 def __gt__(self, module):
243 '''x.__gt__(y) <==> x>y'''
244 result = bool()
245 if type(module) is GLModule:
246 if self.module > module.module:
247 result = True
248 return(result)
250 def __hash__(self):
251 '''x.__hash__() <==> hash(x)'''
252 module = hash(self.module)
253 patched = hash(self.patched)
254 result = module ^ patched
255 return(result)
257 def __le__(self, module):
258 '''x.__le__(y) <==> x<=y'''
259 result = bool()
260 if type(module) is GLModule:
261 if self.module <= module.module:
262 result = True
263 return(result)
265 def __lt__(self, module):
266 '''x.__lt__(y) <==> x<y'''
267 result = bool()
268 if type(module) is GLModule:
269 if self.module < module.module:
270 result = True
271 return(result)
273 def __str__(self):
274 '''x.__str__() <==> str(x)'''
275 result = self.getName()
276 return(result)
278 def __repr__(self):
279 '''x.__repr__ <==> repr(x)'''
280 result = '<pygnulib.GLModule %s %s>' % \
281 (repr(self.getName()), hex(id(self)))
282 return(result)
284 def getName(self):
285 '''GLModule.getName() -> string
287 Return the name of the module.'''
288 pattern = compiler(joinpath('modules', '(.*?)$'))
289 result = pattern.findall(self.module)[0]
290 return(result)
292 def isPatched(self):
293 '''GLModule.isPatched() -> bool
295 Check whether module was created after applying patch.'''
296 return(self.patched)
298 def isTests(self):
299 '''GLModule.isTests() -> bool
301 Check whether module is a -tests version of module.'''
302 result = self.getName().endswith('-tests')
303 return(result)
305 def isNonTests(self):
306 '''GLModule.isTests() -> bool
308 Check whether module is not a -tests version of module.'''
309 result = not(self.isTests())
310 return(result)
312 def getTestsName(self):
313 '''Return -tests version of the module name.'''
314 result = self.getName()
315 if not result.endswith('-tests'):
316 result += '-tests'
317 return(result)
319 def getTestsModule(self):
320 '''Return -tests version of the module as GLModule.'''
321 result = self.modulesystem.find(self.getTestsName())
322 return(result)
324 def getShellFunc(self):
325 '''GLModule.getShellFunc() -> string
327 Computes the shell function name that will contain the m4 macros for the
328 module.'''
329 isalnum = True
330 macro_prefix = self.config['macro_prefix']
331 for char in str(module):
332 if char not in constants.ALPHANUMERIC:
333 isalnum = False
334 break
335 if isalnum:
336 module = str(self)
337 else: # if not isalnum
338 module = '%s\n' % str(self)
339 if type(module) is string:
340 module = module.encode(ENCS['default'])
341 module = hashlib.md5(module).hexdigest()
342 result = 'func_%s_gnulib_m4code_%s' % (macro_prefix, module)
343 if type(result) is bytes:
344 result = result.decode(ENCS['default'])
345 return(result)
347 def getShellVar(self):
348 '''GLModule.getShellVar() -> string
350 Compute the shell variable name the will be set to true once the m4 macros
351 for the module have been executed.'''
352 isalnum = True
353 macro_prefix = self.config['macro_prefix']
354 for char in str(module):
355 if char not in constants.ALPHANUMERIC:
356 isalnum = False
357 break
358 if isalnum:
359 module = str(self)
360 else: # if not isalnum
361 module = '%s\n' % str(self)
362 if type(module) is string:
363 module = module.encode(ENCS['default'])
364 module = hashlib.md5(module).hexdigest()
365 result = '%s_gnulib_enabled_%s' % (macro_prefix, module)
366 if type(result) is bytes:
367 result = result.decode(ENCS['default'])
368 return(result)
370 def getConditionalName(self):
371 '''GLModule.getConditionalName() -> string
373 Return the automake conditional name.
374 GLConfig: macro_prefix.'''
375 macro_prefix = self.config['macro_prefix']
376 nonascii = \
377 [ # Begin to filter non-ascii chars
378 char for char in self.getName() if char not in \
379 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_'
380 ] # Finish to filter non-ascii chars
381 if nonascii:
382 name = self.getName().encode(ENCS['default'])
383 name = hashlib.md5(name).hexdigest()
384 conditional = '%s_GNULIB_ENABLED_%s' % (macro_prefix, name)
385 else: # if not nonascii
386 result = '%s_GNULIB_ENABLED_%s' (macro_prefix, name)
387 if type(result) is bytes:
388 result = result.decode(ENCS['default'])
389 return(result)
391 def getDescription(self):
392 '''GLModule.getDescription() -> string
394 Return description of the module.'''
395 section = 'Description:'
396 if 'description' not in self.cache:
397 if section not in self.content:
398 result = string()
399 else: # if section in self.content
400 pattern = '^%s[\t ]*(.*?)%s' % (section, self.regex)
401 pattern = compiler(pattern, re.S | re.M)
402 result = pattern.findall(self.content)
403 if type(result) is list:
404 if not result:
405 result = string()
406 else: # if result
407 result = result[-1]
408 result = result.strip()
409 self.cache['description'] = result
410 return(self.cache['description'])
412 def getComment(self):
413 '''GLModule.getComment() -> string
415 Return comment to module.'''
416 section = 'Comment:'
417 if 'comment' not in self.cache:
418 if section not in self.content:
419 result = string()
420 else: # if section in self.content
421 pattern = '^%s[\t ]*(.*?)%s' % (section, self.regex)
422 pattern = compiler(pattern, re.S | re.M)
423 result = pattern.findall(self.content)
424 if type(result) is list:
425 if not result:
426 result = string()
427 else: # if result
428 result = result[-1]
429 result = result.strip()
430 self.cache['comment'] = result
431 return(self.cache['comment'])
433 def getStatus(self):
434 '''GLModule.getStatus() -> string
436 Return module status.'''
437 section = 'Status:'
438 if 'status' not in self.cache:
439 if section not in self.content:
440 result = string()
441 else: # if section in self.content
442 snippet = self.content.split(section)[-1]
443 snippet = snippet.replace('\r\n', '\n')
444 lines = ['%s\n' % line for line in snippet.split('\n')]
445 parts = list()
446 for line in lines:
447 regex = '^(Description|Comment|Status|Notice|Applicability|'
448 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
449 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
450 pattern = compiler(regex)
451 findflag = pattern.findall(line)
452 if findflag:
453 break
454 parts += [line]
455 result = [part.strip() for part in parts if part.strip()]
456 self.cache['status'] = list(result)
457 return(list(self.cache['status']))
459 def getNotice(self):
460 '''GLModule.getNotice() -> string
462 Return notice to module.'''
463 section = 'Notice:'
464 if 'notice' not in self.cache:
465 if section not in self.content:
466 result = string()
467 else: # if section in self.content
468 snippet = self.content.split(section)[-1]
469 snippet = snippet.replace('\r\n', '\n')
470 lines = ['%s\n' % line for line in snippet.split('\n')]
471 parts = list()
472 for line in lines:
473 regex = '^(Description|Comment|Status|Notice|Applicability|'
474 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
475 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
476 pattern = compiler(regex)
477 findflag = pattern.findall(line)
478 if findflag:
479 break
480 parts += [line]
481 result = ''.join(parts)
482 self.cache['notice'] = result
483 return(self.cache['notice'])
485 def getApplicability(self):
486 '''GLModule.getApplicability() -> string
488 Return applicability of module.'''
489 section = 'Applicability:'
490 if 'applicability' not in self.cache:
491 if section not in self.content:
492 result = string()
493 else: # if section in self.content
494 snippet = self.content.split(section)[-1]
495 snippet = snippet.replace('\r\n', '\n')
496 lines = ['%s\n' % line for line in snippet.split('\n')]
497 parts = list()
498 for line in lines:
499 regex = '^(Description|Comment|Status|Notice|Applicability|'
500 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
501 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
502 pattern = compiler(regex)
503 findflag = pattern.findall(line)
504 if findflag:
505 break
506 parts += [line]
507 parts = [part.strip() for part in parts]
508 result = ''.join(parts)
509 if not result.strip():
510 if self.getName().endswith('-tests'):
511 result = 'tests'
512 else: # if not self.getName().endswith('-tests')
513 result = 'main'
514 if type(result) is bytes:
515 result = result.decode(ENCS['default'])
516 result = result.strip()
517 self.cache['applicability'] = result
518 return(self.cache['applicability'])
520 def getFiles(self):
521 '''GLModule.getFiles() -> list
523 Return list of files.
524 GLConfig: ac_version.'''
525 ac_version = self.config['ac_version']
526 section = 'Files:'
527 result = list()
528 if 'files' not in self.cache:
529 if section not in self.content:
530 result = list()
531 else: # if section in self.content
532 snippet = self.content.split(section)[-1]
533 snippet = snippet.replace('\r\n', '\n')
534 lines = ['%s\n' % line for line in snippet.split('\n')]
535 parts = list()
536 for line in lines:
537 regex = '^(Description|Comment|Status|Notice|Applicability|'
538 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
539 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
540 pattern = compiler(regex)
541 findflag = pattern.findall(line)
542 if findflag:
543 break
544 parts += [line]
545 result = [part.strip() for part in parts if part.strip()]
546 result += [joinpath('m4', '00gnulib.m4')]
547 result += [joinpath('m4', 'zzgnulib.m4')]
548 result += [joinpath('m4', 'gnulib-common.m4')]
549 self.cache['files'] = list(result)
550 return(list(self.cache['files']))
552 def getDependencies(self):
553 '''GLModule.getDependencies() -> list
555 Return list of dependencies.
556 GLConfig: localdir.'''
557 localdir = self.config['localdir']
558 result = list()
559 section = 'Depends-on:'
560 if 'dependencies' not in self.cache:
561 if section not in self.content:
562 depmodules = list()
563 else: # if section in self.content
564 snippet = self.content.split(section)[-1]
565 snippet = snippet.replace('\r\n', '\n')
566 lines = ['%s\n' % line for line in snippet.split('\n')]
567 parts = list()
568 for line in lines:
569 regex = '^(Description|Comment|Status|Notice|Applicability|'
570 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
571 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
572 pattern = compiler(regex)
573 findflag = pattern.findall(line)
574 if findflag:
575 break
576 parts += [line]
577 modules = ''.join(parts)
578 modules = [line for line in modules.split(
579 '\n') if line.strip()]
580 modules = [
581 module for module in modules if not module.startswith('#')]
582 for line in modules:
583 split = [part for part in line.split(' ') if part.strip()]
584 if len(split) == 1:
585 module = line.strip()
586 condition = None
587 else: # if len(split) != 1
588 module = split[0]
589 condition = split[1]
590 if type(condition) is bytes:
591 condition = condition.decode(ENCS['default'])
592 result += [tuple([self.modulesystem.find(module), condition])]
593 self.cache['dependencies'] = result
594 return(list(self.cache['dependencies']))
596 def getAutoconfSnippet_Early(self):
597 '''GLModule.getAutoconfSnippet_Early() -> string
599 Return autoconf-early snippet.'''
600 section = 'configure.ac-early:'
601 if 'autoconf-early' not in self.cache:
602 if section not in self.content:
603 result = string()
604 else: # if section in self.content
605 snippet = self.content.split(section)[-1]
606 snippet = snippet.replace('\r\n', '\n')
607 lines = ['%s\n' % line for line in snippet.split('\n')]
608 parts = list()
609 for line in lines:
610 regex = '^(Description|Comment|Status|Notice|Applicability|'
611 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
612 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
613 pattern = compiler(regex)
614 findflag = pattern.findall(line)
615 if findflag:
616 break
617 parts += [line]
618 result = ''.join(parts)
619 self.cache['autoconf-early'] = result
620 return(self.cache['autoconf-early'])
622 def getAutoconfSnippet(self):
623 '''GLModule.getAutoconfSnippet() -> string
625 Return autoconf snippet.'''
626 section = 'configure.ac:'
627 if 'autoconf' not in self.cache:
628 if section not in self.content:
629 result = string()
630 else: # if section in self.content
631 snippet = self.content.split(section)[-1]
632 snippet = snippet.replace('\r\n', '\n')
633 lines = ['%s\n' % line for line in snippet.split('\n')]
634 parts = list()
635 for line in lines:
636 regex = '^(Description|Comment|Status|Notice|Applicability|'
637 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
638 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
639 pattern = compiler(regex)
640 findflag = pattern.findall(line)
641 if findflag:
642 break
643 parts += [line]
644 result = ''.join(parts)
645 self.cache['autoconf'] = result
646 return(self.cache['autoconf'])
648 def getAutomakeSnippet(self):
649 '''getAutomakeSnippet() -> string
651 Get automake snippet.
652 GLConfig: auxdir, ac_version.'''
653 result = string() # Define stack variable
654 conditional = self.getAutomakeSnippet_Conditional()
655 if conditional.strip():
656 result += self.getAutomakeSnippet_Conditional()
657 else: # if not conditional.strip()
658 result += '\n'
659 result += self.getAutomakeSnippet_Unconditional()
660 return(result)
662 def getAutomakeSnippet_Conditional(self):
663 '''GLModule.getAutomakeSnippet_Conditional() -> string
665 Return conditional automake snippet.'''
666 section = 'Makefile.am:'
667 if 'makefile-conditional' not in self.cache:
668 if section not in self.content:
669 result = string()
670 else: # if section in self.content
671 snippet = self.content.split(section)[-1]
672 snippet = snippet.replace('\r\n', '\n')
673 lines = ['%s\n' % line for line in snippet.split('\n')]
674 parts = list()
675 for line in lines:
676 regex = '^(Description|Comment|Status|Notice|Applicability|'
677 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
678 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
679 pattern = compiler(regex)
680 findflag = pattern.findall(line)
681 if findflag:
682 break
683 parts += [line]
684 result = ''.join(parts)
685 self.cache['makefile-conditional'] = result
686 return(self.cache['makefile-conditional'])
688 def getAutomakeSnippet_Unconditional(self):
689 '''GLModule.getAutomakeSnippet_Unconditional() -> string
691 Return unconditional automake snippet.
692 GLConfig: auxdir, ac_version.'''
693 auxdir = self.config['auxdir']
694 ac_version = self.config['ac_version']
695 result = string()
696 if 'makefile-unconditional' not in self.cache:
697 if self.isTests():
698 files = self.getFiles()
699 extra_files = filter_filelist(constants.NL, files,
700 'tests/', '', 'tests/', '').split(constants.NL)
701 extra_files = sorted(set(extra_files))
702 if extra_files:
703 result += string('EXTRA_DIST += %s' %
704 ' '.join(extra_files))
705 result += constants.NL * 2
706 else: # if not tests module
707 # TODO: unconditional automake snippet for nontests modules
708 snippet = self.getAutomakeSnippet_Conditional()
709 snippet = constants.combine_lines(snippet)
710 pattern = compiler(
711 '^lib_SOURCES[\t ]*\\+=[\t ]*(.*?)$', re.S | re.M)
712 mentioned_files = pattern.findall(snippet)
713 if mentioned_files != list():
714 mentioned_files = mentioned_files[-1].split(' ')
715 mentioned_files = [f.strip() for f in mentioned_files]
716 mentioned_files = [f for f in mentioned_files if f != '']
717 mentioned_files = sorted(set(mentioned_files))
718 all_files = self.getFiles()
719 lib_files = filter_filelist(constants.NL, all_files,
720 'lib/', '', 'lib/', '').split(constants.NL)
721 extra_files = [
722 f for f in lib_files if f not in mentioned_files]
723 extra_files = sorted(set(extra_files))
724 if extra_files != [''] and extra_files:
725 result += string('EXTRA_DIST += %s' %
726 ' '.join(extra_files))
727 result += '\n\n'
728 # Synthesize also an EXTRA_lib_SOURCES augmentation
729 if str(self) != 'relocatable-prog-wrapper' and str(self) != 'pt_chown':
730 extra_files = filter_filelist(constants.NL, extra_files,
731 '', '.c', '', '').split(constants.NL)
732 extra_files = sorted(set(extra_files))
733 if extra_files != ['']:
734 result += string('EXTRA_lib_SOURCES += %s' %
735 ' '.join(extra_files))
736 result += '\n\n'
737 # Synthesize an EXTRA_DIST augmentation also for the files in build-aux
738 buildaux_files = filter_filelist(constants.NL, all_files,
739 'build-aux/', '', 'build-aux/', '').split(constants.NL)
740 buildaux_files = sorted(set(buildaux_files))
741 if buildaux_files != ['']:
742 buildaux_files = ''.join(buildaux_files)
743 buildaux_files = joinpath(
744 '$(top_srcdir)', auxdir, buildaux_files)
745 result += string('EXTRA_DIST += %s' % buildaux_files)
746 result += '\n\n'
747 # Synthesize an EXTRA_DIST augmentation also for the files from top/.
748 top_files = filter_filelist(constants.NL, all_files,
749 'top/', '', 'top/', '').split(constants.NL)
750 top_files = sorted(set(top_files))
751 if top_files != ['']:
752 top_files = ''.join(top_files)
753 top_files = joinpath('$(top_srcdir)', top_files)
754 result += string('EXTRA_DIST += %s' % top_files)
755 result += '\n\n'
756 result = constants.nlconvert(result)
757 self.cache['makefile-unconditional'] = result
758 return(self.cache['makefile-unconditional'])
760 def getInclude(self):
761 '''GLModule.getInclude() -> string
763 Return include directive.'''
764 section = 'Include:'
765 if 'include' not in self.cache:
766 if section not in self.content:
767 result = string()
768 else: # if section in self.content
769 snippet = self.content.split(section)[-1]
770 snippet = snippet.replace('\r\n', '\n')
771 lines = ['%s\n' % line for line in snippet.split('\n')]
772 parts = list()
773 for line in lines:
774 regex = '^(Description|Comment|Status|Notice|Applicability|'
775 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
776 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
777 pattern = compiler(regex)
778 findflag = pattern.findall(line)
779 if findflag:
780 break
781 parts += [line]
782 result = ''.join(parts)
783 result = result.strip()
784 pattern = compiler('^(["<].*?[>"])', re.S | re.M)
785 result = pattern.sub('#include \\1', result)
786 self.cache['include'] = result
787 return(self.cache['include'])
789 def getLink(self):
790 '''GLModule.getLink() -> string
792 Return link directive.'''
793 section = 'Link:'
794 if 'link' not in self.cache:
795 parts = list()
796 if section in self.content:
797 snippet = self.content.split(section)[-1]
798 snippet = snippet.replace('\r\n', '\n')
799 lines = ['%s\n' % line for line in snippet.split('\n')]
800 for line in lines:
801 regex = '^(Description|Comment|Status|Notice|Applicability|'
802 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
803 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
804 pattern = compiler(regex)
805 findflag = pattern.findall(line)
806 if findflag:
807 break
808 parts += [line]
809 parts = [part.strip() for part in parts if part.strip()]
810 # result = ' '.join(parts)
811 self.cache['link'] = parts
812 return(self.cache['link'])
814 def getLicense(self):
815 '''GLModule.getLicense(self) -> string
817 Get license and warn user if module lacks a license.'''
818 license = self.getLicense_Raw()
819 if not self.isTests():
820 if not license:
821 if self.config['errors']:
822 raise(GLError(18, string(self)))
823 else: # if not self.config['errors']
824 sys.stderr.write('gnulib-tool: warning: ')
825 sys.stderr.write('module %s lacks a license\n' % str(self))
826 if not license:
827 license = 'GPL'
828 return(license)
830 def getLicense_Raw(self):
831 '''GLModule.getLicense_Raw() -> string
833 Return module license.'''
834 section = 'License:'
835 if 'license' not in self.cache:
836 if section not in self.content:
837 result = string()
838 else: # if section in self.content
839 pattern = '^%s[\t ]*(.*?)%s' % (section, self.regex)
840 pattern = compiler(pattern, re.S | re.M)
841 result = pattern.findall(self.content)
842 if type(result) is list:
843 if not result:
844 result = string()
845 else: # if result
846 result = result[-1]
847 result = result.strip()
848 self.cache['license'] = result
849 return(self.cache['license'])
851 def getMaintainer(self):
852 '''GLModule.getMaintainer() -> string
854 Return maintainer directive.'''
855 section = 'Maintainer:'
856 if 'maintainer' not in self.cache:
857 if section not in self.content:
858 result = string()
859 else: # if section in self.content
860 snippet = self.content.split(section)[-1]
861 snippet = snippet.replace('\r\n', '\n')
862 lines = ['%s\n' % line for line in snippet.split('\n')]
863 parts = list()
864 for line in lines:
865 regex = '^(Description|Comment|Status|Notice|Applicability|'
866 regex += 'Files|Depends-on|configure\\.ac-early|configure\\.ac|'
867 regex += 'Makefile\\.am|Include|Link|License|Maintainer):$'
868 pattern = compiler(regex)
869 findflag = pattern.findall(line)
870 if findflag:
871 break
872 parts += [line]
873 result = ''.join(parts)
874 result = result.strip()
875 self.cache['maintainer'] = result
876 return(self.cache['maintainer'])
879 #===============================================================================
880 # Define GLModuleTable class
881 #===============================================================================
882 class GLModuleTable(object):
883 '''GLModuleTable is used to work with the list of the modules.'''
885 def __init__(self, config, avoids=list()):
886 '''GLModuleTable.__init__(config, avoids) -> GLModuleTable
888 Create new GLModuleTable instance. If modules are specified, then add
889 every module from iterable as unconditional module. If avoids is specified,
890 then in transitive_closure every dependency which is in avoids won't be
891 included in the final modules list. If testflags iterable is enabled, then
892 don't add module which status is in the testflags. If conddeps are enabled,
893 then store condition for each dependency if it has a condition.
894 The only necessary argument is localdir, which is needed just to create
895 modulesystem instance to look for dependencies.'''
896 self.avoids = list() # Avoids
897 self.dependers = dict() # Dependencies
898 self.conditionals = dict() # Conditional modules
899 self.unconditionals = dict() # Unconditional modules
900 self.base_modules = list() # Base modules
901 self.main_modules = list() # Main modules
902 self.tests_modules = list() # Tests modules
903 self.final_modules = list() # Final modules
904 if type(config) is not GLConfig:
905 raise(TypeError('config must be a GLConfig, not %s' %
906 type(config).__name__))
907 for avoid in avoids:
908 if type(avoid) is not GLModule:
909 raise(TypeError('each avoid must be a GLModule instance'))
910 self.avoids += [avoids]
911 self.config = config
912 self.filesystem = GLFileSystem(self.config)
913 self.modulesystem = GLModuleSystem(self.config)
915 def __repr__(self):
916 '''x.__repr__() <==> repr(x)'''
917 result = '<pygnulib.GLModuleTable %s>' % hex(id(self))
918 return(result)
920 def __getitem__(self, y):
921 '''x.__getitem__(y) <==> x[y]'''
922 if y in ['base', 'final', 'main', 'tests', 'avoids']:
923 if y == 'base':
924 return(self.getBaseModules())
925 elif y == 'final':
926 return(self.getFinalModules())
927 elif y == 'main':
928 return(self.getMainModules())
929 elif y == 'tests':
930 return(self.getTestsModules())
931 else: # if y == 'avoids'
932 return(self.getAvoids())
933 else: # if y is not in list
934 raise(KeyError('GLModuleTable does not contain key: %s' % repr(y)))
936 def addConditional(self, parent, module, condition):
937 '''GLModuleTable.addConditional(module, condition)
939 Add new conditional dependency from parent to module with condition.'''
940 if type(parent) is not GLModule:
941 raise(TypeError('parent must be a GLModule, not %s' %
942 type(parent).__name__))
943 if type(module) is not GLModule:
944 raise(TypeError('module must be a GLModule, not %s' %
945 type(module).__name__))
946 if type(condition) is bytes or type(condition) is string \
947 or condition == True:
948 if type(condition) is bytes:
949 condition = condition.decode(ENCS['default'])
950 else: # if condition has not bytes or string type or is not True
951 raise(TypeError('condition must be a string or True, not %s' %
952 type(condition).__name__))
953 if not str(module) in self.unconditionals:
954 if str(module) not in self.dependers:
955 self.dependers[module] = list()
956 self.dependers[module] += [module]
957 key = '%s---%s' % (str(parent), str(module))
958 self.conditionals[key] = condition
960 def addUnconditional(self, module):
961 '''GLModuleTable.addUnconditional(module)
963 Add module as unconditional dependency.'''
964 if type(module) is not GLModule:
965 raise(TypeError('module must be a GLModule, not %s' %
966 type(module).__name__))
967 if str(module) in self.dependers:
968 self.dependers.pop(str(module))
969 self.unconditionals[str(module)] = True
971 def isConditional(self, module):
972 '''GLModuleTable.isConditional(module) -> bool
974 Check whether module is unconditional.'''
975 if type(module) is not GLModule:
976 raise(TypeError('module must be a GLModule, not %s' %
977 type(module).__name__))
978 result = str(module) in self.dependers
979 return(result)
981 def getCondition(self, parent, module):
982 '''GLModuleTable.getCondition(module) -> string or True
984 Return condition from parent to module. Condition can be string or True.
985 If module is not in the list of conddeps, method returns None.'''
986 if type(parent) is not GLModule:
987 raise(TypeError('parent must be a GLModule, not %s' %
988 type(parent).__name__))
989 if type(module) is not GLModule:
990 raise(TypeError('module must be a GLModule, not %s' %
991 type(module).__name__))
992 key = '%s---%s' % (str(parent), str(module))
993 result = None
994 if key in self.conditionals:
995 result = self.conditionals[key]
996 return(result)
998 def transitive_closure(self, modules):
999 '''GLModuleTable.transitive_closure(modules) -> list
1001 Use transitive closure to add module and its dependencies. Add every
1002 module and its dependencies from modules list, but do not add dependencies
1003 which contain in avoids list. If any testflag is enabled, then do not add
1004 dependencies which have the status as this flag. If conddeps are enabled,
1005 then store condition for each dependency if it has a condition. This method
1006 is used to update final list of modules. Method returns list of modules.
1007 GLConfig: testflags.'''
1008 for module in modules:
1009 if type(module) is not GLModule:
1010 raise(TypeError('each module must be a GLModule instance'))
1011 handledmodules = list()
1012 inmodules = modules
1013 outmodules = list()
1014 if self.config['conddeps']:
1015 for module in modules:
1016 self.addUnconditional(module)
1017 while inmodules:
1018 inmodules_this_round = inmodules
1019 inmodules = list()
1020 for module in inmodules_this_round:
1021 outmodules += [module]
1022 if self.config['conddeps']:
1023 automake_snippet = \
1024 module.getAutomakeSnippet_Conditional()
1025 pattern = compiler('^if')
1026 if not pattern.findall(automake_snippet):
1027 self.addUnconditional(module)
1028 conditional = self.isConditional(module)
1029 dependencies = module.getDependencies()
1030 depmodules = [pair[0] for pair in dependencies]
1031 conditions = [pair[1] for pair in dependencies]
1032 if TESTS['tests'] in self.config['testflags']:
1033 testsname = module.getTestsName()
1034 if self.modulesystem.exists(testsname):
1035 testsmodule = self.modulesystem.find(testsname)
1036 depmodules += [testsmodule]
1037 conditions += [None]
1038 for depmodule in depmodules:
1039 include = True
1040 includes = list()
1041 status = depmodule.getStatus()
1042 for word in status:
1043 if word == 'obsolete':
1044 if TESTS['obsolete'] in self.config['testflags'] or \
1045 TESTS['all-test'] in self.config['testflags']:
1046 includes += [False]
1047 elif word == 'c++-test':
1048 if TESTS['c++-test'] in self.config['testflags'] or \
1049 TESTS['all-test'] in self.config['testflags']:
1050 includes += [False]
1051 elif word == 'longrunning-test':
1052 if TESTS['longrunning-test'] in self.config['testflags'] or \
1053 TESTS['all-test'] in self.config['testflags']:
1054 includes += [False]
1055 elif word == 'privileged-test':
1056 if TESTS['privileged-test'] in self.config['testflags'] or \
1057 TESTS['all-test'] in self.config['testflags']:
1058 includes += [False]
1059 elif word == 'all-test':
1060 if TESTS['all-test'] in self.config['testflags'] or \
1061 TESTS['all-test'] in self.config['testflags']:
1062 includes += [False]
1063 else: # if any other word
1064 if word.endswith('-tests'):
1065 if TESTS['all-test'] in self.config['testflags']:
1066 includes += [False]
1067 include = any(includes)
1068 if include and depmodule not in self.avoids:
1069 inmodules += [depmodule]
1070 if self.config['conddeps']:
1071 index = depmodules.index(depmodule)
1072 condition = conditions[index]
1073 if condition:
1074 self.addConditional(
1075 module, depmodule, condition)
1076 else: # if condition
1077 if conditional:
1078 self.addConditional(
1079 module, depmodule, True)
1080 else: # if not conditional
1081 self.addUnconditional(module)
1082 listing = list() # Create empty list
1083 inmodules = sorted(set(inmodules))
1084 handledmodules = sorted(set(handledmodules + inmodules_this_round))
1085 inmodules = \
1086 [ # Begin to filter inmodules
1087 module for module in inmodules if module not in handledmodules
1088 ] # Finish to filter inmodules
1089 inmodules = sorted(set(inmodules))
1090 modules = sorted(set(outmodules))
1091 self.modules = modules
1092 return(list(modules))
1094 def transitive_closure_separately(self, basemodules, finalmodules):
1095 '''GLModuleTable.transitive_closure_separately(*args, **kwargs) -> tuple
1097 Determine main module list and tests-related module list separately.
1098 The main module list is the transitive closure of the specified modules,
1099 ignoring tests modules. Its lib/* sources go into $sourcebase/. If lgpl is
1100 specified, it will consist only of LGPLed source.
1101 The tests-related module list is the transitive closure of the specified
1102 modules, including tests modules, minus the main module list excluding
1103 modules of applicability 'all'. Its lib/* sources (brought in through
1104 dependencies of *-tests modules) go into $testsbase/. It may contain GPLed
1105 source, even if lgpl is specified.
1106 Arguments are basemodules and finalmodules, where basemodules argument
1107 represents modules specified by user and finalmodules represents modules
1108 list after previous transitive_closure.
1109 Method returns tuple which contains two lists: the list of main modules and
1110 the list of tests-related modules. Both lists contain dependencies.
1111 GLConfig: testflags.'''
1112 inctests = False
1113 main_modules = list()
1114 tests_modules = list()
1115 if TESTS['tests'] in self.config['testflags']:
1116 self.config['testflags'].pop(TESTS['tests'])
1117 inctests = True
1118 for module in basemodules:
1119 if type(module) is not GLModule:
1120 raise(TypeError('each module must be a GLModule instance'))
1121 for module in finalmodules:
1122 if type(module) is not GLModule:
1123 raise(TypeError('each module must be a GLModule instance'))
1124 main_modules = self.transitive_closure(basemodules)
1125 tests_modules = \
1126 [m for m in finalmodules if m not in main_modules] + \
1127 [m for m in main_modules if m.getApplicability() != 'main']
1128 tests_modules = sorted(set(tests_modules))
1129 if inctests:
1130 testflags = sorted(
1131 set(self.config['testflags'] + [TESTS['tests']]))
1132 self.config.setTestFlags(testflags)
1133 result = tuple([main_modules, tests_modules])
1134 return(result)
1136 def add_dummy(self, modules):
1137 '''GLModuleTable.add_dummy(modules) -> list
1139 Add dummy package to list of modules if dummy package is needed. If not,
1140 return original list of modules.
1141 GLConfig: auxdir, ac_version.'''
1142 auxdir = self.config['auxdir']
1143 ac_version = self.config['ac_version']
1144 have_lib_sources = False
1145 for module in modules:
1146 if type(module) is not GLModule:
1147 raise(TypeError('each module must be a GLModule instance'))
1148 snippet = module.getAutomakeSnippet()
1149 snippet = constants.remove_backslash_newline(snippet)
1150 pattern = compiler(
1151 '^lib_SOURCES[\t ]*\\+=[\t ]*(.*?)$', re.S | re.M)
1152 files = pattern.findall(snippet)
1153 if files: # if source files were found
1154 files = files[-1].split(' ')
1155 for file in files:
1156 if not file.endswith('.h'):
1157 have_lib_sources = True
1158 break
1159 if not have_lib_sources:
1160 dummy = self.modulesystem.find('dummy')
1161 modules = sorted(set(modules + [dummy]))
1162 return(list(modules))
1164 def filelist(self, modules):
1165 '''GLModuleTable.filelist(modules) -> list
1167 Determine the final file list for the given list of modules. The list of
1168 modules must already include dependencies.
1169 GLConfig: ac_version.'''
1170 ac_version = self.config['ac_version']
1171 filelist = list()
1172 for module in modules:
1173 if type(module) is not GLModule:
1174 raise(TypeError('each module must be a GLModule instance'))
1175 listings = [module.getFiles() for module in modules]
1176 for listing in listings:
1177 for file in listing:
1178 if file not in filelist:
1179 filelist += [file]
1180 return(filelist)
1182 def filelist_separately(self, main_modules, tests_modules):
1183 '''GLModuleTable.filelist_separately(**kwargs) -> list
1185 Determine the final file lists. They must be computed separately, because
1186 files in lib/* go into $sourcebase/ if they are in the main file list but
1187 into $testsbase/ if they are in the tests-related file list. Furthermore
1188 lib/dummy.c can be in both.'''
1189 ac_version = self.config['ac_version']
1190 main_filelist = self.filelist(main_modules)
1191 tests_filelist = self.filelist(tests_modules)
1192 tests_filelist = \
1193 [ # Begin to sort filelist
1194 file.replace('lib/', 'tests=lib/', 1) \
1195 if file.startswith('lib/') else file
1196 for file in tests_filelist
1197 ] # Finish to sort filelist
1198 result = tuple([main_filelist, tests_filelist])
1199 return(result)
1201 def getAvoids(self):
1202 '''GLModuleTable.getAvoids() -> list
1204 Return list of avoids.'''
1205 return(list(self.avoids))
1207 def setAvoids(self, modules):
1208 '''GLModuleTable.setAvoids(modules)
1210 Specify list of avoids.'''
1211 for module in modules:
1212 if type(module) is not GLModule:
1213 raise(TypeError('each module must be a GLModule instance'))
1214 self.avoids = sorted(set(modules))
1216 def getBaseModules(self):
1217 '''GLModuleTable.getBaseModules() -> list
1219 Return list of base modules.'''
1220 return(list(self.base_modules))
1222 def setBaseModules(self, modules):
1223 '''GLModuleTable.setBaseModules(modules)
1225 Specify list of base modules.'''
1226 for module in modules:
1227 if type(module) is not GLModule:
1228 raise(TypeError('each module must be a GLModule instance'))
1229 self.base_modules = sorted(set(modules))
1231 def getFinalModules(self):
1232 '''GLModuleTable.getFinalModules() -> list
1234 Return list of final modules.'''
1235 return(list(self.final_modules))
1237 def setFinalModules(self, modules):
1238 '''GLModuleTable.setFinalModules(modules)
1240 Specify list of final modules.'''
1241 for module in modules:
1242 if type(module) is not GLModule:
1243 raise(TypeError('each module must be a GLModule instance'))
1244 self.final_modules = sorted(set(modules))
1246 def getMainModules(self):
1247 '''GLModuleTable.getMainModules() -> list
1249 Return list of main modules.'''
1250 return(list(self.main_modules))
1252 def setMainModules(self, modules):
1253 '''GLModuleTable.setMainModules(modules)
1255 Specify list of main modules.'''
1256 for module in modules:
1257 if type(module) is not GLModule:
1258 raise(TypeError('each module must be a GLModule instance'))
1259 self.main_modules = sorted(set(modules))
1261 def getTestsModules(self):
1262 '''GLModuleTable.getTestsModules() -> list
1264 Return list of tests modules.'''
1265 return(list(self.tests_modules))
1267 def setTestsModules(self, modules):
1268 '''GLModuleTable.setTestsModules(modules)
1270 Specify list of tests modules.'''
1271 for module in modules:
1272 if type(module) is not GLModule:
1273 raise(TypeError('each module must be a GLModule instance'))
1274 self.tests_modules = sorted(set(modules))