3 # Thomas Nagy, 2005-2018 (ita)
6 Node: filesystem structure
8 #. Each file/folder is represented by exactly one node.
10 #. Some potential class properties are stored on :py:class:`waflib.Build.BuildContext` : nodes to depend on, etc.
11 Unused class members can increase the `.wafpickle` file size sensibly.
13 #. Node objects should never be created directly, use
14 the methods :py:func:`Node.make_node` or :py:func:`Node.find_node` for the low-level operations
16 #. The methods :py:func:`Node.find_resource`, :py:func:`Node.find_dir` :py:func:`Node.find_or_declare` must be
17 used when a build context is present
19 #. Each instance of :py:class:`waflib.Context.Context` has a unique :py:class:`Node` subclass required for serialization.
20 (:py:class:`waflib.Node.Nod3`, see the :py:class:`waflib.Context.Context` initializer). A reference to the context
21 owning a node is held as *self.ctx*
24 import os
, re
, sys
, shutil
25 from waflib
import Utils
, Errors
60 Ant patterns for files and folders to exclude while doing the
61 recursive traversal in :py:meth:`waflib.Node.Node.ant_glob`
64 def ant_matcher(s
, ignorecase
):
65 reflags
= re
.I
if ignorecase
else 0
67 for x
in Utils
.to_list(s
):
68 x
= x
.replace('\\', '/').replace('//', '/')
72 for k
in x
.split('/'):
76 k
= k
.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+')
79 exp
= re
.compile(k
, flags
=reflags
)
80 except Exception as e
:
81 raise Errors
.WafError('Invalid pattern: %s' % k
, e
)
87 def ant_sub_filter(name
, nn
):
95 if lst
[1].match(name
):
99 elif lst
[0].match(name
):
103 def ant_sub_matcher(name
, pats
):
104 nacc
= ant_sub_filter(name
, pats
[0])
105 nrej
= ant_sub_filter(name
, pats
[1])
112 This class is organized in two parts:
114 * The basic methods meant for filesystem access (compute paths, create folders, etc)
115 * The methods bound to a :py:class:`waflib.Build.BuildContext` (require ``bld.srcnode`` and ``bld.bldnode``)
120 Subclasses can provide a dict class to enable case insensitivity for example.
123 __slots__
= ('name', 'parent', 'children', 'cache_abspath', 'cache_isdir')
124 def __init__(self
, name
, parent
):
126 .. note:: Use :py:func:`Node.make_node` or :py:func:`Node.find_node` instead of calling this constructor
131 if name
in parent
.children
:
132 raise Errors
.WafError('node %s exists in the parent files %r already' % (name
, parent
))
133 parent
.children
[name
] = self
135 def __setstate__(self
, data
):
136 "Deserializes node information, used for persistence"
138 self
.parent
= data
[1]
139 if data
[2] is not None:
141 self
.children
= self
.dict_class(data
[2])
143 def __getstate__(self
):
144 "Serializes node information, used for persistence"
145 return (self
.name
, self
.parent
, getattr(self
, 'children', None))
149 String representation (abspath), for debugging purposes
153 return self
.abspath()
157 String representation (abspath), for debugging purposes
161 return self
.abspath()
165 Provided to prevent nodes from being copied
167 :raises: :py:class:`waflib.Errors.WafError`
169 raise Errors
.WafError('nodes are not supposed to be copied')
171 def read(self
, flags
='r', encoding
='latin-1'):
173 Reads and returns the contents of the file represented by this node, see :py:func:`waflib.Utils.readf`::
176 bld.path.find_node('wscript').read()
178 :param flags: Open mode
180 :param encoding: encoding value for Python3
181 :type encoding: string
182 :rtype: string or bytes
183 :return: File contents
185 return Utils
.readf(self
.abspath(), flags
, encoding
)
187 def write(self
, data
, flags
='w', encoding
='latin-1'):
189 Writes data to the file represented by this node, see :py:func:`waflib.Utils.writef`::
192 bld.path.make_node('foo.txt').write('Hello, world!')
194 :param data: data to write
196 :param flags: Write mode
198 :param encoding: encoding value for Python3
199 :type encoding: string
201 Utils
.writef(self
.abspath(), data
, flags
, encoding
)
203 def read_json(self
, convert
=True, encoding
='utf-8'):
205 Reads and parses the contents of this node as JSON (Python ≥ 2.6)::
208 bld.path.find_node('abc.json').read_json()
210 Note that this by default automatically decodes unicode strings on Python2, unlike what the Python JSON module does.
212 :type convert: boolean
213 :param convert: Prevents decoding of unicode strings on Python2
214 :type encoding: string
215 :param encoding: The encoding of the file to read. This default to UTF8 as per the JSON standard
217 :return: Parsed file contents
219 import json
# Python 2.6 and up
220 object_pairs_hook
= None
221 if convert
and sys
.hexversion
< 0x3000000:
228 if isinstance(value
, list):
229 return [convert(element
) for element
in value
]
230 elif isinstance(value
, _type
):
235 def object_pairs(pairs
):
236 return dict((str(pair
[0]), convert(pair
[1])) for pair
in pairs
)
238 object_pairs_hook
= object_pairs
240 return json
.loads(self
.read(encoding
=encoding
), object_pairs_hook
=object_pairs_hook
)
242 def write_json(self
, data
, pretty
=True):
244 Writes a python object as JSON to disk (Python ≥ 2.6) as UTF-8 data (JSON standard)::
247 bld.path.find_node('xyz.json').write_json(199)
250 :param data: The data to write to disk
251 :type pretty: boolean
252 :param pretty: Determines if the JSON will be nicely space separated
254 import json
# Python 2.6 and up
256 separators
= (',', ': ')
261 separators
= (',', ':')
263 output
= json
.dumps(data
, indent
=indent
, separators
=separators
, sort_keys
=sort_keys
) + newline
264 self
.write(output
, encoding
='utf-8')
268 Returns whether the Node is present on the filesystem
272 return os
.path
.exists(self
.abspath())
276 Returns whether the Node represents a folder
280 return os
.path
.isdir(self
.abspath())
282 def chmod(self
, val
):
284 Changes the file/dir permissions::
287 bld.path.chmod(493) # 0755
289 os
.chmod(self
.abspath(), val
)
291 def delete(self
, evict
=True):
293 Removes the file/folder from the filesystem (equivalent to `rm -rf`), and remove this object from the Node tree.
294 Do not use this object after calling this method.
298 if os
.path
.isdir(self
.abspath()):
299 shutil
.rmtree(self
.abspath())
301 os
.remove(self
.abspath())
303 if os
.path
.exists(self
.abspath()):
311 Removes this node from the Node tree
313 del self
.parent
.children
[self
.name
]
317 Returns the file rightmost extension, for example `a.b.c.d → .d`
321 k
= max(0, self
.name
.rfind('.'))
326 Returns the depth in the folder hierarchy from the filesystem root or from all the file drives
328 :returns: filesystem depth
340 Lists the folder contents
342 :returns: list of file/folder names ordered alphabetically
343 :rtype: list of string
345 lst
= Utils
.listdir(self
.abspath())
351 Creates a folder represented by this node. Intermediate folders are created as needed.
353 :raises: :py:class:`waflib.Errors.WafError` when the folder is missing
365 os
.makedirs(self
.abspath())
370 raise Errors
.WafError('Could not create the directory %r' % self
)
374 except AttributeError:
375 self
.children
= self
.dict_class()
377 def find_node(self
, lst
):
379 Finds a node on the file system (files or folders), and creates the corresponding Node objects if it exists
381 :param lst: relative path
382 :type lst: string or list of string
383 :returns: The corresponding Node object or None if no entry was found on the filesystem
384 :rtype: :py:class:´waflib.Node.Node´
387 if isinstance(lst
, str):
388 lst
= [x
for x
in Utils
.split_path(lst
) if x
and x
!= '.']
390 if lst
and lst
[0].startswith('\\\\') and not self
.parent
:
391 node
= self
.ctx
.root
.make_node(lst
[0])
392 node
.cache_isdir
= True
393 return node
.find_node(lst
[1:])
398 cur
= cur
.parent
or cur
403 except AttributeError:
404 cur
.children
= self
.dict_class()
412 # optimistic: create the node first then look if it was correct to do so
413 cur
= self
.__class
__(x
, cur
)
424 def make_node(self
, lst
):
426 Returns or creates a Node object corresponding to the input path without considering the filesystem.
428 :param lst: relative path
429 :type lst: string or list of string
430 :rtype: :py:class:´waflib.Node.Node´
432 if isinstance(lst
, str):
433 lst
= [x
for x
in Utils
.split_path(lst
) if x
and x
!= '.']
438 cur
= cur
.parent
or cur
442 cur
= cur
.children
[x
]
443 except AttributeError:
444 cur
.children
= self
.dict_class()
449 cur
= self
.__class
__(x
, cur
)
452 def search_node(self
, lst
):
454 Returns a Node previously defined in the data structure. The filesystem is not considered.
456 :param lst: relative path
457 :type lst: string or list of string
458 :rtype: :py:class:´waflib.Node.Node´ or None if there is no entry in the Node datastructure
460 if isinstance(lst
, str):
461 lst
= [x
for x
in Utils
.split_path(lst
) if x
and x
!= '.']
466 cur
= cur
.parent
or cur
469 cur
= cur
.children
[x
]
470 except (AttributeError, KeyError):
474 def path_from(self
, node
):
476 Path of this node seen from the other::
479 n1 = bld.path.find_node('foo/bar/xyz.txt')
480 n2 = bld.path.find_node('foo/stuff/')
481 n1.path_from(n2) # '../bar/xyz.txt'
483 :param node: path to use as a reference
484 :type node: :py:class:`waflib.Node.Node`
485 :returns: a relative path or an absolute one if that is better
515 lst
.extend(['..'] * up
)
517 return os
.sep
.join(lst
) or '.'
519 return self
.abspath()
523 Returns the absolute path. A cache is kept in the context as ``cache_node_abspath``
528 return self
.cache_abspath
529 except AttributeError:
531 # think twice before touching this (performance + complexity + correctness)
535 elif not self
.parent
.name
:
536 val
= os
.sep
+ self
.name
538 val
= self
.parent
.abspath() + os
.sep
+ self
.name
539 self
.cache_abspath
= val
545 return self
.cache_abspath
546 except AttributeError:
550 elif not self
.parent
.name
:
551 val
= self
.name
+ os
.sep
553 val
= self
.parent
.abspath().rstrip(os
.sep
) + os
.sep
+ self
.name
554 self
.cache_abspath
= val
557 def is_child_of(self
, node
):
559 Returns whether the object belongs to a subtree of the input node::
562 node = bld.path.find_node('wscript')
563 node.is_child_of(bld.path) # True
565 :param node: path to use as a reference
566 :type node: :py:class:`waflib.Node.Node`
570 diff
= self
.height() - node
.height()
576 def ant_iter(self
, accept
=None, maxdepth
=25, pats
=[], dir=False, src
=True, remove
=True, quiet
=False):
578 Recursive method used by :py:meth:`waflib.Node.ant_glob`.
580 :param accept: function used for accepting/rejecting a node, returns the patterns that can be still accepted in recursion
581 :type accept: function
582 :param maxdepth: maximum depth in the filesystem (25)
584 :param pats: list of patterns to accept and list of patterns to exclude
586 :param dir: return folders too (False by default)
588 :param src: return files (True by default)
590 :param remove: remove files/folders that do not exist (True by default)
592 :param quiet: disable build directory traversal warnings (verbose mode)
594 :returns: A generator object to iterate from
597 dircont
= self
.listdir()
601 lst
= set(self
.children
.keys())
602 except AttributeError:
603 self
.children
= self
.dict_class()
606 for x
in lst
- set(dircont
):
607 self
.children
[x
].evict()
610 npats
= accept(name
, pats
)
611 if npats
and npats
[0]:
612 accepted
= [] in npats
[0]
614 node
= self
.make_node([name
])
625 node
.cache_isdir
= True
627 for k
in node
.ant_iter(accept
=accept
, maxdepth
=maxdepth
- 1, pats
=npats
, dir=dir, src
=src
, remove
=remove
, quiet
=quiet
):
630 def ant_glob(self
, *k
, **kw
):
632 Finds files across folders and returns Node objects:
634 * ``**/*`` find all files recursively
635 * ``**/*.class`` find all files ending by .class
636 * ``..`` find files having two dot characters
641 # find all .cpp files
642 cfg.path.ant_glob('**/*.cpp')
643 # find particular files from the root filesystem (can be slow)
644 cfg.root.ant_glob('etc/*.txt')
645 # simple exclusion rule example
646 cfg.path.ant_glob('*.c*', excl=['*.c'], src=True, dir=False)
648 For more information about the patterns, consult http://ant.apache.org/manual/dirtasks.html
649 Please remember that the '..' sequence does not represent the parent directory::
652 cfg.path.ant_glob('../*.h') # incorrect
653 cfg.path.parent.ant_glob('*.h') # correct
655 The Node structure is itself a filesystem cache, so certain precautions must
656 be taken while matching files in the build or installation phases.
657 Nodes objects that do have a corresponding file or folder are garbage-collected by default.
658 This garbage collection is usually required to prevent returning files that do not
659 exist anymore. Yet, this may also remove Node objects of files that are yet-to-be built.
661 This typically happens when trying to match files in the build directory,
662 but there are also cases when files are created in the source directory.
663 Run ``waf -v`` to display any warnings, and try consider passing ``remove=False``
664 when matching files in the build directory.
666 Since ant_glob can traverse both source and build folders, it is a best practice
667 to call this method only from the most specific build node::
670 # traverses the build directory, may need ``remove=False``:
671 bld.path.ant_glob('project/dir/**/*.h')
672 # better, no accidental build directory traversal:
673 bld.path.find_node('project/dir').ant_glob('**/*.h') # best
675 In addition, files and folders are listed immediately. When matching files in the
676 build folders, consider passing ``generator=True`` so that the generator object
677 returned can defer computation to a later stage. For example::
680 bld(rule='tar xvf ${SRC}', source='arch.tar')
682 gen = bld.bldnode.ant_glob("*.h", generator=True, remove=True)
683 # files will be listed only after the arch.tar is unpacked
684 bld(rule='ls ${SRC}', source=gen, name='XYZ')
687 :param incl: ant patterns or list of patterns to include
688 :type incl: string or list of strings
689 :param excl: ant patterns or list of patterns to exclude
690 :type excl: string or list of strings
691 :param dir: return folders too (False by default)
693 :param src: return files (True by default)
695 :param maxdepth: maximum depth of recursion
697 :param ignorecase: ignore case while matching (False by default)
698 :type ignorecase: bool
699 :param generator: Whether to evaluate the Nodes lazily
700 :type generator: bool
701 :param remove: remove files/folders that do not exist (True by default)
703 :param quiet: disable build directory traversal warnings (verbose mode)
705 :returns: The corresponding Node objects as a list or as a generator object (generator=True)
706 :rtype: by default, list of :py:class:`waflib.Node.Node` instances
708 src
= kw
.get('src', True)
710 excl
= kw
.get('excl', exclude_regs
)
711 incl
= k
and k
[0] or kw
.get('incl', '**')
712 remove
= kw
.get('remove', True)
713 maxdepth
= kw
.get('maxdepth', 25)
714 ignorecase
= kw
.get('ignorecase', False)
715 quiet
= kw
.get('quiet', False)
716 pats
= (ant_matcher(incl
, ignorecase
), ant_matcher(excl
, ignorecase
))
718 if kw
.get('generator'):
719 return Utils
.lazy_generator(self
.ant_iter
, (ant_sub_matcher
, maxdepth
, pats
, dir, src
, remove
, quiet
))
721 it
= self
.ant_iter(ant_sub_matcher
, maxdepth
, pats
, dir, src
, remove
, quiet
)
723 # returns relative paths as a space-delimited string
724 # prefer Node objects whenever possible
725 return ' '.join(x
.path_from(self
) for x
in it
)
728 # ----------------------------------------------------------------------------
729 # the methods below require the source/build folders (bld.srcnode/bld.bldnode)
733 Returns True if the node is below the source directory. Note that ``!is_src() ≠ is_bld()``
750 Returns True if the node is below the build directory. Note that ``!is_bld() ≠ is_src()``
764 Returns the corresponding Node object in the source directory (or self if already
765 under the source directory). Use this method only if the purpose is to create
766 a Node object (this is common with folders but not with files, see ticket 1937)
768 :rtype: :py:class:`waflib.Node.Node`
777 return x
.make_node(lst
)
786 Return the corresponding Node object in the build directory (or self if already
787 under the build directory). Use this method only if the purpose is to create
788 a Node object (this is common with folders but not with files, see ticket 1937)
790 :rtype: :py:class:`waflib.Node.Node`
801 return self
.ctx
.bldnode
.make_node(lst
)
804 # the file is external to the current project, make a fake root in the current build directory
806 if lst
and Utils
.is_win32
and len(lst
[0]) == 2 and lst
[0].endswith(':'):
808 return self
.ctx
.bldnode
.make_node(['__root__'] + lst
)
810 def find_resource(self
, lst
):
812 Use this method in the build phase to find source files corresponding to the relative path given.
814 First it looks up the Node data structure to find any declared Node object in the build directory.
815 If None is found, it then considers the filesystem in the source directory.
817 :param lst: relative path
818 :type lst: string or list of string
819 :returns: the corresponding Node object or None
820 :rtype: :py:class:`waflib.Node.Node`
822 if isinstance(lst
, str):
823 lst
= [x
for x
in Utils
.split_path(lst
) if x
and x
!= '.']
825 node
= self
.get_bld().search_node(lst
)
827 node
= self
.get_src().find_node(lst
)
828 if node
and node
.isdir():
832 def find_or_declare(self
, lst
):
834 Use this method in the build phase to declare output files which
835 are meant to be written in the build directory.
837 This method creates the Node object and its parent folder
840 :param lst: relative path
841 :type lst: string or list of string
843 if isinstance(lst
, str) and os
.path
.isabs(lst
):
844 node
= self
.ctx
.root
.make_node(lst
)
846 node
= self
.get_bld().make_node(lst
)
850 def find_dir(self
, lst
):
852 Searches for a folder on the filesystem (see :py:meth:`waflib.Node.Node.find_node`)
854 :param lst: relative path
855 :type lst: string or list of string
856 :returns: The corresponding Node object or None if there is no such folder
857 :rtype: :py:class:`waflib.Node.Node`
859 if isinstance(lst
, str):
860 lst
= [x
for x
in Utils
.split_path(lst
) if x
and x
!= '.']
862 node
= self
.find_node(lst
)
863 if node
and not node
.isdir():
867 # helpers for building things
868 def change_ext(self
, ext
, ext_in
=None):
870 Declares a build node with a distinct extension; this is uses :py:meth:`waflib.Node.Node.find_or_declare`
872 :return: A build node of the same path, but with a different extension
873 :rtype: :py:class:`waflib.Node.Node`
879 name
= name
[:k
] + ext
883 name
= name
[:- len(ext_in
)] + ext
885 return self
.parent
.find_or_declare([name
])
889 Returns the relative path seen from the build directory ``src/foo.cpp``
893 return self
.path_from(self
.ctx
.bldnode
)
897 Returns the relative path seen from the source directory ``../src/foo.cpp``
901 return self
.path_from(self
.ctx
.srcnode
)
905 If a file in the build directory, returns :py:meth:`waflib.Node.Node.bldpath`,
906 else returns :py:meth:`waflib.Node.Node.srcpath`
914 return self
.bldpath()
916 return self
.srcpath()
920 Equivalent to self.parent.bldpath()
924 return self
.parent
.bldpath()
928 See :py:func:`waflib.Utils.h_file`
930 :return: a hash representing the file contents
931 :rtype: string or bytes
933 return Utils
.h_file(self
.abspath())
935 def get_bld_sig(self
):
937 Returns a signature (see :py:meth:`waflib.Node.Node.h_file`) for the purpose
938 of build dependency calculation. This method uses a per-context cache.
940 :return: a hash representing the object contents
941 :rtype: string or bytes
943 # previous behaviour can be set by returning self.ctx.node_sigs[self] when a build node
945 cache
= self
.ctx
.cache_sig
946 except AttributeError:
947 cache
= self
.ctx
.cache_sig
= {}
953 ret
= cache
[self
] = self
.h_file()
954 except EnvironmentError:
956 # allow folders as build nodes, do not use the creation time
958 ret
= cache
[self
] = Utils
.h_list([p
, st
.st_ino
, st
.st_mode
])
963 pickle_lock
= Utils
.threading
.Lock()
964 """Lock mandatory for thread-safe node serialization"""
967 """Mandatory subclass for thread-safe node serialization"""