Include waf as an extracted source directory, rather than as a one-in-a-file script.
[samba.git] / buildtools / wafadmin / Node.py
1 #!/usr/bin/env python
2 # encoding: utf-8
3 # Thomas Nagy, 2005 (ita)
4
5 """
6 Node: filesystem structure, contains lists of nodes
7
8 IMPORTANT:
9 1. Each file/folder is represented by exactly one node.
10
11 2. Most would-be class properties are stored in Build: nodes to depend on, signature, flags, ..
12 unused class members increase the .wafpickle file size sensibly with lots of objects.
13
14 3. The build is launched from the top of the build dir (for example, in _build_/).
15
16 4. Node should not be instantiated directly.
17 Each instance of Build.BuildContext has a Node subclass.
18 (aka: 'Nodu', see BuildContext initializer)
19 The BuildContext is referenced here as self.__class__.bld
20 Its Node class is referenced here as self.__class__
21
22 The public and advertised apis are the following:
23 ${TGT}                 -> dir/to/file.ext
24 ${TGT[0].base()}       -> dir/to/file
25 ${TGT[0].dir(env)}     -> dir/to
26 ${TGT[0].file()}       -> file.ext
27 ${TGT[0].file_base()}   -> file
28 ${TGT[0].suffix()}     -> .ext
29 ${TGT[0].abspath(env)} -> /path/to/dir/to/file.ext
30
31 """
32
33 import os, sys, fnmatch, re, stat
34 import Utils, Constants
35
36 UNDEFINED = 0
37 DIR = 1
38 FILE = 2
39 BUILD = 3
40
41 type_to_string = {UNDEFINED: "unk", DIR: "dir", FILE: "src", BUILD: "bld"}
42
43 # These fnmatch expressions are used by default to prune the directory tree
44 # while doing the recursive traversal in the find_iter method of the Node class.
45 prune_pats = '.git .bzr .hg .svn _MTN _darcs CVS SCCS'.split()
46
47 # These fnmatch expressions are used by default to exclude files and dirs
48 # while doing the recursive traversal in the find_iter method of the Node class.
49 exclude_pats = prune_pats + '*~ #*# .#* %*% ._* .gitignore .cvsignore vssver.scc .DS_Store'.split()
50
51 # These Utils.jar_regexp expressions are used by default to exclude files and dirs and also prune the directory tree
52 # while doing the recursive traversal in the ant_glob method of the Node class.
53 exclude_regs = '''
54 **/*~
55 **/#*#
56 **/.#*
57 **/%*%
58 **/._*
59 **/CVS
60 **/CVS/**
61 **/.cvsignore
62 **/SCCS
63 **/SCCS/**
64 **/vssver.scc
65 **/.svn
66 **/.svn/**
67 **/.git
68 **/.git/**
69 **/.gitignore
70 **/.bzr
71 **/.bzr/**
72 **/.hg
73 **/.hg/**
74 **/_MTN
75 **/_MTN/**
76 **/_darcs
77 **/_darcs/**
78 **/.DS_Store'''
79
80 class Node(object):
81         __slots__ = ("name", "parent", "id", "childs")
82         def __init__(self, name, parent, node_type = UNDEFINED):
83                 self.name = name
84                 self.parent = parent
85
86                 # assumption: one build object at a time
87                 self.__class__.bld.id_nodes += 4
88                 self.id = self.__class__.bld.id_nodes + node_type
89
90                 if node_type == DIR: self.childs = {}
91
92                 # We do not want to add another type attribute (memory)
93                 # use the id to find out: type = id & 3
94                 # for setting: new type = type + x - type & 3
95
96                 if parent and name in parent.childs:
97                         raise Utils.WafError('node %s exists in the parent files %r already' % (name, parent))
98
99                 if parent: parent.childs[name] = self
100
101         def __setstate__(self, data):
102                 if len(data) == 4:
103                         (self.parent, self.name, self.id, self.childs) = data
104                 else:
105                         (self.parent, self.name, self.id) = data
106
107         def __getstate__(self):
108                 if getattr(self, 'childs', None) is None:
109                         return (self.parent, self.name, self.id)
110                 else:
111                         return (self.parent, self.name, self.id, self.childs)
112
113         def __str__(self):
114                 if not self.parent: return ''
115                 return "%s://%s" % (type_to_string[self.id & 3], self.abspath())
116
117         def __repr__(self):
118                 return self.__str__()
119
120         def __hash__(self):
121                 "expensive, make certain it is not used"
122                 raise Utils.WafError('nodes, you are doing it wrong')
123
124         def __copy__(self):
125                 "nodes are not supposed to be copied"
126                 raise Utils.WafError('nodes are not supposed to be cloned')
127
128         def get_type(self):
129                 return self.id & 3
130
131         def set_type(self, t):
132                 "dangerous, you are not supposed to use this"
133                 self.id = self.id + t - self.id & 3
134
135         def dirs(self):
136                 return [x for x in self.childs.values() if x.id & 3 == DIR]
137
138         def files(self):
139                 return [x for x in self.childs.values() if x.id & 3 == FILE]
140
141         def get_dir(self, name, default=None):
142                 node = self.childs.get(name, None)
143                 if not node or node.id & 3 != DIR: return default
144                 return  node
145
146         def get_file(self, name, default=None):
147                 node = self.childs.get(name, None)
148                 if not node or node.id & 3 != FILE: return default
149                 return node
150
151         def get_build(self, name, default=None):
152                 node = self.childs.get(name, None)
153                 if not node or node.id & 3 != BUILD: return default
154                 return node
155
156         def find_resource(self, lst):
157                 "Find an existing input file: either a build node declared previously or a source node"
158                 if isinstance(lst, str):
159                         lst = Utils.split_path(lst)
160
161                 if len(lst) == 1:
162                         parent = self
163                 else:
164                         parent = self.find_dir(lst[:-1])
165                         if not parent: return None
166                 self.__class__.bld.rescan(parent)
167
168                 name = lst[-1]
169                 node = parent.childs.get(name, None)
170                 if node:
171                         tp = node.id & 3
172                         if tp == FILE or tp == BUILD:
173                                 return node
174                         else:
175                                 return None
176
177                 tree = self.__class__.bld
178                 if not name in tree.cache_dir_contents[parent.id]:
179                         return None
180
181                 path = parent.abspath() + os.sep + name
182                 try:
183                         st = Utils.h_file(path)
184                 except IOError:
185                         return None
186
187                 child = self.__class__(name, parent, FILE)
188                 tree.node_sigs[0][child.id] = st
189                 return child
190
191         def find_or_declare(self, lst):
192                 "Used for declaring a build node representing a file being built"
193                 if isinstance(lst, str):
194                         lst = Utils.split_path(lst)
195
196                 if len(lst) == 1:
197                         parent = self
198                 else:
199                         parent = self.find_dir(lst[:-1])
200                         if not parent: return None
201                 self.__class__.bld.rescan(parent)
202
203                 name = lst[-1]
204                 node = parent.childs.get(name, None)
205                 if node:
206                         tp = node.id & 3
207                         if tp != BUILD:
208                                 raise Utils.WafError('find_or_declare found a source file where a build file was expected %r' % '/'.join(lst))
209                         return node
210                 node = self.__class__(name, parent, BUILD)
211                 return node
212
213         def find_dir(self, lst):
214                 "search a folder in the filesystem"
215
216                 if isinstance(lst, str):
217                         lst = Utils.split_path(lst)
218
219                 current = self
220                 for name in lst:
221                         self.__class__.bld.rescan(current)
222                         prev = current
223
224                         if not current.parent and name == current.name:
225                                 continue
226                         elif not name:
227                                 continue
228                         elif name == '.':
229                                 continue
230                         elif name == '..':
231                                 current = current.parent or current
232                         else:
233                                 current = prev.childs.get(name, None)
234                                 if current is None:
235                                         dir_cont = self.__class__.bld.cache_dir_contents
236                                         if prev.id in dir_cont and name in dir_cont[prev.id]:
237                                                 if not prev.name:
238                                                         if os.sep == '/':
239                                                                 # cygwin //machine/share
240                                                                 dirname = os.sep + name
241                                                         else:
242                                                                 # windows c:
243                                                                 dirname = name
244                                                 else:
245                                                         # regular path
246                                                         dirname = prev.abspath() + os.sep + name
247                                                 if not os.path.isdir(dirname):
248                                                         return None
249                                                 current = self.__class__(name, prev, DIR)
250                                         elif (not prev.name and len(name) == 2 and name[1] == ':') or name.startswith('\\\\'):
251                                                 # drive letter or \\ path for windows
252                                                 current = self.__class__(name, prev, DIR)
253                                         else:
254                                                 return None
255                                 else:
256                                         if current.id & 3 != DIR:
257                                                 return None
258                 return current
259
260         def ensure_dir_node_from_path(self, lst):
261                 "used very rarely, force the construction of a branch of node instance for representing folders"
262
263                 if isinstance(lst, str):
264                         lst = Utils.split_path(lst)
265
266                 current = self
267                 for name in lst:
268                         if not name:
269                                 continue
270                         elif name == '.':
271                                 continue
272                         elif name == '..':
273                                 current = current.parent or current
274                         else:
275                                 prev = current
276                                 current = prev.childs.get(name, None)
277                                 if current is None:
278                                         current = self.__class__(name, prev, DIR)
279                 return current
280
281         def exclusive_build_node(self, path):
282                 """
283                 create a hierarchy in the build dir (no source folders) for ill-behaving compilers
284                 the node is not hashed, so you must do it manually
285
286                 after declaring such a node, find_dir and find_resource should work as expected
287                 """
288                 lst = Utils.split_path(path)
289                 name = lst[-1]
290                 if len(lst) > 1:
291                         parent = None
292                         try:
293                                 parent = self.find_dir(lst[:-1])
294                         except OSError:
295                                 pass
296                         if not parent:
297                                 parent = self.ensure_dir_node_from_path(lst[:-1])
298                                 self.__class__.bld.rescan(parent)
299                         else:
300                                 try:
301                                         self.__class__.bld.rescan(parent)
302                                 except OSError:
303                                         pass
304                 else:
305                         parent = self
306
307                 node = parent.childs.get(name, None)
308                 if not node:
309                         node = self.__class__(name, parent, BUILD)
310
311                 return node
312
313         def path_to_parent(self, parent):
314                 "path relative to a direct ancestor, as string"
315                 lst = []
316                 p = self
317                 h1 = parent.height()
318                 h2 = p.height()
319                 while h2 > h1:
320                         h2 -= 1
321                         lst.append(p.name)
322                         p = p.parent
323                 if lst:
324                         lst.reverse()
325                         ret = os.path.join(*lst)
326                 else:
327                         ret = ''
328                 return ret
329
330         def find_ancestor(self, node):
331                 "find a common ancestor for two nodes - for the shortest path in hierarchy"
332                 dist = self.height() - node.height()
333                 if dist < 0: return node.find_ancestor(self)
334                 # now the real code
335                 cand = self
336                 while dist > 0:
337                         cand = cand.parent
338                         dist -= 1
339                 if cand == node: return cand
340                 cursor = node
341                 while cand.parent:
342                         cand = cand.parent
343                         cursor = cursor.parent
344                         if cand == cursor: return cand
345
346         def relpath_gen(self, from_node):
347                 "string representing a relative path between self to another node"
348
349                 if self == from_node: return '.'
350                 if from_node.parent == self: return '..'
351
352                 # up_path is '../../../' and down_path is 'dir/subdir/subdir/file'
353                 ancestor = self.find_ancestor(from_node)
354                 lst = []
355                 cand = self
356                 while not cand.id == ancestor.id:
357                         lst.append(cand.name)
358                         cand = cand.parent
359                 cand = from_node
360                 while not cand.id == ancestor.id:
361                         lst.append('..')
362                         cand = cand.parent
363                 lst.reverse()
364                 return os.sep.join(lst)
365
366         def nice_path(self, env=None):
367                 "printed in the console, open files easily from the launch directory"
368                 tree = self.__class__.bld
369                 ln = tree.launch_node()
370
371                 if self.id & 3 == FILE: return self.relpath_gen(ln)
372                 else: return os.path.join(tree.bldnode.relpath_gen(ln), env.variant(), self.relpath_gen(tree.srcnode))
373
374         def is_child_of(self, node):
375                 "does this node belong to the subtree node"
376                 p = self
377                 diff = self.height() - node.height()
378                 while diff > 0:
379                         diff -= 1
380                         p = p.parent
381                 return p.id == node.id
382
383         def variant(self, env):
384                 "variant, or output directory for this node, a source has for variant 0"
385                 if not env: return 0
386                 elif self.id & 3 == FILE: return 0
387                 else: return env.variant()
388
389         def height(self):
390                 "amount of parents"
391                 # README a cache can be added here if necessary
392                 d = self
393                 val = -1
394                 while d:
395                         d = d.parent
396                         val += 1
397                 return val
398
399         # helpers for building things
400
401         def abspath(self, env=None):
402                 """
403                 absolute path
404                 @param env [Environment]:
405                         * obligatory for build nodes: build/variant/src/dir/bar.o
406                         * optional for dirs: get either src/dir or build/variant/src/dir
407                         * excluded for source nodes: src/dir/bar.c
408
409                 Instead of computing the absolute path each time again,
410                 store the already-computed absolute paths in one of (variants+1) dictionaries:
411                 bld.cache_node_abspath[0] holds absolute paths for source nodes.
412                 bld.cache_node_abspath[variant] holds the absolute path for the build nodes
413                 which reside in the variant given by env.
414                 """
415                 ## absolute path - hot zone, so do not touch
416
417                 # less expensive
418                 variant = (env and (self.id & 3 != FILE) and env.variant()) or 0
419
420                 ret = self.__class__.bld.cache_node_abspath[variant].get(self.id, None)
421                 if ret: return ret
422
423                 if not variant:
424                         # source directory
425                         if not self.parent:
426                                 val = os.sep == '/' and os.sep or ''
427                         elif not self.parent.name: # root
428                                 val = (os.sep == '/' and os.sep or '') + self.name
429                         else:
430                                 val = self.parent.abspath() + os.sep + self.name
431                 else:
432                         # build directory
433                         val = os.sep.join((self.__class__.bld.bldnode.abspath(), variant, self.path_to_parent(self.__class__.bld.srcnode)))
434                 self.__class__.bld.cache_node_abspath[variant][self.id] = val
435                 return val
436
437         def change_ext(self, ext):
438                 "node of the same path, but with a different extension - hot zone so do not touch"
439                 name = self.name
440                 k = name.rfind('.')
441                 if k >= 0:
442                         name = name[:k] + ext
443                 else:
444                         name = name + ext
445
446                 return self.parent.find_or_declare([name])
447
448         def src_dir(self, env):
449                 "src path without the file name"
450                 return self.parent.srcpath(env)
451
452         def bld_dir(self, env):
453                 "build path without the file name"
454                 return self.parent.bldpath(env)
455
456         def bld_base(self, env):
457                 "build path without the extension: src/dir/foo(.cpp)"
458                 s = os.path.splitext(self.name)[0]
459                 return os.path.join(self.bld_dir(env), s)
460
461         def bldpath(self, env=None):
462                 "path seen from the build dir default/src/foo.cpp"
463                 if self.id & 3 == FILE:
464                         return self.relpath_gen(self.__class__.bld.bldnode)
465                 p = self.path_to_parent(self.__class__.bld.srcnode)
466                 if p is not '':
467                         return env.variant() + os.sep + p
468                 return env.variant()
469
470         def srcpath(self, env=None):
471                 "path in the srcdir from the build dir ../src/foo.cpp"
472                 if self.id & 3 == BUILD:
473                         return self.bldpath(env)
474                 return self.relpath_gen(self.__class__.bld.bldnode)
475
476         def read(self, env):
477                 "get the contents of a file, it is not used anywhere for the moment"
478                 return Utils.readf(self.abspath(env))
479
480         def dir(self, env):
481                 "scons-like"
482                 return self.parent.abspath(env)
483
484         def file(self):
485                 "scons-like"
486                 return self.name
487
488         def file_base(self):
489                 "scons-like"
490                 return os.path.splitext(self.name)[0]
491
492         def suffix(self):
493                 "scons-like - hot zone so do not touch"
494                 k = max(0, self.name.rfind('.'))
495                 return self.name[k:]
496
497         def find_iter_impl(self, src=True, bld=True, dir=True, accept_name=None, is_prune=None, maxdepth=25):
498                 """find nodes in the filesystem hierarchy, try to instanciate the nodes passively; same gotcha as ant_glob"""
499                 bld_ctx = self.__class__.bld
500                 bld_ctx.rescan(self)
501                 for name in bld_ctx.cache_dir_contents[self.id]:
502                         if accept_name(self, name):
503                                 node = self.find_resource(name)
504                                 if node:
505                                         if src and node.id & 3 == FILE:
506                                                 yield node
507                                 else:
508                                         node = self.find_dir(name)
509                                         if node and node.id != bld_ctx.bldnode.id:
510                                                 if dir:
511                                                         yield node
512                                                 if not is_prune(self, name):
513                                                         if maxdepth:
514                                                                 for k in node.find_iter_impl(src, bld, dir, accept_name, is_prune, maxdepth=maxdepth - 1):
515                                                                         yield k
516                         else:
517                                 if not is_prune(self, name):
518                                         node = self.find_resource(name)
519                                         if not node:
520                                                 # not a file, it is a dir
521                                                 node = self.find_dir(name)
522                                                 if node and node.id != bld_ctx.bldnode.id:
523                                                         if maxdepth:
524                                                                 for k in node.find_iter_impl(src, bld, dir, accept_name, is_prune, maxdepth=maxdepth - 1):
525                                                                         yield k
526
527                 if bld:
528                         for node in self.childs.values():
529                                 if node.id == bld_ctx.bldnode.id:
530                                         continue
531                                 if node.id & 3 == BUILD:
532                                         if accept_name(self, node.name):
533                                                 yield node
534                 raise StopIteration
535
536         def find_iter(self, in_pat=['*'], ex_pat=exclude_pats, prune_pat=prune_pats, src=True, bld=True, dir=False, maxdepth=25, flat=False):
537                 """find nodes recursively, this returns everything but folders by default; same gotcha as ant_glob"""
538
539                 if not (src or bld or dir):
540                         raise StopIteration
541
542                 if self.id & 3 != DIR:
543                         raise StopIteration
544
545                 in_pat = Utils.to_list(in_pat)
546                 ex_pat = Utils.to_list(ex_pat)
547                 prune_pat = Utils.to_list(prune_pat)
548
549                 def accept_name(node, name):
550                         for pat in ex_pat:
551                                 if fnmatch.fnmatchcase(name, pat):
552                                         return False
553                         for pat in in_pat:
554                                 if fnmatch.fnmatchcase(name, pat):
555                                         return True
556                         return False
557
558                 def is_prune(node, name):
559                         for pat in prune_pat:
560                                 if fnmatch.fnmatchcase(name, pat):
561                                         return True
562                         return False
563
564                 ret = self.find_iter_impl(src, bld, dir, accept_name, is_prune, maxdepth=maxdepth)
565                 if flat:
566                         return " ".join([x.relpath_gen(self) for x in ret])
567
568                 return ret
569
570         def ant_glob(self, *k, **kw):
571                 """
572                 known gotcha: will enumerate the files, but only if the folder exists in the source directory
573                 """
574
575                 src=kw.get('src', 1)
576                 bld=kw.get('bld', 0)
577                 dir=kw.get('dir', 0)
578                 excl = kw.get('excl', exclude_regs)
579                 incl = k and k[0] or kw.get('incl', '**')
580
581                 def to_pat(s):
582                         lst = Utils.to_list(s)
583                         ret = []
584                         for x in lst:
585                                 x = x.replace('//', '/')
586                                 if x.endswith('/'):
587                                         x += '**'
588                                 lst2 = x.split('/')
589                                 accu = []
590                                 for k in lst2:
591                                         if k == '**':
592                                                 accu.append(k)
593                                         else:
594                                                 k = k.replace('.', '[.]').replace('*', '.*').replace('?', '.')
595                                                 k = '^%s$' % k
596                                                 #print "pattern", k
597                                                 accu.append(re.compile(k))
598                                 ret.append(accu)
599                         return ret
600
601                 def filtre(name, nn):
602                         ret = []
603                         for lst in nn:
604                                 if not lst:
605                                         pass
606                                 elif lst[0] == '**':
607                                         ret.append(lst)
608                                         if len(lst) > 1:
609                                                 if lst[1].match(name):
610                                                         ret.append(lst[2:])
611                                         else:
612                                                 ret.append([])
613                                 elif lst[0].match(name):
614                                         ret.append(lst[1:])
615                         return ret
616
617                 def accept(name, pats):
618                         nacc = filtre(name, pats[0])
619                         nrej = filtre(name, pats[1])
620                         if [] in nrej:
621                                 nacc = []
622                         return [nacc, nrej]
623
624                 def ant_iter(nodi, maxdepth=25, pats=[]):
625                         nodi.__class__.bld.rescan(nodi)
626                         tmp = list(nodi.__class__.bld.cache_dir_contents[nodi.id])
627                         tmp.sort()
628                         for name in tmp:
629                                 npats = accept(name, pats)
630                                 if npats and npats[0]:
631                                         accepted = [] in npats[0]
632                                         #print accepted, nodi, name
633
634                                         node = nodi.find_resource(name)
635                                         if node and accepted:
636                                                 if src and node.id & 3 == FILE:
637                                                         yield node
638                                         else:
639                                                 node = nodi.find_dir(name)
640                                                 if node and node.id != nodi.__class__.bld.bldnode.id:
641                                                         if accepted and dir:
642                                                                 yield node
643                                                         if maxdepth:
644                                                                 for k in ant_iter(node, maxdepth=maxdepth - 1, pats=npats):
645                                                                         yield k
646                         if bld:
647                                 for node in nodi.childs.values():
648                                         if node.id == nodi.__class__.bld.bldnode.id:
649                                                 continue
650                                         if node.id & 3 == BUILD:
651                                                 npats = accept(node.name, pats)
652                                                 if npats and npats[0] and [] in npats[0]:
653                                                         yield node
654                         raise StopIteration
655
656                 ret = [x for x in ant_iter(self, pats=[to_pat(incl), to_pat(excl)])]
657
658                 if kw.get('flat', True):
659                         return " ".join([x.relpath_gen(self) for x in ret])
660
661                 return ret
662
663         def update_build_dir(self, env=None):
664
665                 if not env:
666                         for env in bld.all_envs:
667                                 self.update_build_dir(env)
668                         return
669
670                 path = self.abspath(env)
671
672                 lst = Utils.listdir(path)
673                 try:
674                         self.__class__.bld.cache_dir_contents[self.id].update(lst)
675                 except KeyError:
676                         self.__class__.bld.cache_dir_contents[self.id] = set(lst)
677                 self.__class__.bld.cache_scanned_folders[self.id] = True
678
679                 for k in lst:
680                         npath = path + os.sep + k
681                         st = os.stat(npath)
682                         if stat.S_ISREG(st[stat.ST_MODE]):
683                                 ick = self.find_or_declare(k)
684                                 if not (ick.id in self.__class__.bld.node_sigs[env.variant()]):
685                                         self.__class__.bld.node_sigs[env.variant()][ick.id] = Constants.SIG_NIL
686                         elif stat.S_ISDIR(st[stat.ST_MODE]):
687                                 child = self.find_dir(k)
688                                 if not child:
689                                         child = self.ensure_dir_node_from_path(k)
690                                 child.update_build_dir(env)
691
692
693 class Nodu(Node):
694         pass
695