3 # Thomas Nagy, 2005 (ita)
6 Utilities, the stable ones are the following:
8 * h_file: compute a unique value for a file (hash), it uses
9 the module fnv if it is installed (see waf/utils/fnv & http://code.google.com/p/waf/wiki/FAQ)
10 else, md5 (see the python docs)
12 For large projects (projects with more than 15000 files) or slow hard disks and filesystems (HFS)
13 it is possible to use a hashing based on the path and the size (may give broken cache results)
14 The method h_file MUST raise an OSError if the file is a folder
18 st = os.lstat(filename)
19 if stat.S_ISDIR(st[stat.ST_MODE]): raise IOError('not a file')
21 m.update(str(st.st_mtime))
22 m.update(str(st.st_size))
26 To replace the function in your project, use something like this:
37 import os, sys, imp, string, errno, traceback, inspect, re, shutil, datetime, gc
39 # In python 3.0 we can get rid of all this
40 try: from UserDict import UserDict
41 except ImportError: from collections import UserDict
42 if sys.hexversion >= 0x2060000 or os.name == 'java':
43 import subprocess as pproc
47 from Constants import *
50 from collections import deque
56 is_win32 = sys.platform == 'win32'
59 # defaultdict in python 2.5
60 from collections import defaultdict as DefaultDict
62 class DefaultDict(dict):
63 def __init__(self, default_factory):
64 super(DefaultDict, self).__init__()
65 self.default_factory = default_factory
66 def __getitem__(self, key):
68 return super(DefaultDict, self).__getitem__(key)
70 value = self.default_factory()
74 class WafError(Exception):
75 def __init__(self, *args):
78 self.stack = traceback.extract_stack()
81 Exception.__init__(self, *args)
83 return str(len(self.args) == 1 and self.args[0] or self.args)
85 class WscriptError(WafError):
86 def __init__(self, message, wscript_file=None):
88 self.wscript_file = wscript_file
89 self.wscript_line = None
92 (self.wscript_file, self.wscript_line) = self.locate_error()
94 (self.wscript_file, self.wscript_line) = (None, None)
98 msg_file_line = "%s:" % self.wscript_file
100 msg_file_line += "%s:" % self.wscript_line
101 err_message = "%s error: %s" % (msg_file_line, message)
102 WafError.__init__(self, err_message)
104 def locate_error(self):
105 stack = traceback.extract_stack()
108 file_name = os.path.basename(frame[0])
109 is_wscript = (file_name == WSCRIPT_FILE or file_name == WSCRIPT_BUILD_FILE)
111 return (frame[0], frame[1])
114 indicator = is_win32 and '\x1b[A\x1b[K%s%s%s\r' or '\x1b[K%s%s%s\r'
117 from fnv import new as md5
119 Constants.SIG_NIL = 'signofnv'
121 def h_file(filename):
126 if x is None: raise OSError("not a file")
129 raise OSError("not a file" + filename)
134 from hashlib import md5
138 def h_file(filename):
139 f = open(filename, 'rb')
142 filename = f.read(100000)
147 # portability fixes may be added elsewhere (although, md5 should be everywhere by now)
150 def readf(fname, m='r', encoding='ISO8859-1'):
151 """backported from waf 1.8"""
152 if sys.hexversion > 0x3000000 and not 'b' in m:
160 txt = txt.decode(encoding)
171 def writef(fname, data, m='w', encoding='ISO8859-1'):
172 """backported from waf 1.8"""
173 if sys.hexversion > 0x3000000 and not 'b' in m:
174 data = data.encode(encoding)
182 class ordered_dict(UserDict):
183 def __init__(self, dict = None):
185 UserDict.__init__(self, dict)
187 def __delitem__(self, key):
188 self.allkeys.remove(key)
189 UserDict.__delitem__(self, key)
191 def __setitem__(self, key, item):
192 if key not in self.allkeys: self.allkeys.append(key)
193 UserDict.__setitem__(self, key, item)
195 def exec_command(s, **kw):
197 kw['stdout'] = kw['stderr'] = kw['log']
199 kw['shell'] = isinstance(s, str)
202 proc = pproc.Popen(s, **kw)
208 def exec_command(s, **kw):
210 kw['stdout'] = kw['stderr'] = kw['log']
212 kw['shell'] = isinstance(s, str)
215 startupinfo = pproc.STARTUPINFO()
216 startupinfo.dwFlags |= pproc.STARTF_USESHOWWINDOW
217 kw['startupinfo'] = startupinfo
220 if 'stdout' not in kw:
221 kw['stdout'] = pproc.PIPE
222 kw['stderr'] = pproc.PIPE
223 kw['universal_newlines'] = True
224 proc = pproc.Popen(s,**kw)
225 (stdout, stderr) = proc.communicate()
229 return proc.returncode
231 proc = pproc.Popen(s,**kw)
238 def listdir_win32(s):
239 if re.match('^[A-Za-z]:$', s):
240 # os.path.isdir fails if s contains only the drive name... (x:)
242 if not os.path.isdir(s):
244 e.errno = errno.ENOENT
247 listdir = listdir_win32
249 def waf_version(mini = 0x010000, maxi = 0x100000):
250 "Halts if the waf version is wrong"
252 try: min_val = mini + 0
253 except TypeError: min_val = int(mini.replace('.', '0'), 16)
256 Logs.error("waf version should be at least %s (%s found)" % (mini, ver))
259 try: max_val = maxi + 0
260 except TypeError: max_val = int(maxi.replace('.', '0'), 16)
263 Logs.error("waf version should be at most %s (%s found)" % (maxi, ver))
266 def python_24_guard():
267 if sys.hexversion < 0x20400f0 or sys.hexversion >= 0x3000000:
268 raise ImportError("Waf requires Python >= 2.3 but the raw source requires Python 2.4, 2.5 or 2.6")
271 exc_type, exc_value, tb = sys.exc_info()
273 exc_lines = traceback.format_exception(exc_type, exc_value, tb)
274 return ''.join(exc_lines)
275 return str(exc_value)
278 if isinstance(sth, str):
283 g_loaded_modules = {}
284 "index modules by absolute path"
287 "the main module is special"
289 def load_module(file_path, name=WSCRIPT_FILE):
290 "this function requires an absolute path"
292 return g_loaded_modules[file_path]
296 module = imp.new_module(name)
299 code = readf(file_path, m='rU')
300 except (IOError, OSError):
301 raise WscriptError('Could not read the file %r' % file_path)
303 module.waf_hash_val = code
305 dt = os.path.dirname(file_path)
306 sys.path.insert(0, dt)
308 exec(compile(code, file_path, 'exec'), module.__dict__)
310 exc_type, exc_value, tb = sys.exc_info()
311 raise WscriptError("".join(traceback.format_exception(exc_type, exc_value, tb)), file_path)
314 g_loaded_modules[file_path] = module
318 def set_main_module(file_path):
319 "Load custom options, if defined"
321 g_module = load_module(file_path, 'wscript_main')
322 g_module.root_path = file_path
327 g_module.APPNAME = 'noname'
331 g_module.VERSION = '1.0'
333 # note: to register the module globally, use the following:
334 # sys.modules['wscript_main'] = g_module
337 "used for importing env files"
341 if not line: continue
342 mems = line.split('=')
343 tbl[mems[0]] = mems[1]
350 import struct, fcntl, termios
356 dummy_lines, cols = struct.unpack("HHHH", \
357 fcntl.ioctl(sys.stderr.fileno(),termios.TIOCGWINSZ , \
358 struct.pack("HHHH", 0, 0, 0, 0)))[:2]
360 # we actually try the function once to see if it is suitable
366 get_term_cols = myfun
369 rot_chr = ['\\', '|', '/', '-']
370 "the rotation character in the progress bar"
373 def split_path(path):
374 return path.split('/')
376 def split_path_cygwin(path):
377 if path.startswith('//'):
378 ret = path.split('/')[2:]
379 ret[0] = '/' + ret[0]
381 return path.split('/')
383 re_sp = re.compile('[/\\\\]')
384 def split_path_win32(path):
385 if path.startswith('\\\\'):
386 ret = re.split(re_sp, path)[2:]
387 ret[0] = '\\' + ret[0]
389 return re.split(re_sp, path)
391 if sys.platform == 'cygwin':
392 split_path = split_path_cygwin
394 split_path = split_path_win32
396 def copy_attrs(orig, dest, names, only_if_set=False):
397 for a in to_list(names):
398 u = getattr(orig, a, ())
399 if u or not only_if_set:
402 def def_attrs(cls, **kw):
404 set attributes for class.
405 @param cls [any class]: the class to update the given attributes in.
406 @param kw [dictionary]: dictionary of attributes names and values.
408 if the given class hasn't one (or more) of these attributes, add the attribute with its value to the class.
410 for k, v in kw.iteritems():
411 if not hasattr(cls, k):
414 def quote_define_name(path):
415 fu = re.compile("[^a-zA-Z0-9]").sub("_", path)
419 def quote_whitespace(path):
420 return (path.strip().find(' ') > 0 and '"%s"' % path or path).replace('""', '"')
425 if s[0] == "'" and s[-1] == "'": return s[1:-1]
436 except AttributeError:
438 h = inspect.getsource(fun)
443 except AttributeError:
447 def pprint(col, str, label='', sep='\n'):
448 "print messages in color"
449 sys.stderr.write("%s%s%s %s%s" % (Logs.colors(col), str, Logs.colors.NORMAL, label, sep))
452 """If a folder doesn't exists, create it."""
453 if not os.path.isdir(path):
457 if not os.path.isdir(path):
458 raise WafError("Cannot create the folder '%s' (error: %s)" % (path, e))
460 def cmd_output(cmd, **kw):
464 silent = kw['silent']
472 kw['shell'] = isinstance(cmd, str)
473 kw['stdout'] = pproc.PIPE
475 kw['stderr'] = pproc.PIPE
478 p = pproc.Popen(cmd, **kw)
479 output = p.communicate()[0]
481 raise ValueError(str(e))
485 msg = "command execution failed: %s -> %r" % (cmd, str(output))
486 raise ValueError(msg)
490 reg_subst = re.compile(r"(\\\\)|(\$\$)|\$\{([^}]+)\}")
491 def subst_vars(expr, params):
492 "substitute ${PREFIX}/bin in /usr/local/bin"
499 # environments may contain lists
500 return params.get_flat(m.group(3))
501 except AttributeError:
502 return params[m.group(3)]
503 return reg_subst.sub(repl_var, expr)
505 def unversioned_sys_platform_to_binary_format(unversioned_sys_platform):
506 "infers the binary format from the unversioned_sys_platform name."
508 if unversioned_sys_platform in ('linux', 'freebsd', 'netbsd', 'openbsd', 'sunos', 'gnu'):
510 elif unversioned_sys_platform == 'darwin':
512 elif unversioned_sys_platform in ('win32', 'cygwin', 'uwin', 'msys'):
514 # TODO we assume all other operating systems are elf, which is not true.
515 # we may set this to 'unknown' and have ccroot and other tools handle the case "gracefully" (whatever that means).
518 def unversioned_sys_platform():
519 """returns an unversioned name from sys.platform.
520 sys.plaform is not very well defined and depends directly on the python source tree.
521 The version appended to the names is unreliable as it's taken from the build environment at the time python was built,
522 i.e., it's possible to get freebsd7 on a freebsd8 system.
523 So we remove the version from the name, except for special cases where the os has a stupid name like os2 or win32.
524 Some possible values of sys.platform are, amongst others:
525 aix3 aix4 atheos beos5 darwin freebsd2 freebsd3 freebsd4 freebsd5 freebsd6 freebsd7
526 generic gnu0 irix5 irix6 linux2 mac netbsd1 next3 os2emx riscos sunos5 unixware7
527 Investigating the python source tree may reveal more values.
531 # The real OS is hidden under the JVM.
532 from java.lang import System
533 s = System.getProperty('os.name')
534 # see http://lopica.sourceforge.net/os.html for a list of possible values
537 elif s.startswith('Windows '):
543 elif s in ('SunOS', 'Solaris'):
546 if s == 'win32' or s.endswith('os2') and s != 'sunos2': return s
547 return re.split('\d+$', s)[0]
549 #@deprecated('use unversioned_sys_platform instead')
550 def detect_platform():
551 """this function has been in the Utils module for some time.
552 It's hard to guess what people have used it for.
553 It seems its goal is to return an unversionned sys.platform, but it's not handling all platforms.
554 For example, the version is not removed on freebsd and netbsd, amongst others.
559 for x in 'cygwin linux irix sunos hpux aix darwin gnu'.split():
560 # sys.platform may be linux2
565 if os.name in 'posix java os2'.split():
570 def load_tool(tool, tooldir=None):
572 load_tool: import a Python module, optionally using several directories.
573 @param tool [string]: name of tool to import.
574 @param tooldir [list]: directories to look for the tool.
575 @return: the loaded module.
577 Warning: this function is not thread-safe: plays with sys.path,
578 so must run in sequence.
581 assert isinstance(tooldir, list)
582 sys.path = tooldir + sys.path
586 return __import__(tool)
592 """A function that does nothing"""
595 def diff_path(top, subdir):
596 """difference between two absolute paths"""
597 top = os.path.normpath(top).replace('\\', '/').split('/')
598 subdir = os.path.normpath(subdir).replace('\\', '/').split('/')
599 if len(top) == len(subdir): return ''
600 diff = subdir[len(top) - len(subdir):]
601 return os.path.join(*diff)
603 class Context(object):
604 """A base class for commands to be executed from Waf scripts"""
606 def set_curdir(self, dir):
609 def get_curdir(self):
612 except AttributeError:
613 self.curdir_ = os.getcwd()
614 return self.get_curdir()
616 curdir = property(get_curdir, set_curdir)
618 def recurse(self, dirs, name=''):
619 """The function for calling scripts from folders, it tries to call wscript + function_name
620 and if that file does not exist, it will call the method 'function_name' from a file named wscript
621 the dirs can be a list of folders or a string containing space-separated folder paths
624 name = inspect.stack()[1][3]
626 if isinstance(dirs, str):
633 nexdir = os.path.join(self.curdir, x)
635 base = os.path.join(nexdir, WSCRIPT_FILE)
636 file_path = base + '_' + name
639 txt = readf(file_path, m='rU')
640 except (OSError, IOError):
642 module = load_module(base)
644 raise WscriptError('No such script %s' % base)
647 f = module.__dict__[name]
649 raise WscriptError('No function %s defined in %s' % (name, base))
651 if getattr(self.__class__, 'pre_recurse', None):
652 self.pre_recurse(f, base, nexdir)
659 if getattr(self.__class__, 'post_recurse', None):
660 self.post_recurse(module, base, nexdir)
663 if getattr(self.__class__, 'pre_recurse', None):
664 dc = self.pre_recurse(txt, file_path, nexdir)
669 exec(compile(txt, file_path, 'exec'), dc)
671 exc_type, exc_value, tb = sys.exc_info()
672 raise WscriptError("".join(traceback.format_exception(exc_type, exc_value, tb)), base)
675 if getattr(self.__class__, 'post_recurse', None):
676 self.post_recurse(txt, file_path, nexdir)
682 shutil.copystat(src, src)
683 setattr(shutil, 'copy2', copy2)
685 def zip_folder(dir, zip_file_name, prefix):
687 prefix represents the app to add in the archive
690 zip = zipfile.ZipFile(zip_file_name, 'w', compression=zipfile.ZIP_DEFLATED)
691 base = os.path.abspath(dir)
694 if prefix[-1] != os.sep:
698 for root, dirs, files in os.walk(base):
700 archive_name = prefix + root[n:] + os.sep + f
701 zip.write(root + os.sep + f, archive_name, zipfile.ZIP_DEFLATED)
704 def get_elapsed_time(start):
705 "Format a time delta (datetime.timedelta) using the format DdHhMmS.MSs"
706 delta = datetime.datetime.now() - start
707 # cast to int necessary for python 3.0
708 days = int(delta.days)
709 hours = int(delta.seconds / 3600)
710 minutes = int((delta.seconds - hours * 3600) / 60)
711 seconds = delta.seconds - hours * 3600 - minutes * 60 \
712 + float(delta.microseconds) / 1000 / 1000
715 result += '%dd' % days
717 result += '%dh' % hours
718 if days or hours or minutes:
719 result += '%dm' % minutes
720 return '%s%.3fs' % (result, seconds)
722 if os.name == 'java':
723 # For Jython (they should really fix the inconsistency)
727 except NotImplementedError:
728 gc.disable = gc.enable
732 decorator, make a function cache its results, use like this:
746 wrap.__cache__ = cache