third_party/waf/waflib/extras/win32_opts.py

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3
   4 """
   5 Windows-specific optimizations
   6
   7 This module can help reducing the overhead of listing files on windows
   8 (more than 10000 files). Python 3.5 already provides the listdir
   9 optimization though.
  10 """
  11
  12 import os
  13 from waflib import Utils, Build, Node, Logs
  14
  15 try:
  16         TP = '%s\\*'.decode('ascii')
  17 except AttributeError:
  18         TP = '%s\\*'
  19
  20 if Utils.is_win32:
  21         from waflib.Tools import md5_tstamp
  22         import ctypes, ctypes.wintypes
  23
  24         FindFirstFile        = ctypes.windll.kernel32.FindFirstFileW
  25         FindNextFile         = ctypes.windll.kernel32.FindNextFileW
  26         FindClose            = ctypes.windll.kernel32.FindClose
  27         FILE_ATTRIBUTE_DIRECTORY = 0x10
  28         INVALID_HANDLE_VALUE = -1
  29         UPPER_FOLDERS = ('.', '..')
  30         try:
  31                 UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS]
  32         except NameError:
  33                 pass
  34
  35         def cached_hash_file(self):
  36                 try:
  37                         cache = self.ctx.cache_listdir_cache_hash_file
  38                 except AttributeError:
  39                         cache = self.ctx.cache_listdir_cache_hash_file = {}
  40
  41                 if id(self.parent) in cache:
  42                         try:
  43                                 t = cache[id(self.parent)][self.name]
  44                         except KeyError:
  45                                 raise IOError('Not a file')
  46                 else:
  47                         # an opportunity to list the files and the timestamps at once
  48                         findData = ctypes.wintypes.WIN32_FIND_DATAW()
  49                         find     = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData))
  50
  51                         if find == INVALID_HANDLE_VALUE:
  52                                 cache[id(self.parent)] = {}
  53                                 raise IOError('Not a file')
  54
  55                         cache[id(self.parent)] = lst_files = {}
  56                         try:
  57                                 while True:
  58                                         if findData.cFileName not in UPPER_FOLDERS:
  59                                                 thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
  60                                                 if not thatsadir:
  61                                                         ts = findData.ftLastWriteTime
  62                                                         d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime
  63                                                         lst_files[str(findData.cFileName)] = d
  64                                         if not FindNextFile(find, ctypes.byref(findData)):
  65                                                 break
  66                         except Exception:
  67                                 cache[id(self.parent)] = {}
  68                                 raise IOError('Not a file')
  69                         finally:
  70                                 FindClose(find)
  71                         t = lst_files[self.name]
  72
  73                 fname = self.abspath()
  74                 if fname in Build.hashes_md5_tstamp:
  75                         if Build.hashes_md5_tstamp[fname][0] == t:
  76                                 return Build.hashes_md5_tstamp[fname][1]
  77
  78                 try:
  79                         fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT)
  80                 except OSError:
  81                         raise IOError('Cannot read from %r' % fname)
  82                 f = os.fdopen(fd, 'rb')
  83                 m = Utils.md5()
  84                 rb = 1
  85                 try:
  86                         while rb:
  87                                 rb = f.read(200000)
  88                                 m.update(rb)
  89                 finally:
  90                         f.close()
  91
  92                 # ensure that the cache is overwritten
  93                 Build.hashes_md5_tstamp[fname] = (t, m.digest())
  94                 return m.digest()
  95         Node.Node.cached_hash_file = cached_hash_file
  96
  97         def get_bld_sig_win32(self):
  98                 try:
  99                         return self.ctx.hash_cache[id(self)]
 100                 except KeyError:
 101                         pass
 102                 except AttributeError:
 103                         self.ctx.hash_cache = {}
 104                 self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath())
 105                 return ret
 106         Node.Node.get_bld_sig = get_bld_sig_win32
 107
 108         def isfile_cached(self):
 109                 # optimize for nt.stat calls, assuming there are many files for few folders
 110                 try:
 111                         cache = self.__class__.cache_isfile_cache
 112                 except AttributeError:
 113                         cache = self.__class__.cache_isfile_cache = {}
 114
 115                 try:
 116                         c1 = cache[id(self.parent)]
 117                 except KeyError:
 118                         c1 = cache[id(self.parent)] = []
 119
 120                         curpath = self.parent.abspath()
 121                         findData = ctypes.wintypes.WIN32_FIND_DATAW()
 122                         find     = FindFirstFile(TP % curpath, ctypes.byref(findData))
 123
 124                         if find == INVALID_HANDLE_VALUE:
 125                                 Logs.error("invalid win32 handle isfile_cached %r", self.abspath())
 126                                 return os.path.isfile(self.abspath())
 127
 128                         try:
 129                                 while True:
 130                                         if findData.cFileName not in UPPER_FOLDERS:
 131                                                 thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
 132                                                 if not thatsadir:
 133                                                         c1.append(str(findData.cFileName))
 134                                         if not FindNextFile(find, ctypes.byref(findData)):
 135                                                 break
 136                         except Exception as e:
 137                                 Logs.error('exception while listing a folder %r %r', self.abspath(), e)
 138                                 return os.path.isfile(self.abspath())
 139                         finally:
 140                                 FindClose(find)
 141                 return self.name in c1
 142         Node.Node.isfile_cached = isfile_cached
 143
 144         def find_or_declare_win32(self, lst):
 145                 # assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile
 146                 if isinstance(lst, str):
 147                         lst = [x for x in Utils.split_path(lst) if x and x != '.']
 148
 149                 node = self.get_bld().search_node(lst)
 150                 if node:
 151                         if not node.isfile_cached():
 152                                 try:
 153                                         node.parent.mkdir()
 154                                 except OSError:
 155                                         pass
 156                         return node
 157                 self = self.get_src()
 158                 node = self.find_node(lst)
 159                 if node:
 160                         if not node.isfile_cached():
 161                                 try:
 162                                         node.parent.mkdir()
 163                                 except OSError:
 164                                         pass
 165                         return node
 166                 node = self.get_bld().make_node(lst)
 167                 node.parent.mkdir()
 168                 return node
 169         Node.Node.find_or_declare = find_or_declare_win32
 170