third_party/waf/waflib/extras/distnet.py

   1 #! /usr/bin/env python
   2 # encoding: utf-8
   3 # WARNING! Do not edit! https://waf.io/book/index.html#_obtaining_the_waf_file
   4
   5 #! /usr/bin/env python
   6 # encoding: utf-8
   7
   8 """
   9 waf-powered distributed network builds, with a network cache.
  10
  11 Caching files from a server has advantages over a NFS/Samba shared folder:
  12
  13 - builds are much faster because they use local files
  14 - builds just continue to work in case of a network glitch
  15 - permissions are much simpler to manage
  16 """
  17
  18 import os, urllib, tarfile, re, shutil, tempfile, sys
  19 from collections import OrderedDict
  20 from waflib import Context, Utils, Logs
  21
  22 try:
  23         from urllib.parse import urlencode
  24 except ImportError:
  25         urlencode = urllib.urlencode
  26
  27 def safe_urlencode(data):
  28         x = urlencode(data)
  29         try:
  30                 x = x.encode('utf-8')
  31         except Exception:
  32                 pass
  33         return x
  34
  35 try:
  36         from urllib.error import URLError
  37 except ImportError:
  38         from urllib2 import URLError
  39
  40 try:
  41         from urllib.request import Request, urlopen
  42 except ImportError:
  43         from urllib2 import Request, urlopen
  44
  45 DISTNETCACHE = os.environ.get('DISTNETCACHE', '/tmp/distnetcache')
  46 DISTNETSERVER = os.environ.get('DISTNETSERVER', 'http://localhost:8000/cgi-bin/')
  47 TARFORMAT = 'w:bz2'
  48 TIMEOUT = 60
  49 REQUIRES = 'requires.txt'
  50
  51 re_com = re.compile('\s*#.*', re.M)
  52
  53 def total_version_order(num):
  54         lst = num.split('.')
  55         template = '%10s' * len(lst)
  56         ret = template % tuple(lst)
  57         return ret
  58
  59 def get_distnet_cache():
  60         return getattr(Context.g_module, 'DISTNETCACHE', DISTNETCACHE)
  61
  62 def get_server_url():
  63         return getattr(Context.g_module, 'DISTNETSERVER', DISTNETSERVER)
  64
  65 def get_download_url():
  66         return '%s/download.py' % get_server_url()
  67
  68 def get_upload_url():
  69         return '%s/upload.py' % get_server_url()
  70
  71 def get_resolve_url():
  72         return '%s/resolve.py' % get_server_url()
  73
  74 def send_package_name():
  75         out = getattr(Context.g_module, 'out', 'build')
  76         pkgfile = '%s/package_to_upload.tarfile' % out
  77         return pkgfile
  78
  79 class package(Context.Context):
  80         fun = 'package'
  81         cmd = 'package'
  82
  83         def execute(self):
  84                 try:
  85                         files = self.files
  86                 except AttributeError:
  87                         files = self.files = []
  88
  89                 Context.Context.execute(self)
  90                 pkgfile = send_package_name()
  91                 if not pkgfile in files:
  92                         if not REQUIRES in files:
  93                                 files.append(REQUIRES)
  94                         self.make_tarfile(pkgfile, files, add_to_package=False)
  95
  96         def make_tarfile(self, filename, files, **kw):
  97                 if kw.get('add_to_package', True):
  98                         self.files.append(filename)
  99
 100                 with tarfile.open(filename, TARFORMAT) as tar:
 101                         endname = os.path.split(filename)[-1]
 102                         endname = endname.split('.')[0] + '/'
 103                         for x in files:
 104                                 tarinfo = tar.gettarinfo(x, x)
 105                                 tarinfo.uid   = tarinfo.gid   = 0
 106                                 tarinfo.uname = tarinfo.gname = 'root'
 107                                 tarinfo.size = os.stat(x).st_size
 108
 109                                 # TODO - more archive creation options?
 110                                 if kw.get('bare', True):
 111                                         tarinfo.name = os.path.split(x)[1]
 112                                 else:
 113                                         tarinfo.name = endname + x # todo, if tuple, then..
 114                                 Logs.debug('distnet: adding %r to %s', tarinfo.name, filename)
 115                                 with open(x, 'rb') as f:
 116                                         tar.addfile(tarinfo, f)
 117                 Logs.info('Created %s', filename)
 118
 119 class publish(Context.Context):
 120         fun = 'publish'
 121         cmd = 'publish'
 122         def execute(self):
 123                 if hasattr(Context.g_module, 'publish'):
 124                         Context.Context.execute(self)
 125                 mod = Context.g_module
 126
 127                 rfile = getattr(self, 'rfile', send_package_name())
 128                 if not os.path.isfile(rfile):
 129                         self.fatal('Create the release file with "waf release" first! %r' % rfile)
 130
 131                 fdata = Utils.readf(rfile, m='rb')
 132                 data = safe_urlencode([('pkgdata', fdata), ('pkgname', mod.APPNAME), ('pkgver', mod.VERSION)])
 133
 134                 req = Request(get_upload_url(), data)
 135                 response = urlopen(req, timeout=TIMEOUT)
 136                 data = response.read().strip()
 137
 138                 if sys.hexversion>0x300000f:
 139                         data = data.decode('utf-8')
 140
 141                 if data != 'ok':
 142                         self.fatal('Could not publish the package %r' % data)
 143
 144 class constraint(object):
 145         def __init__(self, line=''):
 146                 self.required_line = line
 147                 self.info = []
 148
 149                 line = line.strip()
 150                 if not line:
 151                         return
 152
 153                 lst = line.split(',')
 154                 if lst:
 155                         self.pkgname = lst[0]
 156                         self.required_version = lst[1]
 157                         for k in lst:
 158                                 a, b, c = k.partition('=')
 159                                 if a and c:
 160                                         self.info.append((a, c))
 161         def __str__(self):
 162                 buf = []
 163                 buf.append(self.pkgname)
 164                 buf.append(self.required_version)
 165                 for k in self.info:
 166                         buf.append('%s=%s' % k)
 167                 return ','.join(buf)
 168
 169         def __repr__(self):
 170                 return "requires %s-%s" % (self.pkgname, self.required_version)
 171
 172         def human_display(self, pkgname, pkgver):
 173                 return '%s-%s requires %s-%s' % (pkgname, pkgver, self.pkgname, self.required_version)
 174
 175         def why(self):
 176                 ret = []
 177                 for x in self.info:
 178                         if x[0] == 'reason':
 179                                 ret.append(x[1])
 180                 return ret
 181
 182         def add_reason(self, reason):
 183                 self.info.append(('reason', reason))
 184
 185 def parse_constraints(text):
 186         assert(text is not None)
 187         constraints = []
 188         text = re.sub(re_com, '', text)
 189         lines = text.splitlines()
 190         for line in lines:
 191                 line = line.strip()
 192                 if not line:
 193                         continue
 194                 constraints.append(constraint(line))
 195         return constraints
 196
 197 def list_package_versions(cachedir, pkgname):
 198         pkgdir = os.path.join(cachedir, pkgname)
 199         try:
 200                 versions = os.listdir(pkgdir)
 201         except OSError:
 202                 return []
 203         versions.sort(key=total_version_order)
 204         versions.reverse()
 205         return versions
 206
 207 class package_reader(Context.Context):
 208         cmd = 'solver'
 209         fun = 'solver'
 210
 211         def __init__(self, **kw):
 212                 Context.Context.__init__(self, **kw)
 213
 214                 self.myproject = getattr(Context.g_module, 'APPNAME', 'project')
 215                 self.myversion = getattr(Context.g_module, 'VERSION', '1.0')
 216                 self.cache_constraints = {}
 217                 self.constraints = []
 218
 219         def compute_dependencies(self, filename=REQUIRES):
 220                 text = Utils.readf(filename)
 221                 data = safe_urlencode([('text', text)])
 222
 223                 if '--offline' in sys.argv:
 224                         self.constraints = self.local_resolve(text)
 225                 else:
 226                         req = Request(get_resolve_url(), data)
 227                         try:
 228                                 response = urlopen(req, timeout=TIMEOUT)
 229                         except URLError as e:
 230                                 Logs.warn('The package server is down! %r', e)
 231                                 self.constraints = self.local_resolve(text)
 232                         else:
 233                                 ret = response.read()
 234                                 try:
 235                                         ret = ret.decode('utf-8')
 236                                 except Exception:
 237                                         pass
 238                                 self.trace(ret)
 239                                 self.constraints = parse_constraints(ret)
 240                 self.check_errors()
 241
 242         def check_errors(self):
 243                 errors = False
 244                 for c in self.constraints:
 245                         if not c.required_version:
 246                                 errors = True
 247
 248                                 reasons = c.why()
 249                                 if len(reasons) == 1:
 250                                         Logs.error('%s but no matching package could be found in this repository', reasons[0])
 251                                 else:
 252                                         Logs.error('Conflicts on package %r:', c.pkgname)
 253                                         for r in reasons:
 254                                                 Logs.error('  %s', r)
 255                 if errors:
 256                         self.fatal('The package requirements cannot be satisfied!')
 257
 258         def load_constraints(self, pkgname, pkgver, requires=REQUIRES):
 259                 try:
 260                         return self.cache_constraints[(pkgname, pkgver)]
 261                 except KeyError:
 262                         text = Utils.readf(os.path.join(get_distnet_cache(), pkgname, pkgver, requires))
 263                         ret = parse_constraints(text)
 264                         self.cache_constraints[(pkgname, pkgver)] = ret
 265                         return ret
 266
 267         def apply_constraint(self, domain, constraint):
 268                 vname = constraint.required_version.replace('*', '.*')
 269                 rev = re.compile(vname, re.M)
 270                 ret = [x for x in domain if rev.match(x)]
 271                 return ret
 272
 273         def trace(self, *k):
 274                 if getattr(self, 'debug', None):
 275                         Logs.error(*k)
 276
 277         def solve(self, packages_to_versions={}, packages_to_constraints={}, pkgname='', pkgver='', todo=[], done=[]):
 278                 # breadth first search
 279                 n_packages_to_versions = dict(packages_to_versions)
 280                 n_packages_to_constraints = dict(packages_to_constraints)
 281
 282                 self.trace("calling solve with %r    %r %r" % (packages_to_versions, todo, done))
 283                 done = done + [pkgname]
 284
 285                 constraints = self.load_constraints(pkgname, pkgver)
 286                 self.trace("constraints %r" % constraints)
 287
 288                 for k in constraints:
 289                         try:
 290                                 domain = n_packages_to_versions[k.pkgname]
 291                         except KeyError:
 292                                 domain = list_package_versions(get_distnet_cache(), k.pkgname)
 293
 294
 295                         self.trace("constraints?")
 296                         if not k.pkgname in done:
 297                                 todo = todo + [k.pkgname]
 298
 299                         self.trace("domain before %s -> %s, %r" % (pkgname, k.pkgname, domain))
 300
 301                         # apply the constraint
 302                         domain = self.apply_constraint(domain, k)
 303
 304                         self.trace("domain after %s -> %s, %r" % (pkgname, k.pkgname, domain))
 305
 306                         n_packages_to_versions[k.pkgname] = domain
 307
 308                         # then store the constraint applied
 309                         constraints = list(packages_to_constraints.get(k.pkgname, []))
 310                         constraints.append((pkgname, pkgver, k))
 311                         n_packages_to_constraints[k.pkgname] = constraints
 312
 313                         if not domain:
 314                                 self.trace("no domain while processing constraint %r from %r %r" % (domain, pkgname, pkgver))
 315                                 return (n_packages_to_versions, n_packages_to_constraints)
 316
 317                 # next package on the todo list
 318                 if not todo:
 319                         return (n_packages_to_versions, n_packages_to_constraints)
 320
 321                 n_pkgname = todo[0]
 322                 n_pkgver = n_packages_to_versions[n_pkgname][0]
 323                 tmp = dict(n_packages_to_versions)
 324                 tmp[n_pkgname] = [n_pkgver]
 325
 326                 self.trace("fixed point %s" % n_pkgname)
 327
 328                 return self.solve(tmp, n_packages_to_constraints, n_pkgname, n_pkgver, todo[1:], done)
 329
 330         def get_results(self):
 331                 return '\n'.join([str(c) for c in self.constraints])
 332
 333         def solution_to_constraints(self, versions, constraints):
 334                 solution = []
 335                 for p in versions:
 336                         c = constraint()
 337                         solution.append(c)
 338
 339                         c.pkgname = p
 340                         if versions[p]:
 341                                 c.required_version = versions[p][0]
 342                         else:
 343                                 c.required_version = ''
 344                         for (from_pkgname, from_pkgver, c2) in constraints.get(p, ''):
 345                                 c.add_reason(c2.human_display(from_pkgname, from_pkgver))
 346                 return solution
 347
 348         def local_resolve(self, text):
 349                 self.cache_constraints[(self.myproject, self.myversion)] = parse_constraints(text)
 350                 p2v = OrderedDict({self.myproject: [self.myversion]})
 351                 (versions, constraints) = self.solve(p2v, {}, self.myproject, self.myversion, [])
 352                 return self.solution_to_constraints(versions, constraints)
 353
 354         def download_to_file(self, pkgname, pkgver, subdir, tmp):
 355                 data = safe_urlencode([('pkgname', pkgname), ('pkgver', pkgver), ('pkgfile', subdir)])
 356                 req = urlopen(get_download_url(), data, timeout=TIMEOUT)
 357                 with open(tmp, 'wb') as f:
 358                         while True:
 359                                 buf = req.read(8192)
 360                                 if not buf:
 361                                         break
 362                                 f.write(buf)
 363
 364         def extract_tar(self, subdir, pkgdir, tmpfile):
 365                 with tarfile.open(tmpfile) as f:
 366                         temp = tempfile.mkdtemp(dir=pkgdir)
 367                         try:
 368                                 f.extractall(temp)
 369                                 os.rename(temp, os.path.join(pkgdir, subdir))
 370                         finally:
 371                                 try:
 372                                         shutil.rmtree(temp)
 373                                 except Exception:
 374                                         pass
 375
 376         def get_pkg_dir(self, pkgname, pkgver, subdir):
 377                 pkgdir = os.path.join(get_distnet_cache(), pkgname, pkgver)
 378                 if not os.path.isdir(pkgdir):
 379                         os.makedirs(pkgdir)
 380
 381                 target = os.path.join(pkgdir, subdir)
 382
 383                 if os.path.exists(target):
 384                         return target
 385
 386                 (fd, tmp) = tempfile.mkstemp(dir=pkgdir)
 387                 try:
 388                         os.close(fd)
 389                         self.download_to_file(pkgname, pkgver, subdir, tmp)
 390                         if subdir == REQUIRES:
 391                                 os.rename(tmp, target)
 392                         else:
 393                                 self.extract_tar(subdir, pkgdir, tmp)
 394                 finally:
 395                         try:
 396                                 os.remove(tmp)
 397                         except OSError:
 398                                 pass
 399
 400                 return target
 401
 402         def __iter__(self):
 403                 if not self.constraints:
 404                         self.compute_dependencies()
 405                 for x in self.constraints:
 406                         if x.pkgname == self.myproject:
 407                                 continue
 408                         yield x
 409                 raise StopIteration
 410
 411         def execute(self):
 412                 self.compute_dependencies()
 413
 414 packages = package_reader()
 415
 416 def load_tools(ctx, extra):
 417         global packages
 418         for c in packages:
 419                 packages.get_pkg_dir(c.pkgname, c.required_version, extra)
 420                 noarchdir = packages.get_pkg_dir(c.pkgname, c.required_version, 'noarch')
 421                 for x in os.listdir(noarchdir):
 422                         if x.startswith('waf_') and x.endswith('.py'):
 423                                 ctx.load([x.rstrip('.py')], tooldir=[noarchdir])
 424
 425 def options(opt):
 426         opt.add_option('--offline', action='store_true')
 427         packages.execute()
 428         load_tools(opt, REQUIRES)
 429
 430 def configure(conf):
 431         load_tools(conf, conf.variant)
 432
 433 def build(bld):
 434         load_tools(bld, bld.variant)
 435