third_party/waf/waflib/extras/remote.py

   1 #!/usr/bin/env python
   2 # encoding: utf-8
   3 # Remote Builds tool using rsync+ssh
   4
   5 __author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
   6 __copyright__ = "Jérôme Carretero, 2013"
   7
   8 """
   9 Simple Remote Builds
  10 ********************
  11
  12 This tool is an *experimental* tool (meaning, do not even try to pollute
  13 the waf bug tracker with bugs in here, contact me directly) providing simple
  14 remote builds.
  15
  16 It uses rsync and ssh to perform the remote builds.
  17 It is intended for performing cross-compilation on platforms where
  18 a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product
  19 does not exist (eg. Windows builds using Visual Studio) or simply not installed.
  20 This tool sends the sources and the waf script to the remote host,
  21 and commands the usual waf execution.
  22
  23 There are alternatives to using this tool, such as setting up shared folders,
  24 logging on to remote machines, and building on the shared folders.
  25 Electing one method or another depends on the size of the program.
  26
  27
  28 Usage
  29 =====
  30
  31 1. Set your wscript file so it includes a list of variants,
  32    e.g.::
  33
  34      from waflib import Utils
  35      top = '.'
  36      out = 'build'
  37
  38      variants = [
  39       'linux_64_debug',
  40       'linux_64_release',
  41       'linux_32_debug',
  42       'linux_32_release',
  43       ]
  44
  45      from waflib.extras import remote
  46
  47      def options(opt):
  48          # normal stuff from here on
  49          opt.load('compiler_c')
  50
  51      def configure(conf):
  52          if not conf.variant:
  53              return
  54          # normal stuff from here on
  55          conf.load('compiler_c')
  56
  57      def build(bld):
  58          if not bld.variant:
  59              return
  60          # normal stuff from here on
  61          bld(features='c cprogram', target='app', source='main.c')
  62
  63
  64 2. Build the waf file, so it includes this tool, and put it in the current
  65    directory
  66
  67    .. code:: bash
  68
  69       ./waf-light --tools=remote
  70
  71 3. Set the host names to access the hosts:
  72
  73    .. code:: bash
  74
  75       export REMOTE_QNX=user@kiunix
  76
  77 4. Setup the ssh server and ssh keys
  78
  79    The ssh key should not be protected by a password, or it will prompt for it every time.
  80    Create the key on the client:
  81
  82    .. code:: bash
  83
  84       ssh-keygen -t rsa -f foo.rsa
  85
  86    Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys),
  87    and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys)
  88
  89    A separate key for the build processes can be set in the environment variable WAF_SSH_KEY.
  90    The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so
  91    be warned to use this feature on internal networks only (MITM).
  92
  93    .. code:: bash
  94
  95       export WAF_SSH_KEY=~/foo.rsa
  96
  97 5. Perform the build:
  98
  99    .. code:: bash
 100
 101       waf configure_all build_all --remote
 102
 103 """
 104
 105
 106 import getpass, os, re, sys
 107 from collections import OrderedDict
 108 from waflib import Context, Options, Utils, ConfigSet
 109
 110 from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext
 111 from waflib.Configure import ConfigurationContext
 112
 113
 114 is_remote = False
 115 if '--remote' in sys.argv:
 116         is_remote = True
 117         sys.argv.remove('--remote')
 118
 119 class init(Context.Context):
 120         """
 121         Generates the *_all commands
 122         """
 123         cmd = 'init'
 124         fun = 'init'
 125         def execute(self):
 126                 for x in list(Context.g_module.variants):
 127                         self.make_variant(x)
 128                 lst = ['remote']
 129                 for k in Options.commands:
 130                         if k.endswith('_all'):
 131                                 name = k.replace('_all', '')
 132                                 for x in Context.g_module.variants:
 133                                         lst.append('%s_%s' % (name, x))
 134                         else:
 135                                 lst.append(k)
 136                 del Options.commands[:]
 137                 Options.commands += lst
 138
 139         def make_variant(self, x):
 140                 for y in (BuildContext, CleanContext, InstallContext, UninstallContext):
 141                         name = y.__name__.replace('Context','').lower()
 142                         class tmp(y):
 143                                 cmd = name + '_' + x
 144                                 fun = 'build'
 145                                 variant = x
 146                 class tmp(ConfigurationContext):
 147                         cmd = 'configure_' + x
 148                         fun = 'configure'
 149                         variant = x
 150                         def __init__(self, **kw):
 151                                 ConfigurationContext.__init__(self, **kw)
 152                                 self.setenv(x)
 153
 154 class remote(BuildContext):
 155         cmd = 'remote'
 156         fun = 'build'
 157
 158         def get_ssh_hosts(self):
 159                 lst = []
 160                 for v in Context.g_module.variants:
 161                         self.env.HOST = self.login_to_host(self.variant_to_login(v))
 162                         cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env)
 163                         out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH)
 164                         lst.append(out.strip())
 165                 return lst
 166
 167         def setup_private_ssh_key(self):
 168                 """
 169                 When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory
 170                 Make sure that the ssh key does not prompt for a password
 171                 """
 172                 key = os.environ.get('WAF_SSH_KEY', '')
 173                 if not key:
 174                         return
 175                 if not os.path.isfile(key):
 176                         self.fatal('Key in WAF_SSH_KEY must point to a valid file')
 177                 self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh')
 178                 self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts')
 179                 self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key))
 180                 self.ssh_config = os.path.join(self.ssh_dir, 'config')
 181                 for x in self.ssh_hosts, self.ssh_key, self.ssh_config:
 182                         if not os.path.isfile(x):
 183                                 if not os.path.isdir(self.ssh_dir):
 184                                         os.makedirs(self.ssh_dir)
 185                                 Utils.writef(self.ssh_key, Utils.readf(key), 'wb')
 186                                 os.chmod(self.ssh_key, 448)
 187
 188                                 Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts()))
 189                                 os.chmod(self.ssh_key, 448)
 190
 191                                 Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb')
 192                                 os.chmod(self.ssh_config, 448)
 193                 self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key]
 194                 self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh')
 195
 196         def skip_unbuildable_variant(self):
 197                 # skip variants that cannot be built on this OS
 198                 for k in Options.commands:
 199                         a, _, b = k.partition('_')
 200                         if b in Context.g_module.variants:
 201                                 c, _, _ = b.partition('_')
 202                                 if c != Utils.unversioned_sys_platform():
 203                                         Options.commands.remove(k)
 204
 205         def login_to_host(self, login):
 206                 return re.sub('(\w+@)', '', login)
 207
 208         def variant_to_login(self, variant):
 209                 """linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
 210                 x = variant[:variant.rfind('_')]
 211                 ret = os.environ.get('REMOTE_' + x.upper(), '')
 212                 if not ret:
 213                         x = x[:x.find('_')]
 214                         ret = os.environ.get('REMOTE_' + x.upper(), '')
 215                 if not ret:
 216                         ret = '%s@localhost' % getpass.getuser()
 217                 return ret
 218
 219         def execute(self):
 220                 global is_remote
 221                 if not is_remote:
 222                         self.skip_unbuildable_variant()
 223                 else:
 224                         BuildContext.execute(self)
 225
 226         def restore(self):
 227                 self.top_dir = os.path.abspath(Context.g_module.top)
 228                 self.srcnode = self.root.find_node(self.top_dir)
 229                 self.path = self.srcnode
 230
 231                 self.out_dir = os.path.join(self.top_dir, Context.g_module.out)
 232                 self.bldnode = self.root.make_node(self.out_dir)
 233                 self.bldnode.mkdir()
 234
 235                 self.env = ConfigSet.ConfigSet()
 236
 237         def extract_groups_of_builds(self):
 238                 """Return a dict mapping each variants to the commands to build"""
 239                 self.vgroups = {}
 240                 for x in reversed(Options.commands):
 241                         _, _, variant = x.partition('_')
 242                         if variant in Context.g_module.variants:
 243                                 try:
 244                                         dct = self.vgroups[variant]
 245                                 except KeyError:
 246                                         dct = self.vgroups[variant] = OrderedDict()
 247                                 try:
 248                                         dct[variant].append(x)
 249                                 except KeyError:
 250                                         dct[variant] = [x]
 251                                 Options.commands.remove(x)
 252
 253         def custom_options(self, login):
 254                 try:
 255                         return Context.g_module.host_options[login]
 256                 except (AttributeError, KeyError):
 257                         return {}
 258
 259         def recurse(self, *k, **kw):
 260                 self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx')
 261                 self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh')
 262                 self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan')
 263                 try:
 264                         self.env.WAF = getattr(Context.g_module, 'waf')
 265                 except AttributeError:
 266                         try:
 267                                 os.stat('waf')
 268                         except KeyError:
 269                                 self.fatal('Put a waf file in the directory (./waf-light --tools=remote)')
 270                         else:
 271                                 self.env.WAF = './waf'
 272
 273                 self.extract_groups_of_builds()
 274                 self.setup_private_ssh_key()
 275                 for k, v in self.vgroups.items():
 276                         task = self(rule=rsync_and_ssh, always=True)
 277                         task.env.login = self.variant_to_login(k)
 278
 279                         task.env.commands = []
 280                         for opt, value in v.items():
 281                                 task.env.commands += value
 282                         task.env.variant = task.env.commands[0].partition('_')[2]
 283                         for opt, value in self.custom_options(k):
 284                                 task.env[opt] = value
 285                 self.jobs = len(self.vgroups)
 286
 287         def make_mkdir_command(self, task):
 288                 return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env)
 289
 290         def make_send_command(self, task):
 291                 return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env)
 292
 293         def make_exec_command(self, task):
 294                 txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"'''
 295                 return Utils.subst_vars(txt, task.env)
 296
 297         def make_save_command(self, task):
 298                 return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env)
 299
 300 def rsync_and_ssh(task):
 301
 302         # remove a warning
 303         task.uid_ = id(task)
 304
 305         bld = task.generator.bld
 306
 307         task.env.user, _, _ = task.env.login.partition('@')
 308         task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant)))
 309         task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir)
 310         task.env.local_dir = bld.srcnode.abspath() + '/'
 311
 312         task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant)
 313         task.env.build_dir = bld.bldnode.abspath()
 314
 315         ret = task.exec_command(bld.make_mkdir_command(task))
 316         if ret:
 317                 return ret
 318         ret = task.exec_command(bld.make_send_command(task))
 319         if ret:
 320                 return ret
 321         ret = task.exec_command(bld.make_exec_command(task))
 322         if ret:
 323                 return ret
 324         ret = task.exec_command(bld.make_save_command(task))
 325         if ret:
 326                 return ret
 327