dulwich/client.py

   1 # client.py -- Implementation of the server side git protocols
   2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   3 # Copyright (C) 2008 John Carr
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License
   7 # as published by the Free Software Foundation; either version 2
   8 # or (at your option) a later version of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 # MA  02110-1301, USA.
  19
  20 """Client side support for the Git protocol."""
  21
  22 __docformat__ = 'restructuredText'
  23
  24 import select
  25 import socket
  26 import urlparse
  27
  28 from dulwich.errors import (
  29     SendPackError,
  30     UpdateRefsError,
  31     )
  32 from dulwich.protocol import (
  33     Protocol,
  34     TCP_GIT_PORT,
  35     ZERO_SHA,
  36     extract_capabilities,
  37     )
  38 from dulwich.pack import (
  39     write_pack_data,
  40     )
  41
  42
  43 # Python 2.6.6 included these in urlparse.uses_netloc upstream. Do
  44 # monkeypatching to enable similar behaviour in earlier Pythons:
  45 for scheme in ('git', 'git+ssh'):
  46     if scheme not in urlparse.uses_netloc:
  47         urlparse.uses_netloc.append(scheme)
  48
  49 def _fileno_can_read(fileno):
  50     """Check if a file descriptor is readable."""
  51     return len(select.select([fileno], [], [], 0)[0]) > 0
  52
  53 COMMON_CAPABILITIES = ['ofs-delta']
  54 FETCH_CAPABILITIES = ['multi_ack', 'side-band-64k'] + COMMON_CAPABILITIES
  55 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
  56
  57 # TODO(durin42): this doesn't correctly degrade if the server doesn't
  58 # support some capabilities. This should work properly with servers
  59 # that don't support side-band-64k and multi_ack.
  60 class GitClient(object):
  61     """Git smart server client.
  62
  63     """
  64
  65     def __init__(self, thin_packs=True, report_activity=None):
  66         """Create a new GitClient instance.
  67
  68         :param thin_packs: Whether or not thin packs should be retrieved
  69         :param report_activity: Optional callback for reporting transport
  70             activity.
  71         """
  72         self._report_activity = report_activity
  73         self._fetch_capabilities = list(FETCH_CAPABILITIES)
  74         self._send_capabilities = list(SEND_CAPABILITIES)
  75         if thin_packs:
  76             self._fetch_capabilities.append('thin-pack')
  77
  78     def _connect(self, cmd, path):
  79         """Create a connection to the server.
  80
  81         This method is abstract - concrete implementations should
  82         implement their own variant which connects to the server and
  83         returns an initialized Protocol object with the service ready
  84         for use and a can_read function which may be used to see if
  85         reads would block.
  86
  87         :param cmd: The git service name to which we should connect.
  88         :param path: The path we should pass to the service.
  89         """
  90         raise NotImplementedError()
  91
  92     def read_refs(self, proto):
  93         server_capabilities = None
  94         refs = {}
  95         # Receive refs from server
  96         for pkt in proto.read_pkt_seq():
  97             (sha, ref) = pkt.rstrip('\n').split(' ', 1)
  98             if server_capabilities is None:
  99                 (ref, server_capabilities) = extract_capabilities(ref)
 100             refs[ref] = sha
 101         return refs, server_capabilities
 102
 103     def _parse_status_report(self, proto):
 104         unpack = proto.read_pkt_line().strip()
 105         if unpack != 'unpack ok':
 106             st = True
 107             # flush remaining error data
 108             while st is not None:
 109                 st = proto.read_pkt_line()
 110             raise SendPackError(unpack)
 111         statuses = []
 112         errs = False
 113         ref_status = proto.read_pkt_line()
 114         while ref_status:
 115             ref_status = ref_status.strip()
 116             statuses.append(ref_status)
 117             if not ref_status.startswith('ok '):
 118                 errs = True
 119             ref_status = proto.read_pkt_line()
 120
 121         if errs:
 122             ref_status = {}
 123             ok = set()
 124             for status in statuses:
 125                 if ' ' not in status:
 126                     # malformed response, move on to the next one
 127                     continue
 128                 status, ref = status.split(' ', 1)
 129
 130                 if status == 'ng':
 131                     if ' ' in ref:
 132                         ref, status = ref.split(' ', 1)
 133                 else:
 134                     ok.add(ref)
 135                 ref_status[ref] = status
 136             raise UpdateRefsError('%s failed to update' %
 137                                   ', '.join([ref for ref in ref_status
 138                                              if ref not in ok]),
 139                                   ref_status=ref_status)
 140
 141
 142     # TODO(durin42): add side-band-64k capability support here and advertise it
 143     def send_pack(self, path, determine_wants, generate_pack_contents):
 144         """Upload a pack to a remote repository.
 145
 146         :param path: Repository path
 147         :param generate_pack_contents: Function that can return the shas of the
 148             objects to upload.
 149
 150         :raises SendPackError: if server rejects the pack data
 151         :raises UpdateRefsError: if the server supports report-status
 152                                  and rejects ref updates
 153         """
 154         proto, unused_can_read = self._connect('receive-pack', path)
 155         old_refs, server_capabilities = self.read_refs(proto)
 156         if 'report-status' not in server_capabilities:
 157             self._send_capabilities.remove('report-status')
 158         new_refs = determine_wants(old_refs)
 159         if not new_refs:
 160             proto.write_pkt_line(None)
 161             return {}
 162         want = []
 163         have = [x for x in old_refs.values() if not x == ZERO_SHA]
 164         sent_capabilities = False
 165         for refname in set(new_refs.keys() + old_refs.keys()):
 166             old_sha1 = old_refs.get(refname, ZERO_SHA)
 167             new_sha1 = new_refs.get(refname, ZERO_SHA)
 168             if old_sha1 != new_sha1:
 169                 if sent_capabilities:
 170                     proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
 171                                                             refname))
 172                 else:
 173                     proto.write_pkt_line(
 174                       '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
 175                                         ' '.join(self._send_capabilities)))
 176                     sent_capabilities = True
 177             if new_sha1 not in have and new_sha1 != ZERO_SHA:
 178                 want.append(new_sha1)
 179         proto.write_pkt_line(None)
 180         if not want:
 181             return new_refs
 182         objects = generate_pack_contents(have, want)
 183         entries, sha = write_pack_data(proto.write_file(), objects,
 184                                        len(objects))
 185
 186         if 'report-status' in self._send_capabilities:
 187             self._parse_status_report(proto)
 188         # wait for EOF before returning
 189         data = proto.read()
 190         if data:
 191             raise SendPackError('Unexpected response %r' % data)
 192         return new_refs
 193
 194     def fetch(self, path, target, determine_wants=None, progress=None):
 195         """Fetch into a target repository.
 196
 197         :param path: Path to fetch from
 198         :param target: Target repository to fetch into
 199         :param determine_wants: Optional function to determine what refs
 200             to fetch
 201         :param progress: Optional progress function
 202         :return: remote refs
 203         """
 204         if determine_wants is None:
 205             determine_wants = target.object_store.determine_wants_all
 206         f, commit = target.object_store.add_pack()
 207         try:
 208             return self.fetch_pack(path, determine_wants,
 209                 target.get_graph_walker(), f.write, progress)
 210         finally:
 211             commit()
 212
 213     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 214                    progress):
 215         """Retrieve a pack from a git smart server.
 216
 217         :param determine_wants: Callback that returns list of commits to fetch
 218         :param graph_walker: Object with next() and ack().
 219         :param pack_data: Callback called for each bit of data in the pack
 220         :param progress: Callback for progress reports (strings)
 221         """
 222         proto, can_read = self._connect('upload-pack', path)
 223         (refs, server_capabilities) = self.read_refs(proto)
 224         wants = determine_wants(refs)
 225         if not wants:
 226             proto.write_pkt_line(None)
 227             return refs
 228         assert isinstance(wants, list) and type(wants[0]) == str
 229         proto.write_pkt_line('want %s %s\n' % (
 230             wants[0], ' '.join(self._fetch_capabilities)))
 231         for want in wants[1:]:
 232             proto.write_pkt_line('want %s\n' % want)
 233         proto.write_pkt_line(None)
 234         have = graph_walker.next()
 235         while have:
 236             proto.write_pkt_line('have %s\n' % have)
 237             if can_read():
 238                 pkt = proto.read_pkt_line()
 239                 parts = pkt.rstrip('\n').split(' ')
 240                 if parts[0] == 'ACK':
 241                     graph_walker.ack(parts[1])
 242                     assert parts[2] == 'continue'
 243             have = graph_walker.next()
 244         proto.write_pkt_line('done\n')
 245         pkt = proto.read_pkt_line()
 246         while pkt:
 247             parts = pkt.rstrip('\n').split(' ')
 248             if parts[0] == 'ACK':
 249                 graph_walker.ack(pkt.split(' ')[1])
 250             if len(parts) < 3 or parts[2] != 'continue':
 251                 break
 252             pkt = proto.read_pkt_line()
 253         # TODO(durin42): this is broken if the server didn't support the
 254         # side-band-64k capability.
 255         for pkt in proto.read_pkt_seq():
 256             channel = ord(pkt[0])
 257             pkt = pkt[1:]
 258             if channel == 1:
 259                 pack_data(pkt)
 260             elif channel == 2:
 261                 if progress is not None:
 262                     progress(pkt)
 263             else:
 264                 raise AssertionError('Invalid sideband channel %d' % channel)
 265         return refs
 266
 267
 268 class TCPGitClient(GitClient):
 269     """A Git Client that works over TCP directly (i.e. git://)."""
 270
 271     def __init__(self, host, port=None, *args, **kwargs):
 272         if port is None:
 273             port = TCP_GIT_PORT
 274         self._host = host
 275         self._port = port
 276         GitClient.__init__(self, *args, **kwargs)
 277
 278     def _connect(self, cmd, path):
 279         s = socket.socket(type=socket.SOCK_STREAM)
 280         s.connect((self._host, self._port))
 281         # -1 means system default buffering
 282         rfile = s.makefile('rb', -1)
 283         # 0 means unbuffered
 284         wfile = s.makefile('wb', 0)
 285         proto = Protocol(rfile.read, wfile.write,
 286                          report_activity=self._report_activity)
 287         proto.send_cmd('git-%s' % cmd, path, 'host=%s' % self._host)
 288         return proto, lambda: _fileno_can_read(s)
 289
 290
 291 class SubprocessWrapper(object):
 292     """A socket-like object that talks to a subprocess via pipes."""
 293
 294     def __init__(self, proc):
 295         self.proc = proc
 296         self.read = proc.stdout.read
 297         self.write = proc.stdin.write
 298
 299     def can_read(self):
 300         return _fileno_can_read(self.proc.stdout.fileno())
 301
 302     def close(self):
 303         self.proc.stdin.close()
 304         self.proc.stdout.close()
 305         self.proc.wait()
 306
 307
 308 class SubprocessGitClient(GitClient):
 309     """Git client that talks to a server using a subprocess."""
 310
 311     def __init__(self, *args, **kwargs):
 312         self._connection = None
 313         GitClient.__init__(self, *args, **kwargs)
 314
 315     def _connect(self, service, path):
 316         import subprocess
 317         argv = ['git', service, path]
 318         p = SubprocessWrapper(
 319             subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
 320                              stdout=subprocess.PIPE))
 321         return Protocol(p.read, p.write,
 322                         report_activity=self._report_activity), p.can_read
 323
 324
 325 class SSHVendor(object):
 326
 327     def connect_ssh(self, host, command, username=None, port=None):
 328         import subprocess
 329         #FIXME: This has no way to deal with passwords..
 330         args = ['ssh', '-x']
 331         if port is not None:
 332             args.extend(['-p', str(port)])
 333         if username is not None:
 334             host = '%s@%s' % (username, host)
 335         args.append(host)
 336         proc = subprocess.Popen(args + command,
 337                                 stdin=subprocess.PIPE,
 338                                 stdout=subprocess.PIPE)
 339         return SubprocessWrapper(proc)
 340
 341 # Can be overridden by users
 342 get_ssh_vendor = SSHVendor
 343
 344
 345 class SSHGitClient(GitClient):
 346
 347     def __init__(self, host, port=None, username=None, *args, **kwargs):
 348         self.host = host
 349         self.port = port
 350         self.username = username
 351         GitClient.__init__(self, *args, **kwargs)
 352         self.alternative_paths = {}
 353
 354     def _get_cmd_path(self, cmd):
 355         return self.alternative_paths.get(cmd, 'git-%s' % cmd)
 356
 357     def _connect(self, cmd, path):
 358         con = get_ssh_vendor().connect_ssh(
 359             self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
 360             port=self.port, username=self.username)
 361         return Protocol(con.read, con.write), con.can_read
 362
 363
 364 def get_transport_and_path(uri):
 365     """Obtain a git client from a URI or path.
 366
 367     :param uri: URI or path
 368     :return: Tuple with client instance and relative path.
 369     """
 370     parsed = urlparse.urlparse(uri)
 371     if parsed.scheme == 'git':
 372         return TCPGitClient(parsed.hostname, port=parsed.port), parsed.path
 373     elif parsed.scheme == 'git+ssh':
 374         return SSHGitClient(parsed.hostname, port=parsed.port,
 375                             username=parsed.username), parsed.path
 376
 377     if parsed.scheme and not parsed.netloc:
 378         # SSH with no user@, zero or one leading slash.
 379         return SSHGitClient(parsed.scheme), parsed.path
 380     elif parsed.scheme:
 381         raise ValueError('Unknown git protocol scheme: %s' % parsed.scheme)
 382     elif '@' in parsed.path and ':' in parsed.path:
 383         # SSH with user@host:foo.
 384         user_host, path = parsed.path.split(':')
 385         user, host = user_host.rsplit('@')
 386         return SSHGitClient(host, username=user), path
 387
 388     # Otherwise, assume it's a local path.
 389     return SubprocessGitClient(), uri