dulwich/client.py

   1 # client.py -- Implementation of the server side git protocols
   2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   3 # Copyright (C) 2008 John Carr
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License
   7 # as published by the Free Software Foundation; either version 2
   8 # or (at your option) a later version of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 # MA  02110-1301, USA.
  19
  20 """Client side support for the Git protocol."""
  21
  22 __docformat__ = 'restructuredText'
  23
  24 from cStringIO import StringIO
  25 import select
  26 import socket
  27 import subprocess
  28 import urllib2
  29 import urlparse
  30
  31 from dulwich.errors import (
  32     GitProtocolError,
  33     NotGitRepository,
  34     SendPackError,
  35     UpdateRefsError,
  36     )
  37 from dulwich.protocol import (
  38     _RBUFSIZE,
  39     PktLineParser,
  40     Protocol,
  41     TCP_GIT_PORT,
  42     ZERO_SHA,
  43     extract_capabilities,
  44     )
  45 from dulwich.pack import (
  46     write_pack_objects,
  47     )
  48
  49
  50 # Python 2.6.6 included these in urlparse.uses_netloc upstream. Do
  51 # monkeypatching to enable similar behaviour in earlier Pythons:
  52 for scheme in ('git', 'git+ssh'):
  53     if scheme not in urlparse.uses_netloc:
  54         urlparse.uses_netloc.append(scheme)
  55
  56 def _fileno_can_read(fileno):
  57     """Check if a file descriptor is readable."""
  58     return len(select.select([fileno], [], [], 0)[0]) > 0
  59
  60 COMMON_CAPABILITIES = ['ofs-delta', 'side-band-64k']
  61 FETCH_CAPABILITIES = ['multi_ack', 'multi_ack_detailed'] + COMMON_CAPABILITIES
  62 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
  63
  64
  65 class ReportStatusParser(object):
  66     """Handle status as reported by servers with the 'report-status' capability.
  67     """
  68
  69     def __init__(self):
  70         self._done = False
  71         self._pack_status = None
  72         self._ref_status_ok = True
  73         self._ref_statuses = []
  74
  75     def check(self):
  76         """Check if there were any errors and, if so, raise exceptions.
  77
  78         :raise SendPackError: Raised when the server could not unpack
  79         :raise UpdateRefsError: Raised when refs could not be updated
  80         """
  81         if self._pack_status not in ('unpack ok', None):
  82             raise SendPackError(self._pack_status)
  83         if not self._ref_status_ok:
  84             ref_status = {}
  85             ok = set()
  86             for status in self._ref_statuses:
  87                 if ' ' not in status:
  88                     # malformed response, move on to the next one
  89                     continue
  90                 status, ref = status.split(' ', 1)
  91
  92                 if status == 'ng':
  93                     if ' ' in ref:
  94                         ref, status = ref.split(' ', 1)
  95                 else:
  96                     ok.add(ref)
  97                 ref_status[ref] = status
  98             raise UpdateRefsError('%s failed to update' %
  99                                   ', '.join([ref for ref in ref_status
 100                                              if ref not in ok]),
 101                                   ref_status=ref_status)
 102
 103     def handle_packet(self, pkt):
 104         """Handle a packet.
 105
 106         :raise GitProtocolError: Raised when packets are received after a
 107             flush packet.
 108         """
 109         if self._done:
 110             raise GitProtocolError("received more data after status report")
 111         if pkt is None:
 112             self._done = True
 113             return
 114         if self._pack_status is None:
 115             self._pack_status = pkt.strip()
 116         else:
 117             ref_status = pkt.strip()
 118             self._ref_statuses.append(ref_status)
 119             if not ref_status.startswith('ok '):
 120                 self._ref_status_ok = False
 121
 122
 123 # TODO(durin42): this doesn't correctly degrade if the server doesn't
 124 # support some capabilities. This should work properly with servers
 125 # that don't support multi_ack.
 126 class GitClient(object):
 127     """Git smart server client.
 128
 129     """
 130
 131     def __init__(self, thin_packs=True, report_activity=None):
 132         """Create a new GitClient instance.
 133
 134         :param thin_packs: Whether or not thin packs should be retrieved
 135         :param report_activity: Optional callback for reporting transport
 136             activity.
 137         """
 138         self._report_activity = report_activity
 139         self._fetch_capabilities = list(FETCH_CAPABILITIES)
 140         self._send_capabilities = list(SEND_CAPABILITIES)
 141         if thin_packs:
 142             self._fetch_capabilities.append('thin-pack')
 143
 144     def _read_refs(self, proto):
 145         server_capabilities = None
 146         refs = {}
 147         # Receive refs from server
 148         for pkt in proto.read_pkt_seq():
 149             (sha, ref) = pkt.rstrip('\n').split(' ', 1)
 150             if sha == 'ERR':
 151                 raise GitProtocolError(ref)
 152             if server_capabilities is None:
 153                 (ref, server_capabilities) = extract_capabilities(ref)
 154             refs[ref] = sha
 155         return refs, server_capabilities
 156
 157     def send_pack(self, path, determine_wants, generate_pack_contents,
 158                   progress=None):
 159         """Upload a pack to a remote repository.
 160
 161         :param path: Repository path
 162         :param generate_pack_contents: Function that can return a sequence of the
 163             shas of the objects to upload.
 164         :param progress: Optional progress function
 165
 166         :raises SendPackError: if server rejects the pack data
 167         :raises UpdateRefsError: if the server supports report-status
 168                                  and rejects ref updates
 169         """
 170         raise NotImplementedError(self.send_pack)
 171
 172     def fetch(self, path, target, determine_wants=None, progress=None):
 173         """Fetch into a target repository.
 174
 175         :param path: Path to fetch from
 176         :param target: Target repository to fetch into
 177         :param determine_wants: Optional function to determine what refs
 178             to fetch
 179         :param progress: Optional progress function
 180         :return: remote refs
 181         """
 182         if determine_wants is None:
 183             determine_wants = target.object_store.determine_wants_all
 184         f, commit = target.object_store.add_pack()
 185         try:
 186             return self.fetch_pack(path, determine_wants,
 187                 target.get_graph_walker(), f.write, progress)
 188         finally:
 189             commit()
 190
 191     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 192                    progress):
 193         """Retrieve a pack from a git smart server.
 194
 195         :param determine_wants: Callback that returns list of commits to fetch
 196         :param graph_walker: Object with next() and ack().
 197         :param pack_data: Callback called for each bit of data in the pack
 198         :param progress: Callback for progress reports (strings)
 199         """
 200         raise NotImplementedError(self.fetch_pack)
 201
 202     def _parse_status_report(self, proto):
 203         unpack = proto.read_pkt_line().strip()
 204         if unpack != 'unpack ok':
 205             st = True
 206             # flush remaining error data
 207             while st is not None:
 208                 st = proto.read_pkt_line()
 209             raise SendPackError(unpack)
 210         statuses = []
 211         errs = False
 212         ref_status = proto.read_pkt_line()
 213         while ref_status:
 214             ref_status = ref_status.strip()
 215             statuses.append(ref_status)
 216             if not ref_status.startswith('ok '):
 217                 errs = True
 218             ref_status = proto.read_pkt_line()
 219
 220         if errs:
 221             ref_status = {}
 222             ok = set()
 223             for status in statuses:
 224                 if ' ' not in status:
 225                     # malformed response, move on to the next one
 226                     continue
 227                 status, ref = status.split(' ', 1)
 228
 229                 if status == 'ng':
 230                     if ' ' in ref:
 231                         ref, status = ref.split(' ', 1)
 232                 else:
 233                     ok.add(ref)
 234                 ref_status[ref] = status
 235             raise UpdateRefsError('%s failed to update' %
 236                                   ', '.join([ref for ref in ref_status
 237                                              if ref not in ok]),
 238                                   ref_status=ref_status)
 239
 240     def _read_side_band64k_data(self, proto, channel_callbacks):
 241         """Read per-channel data.
 242
 243         This requires the side-band-64k capability.
 244
 245         :param proto: Protocol object to read from
 246         :param channel_callbacks: Dictionary mapping channels to packet
 247             handlers to use. None for a callback discards channel data.
 248         """
 249         for pkt in proto.read_pkt_seq():
 250             channel = ord(pkt[0])
 251             pkt = pkt[1:]
 252             try:
 253                 cb = channel_callbacks[channel]
 254             except KeyError:
 255                 raise AssertionError('Invalid sideband channel %d' % channel)
 256             else:
 257                 if cb is not None:
 258                     cb(pkt)
 259
 260     def _handle_receive_pack_head(self, proto, capabilities, old_refs, new_refs):
 261         """Handle the head of a 'git-receive-pack' request.
 262
 263         :param proto: Protocol object to read from
 264         :param capabilities: List of negotiated capabilities
 265         :param old_refs: Old refs, as received from the server
 266         :param new_refs: New refs
 267         :return: (have, want) tuple
 268         """
 269         want = []
 270         have = [x for x in old_refs.values() if not x == ZERO_SHA]
 271         sent_capabilities = False
 272         for refname in set(new_refs.keys() + old_refs.keys()):
 273             old_sha1 = old_refs.get(refname, ZERO_SHA)
 274             new_sha1 = new_refs.get(refname, ZERO_SHA)
 275             if old_sha1 != new_sha1:
 276                 if sent_capabilities:
 277                     proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
 278                                                             refname))
 279                 else:
 280                     proto.write_pkt_line(
 281                       '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
 282                                         ' '.join(capabilities)))
 283                     sent_capabilities = True
 284             if new_sha1 not in have and new_sha1 != ZERO_SHA:
 285                 want.append(new_sha1)
 286         proto.write_pkt_line(None)
 287         return (have, want)
 288
 289     def _handle_receive_pack_tail(self, proto, capabilities, progress):
 290         """Handle the tail of a 'git-receive-pack' request.
 291
 292         :param proto: Protocol object to read from
 293         :param capabilities: List of negotiated capabilities
 294         :param progress: Optional progress reporting function
 295         """
 296         if 'report-status' in capabilities:
 297             report_status_parser = ReportStatusParser()
 298         else:
 299             report_status_parser = None
 300         if "side-band-64k" in capabilities:
 301             channel_callbacks = { 2: progress }
 302             if 'report-status' in capabilities:
 303                 channel_callbacks[1] = PktLineParser(
 304                     report_status_parser.handle_packet).parse
 305             self._read_side_band64k_data(proto, channel_callbacks)
 306         else:
 307             if 'report-status':
 308                 for pkt in proto.read_pkt_seq():
 309                     report_status_parser.handle_packet(pkt)
 310         if report_status_parser is not None:
 311             report_status_parser.check()
 312         # wait for EOF before returning
 313         data = proto.read()
 314         if data:
 315             raise SendPackError('Unexpected response %r' % data)
 316
 317     def _handle_upload_pack_head(self, proto, capabilities, graph_walker,
 318                                  wants, can_read):
 319         """Handle the head of a 'git-upload-pack' request.
 320
 321         :param proto: Protocol object to read from
 322         :param capabilities: List of negotiated capabilities
 323         :param graph_walker: GraphWalker instance to call .ack() on
 324         :param wants: List of commits to fetch
 325         :param can_read: function that returns a boolean that indicates
 326             whether there is extra graph data to read on proto
 327         """
 328         assert isinstance(wants, list) and type(wants[0]) == str
 329         proto.write_pkt_line('want %s %s\n' % (
 330             wants[0], ' '.join(capabilities)))
 331         for want in wants[1:]:
 332             proto.write_pkt_line('want %s\n' % want)
 333         proto.write_pkt_line(None)
 334         have = graph_walker.next()
 335         while have:
 336             proto.write_pkt_line('have %s\n' % have)
 337             if can_read():
 338                 pkt = proto.read_pkt_line()
 339                 parts = pkt.rstrip('\n').split(' ')
 340                 if parts[0] == 'ACK':
 341                     graph_walker.ack(parts[1])
 342                     if parts[2] in ('continue', 'common'):
 343                         pass
 344                     elif parts[2] == 'ready':
 345                         break
 346                     else:
 347                         raise AssertionError(
 348                             "%s not in ('continue', 'ready', 'common)" %
 349                             parts[2])
 350             have = graph_walker.next()
 351         proto.write_pkt_line('done\n')
 352
 353     def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,
 354                                  pack_data, progress, rbufsize=_RBUFSIZE):
 355         """Handle the tail of a 'git-upload-pack' request.
 356
 357         :param proto: Protocol object to read from
 358         :param capabilities: List of negotiated capabilities
 359         :param graph_walker: GraphWalker instance to call .ack() on
 360         :param pack_data: Function to call with pack data
 361         :param progress: Optional progress reporting function
 362         :param rbufsize: Read buffer size
 363         """
 364         pkt = proto.read_pkt_line()
 365         while pkt:
 366             parts = pkt.rstrip('\n').split(' ')
 367             if parts[0] == 'ACK':
 368                 graph_walker.ack(pkt.split(' ')[1])
 369             if len(parts) < 3 or parts[2] not in (
 370                     'ready', 'continue', 'common'):
 371                 break
 372             pkt = proto.read_pkt_line()
 373         if "side-band-64k" in capabilities:
 374             self._read_side_band64k_data(proto, {1: pack_data, 2: progress})
 375             # wait for EOF before returning
 376             data = proto.read()
 377             if data:
 378                 raise Exception('Unexpected response %r' % data)
 379         else:
 380             while True:
 381                 data = self.read(rbufsize)
 382                 if data == "":
 383                     break
 384                 pack_data(data)
 385
 386
 387 class TraditionalGitClient(GitClient):
 388     """Traditional Git client."""
 389
 390     def _connect(self, cmd, path):
 391         """Create a connection to the server.
 392
 393         This method is abstract - concrete implementations should
 394         implement their own variant which connects to the server and
 395         returns an initialized Protocol object with the service ready
 396         for use and a can_read function which may be used to see if
 397         reads would block.
 398
 399         :param cmd: The git service name to which we should connect.
 400         :param path: The path we should pass to the service.
 401         """
 402         raise NotImplementedError()
 403
 404     def send_pack(self, path, determine_wants, generate_pack_contents,
 405                   progress=None):
 406         """Upload a pack to a remote repository.
 407
 408         :param path: Repository path
 409         :param generate_pack_contents: Function that can return a sequence of the
 410             shas of the objects to upload.
 411         :param progress: Optional callback called with progress updates
 412
 413         :raises SendPackError: if server rejects the pack data
 414         :raises UpdateRefsError: if the server supports report-status
 415                                  and rejects ref updates
 416         """
 417         proto, unused_can_read = self._connect('receive-pack', path)
 418         old_refs, server_capabilities = self._read_refs(proto)
 419         negotiated_capabilities = list(self._send_capabilities)
 420         if 'report-status' not in server_capabilities:
 421             negotiated_capabilities.remove('report-status')
 422         new_refs = determine_wants(old_refs)
 423         if new_refs is None:
 424             proto.write_pkt_line(None)
 425             return old_refs
 426         (have, want) = self._handle_receive_pack_head(proto,
 427             negotiated_capabilities, old_refs, new_refs)
 428         if not want and old_refs == new_refs:
 429             return new_refs
 430         objects = generate_pack_contents(have, want)
 431         if len(objects) > 0:
 432             entries, sha = write_pack_objects(proto.write_file(), objects)
 433         self._handle_receive_pack_tail(proto, negotiated_capabilities,
 434             progress)
 435         return new_refs
 436
 437     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 438                    progress=None):
 439         """Retrieve a pack from a git smart server.
 440
 441         :param determine_wants: Callback that returns list of commits to fetch
 442         :param graph_walker: Object with next() and ack().
 443         :param pack_data: Callback called for each bit of data in the pack
 444         :param progress: Callback for progress reports (strings)
 445         """
 446         proto, can_read = self._connect('upload-pack', path)
 447         (refs, server_capabilities) = self._read_refs(proto)
 448         negotiated_capabilities = list(self._fetch_capabilities)
 449         wants = determine_wants(refs)
 450         if not wants:
 451             proto.write_pkt_line(None)
 452             return refs
 453         self._handle_upload_pack_head(proto, negotiated_capabilities,
 454             graph_walker, wants, can_read)
 455         self._handle_upload_pack_tail(proto, negotiated_capabilities,
 456             graph_walker, pack_data, progress)
 457         return refs
 458
 459
 460 class TCPGitClient(TraditionalGitClient):
 461     """A Git Client that works over TCP directly (i.e. git://)."""
 462
 463     def __init__(self, host, port=None, *args, **kwargs):
 464         if port is None:
 465             port = TCP_GIT_PORT
 466         self._host = host
 467         self._port = port
 468         GitClient.__init__(self, *args, **kwargs)
 469
 470     def _connect(self, cmd, path):
 471         sockaddrs = socket.getaddrinfo(self._host, self._port,
 472             socket.AF_UNSPEC, socket.SOCK_STREAM)
 473         s = None
 474         err = socket.error("no address found for %s" % self._host)
 475         for (family, socktype, proto, canonname, sockaddr) in sockaddrs:
 476             s = socket.socket(family, socktype, proto)
 477             s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
 478             try:
 479                 s.connect(sockaddr)
 480                 break
 481             except socket.error, err:
 482                 if s is not None:
 483                     s.close()
 484                 s = None
 485         if s is None:
 486             raise err
 487         # -1 means system default buffering
 488         rfile = s.makefile('rb', -1)
 489         # 0 means unbuffered
 490         wfile = s.makefile('wb', 0)
 491         proto = Protocol(rfile.read, wfile.write,
 492                          report_activity=self._report_activity)
 493         if path.startswith("/~"):
 494             path = path[1:]
 495         proto.send_cmd('git-%s' % cmd, path, 'host=%s' % self._host)
 496         return proto, lambda: _fileno_can_read(s)
 497
 498
 499 class SubprocessWrapper(object):
 500     """A socket-like object that talks to a subprocess via pipes."""
 501
 502     def __init__(self, proc):
 503         self.proc = proc
 504         self.read = proc.stdout.read
 505         self.write = proc.stdin.write
 506
 507     def can_read(self):
 508         if subprocess.mswindows:
 509             from msvcrt import get_osfhandle
 510             from win32pipe import PeekNamedPipe
 511             handle = get_osfhandle(self.proc.stdout.fileno())
 512             return PeekNamedPipe(handle, 0)[2] != 0
 513         else:
 514             return _fileno_can_read(self.proc.stdout.fileno())
 515
 516     def close(self):
 517         self.proc.stdin.close()
 518         self.proc.stdout.close()
 519         self.proc.wait()
 520
 521
 522 class SubprocessGitClient(TraditionalGitClient):
 523     """Git client that talks to a server using a subprocess."""
 524
 525     def __init__(self, *args, **kwargs):
 526         self._connection = None
 527         self._stderr = None
 528         self._stderr = kwargs.get('stderr')
 529         if 'stderr' in kwargs:
 530             del kwargs['stderr']
 531         GitClient.__init__(self, *args, **kwargs)
 532
 533     def _connect(self, service, path):
 534         import subprocess
 535         argv = ['git', service, path]
 536         p = SubprocessWrapper(
 537             subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
 538                              stdout=subprocess.PIPE,
 539                              stderr=self._stderr))
 540         return Protocol(p.read, p.write,
 541                         report_activity=self._report_activity), p.can_read
 542
 543
 544 class SSHVendor(object):
 545
 546     def connect_ssh(self, host, command, username=None, port=None):
 547         import subprocess
 548         #FIXME: This has no way to deal with passwords..
 549         args = ['ssh', '-x']
 550         if port is not None:
 551             args.extend(['-p', str(port)])
 552         if username is not None:
 553             host = '%s@%s' % (username, host)
 554         args.append(host)
 555         proc = subprocess.Popen(args + command,
 556                                 stdin=subprocess.PIPE,
 557                                 stdout=subprocess.PIPE)
 558         return SubprocessWrapper(proc)
 559
 560 # Can be overridden by users
 561 get_ssh_vendor = SSHVendor
 562
 563
 564 class SSHGitClient(TraditionalGitClient):
 565
 566     def __init__(self, host, port=None, username=None, *args, **kwargs):
 567         self.host = host
 568         self.port = port
 569         self.username = username
 570         GitClient.__init__(self, *args, **kwargs)
 571         self.alternative_paths = {}
 572
 573     def _get_cmd_path(self, cmd):
 574         return self.alternative_paths.get(cmd, 'git-%s' % cmd)
 575
 576     def _connect(self, cmd, path):
 577         con = get_ssh_vendor().connect_ssh(
 578             self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
 579             port=self.port, username=self.username)
 580         return (Protocol(con.read, con.write, report_activity=self._report_activity),
 581                 con.can_read)
 582
 583
 584 class HttpGitClient(GitClient):
 585
 586     def __init__(self, base_url, dumb=None, *args, **kwargs):
 587         self.base_url = base_url.rstrip("/") + "/"
 588         self.dumb = dumb
 589         GitClient.__init__(self, *args, **kwargs)
 590
 591     def _get_url(self, path):
 592         return urlparse.urljoin(self.base_url, path).rstrip("/") + "/"
 593
 594     def _perform(self, req):
 595         """Perform a HTTP request.
 596
 597         This is provided so subclasses can provide their own version.
 598
 599         :param req: urllib2.Request instance
 600         :return: matching response
 601         """
 602         return urllib2.urlopen(req)
 603
 604     def _discover_references(self, service, url):
 605         assert url[-1] == "/"
 606         url = urlparse.urljoin(url, "info/refs")
 607         headers = {}
 608         if self.dumb != False:
 609             url += "?service=%s" % service
 610             headers["Content-Type"] = "application/x-%s-request" % service
 611         req = urllib2.Request(url, headers=headers)
 612         resp = self._perform(req)
 613         if resp.getcode() == 404:
 614             raise NotGitRepository()
 615         if resp.getcode() != 200:
 616             raise GitProtocolError("unexpected http response %d" %
 617                 resp.getcode())
 618         self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
 619         proto = Protocol(resp.read, None)
 620         if not self.dumb:
 621             # The first line should mention the service
 622             pkts = list(proto.read_pkt_seq())
 623             if pkts != [('# service=%s\n' % service)]:
 624                 raise GitProtocolError(
 625                     "unexpected first line %r from smart server" % pkts)
 626         return self._read_refs(proto)
 627
 628     def _smart_request(self, service, url, data):
 629         assert url[-1] == "/"
 630         url = urlparse.urljoin(url, service)
 631         req = urllib2.Request(url,
 632             headers={"Content-Type": "application/x-%s-request" % service},
 633             data=data)
 634         resp = self._perform(req)
 635         if resp.getcode() == 404:
 636             raise NotGitRepository()
 637         if resp.getcode() != 200:
 638             raise GitProtocolError("Invalid HTTP response from server: %d"
 639                 % resp.getcode())
 640         if resp.info().gettype() != ("application/x-%s-result" % service):
 641             raise GitProtocolError("Invalid content-type from server: %s"
 642                 % resp.info().gettype())
 643         return resp
 644
 645     def send_pack(self, path, determine_wants, generate_pack_contents,
 646                   progress=None):
 647         """Upload a pack to a remote repository.
 648
 649         :param path: Repository path
 650         :param generate_pack_contents: Function that can return a sequence of the
 651             shas of the objects to upload.
 652         :param progress: Optional progress function
 653
 654         :raises SendPackError: if server rejects the pack data
 655         :raises UpdateRefsError: if the server supports report-status
 656                                  and rejects ref updates
 657         """
 658         url = self._get_url(path)
 659         old_refs, server_capabilities = self._discover_references(
 660             "git-receive-pack", url)
 661         negotiated_capabilities = list(self._send_capabilities)
 662         new_refs = determine_wants(old_refs)
 663         if new_refs is None:
 664             return old_refs
 665         if self.dumb:
 666             raise NotImplementedError(self.fetch_pack)
 667         req_data = StringIO()
 668         req_proto = Protocol(None, req_data.write)
 669         (have, want) = self._handle_receive_pack_head(
 670             req_proto, negotiated_capabilities, old_refs, new_refs)
 671         if not want and old_refs == new_refs:
 672             return new_refs
 673         objects = generate_pack_contents(have, want)
 674         if len(objects) > 0:
 675             entries, sha = write_pack_objects(req_proto.write_file(), objects)
 676         resp = self._smart_request("git-receive-pack", url,
 677             data=req_data.getvalue())
 678         resp_proto = Protocol(resp.read, None)
 679         self._handle_receive_pack_tail(resp_proto, negotiated_capabilities,
 680             progress)
 681         return new_refs
 682
 683     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 684                    progress):
 685         """Retrieve a pack from a git smart server.
 686
 687         :param determine_wants: Callback that returns list of commits to fetch
 688         :param graph_walker: Object with next() and ack().
 689         :param pack_data: Callback called for each bit of data in the pack
 690         :param progress: Callback for progress reports (strings)
 691         """
 692         url = self._get_url(path)
 693         refs, server_capabilities = self._discover_references(
 694             "git-upload-pack", url)
 695         negotiated_capabilities = list(server_capabilities)
 696         wants = determine_wants(refs)
 697         if not wants:
 698             return refs
 699         if self.dumb:
 700             raise NotImplementedError(self.send_pack)
 701         req_data = StringIO()
 702         req_proto = Protocol(None, req_data.write)
 703         self._handle_upload_pack_head(req_proto,
 704             negotiated_capabilities, graph_walker, wants,
 705             lambda: False)
 706         resp = self._smart_request("git-upload-pack", url,
 707             data=req_data.getvalue())
 708         resp_proto = Protocol(resp.read, None)
 709         self._handle_upload_pack_tail(resp_proto, negotiated_capabilities,
 710             graph_walker, pack_data, progress)
 711         return refs
 712
 713
 714 def get_transport_and_path(uri):
 715     """Obtain a git client from a URI or path.
 716
 717     :param uri: URI or path
 718     :return: Tuple with client instance and relative path.
 719     """
 720     parsed = urlparse.urlparse(uri)
 721     if parsed.scheme == 'git':
 722         return TCPGitClient(parsed.hostname, port=parsed.port), parsed.path
 723     elif parsed.scheme == 'git+ssh':
 724         return SSHGitClient(parsed.hostname, port=parsed.port,
 725                             username=parsed.username), parsed.path
 726     elif parsed.scheme in ('http', 'https'):
 727         return HttpGitClient(urlparse.urlunparse(parsed)), parsed.path
 728
 729     if parsed.scheme and not parsed.netloc:
 730         # SSH with no user@, zero or one leading slash.
 731         return SSHGitClient(parsed.scheme), parsed.path
 732     elif parsed.scheme:
 733         raise ValueError('Unknown git protocol scheme: %s' % parsed.scheme)
 734     elif '@' in parsed.path and ':' in parsed.path:
 735         # SSH with user@host:foo.
 736         user_host, path = parsed.path.split(':')
 737         user, host = user_host.rsplit('@')
 738         return SSHGitClient(host, username=user), path
 739
 740     # Otherwise, assume it's a local path.
 741     return SubprocessGitClient(), uri