dulwich/client.py

   1 # client.py -- Implementation of the server side git protocols
   2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   3 # Copyright (C) 2008 John Carr
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License
   7 # as published by the Free Software Foundation; either version 2
   8 # or (at your option) a later version of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 # MA  02110-1301, USA.
  19
  20 """Client side support for the Git protocol."""
  21
  22 __docformat__ = 'restructuredText'
  23
  24 import select
  25 import socket
  26 import subprocess
  27
  28 from dulwich.errors import (
  29     SendPackError,
  30     UpdateRefsError,
  31     )
  32 from dulwich.protocol import (
  33     Protocol,
  34     TCP_GIT_PORT,
  35     ZERO_SHA,
  36     extract_capabilities,
  37     )
  38 from dulwich.pack import (
  39     write_pack_data,
  40     )
  41
  42
  43 def _fileno_can_read(fileno):
  44     """Check if a file descriptor is readable."""
  45     return len(select.select([fileno], [], [], 0)[0]) > 0
  46
  47 COMMON_CAPABILITIES = ["ofs-delta"]
  48 FETCH_CAPABILITIES = ["multi_ack", "side-band-64k"] + COMMON_CAPABILITIES
  49 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
  50
  51 # TODO(durin42): this doesn't correctly degrade if the server doesn't
  52 # support some capabilities. This should work properly with servers
  53 # that don't support side-band-64k and multi_ack.
  54 class GitClient(object):
  55     """Git smart server client.
  56
  57     """
  58
  59     def __init__(self, thin_packs=True, report_activity=None):
  60         """Create a new GitClient instance.
  61
  62         :param thin_packs: Whether or not thin packs should be retrieved
  63         :param report_activity: Optional callback for reporting transport
  64             activity.
  65         """
  66         self._report_activity = report_activity
  67         self._fetch_capabilities = list(FETCH_CAPABILITIES)
  68         self._send_capabilities = list(SEND_CAPABILITIES)
  69         if thin_packs:
  70             self._fetch_capabilities.append("thin-pack")
  71
  72     def _connect(self, cmd, path):
  73         """Create a connection to the server.
  74
  75         This method is abstract - concrete implementations should
  76         implement their own variant which connects to the server and
  77         returns an initialized Protocol object with the service ready
  78         for use and a can_read function which may be used to see if
  79         reads would block.
  80
  81         :param cmd: The git service name to which we should connect.
  82         :param path: The path we should pass to the service.
  83         """
  84         raise NotImplementedError()
  85
  86     def read_refs(self, proto):
  87         server_capabilities = None
  88         refs = {}
  89         # Receive refs from server
  90         for pkt in proto.read_pkt_seq():
  91             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
  92             if server_capabilities is None:
  93                 (ref, server_capabilities) = extract_capabilities(ref)
  94             refs[ref] = sha
  95         return refs, server_capabilities
  96
  97     def _parse_status_report(self, proto):
  98         unpack = proto.read_pkt_line().strip()
  99         if unpack != 'unpack ok':
 100             st = True
 101             # flush remaining error data
 102             while st is not None:
 103                 st = proto.read_pkt_line()
 104             raise SendPackError(unpack)
 105         statuses = []
 106         errs = False
 107         ref_status = proto.read_pkt_line()
 108         while ref_status:
 109             ref_status = ref_status.strip()
 110             statuses.append(ref_status)
 111             if not ref_status.startswith('ok '):
 112                 errs = True
 113             ref_status = proto.read_pkt_line()
 114
 115         if errs:
 116             ref_status = {}
 117             ok = set()
 118             for status in statuses:
 119                 if ' ' not in status:
 120                     # malformed response, move on to the next one
 121                     continue
 122                 status, ref = status.split(' ', 1)
 123
 124                 if status == 'ng':
 125                     if ' ' in ref:
 126                         ref, status = ref.split(' ', 1)
 127                 else:
 128                     ok.add(ref)
 129                 ref_status[ref] = status
 130             raise UpdateRefsError('%s failed to update' %
 131                                   ', '.join([ref for ref in ref_status
 132                                              if ref not in ok]),
 133                                   ref_status=ref_status)
 134
 135
 136     # TODO(durin42): add side-band-64k capability support here and advertise it
 137     def send_pack(self, path, determine_wants, generate_pack_contents):
 138         """Upload a pack to a remote repository.
 139
 140         :param path: Repository path
 141         :param generate_pack_contents: Function that can return the shas of the
 142             objects to upload.
 143
 144         :raises SendPackError: if server rejects the pack data
 145         :raises UpdateRefsError: if the server supports report-status
 146                                  and rejects ref updates
 147         """
 148         proto, unused_can_read = self._connect('receive-pack', path)
 149         old_refs, server_capabilities = self.read_refs(proto)
 150         if 'report-status' not in server_capabilities:
 151             self._send_capabilities.remove('report-status')
 152         new_refs = determine_wants(old_refs)
 153         if not new_refs:
 154             proto.write_pkt_line(None)
 155             return {}
 156         want = []
 157         have = [x for x in old_refs.values() if not x == ZERO_SHA]
 158         sent_capabilities = False
 159         for refname in set(new_refs.keys() + old_refs.keys()):
 160             old_sha1 = old_refs.get(refname, ZERO_SHA)
 161             new_sha1 = new_refs.get(refname, ZERO_SHA)
 162             if old_sha1 != new_sha1:
 163                 if sent_capabilities:
 164                     proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1,
 165                                                             refname))
 166                 else:
 167                     proto.write_pkt_line(
 168                       "%s %s %s\0%s" % (old_sha1, new_sha1, refname,
 169                                         ' '.join(self._send_capabilities)))
 170                     sent_capabilities = True
 171             if new_sha1 not in have and new_sha1 != ZERO_SHA:
 172                 want.append(new_sha1)
 173         proto.write_pkt_line(None)
 174         if not want:
 175             return new_refs
 176         objects = generate_pack_contents(have, want)
 177         entries, sha = write_pack_data(proto.write_file(), objects,
 178                                        len(objects))
 179
 180         if 'report-status' in self._send_capabilities:
 181             self._parse_status_report(proto)
 182         # wait for EOF before returning
 183         data = proto.read()
 184         if data:
 185             raise SendPackError('Unexpected response %r' % data)
 186         return new_refs
 187
 188     def fetch(self, path, target, determine_wants=None, progress=None):
 189         """Fetch into a target repository.
 190
 191         :param path: Path to fetch from
 192         :param target: Target repository to fetch into
 193         :param determine_wants: Optional function to determine what refs
 194             to fetch
 195         :param progress: Optional progress function
 196         :return: remote refs
 197         """
 198         if determine_wants is None:
 199             determine_wants = target.object_store.determine_wants_all
 200         f, commit = target.object_store.add_pack()
 201         try:
 202             return self.fetch_pack(path, determine_wants,
 203                 target.get_graph_walker(), f.write, progress)
 204         finally:
 205             commit()
 206
 207     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 208                    progress):
 209         """Retrieve a pack from a git smart server.
 210
 211         :param determine_wants: Callback that returns list of commits to fetch
 212         :param graph_walker: Object with next() and ack().
 213         :param pack_data: Callback called for each bit of data in the pack
 214         :param progress: Callback for progress reports (strings)
 215         """
 216         proto, can_read = self._connect('upload-pack', path)
 217         (refs, server_capabilities) = self.read_refs(proto)
 218         wants = determine_wants(refs)
 219         if not wants:
 220             proto.write_pkt_line(None)
 221             return refs
 222         assert isinstance(wants, list) and type(wants[0]) == str
 223         proto.write_pkt_line("want %s %s\n" % (
 224             wants[0], ' '.join(self._fetch_capabilities)))
 225         for want in wants[1:]:
 226             proto.write_pkt_line("want %s\n" % want)
 227         proto.write_pkt_line(None)
 228         have = graph_walker.next()
 229         while have:
 230             proto.write_pkt_line("have %s\n" % have)
 231             if can_read():
 232                 pkt = proto.read_pkt_line()
 233                 parts = pkt.rstrip("\n").split(" ")
 234                 if parts[0] == "ACK":
 235                     graph_walker.ack(parts[1])
 236                     assert parts[2] == "continue"
 237             have = graph_walker.next()
 238         proto.write_pkt_line("done\n")
 239         pkt = proto.read_pkt_line()
 240         while pkt:
 241             parts = pkt.rstrip("\n").split(" ")
 242             if parts[0] == "ACK":
 243                 graph_walker.ack(pkt.split(" ")[1])
 244             if len(parts) < 3 or parts[2] != "continue":
 245                 break
 246             pkt = proto.read_pkt_line()
 247         # TODO(durin42): this is broken if the server didn't support the
 248         # side-band-64k capability.
 249         for pkt in proto.read_pkt_seq():
 250             channel = ord(pkt[0])
 251             pkt = pkt[1:]
 252             if channel == 1:
 253                 pack_data(pkt)
 254             elif channel == 2:
 255                 if progress is not None:
 256                     progress(pkt)
 257             else:
 258                 raise AssertionError("Invalid sideband channel %d" % channel)
 259         return refs
 260
 261
 262 def can_read(f):
 263     """Check if a file descriptor is readable.
 264
 265     :args f: either the number of the file descriptor or a file-like
 266              object which returns the fileno when f.fileno() is called.
 267     """
 268     return len(select.select([f], [], [], 0)[0]) > 0
 269
 270
 271 class TCPGitClient(GitClient):
 272     """A Git Client that works over TCP directly (i.e. git://)."""
 273
 274     def __init__(self, host, port=None, *args, **kwargs):
 275         if port is None:
 276             port = TCP_GIT_PORT
 277         self._host = host
 278         self._port = port
 279         GitClient.__init__(self, *args, **kwargs)
 280
 281     def _connect(self, cmd, path):
 282         s = socket.socket(type=socket.SOCK_STREAM)
 283         s.connect((self._host, self._port))
 284         # -1 means system default buffering
 285         rfile = s.makefile('rb', -1)
 286         # 0 means unbuffered
 287         wfile = s.makefile('wb', 0)
 288         proto = Protocol(rfile.read, wfile.write,
 289                          report_activity=self._report_activity)
 290         proto.send_cmd('git-%s' % cmd, path, 'host=%s' % self._host)
 291         return proto, lambda: can_read(s)
 292
 293
 294 class SubprocessWrapper(object):
 295     """A socket-like object that talks to a subprocess via pipes."""
 296
 297     def __init__(self, proc):
 298         self.proc = proc
 299         self.read = proc.stdout.read
 300         self.write = proc.stdin.write
 301
 302     def can_read(self):
 303         return can_read(self.proc.stdout.fileno())
 304
 305     def close(self):
 306         self.proc.stdin.close()
 307         self.proc.stdout.close()
 308         self.proc.wait()
 309
 310
 311 class SubprocessGitClient(GitClient):
 312     """Git client that talks to a server using a subprocess."""
 313
 314     def __init__(self, *args, **kwargs):
 315         self._connection = None
 316         GitClient.__init__(self, *args, **kwargs)
 317
 318     def _connect(self, service, path):
 319         argv = ['git', service, path]
 320         p = SubprocessWrapper(
 321             subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
 322                              stdout=subprocess.PIPE))
 323         return Protocol(p.read, p.write,
 324                         report_activity=self._report_activity), p.can_read
 325
 326 class SSHVendor(object):
 327
 328     def connect_ssh(self, host, command, username=None, port=None):
 329         #FIXME: This has no way to deal with passwords..
 330         args = ['ssh', '-x']
 331         if port is not None:
 332             args.extend(['-p', str(port)])
 333         if username is not None:
 334             host = "%s@%s" % (username, host)
 335         args.append(host)
 336         proc = subprocess.Popen(args + command,
 337                                 stdin=subprocess.PIPE,
 338                                 stdout=subprocess.PIPE)
 339         return SubprocessWrapper(proc)
 340
 341 # Can be overridden by users
 342 get_ssh_vendor = SSHVendor
 343
 344
 345 class SSHGitClient(GitClient):
 346
 347     def __init__(self, host, port=None, username=None, *args, **kwargs):
 348         self.host = host
 349         self.port = port
 350         self.username = username
 351         GitClient.__init__(self, *args, **kwargs)
 352
 353     def _connect(self, cmd, path):
 354         con = get_ssh_vendor().connect_ssh(
 355             self.host, ["%s '%s'" % ('git-' + cmd, path)],
 356             port=self.port, username=self.username)
 357         return Protocol(con.read, con.write), con.can_read
 358
 359
 360 def get_transport_and_path(uri):
 361     """Obtain a git client from a URI or path.
 362
 363     :param uri: URI or path
 364     :return: Tuple with client instance and relative path.
 365     """
 366     from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
 367     for handler, transport in (("git://", TCPGitClient), ("git+ssh://", SSHGitClient)):
 368         if uri.startswith(handler):
 369             host, path = uri[len(handler):].split("/", 1)
 370             return transport(host), "/"+path
 371     # FIXME: Parse rsync-like git URLs (user@host:/path), bug 568493
 372     # if its not git or git+ssh, try a local url..
 373     return SubprocessGitClient(), uri