dulwich/client.py

   1 # client.py -- Implementation of the server side git protocols
   2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   3 # Copyright (C) 2008 John Carr
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License
   7 # as published by the Free Software Foundation; either version 2
   8 # or (at your option) a later version of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 # MA  02110-1301, USA.
  19
  20 """Client side support for the Git protocol."""
  21
  22 __docformat__ = 'restructuredText'
  23
  24 import select
  25 import socket
  26 import subprocess
  27
  28 from dulwich.errors import (
  29     SendPackError,
  30     UpdateRefsError,
  31     )
  32 from dulwich.protocol import (
  33     Protocol,
  34     TCP_GIT_PORT,
  35     ZERO_SHA,
  36     extract_capabilities,
  37     )
  38 from dulwich.pack import (
  39     write_pack_data,
  40     )
  41
  42
  43 def _fileno_can_read(fileno):
  44     """Check if a file descriptor is readable."""
  45     return len(select.select([fileno], [], [], 0)[0]) > 0
  46
  47 COMMON_CAPABILITIES = ["ofs-delta"]
  48 FETCH_CAPABILITIES = ["multi_ack", "side-band-64k"] + COMMON_CAPABILITIES
  49 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
  50
  51 # TODO(durin42): this doesn't correctly degrade if the server doesn't
  52 # support some capabilities. This should work properly with servers
  53 # that don't support side-band-64k and multi_ack.
  54 class GitClient(object):
  55     """Git smart server client.
  56
  57     """
  58
  59     def __init__(self, can_read, read, write, thin_packs=True,
  60         report_activity=None):
  61         """Create a new GitClient instance.
  62
  63         :param can_read: Function that returns True if there is data available
  64             to be read.
  65         :param read: Callback for reading data, takes number of bytes to read
  66         :param write: Callback for writing data
  67         :param thin_packs: Whether or not thin packs should be retrieved
  68         :param report_activity: Optional callback for reporting transport
  69             activity.
  70         """
  71         self.proto = Protocol(read, write, report_activity)
  72         self._can_read = can_read
  73         self._fetch_capabilities = list(FETCH_CAPABILITIES)
  74         self._send_capabilities = list(SEND_CAPABILITIES)
  75         if thin_packs:
  76             self._fetch_capabilities.append("thin-pack")
  77
  78     def read_refs(self):
  79         server_capabilities = None
  80         refs = {}
  81         # Receive refs from server
  82         for pkt in self.proto.read_pkt_seq():
  83             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
  84             if server_capabilities is None:
  85                 (ref, server_capabilities) = extract_capabilities(ref)
  86             refs[ref] = sha
  87         return refs, server_capabilities
  88
  89     def _parse_status_report(self):
  90         unpack = self.proto.read_pkt_line().strip()
  91         if unpack != 'unpack ok':
  92             st = True
  93             # flush remaining error data
  94             while st is not None:
  95                 st = self.proto.read_pkt_line()
  96             raise SendPackError(unpack)
  97         statuses = []
  98         errs = False
  99         ref_status = self.proto.read_pkt_line()
 100         while ref_status:
 101             ref_status = ref_status.strip()
 102             statuses.append(ref_status)
 103             if not ref_status.startswith('ok '):
 104                 errs = True
 105             ref_status = self.proto.read_pkt_line()
 106
 107         if errs:
 108             ref_status = {}
 109             ok = set()
 110             for status in statuses:
 111                 if ' ' not in status:
 112                     # malformed response, move on to the next one
 113                     continue
 114                 status, ref = status.split(' ', 1)
 115
 116                 if status == 'ng':
 117                     if ' ' in ref:
 118                         ref, status = ref.split(' ', 1)
 119                 else:
 120                     ok.add(ref)
 121                 ref_status[ref] = status
 122             raise UpdateRefsError('%s failed to update' %
 123                                   ', '.join([ref for ref in ref_status
 124                                              if ref not in ok]),
 125                                   ref_status=ref_status)
 126
 127
 128     # TODO(durin42): add side-band-64k capability support here and advertise it
 129     def send_pack(self, path, determine_wants, generate_pack_contents):
 130         """Upload a pack to a remote repository.
 131
 132         :param path: Repository path
 133         :param generate_pack_contents: Function that can return the shas of the
 134             objects to upload.
 135
 136         :raises SendPackError: if server rejects the pack data
 137         :raises UpdateRefsError: if the server supports report-status
 138                                  and rejects ref updates
 139         """
 140         old_refs, server_capabilities = self.read_refs()
 141         if 'report-status' not in server_capabilities:
 142             self._send_capabilities.remove('report-status')
 143         new_refs = determine_wants(old_refs)
 144         if not new_refs:
 145             self.proto.write_pkt_line(None)
 146             return {}
 147         want = []
 148         have = [x for x in old_refs.values() if not x == ZERO_SHA]
 149         sent_capabilities = False
 150         for refname in set(new_refs.keys() + old_refs.keys()):
 151             old_sha1 = old_refs.get(refname, ZERO_SHA)
 152             new_sha1 = new_refs.get(refname, ZERO_SHA)
 153             if old_sha1 != new_sha1:
 154                 if sent_capabilities:
 155                     self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1,
 156                                                             refname))
 157                 else:
 158                     self.proto.write_pkt_line(
 159                       "%s %s %s\0%s" % (old_sha1, new_sha1, refname,
 160                                         ' '.join(self._send_capabilities)))
 161                     sent_capabilities = True
 162             if new_sha1 not in have and new_sha1 != ZERO_SHA:
 163                 want.append(new_sha1)
 164         self.proto.write_pkt_line(None)
 165         if not want:
 166             return new_refs
 167         objects = generate_pack_contents(have, want)
 168         (entries, sha) = write_pack_data(self.proto.write_file(), objects,
 169                                          len(objects))
 170
 171         if 'report-status' in self._send_capabilities:
 172             self._parse_status_report()
 173         # wait for EOF before returning
 174         data = self.proto.read()
 175         if data:
 176             raise SendPackError('Unexpected response %r' % data)
 177         return new_refs
 178
 179     def fetch(self, path, target, determine_wants=None, progress=None):
 180         """Fetch into a target repository.
 181
 182         :param path: Path to fetch from
 183         :param target: Target repository to fetch into
 184         :param determine_wants: Optional function to determine what refs
 185             to fetch
 186         :param progress: Optional progress function
 187         :return: remote refs
 188         """
 189         if determine_wants is None:
 190             determine_wants = target.object_store.determine_wants_all
 191         f, commit = target.object_store.add_pack()
 192         try:
 193             return self.fetch_pack(path, determine_wants,
 194                 target.get_graph_walker(), f.write, progress)
 195         finally:
 196             commit()
 197
 198     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 199                    progress):
 200         """Retrieve a pack from a git smart server.
 201
 202         :param determine_wants: Callback that returns list of commits to fetch
 203         :param graph_walker: Object with next() and ack().
 204         :param pack_data: Callback called for each bit of data in the pack
 205         :param progress: Callback for progress reports (strings)
 206         """
 207         (refs, server_capabilities) = self.read_refs()
 208         wants = determine_wants(refs)
 209         if not wants:
 210             self.proto.write_pkt_line(None)
 211             return refs
 212         assert isinstance(wants, list) and type(wants[0]) == str
 213         self.proto.write_pkt_line("want %s %s\n" % (
 214             wants[0], ' '.join(self._fetch_capabilities)))
 215         for want in wants[1:]:
 216             self.proto.write_pkt_line("want %s\n" % want)
 217         self.proto.write_pkt_line(None)
 218         have = graph_walker.next()
 219         while have:
 220             self.proto.write_pkt_line("have %s\n" % have)
 221             if self._can_read():
 222                 pkt = self.proto.read_pkt_line()
 223                 parts = pkt.rstrip("\n").split(" ")
 224                 if parts[0] == "ACK":
 225                     graph_walker.ack(parts[1])
 226                     assert parts[2] == "continue"
 227             have = graph_walker.next()
 228         self.proto.write_pkt_line("done\n")
 229         pkt = self.proto.read_pkt_line()
 230         while pkt:
 231             parts = pkt.rstrip("\n").split(" ")
 232             if parts[0] == "ACK":
 233                 graph_walker.ack(pkt.split(" ")[1])
 234             if len(parts) < 3 or parts[2] != "continue":
 235                 break
 236             pkt = self.proto.read_pkt_line()
 237         # TODO(durin42): this is broken if the server didn't support the
 238         # side-band-64k capability.
 239         for pkt in self.proto.read_pkt_seq():
 240             channel = ord(pkt[0])
 241             pkt = pkt[1:]
 242             if channel == 1:
 243                 pack_data(pkt)
 244             elif channel == 2:
 245                 if progress is not None:
 246                     progress(pkt)
 247             else:
 248                 raise AssertionError("Invalid sideband channel %d" % channel)
 249         return refs
 250
 251
 252 class TCPGitClient(GitClient):
 253     """A Git Client that works over TCP directly (i.e. git://)."""
 254
 255     def __init__(self, host, port=None, *args, **kwargs):
 256         self._socket = socket.socket(type=socket.SOCK_STREAM)
 257         if port is None:
 258             port = TCP_GIT_PORT
 259         self._socket.connect((host, port))
 260         self.rfile = self._socket.makefile('rb', -1)
 261         self.wfile = self._socket.makefile('wb', 0)
 262         self.host = host
 263         super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
 264
 265     def send_pack(self, path, changed_refs, generate_pack_contents):
 266         """Send a pack to a remote host.
 267
 268         :param path: Path of the repository on the remote host
 269         """
 270         self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
 271         return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
 272
 273     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
 274         """Fetch a pack from the remote host.
 275
 276         :param path: Path of the reposiutory on the remote host
 277         :param determine_wants: Callback that receives available refs dict and
 278             should return list of sha's to fetch.
 279         :param graph_walker: GraphWalker instance used to find missing shas
 280         :param pack_data: Callback for writing pack data
 281         :param progress: Callback for writing progress
 282         """
 283         self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
 284         return super(TCPGitClient, self).fetch_pack(path, determine_wants,
 285             graph_walker, pack_data, progress)
 286
 287
 288 class SubprocessGitClient(GitClient):
 289     """Git client that talks to a server using a subprocess."""
 290
 291     def __init__(self, *args, **kwargs):
 292         self.proc = None
 293         self._args = args
 294         self._kwargs = kwargs
 295
 296     def _connect(self, service, *args, **kwargs):
 297         argv = [service] + list(args)
 298         self.proc = subprocess.Popen(argv, bufsize=0,
 299                                 stdin=subprocess.PIPE,
 300                                 stdout=subprocess.PIPE)
 301         def read_fn(size):
 302             return self.proc.stdout.read(size)
 303         def write_fn(data):
 304             self.proc.stdin.write(data)
 305             self.proc.stdin.flush()
 306         return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
 307
 308     def send_pack(self, path, changed_refs, generate_pack_contents):
 309         """Upload a pack to the server.
 310
 311         :param path: Path to the git repository on the server
 312         :param changed_refs: Dictionary with new values for the refs
 313         :param generate_pack_contents: Function that returns an iterator over
 314             objects to send
 315         """
 316         client = self._connect("git-receive-pack", path)
 317         return client.send_pack(path, changed_refs, generate_pack_contents)
 318
 319     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 320         progress):
 321         """Retrieve a pack from the server
 322
 323         :param path: Path to the git repository on the server
 324         :param determine_wants: Function that receives existing refs
 325             on the server and returns a list of desired shas
 326         :param graph_walker: GraphWalker instance
 327         :param pack_data: Function that can write pack data
 328         :param progress: Function that can write progress texts
 329         """
 330         client = self._connect("git-upload-pack", path)
 331         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
 332                                  progress)
 333
 334
 335 class SSHSubprocess(object):
 336     """A socket-like object that talks to an ssh subprocess via pipes."""
 337
 338     def __init__(self, proc):
 339         self.proc = proc
 340         self.read = self.recv = proc.stdout.read
 341         self.write = self.send = proc.stdin.write
 342
 343     def close(self):
 344         self.proc.stdin.close()
 345         self.proc.stdout.close()
 346         self.proc.wait()
 347
 348
 349 class SSHVendor(object):
 350
 351     def connect_ssh(self, host, command, username=None, port=None):
 352         #FIXME: This has no way to deal with passwords..
 353         args = ['ssh', '-x']
 354         if port is not None:
 355             args.extend(['-p', str(port)])
 356         if username is not None:
 357             host = "%s@%s" % (username, host)
 358         args.append(host)
 359         proc = subprocess.Popen(args + command,
 360                                 stdin=subprocess.PIPE,
 361                                 stdout=subprocess.PIPE)
 362         return SSHSubprocess(proc)
 363
 364 # Can be overridden by users
 365 get_ssh_vendor = SSHVendor
 366
 367
 368 class SSHGitClient(GitClient):
 369
 370     def __init__(self, host, port=None, username=None, *args, **kwargs):
 371         self.host = host
 372         self.port = port
 373         self.username = username
 374         self.receive_pack_path = "git-receive-pack"
 375         self.upload_pack_path = "git-upload-pack"
 376         self._args = args
 377         self._kwargs = kwargs
 378
 379     def send_pack(self, path, determine_wants, generate_pack_contents):
 380         remote = get_ssh_vendor().connect_ssh(
 381             self.host, ["'%s' '%s'" % (self.receive_pack_path, path)],
 382             port=self.port, username=self.username)
 383         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
 384         return client.send_pack(path, determine_wants, generate_pack_contents)
 385
 386     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 387         progress):
 388         remote = get_ssh_vendor().connect_ssh(
 389             self.host, ["'%s' '%s'" % (self.upload_pack_path, path)],
 390             port=self.port, username=self.username)
 391         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
 392         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
 393                                  progress)
 394
 395
 396 def get_transport_and_path(uri):
 397     """Obtain a git client from a URI or path.
 398
 399     :param uri: URI or path
 400     :return: Tuple with client instance and relative path.
 401     """
 402     from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
 403     for handler, transport in (("git://", TCPGitClient), ("git+ssh://", SSHGitClient)):
 404         if uri.startswith(handler):
 405             host, path = uri[len(handler):].split("/", 1)
 406             return transport(host), "/"+path
 407     # FIXME: Parse rsync-like git URLs (user@host:/path), bug 568493
 408     # if its not git or git+ssh, try a local url..
 409     return SubprocessGitClient(), uri