dulwich/client.py

   1 # client.py -- Implementation of the server side git protocols
   2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   3 # Copyright (C) 2008 John Carr
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License
   7 # as published by the Free Software Foundation; either version 2
   8 # or (at your option) a later version of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 # MA  02110-1301, USA.
  19
  20 """Client side support for the Git protocol."""
  21
  22 __docformat__ = 'restructuredText'
  23
  24 import os
  25 import select
  26 import socket
  27 import subprocess
  28
  29 from dulwich.errors import (
  30     ChecksumMismatch,
  31     )
  32 from dulwich.protocol import (
  33     Protocol,
  34     TCP_GIT_PORT,
  35     ZERO_SHA,
  36     extract_capabilities,
  37     )
  38 from dulwich.pack import (
  39     write_pack_data,
  40     )
  41
  42
  43 def _fileno_can_read(fileno):
  44     """Check if a file descriptor is readable."""
  45     return len(select.select([fileno], [], [], 0)[0]) > 0
  46
  47
  48 CAPABILITIES = ["multi_ack", "side-band-64k", "ofs-delta"]
  49
  50
  51 class GitClient(object):
  52     """Git smart server client.
  53
  54     """
  55
  56     def __init__(self, can_read, read, write, thin_packs=True,
  57         report_activity=None):
  58         """Create a new GitClient instance.
  59
  60         :param can_read: Function that returns True if there is data available
  61             to be read.
  62         :param read: Callback for reading data, takes number of bytes to read
  63         :param write: Callback for writing data
  64         :param thin_packs: Whether or not thin packs should be retrieved
  65         :param report_activity: Optional callback for reporting transport
  66             activity.
  67         """
  68         self.proto = Protocol(read, write, report_activity)
  69         self._can_read = can_read
  70         self._capabilities = list(CAPABILITIES)
  71         if thin_packs:
  72             self._capabilities.append("thin-pack")
  73
  74     def capabilities(self):
  75         return " ".join(self._capabilities)
  76
  77     def read_refs(self):
  78         server_capabilities = None
  79         refs = {}
  80         # Receive refs from server
  81         for pkt in self.proto.read_pkt_seq():
  82             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
  83             if server_capabilities is None:
  84                 (ref, server_capabilities) = extract_capabilities(ref)
  85             refs[ref] = sha
  86         return refs, server_capabilities
  87
  88     def send_pack(self, path, determine_wants, generate_pack_contents):
  89         """Upload a pack to a remote repository.
  90
  91         :param path: Repository path
  92         :param generate_pack_contents: Function that can return the shas of the
  93             objects to upload.
  94         """
  95         old_refs, server_capabilities = self.read_refs()
  96         new_refs = determine_wants(old_refs)
  97         if not new_refs:
  98             self.proto.write_pkt_line(None)
  99             return {}
 100         want = []
 101         have = [x for x in old_refs.values() if not x == ZERO_SHA]
 102         sent_capabilities = False
 103         for refname in set(new_refs.keys() + old_refs.keys()):
 104             old_sha1 = old_refs.get(refname, ZERO_SHA)
 105             new_sha1 = new_refs.get(refname, ZERO_SHA)
 106             if old_sha1 != new_sha1:
 107                 if sent_capabilities:
 108                     self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
 109                 else:
 110                     self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
 111                     sent_capabilities = True
 112             if not new_sha1 in (have, ZERO_SHA):
 113                 want.append(new_sha1)
 114         self.proto.write_pkt_line(None)
 115         if not want:
 116             return new_refs
 117         objects = generate_pack_contents(have, want)
 118         (entries, sha) = write_pack_data(self.proto.write_file(), objects,
 119                                          len(objects))
 120
 121         # read the final confirmation sha
 122         client_sha = self.proto.read(20)
 123         if not client_sha in (None, "", sha):
 124             raise ChecksumMismatch(sha, client_sha)
 125
 126         return new_refs
 127
 128     def fetch(self, path, target, determine_wants=None, progress=None):
 129         """Fetch into a target repository.
 130
 131         :param path: Path to fetch from
 132         :param target: Target repository to fetch into
 133         :param determine_wants: Optional function to determine what refs
 134             to fetch
 135         :param progress: Optional progress function
 136         :return: remote refs
 137         """
 138         if determine_wants is None:
 139             determine_wants = target.object_store.determine_wants_all
 140         f, commit = target.object_store.add_pack()
 141         try:
 142             return self.fetch_pack(path, determine_wants,
 143                 target.get_graph_walker(), f.write, progress)
 144         finally:
 145             commit()
 146
 147     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 148                    progress):
 149         """Retrieve a pack from a git smart server.
 150
 151         :param determine_wants: Callback that returns list of commits to fetch
 152         :param graph_walker: Object with next() and ack().
 153         :param pack_data: Callback called for each bit of data in the pack
 154         :param progress: Callback for progress reports (strings)
 155         """
 156         (refs, server_capabilities) = self.read_refs()
 157         wants = determine_wants(refs)
 158         if not wants:
 159             self.proto.write_pkt_line(None)
 160             return refs
 161         assert isinstance(wants, list) and type(wants[0]) == str
 162         self.proto.write_pkt_line("want %s %s\n" % (wants[0], self.capabilities()))
 163         for want in wants[1:]:
 164             self.proto.write_pkt_line("want %s\n" % want)
 165         self.proto.write_pkt_line(None)
 166         have = graph_walker.next()
 167         while have:
 168             self.proto.write_pkt_line("have %s\n" % have)
 169             if self._can_read():
 170                 pkt = self.proto.read_pkt_line()
 171                 parts = pkt.rstrip("\n").split(" ")
 172                 if parts[0] == "ACK":
 173                     graph_walker.ack(parts[1])
 174                     assert parts[2] == "continue"
 175             have = graph_walker.next()
 176         self.proto.write_pkt_line("done\n")
 177         pkt = self.proto.read_pkt_line()
 178         while pkt:
 179             parts = pkt.rstrip("\n").split(" ")
 180             if parts[0] == "ACK":
 181                 graph_walker.ack(pkt.split(" ")[1])
 182             if len(parts) < 3 or parts[2] != "continue":
 183                 break
 184             pkt = self.proto.read_pkt_line()
 185         for pkt in self.proto.read_pkt_seq():
 186             channel = ord(pkt[0])
 187             pkt = pkt[1:]
 188             if channel == 1:
 189                 pack_data(pkt)
 190             elif channel == 2:
 191                 progress(pkt)
 192             else:
 193                 raise AssertionError("Invalid sideband channel %d" % channel)
 194         return refs
 195
 196
 197 class TCPGitClient(GitClient):
 198     """A Git Client that works over TCP directly (i.e. git://)."""
 199
 200     def __init__(self, host, port=None, *args, **kwargs):
 201         self._socket = socket.socket(type=socket.SOCK_STREAM)
 202         if port is None:
 203             port = TCP_GIT_PORT
 204         self._socket.connect((host, port))
 205         self.rfile = self._socket.makefile('rb', -1)
 206         self.wfile = self._socket.makefile('wb', 0)
 207         self.host = host
 208         super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
 209
 210     def send_pack(self, path, changed_refs, generate_pack_contents):
 211         """Send a pack to a remote host.
 212
 213         :param path: Path of the repository on the remote host
 214         """
 215         self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
 216         return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
 217
 218     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
 219         """Fetch a pack from the remote host.
 220
 221         :param path: Path of the reposiutory on the remote host
 222         :param determine_wants: Callback that receives available refs dict and
 223             should return list of sha's to fetch.
 224         :param graph_walker: GraphWalker instance used to find missing shas
 225         :param pack_data: Callback for writing pack data
 226         :param progress: Callback for writing progress
 227         """
 228         self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
 229         return super(TCPGitClient, self).fetch_pack(path, determine_wants,
 230             graph_walker, pack_data, progress)
 231
 232
 233 class SubprocessGitClient(GitClient):
 234     """Git client that talks to a server using a subprocess."""
 235
 236     def __init__(self, *args, **kwargs):
 237         self.proc = None
 238         self._args = args
 239         self._kwargs = kwargs
 240
 241     def _connect(self, service, *args, **kwargs):
 242         argv = [service] + list(args)
 243         self.proc = subprocess.Popen(argv, bufsize=0,
 244                                 stdin=subprocess.PIPE,
 245                                 stdout=subprocess.PIPE)
 246         def read_fn(size):
 247             return self.proc.stdout.read(size)
 248         def write_fn(data):
 249             self.proc.stdin.write(data)
 250             self.proc.stdin.flush()
 251         return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
 252
 253     def send_pack(self, path, changed_refs, generate_pack_contents):
 254         """Upload a pack to the server.
 255
 256         :param path: Path to the git repository on the server
 257         :param changed_refs: Dictionary with new values for the refs
 258         :param generate_pack_contents: Function that returns an iterator over
 259             objects to send
 260         """
 261         client = self._connect("git-receive-pack", path)
 262         return client.send_pack(path, changed_refs, generate_pack_contents)
 263
 264     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 265         progress):
 266         """Retrieve a pack from the server
 267
 268         :param path: Path to the git repository on the server
 269         :param determine_wants: Function that receives existing refs
 270             on the server and returns a list of desired shas
 271         :param graph_walker: GraphWalker instance
 272         :param pack_data: Function that can write pack data
 273         :param progress: Function that can write progress texts
 274         """
 275         client = self._connect("git-upload-pack", path)
 276         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
 277                                  progress)
 278
 279
 280 class SSHSubprocess(object):
 281     """A socket-like object that talks to an ssh subprocess via pipes."""
 282
 283     def __init__(self, proc):
 284         self.proc = proc
 285
 286     def send(self, data):
 287         return os.write(self.proc.stdin.fileno(), data)
 288
 289     def recv(self, count):
 290         return self.proc.stdout.read(count)
 291
 292     def close(self):
 293         self.proc.stdin.close()
 294         self.proc.stdout.close()
 295         self.proc.wait()
 296
 297
 298 class SSHVendor(object):
 299
 300     def connect_ssh(self, host, command, username=None, port=None):
 301         #FIXME: This has no way to deal with passwords..
 302         args = ['ssh', '-x']
 303         if port is not None:
 304             args.extend(['-p', str(port)])
 305         if username is not None:
 306             host = "%s@%s" % (username, host)
 307         args.append(host)
 308         proc = subprocess.Popen(args + command,
 309                                 stdin=subprocess.PIPE,
 310                                 stdout=subprocess.PIPE)
 311         return SSHSubprocess(proc)
 312
 313 # Can be overridden by users
 314 get_ssh_vendor = SSHVendor
 315
 316
 317 class SSHGitClient(GitClient):
 318
 319     def __init__(self, host, port=None, username=None, *args, **kwargs):
 320         self.host = host
 321         self.port = port
 322         self.username = username
 323         self._args = args
 324         self._kwargs = kwargs
 325
 326     def send_pack(self, path, determine_wants, generate_pack_contents):
 327         remote = get_ssh_vendor().connect_ssh(self.host, ["git-receive-pack '%s'" % path], port=self.port, username=self.username)
 328         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
 329         return client.send_pack(path, determine_wants, generate_pack_contents)
 330
 331     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 332         progress):
 333         remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack '%s'" % path], port=self.port, username=self.username)
 334         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
 335         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
 336                                  progress)
 337
 338
 339 def get_transport_and_path(uri):
 340     """Obtain a git client from a URI or path.
 341
 342     :param uri: URI or path
 343     :return: Tuple with client instance and relative path.
 344     """
 345     from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
 346     for handler, transport in (("git://", TCPGitClient), ("git+ssh://", SSHGitClient)):
 347         if uri.startswith(handler):
 348             host, path = uri[len(handler):].split("/", 1)
 349             return transport(host), "/"+path
 350     # if its not git or git+ssh, try a local url..
 351     return SubprocessGitClient(), uri