dulwich/client.py

   1 # client.py -- Implementation of the server side git protocols
   2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
   3 # Copyright (C) 2008 John Carr
   4 #
   5 # This program is free software; you can redistribute it and/or
   6 # modify it under the terms of the GNU General Public License
   7 # as published by the Free Software Foundation; either version 2
   8 # or (at your option) a later version of the License.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program; if not, write to the Free Software
  17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  18 # MA  02110-1301, USA.
  19
  20 """Client side support for the Git protocol."""
  21
  22 __docformat__ = 'restructuredText'
  23
  24 import os
  25 import select
  26 import socket
  27 import subprocess
  28
  29 from dulwich.errors import (
  30     ChecksumMismatch,
  31     )
  32 from dulwich.protocol import (
  33     Protocol,
  34     TCP_GIT_PORT,
  35     extract_capabilities,
  36     )
  37 from dulwich.pack import (
  38     write_pack_data,
  39     )
  40
  41
  42 def _fileno_can_read(fileno):
  43     """Check if a file descriptor is readable."""
  44     return len(select.select([fileno], [], [], 0)[0]) > 0
  45
  46
  47 CAPABILITIES = ["multi_ack", "side-band-64k", "ofs-delta"]
  48
  49
  50 class GitClient(object):
  51     """Git smart server client.
  52
  53     """
  54
  55     def __init__(self, can_read, read, write, thin_packs=True,
  56         report_activity=None):
  57         """Create a new GitClient instance.
  58
  59         :param can_read: Function that returns True if there is data available
  60             to be read.
  61         :param read: Callback for reading data, takes number of bytes to read
  62         :param write: Callback for writing data
  63         :param thin_packs: Whether or not thin packs should be retrieved
  64         :param report_activity: Optional callback for reporting transport
  65             activity.
  66         """
  67         self.proto = Protocol(read, write, report_activity)
  68         self._can_read = can_read
  69         self._capabilities = list(CAPABILITIES)
  70         if thin_packs:
  71             self._capabilities.append("thin-pack")
  72
  73     def capabilities(self):
  74         return " ".join(self._capabilities)
  75
  76     def read_refs(self):
  77         server_capabilities = None
  78         refs = {}
  79         # Receive refs from server
  80         for pkt in self.proto.read_pkt_seq():
  81             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
  82             if server_capabilities is None:
  83                 (ref, server_capabilities) = extract_capabilities(ref)
  84             refs[ref] = sha
  85         return refs, server_capabilities
  86
  87     def send_pack(self, path, determine_wants, generate_pack_contents):
  88         """Upload a pack to a remote repository.
  89
  90         :param path: Repository path
  91         :param generate_pack_contents: Function that can return the shas of the
  92             objects to upload.
  93         """
  94         refs, server_capabilities = self.read_refs()
  95         changed_refs = determine_wants(refs)
  96         if not changed_refs:
  97             self.proto.write_pkt_line(None)
  98             return {}
  99         want = []
 100         have = [x for x in refs.values() if not x == "0" * 40]
 101         sent_capabilities = False
 102         for changed_ref, new_sha1 in changed_refs.iteritems():
 103             old_sha1 = refs.get(changed_ref, "0" * 40)
 104             if sent_capabilities:
 105                 self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, changed_ref))
 106             else:
 107                 self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, changed_ref, self.capabilities()))
 108                 sent_capabilities = True
 109             if not new_sha1 in have:
 110                 want.append(new_sha1)
 111         self.proto.write_pkt_line(None)
 112         objects = generate_pack_contents(have, want)
 113         (entries, sha) = write_pack_data(self.proto.write_file(), objects,
 114                                          len(objects))
 115         self.proto.write(sha)
 116
 117         # read the final confirmation sha
 118         client_sha = self.proto.read(20)
 119         if not client_sha in (None, "", sha):
 120             raise ChecksumMismatch(sha, client_sha)
 121
 122         return changed_refs
 123
 124     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 125                    progress):
 126         """Retrieve a pack from a git smart server.
 127
 128         :param determine_wants: Callback that returns list of commits to fetch
 129         :param graph_walker: Object with next() and ack().
 130         :param pack_data: Callback called for each bit of data in the pack
 131         :param progress: Callback for progress reports (strings)
 132         """
 133         (refs, server_capabilities) = self.read_refs()
 134         wants = determine_wants(refs)
 135         if not wants:
 136             self.proto.write_pkt_line(None)
 137             return refs
 138         self.proto.write_pkt_line("want %s %s\n" % (wants[0], self.capabilities()))
 139         for want in wants[1:]:
 140             self.proto.write_pkt_line("want %s\n" % want)
 141         self.proto.write_pkt_line(None)
 142         have = graph_walker.next()
 143         while have:
 144             self.proto.write_pkt_line("have %s\n" % have)
 145             if self._can_read():
 146                 pkt = self.proto.read_pkt_line()
 147                 parts = pkt.rstrip("\n").split(" ")
 148                 if parts[0] == "ACK":
 149                     graph_walker.ack(parts[1])
 150                     assert parts[2] == "continue"
 151             have = graph_walker.next()
 152         self.proto.write_pkt_line("done\n")
 153         pkt = self.proto.read_pkt_line()
 154         while pkt:
 155             parts = pkt.rstrip("\n").split(" ")
 156             if parts[0] == "ACK":
 157                 graph_walker.ack(pkt.split(" ")[1])
 158             if len(parts) < 3 or parts[2] != "continue":
 159                 break
 160             pkt = self.proto.read_pkt_line()
 161         for pkt in self.proto.read_pkt_seq():
 162             channel = ord(pkt[0])
 163             pkt = pkt[1:]
 164             if channel == 1:
 165                 pack_data(pkt)
 166             elif channel == 2:
 167                 progress(pkt)
 168             else:
 169                 raise AssertionError("Invalid sideband channel %d" % channel)
 170         return refs
 171
 172
 173 class TCPGitClient(GitClient):
 174     """A Git Client that works over TCP directly (i.e. git://)."""
 175
 176     def __init__(self, host, port=None, *args, **kwargs):
 177         self._socket = socket.socket(type=socket.SOCK_STREAM)
 178         if port is None:
 179             port = TCP_GIT_PORT
 180         self._socket.connect((host, port))
 181         self.rfile = self._socket.makefile('rb', -1)
 182         self.wfile = self._socket.makefile('wb', 0)
 183         self.host = host
 184         super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
 185
 186     def send_pack(self, path, changed_refs, generate_pack_contents):
 187         """Send a pack to a remote host.
 188
 189         :param path: Path of the repository on the remote host
 190         """
 191         self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
 192         return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
 193
 194     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
 195         """Fetch a pack from the remote host.
 196
 197         :param path: Path of the reposiutory on the remote host
 198         :param determine_wants: Callback that receives available refs dict and
 199             should return list of sha's to fetch.
 200         :param graph_walker: GraphWalker instance used to find missing shas
 201         :param pack_data: Callback for writing pack data
 202         :param progress: Callback for writing progress
 203         """
 204         self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
 205         return super(TCPGitClient, self).fetch_pack(path, determine_wants,
 206             graph_walker, pack_data, progress)
 207
 208
 209 class SubprocessGitClient(GitClient):
 210     """Git client that talks to a server using a subprocess."""
 211
 212     def __init__(self, *args, **kwargs):
 213         self.proc = None
 214         self._args = args
 215         self._kwargs = kwargs
 216
 217     def _connect(self, service, *args, **kwargs):
 218         argv = [service] + list(args)
 219         self.proc = subprocess.Popen(argv, bufsize=0,
 220                                 stdin=subprocess.PIPE,
 221                                 stdout=subprocess.PIPE)
 222         def read_fn(size):
 223             return self.proc.stdout.read(size)
 224         def write_fn(data):
 225             self.proc.stdin.write(data)
 226             self.proc.stdin.flush()
 227         return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
 228
 229     def send_pack(self, path, changed_refs, generate_pack_contents):
 230         """Upload a pack to the server.
 231
 232         :param path: Path to the git repository on the server
 233         :param changed_refs: Dictionary with new values for the refs
 234         :param generate_pack_contents: Function that returns an iterator over
 235             objects to send
 236         """
 237         client = self._connect("git-receive-pack", path)
 238         return client.send_pack(path, changed_refs, generate_pack_contents)
 239
 240     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 241         progress):
 242         """Retrieve a pack from the server
 243
 244         :param path: Path to the git repository on the server
 245         :param determine_wants: Function that receives existing refs
 246             on the server and returns a list of desired shas
 247         :param graph_walker: GraphWalker instance
 248         :param pack_data: Function that can write pack data
 249         :param progress: Function that can write progress texts
 250         """
 251         client = self._connect("git-upload-pack", path)
 252         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
 253                                  progress)
 254
 255
 256 class SSHSubprocess(object):
 257     """A socket-like object that talks to an ssh subprocess via pipes."""
 258
 259     def __init__(self, proc):
 260         self.proc = proc
 261
 262     def send(self, data):
 263         return os.write(self.proc.stdin.fileno(), data)
 264
 265     def recv(self, count):
 266         return self.proc.stdout.read(count)
 267
 268     def close(self):
 269         self.proc.stdin.close()
 270         self.proc.stdout.close()
 271         self.proc.wait()
 272
 273
 274 class SSHVendor(object):
 275
 276     def connect_ssh(self, host, command, username=None, port=None):
 277         #FIXME: This has no way to deal with passwords..
 278         args = ['ssh', '-x']
 279         if port is not None:
 280             args.extend(['-p', str(port)])
 281         if username is not None:
 282             host = "%s@%s" % (username, host)
 283         args.append(host)
 284         proc = subprocess.Popen(args + command,
 285                                 stdin=subprocess.PIPE,
 286                                 stdout=subprocess.PIPE)
 287         return SSHSubprocess(proc)
 288
 289 # Can be overridden by users
 290 get_ssh_vendor = SSHVendor
 291
 292
 293 class SSHGitClient(GitClient):
 294
 295     def __init__(self, host, port=None, *args, **kwargs):
 296         self.host = host
 297         self.port = port
 298         self._args = args
 299         self._kwargs = kwargs
 300
 301     def send_pack(self, path, determine_wants, generate_pack_contents):
 302         remote = get_ssh_vendor().connect_ssh(self.host, ["git-receive-pack %s" % path], port=self.port)
 303         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
 304         return client.send_pack(path, determine_wants, generate_pack_contents)
 305
 306     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
 307         progress):
 308         remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack %s" % path], port=self.port)
 309         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
 310         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
 311                                  progress)
 312