1 # client.py -- Implementation of the server side git protocols
2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
3 # Copyright (C) 2008 John Carr
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # or (at your option) a later version of the License.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
20 """Client side support for the Git protocol."""
22 __docformat__ = 'restructuredText'
29 from dulwich.errors import (
34 from dulwich.protocol import (
40 from dulwich.pack import (
45 def _fileno_can_read(fileno):
46 """Check if a file descriptor is readable."""
47 return len(select.select([fileno], [], [], 0)[0]) > 0
49 COMMON_CAPABILITIES = ["ofs-delta"]
50 FETCH_CAPABILITIES = ["multi_ack", "side-band-64k"] + COMMON_CAPABILITIES
51 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
53 # TODO(durin42): this doesn't correctly degrade if the server doesn't
54 # support some capabilities. This should work properly with servers
55 # that don't support side-band-64k and multi_ack.
56 class GitClient(object):
57 """Git smart server client.
61 def __init__(self, can_read, read, write, thin_packs=True,
62 report_activity=None):
63 """Create a new GitClient instance.
65 :param can_read: Function that returns True if there is data available
67 :param read: Callback for reading data, takes number of bytes to read
68 :param write: Callback for writing data
69 :param thin_packs: Whether or not thin packs should be retrieved
70 :param report_activity: Optional callback for reporting transport
73 self.proto = Protocol(read, write, report_activity)
74 self._can_read = can_read
75 self._fetch_capabilities = list(FETCH_CAPABILITIES)
76 self._send_capabilities = list(SEND_CAPABILITIES)
78 self._fetch_capabilities.append("thin-pack")
81 server_capabilities = None
83 # Receive refs from server
84 for pkt in self.proto.read_pkt_seq():
85 (sha, ref) = pkt.rstrip("\n").split(" ", 1)
86 if server_capabilities is None:
87 (ref, server_capabilities) = extract_capabilities(ref)
89 return refs, server_capabilities
91 def _parse_status_report(self):
92 unpack = self.proto.read_pkt_line().strip()
93 if unpack != 'unpack ok':
95 # flush remaining error data
97 st = self.proto.read_pkt_line()
98 raise SendPackError(unpack)
101 ref_status = self.proto.read_pkt_line()
103 ref_status = ref_status.strip()
104 statuses.append(ref_status)
105 if not ref_status.startswith('ok '):
107 ref_status = self.proto.read_pkt_line()
112 for status in statuses:
113 if ' ' not in status:
114 # malformed response, move on to the next one
116 status, ref = status.split(' ', 1)
120 ref, status = ref.split(' ', 1)
123 ref_status[ref] = status
124 raise UpdateRefsError('%s failed to update' %
125 ', '.join([ref for ref in ref_status
127 ref_status=ref_status)
130 # TODO(durin42): add side-band-64k capability support here and advertise it
131 def send_pack(self, path, determine_wants, generate_pack_contents):
132 """Upload a pack to a remote repository.
134 :param path: Repository path
135 :param generate_pack_contents: Function that can return the shas of the
138 :raises SendPackError: if server rejects the pack data
139 :raises UpdateRefsError: if the server supports report-status
140 and rejects ref updates
142 old_refs, server_capabilities = self.read_refs()
143 if 'report-status' not in server_capabilities:
144 self._send_capabilities.remove('report-status')
145 new_refs = determine_wants(old_refs)
147 self.proto.write_pkt_line(None)
150 have = [x for x in old_refs.values() if not x == ZERO_SHA]
151 sent_capabilities = False
152 for refname in set(new_refs.keys() + old_refs.keys()):
153 old_sha1 = old_refs.get(refname, ZERO_SHA)
154 new_sha1 = new_refs.get(refname, ZERO_SHA)
155 if old_sha1 != new_sha1:
156 if sent_capabilities:
157 self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1,
160 self.proto.write_pkt_line(
161 "%s %s %s\0%s" % (old_sha1, new_sha1, refname,
162 ' '.join(self._send_capabilities)))
163 sent_capabilities = True
164 if new_sha1 not in have and new_sha1 != ZERO_SHA:
165 want.append(new_sha1)
166 self.proto.write_pkt_line(None)
169 objects = generate_pack_contents(have, want)
170 (entries, sha) = write_pack_data(self.proto.write_file(), objects,
173 if 'report-status' in self._send_capabilities:
174 self._parse_status_report()
175 # wait for EOF before returning
176 data = self.proto.read()
178 raise SendPackError('Unexpected response %r' % data)
181 def fetch(self, path, target, determine_wants=None, progress=None):
182 """Fetch into a target repository.
184 :param path: Path to fetch from
185 :param target: Target repository to fetch into
186 :param determine_wants: Optional function to determine what refs
188 :param progress: Optional progress function
191 if determine_wants is None:
192 determine_wants = target.object_store.determine_wants_all
193 f, commit = target.object_store.add_pack()
195 return self.fetch_pack(path, determine_wants,
196 target.get_graph_walker(), f.write, progress)
200 def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
202 """Retrieve a pack from a git smart server.
204 :param determine_wants: Callback that returns list of commits to fetch
205 :param graph_walker: Object with next() and ack().
206 :param pack_data: Callback called for each bit of data in the pack
207 :param progress: Callback for progress reports (strings)
209 (refs, server_capabilities) = self.read_refs()
210 wants = determine_wants(refs)
212 self.proto.write_pkt_line(None)
214 assert isinstance(wants, list) and type(wants[0]) == str
215 self.proto.write_pkt_line("want %s %s\n" % (
216 wants[0], ' '.join(self._fetch_capabilities)))
217 for want in wants[1:]:
218 self.proto.write_pkt_line("want %s\n" % want)
219 self.proto.write_pkt_line(None)
220 have = graph_walker.next()
222 self.proto.write_pkt_line("have %s\n" % have)
224 pkt = self.proto.read_pkt_line()
225 parts = pkt.rstrip("\n").split(" ")
226 if parts[0] == "ACK":
227 graph_walker.ack(parts[1])
228 assert parts[2] == "continue"
229 have = graph_walker.next()
230 self.proto.write_pkt_line("done\n")
231 pkt = self.proto.read_pkt_line()
233 parts = pkt.rstrip("\n").split(" ")
234 if parts[0] == "ACK":
235 graph_walker.ack(pkt.split(" ")[1])
236 if len(parts) < 3 or parts[2] != "continue":
238 pkt = self.proto.read_pkt_line()
239 # TODO(durin42): this is broken if the server didn't support the
240 # side-band-64k capability.
241 for pkt in self.proto.read_pkt_seq():
242 channel = ord(pkt[0])
247 if progress is not None:
250 raise AssertionError("Invalid sideband channel %d" % channel)
254 class TCPGitClient(GitClient):
255 """A Git Client that works over TCP directly (i.e. git://)."""
257 def __init__(self, host, port=None, *args, **kwargs):
258 self._socket = socket.socket(type=socket.SOCK_STREAM)
261 self._socket.connect((host, port))
262 self.rfile = self._socket.makefile('rb', -1)
263 self.wfile = self._socket.makefile('wb', 0)
265 super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
267 def send_pack(self, path, changed_refs, generate_pack_contents):
268 """Send a pack to a remote host.
270 :param path: Path of the repository on the remote host
272 self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
273 return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
275 def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
276 """Fetch a pack from the remote host.
278 :param path: Path of the reposiutory on the remote host
279 :param determine_wants: Callback that receives available refs dict and
280 should return list of sha's to fetch.
281 :param graph_walker: GraphWalker instance used to find missing shas
282 :param pack_data: Callback for writing pack data
283 :param progress: Callback for writing progress
285 self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
286 return super(TCPGitClient, self).fetch_pack(path, determine_wants,
287 graph_walker, pack_data, progress)
290 class SubprocessGitClient(GitClient):
291 """Git client that talks to a server using a subprocess."""
293 def __init__(self, *args, **kwargs):
296 self._kwargs = kwargs
298 def _connect(self, service, *args, **kwargs):
299 argv = [service] + list(args)
300 self.proc = subprocess.Popen(argv, bufsize=0,
301 stdin=subprocess.PIPE,
302 stdout=subprocess.PIPE)
304 return self.proc.stdout.read(size)
306 self.proc.stdin.write(data)
307 self.proc.stdin.flush()
308 return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
310 def send_pack(self, path, changed_refs, generate_pack_contents):
311 """Upload a pack to the server.
313 :param path: Path to the git repository on the server
314 :param changed_refs: Dictionary with new values for the refs
315 :param generate_pack_contents: Function that returns an iterator over
318 client = self._connect("git-receive-pack", path)
319 return client.send_pack(path, changed_refs, generate_pack_contents)
321 def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
323 """Retrieve a pack from the server
325 :param path: Path to the git repository on the server
326 :param determine_wants: Function that receives existing refs
327 on the server and returns a list of desired shas
328 :param graph_walker: GraphWalker instance
329 :param pack_data: Function that can write pack data
330 :param progress: Function that can write progress texts
332 client = self._connect("git-upload-pack", path)
333 return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
337 class SSHSubprocess(object):
338 """A socket-like object that talks to an ssh subprocess via pipes."""
340 def __init__(self, proc):
342 self.read = self.recv = proc.stdout.read
343 self.write = self.send = proc.stdin.write
346 self.proc.stdin.close()
347 self.proc.stdout.close()
351 class SSHVendor(object):
353 def connect_ssh(self, host, command, username=None, port=None):
354 #FIXME: This has no way to deal with passwords..
357 args.extend(['-p', str(port)])
358 if username is not None:
359 host = "%s@%s" % (username, host)
361 proc = subprocess.Popen(args + command,
362 stdin=subprocess.PIPE,
363 stdout=subprocess.PIPE)
364 return SSHSubprocess(proc)
366 # Can be overridden by users
367 get_ssh_vendor = SSHVendor
370 class SSHGitClient(GitClient):
372 def __init__(self, host, port=None, username=None, *args, **kwargs):
375 self.username = username
377 self._kwargs = kwargs
379 def send_pack(self, path, determine_wants, generate_pack_contents):
380 remote = get_ssh_vendor().connect_ssh(
381 self.host, ["git-receive-pack '%s'" % path],
382 port=self.port, username=self.username)
383 client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
384 return client.send_pack(path, determine_wants, generate_pack_contents)
386 def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
388 remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack '%s'" % path], port=self.port, username=self.username)
389 client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
390 return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
394 def get_transport_and_path(uri):
395 """Obtain a git client from a URI or path.
397 :param uri: URI or path
398 :return: Tuple with client instance and relative path.
400 from dulwich.client import TCPGitClient, SSHGitClient, SubprocessGitClient
401 for handler, transport in (("git://", TCPGitClient), ("git+ssh://", SSHGitClient)):
402 if uri.startswith(handler):
403 host, path = uri[len(handler):].split("/", 1)
404 return transport(host), "/"+path
405 # if its not git or git+ssh, try a local url..
406 return SubprocessGitClient(), uri