1 # client.py -- Implementation of the server side git protocols
2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
3 # Copyright (C) 2008 John Carr
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # or (at your option) a later version of the License.
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # GNU General Public License for more details.
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
20 """Client side support for the Git protocol."""
22 __docformat__ = 'restructuredText'
28 from dulwich.errors import (
32 from dulwich.protocol import (
38 from dulwich.pack import (
43 # Python 2.6.6 included these in urlparse.uses_netloc upstream. Do
44 # monkeypatching to enable similar behaviour in earlier Pythons:
45 for scheme in ('git', 'git+ssh'):
46 if scheme not in urlparse.uses_netloc:
47 urlparse.uses_netloc.append(scheme)
49 def _fileno_can_read(fileno):
50 """Check if a file descriptor is readable."""
51 return len(select.select([fileno], [], [], 0)[0]) > 0
53 COMMON_CAPABILITIES = ['ofs-delta']
54 FETCH_CAPABILITIES = ['multi_ack', 'side-band-64k'] + COMMON_CAPABILITIES
55 SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
57 # TODO(durin42): this doesn't correctly degrade if the server doesn't
58 # support some capabilities. This should work properly with servers
59 # that don't support side-band-64k and multi_ack.
60 class GitClient(object):
61 """Git smart server client.
65 def __init__(self, thin_packs=True, report_activity=None):
66 """Create a new GitClient instance.
68 :param thin_packs: Whether or not thin packs should be retrieved
69 :param report_activity: Optional callback for reporting transport
72 self._report_activity = report_activity
73 self._fetch_capabilities = list(FETCH_CAPABILITIES)
74 self._send_capabilities = list(SEND_CAPABILITIES)
76 self._fetch_capabilities.append('thin-pack')
78 def _connect(self, cmd, path):
79 """Create a connection to the server.
81 This method is abstract - concrete implementations should
82 implement their own variant which connects to the server and
83 returns an initialized Protocol object with the service ready
84 for use and a can_read function which may be used to see if
87 :param cmd: The git service name to which we should connect.
88 :param path: The path we should pass to the service.
90 raise NotImplementedError()
92 def _read_refs(self, proto):
93 server_capabilities = None
95 # Receive refs from server
96 for pkt in proto.read_pkt_seq():
97 (sha, ref) = pkt.rstrip('\n').split(' ', 1)
98 if server_capabilities is None:
99 (ref, server_capabilities) = extract_capabilities(ref)
101 return refs, server_capabilities
103 def _parse_status_report(self, proto):
104 unpack = proto.read_pkt_line().strip()
105 if unpack != 'unpack ok':
107 # flush remaining error data
108 while st is not None:
109 st = proto.read_pkt_line()
110 raise SendPackError(unpack)
113 ref_status = proto.read_pkt_line()
115 ref_status = ref_status.strip()
116 statuses.append(ref_status)
117 if not ref_status.startswith('ok '):
119 ref_status = proto.read_pkt_line()
124 for status in statuses:
125 if ' ' not in status:
126 # malformed response, move on to the next one
128 status, ref = status.split(' ', 1)
132 ref, status = ref.split(' ', 1)
135 ref_status[ref] = status
136 raise UpdateRefsError('%s failed to update' %
137 ', '.join([ref for ref in ref_status
139 ref_status=ref_status)
142 # TODO(durin42): add side-band-64k capability support here and advertise it
143 def send_pack(self, path, determine_wants, generate_pack_contents):
144 """Upload a pack to a remote repository.
146 :param path: Repository path
147 :param generate_pack_contents: Function that can return the shas of the
150 :raises SendPackError: if server rejects the pack data
151 :raises UpdateRefsError: if the server supports report-status
152 and rejects ref updates
154 proto, unused_can_read = self._connect('receive-pack', path)
155 old_refs, server_capabilities = self._read_refs(proto)
156 if 'report-status' not in server_capabilities:
157 self._send_capabilities.remove('report-status')
158 new_refs = determine_wants(old_refs)
160 proto.write_pkt_line(None)
163 have = [x for x in old_refs.values() if not x == ZERO_SHA]
164 sent_capabilities = False
165 for refname in set(new_refs.keys() + old_refs.keys()):
166 old_sha1 = old_refs.get(refname, ZERO_SHA)
167 new_sha1 = new_refs.get(refname, ZERO_SHA)
168 if old_sha1 != new_sha1:
169 if sent_capabilities:
170 proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
173 proto.write_pkt_line(
174 '%s %s %s\0%s' % (old_sha1, new_sha1, refname,
175 ' '.join(self._send_capabilities)))
176 sent_capabilities = True
177 if new_sha1 not in have and new_sha1 != ZERO_SHA:
178 want.append(new_sha1)
179 proto.write_pkt_line(None)
182 objects = generate_pack_contents(have, want)
183 entries, sha = write_pack_data(proto.write_file(), objects,
186 if 'report-status' in self._send_capabilities:
187 self._parse_status_report(proto)
188 # wait for EOF before returning
191 raise SendPackError('Unexpected response %r' % data)
194 def fetch(self, path, target, determine_wants=None, progress=None):
195 """Fetch into a target repository.
197 :param path: Path to fetch from
198 :param target: Target repository to fetch into
199 :param determine_wants: Optional function to determine what refs
201 :param progress: Optional progress function
204 if determine_wants is None:
205 determine_wants = target.object_store.determine_wants_all
206 f, commit = target.object_store.add_pack()
208 return self.fetch_pack(path, determine_wants,
209 target.get_graph_walker(), f.write, progress)
213 def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
215 """Retrieve a pack from a git smart server.
217 :param determine_wants: Callback that returns list of commits to fetch
218 :param graph_walker: Object with next() and ack().
219 :param pack_data: Callback called for each bit of data in the pack
220 :param progress: Callback for progress reports (strings)
222 proto, can_read = self._connect('upload-pack', path)
223 (refs, server_capabilities) = self._read_refs(proto)
224 wants = determine_wants(refs)
226 proto.write_pkt_line(None)
228 assert isinstance(wants, list) and type(wants[0]) == str
229 proto.write_pkt_line('want %s %s\n' % (
230 wants[0], ' '.join(self._fetch_capabilities)))
231 for want in wants[1:]:
232 proto.write_pkt_line('want %s\n' % want)
233 proto.write_pkt_line(None)
234 have = graph_walker.next()
236 proto.write_pkt_line('have %s\n' % have)
238 pkt = proto.read_pkt_line()
239 parts = pkt.rstrip('\n').split(' ')
240 if parts[0] == 'ACK':
241 graph_walker.ack(parts[1])
242 assert parts[2] == 'continue'
243 have = graph_walker.next()
244 proto.write_pkt_line('done\n')
245 pkt = proto.read_pkt_line()
247 parts = pkt.rstrip('\n').split(' ')
248 if parts[0] == 'ACK':
249 graph_walker.ack(pkt.split(' ')[1])
250 if len(parts) < 3 or parts[2] != 'continue':
252 pkt = proto.read_pkt_line()
253 # TODO(durin42): this is broken if the server didn't support the
254 # side-band-64k capability.
255 for pkt in proto.read_pkt_seq():
256 channel = ord(pkt[0])
261 if progress is not None:
264 raise AssertionError('Invalid sideband channel %d' % channel)
268 class TCPGitClient(GitClient):
269 """A Git Client that works over TCP directly (i.e. git://)."""
271 def __init__(self, host, port=None, *args, **kwargs):
276 GitClient.__init__(self, *args, **kwargs)
278 def _connect(self, cmd, path):
279 s = socket.socket(type=socket.SOCK_STREAM)
280 s.connect((self._host, self._port))
281 # -1 means system default buffering
282 rfile = s.makefile('rb', -1)
284 wfile = s.makefile('wb', 0)
285 proto = Protocol(rfile.read, wfile.write,
286 report_activity=self._report_activity)
287 proto.send_cmd('git-%s' % cmd, path, 'host=%s' % self._host)
288 return proto, lambda: _fileno_can_read(s)
291 class SubprocessWrapper(object):
292 """A socket-like object that talks to a subprocess via pipes."""
294 def __init__(self, proc):
296 self.read = proc.stdout.read
297 self.write = proc.stdin.write
300 return _fileno_can_read(self.proc.stdout.fileno())
303 self.proc.stdin.close()
304 self.proc.stdout.close()
308 class SubprocessGitClient(GitClient):
309 """Git client that talks to a server using a subprocess."""
311 def __init__(self, *args, **kwargs):
312 self._connection = None
313 GitClient.__init__(self, *args, **kwargs)
315 def _connect(self, service, path):
317 argv = ['git', service, path]
318 p = SubprocessWrapper(
319 subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
320 stdout=subprocess.PIPE))
321 return Protocol(p.read, p.write,
322 report_activity=self._report_activity), p.can_read
325 class SSHVendor(object):
327 def connect_ssh(self, host, command, username=None, port=None):
329 #FIXME: This has no way to deal with passwords..
332 args.extend(['-p', str(port)])
333 if username is not None:
334 host = '%s@%s' % (username, host)
336 proc = subprocess.Popen(args + command,
337 stdin=subprocess.PIPE,
338 stdout=subprocess.PIPE)
339 return SubprocessWrapper(proc)
341 # Can be overridden by users
342 get_ssh_vendor = SSHVendor
345 class SSHGitClient(GitClient):
347 def __init__(self, host, port=None, username=None, *args, **kwargs):
350 self.username = username
351 GitClient.__init__(self, *args, **kwargs)
352 self.alternative_paths = {}
354 def _get_cmd_path(self, cmd):
355 return self.alternative_paths.get(cmd, 'git-%s' % cmd)
357 def _connect(self, cmd, path):
358 con = get_ssh_vendor().connect_ssh(
359 self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
360 port=self.port, username=self.username)
361 return Protocol(con.read, con.write), con.can_read
364 def get_transport_and_path(uri):
365 """Obtain a git client from a URI or path.
367 :param uri: URI or path
368 :return: Tuple with client instance and relative path.
370 parsed = urlparse.urlparse(uri)
371 if parsed.scheme == 'git':
372 return TCPGitClient(parsed.hostname, port=parsed.port), parsed.path
373 elif parsed.scheme == 'git+ssh':
374 return SSHGitClient(parsed.hostname, port=parsed.port,
375 username=parsed.username), parsed.path
377 if parsed.scheme and not parsed.netloc:
378 # SSH with no user@, zero or one leading slash.
379 return SSHGitClient(parsed.scheme), parsed.path
381 raise ValueError('Unknown git protocol scheme: %s' % parsed.scheme)
382 elif '@' in parsed.path and ':' in parsed.path:
383 # SSH with user@host:foo.
384 user_host, path = parsed.path.split(':')
385 user, host = user_host.rsplit('@')
386 return SSHGitClient(host, username=user), path
388 # Otherwise, assume it's a local path.
389 return SubprocessGitClient(), uri