c837448229703d5fc47c2c31339c515b48cce967
[jelmer/dulwich-libgit2.git] / dulwich / client.py
1 # client.py -- Implementation of the server side git protocols
2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
3 # Copyright (C) 2008 John Carr
4 #
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # or (at your option) a later version of the License.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
18 # MA  02110-1301, USA.
19
20 """Client side support for the Git protocol."""
21
22 __docformat__ = 'restructuredText'
23
24 import os
25 import select
26 import socket
27 import subprocess
28
29 from dulwich.protocol import (
30     Protocol,
31     TCP_GIT_PORT,
32     extract_capabilities,
33     )
34 from dulwich.pack import (
35     write_pack_data,
36     )
37
38
39 def _fileno_can_read(fileno):
40     """Check if a file descriptor is readable."""
41     return len(select.select([fileno], [], [], 0)[0]) > 0
42
43
44 class SimpleFetchGraphWalker(object):
45
46     def __init__(self, local_heads, get_parents):
47         self.heads = set(local_heads)
48         self.get_parents = get_parents
49         self.parents = {}
50
51     def ack(self, sha):
52         """Ack that a particular revision and its ancestors are present."""
53         if sha in self.heads:
54             self.heads.remove(sha)
55         if sha in self.parents:
56             for p in self.parents[sha]:
57                 self.ack(p)
58
59     def next(self):
60         """Iterate over revisions that might be missing in the target."""
61         if self.heads:
62             ret = self.heads.pop()
63             ps = self.get_parents(ret)
64             self.parents[ret] = ps
65             self.heads.update(ps)
66             return ret
67         return None
68
69
70 CAPABILITIES = ["multi_ack", "side-band-64k", "ofs-delta"]
71
72
73 class GitClient(object):
74     """Git smart server client.
75
76     """
77
78     def __init__(self, can_read, read, write, thin_packs=True, 
79         report_activity=None):
80         """Create a new GitClient instance.
81
82         :param can_read: Function that returns True if there is data available
83             to be read.
84         :param read: Callback for reading data, takes number of bytes to read
85         :param write: Callback for writing data
86         :param thin_packs: Whether or not thin packs should be retrieved
87         :param report_activity: Optional callback for reporting transport
88             activity.
89         """
90         self.proto = Protocol(read, write, report_activity)
91         self._can_read = can_read
92         self._capabilities = list(CAPABILITIES)
93         if thin_packs:
94             self._capabilities.append("thin-pack")
95
96     def capabilities(self):
97         return " ".join(self._capabilities)
98
99     def read_refs(self):
100         server_capabilities = None
101         refs = {}
102         # Receive refs from server
103         for pkt in self.proto.read_pkt_seq():
104             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
105             if server_capabilities is None:
106                 (ref, server_capabilities) = extract_capabilities(ref)
107             refs[ref] = sha
108         return refs, server_capabilities
109
110     def send_pack(self, path, determine_wants, generate_pack_contents):
111         """Upload a pack to a remote repository.
112
113         :param path: Repository path
114         :param generate_pack_contents: Function that can return the shas of the 
115             objects to upload.
116         """
117         refs, server_capabilities = self.read_refs()
118         changed_refs = determine_wants(refs)
119         if not changed_refs:
120             self.proto.write_pkt_line(None)
121             return {}
122         want = []
123         have = []
124         sent_capabilities = False
125         for changed_ref, new_sha1 in changed_refs.iteritems():
126             old_sha1 = refs.get(changed_ref, "0" * 40)
127             if sent_capabilities:
128                 self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, changed_ref))
129             else:
130                 self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, changed_ref, self.capabilities()))
131                 sent_capabilities = True
132             want.append(new_sha1)
133             if old_sha1 != "0"*40:
134                 have.append(old_sha1)
135         self.proto.write_pkt_line(None)
136         objects = generate_pack_contents(want, have)
137         (entries, sha) = write_pack_data(self.proto.write_file(), objects, len(objects))
138         self.proto.write(sha)
139         
140         # read the final confirmation sha
141         sha = self.proto.read(20)
142         if sha:
143             pass # FIXME: Check that this sha is valid
144             
145         return changed_refs
146
147     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
148                    progress):
149         """Retrieve a pack from a git smart server.
150
151         :param determine_wants: Callback that returns list of commits to fetch
152         :param graph_walker: Object with next() and ack().
153         :param pack_data: Callback called for each bit of data in the pack
154         :param progress: Callback for progress reports (strings)
155         """
156         (refs, server_capabilities) = self.read_refs()
157         wants = determine_wants(refs)
158         if not wants:
159             self.proto.write_pkt_line(None)
160             return
161         self.proto.write_pkt_line("want %s %s\n" % (wants[0], self.capabilities()))
162         for want in wants[1:]:
163             self.proto.write_pkt_line("want %s\n" % want)
164         self.proto.write_pkt_line(None)
165         have = graph_walker.next()
166         while have:
167             self.proto.write_pkt_line("have %s\n" % have)
168             if self._can_read():
169                 pkt = self.proto.read_pkt_line()
170                 parts = pkt.rstrip("\n").split(" ")
171                 if parts[0] == "ACK":
172                     graph_walker.ack(parts[1])
173                     assert parts[2] == "continue"
174             have = graph_walker.next()
175         self.proto.write_pkt_line("done\n")
176         pkt = self.proto.read_pkt_line()
177         while pkt:
178             parts = pkt.rstrip("\n").split(" ")
179             if parts[0] == "ACK":
180                 graph_walker.ack(pkt.split(" ")[1])
181             if len(parts) < 3 or parts[2] != "continue":
182                 break
183             pkt = self.proto.read_pkt_line()
184         for pkt in self.proto.read_pkt_seq():
185             channel = ord(pkt[0])
186             pkt = pkt[1:]
187             if channel == 1:
188                 pack_data(pkt)
189             elif channel == 2:
190                 progress(pkt)
191             else:
192                 raise AssertionError("Invalid sideband channel %d" % channel)
193         return refs
194
195
196 class TCPGitClient(GitClient):
197     """A Git Client that works over TCP directly (i.e. git://)."""
198
199     def __init__(self, host, port=None, *args, **kwargs):
200         self._socket = socket.socket(type=socket.SOCK_STREAM)
201         if port is None:
202             port = TCP_GIT_PORT
203         self._socket.connect((host, port))
204         self.rfile = self._socket.makefile('rb', -1)
205         self.wfile = self._socket.makefile('wb', 0)
206         self.host = host
207         super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
208
209     def send_pack(self, path, changed_refs, generate_pack_contents):
210         """Send a pack to a remote host.
211
212         :param path: Path of the repository on the remote host
213         """
214         self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
215         return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
216
217     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
218         """Fetch a pack from the remote host.
219         
220         :param path: Path of the reposiutory on the remote host
221         :param determine_wants: Callback that receives available refs dict and 
222             should return list of sha's to fetch.
223         :param graph_walker: GraphWalker instance used to find missing shas
224         :param pack_data: Callback for writing pack data
225         :param progress: Callback for writing progress
226         """
227         self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
228         return super(TCPGitClient, self).fetch_pack(path, determine_wants,
229             graph_walker, pack_data, progress)
230
231
232 class SubprocessGitClient(GitClient):
233     """Git client that talks to a server using a subprocess."""
234
235     def __init__(self, *args, **kwargs):
236         self.proc = None
237         self._args = args
238         self._kwargs = kwargs
239
240     def _connect(self, service, *args, **kwargs):
241         argv = [service] + list(args)
242         self.proc = subprocess.Popen(argv, bufsize=0,
243                                 stdin=subprocess.PIPE,
244                                 stdout=subprocess.PIPE)
245         def read_fn(size):
246             return self.proc.stdout.read(size)
247         def write_fn(data):
248             self.proc.stdin.write(data)
249             self.proc.stdin.flush()
250         return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
251
252     def send_pack(self, path, changed_refs, generate_pack_contents):
253         """Upload a pack to the server.
254
255         :param path: Path to the git repository on the server
256         :param changed_refs: Dictionary with new values for the refs
257         :param generate_pack_contents: Function that returns an iterator over 
258             objects to send
259         """
260         client = self._connect("git-receive-pack", path)
261         return client.send_pack(path, changed_refs, generate_pack_contents)
262
263     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, 
264         progress):
265         """Retrieve a pack from the server
266
267         :param path: Path to the git repository on the server
268         :param determine_wants: Function that receives existing refs 
269             on the server and returns a list of desired shas
270         :param graph_walker: GraphWalker instance
271         :param pack_data: Function that can write pack data
272         :param progress: Function that can write progress texts
273         """
274         client = self._connect("git-upload-pack", path)
275         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
276                                  progress)
277
278
279 class SSHSubprocess(object):
280     """A socket-like object that talks to an ssh subprocess via pipes."""
281
282     def __init__(self, proc):
283         self.proc = proc
284
285     def send(self, data):
286         return os.write(self.proc.stdin.fileno(), data)
287
288     def recv(self, count):
289         return self.proc.stdout.read(count)
290
291     def close(self):
292         self.proc.stdin.close()
293         self.proc.stdout.close()
294         self.proc.wait()
295
296
297 class SSHVendor(object):
298
299     def connect_ssh(self, host, command, username=None, port=None):
300         #FIXME: This has no way to deal with passwords..
301         args = ['ssh', '-x']
302         if port is not None:
303             args.extend(['-p', str(port)])
304         if username is not None:
305             host = "%s@%s" % (username, host)
306         args.append(host)
307         proc = subprocess.Popen(args + command,
308                                 stdin=subprocess.PIPE,
309                                 stdout=subprocess.PIPE)
310         return SSHSubprocess(proc)
311
312 # Can be overridden by users
313 get_ssh_vendor = SSHVendor
314
315
316 class SSHGitClient(GitClient):
317
318     def __init__(self, host, port=None, *args, **kwargs):
319         self.host = host
320         self.port = port
321         self._args = args
322         self._kwargs = kwargs
323
324     def send_pack(self, path, determine_wants, generate_pack_contents):
325         remote = get_ssh_vendor().connect_ssh(self.host, ["git-receive-pack %s" % path], port=self.port)
326         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
327         return client.send_pack(path, determine_wants, generate_pack_contents)
328
329     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
330         progress):
331         remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack %s" % path], port=self.port)
332         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
333         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
334                                  progress)
335