When there's nothing to push, don't push anything.
[jelmer/dulwich-libgit2.git] / dulwich / client.py
1 # client.py -- Implementation of the server side git protocols
2 # Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
3 # Copyright (C) 2008 John Carr
4 #
5 # This program is free software; you can redistribute it and/or
6 # modify it under the terms of the GNU General Public License
7 # as published by the Free Software Foundation; either version 2
8 # or (at your option) a later version of the License.
9 #
10 # This program is distributed in the hope that it will be useful,
11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13 # GNU General Public License for more details.
14 #
15 # You should have received a copy of the GNU General Public License
16 # along with this program; if not, write to the Free Software
17 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
18 # MA  02110-1301, USA.
19
20 """Client side support for the Git protocol."""
21
22 __docformat__ = 'restructuredText'
23
24 import os
25 import select
26 import socket
27 import subprocess
28
29 from dulwich.errors import (
30     ChecksumMismatch,
31     )
32 from dulwich.protocol import (
33     Protocol,
34     TCP_GIT_PORT,
35     extract_capabilities,
36     )
37 from dulwich.pack import (
38     write_pack_data,
39     )
40
41
42 def _fileno_can_read(fileno):
43     """Check if a file descriptor is readable."""
44     return len(select.select([fileno], [], [], 0)[0]) > 0
45
46
47 CAPABILITIES = ["multi_ack", "side-band-64k", "ofs-delta"]
48
49
50 class GitClient(object):
51     """Git smart server client.
52
53     """
54
55     def __init__(self, can_read, read, write, thin_packs=True, 
56         report_activity=None):
57         """Create a new GitClient instance.
58
59         :param can_read: Function that returns True if there is data available
60             to be read.
61         :param read: Callback for reading data, takes number of bytes to read
62         :param write: Callback for writing data
63         :param thin_packs: Whether or not thin packs should be retrieved
64         :param report_activity: Optional callback for reporting transport
65             activity.
66         """
67         self.proto = Protocol(read, write, report_activity)
68         self._can_read = can_read
69         self._capabilities = list(CAPABILITIES)
70         if thin_packs:
71             self._capabilities.append("thin-pack")
72
73     def capabilities(self):
74         return " ".join(self._capabilities)
75
76     def read_refs(self):
77         server_capabilities = None
78         refs = {}
79         # Receive refs from server
80         for pkt in self.proto.read_pkt_seq():
81             (sha, ref) = pkt.rstrip("\n").split(" ", 1)
82             if server_capabilities is None:
83                 (ref, server_capabilities) = extract_capabilities(ref)
84             refs[ref] = sha
85         return refs, server_capabilities
86
87     def send_pack(self, path, determine_wants, generate_pack_contents):
88         """Upload a pack to a remote repository.
89
90         :param path: Repository path
91         :param generate_pack_contents: Function that can return the shas of the 
92             objects to upload.
93         """
94         old_refs, server_capabilities = self.read_refs()
95         new_refs = determine_wants(old_refs)
96         if not new_refs:
97             self.proto.write_pkt_line(None)
98             return {}
99         want = []
100         have = [x for x in old_refs.values() if not x == "0" * 40]
101         sent_capabilities = False
102         for refname in set(new_refs.keys() + old_refs.keys()):
103             old_sha1 = old_refs.get(refname, "0" * 40)
104             new_sha1 = new_refs.get(refname, "0" * 40)
105             if old_sha1 != new_sha1:
106                 if sent_capabilities:
107                     self.proto.write_pkt_line("%s %s %s" % (old_sha1, new_sha1, refname))
108                 else:
109                     self.proto.write_pkt_line("%s %s %s\0%s" % (old_sha1, new_sha1, refname, self.capabilities()))
110                     sent_capabilities = True
111             if not new_sha1 in (have, "0" * 40):
112                 want.append(new_sha1)
113         self.proto.write_pkt_line(None)
114         if not want:
115             return new_refs
116         objects = generate_pack_contents(have, want)
117         (entries, sha) = write_pack_data(self.proto.write_file(), objects, 
118                                          len(objects))
119         
120         # read the final confirmation sha
121         client_sha = self.proto.read(20)
122         if not client_sha in (None, "", sha):
123             raise ChecksumMismatch(sha, client_sha)
124             
125         return new_refs
126
127     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
128                    progress):
129         """Retrieve a pack from a git smart server.
130
131         :param determine_wants: Callback that returns list of commits to fetch
132         :param graph_walker: Object with next() and ack().
133         :param pack_data: Callback called for each bit of data in the pack
134         :param progress: Callback for progress reports (strings)
135         """
136         (refs, server_capabilities) = self.read_refs()
137         wants = determine_wants(refs)
138         if not wants:
139             self.proto.write_pkt_line(None)
140             return refs
141         assert isinstance(wants, list) and type(wants[0]) == str
142         self.proto.write_pkt_line("want %s %s\n" % (wants[0], self.capabilities()))
143         for want in wants[1:]:
144             self.proto.write_pkt_line("want %s\n" % want)
145         self.proto.write_pkt_line(None)
146         have = graph_walker.next()
147         while have:
148             self.proto.write_pkt_line("have %s\n" % have)
149             if self._can_read():
150                 pkt = self.proto.read_pkt_line()
151                 parts = pkt.rstrip("\n").split(" ")
152                 if parts[0] == "ACK":
153                     graph_walker.ack(parts[1])
154                     assert parts[2] == "continue"
155             have = graph_walker.next()
156         self.proto.write_pkt_line("done\n")
157         pkt = self.proto.read_pkt_line()
158         while pkt:
159             parts = pkt.rstrip("\n").split(" ")
160             if parts[0] == "ACK":
161                 graph_walker.ack(pkt.split(" ")[1])
162             if len(parts) < 3 or parts[2] != "continue":
163                 break
164             pkt = self.proto.read_pkt_line()
165         for pkt in self.proto.read_pkt_seq():
166             channel = ord(pkt[0])
167             pkt = pkt[1:]
168             if channel == 1:
169                 pack_data(pkt)
170             elif channel == 2:
171                 progress(pkt)
172             else:
173                 raise AssertionError("Invalid sideband channel %d" % channel)
174         return refs
175
176
177 class TCPGitClient(GitClient):
178     """A Git Client that works over TCP directly (i.e. git://)."""
179
180     def __init__(self, host, port=None, *args, **kwargs):
181         self._socket = socket.socket(type=socket.SOCK_STREAM)
182         if port is None:
183             port = TCP_GIT_PORT
184         self._socket.connect((host, port))
185         self.rfile = self._socket.makefile('rb', -1)
186         self.wfile = self._socket.makefile('wb', 0)
187         self.host = host
188         super(TCPGitClient, self).__init__(lambda: _fileno_can_read(self._socket.fileno()), self.rfile.read, self.wfile.write, *args, **kwargs)
189
190     def send_pack(self, path, changed_refs, generate_pack_contents):
191         """Send a pack to a remote host.
192
193         :param path: Path of the repository on the remote host
194         """
195         self.proto.send_cmd("git-receive-pack", path, "host=%s" % self.host)
196         return super(TCPGitClient, self).send_pack(path, changed_refs, generate_pack_contents)
197
198     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, progress):
199         """Fetch a pack from the remote host.
200         
201         :param path: Path of the reposiutory on the remote host
202         :param determine_wants: Callback that receives available refs dict and 
203             should return list of sha's to fetch.
204         :param graph_walker: GraphWalker instance used to find missing shas
205         :param pack_data: Callback for writing pack data
206         :param progress: Callback for writing progress
207         """
208         self.proto.send_cmd("git-upload-pack", path, "host=%s" % self.host)
209         return super(TCPGitClient, self).fetch_pack(path, determine_wants,
210             graph_walker, pack_data, progress)
211
212
213 class SubprocessGitClient(GitClient):
214     """Git client that talks to a server using a subprocess."""
215
216     def __init__(self, *args, **kwargs):
217         self.proc = None
218         self._args = args
219         self._kwargs = kwargs
220
221     def _connect(self, service, *args, **kwargs):
222         argv = [service] + list(args)
223         self.proc = subprocess.Popen(argv, bufsize=0,
224                                 stdin=subprocess.PIPE,
225                                 stdout=subprocess.PIPE)
226         def read_fn(size):
227             return self.proc.stdout.read(size)
228         def write_fn(data):
229             self.proc.stdin.write(data)
230             self.proc.stdin.flush()
231         return GitClient(lambda: _fileno_can_read(self.proc.stdout.fileno()), read_fn, write_fn, *args, **kwargs)
232
233     def send_pack(self, path, changed_refs, generate_pack_contents):
234         """Upload a pack to the server.
235
236         :param path: Path to the git repository on the server
237         :param changed_refs: Dictionary with new values for the refs
238         :param generate_pack_contents: Function that returns an iterator over 
239             objects to send
240         """
241         client = self._connect("git-receive-pack", path)
242         return client.send_pack(path, changed_refs, generate_pack_contents)
243
244     def fetch_pack(self, path, determine_wants, graph_walker, pack_data, 
245         progress):
246         """Retrieve a pack from the server
247
248         :param path: Path to the git repository on the server
249         :param determine_wants: Function that receives existing refs 
250             on the server and returns a list of desired shas
251         :param graph_walker: GraphWalker instance
252         :param pack_data: Function that can write pack data
253         :param progress: Function that can write progress texts
254         """
255         client = self._connect("git-upload-pack", path)
256         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
257                                  progress)
258
259
260 class SSHSubprocess(object):
261     """A socket-like object that talks to an ssh subprocess via pipes."""
262
263     def __init__(self, proc):
264         self.proc = proc
265
266     def send(self, data):
267         return os.write(self.proc.stdin.fileno(), data)
268
269     def recv(self, count):
270         return self.proc.stdout.read(count)
271
272     def close(self):
273         self.proc.stdin.close()
274         self.proc.stdout.close()
275         self.proc.wait()
276
277
278 class SSHVendor(object):
279
280     def connect_ssh(self, host, command, username=None, port=None):
281         #FIXME: This has no way to deal with passwords..
282         args = ['ssh', '-x']
283         if port is not None:
284             args.extend(['-p', str(port)])
285         if username is not None:
286             host = "%s@%s" % (username, host)
287         args.append(host)
288         proc = subprocess.Popen(args + command,
289                                 stdin=subprocess.PIPE,
290                                 stdout=subprocess.PIPE)
291         return SSHSubprocess(proc)
292
293 # Can be overridden by users
294 get_ssh_vendor = SSHVendor
295
296
297 class SSHGitClient(GitClient):
298
299     def __init__(self, host, port=None, username=None, *args, **kwargs):
300         self.host = host
301         self.port = port
302         self.username = username
303         self._args = args
304         self._kwargs = kwargs
305
306     def send_pack(self, path, determine_wants, generate_pack_contents):
307         remote = get_ssh_vendor().connect_ssh(self.host, ["git-receive-pack '%s'" % path], port=self.port, username=self.username)
308         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
309         return client.send_pack(path, determine_wants, generate_pack_contents)
310
311     def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
312         progress):
313         remote = get_ssh_vendor().connect_ssh(self.host, ["git-upload-pack '%s'" % path], port=self.port, username=self.username)
314         client = GitClient(lambda: _fileno_can_read(remote.proc.stdout.fileno()), remote.recv, remote.send, *self._args, **self._kwargs)
315         return client.fetch_pack(path, determine_wants, graph_walker, pack_data,
316                                  progress)
317