# client.py -- Implementation of the server side git protocols
-# Copyright (C) 2008-2009 Jelmer Vernooij <jelmer@samba.org>
+# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@samba.org>
# Copyright (C) 2008 John Carr
#
# This program is free software; you can redistribute it and/or
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
-"""Client side support for the Git protocol."""
+"""Client side support for the Git protocol.
+
+The Dulwich client supports the following capabilities:
+
+ * thin-pack
+ * multi_ack_detailed
+ * multi_ack
+ * side-band-64k
+ * ofs-delta
+ * report-status
+ * delete-refs
+
+Known capabilities that are not supported:
+
+ * shallow
+ * no-progress
+ * include-tag
+"""
__docformat__ = 'restructuredText'
from dulwich.pack import (
write_pack_objects,
)
+from dulwich.refs import (
+ read_info_refs,
+ )
# Python 2.6.6 included these in urlparse.uses_netloc upstream. Do
return len(select.select([fileno], [], [], 0)[0]) > 0
COMMON_CAPABILITIES = ['ofs-delta', 'side-band-64k']
-FETCH_CAPABILITIES = ['multi_ack', 'multi_ack_detailed'] + COMMON_CAPABILITIES
+FETCH_CAPABILITIES = ['thin-pack', 'multi_ack', 'multi_ack_detailed'] + COMMON_CAPABILITIES
SEND_CAPABILITIES = ['report-status'] + COMMON_CAPABILITIES
self._ref_status_ok = False
+def read_pkt_refs(proto):
+ server_capabilities = None
+ refs = {}
+ # Receive refs from server
+ for pkt in proto.read_pkt_seq():
+ (sha, ref) = pkt.rstrip('\n').split(None, 1)
+ if sha == 'ERR':
+ raise GitProtocolError(ref)
+ if server_capabilities is None:
+ (ref, server_capabilities) = extract_capabilities(ref)
+ refs[ref] = sha
+
+ if len(refs) == 0:
+ return None, set([])
+ return refs, set(server_capabilities)
+
+
# TODO(durin42): this doesn't correctly degrade if the server doesn't
# support some capabilities. This should work properly with servers
# that don't support multi_ack.
activity.
"""
self._report_activity = report_activity
- self._fetch_capabilities = list(FETCH_CAPABILITIES)
- self._send_capabilities = list(SEND_CAPABILITIES)
- if thin_packs:
- self._fetch_capabilities.append('thin-pack')
-
- def _read_refs(self, proto):
- server_capabilities = None
- refs = {}
- # Receive refs from server
- for pkt in proto.read_pkt_seq():
- (sha, ref) = pkt.rstrip('\n').split(' ', 1)
- if sha == 'ERR':
- raise GitProtocolError(ref)
- if server_capabilities is None:
- (ref, server_capabilities) = extract_capabilities(ref)
- refs[ref] = sha
- return refs, server_capabilities
+ self._report_status_parser = None
+ self._fetch_capabilities = set(FETCH_CAPABILITIES)
+ self._send_capabilities = set(SEND_CAPABILITIES)
+ if not thin_packs:
+ self._fetch_capabilities.remove('thin-pack')
def send_pack(self, path, determine_wants, generate_pack_contents,
progress=None):
:param determine_wants: Optional function to determine what refs
to fetch
:param progress: Optional progress function
- :return: remote refs
+ :return: remote refs as dictionary
"""
if determine_wants is None:
determine_wants = target.object_store.determine_wants_all
- f, commit = target.object_store.add_pack()
+ f, commit, abort = target.object_store.add_pack()
try:
- return self.fetch_pack(path, determine_wants,
- target.get_graph_walker(), f.write, progress)
- finally:
+ result = self.fetch_pack(path, determine_wants,
+ target.get_graph_walker(), f.write, progress)
+ except:
+ abort()
+ raise
+ else:
commit()
+ return result
def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
- progress):
+ progress=None):
"""Retrieve a pack from a git smart server.
:param determine_wants: Callback that returns list of commits to fetch
want = []
have = [x for x in old_refs.values() if not x == ZERO_SHA]
sent_capabilities = False
+
for refname in set(new_refs.keys() + old_refs.keys()):
old_sha1 = old_refs.get(refname, ZERO_SHA)
new_sha1 = new_refs.get(refname, ZERO_SHA)
+
if old_sha1 != new_sha1:
if sent_capabilities:
proto.write_pkt_line('%s %s %s' % (old_sha1, new_sha1,
proto.write_pkt_line(None)
return (have, want)
- def _handle_receive_pack_tail(self, proto, capabilities, progress):
+ def _handle_receive_pack_tail(self, proto, capabilities, progress=None):
"""Handle the tail of a 'git-receive-pack' request.
:param proto: Protocol object to read from
:param capabilities: List of negotiated capabilities
:param progress: Optional progress reporting function
"""
- if 'report-status' in capabilities:
- report_status_parser = ReportStatusParser()
- else:
- report_status_parser = None
if "side-band-64k" in capabilities:
+ if progress is None:
+ progress = lambda x: None
channel_callbacks = { 2: progress }
if 'report-status' in capabilities:
channel_callbacks[1] = PktLineParser(
- report_status_parser.handle_packet).parse
+ self._report_status_parser.handle_packet).parse
self._read_side_band64k_data(proto, channel_callbacks)
else:
- if 'report-status':
+ if 'report-status' in capabilities:
for pkt in proto.read_pkt_seq():
- report_status_parser.handle_packet(pkt)
- if report_status_parser is not None:
- report_status_parser.check()
+ self._report_status_parser.handle_packet(pkt)
+ if self._report_status_parser is not None:
+ self._report_status_parser.check()
# wait for EOF before returning
data = proto.read()
if data:
proto.write_pkt_line('done\n')
def _handle_upload_pack_tail(self, proto, capabilities, graph_walker,
- pack_data, progress, rbufsize=_RBUFSIZE):
+ pack_data, progress=None, rbufsize=_RBUFSIZE):
"""Handle the tail of a 'git-upload-pack' request.
:param proto: Protocol object to read from
break
pkt = proto.read_pkt_line()
if "side-band-64k" in capabilities:
+ if progress is None:
+ # Just ignore progress data
+ progress = lambda x: None
self._read_side_band64k_data(proto, {1: pack_data, 2: progress})
# wait for EOF before returning
data = proto.read()
raise Exception('Unexpected response %r' % data)
else:
while True:
- data = self.read(rbufsize)
+ data = proto.read(rbufsize)
if data == "":
break
pack_data(data)
and rejects ref updates
"""
proto, unused_can_read = self._connect('receive-pack', path)
- old_refs, server_capabilities = self._read_refs(proto)
- negotiated_capabilities = list(self._send_capabilities)
- if 'report-status' not in server_capabilities:
- negotiated_capabilities.remove('report-status')
- new_refs = determine_wants(old_refs)
+ old_refs, server_capabilities = read_pkt_refs(proto)
+ negotiated_capabilities = self._send_capabilities & server_capabilities
+
+ if 'report-status' in negotiated_capabilities:
+ self._report_status_parser = ReportStatusParser()
+ report_status_parser = self._report_status_parser
+
+ try:
+ new_refs = orig_new_refs = determine_wants(dict(old_refs))
+ except:
+ proto.write_pkt_line(None)
+ raise
+
+ if not 'delete-refs' in server_capabilities:
+ # Server does not support deletions. Fail later.
+ def remove_del(pair):
+ if pair[1] == ZERO_SHA:
+ if 'report-status' in negotiated_capabilities:
+ report_status_parser._ref_statuses.append(
+ 'ng %s remote does not support deleting refs'
+ % pair[1])
+ report_status_parser._ref_status_ok = False
+ return False
+ else:
+ return True
+
+ new_refs = dict(
+ filter(
+ remove_del,
+ [(ref, sha) for ref, sha in new_refs.iteritems()]))
+
if new_refs is None:
proto.write_pkt_line(None)
return old_refs
+
+ if len(new_refs) == 0 and len(orig_new_refs):
+ # NOOP - Original new refs filtered out by policy
+ proto.write_pkt_line(None)
+ if self._report_status_parser is not None:
+ self._report_status_parser.check()
+ return old_refs
+
(have, want) = self._handle_receive_pack_head(proto,
negotiated_capabilities, old_refs, new_refs)
if not want and old_refs == new_refs:
objects = generate_pack_contents(have, want)
if len(objects) > 0:
entries, sha = write_pack_objects(proto.write_file(), objects)
+ elif len(set(new_refs.values()) - set([ZERO_SHA])) > 0:
+ # Check for valid create/update refs
+ filtered_new_refs = \
+ dict([(ref, sha) for ref, sha in new_refs.iteritems()
+ if sha != ZERO_SHA])
+ if len(set(filtered_new_refs.iteritems()) -
+ set(old_refs.iteritems())) > 0:
+ entries, sha = write_pack_objects(proto.write_file(), objects)
+
self._handle_receive_pack_tail(proto, negotiated_capabilities,
progress)
return new_refs
:param progress: Callback for progress reports (strings)
"""
proto, can_read = self._connect('upload-pack', path)
- (refs, server_capabilities) = self._read_refs(proto)
- negotiated_capabilities = list(self._fetch_capabilities)
- wants = determine_wants(refs)
+ refs, server_capabilities = read_pkt_refs(proto)
+ negotiated_capabilities = self._fetch_capabilities & server_capabilities
+
+ if refs is None:
+ proto.write_pkt_line(None)
+ return refs
+
+ try:
+ wants = determine_wants(refs)
+ except:
+ proto.write_pkt_line(None)
+ raise
+ if wants is not None:
+ wants = [cid for cid in wants if cid != ZERO_SHA]
if not wants:
proto.write_pkt_line(None)
return refs
graph_walker, pack_data, progress)
return refs
+ def archive(self, path, committish, write_data, progress=None):
+ proto, can_read = self._connect('upload-archive', path)
+ proto.write_pkt_line("argument %s" % committish)
+ proto.write_pkt_line(None)
+ pkt = proto.read_pkt_line()
+ if pkt == "NACK\n":
+ return
+ elif pkt == "ACK\n":
+ pass
+ elif pkt.startswith("ERR "):
+ raise GitProtocolError(pkt[4:].rstrip("\n"))
+ else:
+ raise AssertionError("invalid response %r" % pkt)
+ ret = proto.read_pkt_line()
+ if ret is not None:
+ raise AssertionError("expected pkt tail")
+ self._read_side_band64k_data(proto, {1: write_data, 2: progress})
+
class TCPGitClient(TraditionalGitClient):
"""A Git Client that works over TCP directly (i.e. git://)."""
port = TCP_GIT_PORT
self._host = host
self._port = port
- GitClient.__init__(self, *args, **kwargs)
+ TraditionalGitClient.__init__(self, *args, **kwargs)
def _connect(self, cmd, path):
sockaddrs = socket.getaddrinfo(self._host, self._port,
def __init__(self, *args, **kwargs):
self._connection = None
- GitClient.__init__(self, *args, **kwargs)
+ self._stderr = None
+ self._stderr = kwargs.get('stderr')
+ if 'stderr' in kwargs:
+ del kwargs['stderr']
+ TraditionalGitClient.__init__(self, *args, **kwargs)
def _connect(self, service, path):
import subprocess
argv = ['git', service, path]
p = SubprocessWrapper(
subprocess.Popen(argv, bufsize=0, stdin=subprocess.PIPE,
- stdout=subprocess.PIPE))
+ stdout=subprocess.PIPE,
+ stderr=self._stderr))
return Protocol(p.read, p.write,
report_activity=self._report_activity), p.can_read
+# What Git client to use for local access
+default_local_git_client_cls = SubprocessGitClient
+
class SSHVendor(object):
+ """A client side SSH implementation."""
def connect_ssh(self, host, command, username=None, port=None):
+ import warnings
+ warnings.warn(
+ "SSHVendor.connect_ssh has been renamed to SSHVendor.run_command",
+ DeprecationWarning)
+ return self.run_command(host, command, username=username, port=port)
+
+ def run_command(self, host, command, username=None, port=None):
+ """Connect to an SSH server.
+
+ Run a command remotely and return a file-like object for interaction
+ with the remote command.
+
+ :param host: Host name
+ :param command: Command to run
+ :param username: Optional ame of user to log in as
+ :param port: Optional SSH port to use
+ """
+ raise NotImplementedError(self.run_command)
+
+
+class SubprocessSSHVendor(SSHVendor):
+ """SSH vendor that shells out to the local 'ssh' command."""
+
+ def run_command(self, host, command, username=None, port=None):
import subprocess
#FIXME: This has no way to deal with passwords..
args = ['ssh', '-x']
stdout=subprocess.PIPE)
return SubprocessWrapper(proc)
+
+try:
+ import paramiko
+except ImportError:
+ pass
+else:
+ import threading
+
+ class ParamikoWrapper(object):
+ STDERR_READ_N = 2048 # 2k
+
+ def __init__(self, client, channel, progress_stderr=None):
+ self.client = client
+ self.channel = channel
+ self.progress_stderr = progress_stderr
+ self.should_monitor = bool(progress_stderr) or True
+ self.monitor_thread = None
+ self.stderr = ''
+
+ # Channel must block
+ self.channel.setblocking(True)
+
+ # Start
+ if self.should_monitor:
+ self.monitor_thread = threading.Thread(target=self.monitor_stderr)
+ self.monitor_thread.start()
+
+ def monitor_stderr(self):
+ while self.should_monitor:
+ # Block and read
+ data = self.read_stderr(self.STDERR_READ_N)
+
+ # Socket closed
+ if not data:
+ self.should_monitor = False
+ break
+
+ # Emit data
+ if self.progress_stderr:
+ self.progress_stderr(data)
+
+ # Append to buffer
+ self.stderr += data
+
+ def stop_monitoring(self):
+ # Stop StdErr thread
+ if self.should_monitor:
+ self.should_monitor = False
+ self.monitor_thread.join()
+
+ # Get left over data
+ data = self.channel.in_stderr_buffer.empty()
+ self.stderr += data
+
+ def can_read(self):
+ return self.channel.recv_ready()
+
+ def write(self, data):
+ return self.channel.sendall(data)
+
+ def read_stderr(self, n):
+ return self.channel.recv_stderr(n)
+
+ def read(self, n=None):
+ data = self.channel.recv(n)
+ data_len = len(data)
+
+ # Closed socket
+ if not data:
+ return
+
+ # Read more if needed
+ if n and data_len < n:
+ diff_len = n - data_len
+ return data + self.read(diff_len)
+ return data
+
+ def close(self):
+ self.channel.close()
+ self.stop_monitoring()
+
+ def __del__(self):
+ self.close()
+
+ class ParamikoSSHVendor(object):
+
+ def __init__(self):
+ self.ssh_kwargs = {}
+
+ def run_command(self, host, command, username=None, port=None,
+ progress_stderr=None):
+
+ # Paramiko needs an explicit port. None is not valid
+ if port is None:
+ port = 22
+
+ client = paramiko.SSHClient()
+
+ policy = paramiko.client.MissingHostKeyPolicy()
+ client.set_missing_host_key_policy(policy)
+ client.connect(host, username=username, port=port,
+ **self.ssh_kwargs)
+
+ # Open SSH session
+ channel = client.get_transport().open_session()
+
+ # Run commands
+ apply(channel.exec_command, command)
+
+ return ParamikoWrapper(client, channel,
+ progress_stderr=progress_stderr)
+
+
# Can be overridden by users
-get_ssh_vendor = SSHVendor
+get_ssh_vendor = SubprocessSSHVendor
class SSHGitClient(TraditionalGitClient):
self.host = host
self.port = port
self.username = username
- GitClient.__init__(self, *args, **kwargs)
+ TraditionalGitClient.__init__(self, *args, **kwargs)
self.alternative_paths = {}
def _get_cmd_path(self, cmd):
return self.alternative_paths.get(cmd, 'git-%s' % cmd)
def _connect(self, cmd, path):
- con = get_ssh_vendor().connect_ssh(
+ if path.startswith("/~"):
+ path = path[1:]
+ con = get_ssh_vendor().run_command(
self.host, ["%s '%s'" % (self._get_cmd_path(cmd), path)],
port=self.port, username=self.username)
return (Protocol(con.read, con.write, report_activity=self._report_activity),
def _get_url(self, path):
return urlparse.urljoin(self.base_url, path).rstrip("/") + "/"
+ def _http_request(self, url, headers={}, data=None):
+ req = urllib2.Request(url, headers=headers, data=data)
+ try:
+ resp = self._perform(req)
+ except urllib2.HTTPError, e:
+ if e.code == 404:
+ raise NotGitRepository()
+ if e.code != 200:
+ raise GitProtocolError("unexpected http response %d" % e.code)
+ return resp
+
def _perform(self, req):
"""Perform a HTTP request.
if self.dumb != False:
url += "?service=%s" % service
headers["Content-Type"] = "application/x-%s-request" % service
- req = urllib2.Request(url, headers=headers)
- resp = self._perform(req)
- if resp.getcode() == 404:
- raise NotGitRepository()
- if resp.getcode() != 200:
- raise GitProtocolError("unexpected http response %d" %
- resp.getcode())
+ resp = self._http_request(url, headers)
self.dumb = (not resp.info().gettype().startswith("application/x-git-"))
- proto = Protocol(resp.read, None)
if not self.dumb:
+ proto = Protocol(resp.read, None)
# The first line should mention the service
pkts = list(proto.read_pkt_seq())
if pkts != [('# service=%s\n' % service)]:
raise GitProtocolError(
"unexpected first line %r from smart server" % pkts)
- return self._read_refs(proto)
+ return read_pkt_refs(proto)
+ else:
+ return read_info_refs(resp), set()
def _smart_request(self, service, url, data):
assert url[-1] == "/"
url = urlparse.urljoin(url, service)
- req = urllib2.Request(url,
- headers={"Content-Type": "application/x-%s-request" % service},
- data=data)
- resp = self._perform(req)
- if resp.getcode() == 404:
- raise NotGitRepository()
- if resp.getcode() != 200:
- raise GitProtocolError("Invalid HTTP response from server: %d"
- % resp.getcode())
+ headers = {"Content-Type": "application/x-%s-request" % service}
+ resp = self._http_request(url, headers, data)
if resp.info().gettype() != ("application/x-%s-result" % service):
raise GitProtocolError("Invalid content-type from server: %s"
% resp.info().gettype())
url = self._get_url(path)
old_refs, server_capabilities = self._discover_references(
"git-receive-pack", url)
- negotiated_capabilities = list(self._send_capabilities)
- new_refs = determine_wants(old_refs)
+ negotiated_capabilities = self._send_capabilities & server_capabilities
+
+ if 'report-status' in negotiated_capabilities:
+ self._report_status_parser = ReportStatusParser()
+
+ new_refs = determine_wants(dict(old_refs))
if new_refs is None:
return old_refs
if self.dumb:
return new_refs
def fetch_pack(self, path, determine_wants, graph_walker, pack_data,
- progress):
+ progress=None):
"""Retrieve a pack from a git smart server.
:param determine_wants: Callback that returns list of commits to fetch
:param graph_walker: Object with next() and ack().
:param pack_data: Callback called for each bit of data in the pack
:param progress: Callback for progress reports (strings)
+ :return: Dictionary with the refs of the remote repository
"""
url = self._get_url(path)
refs, server_capabilities = self._discover_references(
"git-upload-pack", url)
- negotiated_capabilities = list(server_capabilities)
+ negotiated_capabilities = self._fetch_capabilities & server_capabilities
wants = determine_wants(refs)
+ if wants is not None:
+ wants = [cid for cid in wants if cid != ZERO_SHA]
if not wants:
return refs
if self.dumb:
return refs
-def get_transport_and_path(uri):
- """Obtain a git client from a URI or path.
+def get_transport_and_path_from_url(url, **kwargs):
+ """Obtain a git client from a URL.
- :param uri: URI or path
+ :param url: URL to open
+ :param thin_packs: Whether or not thin packs should be retrieved
+ :param report_activity: Optional callback for reporting transport
+ activity.
:return: Tuple with client instance and relative path.
"""
- parsed = urlparse.urlparse(uri)
+ parsed = urlparse.urlparse(url)
if parsed.scheme == 'git':
- return TCPGitClient(parsed.hostname, port=parsed.port), parsed.path
+ return (TCPGitClient(parsed.hostname, port=parsed.port, **kwargs),
+ parsed.path)
elif parsed.scheme == 'git+ssh':
+ path = parsed.path
+ if path.startswith('/'):
+ path = parsed.path[1:]
return SSHGitClient(parsed.hostname, port=parsed.port,
- username=parsed.username), parsed.path
+ username=parsed.username, **kwargs), path
elif parsed.scheme in ('http', 'https'):
- return HttpGitClient(urlparse.urlunparse(parsed)), parsed.path
+ return HttpGitClient(urlparse.urlunparse(parsed), **kwargs), parsed.path
+ elif parsed.scheme == 'file':
+ return default_local_git_client_cls(**kwargs), parsed.path
+
+ raise ValueError("unknown scheme '%s'" % parsed.scheme)
+
+
+def get_transport_and_path(location, **kwargs):
+ """Obtain a git client from a URL.
+
+ :param location: URL or path
+ :param thin_packs: Whether or not thin packs should be retrieved
+ :param report_activity: Optional callback for reporting transport
+ activity.
+ :return: Tuple with client instance and relative path.
+ """
+ # First, try to parse it as a URL
+ try:
+ return get_transport_and_path_from_url(location, **kwargs)
+ except ValueError:
+ pass
- if parsed.scheme and not parsed.netloc:
+ if ':' in location and not '@' in location:
# SSH with no user@, zero or one leading slash.
- return SSHGitClient(parsed.scheme), parsed.path
- elif parsed.scheme:
- raise ValueError('Unknown git protocol scheme: %s' % parsed.scheme)
- elif '@' in parsed.path and ':' in parsed.path:
+ (hostname, path) = location.split(':')
+ return SSHGitClient(hostname, **kwargs), path
+ elif '@' in location and ':' in location:
# SSH with user@host:foo.
- user_host, path = parsed.path.split(':')
+ user_host, path = location.split(':')
user, host = user_host.rsplit('@')
- return SSHGitClient(host, username=user), path
+ return SSHGitClient(host, username=user, **kwargs), path
# Otherwise, assume it's a local path.
- return SubprocessGitClient(), uri
+ return default_local_git_client_cls(**kwargs), location
git.samba.org / jelmer / dulwich.git / blobdiff