transport.py

   1 # Copyright (C) 2006 Jelmer Vernooij <jelmer@samba.org>
   2 # -*- coding: utf-8 -*-
   3
   4 # This program is free software; you can redistribute it and/or modify
   5 # it under the terms of the GNU General Public License as published by
   6 # the Free Software Foundation; either version 3 of the License, or
   7 # (at your option) any later version.
   8
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU General Public License for more details.
  13
  14 # You should have received a copy of the GNU General Public License
  15 # along with this program; if not, write to the Free Software
  16 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  17 """Simple transport for accessing Subversion smart servers."""
  18
  19 import bzrlib
  20 from bzrlib import debug, urlutils
  21 from bzrlib.errors import (NoSuchFile, TransportNotPossible,
  22                            FileExists, NotLocalUrl, InvalidURL, RedirectRequested)
  23 from bzrlib.trace import mutter, warning
  24 from bzrlib.transport import Transport
  25
  26 import bzrlib.plugins.svn
  27 from bzrlib.plugins.svn import ra
  28 from bzrlib.plugins.svn.auth import create_auth_baton
  29 from bzrlib.plugins.svn.client import get_config
  30 from bzrlib.plugins.svn.core import SubversionException
  31 from bzrlib.plugins.svn.errors import convert_svn_error, NoSvnRepositoryPresent, ERR_BAD_URL, ERR_RA_SVN_REPOS_NOT_FOUND, ERR_FS_ALREADY_EXISTS, ERR_FS_NOT_DIRECTORY, ERR_RA_DAV_RELOCATED, ERR_RA_DAV_PATH_NOT_FOUND
  32 import urlparse
  33 import urllib
  34
  35 svn_config = get_config()
  36
  37 def get_client_string():
  38     """Return a string that can be send as part of the User Agent string."""
  39     return "bzr%s+bzr-svn%s" % (bzrlib.__version__, bzrlib.plugins.svn.__version__)
  40
  41
  42 # Don't run any tests on SvnTransport as it is not intended to be
  43 # a full implementation of Transport
  44 def get_test_permutations():
  45     return []
  46
  47
  48 def get_svn_ra_transport(bzr_transport):
  49     """Obtain corresponding SvnRaTransport for a stock Bazaar transport."""
  50     if isinstance(bzr_transport, SvnRaTransport):
  51         return bzr_transport
  52
  53     ra_transport = getattr(bzr_transport, "_svn_ra", None)
  54     if ra_transport is not None:
  55         return ra_transport
  56
  57     # Save _svn_ra transport here so we don't have to connect again next time
  58     # we try to use bzr svn on this transport
  59     ra_transport = SvnRaTransport(bzr_transport.base)
  60     bzr_transport._svn_ra = ra_transport
  61     return ra_transport
  62
  63
  64 def _url_unescape_uri(url):
  65     (scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
  66     if scheme in ("http", "https"):
  67         # Without this, URLs with + in them break
  68         path = urllib.unquote(path)
  69     return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
  70
  71
  72 def _url_escape_uri(url):
  73     (scheme, netloc, path, query, fragment) = urlparse.urlsplit(url)
  74     if scheme in ("http", "https"):
  75         # Without this, URLs with + in them break
  76         path = urllib.quote(path)
  77     return urlparse.urlunsplit((scheme, netloc, path, query, fragment))
  78
  79
  80 svnplus_warning_showed = False
  81
  82 def warn_svnplus(url):
  83     global svnplus_warning_showed
  84     if not svnplus_warning_showed:
  85         warning("The svn+ syntax is deprecated, use %s instead.", url)
  86         svnplus_warning_showed = True
  87
  88
  89 def bzr_to_svn_url(url):
  90     """Convert a Bazaar URL to a URL understood by Subversion.
  91
  92     This will possibly remove the svn+ prefix.
  93     """
  94     if (url.startswith("svn+http://") or
  95         url.startswith("svn+file://") or
  96         url.startswith("svn+https://")):
  97         url = url[len("svn+"):] # Skip svn+
  98         warn_svnplus(url)
  99
 100     url = _url_unescape_uri(url)
 101
 102     # The SVN libraries don't like trailing slashes...
 103     url = url.rstrip('/')
 104
 105     return url
 106
 107
 108 def Connection(url):
 109     try:
 110         mutter('opening SVN RA connection to %r' % url)
 111         ret = ra.RemoteAccess(url.encode('utf8'),
 112                 auth=create_auth_baton(url),
 113                 client_string_func=get_client_string)
 114     except SubversionException, (msg, num):
 115         if num in (ERR_RA_SVN_REPOS_NOT_FOUND,):
 116             raise NoSvnRepositoryPresent(url=url)
 117         if num == ERR_BAD_URL:
 118             raise InvalidURL(url)
 119         if num == ERR_RA_DAV_PATH_NOT_FOUND:
 120             raise NoSuchFile(url)
 121         if num == ERR_RA_DAV_RELOCATED:
 122             # Try to guess the new url
 123             if "'" in msg:
 124                 new_url = msg.split("'")[1]
 125             elif "«" in msg:
 126                 new_url = msg[msg.index("»")+2:msg.index("«")]
 127             else:
 128                 new_url = None
 129             raise RedirectRequested(source=url, target=new_url,
 130                                     is_permanent=True)
 131         raise
 132
 133     from bzrlib.plugins.svn import lazy_check_versions
 134     lazy_check_versions()
 135
 136     return ret
 137
 138
 139 class ConnectionPool(object):
 140     """Collection of connections to a Subversion repository."""
 141     def __init__(self):
 142         self.connections = set()
 143
 144     def get(self, url):
 145         # Check if there is an existing connection we can use
 146         for c in self.connections:
 147             assert not c.busy, "busy connection in pool"
 148             if c.url == url:
 149                 self.connections.remove(c)
 150                 return c
 151         # Nothing available? Just pick an existing one and reparent:
 152         if len(self.connections) == 0:
 153             return Connection(url)
 154         c = self.connections.pop()
 155         try:
 156             c.reparent(_url_escape_uri(url))
 157             return c
 158         except NotImplementedError:
 159             self.connections.add(c)
 160             return Connection(url)
 161         except:
 162             self.connections.add(c)
 163             raise
 164
 165     def add(self, connection):
 166         assert not connection.busy, "adding busy connection in pool"
 167         self.connections.add(connection)
 168
 169
 170 class SvnRaTransport(Transport):
 171     """Fake transport for Subversion-related namespaces.
 172
 173     This implements just as much of Transport as is necessary
 174     to fool Bazaar. """
 175     @convert_svn_error
 176     def __init__(self, url="", _backing_url=None, pool=None, _uuid=None, _repos_root=None):
 177         bzr_url = url
 178         self.svn_url = bzr_to_svn_url(url)
 179         # _backing_url is an evil hack so the root directory of a repository
 180         # can be accessed on some HTTP repositories.
 181         if _backing_url is None:
 182             _backing_url = self.svn_url
 183         self._backing_url = _backing_url.rstrip("/")
 184         Transport.__init__(self, bzr_url)
 185
 186         if pool is None:
 187             self.connections = ConnectionPool()
 188
 189             # Make sure that the URL is valid by connecting to it.
 190             self.connections.add(self.connections.get(self._backing_url))
 191         else:
 192             self.connections = pool
 193
 194         self._repos_root = _repos_root
 195         self._uuid = _uuid
 196         self.capabilities = {}
 197
 198         from bzrlib.plugins.svn import lazy_check_versions
 199         lazy_check_versions()
 200
 201     def get_connection(self):
 202         return self.connections.get(self._backing_url)
 203
 204     def add_connection(self, conn):
 205         self.connections.add(conn)
 206
 207     def has(self, relpath):
 208         """See Transport.has()."""
 209         # TODO: Raise TransportNotPossible here instead and
 210         # catch it in bzrdir.py
 211         return False
 212
 213     def get(self, relpath):
 214         """See Transport.get()."""
 215         # TODO: Raise TransportNotPossible here instead and
 216         # catch it in bzrdir.py
 217         raise NoSuchFile(path=relpath)
 218
 219     def stat(self, relpath):
 220         """See Transport.stat()."""
 221         raise TransportNotPossible('stat not supported on Subversion')
 222
 223     def put_file(self, name, file, mode=0):
 224         raise TransportNotPossible("put_file not supported on Subversion")
 225
 226     def get_uuid(self):
 227         if self._uuid is None:
 228             conn = self.get_connection()
 229             self.mutter('svn get-uuid')
 230             try:
 231                 return conn.get_uuid()
 232             finally:
 233                 self.add_connection(conn)
 234         return self._uuid
 235
 236     def get_repos_root(self):
 237         root = self.get_svn_repos_root()
 238         if (self.base.startswith("svn+http:") or
 239             self.base.startswith("svn+https:")):
 240             return "svn+%s" % root
 241         return root
 242
 243     def get_svn_repos_root(self):
 244         if self._repos_root is None:
 245             self.mutter('svn get-repos-root')
 246             conn = self.get_connection()
 247             try:
 248                 self._repos_root = conn.get_repos_root()
 249             finally:
 250                 self.add_connection(conn)
 251         return self._repos_root
 252
 253     def get_latest_revnum(self):
 254         conn = self.get_connection()
 255         self.mutter('svn get-latest-revnum')
 256         try:
 257             return conn.get_latest_revnum()
 258         finally:
 259             self.add_connection(conn)
 260
 261     def iter_log(self, paths, from_revnum, to_revnum, limit, discover_changed_paths,
 262                  strict_node_history, include_merged_revisions, revprops):
 263         assert paths is None or isinstance(paths, list)
 264         assert isinstance(from_revnum, int) and isinstance(to_revnum, int)
 265         assert isinstance(limit, int)
 266         from threading import Thread, Semaphore
 267
 268         class logfetcher(Thread):
 269             def __init__(self, transport, *args, **kwargs):
 270                 Thread.__init__(self)
 271                 self.setDaemon(True)
 272                 self.transport = transport
 273                 self.args = args
 274                 self.kwargs = kwargs
 275                 self.pending = []
 276                 self.conn = self.transport.get_connection()
 277                 self.semaphore = Semaphore(0)
 278                 self.busy = False
 279
 280             def next(self):
 281                 self.semaphore.acquire()
 282                 ret = self.pending.pop(0)
 283                 if ret is None:
 284                     self.transport.add_connection(self.conn)
 285                 elif isinstance(ret, Exception):
 286                     self.transport.add_connection(self.conn)
 287                     raise ret
 288                 return ret
 289
 290             def run(self):
 291                 assert not self.busy, "already running"
 292                 self.busy = True
 293                 def rcvr(*args):
 294                     self.pending.append(args)
 295                     self.semaphore.release()
 296                 try:
 297                     try:
 298                         self.conn.get_log(callback=rcvr, *self.args, **self.kwargs)
 299                         self.pending.append(None)
 300                     except Exception, e:
 301                         self.pending.append(e)
 302                 finally:
 303                     self.pending.append(Exception("Some exception was not handled"))
 304                     self.semaphore.release()
 305
 306         if paths is None:
 307             newpaths = None
 308         else:
 309             newpaths = [self._request_path(path) for path in paths]
 310
 311         fetcher = logfetcher(self, paths=newpaths, start=from_revnum, end=to_revnum, limit=limit, discover_changed_paths=discover_changed_paths, strict_node_history=strict_node_history, include_merged_revisions=include_merged_revisions, revprops=revprops)
 312         fetcher.start()
 313         return iter(fetcher.next, None)
 314
 315     def get_log(self, rcvr, paths, from_revnum, to_revnum, limit, discover_changed_paths,
 316                 strict_node_history, include_merged_revisions, revprops):
 317         assert paths is None or isinstance(paths, list), "Invalid paths"
 318
 319         all_true = True
 320         for item in [isinstance(x, str) for x in paths]:
 321             if not item:
 322                 all_true = False
 323                 break
 324
 325         assert paths is None or all_true
 326
 327         self.mutter('svn log -r%d:%d %r' % (from_revnum, to_revnum, paths))
 328
 329         if paths is None:
 330             newpaths = None
 331         else:
 332             newpaths = [self._request_path(path) for path in paths]
 333
 334         conn = self.get_connection()
 335         try:
 336             return conn.get_log(rcvr, newpaths,
 337                     from_revnum, to_revnum,
 338                     limit, discover_changed_paths, strict_node_history,
 339                     include_merged_revisions,
 340                     revprops)
 341         finally:
 342             self.add_connection(conn)
 343
 344     def _open_real_transport(self):
 345         if self._backing_url != self.svn_url:
 346             return self.connections.get(self.svn_url)
 347         return self.get_connection()
 348
 349     def change_rev_prop(self, revnum, name, value):
 350         conn = self.get_connection()
 351         self.mutter('svn change-revprop -r%d %s=%s' % (revnum, name, value))
 352         try:
 353             return conn.change_rev_prop(revnum, name, value)
 354         finally:
 355             self.add_connection(conn)
 356
 357     def get_dir(self, path, revnum, kind=False):
 358         path = self._request_path(path)
 359         conn = self.get_connection()
 360         self.mutter('svn get-dir -r%d %s' % (revnum, path))
 361         try:
 362             return conn.get_dir(path, revnum, kind)
 363         finally:
 364             self.add_connection(conn)
 365
 366     def get_file(self, path, stream, revnum):
 367         path = self._request_path(path)
 368         conn = self.get_connection()
 369         self.mutter('svn get-file -r%d %s' % (revnum, path))
 370         try:
 371             return conn.get_file(path, stream, revnum)
 372         finally:
 373             self.add_connection(conn)
 374
 375     def mutter(self, text, *args):
 376         if 'transport' in debug.debug_flags:
 377             mutter(text, *args)
 378
 379     def _request_path(self, relpath):
 380         if self._backing_url == self.svn_url:
 381             return relpath.strip("/")
 382         newsvnurl = urlutils.join(self.svn_url, relpath)
 383         if newsvnurl == self._backing_url:
 384             return ""
 385         newrelpath = urlutils.relative_url(self._backing_url+"/", newsvnurl+"/").strip("/")
 386         self.mutter('request path %r -> %r', relpath, newrelpath)
 387         return newrelpath
 388
 389     def list_dir(self, relpath):
 390         assert len(relpath) == 0 or relpath[0] != "/"
 391         if relpath == ".":
 392             relpath = ""
 393         try:
 394             (dirents, _, _) = self.get_dir(relpath, self.get_latest_revnum())
 395         except SubversionException, (msg, num):
 396             if num == ERR_FS_NOT_DIRECTORY:
 397                 raise NoSuchFile(relpath)
 398             raise
 399         return dirents.keys()
 400
 401     def check_path(self, path, revnum):
 402         path = self._request_path(path)
 403         conn = self.get_connection()
 404         self.mutter('svn check-path -r%d %s' % (revnum, path))
 405         try:
 406             return conn.check_path(path, revnum)
 407         finally:
 408             self.add_connection(conn)
 409
 410     @convert_svn_error
 411     def mkdir(self, relpath, message="Creating directory"):
 412         conn = self.get_connection()
 413         self.mutter('svn mkdir %s' % (relpath,))
 414         try:
 415             ce = conn.get_commit_editor({"svn:log": message})
 416             try:
 417                 node = ce.open_root(-1)
 418                 batons = relpath.split("/")
 419                 toclose = [node]
 420                 for i in range(len(batons)):
 421                     node = node.open_directory("/".join(batons[:i]), -1)
 422                     toclose.append(node)
 423                 toclose.append(node.add_directory(relpath, None, -1))
 424                 for c in reversed(toclose):
 425                     c.close()
 426                 ce.close()
 427             except SubversionException, (msg, num):
 428                 ce.abort()
 429                 if num == ERR_FS_NOT_DIRECTORY:
 430                     raise NoSuchFile(msg)
 431                 if num == ERR_FS_ALREADY_EXISTS:
 432                     raise FileExists(msg)
 433                 raise
 434         finally:
 435             self.add_connection(conn)
 436
 437     def has_capability(self, cap):
 438         if cap in self.capabilities:
 439             return self.capabilities[cap]
 440         conn = self.get_connection()
 441         self.mutter('svn has-capability %s' % (cap,))
 442         try:
 443             try:
 444                 self.capabilities[cap] = conn.has_capability(cap)
 445             except NotImplementedError:
 446                 self.capabilities[cap] = False # Assume the worst
 447             return self.capabilities[cap]
 448         finally:
 449             self.add_connection(conn)
 450
 451     def revprop_list(self, revnum):
 452         conn = self.get_connection()
 453         self.mutter('svn revprop-list -r%d' % (revnum,))
 454         try:
 455             return conn.rev_proplist(revnum)
 456         finally:
 457             self.add_connection(conn)
 458
 459     def get_locations(self, path, peg_revnum, revnums):
 460         conn = self.get_connection()
 461         self.mutter('svn get_locations -r%d %s (%r)' % (peg_revnum, path, revnums))
 462         try:
 463             return conn.get_locations(path, peg_revnum, revnums)
 464         finally:
 465             self.add_connection(conn)
 466
 467     def listable(self):
 468         """See Transport.listable().
 469         """
 470         return True
 471
 472     # There is no real way to do locking directly on the transport
 473     # nor is there a need to as the remote server will take care of
 474     # locking
 475     class PhonyLock(object):
 476         def unlock(self):
 477             pass
 478
 479     def lock_read(self, relpath):
 480         """See Transport.lock_read()."""
 481         return self.PhonyLock()
 482
 483     def lock_write(self, path_revs, comment=None, steal_lock=False):
 484         return self.PhonyLock() # FIXME
 485
 486     def _is_http_transport(self):
 487         return False
 488         return (self.svn_url.startswith("http://") or
 489                 self.svn_url.startswith("https://"))
 490
 491     def clone_root(self):
 492         if self._is_http_transport():
 493             return SvnRaTransport(self.get_repos_root(),
 494                                   bzr_to_svn_url(self.base),
 495                                   pool=self.connections)
 496         return SvnRaTransport(self.get_repos_root(),
 497                               pool=self.connections)
 498
 499     def clone(self, offset=None):
 500         """See Transport.clone()."""
 501         if offset is None:
 502             newurl = self.base
 503         else:
 504             newurl = urlutils.join(self.base, offset)
 505
 506         return SvnRaTransport(newurl, pool=self.connections)
 507
 508     def local_abspath(self, relpath):
 509         """See Transport.local_abspath()."""
 510         absurl = self.abspath(relpath)
 511         if self.base.startswith("file:///"):
 512             return urlutils.local_path_from_url(absurl)
 513         raise NotLocalUrl(absurl)
 514
 515     def abspath(self, relpath):
 516         """See Transport.abspath()."""
 517         return urlutils.join(self.base, relpath)