Use locking and write groups properly, fixes compatibility with packs.
[jelmer/subvertpy.git] / fetch.py
index 3b5912238f3ecf441f7bd2f6f7b2fcc44f664fff..bcde7d245b8d6ead5d19083a60ffb03300293c97 100644 (file)
--- a/fetch.py
+++ b/fetch.py
 import bzrlib
 from bzrlib import osutils, ui, urlutils
 from bzrlib.inventory import Inventory
-from bzrlib.revision import Revision
+from bzrlib.revision import Revision, NULL_REVISION
 from bzrlib.repository import InterRepository
 from bzrlib.trace import mutter
 
 from copy import copy
 from cStringIO import StringIO
 import md5
-import os
 
 from svn.core import Pool
 import svn.core
@@ -34,13 +33,18 @@ from fileids import generate_file_id
 from repository import (SvnRepository, SVN_PROP_BZR_ANCESTRY, 
                 SVN_PROP_SVK_MERGE, SVN_PROP_BZR_MERGE,
                 SVN_PROP_BZR_PREFIX, SVN_PROP_BZR_REVISION_INFO, 
-                SVN_PROP_BZR_BRANCHING_SCHEME,
-                SvnRepositoryFormat, parse_revision_metadata,
-                parse_merge_property)
+                SVN_PROP_BZR_BRANCHING_SCHEME, SVN_PROP_BZR_REVISION_ID,
+                SVN_PROP_BZR_FILEIDS, SvnRepositoryFormat, 
+                parse_revision_metadata, parse_merge_property)
 from tree import apply_txdelta_handler
 
 
 def md5_strings(strings):
+    """Return the MD5sum of the concatenation of strings.
+
+    :param strings: Strings to find the MD5sum of.
+    :return: MD5sum
+    """
     s = md5.new()
     map(s.update, strings)
     return s.hexdigest()
@@ -67,7 +71,9 @@ class RevisionBuildEditor(svn.delta.Editor):
         self._svk_merges = []
         self._revinfo = None
         self._svn_revprops = svn_revprops
+        self._premature_deletes = set()
         self.pool = Pool()
+        self.target.start_write_group()
 
     def _get_revision(self, revid):
         """Creates the revision object.
@@ -123,7 +129,7 @@ class RevisionBuildEditor(svn.delta.Editor):
         return self._get_old_id(parent_id, path)
 
     def _get_old_id(self, parent_id, old_path):
-        return self.old_inventory[parent_id].children[os.path.basename(old_path)].file_id
+        return self.old_inventory[parent_id].children[urlutils.basename(old_path)].file_id
 
     def _get_new_id(self, parent_id, new_path):
         if self.id_map.has_key(new_path):
@@ -132,7 +138,10 @@ class RevisionBuildEditor(svn.delta.Editor):
 
     def delete_entry(self, path, revnum, parent_id, pool):
         path = path.decode("utf-8")
-        del self.inventory[self._get_old_id(parent_id, path)]
+        if path in self._premature_deletes:
+            self._premature_deletes.remove(path)
+        else:
+            self.inventory.remove_recursive_id(self._get_old_id(parent_id, path))
 
     def close_directory(self, id):
         self.inventory[id].revision = self.revid
@@ -149,7 +158,19 @@ class RevisionBuildEditor(svn.delta.Editor):
         file_id = self._get_new_id(parent_id, path)
 
         self.dir_baserev[file_id] = []
-        ie = self.inventory.add_path(path, 'directory', file_id)
+        if file_id in self.inventory:
+            # This directory was moved here from somewhere else, but the 
+            # other location hasn't been removed yet. 
+            if copyfrom_path is None:
+                # FIXME: This should never happen!
+                copyfrom_path = self.old_inventory.id2path(file_id)
+            assert copyfrom_path == self.old_inventory.id2path(file_id)
+            assert copyfrom_path not in self._premature_deletes
+            self._premature_deletes.add(copyfrom_path)
+            self.inventory.rename(file_id, parent_id, urlutils.basename(path))
+            ie = self.inventory[file_id]
+        else:
+            ie = self.inventory.add_path(path, 'directory', file_id)
         ie.revision = self.revid
 
         return file_id
@@ -187,7 +208,8 @@ class RevisionBuildEditor(svn.delta.Editor):
                 return
             
             self._bzr_merges = parse_merge_property(value.splitlines()[-1])
-        elif name.startswith(SVN_PROP_BZR_ANCESTRY):
+        elif (name.startswith(SVN_PROP_BZR_ANCESTRY) or 
+              name.startswith(SVN_PROP_BZR_REVISION_ID)):
             pass
         elif name == SVN_PROP_SVK_MERGE:
             self._svk_merges = None # Force Repository.revision_parents() to look it up
@@ -206,11 +228,11 @@ class RevisionBuildEditor(svn.delta.Editor):
             pass
         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
             pass
-        elif name == SVN_PROP_BZR_MERGE:
+        elif name in (SVN_PROP_BZR_MERGE, SVN_PROP_BZR_FILEIDS):
             pass
         elif (name.startswith(svn.core.SVN_PROP_PREFIX) or
               name.startswith(SVN_PROP_BZR_PREFIX)):
-            mutter('unsupported file property %r' % name)
+            mutter('unsupported dir property %r' % name)
 
     def change_file_prop(self, id, name, value, pool):
         if name == svn.core.SVN_PROP_EXECUTABLE: 
@@ -243,6 +265,16 @@ class RevisionBuildEditor(svn.delta.Editor):
         self.file_parents = []
         self.file_stream = None
         self.file_id = self._get_new_id(parent_id, path)
+        if self.file_id in self.inventory:
+            # This file was moved here from somewhere else, but the 
+            # other location hasn't been removed yet. 
+            if copyfrom_path is None:
+                # FIXME: This should never happen!
+                copyfrom_path = self.old_inventory.id2path(self.file_id)
+            assert copyfrom_path == self.old_inventory.id2path(self.file_id)
+            assert copyfrom_path not in self._premature_deletes
+            self._premature_deletes.add(copyfrom_path)
+            self.inventory.rename(self.file_id, parent_id, urlutils.basename(path))
         return path
 
     def open_file(self, path, parent_id, base_revnum, pool):
@@ -301,12 +333,17 @@ class RevisionBuildEditor(svn.delta.Editor):
         self.file_stream = None
 
     def close_edit(self):
+        assert len(self._premature_deletes) == 0
         rev = self._get_revision(self.revid)
         self.inventory.revision_id = self.revid
         rev.inventory_sha1 = osutils.sha_string(
             bzrlib.xml5.serializer_v5.write_inventory_to_string(
                 self.inventory))
-        self.target.add_revision(self.revid, rev, self.inventory)
+        try:
+            self.target.add_revision(self.revid, rev, self.inventory)
+            self.target.commit_write_group()
+        except:
+            self.target.abort_write_group()
         self.pool.destroy()
 
     def abort_edit(self):
@@ -342,6 +379,12 @@ class InterFromSvnRepository(InterRepository):
         return (needed, parents)
 
     def _find_until(self, revision_id):
+        """Find all missing revisions until revision_id
+
+        :param revision_id: Stop revision
+        :return: Tuple with revisions missing and a dictionary with 
+            parents for those revision.
+        """
         needed = []
         parents = {}
         (path, until_revnum, scheme) = self.source.lookup_revision_id(
@@ -363,9 +406,10 @@ class InterFromSvnRepository(InterRepository):
         parents[prev_revid] = None
         return (needed, parents)
 
-    def copy_content(self, revision_id=None, basis=None, pb=None):
+    def copy_content(self, revision_id=None, pb=None):
         """See InterRepository.copy_content."""
-        # FIXME: Use basis
+        if revision_id == NULL_REVISION:
+            return
         # Dictionary with paths as keys, revnums as values
 
         # Loop over all the revnums until revision_id
@@ -384,11 +428,10 @@ class InterFromSvnRepository(InterRepository):
             # Nothing to fetch
             return
 
-        repos_root = self.source.transport.get_repos_root()
+        repos_root = self.source.transport.get_svn_repos_root()
 
         prev_revid = None
         transport = self.source.transport
-        self.target.lock_write()
         if pb is None:
             pb = ui.ui_factory.nested_progress_bar()
             nested_pb = pb
@@ -396,6 +439,7 @@ class InterFromSvnRepository(InterRepository):
             nested_pb = None
         num = 0
         prev_inv = None
+        self.target.lock_write()
         try:
             for revid in reversed(needed):
                 (branch, revnum, scheme) = self.source.lookup_revision_id(revid)
@@ -425,7 +469,10 @@ class InterFromSvnRepository(InterRepository):
                 edit, edit_baton = svn.delta.make_editor(editor, pool)
 
                 if parent_revid is None:
-                    transport.reparent(urlutils.join(repos_root, branch))
+                    branch_url = urlutils.join(repos_root, branch)
+                    transport.reparent(branch_url)
+                    assert transport.svn_url == branch_url.rstrip("/"), \
+                        "Expected %r, got %r" % (transport.svn_url, branch_url)
                     reporter = transport.do_update(
                                    revnum, True, edit, edit_baton, pool)