Fix bug in revid caching.
[jelmer/subvertpy.git] / logwalker.py
index 530b03d39a0cdc37a8c79619ac1b0a84c8f3daef..e85bcbf5fc639b5ef7d8e8818958c05c077ec778 100644 (file)
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""Cache of the Subversion history log."""
 
-from bzrlib.errors import NoSuchRevision, BzrError, NotBranchError
-from bzrlib.progress import ProgressBar, DummyProgress
-from bzrlib.trace import mutter
+from bzrlib.errors import NoSuchRevision
+import bzrlib.ui as ui
 
 import os
 
@@ -26,12 +26,7 @@ import svn.core
 
 import base64
 
-try:
-    import sqlite3
-except ImportError:
-    from pysqlite2 import dbapi2 as sqlite3
-
-shelves = {}
+from cache import sqlite3
 
 def _escape_commit_message(message):
     """Replace xml-incompatible control characters."""
@@ -54,27 +49,16 @@ def _escape_commit_message(message):
     return message
 
 
-class NotSvnBranchPath(BzrError):
-    _fmt = """{%(branch_path)s}:%(revnum)s is not a valid Svn branch path"""
-
-    def __init__(self, branch_path, revnum=None):
-        BzrError.__init__(self)
-        self.branch_path = branch_path
-        self.revnum = revnum
-
-
 class LogWalker(object):
     """Easy way to access the history of a Subversion repository."""
-    def __init__(self, scheme, transport=None, cache_db=None, last_revnum=None, pb=None):
+    def __init__(self, transport=None, cache_db=None, last_revnum=None):
         """Create a new instance.
 
-        :param scheme:  Branching scheme to use.
         :param transport:   SvnRaTransport to use to access the repository.
         :param cache_db:    Optional sql database connection to use. Doesn't 
                             cache if not set.
         :param last_revnum: Last known revnum in the repository. Will be 
                             determined if not specified.
-        :param pb:          Progress bar to report progress to.
         """
         assert isinstance(transport, SvnRaTransport)
 
@@ -83,8 +67,7 @@ class LogWalker(object):
 
         self.last_revnum = last_revnum
 
-        self.transport = SvnRaTransport(transport.get_repos_root())
-        self.scheme = scheme
+        self.transport = SvnRaTransport(transport.base)
 
         if cache_db is None:
             self.db = sqlite3.connect(":memory:")
@@ -95,6 +78,7 @@ class LogWalker(object):
           create table if not exists revision(revno integer unique, author text, message text, date text);
           create unique index if not exists revision_revno on revision (revno);
           create table if not exists changed_path(rev integer, action text, path text, copyfrom_path text, copyfrom_rev integer);
+          create index if not exists path_rev on changed_path(rev);
           create index if not exists path_rev_path on changed_path(rev, path);
         """)
         self.db.commit()
@@ -102,16 +86,18 @@ class LogWalker(object):
         if self.saved_revnum is None:
             self.saved_revnum = 0
 
-    def fetch_revisions(self, to_revnum, pb=None):
+    def fetch_revisions(self, to_revnum):
         """Fetch information about all revisions in the remote repository
         until to_revnum.
 
         :param to_revnum: End of range to fetch information for
-        :param pb: Optional progress bar to use
         """
+        to_revnum = max(self.last_revnum, to_revnum)
+
+        pb = ui.ui_factory.nested_progress_bar()
+
         def rcvr(orig_paths, rev, author, date, message, pool):
             pb.update('fetching svn revision info', rev, to_revnum)
-            paths = {}
             if orig_paths is None:
                 orig_paths = {}
             for p in orig_paths:
@@ -129,23 +115,16 @@ class LogWalker(object):
             self.db.execute("replace into revision (revno, author, date, message) values (?,?,?,?)", (rev, author, date, message))
 
             self.saved_revnum = rev
-
-        to_revnum = max(self.last_revnum, to_revnum)
-
-        # Don't bother for only a few revisions
-        if abs(self.saved_revnum-to_revnum) < 10:
-            pb = DummyProgress()
-        else:
-            pb = ProgressBar()
+            if self.saved_revnum % 1000 == 0:
+                self.db.commit()
 
         pool = Pool()
         try:
             try:
-                mutter('getting log %r:%r' % (self.saved_revnum, to_revnum))
-                self.transport.get_log(["/"], self.saved_revnum, to_revnum, 
+                self.transport.get_log("/", self.saved_revnum, to_revnum, 
                                0, True, True, rcvr, pool)
             finally:
-                pb.clear()
+                pb.finished()
         except SubversionException, (_, num):
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise NoSuchRevision(branch=self, 
@@ -154,131 +133,106 @@ class LogWalker(object):
         self.db.commit()
         pool.destroy()
 
-    def follow_history(self, branch_path, revnum):
+    def follow_path(self, path, revnum):
         """Return iterator over all the revisions between revnum and 
-        0 that touch branch_path.
-        
-        :param branch_path:   Branch path to start reporting (in revnum)
+        0 named path or inside path.
+
+        :param path:   Branch path to start reporting (in revnum)
         :param revnum:        Start revision.
+
+        :return: An iterators that yields tuples with (path, paths, revnum)
+        where paths is a dictionary with all changes that happened in path 
+        in revnum.
         """
         assert revnum >= 0
 
-        if revnum == 0 and branch_path in (None, ""):
+        if revnum == 0 and path == "":
             return
 
-        if not branch_path is None and not self.scheme.is_branch(branch_path):
-            raise NotSvnBranchPath(branch_path, revnum)
-
-        if branch_path:
-            branch_path = branch_path.strip("/")
-
-        if revnum > self.saved_revnum:
-            self.fetch_revisions(revnum)
-
-        continue_revnum = None
-        for i in range(revnum+1):
-            i = revnum - i
-
-            if i == 0:
-                continue
-
-            if not (continue_revnum is None or continue_revnum == i):
-                continue
-
-            continue_revnum = None
-
-            changed_paths = {}
-            revpaths = self._get_revision_paths(i)
-            for p in revpaths:
-                if (branch_path is None or 
-                    p == branch_path or
-                    branch_path == "" or
-                    p.startswith(branch_path+"/")):
-
-                    try:
-                        (bp, rp) = self.scheme.unprefix(p)
-                        if not changed_paths.has_key(bp):
-                            changed_paths[bp] = {}
-                        changed_paths[bp][p] = revpaths[p]
-                    except NotBranchError:
-                        pass
-
-            assert branch_path is None or len(changed_paths) <= 1
-
-            for bp in changed_paths:
-                yield (bp, changed_paths[bp], i)
-
-            if (not branch_path is None and 
-                branch_path in revpaths and 
-                not revpaths[branch_path][1] is None):
-                # In this revision, this branch was copied from 
-                # somewhere else
-                # FIXME: What if copyfrom_path is not a branch path?
-                continue_revnum = revpaths[branch_path][2]
-                branch_path = revpaths[branch_path][1]
-
-    def find_branches(self, revnum):
-        """Find all branches that were changed in the specified revision number.
-
-        :param revnum: Revision to search for branches.
+        path = path.strip("/")
+
+        while revnum >= 0:
+            revpaths = self.get_revision_paths(revnum, path)
+
+            if revpaths != {}:
+                yield (path, revpaths, revnum)
+
+            if revpaths.has_key(path):
+                if revpaths[path][1] is None:
+                    if revpaths[path][0] in ('A', 'R'):
+                        # this path didn't exist before this revision
+                        return
+                else:
+                    # In this revision, this path was copied from 
+                    # somewhere else
+                    revnum = revpaths[path][2]
+                    path = revpaths[path][1]
+                    continue
+            revnum -= 1
+
+    def get_revision_paths(self, revnum, path=None):
+        """Obtain dictionary with all the changes in a particular revision.
+
+        :param revnum: Subversion revision number
+        :param path: optional path under which to return all entries
+        :returns: dictionary with paths as keys and 
+                  (action, copyfrom_path, copyfrom_rev) as values.
         """
-        created_branches = {}
 
+        if revnum == 0:
+            return {'': ('A', None, -1)}
+                
         if revnum > self.saved_revnum:
             self.fetch_revisions(revnum)
 
-        for i in range(revnum+1):
-            if i == 0:
-                paths = {'': ('A', None, None)}
-            else:
-                paths = self._get_revision_paths(i)
-            for p in paths:
-                if self.scheme.is_branch(p):
-                    if paths[p][0] in ('R', 'D'):
-                        del created_branches[p]
-                        yield (p, i, False)
-
-                    if paths[p][0] in ('A', 'R'): 
-                        created_branches[p] = i
+        query = "select path, action, copyfrom_path, copyfrom_rev from changed_path where rev="+str(revnum)
+        if path is not None and path != "":
+            query += " and (path='%s' or path like '%s/%%')" % (path, path)
 
-        for p in created_branches:
-            yield (p, i, True)
-
-    def _get_revision_paths(self, revnum):
         paths = {}
-        for p, act, cf, cr in self.db.execute("select path, action, copyfrom_path, copyfrom_rev from changed_path where rev="+str(revnum)):
+        for p, act, cf, cr in self.db.execute(query):
             paths[p] = (act, cf, cr)
         return paths
 
-    def get_revision_info(self, revnum, pb=None):
+    def get_revision_info(self, revnum):
         """Obtain basic information for a specific revision.
 
         :param revnum: Revision number.
         :returns: Tuple with author, log message and date of the revision.
         """
+        assert revnum >= 0
+        if revnum == 0:
+            return (None, None, None)
         if revnum > self.saved_revnum:
-            self.fetch_revisions(revnum, pb)
+            self.fetch_revisions(revnum)
         (author, message, date) = self.db.execute("select author, message, date from revision where revno="+ str(revnum)).fetchone()
-        if author is None:
-            author = None
-        return (author, _escape_commit_message(base64.b64decode(message)), date)
+        if message is not None:
+            message = _escape_commit_message(base64.b64decode(message))
+        return (author, message, date)
 
-    
-    def find_latest_change(self, path, revnum):
+    def find_latest_change(self, path, revnum, recurse=False):
         """Find latest revision that touched path.
 
         :param path: Path to check for changes
         :param revnum: First revision to check
         """
+        assert isinstance(path, basestring)
+        assert isinstance(revnum, int) and revnum >= 0
         if revnum > self.saved_revnum:
             self.fetch_revisions(revnum)
 
-        row = self.db.execute(
-             "select rev from changed_path where path='%s' and rev <= %d order by rev desc limit 1" % (path.strip("/"), revnum)).fetchone()
+        if recurse:
+            extra = " or path like '%s/%%'" % path.strip("/")
+        else:
+            extra = ""
+        query = "select rev from changed_path where (path='%s' or ('%s' like (path || '/%%') and (action = 'R' or action = 'A'))%s) and rev <= %d order by rev desc limit 1" % (path.strip("/"), path.strip("/"), extra, revnum)
+
+        row = self.db.execute(query).fetchone()
         if row is None and path == "":
             return 0
 
-        assert row is not None, "now latest change for %r:%d" % (path, revnum)
+        if row is None:
+            return None
 
         return row[0]
 
@@ -296,24 +250,58 @@ class LogWalker(object):
 
     def find_children(self, path, revnum):
         """Find all children of path in revnum."""
-        # TODO: Find children by walking history, or use 
-        # cache?
-        mutter("svn ls -r %d '%r' (logwalker)" % (revnum, path))
+        path = path.strip("/")
+        if self.transport.check_path(path, revnum) == svn.core.svn_node_file:
+            return []
+        class TreeLister(svn.delta.Editor):
+            def __init__(self, base):
+                self.files = []
+                self.base = base
 
-        try:
-            (dirents, _, _) = self.transport.get_dir2(
-                path.lstrip("/").encode('utf8'), revnum, 
-                svn.core.SVN_DIRENT_KIND)
-        except SubversionException, (_, num):
-            if num == svn.core.SVN_ERR_FS_NOT_DIRECTORY:
-                return
-            raise
+            def set_target_revision(self, revnum):
+                pass
+
+            def open_root(self, revnum, baton):
+                return path
+
+            def add_directory(self, path, parent_baton, copyfrom_path, copyfrom_revnum, pool):
+                self.files.append(os.path.join(self.base, path))
+                return path
+
+            def change_dir_prop(self, id, name, value, pool):
+                pass
+
+            def change_file_prop(self, id, name, value, pool):
+                pass
+
+            def add_file(self, path, parent_id, copyfrom_path, copyfrom_revnum, baton):
+                self.files.append(os.path.join(self.base, path))
+                return path
 
-        for p in dirents:
-            yield os.path.join(path, p)
-            if dirents[p].kind == svn.core.svn_node_dir:
-                for c in self.find_children(os.path.join(path, p), revnum):
-                    yield c
+            def close_dir(self, id):
+                pass
+
+            def close_file(self, path, checksum):
+                pass
+
+            def close_edit(self):
+                pass
+
+            def abort_edit(self):
+                pass
+
+            def apply_textdelta(self, file_id, base_checksum):
+                pass
+        pool = Pool()
+        editor = TreeLister(path)
+        edit, baton = svn.delta.make_editor(editor, pool)
+        root_repos = self.transport.get_repos_root()
+        self.transport.reparent(os.path.join(root_repos, path))
+        reporter = self.transport.do_update(
+                        revnum, "", True, edit, baton, pool)
+        reporter.set_path("", revnum, True, None, pool)
+        reporter.finish_report(pool)
+        return editor.files
 
     def get_previous(self, path, revnum):
         """Return path,revnum pair specified pair was derived from.
@@ -321,6 +309,7 @@ class LogWalker(object):
         :param path:  Path to check
         :param revnum:  Revision to check
         """
+        assert revnum >= 0
         if revnum > self.saved_revnum:
             self.fetch_revisions(revnum)
         if revnum == 0: