Fix bug in revid caching.
[jelmer/subvertpy.git] / logwalker.py
index 71f1a3c83b515f8b8fedf9367c413fedc0da1ae0..e85bcbf5fc639b5ef7d8e8818958c05c077ec778 100644 (file)
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+"""Cache of the Subversion history log."""
 
-from bzrlib.errors import NoSuchRevision, BzrNewError, NotBranchError
-from bzrlib.progress import ProgressBar, DummyProgress
-from bzrlib.trace import mutter
+from bzrlib.errors import NoSuchRevision
+import bzrlib.ui as ui
 
 import os
-from bsddb import dbshelve as shelve
 
-from svn.core import SubversionException
-import svn.ra
+from svn.core import SubversionException, Pool
 from transport import SvnRaTransport
+import svn.core
 
-class NotSvnBranchPath(BzrNewError):
-    """{%(branch_path)s} is not a valid Svn branch path"""
+import base64
 
-    def __init__(self, branch_path):
-        BzrNewError.__init__(self)
-        self.branch_path = branch_path
+from cache import sqlite3
+
+def _escape_commit_message(message):
+    """Replace xml-incompatible control characters."""
+    if message is None:
+        return None
+    import re
+    # FIXME: RBC 20060419 this should be done by the revision
+    # serialiser not by commit. Then we can also add an unescaper
+    # in the deserializer and start roundtripping revision messages
+    # precisely. See repository_implementations/test_repository.py
+    
+    # Python strings can include characters that can't be
+    # represented in well-formed XML; escape characters that
+    # aren't listed in the XML specification
+    # (http://www.w3.org/TR/REC-xml/#NT-Char).
+    message, _ = re.subn(
+        u'[^\x09\x0A\x0D\u0020-\uD7FF\uE000-\uFFFD]+',
+        lambda match: match.group(0).encode('unicode_escape'),
+        message)
+    return message
 
 
 class LogWalker(object):
-    def __init__(self, scheme, ra=None, cache_dir=None, last_revnum=None, repos_url=None, pb=None):
-        if ra is None:
-            ra = SvnRaTransport(repos_url).ra
+    """Easy way to access the history of a Subversion repository."""
+    def __init__(self, transport=None, cache_db=None, last_revnum=None):
+        """Create a new instance.
+
+        :param transport:   SvnRaTransport to use to access the repository.
+        :param cache_db:    Optional sql database connection to use. Doesn't 
+                            cache if not set.
+        :param last_revnum: Last known revnum in the repository. Will be 
+                            determined if not specified.
+        """
+        assert isinstance(transport, SvnRaTransport)
 
         if last_revnum is None:
-            last_revnum = svn.ra.get_latest_revnum(ra)
+            last_revnum = transport.get_latest_revnum()
 
-        self.ra = ra
-        self.scheme = scheme
+        self.last_revnum = last_revnum
 
-        # Try to load cache from file
-        if cache_dir is not None:
-            self.revisions = shelve.open(os.path.join(cache_dir, 'log'))
-        else:
-            self.revisions = {}
-        self.saved_revnum = max(len(self.revisions)-1, 0)
+        self.transport = SvnRaTransport(transport.base)
 
-        if self.saved_revnum < last_revnum:
-            self.fetch_revisions(self.saved_revnum, last_revnum, pb)
+        if cache_db is None:
+            self.db = sqlite3.connect(":memory:")
         else:
-            self.last_revnum = self.saved_revnum
+            self.db = cache_db
+
+        self.db.executescript("""
+          create table if not exists revision(revno integer unique, author text, message text, date text);
+          create unique index if not exists revision_revno on revision (revno);
+          create table if not exists changed_path(rev integer, action text, path text, copyfrom_path text, copyfrom_rev integer);
+          create index if not exists path_rev on changed_path(rev);
+          create index if not exists path_rev_path on changed_path(rev, path);
+        """)
+        self.db.commit()
+        self.saved_revnum = self.db.execute("SELECT MAX(revno) FROM revision").fetchone()[0]
+        if self.saved_revnum is None:
+            self.saved_revnum = 0
+
+    def fetch_revisions(self, to_revnum):
+        """Fetch information about all revisions in the remote repository
+        until to_revnum.
+
+        :param to_revnum: End of range to fetch information for
+        """
+        to_revnum = max(self.last_revnum, to_revnum)
+
+        pb = ui.ui_factory.nested_progress_bar()
 
-    def fetch_revisions(self, from_revnum, to_revnum, pb=None):
         def rcvr(orig_paths, rev, author, date, message, pool):
             pb.update('fetching svn revision info', rev, to_revnum)
-            paths = {}
             if orig_paths is None:
                 orig_paths = {}
             for p in orig_paths:
                 copyfrom_path = orig_paths[p].copyfrom_path
                 if copyfrom_path:
                     copyfrom_path = copyfrom_path.strip("/")
-                paths[p.strip("/")] = (orig_paths[p].action,
-                            copyfrom_path, orig_paths[p].copyfrom_rev)
-
-            self.revisions[str(rev)] = {
-                    'paths': paths,
-                    'author': author,
-                    'date': date,
-                    'message': message
-                    }
-
-        # Don't bother for only a few revisions
-        if abs(self.saved_revnum-to_revnum) < 10:
-            pb = DummyProgress()
-        else:
-            pb = ProgressBar()
 
+                self.db.execute(
+                     "insert into changed_path (rev, path, action, copyfrom_path, copyfrom_rev) values (?, ?, ?, ?, ?)", 
+                     (rev, p.strip("/"), orig_paths[p].action, copyfrom_path, orig_paths[p].copyfrom_rev))
+
+            if message is not None:
+                message = base64.b64encode(message)
+
+            self.db.execute("replace into revision (revno, author, date, message) values (?,?,?,?)", (rev, author, date, message))
+
+            self.saved_revnum = rev
+            if self.saved_revnum % 1000 == 0:
+                self.db.commit()
+
+        pool = Pool()
         try:
             try:
-                mutter('getting log %r:%r' % (self.saved_revnum, to_revnum))
-                svn.ra.get_log(self.ra, ["/"], self.saved_revnum, to_revnum, 
-                               0, True, True, rcvr)
-                self.last_revnum = to_revnum
+                self.transport.get_log("/", self.saved_revnum, to_revnum, 
+                               0, True, True, rcvr, pool)
             finally:
-                pb.clear()
+                pb.finished()
         except SubversionException, (_, num):
             if num == svn.core.SVN_ERR_FS_NO_SUCH_REVISION:
                 raise NoSuchRevision(branch=self, 
                     revision="Revision number %d" % to_revnum)
             raise
+        self.db.commit()
+        pool.destroy()
+
+    def follow_path(self, path, revnum):
+        """Return iterator over all the revisions between revnum and 
+        0 named path or inside path.
+
+        :param path:   Branch path to start reporting (in revnum)
+        :param revnum:        Start revision.
 
-    def follow_history(self, branch_path, revnum):
-        """Return iterator over all the revisions between from_revnum and 
-        to_revnum that touch branch_path."""
+        :return: An iterators that yields tuples with (path, paths, revnum)
+        where paths is a dictionary with all changes that happened in path 
+        in revnum.
+        """
         assert revnum >= 0
 
-        if not branch_path is None and not self.scheme.is_branch(branch_path):
-            raise NotSvnBranchPath(branch_path)
-
-        if branch_path:
-            branch_path = branch_path.strip("/")
-
-        if revnum > self.last_revnum:
-            self.fetch_revisions(self.last_revnum, revnum)
-
-        continue_revnum = None
-        for i in range(revnum+1):
-            i = revnum - i
-
-            if i == 0:
-                continue
-
-            if not (continue_revnum is None or continue_revnum == i):
-                continue
-
-            continue_revnum = None
-
-            rev = self.revisions[str(i)]
-            changed_paths = {}
-            for p in rev['paths']:
-                if (branch_path is None or 
-                    p == branch_path or
-                    branch_path == "" or
-                    p.startswith(branch_path+"/")):
-
-                    try:
-                        (bp, rp) = self.scheme.unprefix(p)
-                        if not changed_paths.has_key(bp):
-                            changed_paths[bp] = {}
-                        changed_paths[bp][p] = rev['paths'][p]
-                    except NotBranchError:
-                        pass
-
-            assert branch_path is None or len(changed_paths) <= 1
-
-            for bp in changed_paths:
-                yield (bp, changed_paths[bp], i)
-
-            if (not branch_path is None and 
-                branch_path in rev['paths'] and 
-                not rev['paths'][branch_path][1] is None):
-                # In this revision, this branch was copied from 
-                # somewhere else
-                # FIXME: What if copyfrom_path is not a branch path?
-                continue_revnum = rev['paths'][branch_path][2]
-                branch_path = rev['paths'][branch_path][1]
-
-    def find_branches(self, revnum):
-        created_branches = {}
-
-        for i in range(revnum):
-            if i == 0:
-                continue
-            rev = self.revisions[str(i)]
-            for p in rev['paths']:
-                if self.scheme.is_branch(p):
-                    if rev['paths'][p][0] in ('R', 'D'):
-                        del created_branches[p]
-                        yield (p, i, False)
-
-                    if rev['paths'][p][0] in ('A', 'R'): 
-                        created_branches[p] = i
-
-        for p in created_branches:
-            yield (p, i, True)
-
-    def get_revision_info(self, revnum, pb=None):
+        if revnum == 0 and path == "":
+            return
+
+        path = path.strip("/")
+
+        while revnum >= 0:
+            revpaths = self.get_revision_paths(revnum, path)
+
+            if revpaths != {}:
+                yield (path, revpaths, revnum)
+
+            if revpaths.has_key(path):
+                if revpaths[path][1] is None:
+                    if revpaths[path][0] in ('A', 'R'):
+                        # this path didn't exist before this revision
+                        return
+                else:
+                    # In this revision, this path was copied from 
+                    # somewhere else
+                    revnum = revpaths[path][2]
+                    path = revpaths[path][1]
+                    continue
+            revnum -= 1
+
+    def get_revision_paths(self, revnum, path=None):
+        """Obtain dictionary with all the changes in a particular revision.
+
+        :param revnum: Subversion revision number
+        :param path: optional path under which to return all entries
+        :returns: dictionary with paths as keys and 
+                  (action, copyfrom_path, copyfrom_rev) as values.
+        """
+
+        if revnum == 0:
+            return {'': ('A', None, -1)}
+                
+        if revnum > self.saved_revnum:
+            self.fetch_revisions(revnum)
+
+        query = "select path, action, copyfrom_path, copyfrom_rev from changed_path where rev="+str(revnum)
+        if path is not None and path != "":
+            query += " and (path='%s' or path like '%s/%%')" % (path, path)
+
+        paths = {}
+        for p, act, cf, cr in self.db.execute(query):
+            paths[p] = (act, cf, cr)
+        return paths
+
+    def get_revision_info(self, revnum):
         """Obtain basic information for a specific revision.
 
         :param revnum: Revision number.
         :returns: Tuple with author, log message and date of the revision.
         """
-        if revnum > self.last_revnum:
-            self.fetch_revisions(self.saved_revnum, revnum, pb)
-        rev = self.revisions[str(revnum)]
-        if rev['author'] is None:
-            author = None
+        assert revnum >= 0
+        if revnum == 0:
+            return (None, None, None)
+        if revnum > self.saved_revnum:
+            self.fetch_revisions(revnum)
+        (author, message, date) = self.db.execute("select author, message, date from revision where revno="+ str(revnum)).fetchone()
+        if message is not None:
+            message = _escape_commit_message(base64.b64decode(message))
+        return (author, message, date)
+
+    def find_latest_change(self, path, revnum, recurse=False):
+        """Find latest revision that touched path.
+
+        :param path: Path to check for changes
+        :param revnum: First revision to check
+        """
+        assert isinstance(path, basestring)
+        assert isinstance(revnum, int) and revnum >= 0
+        if revnum > self.saved_revnum:
+            self.fetch_revisions(revnum)
+
+        if recurse:
+            extra = " or path like '%s/%%'" % path.strip("/")
         else:
-            author = rev['author'].decode('utf-8', 'ignore')
-        return (author, rev['message'].decode('utf-8', 'ignore'), 
-                rev['date'], rev['paths'])
+            extra = ""
+        query = "select rev from changed_path where (path='%s' or ('%s' like (path || '/%%') and (action = 'R' or action = 'A'))%s) and rev <= %d order by rev desc limit 1" % (path.strip("/"), path.strip("/"), extra, revnum)
 
-    
-    def find_latest_change(self, path, revnum):
-        while revnum > 0 and not self.touches_path(path, revnum):
-            revnum = revnum - 1
-        return revnum
+        row = self.db.execute(query).fetchone()
+        if row is None and path == "":
+            return 0
 
-    def touches_path(self, path, revnum):
-        return (path in self.revisions[str(revnum)]['paths'])
+        if row is None:
+            return None
 
-    def find_children(self, path, revnum):
-        # TODO: Find children by walking history, or use 
-        # cache?
-        mutter("svn ls -r %d '%r'" % (revnum, path))
+        return row[0]
 
-        try:
-            (dirents, _, _) = svn.ra.get_dir(
-                self.ra, path.encode('utf8'), revnum)
-        except SubversionException, (_, num):
-            if num == svn.core.SVN_ERR_FS_NOT_DIRECTORY:
-                return
-            raise
+    def touches_path(self, path, revnum):
+        """Check whether path was changed in specified revision.
+
+        :param path:  Path to check
+        :param revnum:  Revision to check
+        """
+        if revnum > self.saved_revnum:
+            self.fetch_revisions(revnum)
+        if revnum == 0:
+            return (path == "")
+        return (self.db.execute("select 1 from changed_path where path='%s' and rev=%d" % (path, revnum)).fetchone() is not None)
 
-        for p in dirents:
-            yield os.path.join(path, p)
-            for c in self.find_children(os.path.join(path, p), revnum):
-                yield c
+    def find_children(self, path, revnum):
+        """Find all children of path in revnum."""
+        path = path.strip("/")
+        if self.transport.check_path(path, revnum) == svn.core.svn_node_file:
+            return []
+        class TreeLister(svn.delta.Editor):
+            def __init__(self, base):
+                self.files = []
+                self.base = base
+
+            def set_target_revision(self, revnum):
+                pass
+
+            def open_root(self, revnum, baton):
+                return path
+
+            def add_directory(self, path, parent_baton, copyfrom_path, copyfrom_revnum, pool):
+                self.files.append(os.path.join(self.base, path))
+                return path
+
+            def change_dir_prop(self, id, name, value, pool):
+                pass
+
+            def change_file_prop(self, id, name, value, pool):
+                pass
+
+            def add_file(self, path, parent_id, copyfrom_path, copyfrom_revnum, baton):
+                self.files.append(os.path.join(self.base, path))
+                return path
+
+            def close_dir(self, id):
+                pass
+
+            def close_file(self, path, checksum):
+                pass
+
+            def close_edit(self):
+                pass
+
+            def abort_edit(self):
+                pass
+
+            def apply_textdelta(self, file_id, base_checksum):
+                pass
+        pool = Pool()
+        editor = TreeLister(path)
+        edit, baton = svn.delta.make_editor(editor, pool)
+        root_repos = self.transport.get_repos_root()
+        self.transport.reparent(os.path.join(root_repos, path))
+        reporter = self.transport.do_update(
+                        revnum, "", True, edit, baton, pool)
+        reporter.set_path("", revnum, True, None, pool)
+        reporter.finish_report(pool)
+        return editor.files
+
+    def get_previous(self, path, revnum):
+        """Return path,revnum pair specified pair was derived from.
+
+        :param path:  Path to check
+        :param revnum:  Revision to check
+        """
+        assert revnum >= 0
+        if revnum > self.saved_revnum:
+            self.fetch_revisions(revnum)
+        if revnum == 0:
+            return (None, -1)
+        row = self.db.execute("select action, copyfrom_path, copyfrom_rev from changed_path where path='%s' and rev=%d" % (path, revnum)).fetchone()
+        if row[2] == -1:
+            if row[0] == 'A':
+                return (None, -1)
+            return (path, revnum-1)
+        return (row[1], row[2])