Fetch revisions in chunks of a 1000.
authorJelmer Vernooij <jelmer@samba.org>
Thu, 20 Sep 2007 19:28:59 +0000 (21:28 +0200)
committerJelmer Vernooij <jelmer@samba.org>
Thu, 20 Sep 2007 19:28:59 +0000 (21:28 +0200)
logwalker.py

index c03fd7f74da2355788c3cb61f97eef3c2d610b13..80cb025bf6bb934284bd260cffdad92b0351df6d 100644 (file)
@@ -28,6 +28,8 @@ import base64
 
 from cache import sqlite3
 
+LOG_CHUNK_LIMIT = 1000
+
 def _escape_commit_message(message):
     """Replace xml-incompatible control characters."""
     if message is None:
@@ -51,7 +53,7 @@ def _escape_commit_message(message):
 
 class LogWalker(object):
     """Easy way to access the history of a Subversion repository."""
-    def __init__(self, transport, cache_db=None):
+    def __init__(self, transport, cache_db=None, limit=None):
         """Create a new instance.
 
         :param transport:   SvnRaTransport to use to access the repository.
@@ -63,6 +65,11 @@ class LogWalker(object):
         self.url = transport.base
         self._transport = None
 
+        if limit is not None:
+            self._limit = limit
+        else:
+            self._limit = LOG_CHUNK_LIMIT
+
         if cache_db is None:
             self.db = sqlite3.connect(":memory:")
         else:
@@ -119,12 +126,14 @@ class LogWalker(object):
             if self.saved_revnum % 1000 == 0:
                 self.db.commit()
 
-        pool = Pool()
         try:
             try:
-                self._get_transport().get_log("/", self.saved_revnum, 
-                                             to_revnum, 0, True, True, rcvr, 
-                                             pool)
+                while self.saved_revnum < to_revnum:
+                    pool = Pool()
+                    self._get_transport().get_log("/", self.saved_revnum, 
+                                             to_revnum, self._limit, True, 
+                                             True, rcvr, pool)
+                    pool.destroy()
             finally:
                 pb.finished()
         except SubversionException, (_, num):
@@ -133,7 +142,6 @@ class LogWalker(object):
                     revision="Revision number %d" % to_revnum)
             raise
         self.db.commit()
-        pool.destroy()
 
     def follow_path(self, path, revnum):
         """Return iterator over all the revisions between revnum and