fetch.py

   1 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   2
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program; if not, write to the Free Software
  15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16 """Fetching revisions from Subversion repositories in batches."""
  17
  18 import bzrlib
  19 from bzrlib.inventory import Inventory
  20 import bzrlib.osutils as osutils
  21 from bzrlib.revision import Revision
  22 from bzrlib.repository import InterRepository
  23 from bzrlib.trace import mutter
  24 import bzrlib.ui as ui
  25
  26 from copy import copy
  27 from cStringIO import StringIO
  28 import md5
  29 import os
  30
  31 from svn.core import Pool
  32 import svn.core
  33
  34 from fileids import generate_file_id
  35 from repository import (SvnRepository, SVN_PROP_BZR_ANCESTRY,
  36                 SVN_PROP_SVK_MERGE, SVN_PROP_BZR_MERGE,
  37                 SVN_PROP_BZR_PREFIX, SVN_PROP_BZR_REVISION_INFO,
  38                 SVN_PROP_BZR_BRANCHING_SCHEME,
  39                 SvnRepositoryFormat, parse_revision_metadata,
  40                 parse_merge_property)
  41 from tree import apply_txdelta_handler
  42
  43
  44 def md5_strings(strings):
  45     s = md5.new()
  46     map(s.update, strings)
  47     return s.hexdigest()
  48
  49
  50 class RevisionBuildEditor(svn.delta.Editor):
  51     """Implementation of the Subversion commit editor interface that builds a
  52     Bazaar revision.
  53     """
  54     def __init__(self, source, target, branch_path, prev_inventory, revid,
  55                  svn_revprops, id_map, scheme):
  56         self.branch_path = branch_path
  57         self.old_inventory = prev_inventory
  58         self.inventory = copy(prev_inventory)
  59         self.revid = revid
  60         self.id_map = id_map
  61         self.scheme = scheme
  62         self.source = source
  63         self.target = target
  64         self.transact = target.get_transaction()
  65         self.weave_store = target.weave_store
  66         self.dir_baserev = {}
  67         self._bzr_merges = []
  68         self._svk_merges = []
  69         self._revinfo = None
  70         self._svn_revprops = svn_revprops
  71         self.pool = Pool()
  72
  73     def _get_revision(self, revid):
  74         """Creates the revision object.
  75
  76         :param revid: Revision id of the revision to create.
  77         """
  78         parent_ids = self.source.revision_parents(revid, self._bzr_merges)
  79
  80         # Commit SVN revision properties to a Revision object
  81         rev = Revision(revision_id=revid, parent_ids=parent_ids)
  82
  83         if self._svn_revprops[2] is not None:
  84             rev.timestamp = 1.0 * svn.core.secs_from_timestr(
  85                 self._svn_revprops[2], None) #date
  86         else:
  87             rev.timestamp = 0 # FIXME: Obtain repository creation time
  88         rev.timezone = None
  89
  90         rev.committer = self._svn_revprops[0] # author
  91         if rev.committer is None:
  92             rev.committer = ""
  93         rev.message = self._svn_revprops[1] # message
  94
  95         if self._revinfo:
  96             parse_revision_metadata(self._revinfo, rev)
  97
  98         return rev
  99
 100     def open_root(self, base_revnum, baton):
 101         if self.old_inventory.root is None:
 102             # First time the root is set
 103             file_id = generate_file_id(self.source, self.revid, "")
 104             self.dir_baserev[file_id] = []
 105         else:
 106             assert self.old_inventory.root.revision is not None
 107             if self.id_map.has_key(""):
 108                 file_id = self.id_map[""]
 109             else:
 110                 file_id = self.old_inventory.root.file_id
 111             self.dir_baserev[file_id] = [self.old_inventory.root.revision]
 112
 113         if self.inventory.root is not None and \
 114                 file_id == self.inventory.root.file_id:
 115             ie = self.inventory.root
 116         else:
 117             ie = self.inventory.add_path("", 'directory', file_id)
 118         ie.revision = self.revid
 119         return file_id
 120
 121     def _get_existing_id(self, parent_id, path):
 122         if self.id_map.has_key(path):
 123             return self.id_map[path]
 124         return self._get_old_id(parent_id, path)
 125
 126     def _get_old_id(self, parent_id, old_path):
 127         return self.old_inventory[parent_id].children[os.path.basename(old_path)].file_id
 128
 129     def _get_new_id(self, parent_id, new_path):
 130         if self.id_map.has_key(new_path):
 131             return self.id_map[new_path]
 132         return generate_file_id(self.source, self.revid, new_path)
 133
 134     def delete_entry(self, path, revnum, parent_id, pool):
 135         path = path.decode("utf-8")
 136         del self.inventory[self._get_old_id(parent_id, path)]
 137
 138     def close_directory(self, id):
 139         self.inventory[id].revision = self.revid
 140
 141         # Only record root if the target repository supports it
 142         if self.target.supports_rich_root:
 143             file_weave = self.weave_store.get_weave_or_empty(id, self.transact)
 144             if not file_weave.has_version(self.revid):
 145                 file_weave.add_lines(self.revid, self.dir_baserev[id], [])
 146
 147     def add_directory(self, path, parent_id, copyfrom_path, copyfrom_revnum,
 148                       pool):
 149         path = path.decode("utf-8")
 150         file_id = self._get_new_id(parent_id, path)
 151
 152         self.dir_baserev[file_id] = []
 153         ie = self.inventory.add_path(path, 'directory', file_id)
 154         ie.revision = self.revid
 155
 156         return file_id
 157
 158     def open_directory(self, path, parent_id, base_revnum, pool):
 159         assert base_revnum >= 0
 160         base_file_id = self._get_old_id(parent_id, path)
 161         base_revid = self.old_inventory[base_file_id].revision
 162         file_id = self._get_existing_id(parent_id, path)
 163         if file_id == base_file_id:
 164             self.dir_baserev[file_id] = [base_revid]
 165             ie = self.inventory[file_id]
 166         else:
 167             # Replace if original was inside this branch
 168             # change id of base_file_id to file_id
 169             ie = self.inventory[base_file_id]
 170             for name in ie.children:
 171                 ie.children[name].parent_id = file_id
 172             # FIXME: Don't touch inventory internals
 173             del self.inventory._byid[base_file_id]
 174             self.inventory._byid[file_id] = ie
 175             ie.file_id = file_id
 176             self.dir_baserev[file_id] = []
 177         ie.revision = self.revid
 178         return file_id
 179
 180     def change_dir_prop(self, id, name, value, pool):
 181         if name == SVN_PROP_BZR_BRANCHING_SCHEME:
 182             if id != self.inventory.root.file_id:
 183                 mutter('rogue %r on non-root directory' % name)
 184                 return
 185         elif name == SVN_PROP_BZR_ANCESTRY+str(self.scheme):
 186             if id != self.inventory.root.file_id:
 187                 mutter('rogue %r on non-root directory' % name)
 188                 return
 189
 190             self._bzr_merges = parse_merge_property(value.splitlines()[-1])
 191         elif name.startswith(SVN_PROP_BZR_ANCESTRY):
 192             pass
 193         elif name == SVN_PROP_SVK_MERGE:
 194             self._svk_merges = None # Force Repository.revision_parents() to look it up
 195         elif name == SVN_PROP_BZR_REVISION_INFO:
 196             if id != self.inventory.root.file_id:
 197                 mutter('rogue %r on non-root directory' % SVN_PROP_BZR_REVISION_INFO)
 198                 return
 199
 200             self._revinfo = value
 201         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 202                       svn.core.SVN_PROP_ENTRY_COMMITTED_REV,
 203                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 204                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 205                       svn.core.SVN_PROP_ENTRY_UUID,
 206                       svn.core.SVN_PROP_EXECUTABLE):
 207             pass
 208         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 209             pass
 210         elif name == SVN_PROP_BZR_MERGE:
 211             pass
 212         elif (name.startswith(svn.core.SVN_PROP_PREFIX) or
 213               name.startswith(SVN_PROP_BZR_PREFIX)):
 214             mutter('unsupported file property %r' % name)
 215
 216     def change_file_prop(self, id, name, value, pool):
 217         if name == svn.core.SVN_PROP_EXECUTABLE:
 218             # You'd expect executable to match
 219             # svn.core.SVN_PROP_EXECUTABLE_VALUE, but that's not
 220             # how SVN behaves. It appears to consider the presence
 221             # of the property sufficient to mark it executable.
 222             self.is_executable = (value != None)
 223         elif (name == svn.core.SVN_PROP_SPECIAL):
 224             self.is_symlink = (value != None)
 225         elif name == svn.core.SVN_PROP_ENTRY_COMMITTED_REV:
 226             self.last_file_rev = int(value)
 227         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 228                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 229                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 230                       svn.core.SVN_PROP_ENTRY_UUID,
 231                       svn.core.SVN_PROP_MIME_TYPE):
 232             pass
 233         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 234             pass
 235         elif (name.startswith(svn.core.SVN_PROP_PREFIX) or
 236               name.startswith(SVN_PROP_BZR_PREFIX)):
 237             mutter('unsupported file property %r' % name)
 238
 239     def add_file(self, path, parent_id, copyfrom_path, copyfrom_revnum, baton):
 240         path = path.decode("utf-8")
 241         self.is_symlink = False
 242         self.is_executable = None
 243         self.file_data = ""
 244         self.file_parents = []
 245         self.file_stream = None
 246         self.file_id = self._get_new_id(parent_id, path)
 247         return path
 248
 249     def open_file(self, path, parent_id, base_revnum, pool):
 250         base_file_id = self._get_old_id(parent_id, path)
 251         base_revid = self.old_inventory[base_file_id].revision
 252         self.file_id = self._get_existing_id(parent_id, path)
 253         self.is_executable = None
 254         self.is_symlink = (self.inventory[base_file_id].kind == 'symlink')
 255         file_weave = self.weave_store.get_weave_or_empty(base_file_id,
 256                                                          self.transact)
 257         self.file_data = file_weave.get_text(base_revid)
 258         self.file_stream = None
 259         if self.file_id == base_file_id:
 260             self.file_parents = [base_revid]
 261         else:
 262             # Replace
 263             del self.inventory[base_file_id]
 264             self.file_parents = []
 265         return path
 266
 267     def close_file(self, path, checksum):
 268         if self.file_stream is not None:
 269             self.file_stream.seek(0)
 270             lines = osutils.split_lines(self.file_stream.read())
 271         else:
 272             # Data didn't change or file is new
 273             lines = osutils.split_lines(self.file_data)
 274
 275         actual_checksum = md5_strings(lines)
 276         assert checksum is None or checksum == actual_checksum
 277
 278         file_weave = self.weave_store.get_weave_or_empty(self.file_id,
 279                                                          self.transact)
 280         if not file_weave.has_version(self.revid):
 281             file_weave.add_lines(self.revid, self.file_parents, lines)
 282
 283         if self.file_id in self.inventory:
 284             ie = self.inventory[self.file_id]
 285         elif self.is_symlink:
 286             ie = self.inventory.add_path(path, 'symlink', self.file_id)
 287         else:
 288             ie = self.inventory.add_path(path, 'file', self.file_id)
 289         ie.revision = self.revid
 290
 291         if self.is_symlink:
 292             ie.symlink_target = lines[0][len("link "):]
 293             ie.text_sha1 = None
 294             ie.text_size = None
 295             ie.text_id = None
 296         else:
 297             ie.text_sha1 = osutils.sha_strings(lines)
 298             ie.text_size = sum(map(len, lines))
 299             if self.is_executable is not None:
 300                 ie.executable = self.is_executable
 301
 302         self.file_stream = None
 303
 304     def close_edit(self):
 305         rev = self._get_revision(self.revid)
 306         self.inventory.revision_id = self.revid
 307         rev.inventory_sha1 = osutils.sha_string(
 308             bzrlib.xml5.serializer_v5.write_inventory_to_string(
 309                 self.inventory))
 310         self.target.add_revision(self.revid, rev, self.inventory)
 311         self.pool.destroy()
 312
 313     def abort_edit(self):
 314         pass
 315
 316     def apply_textdelta(self, file_id, base_checksum):
 317         actual_checksum = md5.new(self.file_data).hexdigest(),
 318         assert (base_checksum is None or base_checksum == actual_checksum,
 319             "base checksum mismatch: %r != %r" % (base_checksum,
 320                                                   actual_checksum))
 321         self.file_stream = StringIO()
 322         return apply_txdelta_handler(StringIO(self.file_data),
 323                                      self.file_stream, self.pool)
 324
 325
 326 class InterFromSvnRepository(InterRepository):
 327     """Svn to any repository actions."""
 328
 329     _matching_repo_format = SvnRepositoryFormat()
 330
 331     @staticmethod
 332     def _get_repo_format_to_test():
 333         return SvnRepositoryFormat()
 334
 335     def _find_all(self):
 336         parents = {}
 337         needed = filter(lambda x: not self.target.has_revision(x),
 338                         self.source.all_revision_ids())
 339         for revid in needed:
 340             (branch, revnum, scheme) = self.source.lookup_revision_id(revid)
 341             parents[revid] = self.source._mainline_revision_parent(branch,
 342                                                revnum, scheme)
 343         return (needed, parents)
 344
 345     def _find_until(self, revision_id):
 346         needed = []
 347         parents = {}
 348         (path, until_revnum, scheme) = self.source.lookup_revision_id(
 349                                                                     revision_id)
 350
 351         prev_revid = None
 352         for (branch, revnum) in self.source.follow_branch(path,
 353                                                           until_revnum, scheme):
 354             revid = self.source.generate_revision_id(revnum, branch, str(scheme))
 355
 356             if prev_revid is not None:
 357                 parents[prev_revid] = revid
 358
 359             prev_revid = revid
 360
 361             if not self.target.has_revision(revid):
 362                 needed.append(revid)
 363
 364         parents[prev_revid] = None
 365         return (needed, parents)
 366
 367     def copy_content(self, revision_id=None, basis=None, pb=None):
 368         """See InterRepository.copy_content."""
 369         # FIXME: Use basis
 370         # Dictionary with paths as keys, revnums as values
 371
 372         # Loop over all the revnums until revision_id
 373         # (or youngest_revnum) and call self.target.add_revision()
 374         # or self.target.add_inventory() each time
 375         self.target.lock_read()
 376         try:
 377             if revision_id is None:
 378                 (needed, parents) = self._find_all()
 379             else:
 380                 (needed, parents) = self._find_until(revision_id)
 381         finally:
 382             self.target.unlock()
 383
 384         if len(needed) == 0:
 385             # Nothing to fetch
 386             return
 387
 388         repos_root = self.source.transport.get_repos_root()
 389
 390         prev_revid = None
 391         transport = self.source.transport
 392         self.target.lock_write()
 393         if pb is None:
 394             pb = ui.ui_factory.nested_progress_bar()
 395             nested_pb = pb
 396         else:
 397             nested_pb = None
 398         num = 0
 399         prev_inv = None
 400         try:
 401             for revid in reversed(needed):
 402                 (branch, revnum, scheme) = self.source.lookup_revision_id(revid)
 403                 pb.update('copying revision', num, len(needed))
 404
 405                 parent_revid = parents[revid]
 406
 407                 if parent_revid is None:
 408                     parent_inv = Inventory(root_id=None)
 409                 elif prev_revid != parent_revid:
 410                     parent_inv = self.target.get_inventory(parent_revid)
 411                 else:
 412                     assert prev_inv is not None
 413                     parent_inv = prev_inv
 414
 415                 changes = self.source._log.get_revision_paths(revnum, branch)
 416                 renames = self.source.revision_fileid_renames(revid)
 417                 id_map = self.source.transform_fileid_map(self.source.uuid,
 418                                       revnum, branch, changes, renames, scheme)
 419
 420                 editor = RevisionBuildEditor(self.source, self.target, branch,
 421                              parent_inv, revid,
 422                              self.source._log.get_revision_info(revnum),
 423                              id_map, scheme)
 424
 425                 pool = Pool()
 426                 edit, edit_baton = svn.delta.make_editor(editor, pool)
 427
 428                 if parent_revid is None:
 429                     transport.reparent("%s/%s" % (repos_root, branch))
 430                     reporter = transport.do_update(
 431                                    revnum, "", True, edit, edit_baton, pool)
 432
 433                     # Report status of existing paths
 434                     reporter.set_path("", revnum, True, None, pool)
 435                 else:
 436                     (parent_branch, parent_revnum, scheme) = \
 437                             self.source.lookup_revision_id(parent_revid)
 438                     transport.reparent("%s/%s" % (repos_root, parent_branch))
 439
 440                     if parent_branch != branch:
 441                         switch_url = "%s/%s" % (repos_root, branch)
 442                         reporter = transport.do_switch(
 443                                    revnum, "", True,
 444                                    switch_url, edit, edit_baton, pool)
 445                     else:
 446                         reporter = transport.do_update(
 447                                    revnum, "", True, edit, edit_baton, pool)
 448
 449                     # Report status of existing paths
 450                     reporter.set_path("", parent_revnum, False, None, pool)
 451
 452                 lock = transport.lock_read(".")
 453                 reporter.finish_report(pool)
 454                 lock.unlock()
 455
 456                 prev_inv = editor.inventory
 457                 prev_revid = revid
 458                 pool.destroy()
 459                 num += 1
 460         finally:
 461             self.target.unlock()
 462             if nested_pb is not None:
 463                 nested_pb.finished()
 464         self.source.transport.reparent(repos_root)
 465
 466     def fetch(self, revision_id=None, pb=None):
 467         """Fetch revisions. """
 468         self.copy_content(revision_id=revision_id, pb=pb)
 469
 470     @staticmethod
 471     def is_compatible(source, target):
 472         """Be compatible with SvnRepository."""
 473         # FIXME: Also check target uses VersionedFile
 474         return isinstance(source, SvnRepository) and \
 475                 target.supports_rich_root()
 476