fetch.py

   1 # Copyright (C) 2005-2007 Jelmer Vernooij <jelmer@samba.org>
   2
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program; if not, write to the Free Software
  15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16
  17 import bzrlib
  18 from bzrlib.inventory import Inventory, ROOT_ID
  19 import bzrlib.osutils as osutils
  20 from bzrlib.revision import Revision
  21 from bzrlib.repository import InterRepository
  22 from bzrlib.trace import mutter
  23 import bzrlib.ui as ui
  24
  25 from copy import copy
  26 from cStringIO import StringIO
  27 import md5
  28 import os
  29
  30 from svn.core import SubversionException, Pool
  31 import svn.core, svn.ra
  32
  33 from fileids import generate_file_id
  34 from repository import (SvnRepository, SVN_PROP_BZR_MERGE, SVN_PROP_SVK_MERGE,
  35                 SVN_PROP_BZR_PREFIX, SVN_PROP_BZR_REVPROP_PREFIX,
  36                 SvnRepositoryFormat)
  37 from tree import apply_txdelta_handler
  38
  39
  40 def md5_strings(strings):
  41     s = md5.new()
  42     map(s.update, strings)
  43     return s.hexdigest()
  44
  45
  46 class RevisionBuildEditor(svn.delta.Editor):
  47     def __init__(self, source, target, branch_path, prev_inventory, revid,
  48                  svn_revprops, id_map):
  49         self.branch_path = branch_path
  50         self.old_inventory = prev_inventory
  51         self.inventory = copy(prev_inventory)
  52         self.revid = revid
  53         self.id_map = id_map
  54         self.source = source
  55         self.target = target
  56         self.transact = target.get_transaction()
  57         self.weave_store = target.weave_store
  58         self.dir_baserev = {}
  59         self._parent_ids = None
  60         self._revprops = {}
  61         self._svn_revprops = svn_revprops
  62         self.pool = Pool()
  63
  64     def _get_revision(self, revid):
  65         if self._parent_ids is None:
  66             self._parent_ids = ""
  67
  68         parent_ids = self.source.revision_parents(revid, self._parent_ids)
  69
  70         # Commit SVN revision properties to a Revision object
  71         rev = Revision(revision_id=revid, parent_ids=parent_ids)
  72
  73         rev.timestamp = 1.0 * svn.core.secs_from_timestr(
  74             self._svn_revprops[2], None) #date
  75         rev.timezone = None
  76
  77         rev.committer = self._svn_revprops[0] # author
  78         if rev.committer is None:
  79             rev.committer = ""
  80         rev.message = self._svn_revprops[1] # message
  81
  82         rev.properties = self._revprops
  83         return rev
  84
  85     def open_root(self, base_revnum, baton):
  86         if self.inventory.revision_id is None:
  87             self.dir_baserev[ROOT_ID] = []
  88         else:
  89             self.dir_baserev[ROOT_ID] = [self.inventory.revision_id]
  90         self.inventory.revision_id = self.revid
  91         return ROOT_ID
  92
  93     def _get_existing_id(self, parent_id, path):
  94         if self.id_map.has_key(path):
  95             return self.id_map[path]
  96         return self._get_old_id(parent_id, path)
  97
  98     def _get_old_id(self, parent_id, old_path):
  99         return self.old_inventory[parent_id].children[os.path.basename(old_path)].file_id
 100
 101     def _get_new_id(self, parent_id, new_path):
 102         if self.id_map.has_key(new_path):
 103             return self.id_map[new_path]
 104         return generate_file_id(self.revid, new_path)
 105
 106     def delete_entry(self, path, revnum, parent_id, pool):
 107         path = path.decode("utf-8")
 108         del self.inventory[self._get_old_id(parent_id, path)]
 109
 110     def close_directory(self, id):
 111         if id != ROOT_ID:
 112             self.inventory[id].revision = self.revid
 113
 114             file_weave = self.weave_store.get_weave_or_empty(id, self.transact)
 115             if not file_weave.has_version(self.revid):
 116                 file_weave.add_lines(self.revid, self.dir_baserev[id], [])
 117
 118     def add_directory(self, path, parent_id, copyfrom_path, copyfrom_revnum, pool):
 119         path = path.decode("utf-8")
 120         file_id = self._get_new_id(parent_id, path)
 121
 122         self.dir_baserev[file_id] = []
 123         ie = self.inventory.add_path(path, 'directory', file_id)
 124         ie.revision = self.revid
 125
 126         return file_id
 127
 128     def open_directory(self, path, parent_id, base_revnum, pool):
 129         assert base_revnum >= 0
 130         base_file_id = self._get_old_id(parent_id, path)
 131         base_revid = self.old_inventory[base_file_id].revision
 132         file_id = self._get_existing_id(parent_id, path)
 133         if file_id == base_file_id:
 134             self.dir_baserev[file_id] = [base_revid]
 135             ie = self.inventory[file_id]
 136         else:
 137             # Replace if original was inside this branch
 138             # change id of base_file_id to file_id
 139             ie = self.inventory[base_file_id]
 140             for name in ie.children:
 141                 ie.children[name].parent_id = file_id
 142             # FIXME: Don't touch inventory internals
 143             del self.inventory._byid[base_file_id]
 144             self.inventory._byid[file_id] = ie
 145             ie.file_id = file_id
 146             self.dir_baserev[file_id] = []
 147         ie.revision = self.revid
 148         return file_id
 149
 150     def change_dir_prop(self, id, name, value, pool):
 151         if name == SVN_PROP_BZR_MERGE:
 152             if id != ROOT_ID:
 153                 mutter('rogue %r on non-root directory' % SVN_PROP_BZR_MERGE)
 154                 return
 155
 156             self._parent_ids = value.splitlines()[-1]
 157         elif name == SVN_PROP_SVK_MERGE:
 158             if self._parent_ids is None:
 159                 # Only set parents using svk:merge if no
 160                 # bzr:merge set.
 161                 pass # FIXME
 162         elif name.startswith(SVN_PROP_BZR_REVPROP_PREFIX):
 163             self._revprops[name[len(SVN_PROP_BZR_REVPROP_PREFIX):]] = value
 164         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 165                       svn.core.SVN_PROP_ENTRY_COMMITTED_REV,
 166                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 167                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 168                       svn.core.SVN_PROP_ENTRY_UUID,
 169                       svn.core.SVN_PROP_EXECUTABLE):
 170             pass
 171         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 172             pass
 173         elif (name.startswith(svn.core.SVN_PROP_PREFIX) or
 174               name.startswith(SVN_PROP_BZR_PREFIX)):
 175             mutter('unsupported file property %r' % name)
 176
 177     def change_file_prop(self, id, name, value, pool):
 178         if name == svn.core.SVN_PROP_EXECUTABLE:
 179             # You'd expect executable to match
 180             # svn.core.SVN_PROP_EXECUTABLE_VALUE, but that's not
 181             # how SVN behaves. It appears to consider the presence
 182             # of the property sufficient to mark it executable.
 183             self.is_executable = (value != None)
 184         elif (name == svn.core.SVN_PROP_SPECIAL):
 185             self.is_symlink = (value != None)
 186         elif name == svn.core.SVN_PROP_ENTRY_COMMITTED_REV:
 187             self.last_file_rev = int(value)
 188         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 189                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 190                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 191                       svn.core.SVN_PROP_ENTRY_UUID,
 192                       svn.core.SVN_PROP_MIME_TYPE):
 193             pass
 194         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 195             pass
 196         elif (name.startswith(svn.core.SVN_PROP_PREFIX) or
 197               name.startswith(SVN_PROP_BZR_PREFIX)):
 198             mutter('unsupported file property %r' % name)
 199
 200     def add_file(self, path, parent_id, copyfrom_path, copyfrom_revnum, baton):
 201         path = path.decode("utf-8")
 202         self.is_symlink = False
 203         self.is_executable = None
 204         self.file_data = ""
 205         self.file_parents = []
 206         self.file_stream = None
 207         self.file_id = self._get_new_id(parent_id, path)
 208         return path
 209
 210     def open_file(self, path, parent_id, base_revnum, pool):
 211         base_file_id = self._get_old_id(parent_id, path)
 212         base_revid = self.old_inventory[base_file_id].revision
 213         self.file_id = self._get_existing_id(parent_id, path)
 214         self.is_executable = None
 215         self.is_symlink = (self.inventory[base_file_id].kind == 'symlink')
 216         file_weave = self.weave_store.get_weave_or_empty(base_file_id, self.transact)
 217         self.file_data = file_weave.get_text(base_revid)
 218         self.file_stream = None
 219         if self.file_id == base_file_id:
 220             self.file_parents = [base_revid]
 221         else:
 222             # Replace
 223             del self.inventory[base_file_id]
 224             self.file_parents = []
 225         return path
 226
 227     def close_file(self, path, checksum):
 228         if self.file_stream is not None:
 229             self.file_stream.seek(0)
 230             lines = osutils.split_lines(self.file_stream.read())
 231         else:
 232             # Data didn't change or file is new
 233             lines = osutils.split_lines(self.file_data)
 234
 235         actual_checksum = md5_strings(lines)
 236         assert checksum is None or checksum == actual_checksum
 237
 238         file_weave = self.weave_store.get_weave_or_empty(self.file_id, self.transact)
 239         if not file_weave.has_version(self.revid):
 240             file_weave.add_lines(self.revid, self.file_parents, lines)
 241
 242         if self.file_id in self.inventory:
 243             ie = self.inventory[self.file_id]
 244         elif self.is_symlink:
 245             ie = self.inventory.add_path(path, 'symlink', self.file_id)
 246         else:
 247             ie = self.inventory.add_path(path, 'file', self.file_id)
 248         ie.revision = self.revid
 249
 250         if self.is_symlink:
 251             ie.symlink_target = lines[0][len("link "):]
 252             ie.text_sha1 = None
 253             ie.text_size = None
 254             ie.text_id = None
 255         else:
 256             ie.text_sha1 = osutils.sha_strings(lines)
 257             ie.text_size = sum(map(len, lines))
 258             if self.is_executable is not None:
 259                 ie.executable = self.is_executable
 260
 261         self.file_stream = None
 262
 263     def close_edit(self):
 264         rev = self._get_revision(self.revid)
 265         self.inventory.revision_id = self.revid
 266         rev.inventory_sha1 = osutils.sha_string(
 267             bzrlib.xml5.serializer_v5.write_inventory_to_string(
 268                 self.inventory))
 269         self.target.add_revision(self.revid, rev, self.inventory)
 270         self.pool.destroy()
 271
 272     def abort_edit(self):
 273         pass
 274
 275     def apply_textdelta(self, file_id, base_checksum):
 276         actual_checksum = md5.new(self.file_data).hexdigest(),
 277         assert (base_checksum is None or base_checksum == actual_checksum,
 278             "base checksum mismatch: %r != %r" % (base_checksum, actual_checksum))
 279         self.file_stream = StringIO()
 280         return apply_txdelta_handler(StringIO(self.file_data), self.file_stream, self.pool)
 281
 282
 283 class InterSvnRepository(InterRepository):
 284     """Svn to any repository actions."""
 285
 286     _matching_repo_format = SvnRepositoryFormat()
 287
 288     @staticmethod
 289     def _get_repo_format_to_test():
 290         return None
 291
 292     def _find_all(self):
 293         needed = []
 294         parents = {}
 295         for (branch, revnum) in self.source.follow_history(
 296                                                 self.source._latest_revnum):
 297             revid = self.source.generate_revision_id(revnum, branch)
 298             parents[revid] = self.source._mainline_revision_parent(branch, revnum)
 299
 300             if not self.target.has_revision(revid):
 301                 needed.append(revid)
 302         return (needed, parents)
 303
 304     def _find_until(self, revision_id):
 305         needed = []
 306         parents = {}
 307         (path, until_revnum) = self.source.parse_revision_id(revision_id)
 308
 309         prev_revid = None
 310         for (branch, revnum) in self.source.follow_branch(path,
 311                                                           until_revnum):
 312             revid = self.source.generate_revision_id(revnum, branch)
 313
 314             if prev_revid is not None:
 315                 parents[prev_revid] = revid
 316
 317             prev_revid = revid
 318
 319             if not self.target.has_revision(revid):
 320                 needed.append(revid)
 321
 322         parents[prev_revid] = None
 323         return (needed, parents)
 324
 325     def copy_content(self, revision_id=None, basis=None, pb=None):
 326         """See InterRepository.copy_content."""
 327         # Dictionary with paths as keys, revnums as values
 328
 329         # Loop over all the revnums until revision_id
 330         # (or youngest_revnum) and call self.target.add_revision()
 331         # or self.target.add_inventory() each time
 332         needed = []
 333         parents = {}
 334         self.target.lock_read()
 335         try:
 336             if revision_id is None:
 337                 (needed, parents) = self._find_all()
 338             else:
 339                 (needed, parents) = self._find_until(revision_id)
 340         finally:
 341             self.target.unlock()
 342
 343         if len(needed) == 0:
 344             # Nothing to fetch
 345             return
 346
 347         repos_root = self.source.transport.get_repos_root()
 348
 349         needed.reverse()
 350         prev_revid = None
 351         transport = self.source.transport
 352         self.target.lock_write()
 353         if pb is None:
 354             pb = ui.ui_factory.nested_progress_bar()
 355             nested_pb = pb
 356         else:
 357             nested_pb = None
 358         num = 0
 359         try:
 360             for revid in needed:
 361                 (branch, revnum) = self.source.parse_revision_id(revid)
 362                 pb.update('copying revision', num, len(needed))
 363
 364                 parent_revid = parents[revid]
 365
 366                 if parent_revid is None:
 367                     parent_inv = Inventory()
 368                 elif prev_revid != parent_revid:
 369                     parent_inv = self.target.get_inventory(parent_revid)
 370                 else:
 371                     parent_inv = prev_inv
 372
 373                 changes = self.source._log.get_revision_paths(revnum, branch)
 374                 renames = self.source.revision_fileid_renames(revid)
 375                 id_map = self.source.transform_fileid_map(self.source.uuid,
 376                                             revnum, branch, changes, renames)
 377
 378                 editor = RevisionBuildEditor(self.source, self.target, branch,
 379                                              parent_inv, revid,
 380                                          self.source._log.get_revision_info(revnum),
 381                                          id_map)
 382
 383                 pool = Pool()
 384                 edit, edit_baton = svn.delta.make_editor(editor, pool)
 385
 386                 if parent_revid is None:
 387                     transport.reparent("%s/%s" % (repos_root, branch))
 388                     reporter, reporter_baton = transport.do_update(
 389                                    revnum, "", True, edit, edit_baton, pool)
 390
 391                     # Report status of existing paths
 392                     svn.ra.reporter2_invoke_set_path(reporter, reporter_baton,
 393                         "", revnum, True, None, pool)
 394                 else:
 395                     (parent_branch, parent_revnum) = self.source.parse_revision_id(parent_revid)
 396                     transport.reparent("%s/%s" % (repos_root, parent_branch))
 397
 398                     if parent_branch != branch:
 399                         switch_url = "%s/%s" % (repos_root, branch)
 400                         reporter, reporter_baton = transport.do_switch(
 401                                    revnum, "", True,
 402                                    switch_url, edit, edit_baton, pool)
 403                     else:
 404                         reporter, reporter_baton = transport.do_update(
 405                                    revnum, "", True, edit, edit_baton, pool)
 406
 407                     # Report status of existing paths
 408                     svn.ra.reporter2_invoke_set_path(reporter, reporter_baton,
 409                         "", parent_revnum, False, None, pool)
 410
 411                 transport.lock()
 412                 svn.ra.reporter2_invoke_finish_report(reporter, reporter_baton, pool)
 413                 transport.unlock()
 414
 415                 prev_inv = editor.inventory
 416                 prev_revid = revid
 417                 pool.destroy()
 418                 num += 1
 419         finally:
 420             self.target.unlock()
 421             if nested_pb is not None:
 422                 nested_pb.finished()
 423         self.source.transport.reparent(repos_root)
 424
 425     def fetch(self, revision_id=None, pb=None):
 426         """Fetch revisions. """
 427         self.copy_content(revision_id=revision_id, pb=pb)
 428
 429     @staticmethod
 430     def is_compatible(source, target):
 431         """Be compatible with SvnRepository."""
 432         # FIXME: Also check target uses VersionedFile
 433         return isinstance(source, SvnRepository)
 434