fetch.py

   1 # Copyright (C) 2005-2006 Jelmer Vernooij <jelmer@samba.org>
   2
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program; if not, write to the Free Software
  15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16
  17 import bzrlib
  18 from bzrlib.decorators import needs_write_lock
  19 from bzrlib.inventory import Inventory, ROOT_ID
  20 import bzrlib.osutils as osutils
  21 from bzrlib.progress import ProgressBar
  22 from bzrlib.revision import Revision
  23 from bzrlib.repository import InterRepository
  24 from bzrlib.trace import mutter
  25
  26 from copy import copy
  27 from cStringIO import StringIO
  28 import md5
  29 import os
  30
  31 from svn.core import SubversionException, Pool
  32 import svn.core, svn.ra
  33
  34 from repository import (SvnRepository, SVN_PROP_BZR_MERGE, SVN_PROP_SVK_MERGE,
  35                 SVN_PROP_BZR_REVPROP_PREFIX, SvnRepositoryFormat)
  36 from tree import apply_txdelta_handler
  37
  38
  39 def md5_strings(strings):
  40     s = md5.new()
  41     map(s.update, strings)
  42     return s.hexdigest()
  43
  44 class RevisionBuildEditor(svn.delta.Editor):
  45     def __init__(self, source, target, branch_path, revnum, prev_inventory, revid, svn_revprops, id_map, parent_branch):
  46         self.branch_path = branch_path
  47         self.old_inventory = prev_inventory
  48         self.inventory = copy(prev_inventory)
  49         self.revid = revid
  50         self.revnum = revnum
  51         self.id_map = id_map
  52         self.parent_branch = parent_branch
  53         self.source = source
  54         self.target = target
  55         self.transact = target.get_transaction()
  56         self.weave_store = target.weave_store
  57
  58         self.dir_baserev = {}
  59
  60         self._parent_ids = None
  61         self._revprops = {}
  62         self._svn_revprops = svn_revprops
  63
  64         self.pool = Pool()
  65
  66     def _get_revision(self, revid):
  67         if self._parent_ids is None:
  68             self._parent_ids = ""
  69
  70         parent_ids = self.source.revision_parents(revid, self._parent_ids)
  71
  72         # Commit SVN revision properties to a Revision object
  73         rev = Revision(revision_id=revid, parent_ids=parent_ids)
  74
  75         rev.timestamp = 1.0 * svn.core.secs_from_timestr(
  76             self._svn_revprops[2], None) #date
  77         rev.timezone = None
  78
  79         rev.committer = self._svn_revprops[0] # author
  80         if rev.committer is None:
  81             rev.committer = ""
  82         rev.message = self._svn_revprops[1] # message
  83
  84         rev.properties = self._revprops
  85         return rev
  86
  87     def open_root(self, base_revnum, baton):
  88         if self.inventory.revision_id is None:
  89             self.dir_baserev[ROOT_ID] = []
  90         else:
  91             self.dir_baserev[ROOT_ID] = [self.inventory.revision_id]
  92         self.inventory.revision_id = self.revid
  93         return ROOT_ID
  94
  95     def relpath(self, path):
  96         return path.strip("/")
  97
  98     def delete_entry(self, path, revnum, parent_baton, pool):
  99         del self.inventory[self.inventory.path2id(path)]
 100
 101     def close_directory(self, id):
 102         revid = self.revid
 103
 104         if id != ROOT_ID:
 105             self.inventory[id].revision = revid
 106
 107             file_weave = self.weave_store.get_weave_or_empty(id, self.transact)
 108             if not file_weave.has_version(revid):
 109                 file_weave.add_lines(revid, self.dir_baserev[id], [])
 110
 111     def add_directory(self, path, parent_baton, copyfrom_path, copyfrom_revnum, pool):
 112         file_id, revision_id = self.id_map[path]
 113
 114         self.dir_baserev[file_id] = []
 115         ie = self.inventory.add_path(path, 'directory', file_id)
 116         ie.revision = revision_id
 117
 118         return file_id
 119
 120     def open_directory(self, path, parent_baton, base_revnum, pool):
 121         file_id, revision_id = self.id_map[path]
 122         assert base_revnum >= 0
 123         base_file_id = self.old_inventory.path2id(path)
 124         base_revid = self.old_inventory[base_file_id].revision
 125         if file_id == base_file_id:
 126             self.dir_baserev[file_id] = [base_revid]
 127             ie = self.inventory[file_id]
 128         else:
 129             # Replace if original was inside this branch
 130             # change id of base_file_id to file_id
 131             ie = self.inventory[base_file_id]
 132             for name in ie.children:
 133                 ie.children[name].parent_id = file_id
 134             # FIXME: Don't touch inventory internals
 135             del self.inventory._byid[base_file_id]
 136             self.inventory._byid[file_id] = ie
 137             ie.file_id = file_id
 138             self.dir_baserev[file_id] = []
 139         ie.revision = revision_id
 140         return file_id
 141
 142     def change_dir_prop(self, id, name, value, pool):
 143         if name == SVN_PROP_BZR_MERGE:
 144             if id != ROOT_ID:
 145                 mutter('rogue %r on non-root directory' % SVN_PROP_BZR_MERGE)
 146                 return
 147
 148             self._parent_ids = value.splitlines()[-1]
 149         elif name == SVN_PROP_SVK_MERGE:
 150             if self._parent_ids is None:
 151                 # Only set parents using svk:merge if no
 152                 # bzr:merge set.
 153                 pass # FIXME
 154         elif name.startswith(SVN_PROP_BZR_REVPROP_PREFIX):
 155             self._revprops[name[len(SVN_PROP_BZR_REVPROP_PREFIX):]] = value
 156         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 157                       svn.core.SVN_PROP_ENTRY_COMMITTED_REV,
 158                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 159                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 160                       svn.core.SVN_PROP_ENTRY_UUID,
 161                       svn.core.SVN_PROP_EXECUTABLE):
 162             pass
 163         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 164             pass
 165         else:
 166             mutter('unsupported file property %r' % name)
 167
 168     def change_file_prop(self, id, name, value, pool):
 169         if name == svn.core.SVN_PROP_EXECUTABLE:
 170             # Strange, you'd expect executable to match svn.core.SVN_PROP_EXECUTABLE_VALUE, but that's not how SVN behaves.
 171             self.is_executable = (value != None)
 172         elif (name == svn.core.SVN_PROP_SPECIAL):
 173             self.is_symlink = (value != None)
 174         elif name == svn.core.SVN_PROP_ENTRY_COMMITTED_REV:
 175             self.last_file_rev = int(value)
 176         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 177                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 178                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 179                       svn.core.SVN_PROP_ENTRY_UUID,
 180                       svn.core.SVN_PROP_MIME_TYPE):
 181             pass
 182         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 183             pass
 184         else:
 185             mutter('unsupported file property %r' % name)
 186
 187     def add_file(self, path, parent_id, copyfrom_path, copyfrom_revnum, baton):
 188         self.is_symlink = False
 189         self.is_executable = None
 190         self.file_data = ""
 191         self.file_parents = []
 192         self.file_stream = None
 193         return path
 194
 195     def open_file(self, path, parent_id, base_revnum, pool):
 196         base_file_id = self.old_inventory.path2id(path)
 197         base_revid = self.old_inventory[base_file_id].revision
 198         file_id, revid = self.id_map[path]
 199         self.is_executable = None
 200         self.is_symlink = (self.inventory[base_file_id].kind == 'symlink')
 201         file_weave = self.weave_store.get_weave_or_empty(base_file_id, self.transact)
 202         self.file_data = file_weave.get_text(base_revid)
 203         self.file_stream = None
 204         if file_id == base_file_id:
 205             self.file_parents = [base_revid]
 206         else:
 207             # Replace
 208             del self.inventory[base_file_id]
 209             self.file_parents = []
 210         return path
 211
 212     def close_file(self, path, checksum):
 213         if self.file_stream is not None:
 214             self.file_stream.seek(0)
 215             lines = osutils.split_lines(self.file_stream.read())
 216         else:
 217             # Data didn't change or file is new
 218             lines = osutils.split_lines(self.file_data)
 219
 220         actual_checksum = md5_strings(lines)
 221         assert checksum is None or checksum == actual_checksum
 222
 223         file_id, revision_id = self.id_map[path]
 224         file_weave = self.weave_store.get_weave_or_empty(file_id, self.transact)
 225         if not file_weave.has_version(revision_id):
 226             file_weave.add_lines(revision_id, self.file_parents, lines)
 227
 228         if file_id in self.inventory:
 229             ie = self.inventory[file_id]
 230         elif self.is_symlink:
 231             ie = self.inventory.add_path(path, 'symlink', file_id)
 232         else:
 233             ie = self.inventory.add_path(path, 'file', file_id)
 234         ie.revision = revision_id
 235
 236         if self.is_symlink:
 237             ie.symlink_target = lines[0][len("link "):]
 238             ie.text_sha1 = None
 239             ie.text_size = None
 240             ie.text_id = None
 241         else:
 242             ie.text_sha1 = osutils.sha_strings(lines)
 243             ie.text_size = sum(map(len, lines))
 244             if self.is_executable is not None:
 245                 ie.executable = self.is_executable
 246
 247         self.file_stream = None
 248
 249     def close_edit(self):
 250         rev = self._get_revision(self.revid)
 251         self.inventory.revision_id = self.revid
 252         rev.inventory_sha1 = osutils.sha_string(
 253             bzrlib.xml5.serializer_v5.write_inventory_to_string(
 254                 self.inventory))
 255         self.target.add_revision(self.revid, rev, self.inventory)
 256         self.pool.destroy()
 257
 258     def abort_edit(self):
 259         pass
 260
 261     def apply_textdelta(self, file_id, base_checksum):
 262         actual_checksum = md5.new(self.file_data).hexdigest(),
 263         assert (base_checksum is None or base_checksum == actual_checksum,
 264             "base checksum mismatch: %r != %r" % (base_checksum, actual_checksum))
 265         self.file_stream = StringIO()
 266         return apply_txdelta_handler(StringIO(self.file_data), self.file_stream, self.pool)
 267
 268
 269 class InterSvnRepository(InterRepository):
 270     """Svn to any repository actions."""
 271
 272     _matching_repo_format = SvnRepositoryFormat()
 273     """The format to test with."""
 274
 275     @needs_write_lock
 276     def copy_content(self, revision_id=None, basis=None, pb=ProgressBar()):
 277         """See InterRepository.copy_content."""
 278         # Dictionary with paths as keys, revnums as values
 279
 280         # Loop over all the revnums until revision_id
 281         # (or youngest_revnum) and call self.target.add_revision()
 282         # or self.target.add_inventory() each time
 283         if revision_id is None:
 284             path = None
 285             until_revnum = self.source._latest_revnum
 286         else:
 287             (path, until_revnum) = self.source.parse_revision_id(revision_id)
 288
 289         repos_root = self.source.transport.get_repos_root()
 290
 291         needed = []
 292         parents = {}
 293         prev_revid = None
 294         if path is None:
 295             it = self.source.follow_history(until_revnum)
 296         else:
 297             it = self.source.follow_branch_history(path, until_revnum)
 298         for (branch, changes, revnum) in it:
 299             revid = self.source.generate_revision_id(revnum, branch)
 300
 301             if prev_revid is not None:
 302                 parents[prev_revid] = revid
 303
 304             prev_revid = revid
 305
 306             if not self.target.has_revision(revid):
 307                 needed.append((branch, revnum, revid, changes))
 308
 309         parents[prev_revid] = None
 310
 311         num = 0
 312         needed.reverse()
 313         prev_revid = None
 314         transport = self.source.transport
 315         for (branch, revnum, revid, changes) in needed:
 316             if pb is not None:
 317                 pb.update('copying revision', num+1, len(needed)+1)
 318             num += 1
 319
 320             parent_revid = parents[revid]
 321
 322             if parent_revid is not None:
 323                 (parent_branch, parent_revnum) = self.source.parse_revision_id(parent_revid)
 324             else:
 325                 parent_revnum = 0
 326                 parent_branch = None
 327
 328             if parent_revid is None:
 329                 id_map = self.source.get_fileid_map(revnum, branch)
 330                 parent_inv = Inventory(ROOT_ID)
 331             elif prev_revid != parent_revid:
 332                 id_map = self.source.get_fileid_map(revnum, branch)
 333                 parent_inv = self.target.get_inventory(parent_revid)
 334             else:
 335                 self.source.transform_fileid_map(self.source.uuid,
 336                                         revnum, branch,
 337                                         changes, id_map)
 338                 parent_inv = prev_inv
 339
 340
 341             editor = RevisionBuildEditor(self.source, self.target, branch,
 342                                          revnum, parent_inv, revid,
 343                                      self.source._log.get_revision_info(revnum),
 344                                      id_map, parent_branch)
 345
 346             pool = Pool()
 347             edit, edit_baton = svn.delta.make_editor(editor, pool)
 348
 349             if parent_branch is None:
 350                 transport.reparent(repos_root)
 351             else:
 352                 transport.reparent("%s/%s" % (repos_root, parent_branch))
 353             if parent_branch != branch:
 354                 switch_url = "%s/%s" % (repos_root, branch)
 355                 reporter, reporter_baton = transport.do_switch(
 356                            revnum, "", True,
 357                            switch_url, edit, edit_baton, pool)
 358             else:
 359                 reporter, reporter_baton = transport.do_update(
 360                            revnum, "", True, edit, edit_baton, pool)
 361
 362             # Report status of existing paths
 363             svn.ra.reporter2_invoke_set_path(reporter, reporter_baton,
 364                 "", parent_revnum, False, None, pool)
 365
 366             transport.lock()
 367             svn.ra.reporter2_invoke_finish_report(reporter, reporter_baton, pool)
 368             transport.unlock()
 369
 370             prev_inv = editor.inventory
 371             prev_revid = revid
 372
 373             pool.destroy()
 374
 375         if pb is not None:
 376             pb.clear()
 377
 378         self.source.transport.reparent(repos_root)
 379
 380     @needs_write_lock
 381     def fetch(self, revision_id=None, pb=ProgressBar()):
 382         """Fetch revisions. """
 383         self.copy_content(revision_id=revision_id, pb=pb)
 384
 385     @staticmethod
 386     def is_compatible(source, target):
 387         """Be compatible with SvnRepository."""
 388         # FIXME: Also check target uses VersionedFile
 389         mutter('test %r' % source)
 390         return isinstance(source, SvnRepository)
 391
 392
 393