fetch.py

   1 # Copyright (C) 2005-2006 Jelmer Vernooij <jelmer@samba.org>
   2
   3 # This program is free software; you can redistribute it and/or modify
   4 # it under the terms of the GNU General Public License as published by
   5 # the Free Software Foundation; either version 2 of the License, or
   6 # (at your option) any later version.
   7
   8 # This program is distributed in the hope that it will be useful,
   9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  11 # GNU General Public License for more details.
  12
  13 # You should have received a copy of the GNU General Public License
  14 # along with this program; if not, write to the Free Software
  15 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  16
  17 import bzrlib
  18 from bzrlib.decorators import needs_write_lock
  19 from bzrlib.inventory import Inventory, ROOT_ID
  20 import bzrlib.osutils as osutils
  21 from bzrlib.progress import ProgressBar
  22 from bzrlib.revision import Revision
  23 from bzrlib.repository import InterRepository
  24 from bzrlib.trace import mutter
  25
  26 from copy import copy
  27 from cStringIO import StringIO
  28 import md5
  29 import os
  30
  31 from svn.core import SubversionException, Pool
  32 import svn.core, svn.ra
  33
  34 from repository import (SvnRepository, SVN_PROP_BZR_MERGE, SVN_PROP_SVK_MERGE,
  35                 SVN_PROP_BZR_REVPROP_PREFIX, SvnRepositoryFormat)
  36 from tree import apply_txdelta_handler
  37
  38
  39 def md5_strings(strings):
  40     s = md5.new()
  41     map(s.update, strings)
  42     return s.hexdigest()
  43
  44 class RevisionBuildEditor(svn.delta.Editor):
  45     def __init__(self, source, target, branch_path, revnum, prev_inventory, revid, svn_revprops, id_map, parent_branch, parent_id_map):
  46         self.branch_path = branch_path
  47         self.inventory = copy(prev_inventory)
  48         self.revid = revid
  49         self.revnum = revnum
  50         self.id_map = id_map
  51         self.parent_branch = parent_branch
  52         self.parent_id_map = parent_id_map
  53         self.source = source
  54         self.target = target
  55         self.transact = target.get_transaction()
  56         self.weave_store = target.weave_store
  57
  58         self.dir_baserev = {}
  59
  60         self._parent_ids = None
  61         self._revprops = {}
  62         self._svn_revprops = svn_revprops
  63
  64     def _get_revision(self, revid):
  65         if self._parent_ids is None:
  66             self._parent_ids = ""
  67
  68         parent_ids = self.source.revision_parents(revid, self._parent_ids)
  69
  70         # Commit SVN revision properties to a Revision object
  71         rev = Revision(revision_id=revid, parent_ids=parent_ids)
  72
  73         rev.timestamp = 1.0 * svn.core.secs_from_timestr(
  74             self._svn_revprops[2], None) #date
  75         rev.timezone = None
  76
  77         rev.committer = self._svn_revprops[0] # author
  78         if rev.committer is None:
  79             rev.committer = ""
  80         rev.message = self._svn_revprops[1] # message
  81
  82         rev.properties = self._revprops
  83         return rev
  84
  85     def open_root(self, base_revnum, baton):
  86         if self.inventory.revision_id is None:
  87             self.dir_baserev[ROOT_ID] = []
  88         else:
  89             self.dir_baserev[ROOT_ID] = [self.inventory.revision_id]
  90         self.inventory.revision_id = self.revid
  91         return ROOT_ID
  92
  93     def relpath(self, path):
  94         return path.strip("/")
  95
  96     def delete_entry(self, path, revnum, parent_baton, pool):
  97         del self.inventory[self.inventory.path2id(path)]
  98
  99     def close_directory(self, id):
 100         revid = self.revid
 101
 102         if id != ROOT_ID:
 103             self.inventory[id].revision = revid
 104
 105             file_weave = self.weave_store.get_weave_or_empty(id, self.transact)
 106             if not file_weave.has_version(revid):
 107                 file_weave.add_lines(revid, self.dir_baserev[id], [])
 108
 109     def add_directory(self, path, parent_baton, copyfrom_path, copyfrom_revnum, pool):
 110         file_id, revision_id = self.id_map[path]
 111
 112         if copyfrom_path is not None:
 113             base_file_id, base_revid = self.source.path_to_file_id(copyfrom_revnum, os.path.join(self.parent_branch, copyfrom_path))
 114             if base_file_id == file_id:
 115                 self.dir_baserev[file_id] = [base_revid]
 116                 ie = self.inventory[file_id]
 117                 ie.revision = revision_id
 118                 return file_id
 119
 120         self.dir_baserev[file_id] = []
 121         ie = self.inventory.add_path(path, 'directory', file_id)
 122         if ie:
 123             ie.revision = revision_id
 124
 125         return file_id
 126
 127     def open_directory(self, path, parent_baton, base_revnum, pool):
 128         return self.add_directory(path, parent_baton, path, base_revnum, pool)
 129
 130     def change_dir_prop(self, id, name, value, pool):
 131         if name == SVN_PROP_BZR_MERGE:
 132             if id != ROOT_ID:
 133                 mutter('rogue %r on non-root directory' % SVN_PROP_BZR_MERGE)
 134                 return
 135
 136             self._parent_ids = value.splitlines()[-1]
 137         elif name == SVN_PROP_SVK_MERGE:
 138             if self._parent_ids is None:
 139                 # Only set parents using svk:merge if no
 140                 # bzr:merge set.
 141                 pass # FIXME
 142         elif name.startswith(SVN_PROP_BZR_REVPROP_PREFIX):
 143             self._revprops[name[len(SVN_PROP_BZR_REVPROP_PREFIX):]] = value
 144         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 145                       svn.core.SVN_PROP_ENTRY_COMMITTED_REV,
 146                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 147                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 148                       svn.core.SVN_PROP_ENTRY_UUID,
 149                       svn.core.SVN_PROP_EXECUTABLE):
 150             pass
 151         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 152             pass
 153         else:
 154             mutter('unsupported file property %r' % name)
 155
 156     def change_file_prop(self, id, name, value, pool):
 157         if name == svn.core.SVN_PROP_EXECUTABLE:
 158             # Strange, you'd expect executable to match svn.core.SVN_PROP_EXECUTABLE_VALUE, but that's not how SVN behaves.
 159             self.is_executable = (value != None)
 160         elif (name == svn.core.SVN_PROP_SPECIAL):
 161             self.is_symlink = (value != None)
 162         elif name == svn.core.SVN_PROP_ENTRY_COMMITTED_REV:
 163             self.last_file_rev = int(value)
 164         elif name in (svn.core.SVN_PROP_ENTRY_COMMITTED_DATE,
 165                       svn.core.SVN_PROP_ENTRY_LAST_AUTHOR,
 166                       svn.core.SVN_PROP_ENTRY_LOCK_TOKEN,
 167                       svn.core.SVN_PROP_ENTRY_UUID,
 168                       svn.core.SVN_PROP_MIME_TYPE):
 169             pass
 170         elif name.startswith(svn.core.SVN_PROP_WC_PREFIX):
 171             pass
 172         else:
 173             mutter('unsupported file property %r' % name)
 174
 175     def add_file(self, path, parent_id, copyfrom_path, copyfrom_revnum, baton):
 176         self.is_symlink = False
 177         self.is_executable = None
 178         self.file_data = ""
 179         self.file_parents = []
 180         self.file_stream = None
 181         return path
 182
 183     def open_file(self, path, parent_id, base_revnum, pool):
 184         base_file_id, base_revid = self.source.path_to_file_id(base_revnum, os.path.join(self.parent_branch, path))
 185         file_id, revid = self.id_map[path]
 186         self.is_executable = None
 187         self.is_symlink = (self.inventory[base_file_id].kind == 'symlink')
 188         file_weave = self.weave_store.get_weave_or_empty(base_file_id, self.transact)
 189         self.file_data = file_weave.get_text(base_revid)
 190         self.file_stream = None
 191         if file_id == base_file_id:
 192             self.file_parents = [base_revid]
 193         else:
 194             # Replace
 195             del self.inventory[base_file_id]
 196             self.file_parents = []
 197         return path
 198
 199     def close_file(self, path, checksum):
 200         if self.file_stream is not None:
 201             self.file_stream.seek(0)
 202             lines = osutils.split_lines(self.file_stream.read())
 203         else:
 204             # Data didn't change or file is new
 205             lines = osutils.split_lines(self.file_data)
 206
 207         actual_checksum = md5_strings(lines)
 208         assert checksum is None or checksum == actual_checksum
 209
 210         file_id, revision_id = self.id_map[path]
 211         file_weave = self.weave_store.get_weave_or_empty(file_id, self.transact)
 212         if not file_weave.has_version(revision_id):
 213             file_weave.add_lines(revision_id, self.file_parents, lines)
 214
 215         if file_id in self.inventory:
 216             ie = self.inventory[file_id]
 217         elif self.is_symlink:
 218             ie = self.inventory.add_path(path, 'symlink', file_id)
 219         else:
 220             ie = self.inventory.add_path(path, 'file', file_id)
 221         ie.revision = revision_id
 222
 223         if self.is_symlink:
 224             ie.symlink_target = lines[0][len("link "):]
 225             ie.text_sha1 = None
 226             ie.text_size = None
 227             ie.text_id = None
 228         else:
 229             ie.text_sha1 = osutils.sha_strings(lines)
 230             ie.text_size = sum(map(len, lines))
 231             if self.is_executable is not None:
 232                 ie.executable = self.is_executable
 233
 234         self.file_stream = None
 235
 236     def close_edit(self):
 237         rev = self._get_revision(self.revid)
 238         self.inventory.revision_id = self.revid
 239         rev.inventory_sha1 = osutils.sha_string(
 240             bzrlib.xml5.serializer_v5.write_inventory_to_string(
 241                 self.inventory))
 242         self.target.add_revision(self.revid, rev, self.inventory)
 243
 244     def abort_edit(self):
 245         pass
 246
 247     def apply_textdelta(self, file_id, base_checksum):
 248         actual_checksum = md5.new(self.file_data).hexdigest(),
 249         assert (base_checksum is None or base_checksum == actual_checksum,
 250             "base checksum mismatch: %r != %r" % (base_checksum, actual_checksum))
 251         self.file_stream = StringIO()
 252         return apply_txdelta_handler(StringIO(self.file_data), self.file_stream)
 253
 254
 255 class InterSvnRepository(InterRepository):
 256     """Svn to any repository actions."""
 257
 258     _matching_repo_format = SvnRepositoryFormat
 259     """The format to test with."""
 260
 261     @needs_write_lock
 262     def copy_content(self, revision_id=None, basis=None, pb=ProgressBar()):
 263         """See InterRepository.copy_content."""
 264         # Dictionary with paths as keys, revnums as values
 265
 266         # Loop over all the revnums until revision_id
 267         # (or youngest_revnum) and call self.target.add_revision()
 268         # or self.target.add_inventory() each time
 269         if revision_id is None:
 270             path = None
 271             until_revnum = self.source._latest_revnum
 272         else:
 273             (path, until_revnum) = self.source.parse_revision_id(revision_id)
 274
 275         repos_root = self.source.transport.get_repos_root()
 276
 277         needed = []
 278         parents = {}
 279         prev_revid = None
 280         for (branch, changes, revnum) in \
 281             self.source._log.follow_history(path, until_revnum):
 282             revid = self.source.generate_revision_id(revnum, branch)
 283
 284             if prev_revid is not None:
 285                 parents[prev_revid] = revid
 286
 287             prev_revid = revid
 288
 289             if not self.target.has_revision(revid):
 290                 needed.append((branch, revnum, revid, changes))
 291
 292         parents[prev_revid] = None
 293
 294         num = 0
 295         needed.reverse()
 296         prev_revid = None
 297         transport = self.source.transport
 298         for (branch, revnum, revid, changes) in needed:
 299             if pb is not None:
 300                 pb.update('copying revision', num+1, len(needed)+1)
 301             num += 1
 302
 303             parent_revid = parents[revid]
 304
 305             if parent_revid is not None:
 306                 (parent_branch, parent_revnum) = self.source.parse_revision_id(parent_revid)
 307             else:
 308                 parent_revnum = 0
 309                 parent_branch = None
 310
 311             if parent_revid is None:
 312                 parent_id_map = {"": (ROOT_ID, None)}
 313                 id_map = self.source.get_fileid_map(revnum, branch)
 314                 parent_inv = Inventory(ROOT_ID)
 315             elif prev_revid != parent_revid:
 316                 parent_id_map = self.source.get_fileid_map(parent_revnum, parent_branch)
 317                 id_map = self.source.get_fileid_map(revnum, branch)
 318                 parent_inv = self.target.get_inventory(parent_revid)
 319             else:
 320                 parent_id_map = copy(id_map)
 321                 self.source.transform_fileid_map(self.source.uuid,
 322                                         revnum, branch,
 323                                         changes, id_map)
 324                 parent_inv = prev_inv
 325
 326
 327             editor = RevisionBuildEditor(self.source, self.target, branch,
 328                                          revnum, parent_inv, revid,
 329                                      self.source._log.get_revision_info(revnum),
 330                                      id_map, parent_branch, parent_id_map)
 331
 332             edit, edit_baton = svn.delta.make_editor(editor)
 333
 334             if parent_branch is None:
 335                 transport.reparent(repos_root)
 336             else:
 337                 transport.reparent("%s/%s" % (repos_root, parent_branch))
 338             pool = Pool()
 339             if parent_branch != branch:
 340                 mutter('svn switch %r:%r -> %r:%r' %
 341                                (parent_branch, parent_revnum, branch, revnum))
 342                 reporter, reporter_baton = transport.do_switch(
 343                            revnum, "", True,
 344                            "%s/%s" % (repos_root, branch),
 345                            edit, edit_baton, pool)
 346             else:
 347                 mutter('svn update -r %r:%r %r' %
 348                                (parent_revnum, revnum, branch))
 349                 reporter, reporter_baton = transport.do_update(
 350                            revnum, "", True,
 351                            edit, edit_baton, pool)
 352
 353             # Report status of existing paths
 354             svn.ra.reporter2_invoke_set_path(reporter, reporter_baton,
 355                 "", parent_revnum, False, None)
 356
 357             transport.lock()
 358             svn.ra.reporter2_invoke_finish_report(reporter, reporter_baton)
 359             transport.unlock()
 360
 361             prev_inv = editor.inventory
 362             prev_revid = revid
 363
 364         if pb is not None:
 365             pb.clear()
 366
 367         self.source.transport.reparent(repos_root)
 368
 369     @needs_write_lock
 370     def fetch(self, revision_id=None, pb=ProgressBar()):
 371         """Fetch revisions. """
 372         self.copy_content(revision_id=revision_id, pb=pb)
 373
 374     @staticmethod
 375     def is_compatible(source, target):
 376         """Be compatible with SvnRepository."""
 377         # FIXME: Also check target uses VersionedFile
 378         mutter('test %r' % source)
 379         return isinstance(source, SvnRepository)
 380
 381
 382