1 # object_store.py -- Object store for git objects
2 # Copyright (C) 2008 Jelmer Vernooij <jelmer@samba.org>
4 # This program is free software; you can redistribute it and/or
5 # modify it under the terms of the GNU General Public License
6 # as published by the Free Software Foundation; either version 2
7 # or (at your option) a later version of the License.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software
16 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
23 from dulwich.objects import (
28 from dulwich.pack import (
40 class ObjectStore(object):
43 def __init__(self, path):
44 """Open an object store.
46 :param path: Path of the object store.
49 self._pack_cache = None
50 self.pack_dir = os.path.join(self.path, PACKDIR)
52 def determine_wants_all(self, refs):
53 return [sha for (ref, sha) in refs.iteritems() if not sha in self and not ref.endswith("^{}")]
55 def iter_shas(self, shas):
56 """Iterate over the objects for the specified shas.
58 :param shas: Iterable object with SHAs
60 return ObjectStoreIterator(self, shas)
62 def __contains__(self, sha):
63 # TODO: This can be more efficient
72 """List with pack objects."""
73 if self._pack_cache is None:
74 self._pack_cache = list(load_packs(self.pack_dir))
75 return self._pack_cache
77 def _add_known_pack(self, path):
78 """Add a newly appeared pack to the cache by path.
81 if self._pack_cache is not None:
82 self._pack_cache.append(Pack(path))
84 def _get_shafile_path(self, sha):
87 # Check from object dir
88 return os.path.join(self.path, dir, file)
90 def _get_shafile(self, sha):
91 path = self._get_shafile_path(sha)
92 if os.path.exists(path):
93 return ShaFile.from_file(path)
96 def _add_shafile(self, sha, o):
97 path = self._get_shafile_path(sha)
98 f = os.path.open(path, 'w')
105 def get_raw(self, sha):
106 """Obtain the raw text for an object.
108 :param sha: Sha for the object.
109 :return: tuple with object type and object contents.
111 for pack in self.packs:
113 return pack.get_raw(sha, self.get_raw)
114 # FIXME: Are thin pack deltas ever against on-disk shafiles ?
115 ret = self._get_shafile(sha)
117 return ret.as_raw_string()
120 def __getitem__(self, sha):
121 assert len(sha) == 40, "Incorrect length sha: %s" % str(sha)
122 ret = self._get_shafile(sha)
126 type, uncomp = self.get_raw(sha)
127 return ShaFile.from_raw_string(type, uncomp)
129 def move_in_thin_pack(self, path):
130 """Move a specific file containing a pack into the pack directory.
132 :note: The file should be on the same file system as the
135 :param path: Path to the pack file.
138 temppath = os.path.join(self.pack_dir,
139 sha_to_hex(urllib2.randombytes(20))+".temppack")
140 write_pack(temppath, p.iterobjects(self.get_raw), len(p))
141 pack_sha = PackIndex(temppath+".idx").objects_sha1()
142 newbasename = os.path.join(self.pack_dir, "pack-%s" % pack_sha)
143 os.rename(temppath+".pack", newbasename+".pack")
144 os.rename(temppath+".idx", newbasename+".idx")
145 self._add_known_pack(newbasename)
147 def move_in_pack(self, path):
148 """Move a specific file containing a pack into the pack directory.
150 :note: The file should be on the same file system as the
153 :param path: Path to the pack file.
156 entries = p.sorted_entries()
157 basename = os.path.join(self.pack_dir,
158 "pack-%s" % iter_sha1(entry[0] for entry in entries))
159 write_pack_index_v2(basename+".idx", entries, p.get_stored_checksum())
160 os.rename(path, basename + ".pack")
161 self._add_known_pack(basename)
163 def add_thin_pack(self):
164 """Add a new thin pack to this object store.
166 Thin packs are packs that contain deltas with parents that exist
169 fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
170 f = os.fdopen(fd, 'w')
174 if os.path.getsize(path) > 0:
175 self.move_in_thin_pack(path)
179 """Add a new pack to this object store.
181 :return: Fileobject to write to and a commit function to
182 call when the pack is finished.
184 fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
185 f = os.fdopen(fd, 'w')
189 if os.path.getsize(path) > 0:
190 self.move_in_pack(path)
193 def add_objects(self, objects):
194 """Add a set of objects to this object store.
196 :param objects: Iterable over a list of objects.
198 if len(objects) == 0:
200 f, commit = self.add_pack()
201 write_pack_data(f, objects, len(objects))
205 class ObjectImporter(object):
206 """Interface for importing objects."""
208 def __init__(self, count):
209 """Create a new ObjectImporter.
211 :param count: Number of objects that's going to be imported.
215 def add_object(self, object):
217 raise NotImplementedError(self.add_object)
219 def finish(self, object):
220 """Finish the imoprt and write objects to disk."""
221 raise NotImplementedError(self.finish)
224 class ObjectIterator(object):
225 """Interface for iterating over objects."""
227 def iterobjects(self):
228 raise NotImplementedError(self.iterobjects)
231 class ObjectStoreIterator(ObjectIterator):
232 """ObjectIterator that works on top of an ObjectStore."""
234 def __init__(self, store, sha_iter):
236 self.shas = list(sha_iter)
239 return ((self.store[sha], path) for sha, path in self.shas)
241 def iterobjects(self):
245 def __contains__(self, needle):
246 """Check if an object is present.
248 :param needle: SHA1 of the object to check for
250 # FIXME: This could be more efficient
251 for sha, path in self.shas:
256 def __getitem__(self, key):
257 """Find an object by SHA1."""
258 return self.store[key]
261 """Return the number of objects."""
262 return len(self.shas)