[PATCH] Finish initial cut of git-pack-object/git-unpack-object pair.
This finishes the initial round of git-pack-object /
git-unpack-object pair. They are now good enough to be used as
a transport medium:
- Fix delta direction in pack-objects; the original was
computing delta to create the base object from the object to
be squashed, which was quite unfriendly for unpacker ;-).
- Add a script to test the very basics.
- Implement unpacker for both regular and deltified objects.
Signed-off-by: Junio C Hamano <junkio@cox.net>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>
diff --git a/pack-objects.c b/pack-objects.c
index 9d87a64..3900df3 100644
--- a/pack-objects.c
+++ b/pack-objects.c
@@ -82,7 +82,8 @@
if (!otherbuf)
die("unable to read %s", sha1_to_hex(entry->delta->sha1));
- delta_buf = diff_delta(buf, size, otherbuf, othersize, &delta_size, ~0UL);
+ delta_buf = diff_delta(otherbuf, othersize,
+ buf, size, &delta_size, ~0UL);
if (!delta_buf || delta_size != entry->delta_size)
die("delta size changed");
free(buf);
@@ -318,7 +319,8 @@
max_size = size / 2 - 20;
if (cur_entry->delta)
max_size = cur_entry->delta_size-1;
- delta_buf = diff_delta(cur->data, size, old->data, oldsize, &delta_size, max_size);
+ delta_buf = diff_delta(old->data, oldsize,
+ cur->data, size, &delta_size, max_size);
if (!delta_buf)
return 0;
cur_entry->delta = old_entry;
diff --git a/t/t5300-pack-object.sh b/t/t5300-pack-object.sh
new file mode 100755
index 0000000..171af64
--- /dev/null
+++ b/t/t5300-pack-object.sh
@@ -0,0 +1,85 @@
+#!/bin/sh
+#
+# Copyright (c) 2005 Junio C Hamano
+#
+
+test_description='git-pack-object
+
+'
+. ./test-lib.sh
+
+TRASH=`pwd`
+
+test_expect_success \
+ 'setup' \
+ 'rm -f .git/index*
+ for i in a b c
+ do
+ dd if=/dev/zero bs=4k count=1 | tr "\\0" $i >$i &&
+ git-update-cache --add $i || exit
+ done &&
+ cat c >d && echo foo >>d && git-update-cache --add d &&
+ tree=`git-write-tree` && {
+ echo $tree &&
+ git-ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\) .*/\\1/"
+ } >obj-list'
+
+test_expect_success \
+ 'pack without delta' \
+ 'git-pack-objects --window=0 test-1 <obj-list'
+
+rm -fr .git2
+mkdir .git2
+
+test_expect_success \
+ 'unpack without delta' \
+ 'GIT_OBJECT_DIRECTORY=.git2/objects &&
+ export GIT_OBJECT_DIRECTORY &&
+ git-init-db &&
+ git-unpack-objects test-1'
+
+unset GIT_OBJECT_DIRECTORY
+cd $TRASH/.git2
+
+test_expect_success \
+ 'check unpack without delta' \
+ '(cd ../.git && find objects -type f -print) |
+ while read path
+ do
+ cmp $path ../.git/$path || {
+ echo $path differs.
+ exit 1
+ }
+ done'
+cd $TRASH
+
+test_expect_success \
+ 'pack with delta' \
+ 'pwd &&
+ git-pack-objects test-2 <obj-list'
+
+rm -fr .git2
+mkdir .git2
+
+test_expect_success \
+ 'unpack with delta' \
+ 'GIT_OBJECT_DIRECTORY=.git2/objects &&
+ export GIT_OBJECT_DIRECTORY &&
+ git-init-db &&
+ git-unpack-objects test-2'
+
+unset GIT_OBJECT_DIRECTORY
+cd $TRASH/.git2
+test_expect_success \
+ 'check unpack with delta' \
+ '(cd ../.git && find objects -type f -print) |
+ while read path
+ do
+ cmp $path ../.git/$path || {
+ echo $path differs.
+ exit 1
+ }
+ done'
+cd $TRASH
+
+test_done
diff --git a/unpack-objects.c b/unpack-objects.c
index 7b175c9..a7b8df0 100644
--- a/unpack-objects.c
+++ b/unpack-objects.c
@@ -1,5 +1,6 @@
#include "cache.h"
#include "object.h"
+#include "delta.h"
static int dry_run;
static int nr_entries;
@@ -7,12 +8,14 @@
static const char unpack_usage[] = "git-unpack-objects basename";
struct pack_entry {
- unsigned int offset;
+ unsigned int offset; /* network byte order */
unsigned char sha1[20];
};
static void *pack_base;
static unsigned long pack_size;
+static void *index_base;
+static unsigned long index_size;
static struct pack_entry **pack_list;
@@ -52,13 +55,13 @@
return o1 < o2 ? -1 : 1;
}
-static int check_index(void *index, unsigned long idx_size)
+static int check_index(void)
{
- unsigned int *array = index;
+ unsigned int *array = index_base;
unsigned int nr;
int i;
- if (idx_size < 4*256)
+ if (index_size < 4*256)
return error("index file too small");
nr = 0;
for (i = 0; i < 256; i++) {
@@ -67,15 +70,16 @@
return error("non-monotonic index");
nr = n;
}
- if (idx_size != 4*256 + nr * 24) {
- printf("idx_size=%lu, expected %u (%u)\n", idx_size, 4*256 + nr * 24, nr);
+ if (index_size != 4*256 + nr * 24) {
+ printf("index_size=%lu, expected %u (%u)\n",
+ index_size, 4*256 + nr * 24, nr);
return error("wrong index file size");
}
nr_entries = nr;
pack_list = xmalloc(nr * sizeof(struct pack_entry *));
for (i = 0; i < nr; i++)
- pack_list[i] = index + 4*256 + i*24;
+ pack_list[i] = index_base + 4*256 + i*24;
qsort(pack_list, nr, sizeof(*pack_list), sort_by_offset);
@@ -83,9 +87,165 @@
return 0;
}
+static int unpack_non_delta_entry(struct pack_entry *entry,
+ unsigned char *pack)
+{
+ int st, kind;
+ unsigned long size;
+ z_stream stream;
+ char *buffer;
+ unsigned char sha1[20];
+ char *type_s;
+ unsigned long offset = ntohl(entry->offset);
+
+ kind = pack[0];
+ size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
+ printf("%s %c %lu\n", sha1_to_hex(entry->sha1), kind, size);
+ pack += 5;
+
+ buffer = xmalloc(size + 1);
+ buffer[size] = 0;
+ memset(&stream, 0, sizeof(stream));
+ stream.next_in = pack;
+ stream.avail_in = pack_size - offset; /* sheesh. */
+ stream.next_out = buffer;
+ stream.avail_out = size;
+
+ inflateInit(&stream);
+ st = inflate(&stream, Z_FINISH);
+ inflateEnd(&stream);
+ if ((st != Z_STREAM_END) || stream.total_out != size)
+ goto err_finish;
+ switch (kind) {
+ case 'C': type_s = "commit"; break;
+ case 'T': type_s = "tree"; break;
+ case 'B': type_s = "blob"; break;
+ default: goto err_finish;
+ }
+ if (write_sha1_file(buffer, size, type_s, sha1) < 0)
+ die("failed to write %s (%s)",
+ sha1_to_hex(entry->sha1), type_s);
+ printf("%s %s\n", sha1_to_hex(sha1), type_s);
+ if (memcmp(sha1, entry->sha1, 20))
+ die("resulting %s have wrong SHA1", type_s);
+
+ finish:
+ st = 0;
+ free(buffer);
+ return st;
+ err_finish:
+ st = -1;
+ goto finish;
+}
+
+static int find_pack_entry(unsigned char *sha1, struct pack_entry **ent)
+{
+ int *level1_ofs = index_base;
+ int hi = ntohl(level1_ofs[*sha1]);
+ int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
+ void *index = index_base + 4*256;
+
+ do {
+ int mi = (lo + hi) / 2;
+ int cmp = memcmp(index + 24 * mi + 4, sha1, 20);
+ if (!cmp) {
+ *ent = index + 24 * mi;
+ return 1;
+ }
+ if (cmp < 0)
+ hi = mi;
+ else
+ lo = mi;
+ } while (lo < hi);
+ return 0;
+}
+
+/* forward declaration for a mutually recursive function */
+static void unpack_entry(struct pack_entry *);
+
+static int unpack_delta_entry(struct pack_entry *entry, unsigned char *pack)
+{
+ void *delta_data, *result, *base;
+ unsigned long delta_alloc, delta_size, result_size, base_size;
+ z_stream stream;
+ int st;
+ char type[20];
+ unsigned char sha1[20];
+
+ printf("%s D", sha1_to_hex(entry->sha1));
+ printf(" %s\n", sha1_to_hex(pack+1));
+
+ /* pack+1 is the base sha1, unless we have it, we need to
+ * unpack it first.
+ */
+ if (!has_sha1_file(pack+1)) {
+ struct pack_entry *base;
+ if (!find_pack_entry(pack+1, &base))
+ die("cannot find delta-pack base object");
+ unpack_entry(base);
+ }
+
+ /* pack+1 thru pack+20 is the base sha1 and
+ * pack+21 thru unknown number is the delta data.
+ * we do not even have size of the delta data uncompressed.
+ * sheesh!
+ */
+ delta_alloc = 1024;
+ delta_data = xmalloc(delta_alloc);
+
+ memset(&stream, 0, sizeof(stream));
+
+ stream.next_in = pack + 21;
+ stream.avail_in = pack_size - ntohl(entry->offset); /* sheesh. */
+ stream.next_out = delta_data;
+ stream.avail_out = delta_alloc;
+ delta_size = 0;
+
+ inflateInit(&stream);
+ while (1) {
+ st = inflate(&stream, Z_FINISH);
+ if (st == Z_STREAM_END) {
+ delta_size = stream.total_out;
+ break;
+ }
+ if (st < 0)
+ break;
+
+ if (delta_alloc <= stream.total_out) {
+ delta_alloc = (delta_alloc +1024) * 3 / 2;
+ delta_data = xrealloc(delta_data, delta_alloc);
+ stream.next_out = delta_data + stream.total_out;
+ stream.avail_out = delta_alloc - stream.total_out;
+ }
+ }
+ inflateEnd(&stream);
+ if (st != Z_STREAM_END) {
+ free(delta_data);
+ return -1;
+ }
+
+ base = read_sha1_file(pack+1, type, &base_size);
+ if (!base)
+ die("failed to read delta-pack base object");
+ result = patch_delta(base, base_size,
+ delta_data, delta_size,
+ &result_size);
+ if (!result)
+ die("failed to apply delta");
+ free(delta_data);
+
+ if (write_sha1_file(result, result_size, type, sha1) < 0)
+ die("failed to write %s (%s)",
+ sha1_to_hex(entry->sha1), type);
+ free(result);
+ printf("%s %s\n", sha1_to_hex(sha1), type);
+ if (memcmp(sha1, entry->sha1, 20))
+ die("resulting %s have wrong SHA1", type);
+ return 0;
+}
+
static void unpack_entry(struct pack_entry *entry)
{
- unsigned long size;
unsigned long offset;
unsigned char *pack;
@@ -100,12 +260,10 @@
offset = pack_size - offset;
switch (*pack) {
case 'C': case 'T': case 'B':
- size = (pack[1] << 24) + (pack[2] << 16) + (pack[3] << 8) + pack[4];
- printf("%s %c %lu\n", sha1_to_hex(entry->sha1), *pack, size);
+ unpack_non_delta_entry(entry, pack);
break;
case 'D':
- printf("%s D", sha1_to_hex(entry->sha1));
- printf(" %s\n", sha1_to_hex(pack+1));
+ unpack_delta_entry(entry, pack);
break;
default:
die("corrupted pack file");
@@ -130,8 +288,6 @@
int main(int argc, char **argv)
{
int i;
- unsigned long idx_size;
- void *index;
for (i = 1 ; i < argc; i++) {
const char *arg = argv[i];
@@ -149,9 +305,9 @@
}
if (!base_name)
usage(unpack_usage);
- index = map_file("idx", &idx_size);
+ index_base = map_file("idx", &index_size);
pack_base = map_file("pack", &pack_size);
- if (check_index(index, idx_size) < 0)
+ if (check_index() < 0)
die("bad index file");
unpack_all();
return 0;
