reftable: debug tools
Simple debug programs to experiment with the reftable file format:
debug-read-reftable
debug-write-reftable
debug-verify-reftable
debug-benchmark-reftable
Change-Id: I79db351d86900f1e58b17e922e195dff06ee71f1
diff --git a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
index e73b352..c59f636 100644
--- a/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit.pgm/META-INF/MANIFEST.MF
@@ -37,8 +37,11 @@
org.eclipse.jgit.errors;version="[4.9.0,4.10.0)",
org.eclipse.jgit.gitrepo;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.ketch;version="[4.9.0,4.10.0)",
+ org.eclipse.jgit.internal.storage.dfs;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.file;version="[4.9.0,4.10.0)",
+ org.eclipse.jgit.internal.storage.io;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.pack;version="[4.9.0,4.10.0)",
+ org.eclipse.jgit.internal.storage.reftable;version="[4.9.0,4.10.0)",
org.eclipse.jgit.internal.storage.reftree;version="[4.9.0,4.10.0)",
org.eclipse.jgit.lfs;version="[4.9.0,4.10.0)",
org.eclipse.jgit.lfs.lib;version="[4.9.0,4.10.0)",
diff --git a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
index 5495be6..9025473 100644
--- a/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
+++ b/org.eclipse.jgit.pgm/META-INF/services/org.eclipse.jgit.pgm.TextBuiltin
@@ -38,10 +38,12 @@
org.eclipse.jgit.pgm.UploadPack
org.eclipse.jgit.pgm.Version
+org.eclipse.jgit.pgm.debug.BenchmarkReftable
org.eclipse.jgit.pgm.debug.DiffAlgorithms
org.eclipse.jgit.pgm.debug.LfsStore
org.eclipse.jgit.pgm.debug.MakeCacheTree
org.eclipse.jgit.pgm.debug.ReadDirCache
+org.eclipse.jgit.pgm.debug.ReadReftable
org.eclipse.jgit.pgm.debug.RebuildCommitGraph
org.eclipse.jgit.pgm.debug.RebuildRefTree
org.eclipse.jgit.pgm.debug.ShowCacheTree
@@ -49,5 +51,6 @@
org.eclipse.jgit.pgm.debug.ShowDirCache
org.eclipse.jgit.pgm.debug.ShowPackDelta
org.eclipse.jgit.pgm.debug.TextHashFunctions
-org.eclipse.jgit.pgm.debug.WriteDirCache
-
+org.eclipse.jgit.pgm.debug.VerifyReftable
+org.eclipse.jgit.pgm.debug.WriteReftable
+org.eclipse.jgit.pgm.debug.WriteReftable
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java
new file mode 100644
index 0000000..71c8db8
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/BenchmarkReftable.java
@@ -0,0 +1,315 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.HEAD;
+import static org.eclipse.jgit.lib.Constants.MASTER;
+import static org.eclipse.jgit.lib.Constants.R_HEADS;
+import static org.eclipse.jgit.lib.Ref.Storage.NEW;
+import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.SymbolicRef;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.eclipse.jgit.util.RefList;
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.Option;
+
+@Command
+class BenchmarkReftable extends TextBuiltin {
+ enum Test {
+ SCAN,
+ SEEK_COLD, SEEK_HOT,
+ BY_ID_COLD, BY_ID_HOT;
+ }
+
+ @Option(name = "--tries")
+ private int tries = 10;
+
+ @Option(name = "--test")
+ private Test test = Test.SCAN;
+
+ @Option(name = "--ref")
+ private String ref;
+
+ @Option(name = "--object-id")
+ private String objectId;
+
+ @Argument(index = 0)
+ private String lsRemotePath;
+
+ @Argument(index = 1)
+ private String reftablePath;
+
+ @Override
+ protected void run() throws Exception {
+ switch (test) {
+ case SCAN:
+ scan();
+ break;
+
+ case SEEK_COLD:
+ seekCold(ref);
+ break;
+ case SEEK_HOT:
+ seekHot(ref);
+ break;
+
+ case BY_ID_COLD:
+ byIdCold(ObjectId.fromString(objectId));
+ break;
+ case BY_ID_HOT:
+ byIdHot(ObjectId.fromString(objectId));
+ break;
+ }
+ }
+
+ private void printf(String fmt, Object... args) throws IOException {
+ errw.println(String.format(fmt, args));
+ }
+
+ @SuppressWarnings({ "nls", "boxing" })
+ private void scan() throws Exception {
+ long start, tot;
+
+ start = System.currentTimeMillis();
+ for (int i = 0; i < tries; i++) {
+ readLsRemote();
+ }
+ tot = System.currentTimeMillis() - start;
+ printf("%12s %10d ms %6d ms/run", "packed-refs", tot, tot / tries);
+
+ start = System.currentTimeMillis();
+ for (int i = 0; i < tries; i++) {
+ try (FileInputStream in = new FileInputStream(reftablePath);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ try (RefCursor rc = reader.allRefs()) {
+ while (rc.next()) {
+ rc.getRef();
+ }
+ }
+ }
+ }
+ tot = System.currentTimeMillis() - start;
+ printf("%12s %10d ms %6d ms/run", "reftable", tot, tot / tries);
+ }
+
+ private RefList<Ref> readLsRemote()
+ throws IOException, FileNotFoundException {
+ RefList.Builder<Ref> list = new RefList.Builder<>();
+ try (BufferedReader br = new BufferedReader(new InputStreamReader(
+ new FileInputStream(lsRemotePath), UTF_8))) {
+ Ref last = null;
+ String line;
+ while ((line = br.readLine()) != null) {
+ ObjectId id = ObjectId.fromString(line.substring(0, 40));
+ String name = line.substring(41, line.length());
+ if (last != null && name.endsWith("^{}")) { //$NON-NLS-1$
+ last = new ObjectIdRef.PeeledTag(PACKED, last.getName(),
+ last.getObjectId(), id);
+ list.set(list.size() - 1, last);
+ continue;
+ }
+
+ if (name.equals(HEAD)) {
+ last = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
+ R_HEADS + MASTER, null));
+ } else {
+ last = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
+ }
+ list.add(last);
+ }
+ }
+ list.sort();
+ return list.toRefList();
+ }
+
+ @SuppressWarnings({ "nls", "boxing" })
+ private void seekCold(String refName) throws Exception {
+ long start, tot;
+
+ int lsTries = Math.min(tries, 64);
+ start = System.nanoTime();
+ for (int i = 0; i < lsTries; i++) {
+ readLsRemote().get(refName);
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
+ tot / 1000,
+ (((double) tot) / lsTries) / 1000,
+ lsTries);
+
+ start = System.nanoTime();
+ for (int i = 0; i < tries; i++) {
+ try (FileInputStream in = new FileInputStream(reftablePath);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ try (RefCursor rc = reader.seekRef(refName)) {
+ while (rc.next()) {
+ rc.getRef();
+ }
+ }
+ }
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
+ tot / 1000,
+ (((double) tot) / tries) / 1000,
+ tries);
+ }
+
+ @SuppressWarnings({ "nls", "boxing" })
+ private void seekHot(String refName) throws Exception {
+ long start, tot;
+
+ int lsTries = Math.min(tries, 64);
+ start = System.nanoTime();
+ RefList<Ref> lsRemote = readLsRemote();
+ for (int i = 0; i < lsTries; i++) {
+ lsRemote.get(refName);
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
+ tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
+
+ start = System.nanoTime();
+ try (FileInputStream in = new FileInputStream(reftablePath);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ for (int i = 0; i < tries; i++) {
+ try (RefCursor rc = reader.seekRef(refName)) {
+ while (rc.next()) {
+ rc.getRef();
+ }
+ }
+ }
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
+ tot / 1000, (((double) tot) / tries) / 1000, tries);
+ }
+
+ @SuppressWarnings({ "nls", "boxing" })
+ private void byIdCold(ObjectId id) throws Exception {
+ long start, tot;
+
+ int lsTries = Math.min(tries, 64);
+ start = System.nanoTime();
+ for (int i = 0; i < lsTries; i++) {
+ for (Ref r : readLsRemote()) {
+ if (id.equals(r.getObjectId())) {
+ continue;
+ }
+ }
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
+ tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
+
+ start = System.nanoTime();
+ for (int i = 0; i < tries; i++) {
+ try (FileInputStream in = new FileInputStream(reftablePath);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ try (RefCursor rc = reader.byObjectId(id)) {
+ while (rc.next()) {
+ rc.getRef();
+ }
+ }
+ }
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
+ tot / 1000, (((double) tot) / tries) / 1000, tries);
+ }
+
+ @SuppressWarnings({ "nls", "boxing" })
+ private void byIdHot(ObjectId id) throws Exception {
+ long start, tot;
+
+ int lsTries = Math.min(tries, 64);
+ start = System.nanoTime();
+ RefList<Ref> lsRemote = readLsRemote();
+ for (int i = 0; i < lsTries; i++) {
+ for (Ref r : lsRemote) {
+ if (id.equals(r.getObjectId())) {
+ continue;
+ }
+ }
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "packed-refs",
+ tot / 1000, (((double) tot) / lsTries) / 1000, lsTries);
+
+ start = System.nanoTime();
+ try (FileInputStream in = new FileInputStream(reftablePath);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ for (int i = 0; i < tries; i++) {
+ try (RefCursor rc = reader.byObjectId(id)) {
+ while (rc.next()) {
+ rc.getRef();
+ }
+ }
+ }
+ }
+ tot = System.nanoTime() - start;
+ printf("%12s %10d usec %9.1f usec/run %5d runs", "reftable",
+ tot / 1000, (((double) tot) / tries) / 1000, tries);
+ }
+}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java
new file mode 100644
index 0000000..9b8db3e
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/ReadReftable.java
@@ -0,0 +1,97 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.kohsuke.args4j.Argument;
+
+@Command
+class ReadReftable extends TextBuiltin {
+ @Argument(index = 0)
+ private String input;
+
+ @Argument(index = 1, required = false)
+ private String ref;
+
+ @Override
+ protected void run() throws Exception {
+ try (FileInputStream in = new FileInputStream(input);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ try (RefCursor rc = ref != null
+ ? reader.seekRef(ref)
+ : reader.allRefs()) {
+ while (rc.next()) {
+ write(rc.getRef());
+ }
+ }
+ }
+ }
+
+ private void write(Ref r) throws IOException {
+ if (r.isSymbolic()) {
+ outw.println(r.getTarget().getName() + '\t' + r.getName());
+ return;
+ }
+
+ ObjectId id1 = r.getObjectId();
+ if (id1 != null) {
+ outw.println(id1.name() + '\t' + r.getName());
+ }
+
+ ObjectId id2 = r.getPeeledObjectId();
+ if (id2 != null) {
+ outw.println('^' + id2.name());
+ }
+ }
+}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
new file mode 100644
index 0000000..dffb579
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.eclipse.jgit.internal.storage.io.BlockSource;
+import org.eclipse.jgit.internal.storage.reftable.RefCursor;
+import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
+import org.eclipse.jgit.lib.AnyObjectId;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.RefComparator;
+import org.eclipse.jgit.lib.TextProgressMonitor;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.kohsuke.args4j.Argument;
+
+@Command
+class VerifyReftable extends TextBuiltin {
+ private static final long SEED1 = 0xaba8bb4de4caf86cL;
+ private static final long SEED2 = 0x28bb5c25ad43ecb5L;
+
+ @Argument(index = 0)
+ private String lsRemotePath;
+
+ @Argument(index = 1)
+ private String reftablePath;
+
+ @Override
+ protected void run() throws Exception {
+ List<Ref> refs = WriteReftable.readRefs(lsRemotePath);
+
+ try (FileInputStream in = new FileInputStream(reftablePath);
+ BlockSource src = BlockSource.from(in);
+ ReftableReader reader = new ReftableReader(src)) {
+ scan(refs, reader);
+ seek(refs, reader);
+ byId(refs, reader);
+ }
+ }
+
+ @SuppressWarnings("nls")
+ private void scan(List<Ref> refs, ReftableReader reader)
+ throws IOException {
+ errw.print(String.format("%-20s", "sequential scan..."));
+ errw.flush();
+ try (RefCursor rc = reader.allRefs()) {
+ for (Ref exp : refs) {
+ verify(exp, rc);
+ }
+ if (rc.next()) {
+ throw die("expected end of table");
+ }
+ }
+ errw.println(" OK");
+ }
+
+ @SuppressWarnings("nls")
+ private void seek(List<Ref> refs, ReftableReader reader)
+ throws IOException {
+ List<Ref> rnd = new ArrayList<>(refs);
+ Collections.shuffle(rnd, new Random(SEED1));
+
+ TextProgressMonitor pm = new TextProgressMonitor(errw);
+ pm.beginTask("random seek", rnd.size());
+ for (Ref exp : rnd) {
+ try (RefCursor rc = reader.seekRef(exp.getName())) {
+ verify(exp, rc);
+ if (rc.next()) {
+ throw die("should not have ref after " + exp.getName());
+ }
+ }
+ pm.update(1);
+ }
+ pm.endTask();
+ }
+
+ @SuppressWarnings("nls")
+ private void byId(List<Ref> refs, ReftableReader reader)
+ throws IOException {
+ Map<ObjectId, List<Ref>> want = groupById(refs);
+ List<List<Ref>> rnd = new ArrayList<>(want.values());
+ Collections.shuffle(rnd, new Random(SEED2));
+
+ TextProgressMonitor pm = new TextProgressMonitor(errw);
+ pm.beginTask("byObjectId", rnd.size());
+ for (List<Ref> exp : rnd) {
+ Collections.sort(exp, RefComparator.INSTANCE);
+ ObjectId id = exp.get(0).getObjectId();
+ try (RefCursor rc = reader.byObjectId(id)) {
+ for (Ref r : exp) {
+ verify(r, rc);
+ }
+ }
+ pm.update(1);
+ }
+ pm.endTask();
+ }
+
+ private static Map<ObjectId, List<Ref>> groupById(List<Ref> refs) {
+ Map<ObjectId, List<Ref>> m = new HashMap<>();
+ for (Ref r : refs) {
+ ObjectId id = r.getObjectId();
+ if (id != null) {
+ List<Ref> c = m.get(id);
+ if (c == null) {
+ c = new ArrayList<>(2);
+ m.put(id, c);
+ }
+ c.add(r);
+ }
+ }
+ return m;
+ }
+
+ @SuppressWarnings("nls")
+ private void verify(Ref exp, RefCursor rc) throws IOException {
+ if (!rc.next()) {
+ throw die("ended before " + exp.getName());
+ }
+
+ Ref act = rc.getRef();
+ if (!exp.getName().equals(act.getName())) {
+ throw die(String.format("expected %s, found %s",
+ exp.getName(),
+ act.getName()));
+ }
+
+ if (exp.isSymbolic()) {
+ if (!act.isSymbolic()) {
+ throw die("expected " + act.getName() + " to be symbolic");
+ }
+ if (!exp.getTarget().getName().equals(act.getTarget().getName())) {
+ throw die(String.format("expected %s to be %s, found %s",
+ exp.getName(),
+ exp.getLeaf().getName(),
+ act.getLeaf().getName()));
+ }
+ return;
+ }
+
+ if (!AnyObjectId.equals(exp.getObjectId(), act.getObjectId())) {
+ throw die(String.format("expected %s to be %s, found %s",
+ exp.getName(),
+ id(exp.getObjectId()),
+ id(act.getObjectId())));
+ }
+
+ if (exp.getPeeledObjectId() != null
+ && !AnyObjectId.equals(exp.getPeeledObjectId(), act.getPeeledObjectId())) {
+ throw die(String.format("expected %s to be %s, found %s",
+ exp.getName(),
+ id(exp.getPeeledObjectId()),
+ id(act.getPeeledObjectId())));
+ }
+ }
+
+ @SuppressWarnings("nls")
+ private static String id(ObjectId id) {
+ return id != null ? id.name() : "<null>";
+ }
+}
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java
new file mode 100644
index 0000000..76ffa19
--- /dev/null
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java
@@ -0,0 +1,301 @@
+/*
+ * Copyright (C) 2017, Google Inc.
+ * and other copyright owners as documented in the project's IP log.
+ *
+ * This program and the accompanying materials are made available
+ * under the terms of the Eclipse Distribution License v1.0 which
+ * accompanies this distribution, is reproduced below, and is
+ * available at http://www.eclipse.org/org/documents/edl-v10.php
+ *
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials provided
+ * with the distribution.
+ *
+ * - Neither the name of the Eclipse Foundation, Inc. nor the
+ * names of its contributors may be used to endorse or promote
+ * products derived from this software without specific prior
+ * written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package org.eclipse.jgit.pgm.debug;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.eclipse.jgit.lib.Constants.HEAD;
+import static org.eclipse.jgit.lib.Constants.MASTER;
+import static org.eclipse.jgit.lib.Constants.R_HEADS;
+import static org.eclipse.jgit.lib.Ref.Storage.NEW;
+import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
+import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
+import org.eclipse.jgit.lib.ObjectId;
+import org.eclipse.jgit.lib.ObjectIdRef;
+import org.eclipse.jgit.lib.PersonIdent;
+import org.eclipse.jgit.lib.Ref;
+import org.eclipse.jgit.lib.SymbolicRef;
+import org.eclipse.jgit.pgm.Command;
+import org.eclipse.jgit.pgm.TextBuiltin;
+import org.kohsuke.args4j.Argument;
+import org.kohsuke.args4j.Option;
+
+@Command
+class WriteReftable extends TextBuiltin {
+ private static final int KIB = 1 << 10;
+ private static final int MIB = 1 << 20;
+
+ @Option(name = "--block-size")
+ private int refBlockSize;
+
+ @Option(name = "--log-block-size")
+ private int logBlockSize;
+
+ @Option(name = "--restart-interval")
+ private int restartInterval;
+
+ @Option(name = "--index-levels")
+ private int indexLevels;
+
+ @Option(name = "--reflog-in")
+ private String reflogIn;
+
+ @Option(name = "--no-index-objects")
+ private boolean noIndexObjects;
+
+ @Argument(index = 0)
+ private String in;
+
+ @Argument(index = 1)
+ private String out;
+
+ @SuppressWarnings({ "nls", "boxing" })
+ @Override
+ protected void run() throws Exception {
+ List<Ref> refs = readRefs(in);
+ List<LogEntry> logs = readLog(reflogIn);
+
+ ReftableWriter.Stats stats;
+ try (OutputStream os = new FileOutputStream(out)) {
+ ReftableConfig cfg = new ReftableConfig();
+ cfg.setIndexObjects(!noIndexObjects);
+ if (refBlockSize > 0) {
+ cfg.setRefBlockSize(refBlockSize);
+ }
+ if (logBlockSize > 0) {
+ cfg.setLogBlockSize(logBlockSize);
+ }
+ if (restartInterval > 0) {
+ cfg.setRestartInterval(restartInterval);
+ }
+ if (indexLevels > 0) {
+ cfg.setMaxIndexLevels(indexLevels);
+ }
+
+ ReftableWriter w = new ReftableWriter(cfg);
+ w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
+ w.begin(os);
+ w.sortAndWriteRefs(refs);
+ for (LogEntry e : logs) {
+ w.writeLog(e.ref, e.updateIndex, e.who,
+ e.oldId, e.newId, e.message);
+ }
+ stats = w.finish().getStats();
+ }
+
+ double fileMiB = ((double) stats.totalBytes()) / MIB;
+ printf("Summary:");
+ printf(" file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
+ printf(" padding : %d KiB", stats.paddingBytes() / KIB);
+ errw.println();
+
+ printf("Refs:");
+ printf(" ref blk : %d", stats.refBlockSize());
+ printf(" restarts: %d", stats.restartInterval());
+ printf(" refs : %d", stats.refCount());
+ if (stats.refIndexLevels() > 0) {
+ int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
+ printf(" idx sz : %d KiB", idxSize);
+ printf(" idx lvl : %d", stats.refIndexLevels());
+ }
+ printf(" avg ref : %d bytes", stats.refBytes() / refs.size());
+ errw.println();
+
+ if (stats.objCount() > 0) {
+ int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
+ int idLen = stats.objIdLength();
+ printf("Objects:");
+ printf(" obj blk : %d", stats.refBlockSize());
+ printf(" restarts: %d", stats.restartInterval());
+ printf(" objects : %d", stats.objCount());
+ printf(" obj sz : %d MiB (%d bytes)", objMiB, stats.objBytes());
+ if (stats.objIndexSize() > 0) {
+ int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
+ printf(" idx sz : %d KiB", s);
+ printf(" idx lvl : %d", stats.objIndexLevels());
+ }
+ printf(" id len : %d bytes (%d hex digits)", idLen, 2 * idLen);
+ printf(" avg obj : %d bytes", stats.objBytes() / stats.objCount());
+ errw.println();
+ }
+ if (stats.logCount() > 0) {
+ int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
+ printf("Log:");
+ printf(" log blk : %d", stats.logBlockSize());
+ printf(" logs : %d", stats.logCount());
+ printf(" log sz : %d MiB (%d bytes)", logMiB, stats.logBytes());
+ printf(" avg log : %d bytes", stats.logBytes() / logs.size());
+ errw.println();
+ }
+ }
+
+ private void printf(String fmt, Object... args) throws IOException {
+ errw.println(String.format(fmt, args));
+ }
+
+ static List<Ref> readRefs(String inputFile) throws IOException {
+ List<Ref> refs = new ArrayList<>();
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
+ String line;
+ while ((line = br.readLine()) != null) {
+ ObjectId id = ObjectId.fromString(line.substring(0, 40));
+ String name = line.substring(41, line.length());
+ if (name.endsWith("^{}")) { //$NON-NLS-1$
+ int lastIdx = refs.size() - 1;
+ Ref last = refs.get(lastIdx);
+ refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
+ last.getName(), last.getObjectId(), id));
+ continue;
+ }
+
+ Ref ref;
+ if (name.equals(HEAD)) {
+ ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
+ R_HEADS + MASTER, null));
+ } else {
+ ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
+ }
+ refs.add(ref);
+ }
+ }
+ Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
+ return refs;
+ }
+
+ private static List<LogEntry> readLog(String logPath)
+ throws FileNotFoundException, IOException {
+ if (logPath == null) {
+ return Collections.emptyList();
+ }
+
+ List<LogEntry> log = new ArrayList<>();
+ try (BufferedReader br = new BufferedReader(
+ new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
+ @SuppressWarnings("nls")
+ Pattern pattern = Pattern.compile("([^,]+)" // 1: ref
+ + ",([0-9]+(?:[.][0-9]+)?)" // 2: time
+ + ",([^,]+)" // 3: who
+ + ",([^,]+)" // 4: old
+ + ",([^,]+)" // 5: new
+ + ",(.*)"); // 6: msg
+ String line;
+ while ((line = br.readLine()) != null) {
+ Matcher m = pattern.matcher(line);
+ if (!m.matches()) {
+ throw new IOException("unparsed line: " + line); //$NON-NLS-1$
+ }
+ String ref = m.group(1);
+ double t = Double.parseDouble(m.group(2));
+ long time = ((long) t) * 1000L;
+ long index = (long) (t * 1e6);
+ String user = m.group(3);
+ ObjectId oldId = parseId(m.group(4));
+ ObjectId newId = parseId(m.group(5));
+ String msg = m.group(6);
+ String email = user + "@gerrit"; //$NON-NLS-1$
+ PersonIdent who = new PersonIdent(user, email, time, -480);
+ log.add(new LogEntry(ref, index, who, oldId, newId, msg));
+ }
+ }
+ Collections.sort(log, LogEntry::compare);
+ return log;
+ }
+
+ private static long min(List<LogEntry> log) {
+ return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
+ }
+
+ private static long max(List<LogEntry> log) {
+ return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
+ }
+
+ private static ObjectId parseId(String s) {
+ if ("NULL".equals(s)) { //$NON-NLS-1$
+ return ObjectId.zeroId();
+ }
+ return ObjectId.fromString(s);
+ }
+
+ private static class LogEntry {
+ static int compare(LogEntry a, LogEntry b) {
+ int cmp = a.ref.compareTo(b.ref);
+ if (cmp == 0) {
+ cmp = Long.signum(b.updateIndex - a.updateIndex);
+ }
+ return cmp;
+ }
+
+ final String ref;
+ final long updateIndex;
+ final PersonIdent who;
+ final ObjectId oldId;
+ final ObjectId newId;
+ final String message;
+
+ LogEntry(String ref, long updateIndex, PersonIdent who,
+ ObjectId oldId, ObjectId newId, String message) {
+ this.ref = ref;
+ this.updateIndex = updateIndex;
+ this.who = who;
+ this.oldId = oldId;
+ this.newId = newId;
+ this.message = message;
+ }
+ }
+}
diff --git a/org.eclipse.jgit/META-INF/MANIFEST.MF b/org.eclipse.jgit/META-INF/MANIFEST.MF
index bbc500f..e326081 100644
--- a/org.eclipse.jgit/META-INF/MANIFEST.MF
+++ b/org.eclipse.jgit/META-INF/MANIFEST.MF
@@ -74,6 +74,7 @@
org.eclipse.jgit.lfs,
org.eclipse.jgit.pgm,
org.eclipse.jgit.pgm.test",
+ org.eclipse.jgit.internal.storage.io;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.pack;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.reftable;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
org.eclipse.jgit.internal.storage.reftree;version="4.9.0";x-friends:="org.eclipse.jgit.junit,org.eclipse.jgit.test,org.eclipse.jgit.pgm",
