DO NOT SUBMIT mh hack
Change-Id: Iade6952edd09be482ae10f956ee7a611032374b4
diff --git a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
index 2393340..1029668 100644
--- a/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
+++ b/org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/VerifyReftable.java
@@ -84,7 +84,7 @@
ReftableReader reader = new ReftableReader(src)) {
scan(refs, reader);
seek(refs, reader);
- byId(refs, reader);
+ // byId(refs, reader);
}
}
@@ -125,7 +125,7 @@
}
@SuppressWarnings("nls")
- private void byId(List<Ref> refs, ReftableReader reader)
+ void byId(List<Ref> refs, ReftableReader reader)
throws IOException {
Map<ObjectId, List<Ref>> want = groupById(refs);
List<List<Ref>> rnd = new ArrayList<>(want.values());
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java
index c8ea9e2..36a5f31 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockReader.java
@@ -44,19 +44,15 @@
package org.eclipse.jgit.internal.storage.reftable;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.eclipse.jgit.internal.storage.reftable.BlockWriter.compare;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_BLOCK_TYPE;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
+import static org.eclipse.jgit.internal.storage.reftable.ChunkWriter.compare;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.OBJ_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.REF_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_1ID;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_2ID;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_NONE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_TEXT;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_TYPE_MASK;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.reverseUpdateIndex;
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_STRING_LENGTH;
import static org.eclipse.jgit.lib.Ref.Storage.NEW;
@@ -65,27 +61,19 @@
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Arrays;
-import java.util.zip.DataFormatException;
-import java.util.zip.Inflater;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.io.BlockSource;
-import org.eclipse.jgit.lib.CheckoutEntry;
-import org.eclipse.jgit.lib.InflaterCache;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
-import org.eclipse.jgit.lib.PersonIdent;
import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.ReflogEntry;
import org.eclipse.jgit.lib.SymbolicRef;
-import org.eclipse.jgit.util.IntList;
-import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.RawParseUtils;
/** Reads a single block for {@link ReftableReader}. */
class BlockReader {
private byte blockType;
- private long endPosition;
+ private long addr;
private byte[] buf;
private int bufLen;
@@ -93,19 +81,16 @@
private int keysStart;
private int keysEnd;
- private int restartIdx;
- private int restartCount;
private byte[] nameBuf = new byte[256];
private int nameLen;
- private int valueType;
byte type() {
return blockType;
}
long endPosition() {
- return endPosition;
+ return addr + bufLen;
}
boolean next() {
@@ -113,9 +98,8 @@
}
void parseKey() {
- int pfx = readVarint32();
- valueType = readVarint32();
- int sfx = valueType >>> 3;
+ int pfx = ptr == keysStart ? 0 : readVarint32();
+ int sfx = readVarint32();
if (pfx + sfx > nameBuf.length) {
int n = Math.max(pfx + sfx, nameBuf.length * 2);
nameBuf = Arrays.copyOf(nameBuf, n);
@@ -147,24 +131,21 @@
return compare(match, 0, match.length, nameBuf, 0, len) == 0;
}
- boolean matchAbbrevId(byte[] match) {
- int n = Math.min(match.length, nameLen);
- return compare(match, 0, n, nameBuf, 0, n) == 0;
- }
-
long readIndex() throws IOException {
if (blockType != INDEX_BLOCK_TYPE) {
throw invalidBlock();
}
-
- readVarint32(); // skip prefix length
- int n = readVarint32() >>> 3;
- ptr += n; // skip name
- return readVarint64();
+ if (ptr > keysStart) {
+ readVarint32();
+ }
+ int sfx = readVarint32();
+ ptr += sfx;
+ return addr - readVarint64();
}
Ref readRef() throws IOException {
String name = RawParseUtils.decode(UTF_8, nameBuf, 0, nameLen);
+ int valueType = buf[ptr++];
switch (valueType & VALUE_TYPE_MASK) {
case VALUE_NONE: // delete
return newRef(name);
@@ -197,61 +178,6 @@
}
}
- IntList readBlockList() {
- int n = valueType & VALUE_TYPE_MASK;
- if (n == 0) {
- n = readVarint32();
- }
- IntList b = new IntList(n);
- b.add(readVarint32());
- for (int j = 1; j < n; j++) {
- int prior = b.get(j - 1);
- b.add(prior + readVarint32());
- }
- return b;
- }
-
- long readLogUpdateIndex() {
- return reverseUpdateIndex(NB.decodeUInt64(nameBuf, nameLen - 8));
- }
-
- ReflogEntry readLog() {
- ObjectId oldId = readValueId();
- ObjectId newId = readValueId();
- long ms = readVarint64() * 1000L;
- short tz = readInt16();
- String name = readValueString();
- String email = readValueString();
- String comment = readValueString();
-
- return new ReflogEntry() {
- @Override
- public ObjectId getOldId() {
- return oldId;
- }
-
- @Override
- public ObjectId getNewId() {
- return newId;
- }
-
- @Override
- public PersonIdent getWho() {
- return new PersonIdent(name, email, ms, tz);
- }
-
- @Override
- public String getComment() {
- return comment;
- }
-
- @Override
- public CheckoutEntry parseCheckout() {
- return null;
- }
- };
- }
-
private ObjectId readValueId() {
ObjectId id = ObjectId.fromRaw(buf, ptr);
ptr += OBJECT_ID_LENGTH;
@@ -286,109 +212,21 @@
b.flip();
b.get(buf);
}
- endPosition = pos + bufLen;
}
private void parseBlockStart(long pos) throws IOException {
+ addr = pos;
ptr = 0;
- if (pos == 0) {
- if (bufLen == FILE_HEADER_LEN) {
- setupEmptyFileBlock();
- return;
- }
- ptr += FILE_HEADER_LEN; // first block begins with file header
- }
-
- int typeAndSize = NB.decodeInt32(buf, ptr);
- ptr += 4;
-
- blockType = (byte) (typeAndSize >>> 24);
- int blockLen;
- if ((blockType & INDEX_BLOCK_TYPE) == INDEX_BLOCK_TYPE) {
- // Index blocks are allowed to grow up to 31-bit blockSize.
- blockType = INDEX_BLOCK_TYPE;
- blockLen = decodeIndexSize(typeAndSize);
- } else {
- blockLen = typeAndSize & 0xffffff;
- }
- if (blockType == LOG_BLOCK_TYPE) {
- // Log blocks must be inflated after the header.
- long deflatedSize = inflateBuf(blockLen);
- endPosition = pos + 4 + deflatedSize;
- }
- if (bufLen < blockLen) {
+ blockType = buf[ptr++];
+ int n = readVarint32();
+ if (ptr + n > bufLen || ptr + n > buf.length) {
throw invalidBlock();
- } else if (bufLen > blockLen) {
- bufLen = blockLen;
}
-
+ bufLen = ptr + n;
keysStart = ptr;
- if (blockType != FILE_BLOCK_TYPE) {
- restartCount = NB.decodeUInt16(buf, bufLen - 2);
- restartIdx = bufLen - (restartCount * 3 + 2);
- keysEnd = restartIdx;
- } else {
- keysEnd = keysStart;
- }
- }
-
- static int decodeIndexSize(int typeAndSize) {
- return typeAndSize & 0x7fffffff;
- }
-
- private long inflateBuf(int blockLen) throws IOException {
- byte[] dst = new byte[4 + blockLen];
- System.arraycopy(buf, 0, dst, 0, 4);
-
- long deflatedSize = 0;
- Inflater inf = InflaterCache.get();
- try {
- inf.setInput(buf, ptr, bufLen - ptr);
- for (int o = 4;;) {
- int n = inf.inflate(dst, o, dst.length - o);
- o += n;
- if (inf.finished()) {
- deflatedSize = inf.getBytesRead();
- break;
- } else if (n <= 0) {
- throw invalidBlock();
- }
- }
- } catch (DataFormatException e) {
- throw invalidBlock(e);
- } finally {
- InflaterCache.release(inf);
- }
-
- buf = dst;
- bufLen = dst.length;
- return deflatedSize;
- }
-
- private void setupEmptyFileBlock() {
- // An empty reftable has only the file header in first block.
- blockType = FILE_BLOCK_TYPE;
- ptr = FILE_HEADER_LEN;
- restartCount = 0;
- restartIdx = bufLen;
- keysStart = bufLen;
keysEnd = bufLen;
}
- void verifyIndex() throws IOException {
- if (blockType != INDEX_BLOCK_TYPE) {
- throw invalidBlock();
- }
- }
-
- int seekKey(byte[] key) {
- return seek(key, true);
- }
-
- int seekAbbrevId(byte[] key) {
- return seek(key, false);
- }
-
private static int compareKey(boolean useKeyLen, byte[] a,
byte[] b, int bi, int bLen) {
if (useKeyLen) {
@@ -398,46 +236,13 @@
return compare(a, 0, n, b, bi, n);
}
- private int seek(byte[] key, boolean useKeyLen) {
- int low = 0;
- int end = restartCount;
- for (;;) {
- int mid = (low + end) >>> 1;
- int p = NB.decodeUInt24(buf, restartIdx + mid * 3);
- ptr = p + 1; // skip 0 prefix length
- int n = readVarint32() >>> 3;
- int cmp = compareKey(useKeyLen, key, buf, ptr, n);
- if (cmp < 0) {
- end = mid;
- } else if (cmp == 0) {
- ptr = p;
- return 0;
- } else /* if (cmp > 0) */ {
- low = mid + 1;
- }
- if (low >= end) {
- return seekToKey(key, useKeyLen, p, low, cmp);
- }
- }
- }
-
- private int seekToKey(byte[] key, boolean useKeyLen,
- int rPtr, int rIdx, int rCmp) {
- if (rCmp < 0) {
- if (rIdx == 0) {
- ptr = keysStart;
- return -1;
- }
- ptr = NB.decodeUInt24(buf, restartIdx + (rIdx - 1) * 3);
- } else {
- ptr = rPtr;
- }
-
+ int seekToKey(byte[] key) {
+ ptr = keysStart;
int cmp;
do {
int savePtr = ptr;
parseKey();
- cmp = compareKey(useKeyLen, key, nameBuf, 0, nameLen);
+ cmp = compareKey(true, key, nameBuf, 0, nameLen);
if (cmp <= 0) {
// cmp < 0, name should be in this block, but is not.
// cmp = 0, block is positioned at name.
@@ -452,6 +257,7 @@
void skipValue() {
switch (blockType) {
case REF_BLOCK_TYPE:
+ int valueType = buf[ptr++];
switch (valueType & VALUE_TYPE_MASK) {
case VALUE_NONE:
return;
@@ -467,31 +273,10 @@
}
break;
- case OBJ_BLOCK_TYPE: {
- int n = valueType & VALUE_TYPE_MASK;
- if (n == 0) {
- n = readVarint32();
- }
- while (n-- > 0) {
- readVarint32();
- }
- return;
- }
-
case INDEX_BLOCK_TYPE:
- readVarint32();
- return;
-
- case LOG_BLOCK_TYPE:
- ptr += 2 * OBJECT_ID_LENGTH; // 2x id;
- readVarint64(); // time
- ptr += 2; // 2-byte tz
- skipString(); // name
- skipString(); // email
- skipString(); // comment
+ readVarint64();
return;
}
-
throw new IllegalStateException();
}
@@ -500,10 +285,6 @@
ptr += n;
}
- private short readInt16() {
- return (short) NB.decodeUInt16(buf, ptr += 2);
- }
-
private int readVarint32() {
byte c = buf[ptr++];
int val = c & 0x7f;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ChunkWriter.java
similarity index 74%
rename from org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java
rename to org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ChunkWriter.java
index 9c05381..df94e02 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/BlockWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ChunkWriter.java
@@ -46,12 +46,10 @@
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.MAX_RESTARTS;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.OBJ_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.REF_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_1ID;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_2ID;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_INDEX_RECORD;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_LOG_RECORD;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_NONE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VALUE_TEXT;
@@ -73,27 +71,20 @@
import org.eclipse.jgit.util.IntList;
import org.eclipse.jgit.util.NB;
-class BlockWriter {
- private final byte blockType;
+class ChunkWriter {
+ private final byte chunkType;
private final List<Entry> entries = new ArrayList<>();
private final int blockSize;
- private final int restartInterval;
private int bytesInKeyTable;
- private int restartCnt;
- BlockWriter(byte type, int bs, int ri) {
- blockType = type;
+ ChunkWriter(byte type, int bs) {
+ chunkType = type;
blockSize = bs;
- restartInterval = ri;
}
byte blockType() {
- return blockType;
- }
-
- boolean padBetweenBlocks() {
- return blockType == REF_BLOCK_TYPE || blockType == OBJ_BLOCK_TYPE;
+ return chunkType;
}
byte[] lastKey() {
@@ -105,120 +96,56 @@
}
int currentSize() {
- return computeBlockSize(0, false);
- }
-
- int estimateIndexSizeIfAdding(byte[] lastKey, long blockOffset) {
- IndexEntry entry = new IndexEntry(lastKey, blockOffset);
- return computeBlockSize(entry.size(0), true);
- }
-
- void addIndex(byte[] lastKey, long blockOffset) {
- entries.add(new IndexEntry(lastKey, blockOffset));
+ return computeBlockSize(0);
}
void addFirst(Entry entry) throws BlockSizeTooSmallException {
- if (!tryAdd(entry, true)) {
+ if (!tryAdd(entry)) {
// Insanely long names need a larger block size.
throw blockSizeTooSmall(entry);
}
}
boolean tryAdd(Entry entry) {
- if (tryAdd(entry, true)) {
- return true;
- } else if (nextShouldBeRestart()) {
- // It was time for another restart, but the entry doesn't fit
- // with its complete name, as the block is nearly full. Try to
- // force it to fit with prefix compression rather than waste
- // the tail of the block with padding.
- return tryAdd(entry, false);
- }
- return false;
- }
-
- private boolean tryAdd(Entry entry, boolean tryRestart) {
byte[] key = entry.key;
int prefixLen = 0;
- boolean restart = tryRestart && nextShouldBeRestart();
- if (!restart) {
+ boolean first = entries.isEmpty();
+ if (!first) {
byte[] prior = entries.get(entries.size() - 1).key;
prefixLen = commonPrefix(prior, prior.length, key);
- if (prefixLen == 0) {
- restart = true;
- }
}
- int entrySize = entry.size(prefixLen);
- if (computeBlockSize(entrySize, restart) > blockSize) {
+ int entrySize = entry.size(first, prefixLen);
+ if (computeBlockSize(entrySize) > blockSize) {
return false;
}
bytesInKeyTable += entrySize;
entries.add(entry);
- if (restart) {
- entry.restart = true;
- restartCnt++;
- }
return true;
}
- private boolean nextShouldBeRestart() {
- int cnt = entries.size();
- return (cnt == 0 || ((cnt + 1) % restartInterval) == 0)
- && restartCnt < MAX_RESTARTS;
- }
-
- private int computeBlockSize(int key, boolean restart) {
- return 4 // 4-byte block header
- + bytesInKeyTable + key
- + (restartCnt + (restart ? 1 : 0)) * 3
- + 2; // 2-byte restart_count
+ private int computeBlockSize(int key) {
+ int sz = bytesInKeyTable + key;
+ return 1 + computeVarintSize(sz) + sz;
}
void writeTo(ReftableOutputStream os) throws IOException {
- if (blockType == INDEX_BLOCK_TYPE) {
- selectIndexRestarts();
- }
- if (restartCnt > MAX_RESTARTS) {
- throw new IllegalStateException();
- }
-
- IntList restartOffsets = new IntList(restartCnt);
- byte[] prior = {};
-
- os.beginBlock(blockType);
+ byte[] prior = null;
+ os.beginBlock(chunkType);
+ os.writeVarint(bytesInKeyTable);
for (int entryIdx = 0; entryIdx < entries.size(); entryIdx++) {
Entry entry = entries.get(entryIdx);
- if (entry.restart) {
- restartOffsets.add(os.bytesWrittenInBlock());
- }
entry.writeKey(os, prior);
entry.writeValue(os);
prior = entry.key;
}
- for (int i = 0; i < restartOffsets.size(); i++) {
- os.writeInt24(restartOffsets.get(i));
- }
- os.writeInt16(restartOffsets.size());
os.flushBlock();
}
- private void selectIndexRestarts() {
- // Indexes grow without bound, but the restart table has a limit.
- // Select restarts in the index as far apart as possible to stay
- // within the MAX_RESTARTS limit defined by the file format.
- int ir = Math.max(restartInterval, entries.size() / MAX_RESTARTS);
- for (int k = 0; k < entries.size(); k++) {
- if ((k % ir) == 0) {
- entries.get(k).restart = true;
- }
- }
- }
-
private BlockSizeTooSmallException blockSizeTooSmall(Entry entry) {
// Compute size required to fit this entry by itself.
- int min = computeBlockSize(entry.size(0), true);
+ int min = computeBlockSize(entry.size(true, 0));
return new BlockSizeTooSmallException(min);
}
@@ -254,35 +181,32 @@
static int compare(Entry ea, Entry eb) {
byte[] a = ea.key;
byte[] b = eb.key;
- return BlockWriter.compare(a, 0, a.length, b, 0, b.length);
+ return ChunkWriter.compare(a, 0, a.length, b, 0, b.length);
}
final byte[] key;
- boolean restart;
Entry(byte[] key) {
this.key = key;
}
void writeKey(ReftableOutputStream os, byte[] prior) {
- int pfx;
- int sfx;
- if (restart) {
- pfx = 0;
- sfx = key.length;
+ if (prior == null) {
+ os.writeVarint(key.length);
+ os.write(key, 0, key.length);
} else {
- pfx = commonPrefix(prior, prior.length, key);
- sfx = key.length - pfx;
+ int pfx = commonPrefix(prior, prior.length, key);
+ int sfx = key.length - pfx;
+ os.writeVarint(pfx);
+ os.writeVarint(sfx);
+ os.write(key, pfx, sfx);
}
- os.writeVarint(pfx);
- os.writeVarint(encodeSuffixAndType(sfx, valueType()));
- os.write(key, pfx, sfx);
}
- int size(int prefixLen) {
+ int size(boolean first, int prefixLen) {
int sfx = key.length - prefixLen;
- return computeVarintSize(prefixLen)
- + computeVarintSize(encodeSuffixAndType(sfx, valueType()))
+ return (first ? 0 : computeVarintSize(prefixLen))
+ + computeVarintSize(sfx)
+ sfx
+ valueSize();
}
@@ -294,11 +218,11 @@
}
static class IndexEntry extends Entry {
- private final long blockOffset;
+ private final long childAddr;
- IndexEntry(byte[] key, long blockOffset) {
+ IndexEntry(byte[] key, long childAddr) {
super(key);
- this.blockOffset = blockOffset;
+ this.childAddr = childAddr;
}
@Override
@@ -308,17 +232,24 @@
@Override
int valueType() {
- return VALUE_INDEX_RECORD;
+ return 0;
}
@Override
int valueSize() {
- return computeVarintSize(blockOffset);
+ return computeVarintSize(childAddr);
}
@Override
void writeValue(ReftableOutputStream os) {
- os.writeVarint(blockOffset);
+ os.writeVarint(childAddr);
+ }
+
+ IndexEntry forIndexAt(long indexAddr) {
+ if (indexAddr <= childAddr) {
+ throw new IllegalArgumentException();
+ }
+ return new IndexEntry(key, indexAddr - childAddr);
}
}
@@ -352,18 +283,20 @@
int valueSize() {
if (ref.isSymbolic()) {
int nameLen = 5 + nameUtf8(ref.getTarget()).length;
- return computeVarintSize(nameLen) + nameLen;
+ return 1 + computeVarintSize(nameLen) + nameLen;
} else if (ref.getStorage() == NEW && ref.getObjectId() == null) {
- return 0;
+ return 1;
} else if (ref.getPeeledObjectId() != null) {
- return 2 * OBJECT_ID_LENGTH;
+ return 1 + 2 * OBJECT_ID_LENGTH;
} else {
- return OBJECT_ID_LENGTH;
+ return 1 + OBJECT_ID_LENGTH;
}
}
@Override
void writeValue(ReftableOutputStream os) throws IOException {
+ os.write(valueType());
+
if (ref.isSymbolic()) {
String target = ref.getTarget().getName();
os.writeVarintString("ref: " + target); //$NON-NLS-1$
@@ -412,11 +345,12 @@
@Override
int valueSize() {
- return computeVarintSize(value.length) + value.length;
+ return 1 + computeVarintSize(value.length) + value.length;
}
@Override
void writeValue(ReftableOutputStream os) throws IOException {
+ os.write(valueType());
os.writeVarint(value.length);
os.write(value);
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConfig.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConfig.java
index ed7d03e..643cc59 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConfig.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConfig.java
@@ -50,10 +50,10 @@
/** Configuration used by a reftable writer when constructing the stream. */
public class ReftableConfig {
- private int refBlockSize = 4 << 10;
+ private int refBlockSize = 1 << 10;
private int logBlockSize = 64 << 10;
private int restartInterval;
- private boolean indexObjects = true;
+ private boolean indexObjects = false;
/** Create a default configuration. */
public ReftableConfig() {
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConstants.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConstants.java
index 80c3a7d..c0f4f8f 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConstants.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableConstants.java
@@ -47,8 +47,8 @@
static final byte[] FILE_HEADER_MAGIC = { 'R', 'E', 'F', 'T' };
static final byte VERSION_1 = (byte) 1;
- static final int FILE_HEADER_LEN = 24;
- static final int FILE_FOOTER_LEN = 68;
+ static final int FILE_HEADER_LEN = 40;
+ static final int FILE_FOOTER_LEN = FILE_HEADER_LEN + 4;
static final int MAX_BLOCK_SIZE = (1 << 24) - 1;
static final byte FILE_BLOCK_TYPE = 'R';
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java
index 8e5222d..6c8c6f5 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableOutputStream.java
@@ -44,18 +44,12 @@
package org.eclipse.jgit.internal.storage.reftable;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
-import java.util.zip.Deflater;
-import java.util.zip.DeflaterOutputStream;
-import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.util.NB;
import org.eclipse.jgit.util.io.CountingOutputStream;
@@ -69,12 +63,8 @@
class ReftableOutputStream extends OutputStream {
private final byte[] tmp = new byte[10];
private final CountingOutputStream out;
- private final Deflater deflater;
- private final DeflaterOutputStream compressor;
- private int blockType;
private int blockSize;
- private int blockStart;
private byte[] blockBuf;
private int cur;
@@ -86,8 +76,6 @@
blockBuf = new byte[bs];
out = new CountingOutputStream(os);
- deflater = new Deflater(Deflater.BEST_COMPRESSION);
- compressor = new DeflaterOutputStream(out, deflater);
}
void setBlockSize(int bs) {
@@ -180,67 +168,25 @@
}
void flushFileHeader() throws IOException {
- if (cur == FILE_HEADER_LEN && out.getCount() == 0) {
- // If the file will be a log only file, flush the file header.
- out.write(blockBuf, 0, cur);
- cur = 0;
- }
+ out.write(blockBuf, 0, cur);
+ cur = 0;
}
void beginBlock(byte id) {
- blockType = id;
- blockStart = cur;
- cur += 4; // reserve space for 4-byte block header.
+ blockBuf[cur++] = id;
}
void flushBlock() throws IOException {
- if (cur > blockSize && !isIndexBlock()) {
- throw new IOException(JGitText.get().overflowedReftableBlock);
- }
- NB.encodeInt32(blockBuf, blockStart, (blockType << 24) | cur);
-
- if (blockType == LOG_BLOCK_TYPE) {
- // Log blocks are deflated after the block header.
- deflater.reset();
- out.write(blockBuf, 0, 4);
- compressor.write(blockBuf, 4, cur - 4);
- compressor.finish();
- } else {
- // Other blocks are uncompressed.
- out.write(blockBuf, 0, cur);
- }
+ out.write(blockBuf, 0, cur);
cur = 0;
- blockType = 0;
- blockStart = 0;
blockCount++;
}
- void padBetweenBlocksToNextBlock() throws IOException {
- long m = size() % blockSize;
- if (m > 0) {
- int pad = blockSize - (int) m;
- ensureBytesAvailableInBlockBuf(pad);
- Arrays.fill(blockBuf, 0, pad, (byte) 0);
- out.write(blockBuf, 0, pad);
- paddingUsed += pad;
- }
- }
-
- int estimatePadBetweenBlocks(int currentBlockSize) {
- long m = (size() + currentBlockSize) % blockSize;
- return m > 0 ? blockSize - (int) m : 0;
- }
-
- private boolean isIndexBlock() {
- return (blockType & INDEX_BLOCK_TYPE) == INDEX_BLOCK_TYPE;
- }
-
void finishFile() throws IOException {
// File footer doesn't need patching for the block start.
// Just flush what has been buffered.
out.write(blockBuf, 0, cur);
cur = 0;
- deflater.end();
}
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java
index 3360eb7..b3bf33a 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java
@@ -44,14 +44,11 @@
package org.eclipse.jgit.internal.storage.reftable;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.eclipse.jgit.internal.storage.reftable.BlockReader.decodeIndexSize;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_FOOTER_LEN;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.REF_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VERSION_1;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.isFileHeaderMagic;
-import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -61,12 +58,8 @@
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.internal.storage.io.BlockSource;
-import org.eclipse.jgit.internal.storage.reftable.BlockWriter.LogEntry;
import org.eclipse.jgit.lib.AnyObjectId;
-import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.Ref;
-import org.eclipse.jgit.lib.ReflogEntry;
-import org.eclipse.jgit.util.IntList;
import org.eclipse.jgit.util.NB;
/**
@@ -78,23 +71,12 @@
public class ReftableReader extends Reftable {
private final BlockSource src;
- private int blockSize;
private long minUpdateIndex;
private long maxUpdateIndex;
+ private long refRootChunkAddr;
+ long objRootChunkAddr;
- private long refEnd;
- private long objOffset;
- private long objEnd;
- private long logOffset;
- private long logEnd;
-
- private long refIndexOffset = -1;
- private long objIndexOffset = -1;
- private long logIndexOffset = -1;
-
- private BlockReader refIndex;
- private BlockReader objIndex;
- private BlockReader logIndex;
+ private BlockReader refRoot;
/**
* Initialize a new reftable reader.
@@ -114,10 +96,7 @@
* file cannot be read.
*/
public int blockSize() throws IOException {
- if (blockSize == 0) {
- readFileHeader();
- }
- return blockSize;
+ return 0;
}
/**
@@ -128,7 +107,7 @@
* file cannot be read.
*/
public long minUpdateIndex() throws IOException {
- if (blockSize == 0) {
+ if (minUpdateIndex == 0) {
readFileHeader();
}
return minUpdateIndex;
@@ -142,7 +121,7 @@
* file cannot be read.
*/
public long maxUpdateIndex() throws IOException {
- if (blockSize == 0) {
+ if (minUpdateIndex == 0) {
readFileHeader();
}
return maxUpdateIndex;
@@ -150,21 +129,17 @@
@Override
public RefCursor allRefs() throws IOException {
- if (blockSize == 0) {
- readFileHeader();
- }
-
- long end = refEnd > 0 ? refEnd : (src.size() - FILE_FOOTER_LEN);
- src.adviseSequentialRead(0, end);
-
+ long end = src.size() - FILE_FOOTER_LEN;
RefCursorImpl i = new RefCursorImpl(end, null, false);
- i.block = readBlock(0, end);
+ i.block = readBlock(FILE_HEADER_LEN, end);
return i;
}
@Override
public RefCursor seek(String refName) throws IOException {
- initRefIndex();
+ if (refRootChunkAddr == 0) {
+ readFileFooter();
+ }
byte[] match = refName.getBytes(UTF_8);
boolean prefix = match[match.length - 1] == '/';
@@ -174,87 +149,46 @@
key[key.length - 1] = '\1';
}
- RefCursorImpl i = new RefCursorImpl(refEnd, match, prefix);
- i.block = seek(REF_BLOCK_TYPE, key, refIndex, 0, refEnd);
+ long end = src.size() - FILE_FOOTER_LEN;
+ RefCursorImpl i = new RefCursorImpl(end, match, prefix);
+ i.block = seek(key, refRootChunkAddr, end);
return i;
}
@Override
public RefCursor byObjectId(AnyObjectId id) throws IOException {
- initObjIndex();
- ObjCursorImpl i = new ObjCursorImpl(refEnd, id);
- if (objIndex != null) {
- i.initSeek();
- } else {
- i.initScan();
- }
- return i;
+ throw new UnsupportedOperationException();
}
@Override
public LogCursor allLogs() throws IOException {
- initLogIndex();
- if (logOffset > 0) {
- src.adviseSequentialRead(logOffset, logEnd);
- LogCursorImpl i = new LogCursorImpl(logEnd, null);
- i.block = readBlock(logOffset, logEnd);
- return i;
- }
return new EmptyLogCursor();
}
@Override
public LogCursor seekLog(String refName, long updateIndex)
throws IOException {
- initLogIndex();
- if (logOffset > 0) {
- byte[] key = LogEntry.key(refName, updateIndex);
- byte[] match = refName.getBytes(UTF_8);
- LogCursorImpl i = new LogCursorImpl(logEnd, match);
- i.block = seek(LOG_BLOCK_TYPE, key, logIndex, logOffset, logEnd);
- return i;
- }
return new EmptyLogCursor();
}
- private BlockReader seek(byte blockType, byte[] key, BlockReader idx,
- long start, long end) throws IOException {
- if (idx != null) {
- long blockOffset = idx.seekKey(key) <= 0
- ? idx.readIndex()
- : ((blocksIn(start, end) - 1) * blockSize);
- BlockReader block = readBlock(blockOffset, end);
- block.seekKey(key);
- return block;
+ private BlockReader seek(byte[] key, long addr, long end)
+ throws IOException {
+ BlockReader block = refRoot;
+ if (block == null) {
+ block = readBlock(addr, end);
+ refRoot = block;
}
- return binarySearch(blockType, key, start, end);
- }
-
- private BlockReader binarySearch(byte blockType, byte[] name,
- long startPos, long endPos) throws IOException {
- int low = (int) (startPos / blockSize);
- int end = blocksIn(startPos, endPos);
- BlockReader block = null;
do {
- int mid = (low + end) >>> 1;
- block = readBlock(((long) mid) * blockSize, endPos);
- if (blockType != block.type()) {
- return null;
- }
- int cmp = block.seekKey(name);
- if (cmp < 0) {
- end = mid;
- } else if (cmp == 0) {
- break;
- } else /* if (cmp > 0) */ {
- low = mid + 1;
- }
- } while (low < end);
+ block.seekToKey(key);
+ addr = block.readIndex();
+ block = readBlock(addr, end);
+ } while (block.type() == ReftableConstants.INDEX_BLOCK_TYPE);
+ block.seekToKey(key);
return block;
}
private void readFileHeader() throws IOException {
- readHeaderOrFooter(0, FILE_HEADER_LEN);
+ readFileFooter();
}
private void readFileFooter() throws IOException {
@@ -266,40 +200,6 @@
if (crc.getValue() != NB.decodeUInt32(ftr, ftrLen - 4)) {
throw new IOException(JGitText.get().invalidReftableCRC);
}
-
- refIndexOffset = NB.decodeInt64(ftr, 24);
- objOffset = NB.decodeInt64(ftr, 32);
- objIndexOffset = NB.decodeInt64(ftr, 40);
- logOffset = NB.decodeInt64(ftr, 48);
- logIndexOffset = NB.decodeInt64(ftr, 56);
-
- if (refIndexOffset > 0) {
- refEnd = refIndexOffset;
- } else if (objOffset > 0) {
- refEnd = objOffset;
- } else if (logOffset > 0) {
- refEnd = logOffset;
- } else {
- refEnd = src.size() - ftrLen;
- }
-
- if (objOffset > 0) {
- if (objIndexOffset > 0) {
- objEnd = objIndexOffset;
- } else if (logOffset > 0) {
- objEnd = logOffset;
- } else {
- objEnd = src.size() - ftrLen;
- }
- }
-
- if (logOffset > 0) {
- if (logIndexOffset > 0) {
- logEnd = logIndexOffset;
- } else {
- logEnd = src.size() - ftrLen;
- }
- }
}
private byte[] readHeaderOrFooter(long pos, int len) throws IOException {
@@ -322,81 +222,23 @@
JGitText.get().unsupportedReftableVersion,
Integer.valueOf(version)));
}
- if (blockSize == 0) {
- blockSize = v & 0xffffff;
- }
minUpdateIndex = NB.decodeInt64(tmp, 8);
maxUpdateIndex = NB.decodeInt64(tmp, 16);
+ refRootChunkAddr = NB.decodeInt64(tmp, 24);
+ objRootChunkAddr = NB.decodeInt64(tmp, 32);
return tmp;
}
- private void initRefIndex() throws IOException {
- if (blockSize == 0 || refIndexOffset < 0) {
- readFileFooter();
- }
- if (refIndex == null && refIndexOffset > 0) {
- refIndex = readIndex(refIndexOffset);
- }
- }
-
- private void initObjIndex() throws IOException {
- if (blockSize == 0 || objIndexOffset < 0) {
- readFileFooter();
- }
- if (objIndex == null && objIndexOffset > 0) {
- objIndex = readIndex(objIndexOffset);
- }
- }
-
- private void initLogIndex() throws IOException {
- if (blockSize == 0 || logIndexOffset < 0) {
- readFileFooter();
- }
- if (logIndex == null && logIndexOffset > 0) {
- logIndex = readIndex(logIndexOffset);
- }
- }
-
- private BlockReader readIndex(long pos) throws IOException {
- int sz = readIndexSize(pos);
- BlockReader i = new BlockReader();
- i.readBlock(src, pos, sz);
- i.verifyIndex();
- return i;
- }
-
- private int readIndexSize(long pos) throws IOException {
- ByteBuffer tmp = src.read(pos, 4);
- if (tmp.position() < 4) {
- throw new IOException(JGitText.get().invalidReftableFile);
- }
- byte[] buf;
- if (tmp.hasArray() && tmp.arrayOffset() == 0) {
- buf = tmp.array();
- } else {
- buf = new byte[4];
- tmp.flip();
- tmp.get(buf, 0, 4);
- }
- return decodeIndexSize(NB.decodeInt32(buf, 0));
- }
-
private BlockReader readBlock(long position, long end) throws IOException {
- int sz = blockSize;
+ int sz = 1024; // what a fucking hack
if (position + sz > end) {
- sz = (int) (end - position); // last block may omit padding.
+ sz = (int) (end - position);
}
-
BlockReader b = new BlockReader();
b.readBlock(src, position, sz);
return b;
}
- private int blocksIn(long pos, long end) {
- int blocks = (int) ((end - pos) / blockSize);
- return end % blockSize == 0 ? blocks : (blocks + 1);
- }
-
/**
* Get size of the reftable, in bytes.
*
@@ -465,154 +307,4 @@
// Do nothing.
}
}
-
- private class LogCursorImpl extends LogCursor {
- private final long scanEnd;
- private final byte[] match;
-
- private String refName;
- private long updateIndex;
- private ReflogEntry entry;
- BlockReader block;
-
- LogCursorImpl(long scanEnd, byte[] match) {
- this.scanEnd = scanEnd;
- this.match = match;
- }
-
- @Override
- public boolean next() throws IOException {
- for (;;) {
- if (block == null || block.type() != LOG_BLOCK_TYPE) {
- return false;
- } else if (!block.next()) {
- long p = block.endPosition();
- if (p >= scanEnd) {
- return false;
- }
- block = readBlock(p, scanEnd);
- continue;
- }
-
- block.parseKey();
- if (match != null && !block.match(match, false)) {
- block.skipValue();
- return false;
- }
-
- refName = block.name();
- updateIndex = block.readLogUpdateIndex();
- entry = block.readLog();
- return true;
- }
- }
-
- @Override
- public String getRefName() {
- return refName;
- }
-
- @Override
- public long getUpdateIndex() {
- return updateIndex;
- }
-
- @Override
- public ReflogEntry getReflogEntry() {
- return entry;
- }
-
- @Override
- public void close() {
- // Do nothing.
- }
- }
-
- private class ObjCursorImpl extends RefCursor {
- private final long scanEnd;
- private final ObjectId match;
-
- private Ref ref;
- private int listIdx;
- private IntList blockList;
- private BlockReader block;
-
- ObjCursorImpl(long scanEnd, AnyObjectId id) {
- this.scanEnd = scanEnd;
- this.match = id.copy();
- }
-
- void initSeek() throws IOException {
- byte[] key = new byte[OBJECT_ID_LENGTH];
- match.copyRawTo(key, 0);
-
- long blockOffset = objIndex.seekAbbrevId(key) <= 0
- ? objIndex.readIndex()
- : ((blocksIn(objOffset, objEnd) - 1) * blockSize);
- BlockReader b = readBlock(blockOffset, objEnd);
- b.seekAbbrevId(key);
- while (b.next()) {
- b.parseKey();
- if (b.matchAbbrevId(key)) {
- blockList = b.readBlockList();
- break;
- }
- b.skipValue();
- }
- if (blockList == null) {
- blockList = new IntList(0);
- }
- if (blockList.size() > 0) {
- int blockIdx = blockList.get(listIdx++);
- block = readBlock(blockIdx * blockSize, scanEnd);
- }
- }
-
- void initScan() throws IOException {
- block = readBlock(0, scanEnd);
- }
-
- @Override
- public boolean next() throws IOException {
- for (;;) {
- if (block == null || block.type() != REF_BLOCK_TYPE) {
- return false;
- } else if (!block.next()) {
- long p;
- if (blockList != null) {
- if (listIdx >= blockList.size()) {
- return false;
- }
- int blockIdx = blockList.get(listIdx++);
- p = blockIdx * blockSize;
- } else {
- p = block.endPosition();
- }
- if (p >= scanEnd) {
- return false;
- }
- block = readBlock(p, scanEnd);
- continue;
- }
-
- block.parseKey();
- ref = block.readRef();
- ObjectId id = ref.getObjectId();
- if (id != null && match.equals(id)
- && (includeDeletes || !wasDeleted())) {
- return true;
- }
- }
- }
-
- @Override
- public Ref getRef() {
- return ref;
- }
-
- @Override
- public void close() {
- // Do nothing.
- }
- }
}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableWriter.java
index d270ea9..9dd6dd7 100644
--- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableWriter.java
+++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableWriter.java
@@ -43,12 +43,10 @@
package org.eclipse.jgit.internal.storage.reftable;
-import static java.lang.Math.log;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_FOOTER_LEN;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_MAGIC;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;
-import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.MAX_BLOCK_SIZE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.REF_BLOCK_TYPE;
import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VERSION_1;
@@ -65,11 +63,11 @@
import java.util.zip.CRC32;
import org.eclipse.jgit.annotations.Nullable;
-import org.eclipse.jgit.internal.storage.reftable.BlockWriter.Entry;
-import org.eclipse.jgit.internal.storage.reftable.BlockWriter.LogEntry;
-import org.eclipse.jgit.internal.storage.reftable.BlockWriter.ObjEntry;
-import org.eclipse.jgit.internal.storage.reftable.BlockWriter.RefEntry;
-import org.eclipse.jgit.internal.storage.reftable.BlockWriter.TextEntry;
+import org.eclipse.jgit.internal.storage.reftable.ChunkWriter.Entry;
+import org.eclipse.jgit.internal.storage.reftable.ChunkWriter.IndexEntry;
+import org.eclipse.jgit.internal.storage.reftable.ChunkWriter.ObjEntry;
+import org.eclipse.jgit.internal.storage.reftable.ChunkWriter.RefEntry;
+import org.eclipse.jgit.internal.storage.reftable.ChunkWriter.TextEntry;
import org.eclipse.jgit.lib.AbbreviatedObjectId;
import org.eclipse.jgit.lib.AnyObjectId;
import org.eclipse.jgit.lib.Constants;
@@ -96,33 +94,28 @@
private long minUpdateIndex;
private long maxUpdateIndex;
+ private long refRootChunkAddr;
+ private long objRootChunkAddr;
private ReftableOutputStream out;
private ObjectIdSubclassMap<RefList> obj2ref;
- private BlockWriter refIndex;
- private BlockWriter objIndex;
- private BlockWriter logIndex;
- private BlockWriter cur;
-
- private long objOffset;
- private long logOffset;
- private long refIndexOffset;
- private long objIndexOffset;
- private long logIndexOffset;
+ private List<IndexEntry> refIndex;
+ private List<IndexEntry> objIndex;
+ private ChunkWriter cur;
private long refCnt;
private int objCnt;
private long logCnt;
private long refBytes;
private long objBytes;
- private long logBytes;
private int refBlocks;
private int objBlocks;
- private int logBlocks;
private int refIndexSize;
private int objIndexSize;
private int objIdLen;
+
+ private int indexLevels;
private Stats stats;
/** Initialize a writer with a default configuration. */
@@ -203,7 +196,7 @@
restartInterval = refBlockSize < (60 << 10) ? 16 : 64;
}
- refIndex = newIndex(refBlockSize);
+ refIndex = new ArrayList<>();
out = new ReftableOutputStream(os, refBlockSize);
if (indexObjects) {
obj2ref = new ObjectIdSubclassMap<>();
@@ -300,53 +293,28 @@
public void writeLog(String ref, long updateIndex, PersonIdent who,
ObjectId oldId, ObjectId newId, @Nullable String message)
throws IOException {
- String msg = message != null ? message : ""; //$NON-NLS-1$
- beginLog();
- logCnt++;
- write(logIndex, new LogEntry(ref, updateIndex, who, oldId, newId, msg));
}
- private void beginLog() throws IOException {
- if (logOffset == 0) {
- finishRef(); // close prior ref blocks and their index, if present.
- out.flushFileHeader();
-
- if (logBlockSize == 0) {
- logBlockSize = refBlockSize * 2;
- }
- logIndex = newIndex(logBlockSize);
- out.setBlockSize(logBlockSize);
- logOffset = out.size();
- }
- }
-
- private int write(BlockWriter idx, BlockWriter.Entry entry)
+ private int write(List<IndexEntry> idx, ChunkWriter.Entry entry)
throws IOException {
if (cur == null) {
beginBlock(entry);
} else if (!cur.tryAdd(entry)) {
- idx.addIndex(cur.lastKey(), out.size());
+ idx.add(new IndexEntry(cur.lastKey(), out.size()));
cur.writeTo(out);
- if (cur.padBetweenBlocks()) {
- out.padBetweenBlocksToNextBlock();
- }
beginBlock(entry);
}
return out.blockCount();
}
- private void beginBlock(BlockWriter.Entry entry)
+ private void beginBlock(ChunkWriter.Entry entry)
throws BlockSizeTooSmallException {
byte type = entry.blockType();
int bs = out.bytesAvailableInBlock();
- cur = new BlockWriter(type, bs, restartInterval);
+ cur = new ChunkWriter(type, bs);
cur.addFirst(entry);
}
- private BlockWriter newIndex(int bs) {
- return new BlockWriter(INDEX_BLOCK_TYPE, bs, restartInterval);
- }
-
/**
* @return an estimate of the current size in bytes of the reftable, if it
* was finished right now. The estimate is only accurate if the
@@ -354,30 +322,7 @@
* to {@code false}.
*/
public long estimateTotalBytes() {
- long bytes = out.size();
- if (bytes == 0) {
- bytes += FILE_HEADER_LEN;
- }
- if (cur != null) {
- long offset = out.size();
- int sz = cur.currentSize();
- bytes += sz;
-
- BlockWriter idx = null;
- if (cur.blockType() == REF_BLOCK_TYPE) {
- idx = refIndex;
- } else if (cur.blockType() == LOG_BLOCK_TYPE) {
- idx = logIndex;
- }
- if (idx != null && shouldHaveIndex(idx)) {
- if (idx == refIndex) {
- bytes += out.estimatePadBetweenBlocks(sz);
- }
- bytes += idx.estimateIndexSizeIfAdding(cur.lastKey(), offset);
- }
- }
- bytes += FILE_FOOTER_LEN;
- return bytes;
+ return 0;
}
/**
@@ -389,12 +334,10 @@
*/
public ReftableWriter finish() throws IOException {
finishRef();
- finishLog();
writeFileFooter();
- stats = new Stats(this, out, refIndex);
+ stats = new Stats(this, out);
refIndex = null;
- logIndex = null;
cur = null;
out = null;
obj2ref = null;
@@ -403,14 +346,18 @@
private void finishRef() throws IOException {
if (cur != null && cur.blockType() == REF_BLOCK_TYPE) {
- refBlocks = out.blockCount() + 1;
- refIndexOffset = finishBlockMaybeWriteIndex(refIndex);
- if (refIndexOffset > 0) {
- refIndexSize = (int) (out.size() - refIndexOffset);
+ refIndex.add(new IndexEntry(cur.lastKey(), out.size()));
+ cur.writeTo(out);
+ cur = null;
+
+ refBlocks = out.blockCount();
+ refRootChunkAddr = writeIndex(refIndex);
+ if (refRootChunkAddr > 0) {
+ refIndexSize = (int) (out.size() - refRootChunkAddr);
}
refBytes = out.size();
- if (indexObjects && !obj2ref.isEmpty() && refIndexOffset > 0) {
+ if (indexObjects && !obj2ref.isEmpty()) {
writeObjBlocks();
}
obj2ref = null;
@@ -422,76 +369,67 @@
obj2ref = null;
objIdLen = shortestUniqueAbbreviation(sorted);
- out.padBetweenBlocksToNextBlock();
objCnt = sorted.size();
- objOffset = out.size();
- objIndex = newIndex(refBlockSize);
+ objIndex = new ArrayList<>();
for (RefList l : sorted) {
write(objIndex, new ObjEntry(objIdLen, l, l.blockIds));
}
objBlocks = (out.blockCount() + 1) - refBlocks;
- objIndexOffset = finishBlockMaybeWriteIndex(objIndex);
- if (objIndexOffset > 0) {
- objIndexSize = (int) (out.size() - objIndexOffset);
+ objRootChunkAddr = 0;
+ if (objRootChunkAddr > 0) {
+ objIndexSize = (int) (out.size() - objRootChunkAddr);
}
- objBytes = out.size() - objOffset;
+ objBytes = out.size();
}
- private void finishLog() throws IOException {
- if (cur != null && cur.blockType() == LOG_BLOCK_TYPE) {
- logBlocks = (out.blockCount() + 1) - (refBlocks + objBlocks);
- logIndexOffset = finishBlockMaybeWriteIndex(logIndex);
- logBytes = out.size() - logOffset;
- }
- }
+ private long writeIndex(List<IndexEntry> idx) throws IOException {
+ long addr = 0;
+ while (!idx.isEmpty()) {
+ addr = out.size();
+ List<IndexEntry> l2 = new ArrayList<>();
+ int bs = out.bytesAvailableInBlock();
+ ChunkWriter c = new ChunkWriter(INDEX_BLOCK_TYPE, bs);
+ for (IndexEntry e : idx) {
+ if (!c.tryAdd(e.forIndexAt(addr))) {
+ l2.add(new IndexEntry(c.lastKey(), addr));
+ c.writeTo(out);
- private long finishBlockMaybeWriteIndex(BlockWriter idx)
- throws IOException {
- idx.addIndex(cur.lastKey(), out.size());
- cur.writeTo(out);
- cur = null;
-
- if (shouldHaveIndex(idx)) {
- if (idx == refIndex || idx == objIndex) {
- out.padBetweenBlocksToNextBlock();
+ addr = out.size();
+ c = new ChunkWriter(INDEX_BLOCK_TYPE, bs);
+ c.addFirst(e.forIndexAt(addr));
+ }
}
- long offset = out.size();
- idx.writeTo(out);
- return offset;
- } else {
- return 0;
- }
- }
+ if (!l2.isEmpty()) {
+ l2.add(new IndexEntry(c.lastKey(), addr));
+ }
+ c.writeTo(out);
- private boolean shouldHaveIndex(BlockWriter idx) {
- int threshold = idx == refIndex ? 4 : 1;
- return idx.entryCount() + (cur != null ? 1 : 0) > threshold;
+ indexLevels++;
+ idx = l2;
+ }
+ return addr;
}
private void writeFileHeader() throws IOException {
byte[] hdr = new byte[FILE_HEADER_LEN];
encodeHeader(hdr);
- out.write(hdr);
+ out.write(hdr, 0, FILE_HEADER_LEN);
+ out.flushFileHeader();
}
private void encodeHeader(byte[] hdr) {
System.arraycopy(FILE_HEADER_MAGIC, 0, hdr, 0, 4);
- NB.encodeInt32(hdr, 4, (VERSION_1 << 24) | refBlockSize);
+ NB.encodeInt32(hdr, 4, (VERSION_1 << 24));
NB.encodeInt64(hdr, 8, minUpdateIndex);
NB.encodeInt64(hdr, 16, maxUpdateIndex);
+ NB.encodeInt64(hdr, 24, refRootChunkAddr);
+ NB.encodeInt64(hdr, 32, objRootChunkAddr);
}
private void writeFileFooter() throws IOException {
int ftrLen = FILE_FOOTER_LEN;
byte[] ftr = new byte[ftrLen];
encodeHeader(ftr);
-
- NB.encodeInt64(ftr, 24, refIndexOffset);
- NB.encodeInt64(ftr, 32, objOffset);
- NB.encodeInt64(ftr, 40, objIndexOffset);
- NB.encodeInt64(ftr, 48, logOffset);
- NB.encodeInt64(ftr, 56, logIndexOffset);
-
CRC32 crc = new CRC32();
crc.update(ftr, 0, ftrLen - 4);
NB.encodeInt32(ftr, ftrLen - 4, (int) crc.getValue());
@@ -520,18 +458,19 @@
private final long logCnt;
private final long refBytes;
private final long objBytes;
- private final long logBytes;
+ private long logBytes;
private final long paddingUsed;
private final long totalBytes;
private final int refBlocks;
private final int objBlocks;
- private final int logBlocks;
+ private int logBlocks;
- private final int refIndexKeys;
+ private int refIndexKeys;
+ private final int indexLevels;
private final int refIndexSize;
private final int objIndexSize;
- Stats(ReftableWriter w, ReftableOutputStream o, BlockWriter refIdx) {
+ Stats(ReftableWriter w, ReftableOutputStream o) {
refBlockSize = w.refBlockSize;
logBlockSize = w.logBlockSize;
restartInterval = w.restartInterval;
@@ -545,14 +484,12 @@
logCnt = w.logCnt;
refBytes = w.refBytes;
objBytes = w.objBytes;
- logBytes = w.logBytes;
paddingUsed = o.paddingUsed();
totalBytes = o.size();
refBlocks = w.refBlocks;
objBlocks = w.objBlocks;
- logBlocks = w.logBlocks;
+ indexLevels = w.indexLevels;
- refIndexKeys = w.refIndexOffset > 0 ? refIdx.entryCount() : 0;
refIndexSize = w.refIndexSize;
objIndexSize = w.objIndexSize;
}
@@ -663,10 +600,7 @@
/** @return estimated number of disk seeks per ref read. */
public double diskSeeksPerRead() {
- if (refIndexKeys() > 0) {
- return 1;
- }
- return log(refBlockCount()) / log(2);
+ return indexLevels + 1;
}
}
