org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/reftable/ReftableReader.java - sop/jgit - Git at Google

 /*
  * Copyright (C) 2017, Google Inc.
  * and other copyright owners as documented in the project's IP log.
  *
  * This program and the accompanying materials are made available
  * under the terms of the Eclipse Distribution License v1.0 which
  * accompanies this distribution, is reproduced below, and is
  * available at http://www.eclipse.org/org/documents/edl-v10.php
  *
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or
  * without modification, are permitted provided that the following
  * conditions are met:
  *
  * - Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  *
  * - Redistributions in binary form must reproduce the above
  *   copyright notice, this list of conditions and the following
  *   disclaimer in the documentation and/or other materials provided
  *   with the distribution.
  *
  * - Neither the name of the Eclipse Foundation, Inc. nor the
  *   names of its contributors may be used to endorse or promote
  *   products derived from this software without specific prior
  *   written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 package org.eclipse.jgit.internal.storage.reftable;

 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_FOOTER_LEN;
 import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
 import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.REF_BLOCK_TYPE;
 import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VERSION_1;
 import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.isFileHeaderMagic;
 import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;

 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.text.MessageFormat;
 import java.util.Arrays;
 import java.util.zip.CRC32;

 import org.eclipse.jgit.internal.JGitText;
 import org.eclipse.jgit.internal.storage.io.BlockSource;
 import org.eclipse.jgit.lib.AnyObjectId;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.util.LongMap;
 import org.eclipse.jgit.util.NB;

 /**
  * Reads a reftable formatted file.
  * <p>
  * {@code ReftableReader} is not thread-safe. Concurrent readers need their own
  * instance to read from the same file.
  */
 public class ReftableReader extends Reftable {
 	private final BlockSource src;

 	private long minUpdateIndex;
 	private long maxUpdateIndex;
 	private long refRootChunkAddr;
 	long objRootChunkAddr;

 	private BlockReader refRoot;
 	private LongMap<BlockReader> idxCache;

 	/**
 	 * Initialize a new reftable reader.
 	 *
 	 * @param src
 	 *            the file content to read.
 	 */
 	public ReftableReader(BlockSource src) {
 		this.src = src;
 	}

 	/**
 	 * @return the block size in bytes chosen for this file by the writer. Most
 	 *         reads from the {@link BlockSource} will be aligned to the block
 	 *         size.
 	 * @throws IOException
 	 *             file cannot be read.
 	 */
 	public int blockSize() throws IOException {
 		return 0;
 	}

 	/**
 	 * @return the minimum update index for log entries that appear in this
 	 *         reftable. This should be 1 higher than the prior reftable's
 	 *         {@code maxUpdateIndex} if this table is used in a stack.
 	 * @throws IOException
 	 *             file cannot be read.
 	 */
 	public long minUpdateIndex() throws IOException {
 		if (minUpdateIndex == 0) {
 			readFileHeader();
 		}
 		return minUpdateIndex;
 	}

 	/**
 	 * @return the maximum update index for log entries that appear in this
 	 *         reftable. This should be 1 higher than the prior reftable's
 	 *         {@code maxUpdateIndex} if this table is used in a stack.
 	 * @throws IOException
 	 *             file cannot be read.
 	 */
 	public long maxUpdateIndex() throws IOException {
 		if (minUpdateIndex == 0) {
 			readFileHeader();
 		}
 		return maxUpdateIndex;
 	}

 	@Override
 	public RefCursor allRefs() throws IOException {
 		long end = src.size() - FILE_FOOTER_LEN;
 		RefCursorImpl i = new RefCursorImpl(end, null, false);
 		i.block = readBlock(FILE_HEADER_LEN, end);
 		return i;
 	}

 	@Override
 	public RefCursor seek(String refName) throws IOException {
 		if (refRootChunkAddr == 0) {
 			readFileFooter();
 		}

 		byte[] match = refName.getBytes(UTF_8);
 		boolean prefix = match[match.length - 1] == '/';
 		byte[] key = match;
 		if (prefix) {
 			key = Arrays.copyOf(key, key.length + 1);
 			key[key.length - 1] = '\1';
 		}

 		long end = src.size() - FILE_FOOTER_LEN;
 		RefCursorImpl i = new RefCursorImpl(end, match, prefix);
 		i.block = seek(key, refRootChunkAddr, end);
 		return i;
 	}

 	@Override
 	public RefCursor byObjectId(AnyObjectId id) throws IOException {
 		throw new UnsupportedOperationException();
 	}

 	@Override
 	public LogCursor allLogs() throws IOException {
 		return new EmptyLogCursor();
 	}

 	@Override
 	public LogCursor seekLog(String refName, long updateIndex)
 			throws IOException {
 		return new EmptyLogCursor();
 	}

 	private BlockReader seek(byte[] key, long addr, long end)
 			throws IOException {
 		BlockReader block = refRoot;
 		if (block == null) {
 			block = readBlock(addr, end);
 			refRoot = block;
 		}
 		do {
 			block.seekToKey(key);
 			addr = block.readIndex();
 			block = readBlock(addr, end);
 		} while (block.type() == ReftableConstants.INDEX_BLOCK_TYPE);
 		block.seekToKey(key);
 		return block;
 	}

 	private void readFileHeader() throws IOException {
 		readFileFooter();
 	}

 	private void readFileFooter() throws IOException {
 		int ftrLen = FILE_FOOTER_LEN;
 		byte[] ftr = readHeaderOrFooter(src.size() - ftrLen, ftrLen);

 		CRC32 crc = new CRC32();
 		crc.update(ftr, 0, ftrLen - 4);
 		if (crc.getValue() != NB.decodeUInt32(ftr, ftrLen - 4)) {
 			throw new IOException(JGitText.get().invalidReftableCRC);
 		}
 	}

 	private byte[] readHeaderOrFooter(long pos, int len) throws IOException {
 		ByteBuffer buf = src.read(pos, len);
 		if (buf.position() != len) {
 			throw new IOException(JGitText.get().shortReadOfBlock);
 		}

 		byte[] tmp = new byte[len];
 		buf.flip();
 		buf.get(tmp);
 		if (!isFileHeaderMagic(tmp, 0, len)) {
 			throw new IOException(JGitText.get().invalidReftableFile);
 		}

 		int v = NB.decodeInt32(tmp, 4);
 		int version = v >>> 24;
 		if (VERSION_1 != version) {
 			throw new IOException(MessageFormat.format(
 					JGitText.get().unsupportedReftableVersion,
 					Integer.valueOf(version)));
 		}
 		minUpdateIndex = NB.decodeInt64(tmp, 8);
 		maxUpdateIndex = NB.decodeInt64(tmp, 16);
 		refRootChunkAddr = NB.decodeInt64(tmp, 24);
 		objRootChunkAddr = NB.decodeInt64(tmp, 32);
 		return tmp;
 	}

 	private BlockReader readBlock(long position, long end) throws IOException {
 		if (idxCache != null) {
 			BlockReader b = idxCache.get(position);
 			if (b != null) {
 				return b;
 			}
 		}

 		int sz = 4096; // what a fucking hack
 		if (position + sz > end) {
 			sz = (int) (end - position);
 		}
 		BlockReader b = new BlockReader();
 		b.readBlock(src, position, sz);
 		if (b.type() == INDEX_BLOCK_TYPE) {
 			if (idxCache == null) {
 				idxCache = new LongMap<>();
 			}
 			idxCache.put(position, b);
 		}
 		return b;
 	}

 	/**
 	 * Get size of the reftable, in bytes.
 	 *
 	 * @return size of the reftable, in bytes.
 	 * @throws IOException
 	 *             size cannot be obtained.
 	 */
 	public long size() throws IOException {
 		return src.size();
 	}

 	@Override
 	public void close() throws IOException {
 		src.close();
 	}

 	private class RefCursorImpl extends RefCursor {
 		private final long scanEnd;
 		private final byte[] match;
 		private final boolean prefix;

 		private Ref ref;
 		BlockReader block;

 		RefCursorImpl(long scanEnd, byte[] match, boolean prefix) {
 			this.scanEnd = scanEnd;
 			this.match = match;
 			this.prefix = prefix;
 		}

 		@Override
 		public boolean next() throws IOException {
 			for (;;) {
 				if (block == null || block.type() != REF_BLOCK_TYPE) {
 					return false;
 				} else if (!block.next()) {
 					long p = block.endPosition();
 					if (p >= scanEnd) {
 						return false;
 					}
 					block = readBlock(p, scanEnd);
 					continue;
 				}

 				block.parseKey();
 				if (match != null && !block.match(match, prefix)) {
 					block.skipValue();
 					return false;
 				}

 				ref = block.readRef();
 				if (!includeDeletes && wasDeleted()) {
 					continue;
 				}
 				return true;
 			}
 		}

 		@Override
 		public Ref getRef() {
 			return ref;
 		}

 		@Override
 		public void close() {
 			// Do nothing.
 		}
 	}
 }
	/*
	* Copyright (C) 2017, Google Inc.
	* and other copyright owners as documented in the project's IP log.
	*
	* This program and the accompanying materials are made available
	* under the terms of the Eclipse Distribution License v1.0 which
	* accompanies this distribution, is reproduced below, and is
	* available at http://www.eclipse.org/org/documents/edl-v10.php
	*
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or
	* without modification, are permitted provided that the following
	* conditions are met:
	*
	* - Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* - Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following
	* disclaimer in the documentation and/or other materials provided
	* with the distribution.
	*
	* - Neither the name of the Eclipse Foundation, Inc. nor the
	* names of its contributors may be used to endorse or promote
	* products derived from this software without specific prior
	* written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
	* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
	* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
	* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	package org.eclipse.jgit.internal.storage.reftable;

	import static java.nio.charset.StandardCharsets.UTF_8;
	import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_FOOTER_LEN;
	import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
	import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.REF_BLOCK_TYPE;
	import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.VERSION_1;
	import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.isFileHeaderMagic;
	import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;

	import java.io.IOException;
	import java.nio.ByteBuffer;
	import java.text.MessageFormat;
	import java.util.Arrays;
	import java.util.zip.CRC32;

	import org.eclipse.jgit.internal.JGitText;
	import org.eclipse.jgit.internal.storage.io.BlockSource;
	import org.eclipse.jgit.lib.AnyObjectId;
	import org.eclipse.jgit.lib.Ref;
	import org.eclipse.jgit.util.LongMap;
	import org.eclipse.jgit.util.NB;

	/**
	* Reads a reftable formatted file.
	* <p>
	* {@code ReftableReader} is not thread-safe. Concurrent readers need their own
	* instance to read from the same file.
	*/
	public class ReftableReader extends Reftable {
	private final BlockSource src;

	private long minUpdateIndex;
	private long maxUpdateIndex;
	private long refRootChunkAddr;
	long objRootChunkAddr;

	private BlockReader refRoot;
	private LongMap<BlockReader> idxCache;

	/**
	* Initialize a new reftable reader.
	*
	* @param src
	* the file content to read.
	*/
	public ReftableReader(BlockSource src) {
	this.src = src;
	}

	/**
	* @return the block size in bytes chosen for this file by the writer. Most
	* reads from the {@link BlockSource} will be aligned to the block
	* size.
	* @throws IOException
	* file cannot be read.
	*/
	public int blockSize() throws IOException {
	return 0;
	}

	/**
	* @return the minimum update index for log entries that appear in this
	* reftable. This should be 1 higher than the prior reftable's
	* {@code maxUpdateIndex} if this table is used in a stack.
	* @throws IOException
	* file cannot be read.
	*/
	public long minUpdateIndex() throws IOException {
	if (minUpdateIndex == 0) {
	readFileHeader();
	}
	return minUpdateIndex;
	}

	/**
	* @return the maximum update index for log entries that appear in this
	* reftable. This should be 1 higher than the prior reftable's
	* {@code maxUpdateIndex} if this table is used in a stack.
	* @throws IOException
	* file cannot be read.
	*/
	public long maxUpdateIndex() throws IOException {
	if (minUpdateIndex == 0) {
	readFileHeader();
	}
	return maxUpdateIndex;
	}

	@Override
	public RefCursor allRefs() throws IOException {
	long end = src.size() - FILE_FOOTER_LEN;
	RefCursorImpl i = new RefCursorImpl(end, null, false);
	i.block = readBlock(FILE_HEADER_LEN, end);
	return i;
	}

	@Override
	public RefCursor seek(String refName) throws IOException {
	if (refRootChunkAddr == 0) {
	readFileFooter();
	}

	byte[] match = refName.getBytes(UTF_8);
	boolean prefix = match[match.length - 1] == '/';
	byte[] key = match;
	if (prefix) {
	key = Arrays.copyOf(key, key.length + 1);
	key[key.length - 1] = '\1';
	}

	long end = src.size() - FILE_FOOTER_LEN;
	RefCursorImpl i = new RefCursorImpl(end, match, prefix);
	i.block = seek(key, refRootChunkAddr, end);
	return i;
	}

	@Override
	public RefCursor byObjectId(AnyObjectId id) throws IOException {
	throw new UnsupportedOperationException();
	}

	@Override
	public LogCursor allLogs() throws IOException {
	return new EmptyLogCursor();
	}

	@Override
	public LogCursor seekLog(String refName, long updateIndex)
	throws IOException {
	return new EmptyLogCursor();
	}

	private BlockReader seek(byte[] key, long addr, long end)
	throws IOException {
	BlockReader block = refRoot;
	if (block == null) {
	block = readBlock(addr, end);
	refRoot = block;
	}
	do {
	block.seekToKey(key);
	addr = block.readIndex();
	block = readBlock(addr, end);
	} while (block.type() == ReftableConstants.INDEX_BLOCK_TYPE);
	block.seekToKey(key);
	return block;
	}

	private void readFileHeader() throws IOException {
	readFileFooter();
	}

	private void readFileFooter() throws IOException {
	int ftrLen = FILE_FOOTER_LEN;
	byte[] ftr = readHeaderOrFooter(src.size() - ftrLen, ftrLen);

	CRC32 crc = new CRC32();
	crc.update(ftr, 0, ftrLen - 4);
	if (crc.getValue() != NB.decodeUInt32(ftr, ftrLen - 4)) {
	throw new IOException(JGitText.get().invalidReftableCRC);
	}
	}

	private byte[] readHeaderOrFooter(long pos, int len) throws IOException {
	ByteBuffer buf = src.read(pos, len);
	if (buf.position() != len) {
	throw new IOException(JGitText.get().shortReadOfBlock);
	}

	byte[] tmp = new byte[len];
	buf.flip();
	buf.get(tmp);
	if (!isFileHeaderMagic(tmp, 0, len)) {
	throw new IOException(JGitText.get().invalidReftableFile);
	}

	int v = NB.decodeInt32(tmp, 4);
	int version = v >>> 24;
	if (VERSION_1 != version) {
	throw new IOException(MessageFormat.format(
	JGitText.get().unsupportedReftableVersion,
	Integer.valueOf(version)));
	}
	minUpdateIndex = NB.decodeInt64(tmp, 8);
	maxUpdateIndex = NB.decodeInt64(tmp, 16);
	refRootChunkAddr = NB.decodeInt64(tmp, 24);
	objRootChunkAddr = NB.decodeInt64(tmp, 32);
	return tmp;
	}

	private BlockReader readBlock(long position, long end) throws IOException {
	if (idxCache != null) {
	BlockReader b = idxCache.get(position);
	if (b != null) {
	return b;
	}
	}

	int sz = 4096; // what a fucking hack
	if (position + sz > end) {
	sz = (int) (end - position);
	}
	BlockReader b = new BlockReader();
	b.readBlock(src, position, sz);
	if (b.type() == INDEX_BLOCK_TYPE) {
	if (idxCache == null) {
	idxCache = new LongMap<>();
	}
	idxCache.put(position, b);
	}
	return b;
	}

	/**
	* Get size of the reftable, in bytes.
	*
	* @return size of the reftable, in bytes.
	* @throws IOException
	* size cannot be obtained.
	*/
	public long size() throws IOException {
	return src.size();
	}

	@Override
	public void close() throws IOException {
	src.close();
	}

	private class RefCursorImpl extends RefCursor {
	private final long scanEnd;
	private final byte[] match;
	private final boolean prefix;

	private Ref ref;
	BlockReader block;

	RefCursorImpl(long scanEnd, byte[] match, boolean prefix) {
	this.scanEnd = scanEnd;
	this.match = match;
	this.prefix = prefix;
	}

	@Override
	public boolean next() throws IOException {
	for (;;) {
	if (block == null \|\| block.type() != REF_BLOCK_TYPE) {
	return false;
	} else if (!block.next()) {
	long p = block.endPosition();
	if (p >= scanEnd) {
	return false;
	}
	block = readBlock(p, scanEnd);
	continue;
	}

	block.parseKey();
	if (match != null && !block.match(match, prefix)) {
	block.skipValue();
	return false;
	}

	ref = block.readRef();
	if (!includeDeletes && wasDeleted()) {
	continue;
	}
	return true;
	}
	}

	@Override
	public Ref getRef() {
	return ref;
	}

	@Override
	public void close() {
	// Do nothing.
	}
	}
	}