org.eclipse.jgit.pgm/src/org/eclipse/jgit/pgm/debug/WriteReftable.java - sop/jgit - Git at Google

 /*
  * Copyright (C) 2017, Google Inc.
  * and other copyright owners as documented in the project's IP log.
  *
  * This program and the accompanying materials are made available
  * under the terms of the Eclipse Distribution License v1.0 which
  * accompanies this distribution, is reproduced below, and is
  * available at http://www.eclipse.org/org/documents/edl-v10.php
  *
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or
  * without modification, are permitted provided that the following
  * conditions are met:
  *
  * - Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  *
  * - Redistributions in binary form must reproduce the above
  *   copyright notice, this list of conditions and the following
  *   disclaimer in the documentation and/or other materials provided
  *   with the distribution.
  *
  * - Neither the name of the Eclipse Foundation, Inc. nor the
  *   names of its contributors may be used to endorse or promote
  *   products derived from this software without specific prior
  *   written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
  * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
  * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */

 package org.eclipse.jgit.pgm.debug;

 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.eclipse.jgit.lib.Constants.HEAD;
 import static org.eclipse.jgit.lib.Constants.MASTER;
 import static org.eclipse.jgit.lib.Constants.R_HEADS;
 import static org.eclipse.jgit.lib.Ref.Storage.NEW;
 import static org.eclipse.jgit.lib.Ref.Storage.PACKED;

 import java.io.BufferedReader;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
 import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
 import org.eclipse.jgit.lib.ObjectId;
 import org.eclipse.jgit.lib.ObjectIdRef;
 import org.eclipse.jgit.lib.PersonIdent;
 import org.eclipse.jgit.lib.Ref;
 import org.eclipse.jgit.lib.SymbolicRef;
 import org.eclipse.jgit.pgm.Command;
 import org.eclipse.jgit.pgm.TextBuiltin;
 import org.kohsuke.args4j.Argument;
 import org.kohsuke.args4j.Option;

 @Command
 class WriteReftable extends TextBuiltin {
 	private static final int KIB = 1 << 10;
 	private static final int MIB = 1 << 20;

 	@Option(name = "--block-size")
 	private int refBlockSize;

 	@Option(name = "--log-block-size")
 	private int logBlockSize;

 	@Option(name = "--restart-interval")
 	private int restartInterval;

 	@Option(name = "--index-levels")
 	private int indexLevels;

 	@Option(name = "--reflog-in")
 	private String reflogIn;

 	@Option(name = "--no-index-objects")
 	private boolean noIndexObjects;

 	@Argument(index = 0)
 	private String in;

 	@Argument(index = 1)
 	private String out;

 	@SuppressWarnings({ "nls", "boxing" })
 	@Override
 	protected void run() throws Exception {
 		List<Ref> refs = readRefs(in);
 		List<LogEntry> logs = readLog(reflogIn);

 		ReftableWriter.Stats stats;
 		try (OutputStream os = new FileOutputStream(out)) {
 			ReftableConfig cfg = new ReftableConfig();
 			cfg.setIndexObjects(!noIndexObjects);
 			if (refBlockSize > 0) {
 				cfg.setRefBlockSize(refBlockSize);
 			}
 			if (logBlockSize > 0) {
 				cfg.setLogBlockSize(logBlockSize);
 			}
 			if (restartInterval > 0) {
 				cfg.setRestartInterval(restartInterval);
 			}
 			if (indexLevels > 0) {
 				cfg.setMaxIndexLevels(indexLevels);
 			}

 			ReftableWriter w = new ReftableWriter(cfg);
 			w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
 			w.begin(os);
 			w.sortAndWriteRefs(refs);
 			for (LogEntry e : logs) {
 				w.writeLog(e.ref, e.updateIndex, e.who,
 						e.oldId, e.newId, e.message);
 			}
 			stats = w.finish().getStats();
 		}

 		double fileMiB = ((double) stats.totalBytes()) / MIB;
 		printf("Summary:");
 		printf("  file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
 		printf("  padding : %d KiB", stats.paddingBytes() / KIB);
 		errw.println();

 		printf("Refs:");
 		printf("  ref blk : %d", stats.refBlockSize());
 		printf("  restarts: %d", stats.restartInterval());
 		printf("  refs    : %d", stats.refCount());
 		if (stats.refIndexLevels() > 0) {
 			int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
 			printf("  idx sz  : %d KiB", idxSize);
 			printf("  idx lvl : %d", stats.refIndexLevels());
 		}
 		printf("  avg ref : %d bytes", stats.refBytes() / refs.size());
 		errw.println();

 		if (stats.objCount() > 0) {
 			int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
 			int idLen = stats.objIdLength();
 			printf("Objects:");
 			printf("  obj blk : %d", stats.refBlockSize());
 			printf("  restarts: %d", stats.restartInterval());
 			printf("  objects : %d", stats.objCount());
 			printf("  obj sz  : %d MiB (%d bytes)", objMiB, stats.objBytes());
 			if (stats.objIndexSize() > 0) {
 				int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
 				printf("  idx sz  : %d KiB", s);
 				printf("  idx lvl : %d", stats.objIndexLevels());
 			}
 			printf("  id len  : %d bytes (%d hex digits)", idLen, 2 * idLen);
 			printf("  avg obj : %d bytes", stats.objBytes() / stats.objCount());
 			errw.println();
 		}
 		if (stats.logCount() > 0) {
 			int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
 			printf("Log:");
 			printf("  log blk : %d", stats.logBlockSize());
 			printf("  logs    : %d", stats.logCount());
 			printf("  log sz  : %d MiB (%d bytes)", logMiB, stats.logBytes());
 			printf("  avg log : %d bytes", stats.logBytes() / logs.size());
 			errw.println();
 		}
 	}

 	private void printf(String fmt, Object... args) throws IOException {
 		errw.println(String.format(fmt, args));
 	}

 	static List<Ref> readRefs(String inputFile) throws IOException {
 		List<Ref> refs = new ArrayList<>();
 		try (BufferedReader br = new BufferedReader(
 				new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
 			String line;
 			while ((line = br.readLine()) != null) {
 				ObjectId id = ObjectId.fromString(line.substring(0, 40));
 				String name = line.substring(41, line.length());
 				if (name.endsWith("^{}")) { //$NON-NLS-1$
 					int lastIdx = refs.size() - 1;
 					Ref last = refs.get(lastIdx);
 					refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
 							last.getName(), last.getObjectId(), id));
 					continue;
 				}

 				Ref ref;
 				if (name.equals(HEAD)) {
 					ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
 							R_HEADS + MASTER, null));
 				} else {
 					ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
 				}
 				refs.add(ref);
 			}
 		}
 		Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
 		return refs;
 	}

 	private static List<LogEntry> readLog(String logPath)
 			throws FileNotFoundException, IOException {
 		if (logPath == null) {
 			return Collections.emptyList();
 		}

 		List<LogEntry> log = new ArrayList<>();
 		try (BufferedReader br = new BufferedReader(
 				new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
 			@SuppressWarnings("nls")
 			Pattern pattern = Pattern.compile("([^,]+)" // 1: ref
 					+ ",([0-9]+(?:[.][0-9]+)?)" // 2: time
 					+ ",([^,]+)" // 3: who
 					+ ",([^,]+)" // 4: old
 					+ ",([^,]+)" // 5: new
 					+ ",(.*)"); // 6: msg
 			String line;
 			while ((line = br.readLine()) != null) {
 				Matcher m = pattern.matcher(line);
 				if (!m.matches()) {
 					throw new IOException("unparsed line: " + line); //$NON-NLS-1$
 				}
 				String ref = m.group(1);
 				double t = Double.parseDouble(m.group(2));
 				long time = ((long) t) * 1000L;
 				long index = (long) (t * 1e6);
 				String user = m.group(3);
 				ObjectId oldId = parseId(m.group(4));
 				ObjectId newId = parseId(m.group(5));
 				String msg = m.group(6);
 				String email = user + "@gerrit"; //$NON-NLS-1$
 				PersonIdent who = new PersonIdent(user, email, time, -480);
 				log.add(new LogEntry(ref, index, who, oldId, newId, msg));
 			}
 		}
 		Collections.sort(log, LogEntry::compare);
 		return log;
 	}

 	private static long min(List<LogEntry> log) {
 		return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
 	}

 	private static long max(List<LogEntry> log) {
 		return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
 	}

 	private static ObjectId parseId(String s) {
 		if ("NULL".equals(s)) { //$NON-NLS-1$
 			return ObjectId.zeroId();
 		}
 		return ObjectId.fromString(s);
 	}

 	private static class LogEntry {
 		static int compare(LogEntry a, LogEntry b) {
 			int cmp = a.ref.compareTo(b.ref);
 			if (cmp == 0) {
 				cmp = Long.signum(b.updateIndex - a.updateIndex);
 			}
 			return cmp;
 		}

 		final String ref;
 		final long updateIndex;
 		final PersonIdent who;
 		final ObjectId oldId;
 		final ObjectId newId;
 		final String message;

 		LogEntry(String ref, long updateIndex, PersonIdent who,
 				ObjectId oldId, ObjectId newId, String message) {
 			this.ref = ref;
 			this.updateIndex = updateIndex;
 			this.who = who;
 			this.oldId = oldId;
 			this.newId = newId;
 			this.message = message;
 		}
 	}
 }
	/*
	* Copyright (C) 2017, Google Inc.
	* and other copyright owners as documented in the project's IP log.
	*
	* This program and the accompanying materials are made available
	* under the terms of the Eclipse Distribution License v1.0 which
	* accompanies this distribution, is reproduced below, and is
	* available at http://www.eclipse.org/org/documents/edl-v10.php
	*
	* All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or
	* without modification, are permitted provided that the following
	* conditions are met:
	*
	* - Redistributions of source code must retain the above copyright
	* notice, this list of conditions and the following disclaimer.
	*
	* - Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following
	* disclaimer in the documentation and/or other materials provided
	* with the distribution.
	*
	* - Neither the name of the Eclipse Foundation, Inc. nor the
	* names of its contributors may be used to endorse or promote
	* products derived from this software without specific prior
	* written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
	* CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
	* INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
	* CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
	* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
	* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
	* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	*/

	package org.eclipse.jgit.pgm.debug;

	import static java.nio.charset.StandardCharsets.UTF_8;
	import static org.eclipse.jgit.lib.Constants.HEAD;
	import static org.eclipse.jgit.lib.Constants.MASTER;
	import static org.eclipse.jgit.lib.Constants.R_HEADS;
	import static org.eclipse.jgit.lib.Ref.Storage.NEW;
	import static org.eclipse.jgit.lib.Ref.Storage.PACKED;

	import java.io.BufferedReader;
	import java.io.FileInputStream;
	import java.io.FileNotFoundException;
	import java.io.FileOutputStream;
	import java.io.IOException;
	import java.io.InputStreamReader;
	import java.io.OutputStream;
	import java.util.ArrayList;
	import java.util.Collections;
	import java.util.List;
	import java.util.regex.Matcher;
	import java.util.regex.Pattern;

	import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
	import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
	import org.eclipse.jgit.lib.ObjectId;
	import org.eclipse.jgit.lib.ObjectIdRef;
	import org.eclipse.jgit.lib.PersonIdent;
	import org.eclipse.jgit.lib.Ref;
	import org.eclipse.jgit.lib.SymbolicRef;
	import org.eclipse.jgit.pgm.Command;
	import org.eclipse.jgit.pgm.TextBuiltin;
	import org.kohsuke.args4j.Argument;
	import org.kohsuke.args4j.Option;

	@Command
	class WriteReftable extends TextBuiltin {
	private static final int KIB = 1 << 10;
	private static final int MIB = 1 << 20;

	@Option(name = "--block-size")
	private int refBlockSize;

	@Option(name = "--log-block-size")
	private int logBlockSize;

	@Option(name = "--restart-interval")
	private int restartInterval;

	@Option(name = "--index-levels")
	private int indexLevels;

	@Option(name = "--reflog-in")
	private String reflogIn;

	@Option(name = "--no-index-objects")
	private boolean noIndexObjects;

	@Argument(index = 0)
	private String in;

	@Argument(index = 1)
	private String out;

	@SuppressWarnings({ "nls", "boxing" })
	@Override
	protected void run() throws Exception {
	List<Ref> refs = readRefs(in);
	List<LogEntry> logs = readLog(reflogIn);

	ReftableWriter.Stats stats;
	try (OutputStream os = new FileOutputStream(out)) {
	ReftableConfig cfg = new ReftableConfig();
	cfg.setIndexObjects(!noIndexObjects);
	if (refBlockSize > 0) {
	cfg.setRefBlockSize(refBlockSize);
	}
	if (logBlockSize > 0) {
	cfg.setLogBlockSize(logBlockSize);
	}
	if (restartInterval > 0) {
	cfg.setRestartInterval(restartInterval);
	}
	if (indexLevels > 0) {
	cfg.setMaxIndexLevels(indexLevels);
	}

	ReftableWriter w = new ReftableWriter(cfg);
	w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
	w.begin(os);
	w.sortAndWriteRefs(refs);
	for (LogEntry e : logs) {
	w.writeLog(e.ref, e.updateIndex, e.who,
	e.oldId, e.newId, e.message);
	}
	stats = w.finish().getStats();
	}

	double fileMiB = ((double) stats.totalBytes()) / MIB;
	printf("Summary:");
	printf(" file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
	printf(" padding : %d KiB", stats.paddingBytes() / KIB);
	errw.println();

	printf("Refs:");
	printf(" ref blk : %d", stats.refBlockSize());
	printf(" restarts: %d", stats.restartInterval());
	printf(" refs : %d", stats.refCount());
	if (stats.refIndexLevels() > 0) {
	int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
	printf(" idx sz : %d KiB", idxSize);
	printf(" idx lvl : %d", stats.refIndexLevels());
	}
	printf(" avg ref : %d bytes", stats.refBytes() / refs.size());
	errw.println();

	if (stats.objCount() > 0) {
	int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
	int idLen = stats.objIdLength();
	printf("Objects:");
	printf(" obj blk : %d", stats.refBlockSize());
	printf(" restarts: %d", stats.restartInterval());
	printf(" objects : %d", stats.objCount());
	printf(" obj sz : %d MiB (%d bytes)", objMiB, stats.objBytes());
	if (stats.objIndexSize() > 0) {
	int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
	printf(" idx sz : %d KiB", s);
	printf(" idx lvl : %d", stats.objIndexLevels());
	}
	printf(" id len : %d bytes (%d hex digits)", idLen, 2 * idLen);
	printf(" avg obj : %d bytes", stats.objBytes() / stats.objCount());
	errw.println();
	}
	if (stats.logCount() > 0) {
	int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
	printf("Log:");
	printf(" log blk : %d", stats.logBlockSize());
	printf(" logs : %d", stats.logCount());
	printf(" log sz : %d MiB (%d bytes)", logMiB, stats.logBytes());
	printf(" avg log : %d bytes", stats.logBytes() / logs.size());
	errw.println();
	}
	}

	private void printf(String fmt, Object... args) throws IOException {
	errw.println(String.format(fmt, args));
	}

	static List<Ref> readRefs(String inputFile) throws IOException {
	List<Ref> refs = new ArrayList<>();
	try (BufferedReader br = new BufferedReader(
	new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
	String line;
	while ((line = br.readLine()) != null) {
	ObjectId id = ObjectId.fromString(line.substring(0, 40));
	String name = line.substring(41, line.length());
	if (name.endsWith("^{}")) { //$NON-NLS-1$
	int lastIdx = refs.size() - 1;
	Ref last = refs.get(lastIdx);
	refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
	last.getName(), last.getObjectId(), id));
	continue;
	}

	Ref ref;
	if (name.equals(HEAD)) {
	ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
	R_HEADS + MASTER, null));
	} else {
	ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
	}
	refs.add(ref);
	}
	}
	Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
	return refs;
	}

	private static List<LogEntry> readLog(String logPath)
	throws FileNotFoundException, IOException {
	if (logPath == null) {
	return Collections.emptyList();
	}

	List<LogEntry> log = new ArrayList<>();
	try (BufferedReader br = new BufferedReader(
	new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
	@SuppressWarnings("nls")
	Pattern pattern = Pattern.compile("([^,]+)" // 1: ref
	+ ",([0-9]+(?:[.][0-9]+)?)" // 2: time
	+ ",([^,]+)" // 3: who
	+ ",([^,]+)" // 4: old
	+ ",([^,]+)" // 5: new
	+ ",(.*)"); // 6: msg
	String line;
	while ((line = br.readLine()) != null) {
	Matcher m = pattern.matcher(line);
	if (!m.matches()) {
	throw new IOException("unparsed line: " + line); //$NON-NLS-1$
	}
	String ref = m.group(1);
	double t = Double.parseDouble(m.group(2));
	long time = ((long) t) * 1000L;
	long index = (long) (t * 1e6);
	String user = m.group(3);
	ObjectId oldId = parseId(m.group(4));
	ObjectId newId = parseId(m.group(5));
	String msg = m.group(6);
	String email = user + "@gerrit"; //$NON-NLS-1$
	PersonIdent who = new PersonIdent(user, email, time, -480);
	log.add(new LogEntry(ref, index, who, oldId, newId, msg));
	}
	}
	Collections.sort(log, LogEntry::compare);
	return log;
	}

	private static long min(List<LogEntry> log) {
	return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
	}

	private static long max(List<LogEntry> log) {
	return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
	}

	private static ObjectId parseId(String s) {
	if ("NULL".equals(s)) { //$NON-NLS-1$
	return ObjectId.zeroId();
	}
	return ObjectId.fromString(s);
	}

	private static class LogEntry {
	static int compare(LogEntry a, LogEntry b) {
	int cmp = a.ref.compareTo(b.ref);
	if (cmp == 0) {
	cmp = Long.signum(b.updateIndex - a.updateIndex);
	}
	return cmp;
	}

	final String ref;
	final long updateIndex;
	final PersonIdent who;
	final ObjectId oldId;
	final ObjectId newId;
	final String message;

	LogEntry(String ref, long updateIndex, PersonIdent who,
	ObjectId oldId, ObjectId newId, String message) {
	this.ref = ref;
	this.updateIndex = updateIndex;
	this.who = who;
	this.oldId = oldId;
	this.newId = newId;
	this.message = message;
	}
	}
	}