reftable/readwrite_test.c

/*
Copyright 2020 Google LLC

Use of this source code is governed by a BSD-style
license that can be found in the LICENSE file or at
https://developers.google.com/open-source/licenses/bsd
*/

#include "system.h"

#include "basics.h"
#include "block.h"
#include "blocksource.h"
#include "constants.h"
#include "reader.h"
#include "record.h"
#include "test_framework.h"
#include "reftable-tests.h"
#include "reftable-writer.h"

static const int update_index = 5;

static void test_buffer(void)
{
	struct strbuf buf = STRBUF_INIT;
	struct reftable_block_source source = { NULL };
	struct reftable_block out = { NULL };
	int n;
	uint8_t in[] = "hello";
	strbuf_add(&buf, in, sizeof(in));
	block_source_from_strbuf(&source, &buf);
	EXPECT(block_source_size(&source) == 6);
	n = block_source_read_block(&source, &out, 0, sizeof(in));
	EXPECT(n == sizeof(in));
	EXPECT(!memcmp(in, out.data, n));
	reftable_block_done(&out);

	n = block_source_read_block(&source, &out, 1, 2);
	EXPECT(n == 2);
	EXPECT(!memcmp(out.data, "el", 2));

	reftable_block_done(&out);
	block_source_close(&source);
	strbuf_release(&buf);
}

static void write_table(char ***names, struct strbuf *buf, int N,
			int block_size, uint32_t hash_id)
{
	struct reftable_write_options opts = {
		.block_size = block_size,
		.hash_id = hash_id,
	};
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, buf, &opts);
	struct reftable_ref_record ref = { NULL };
	int i = 0, n;
	struct reftable_log_record log = { NULL };
	const struct reftable_stats *stats = NULL;
	*names = reftable_calloc(sizeof(char *) * (N + 1));
	reftable_writer_set_limits(w, update_index, update_index);
	for (i = 0; i < N; i++) {
		uint8_t hash[GIT_SHA256_RAWSZ] = { 0 };
		char name[100];
		int n;

		set_test_hash(hash, i);

		snprintf(name, sizeof(name), "refs/heads/branch%02d", i);

		ref.refname = name;
		ref.update_index = update_index;
		ref.value_type = REFTABLE_REF_VAL1;
		ref.value.val1 = hash;
		(*names)[i] = xstrdup(name);

		n = reftable_writer_add_ref(w, &ref);
		EXPECT(n == 0);
	}

	for (i = 0; i < N; i++) {
		uint8_t hash[GIT_SHA256_RAWSZ] = { 0 };
		char name[100];
		int n;

		set_test_hash(hash, i);

		snprintf(name, sizeof(name), "refs/heads/branch%02d", i);

		log.refname = name;
		log.update_index = update_index;
		log.value_type = REFTABLE_LOG_UPDATE;
		log.value.update.new_hash = hash;
		log.value.update.message = "message";

		n = reftable_writer_add_log(w, &log);
		EXPECT(n == 0);
	}

	n = reftable_writer_close(w);
	EXPECT(n == 0);

	stats = writer_stats(w);
	for (i = 0; i < stats->ref_stats.blocks; i++) {
		int off = i * opts.block_size;
		if (off == 0) {
			off = header_size(
				(hash_id == GIT_SHA256_FORMAT_ID) ? 2 : 1);
		}
		EXPECT(buf->buf[off] == 'r');
	}

	EXPECT(stats->log_stats.blocks > 0);
	reftable_writer_free(w);
}

static void test_log_buffer_size(void)
{
	struct strbuf buf = STRBUF_INIT;
	struct reftable_write_options opts = {
		.block_size = 4096,
	};
	int err;
	int i;
	struct reftable_log_record
		log = { .refname = "refs/heads/master",
			.update_index = 0xa,
			.value_type = REFTABLE_LOG_UPDATE,
			.value = { .update = {
					   .name = "Han-Wen Nienhuys",
					   .email = "hanwen@google.com",
					   .tz_offset = 100,
					   .time = 0x5e430672,
					   .message = "commit: 9\n",
				   } } };
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, &buf, &opts);

	/* This tests buffer extension for log compression. Must use a random
	   hash, to ensure that the compressed part is larger than the original.
	*/
	uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ];
	for (i = 0; i < GIT_SHA1_RAWSZ; i++) {
		hash1[i] = (uint8_t)(rand() % 256);
		hash2[i] = (uint8_t)(rand() % 256);
	}
	log.value.update.old_hash = hash1;
	log.value.update.new_hash = hash2;
	reftable_writer_set_limits(w, update_index, update_index);
	err = reftable_writer_add_log(w, &log);
	EXPECT_ERR(err);
	err = reftable_writer_close(w);
	EXPECT_ERR(err);
	reftable_writer_free(w);
	strbuf_release(&buf);
}

static void test_log_overflow(void)
{
	struct strbuf buf = STRBUF_INIT;
	char msg[256] = { 0 };
	struct reftable_write_options opts = {
		.block_size = ARRAY_SIZE(msg),
	};
	int err;
	struct reftable_log_record
		log = { .refname = "refs/heads/master",
			.update_index = 0xa,
			.value_type = REFTABLE_LOG_UPDATE,
			.value = { .update = {
					   .name = "Han-Wen Nienhuys",
					   .email = "hanwen@google.com",
					   .tz_offset = 100,
					   .time = 0x5e430672,
					   .message = msg,
				   } } };
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, &buf, &opts);

	uint8_t hash1[GIT_SHA1_RAWSZ]  = {1}, hash2[GIT_SHA1_RAWSZ] = { 2 };

	memset(msg, 'x', sizeof(msg) - 1);
	log.value.update.old_hash = hash1;
	log.value.update.new_hash = hash2;
	reftable_writer_set_limits(w, update_index, update_index);
	err = reftable_writer_add_log(w, &log);
	EXPECT(err == REFTABLE_ENTRY_TOO_BIG_ERROR);
	reftable_writer_free(w);
	strbuf_release(&buf);
}

static void test_log_write_read(void)
{
	int N = 2;
	char **names = reftable_calloc(sizeof(char *) * (N + 1));
	int err;
	struct reftable_write_options opts = {
		.block_size = 256,
	};
	struct reftable_ref_record ref = { NULL };
	int i = 0;
	struct reftable_log_record log = { NULL };
	int n;
	struct reftable_iterator it = { NULL };
	struct reftable_reader rd = { NULL };
	struct reftable_block_source source = { NULL };
	struct strbuf buf = STRBUF_INIT;
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, &buf, &opts);
	const struct reftable_stats *stats = NULL;
	reftable_writer_set_limits(w, 0, N);
	for (i = 0; i < N; i++) {
		char name[256];
		struct reftable_ref_record ref = { NULL };
		snprintf(name, sizeof(name), "b%02d%0*d", i, 130, 7);
		names[i] = xstrdup(name);
		ref.refname = name;
		ref.update_index = i;

		err = reftable_writer_add_ref(w, &ref);
		EXPECT_ERR(err);
	}
	for (i = 0; i < N; i++) {
		uint8_t hash1[GIT_SHA1_RAWSZ], hash2[GIT_SHA1_RAWSZ];
		struct reftable_log_record log = { NULL };
		set_test_hash(hash1, i);
		set_test_hash(hash2, i + 1);

		log.refname = names[i];
		log.update_index = i;
		log.value_type = REFTABLE_LOG_UPDATE;
		log.value.update.old_hash = hash1;
		log.value.update.new_hash = hash2;

		err = reftable_writer_add_log(w, &log);
		EXPECT_ERR(err);
	}

	n = reftable_writer_close(w);
	EXPECT(n == 0);

	stats = writer_stats(w);
	EXPECT(stats->log_stats.blocks > 0);
	reftable_writer_free(w);
	w = NULL;

	block_source_from_strbuf(&source, &buf);

	err = init_reader(&rd, &source, "file.log");
	EXPECT_ERR(err);

	err = reftable_reader_seek_ref(&rd, &it, names[N - 1]);
	EXPECT_ERR(err);

	err = reftable_iterator_next_ref(&it, &ref);
	EXPECT_ERR(err);

	/* end of iteration. */
	err = reftable_iterator_next_ref(&it, &ref);
	EXPECT(0 < err);

	reftable_iterator_destroy(&it);
	reftable_ref_record_release(&ref);

	err = reftable_reader_seek_log(&rd, &it, "");
	EXPECT_ERR(err);

	i = 0;
	while (1) {
		int err = reftable_iterator_next_log(&it, &log);
		if (err > 0) {
			break;
		}

		EXPECT_ERR(err);
		EXPECT_STREQ(names[i], log.refname);
		EXPECT(i == log.update_index);
		i++;
		reftable_log_record_release(&log);
	}

	EXPECT(i == N);
	reftable_iterator_destroy(&it);

	/* cleanup. */
	strbuf_release(&buf);
	free_names(names);
	reader_close(&rd);
}

static void test_table_read_write_sequential(void)
{
	char **names;
	struct strbuf buf = STRBUF_INIT;
	int N = 50;
	struct reftable_iterator it = { NULL };
	struct reftable_block_source source = { NULL };
	struct reftable_reader rd = { NULL };
	int err = 0;
	int j = 0;

	write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID);

	block_source_from_strbuf(&source, &buf);

	err = init_reader(&rd, &source, "file.ref");
	EXPECT_ERR(err);

	err = reftable_reader_seek_ref(&rd, &it, "");
	EXPECT_ERR(err);

	while (1) {
		struct reftable_ref_record ref = { NULL };
		int r = reftable_iterator_next_ref(&it, &ref);
		EXPECT(r >= 0);
		if (r > 0) {
			break;
		}
		EXPECT(0 == strcmp(names[j], ref.refname));
		EXPECT(update_index == ref.update_index);

		j++;
		reftable_ref_record_release(&ref);
	}
	EXPECT(j == N);
	reftable_iterator_destroy(&it);
	strbuf_release(&buf);
	free_names(names);

	reader_close(&rd);
}

static void test_table_write_small_table(void)
{
	char **names;
	struct strbuf buf = STRBUF_INIT;
	int N = 1;
	write_table(&names, &buf, N, 4096, GIT_SHA1_FORMAT_ID);
	EXPECT(buf.len < 200);
	strbuf_release(&buf);
	free_names(names);
}

static void test_table_read_api(void)
{
	char **names;
	struct strbuf buf = STRBUF_INIT;
	int N = 50;
	struct reftable_reader rd = { NULL };
	struct reftable_block_source source = { NULL };
	int err;
	int i;
	struct reftable_log_record log = { NULL };
	struct reftable_iterator it = { NULL };

	write_table(&names, &buf, N, 256, GIT_SHA1_FORMAT_ID);

	block_source_from_strbuf(&source, &buf);

	err = init_reader(&rd, &source, "file.ref");
	EXPECT_ERR(err);

	err = reftable_reader_seek_ref(&rd, &it, names[0]);
	EXPECT_ERR(err);

	err = reftable_iterator_next_log(&it, &log);
	EXPECT(err == REFTABLE_API_ERROR);

	strbuf_release(&buf);
	for (i = 0; i < N; i++) {
		reftable_free(names[i]);
	}
	reftable_iterator_destroy(&it);
	reftable_free(names);
	reader_close(&rd);
	strbuf_release(&buf);
}

static void test_table_read_write_seek(int index, int hash_id)
{
	char **names;
	struct strbuf buf = STRBUF_INIT;
	int N = 50;
	struct reftable_reader rd = { NULL };
	struct reftable_block_source source = { NULL };
	int err;
	int i = 0;

	struct reftable_iterator it = { NULL };
	struct strbuf pastLast = STRBUF_INIT;
	struct reftable_ref_record ref = { NULL };

	write_table(&names, &buf, N, 256, hash_id);

	block_source_from_strbuf(&source, &buf);

	err = init_reader(&rd, &source, "file.ref");
	EXPECT_ERR(err);
	EXPECT(hash_id == reftable_reader_hash_id(&rd));

	if (!index) {
		rd.ref_offsets.index_offset = 0;
	} else {
		EXPECT(rd.ref_offsets.index_offset > 0);
	}

	for (i = 1; i < N; i++) {
		int err = reftable_reader_seek_ref(&rd, &it, names[i]);
		EXPECT_ERR(err);
		err = reftable_iterator_next_ref(&it, &ref);
		EXPECT_ERR(err);
		EXPECT(0 == strcmp(names[i], ref.refname));
		EXPECT(REFTABLE_REF_VAL1 == ref.value_type);
		EXPECT(i == ref.value.val1[0]);

		reftable_ref_record_release(&ref);
		reftable_iterator_destroy(&it);
	}

	strbuf_addstr(&pastLast, names[N - 1]);
	strbuf_addstr(&pastLast, "/");

	err = reftable_reader_seek_ref(&rd, &it, pastLast.buf);
	if (err == 0) {
		struct reftable_ref_record ref = { NULL };
		int err = reftable_iterator_next_ref(&it, &ref);
		EXPECT(err > 0);
	} else {
		EXPECT(err > 0);
	}

	strbuf_release(&pastLast);
	reftable_iterator_destroy(&it);

	strbuf_release(&buf);
	for (i = 0; i < N; i++) {
		reftable_free(names[i]);
	}
	reftable_free(names);
	reader_close(&rd);
}

static void test_table_read_write_seek_linear(void)
{
	test_table_read_write_seek(0, GIT_SHA1_FORMAT_ID);
}

static void test_table_read_write_seek_linear_sha256(void)
{
	test_table_read_write_seek(0, GIT_SHA256_FORMAT_ID);
}

static void test_table_read_write_seek_index(void)
{
	test_table_read_write_seek(1, GIT_SHA1_FORMAT_ID);
}

static void test_table_refs_for(int indexed)
{
	int N = 50;
	char **want_names = reftable_calloc(sizeof(char *) * (N + 1));
	int want_names_len = 0;
	uint8_t want_hash[GIT_SHA1_RAWSZ];

	struct reftable_write_options opts = {
		.block_size = 256,
	};
	struct reftable_ref_record ref = { NULL };
	int i = 0;
	int n;
	int err;
	struct reftable_reader rd;
	struct reftable_block_source source = { NULL };

	struct strbuf buf = STRBUF_INIT;
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, &buf, &opts);

	struct reftable_iterator it = { NULL };
	int j;

	set_test_hash(want_hash, 4);

	for (i = 0; i < N; i++) {
		uint8_t hash[GIT_SHA1_RAWSZ];
		char fill[51] = { 0 };
		char name[100];
		uint8_t hash1[GIT_SHA1_RAWSZ];
		uint8_t hash2[GIT_SHA1_RAWSZ];
		struct reftable_ref_record ref = { NULL };

		memset(hash, i, sizeof(hash));
		memset(fill, 'x', 50);
		/* Put the variable part in the start */
		snprintf(name, sizeof(name), "br%02d%s", i, fill);
		name[40] = 0;
		ref.refname = name;

		set_test_hash(hash1, i / 4);
		set_test_hash(hash2, 3 + i / 4);
		ref.value_type = REFTABLE_REF_VAL2;
		ref.value.val2.value = hash1;
		ref.value.val2.target_value = hash2;

		/* 80 bytes / entry, so 3 entries per block. Yields 17
		 */
		/* blocks. */
		n = reftable_writer_add_ref(w, &ref);
		EXPECT(n == 0);

		if (!memcmp(hash1, want_hash, GIT_SHA1_RAWSZ) ||
		    !memcmp(hash2, want_hash, GIT_SHA1_RAWSZ)) {
			want_names[want_names_len++] = xstrdup(name);
		}
	}

	n = reftable_writer_close(w);
	EXPECT(n == 0);

	reftable_writer_free(w);
	w = NULL;

	block_source_from_strbuf(&source, &buf);

	err = init_reader(&rd, &source, "file.ref");
	EXPECT_ERR(err);
	if (!indexed) {
		rd.obj_offsets.is_present = 0;
	}

	err = reftable_reader_seek_ref(&rd, &it, "");
	EXPECT_ERR(err);
	reftable_iterator_destroy(&it);

	err = reftable_reader_refs_for(&rd, &it, want_hash);
	EXPECT_ERR(err);

	j = 0;
	while (1) {
		int err = reftable_iterator_next_ref(&it, &ref);
		EXPECT(err >= 0);
		if (err > 0) {
			break;
		}

		EXPECT(j < want_names_len);
		EXPECT(0 == strcmp(ref.refname, want_names[j]));
		j++;
		reftable_ref_record_release(&ref);
	}
	EXPECT(j == want_names_len);

	strbuf_release(&buf);
	free_names(want_names);
	reftable_iterator_destroy(&it);
	reader_close(&rd);
}

static void test_table_refs_for_no_index(void)
{
	test_table_refs_for(0);
}

static void test_table_refs_for_obj_index(void)
{
	test_table_refs_for(1);
}

static void test_write_empty_table(void)
{
	struct reftable_write_options opts = { 0 };
	struct strbuf buf = STRBUF_INIT;
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, &buf, &opts);
	struct reftable_block_source source = { NULL };
	struct reftable_reader *rd = NULL;
	struct reftable_ref_record rec = { NULL };
	struct reftable_iterator it = { NULL };
	int err;

	reftable_writer_set_limits(w, 1, 1);

	err = reftable_writer_close(w);
	EXPECT(err == REFTABLE_EMPTY_TABLE_ERROR);
	reftable_writer_free(w);

	EXPECT(buf.len == header_size(1) + footer_size(1));

	block_source_from_strbuf(&source, &buf);

	err = reftable_new_reader(&rd, &source, "filename");
	EXPECT_ERR(err);

	err = reftable_reader_seek_ref(rd, &it, "");
	EXPECT_ERR(err);

	err = reftable_iterator_next_ref(&it, &rec);
	EXPECT(err > 0);

	reftable_iterator_destroy(&it);
	reftable_reader_free(rd);
	strbuf_release(&buf);
}

static void test_write_key_order(void)
{
	struct reftable_write_options opts = { 0 };
	struct strbuf buf = STRBUF_INIT;
	struct reftable_writer *w =
		reftable_new_writer(&strbuf_add_void, &buf, &opts);
	struct reftable_ref_record refs[2] = {
		{
			.refname = "b",
			.update_index = 1,
			.value_type = REFTABLE_REF_SYMREF,
			.value = {
				.symref = "target",
			},
		}, {
			.refname = "a",
			.update_index = 1,
			.value_type = REFTABLE_REF_SYMREF,
			.value = {
				.symref = "target",
			},
		}
	};
	int err;

	reftable_writer_set_limits(w, 1, 1);
	err = reftable_writer_add_ref(w, &refs[0]);
	EXPECT_ERR(err);
	err = reftable_writer_add_ref(w, &refs[1]);
	printf("%d\n", err);
	EXPECT(err == REFTABLE_API_ERROR);
	reftable_writer_close(w);
	reftable_writer_free(w);
	strbuf_release(&buf);
}

static void test_corrupt_table_empty(void)
{
	struct strbuf buf = STRBUF_INIT;
	struct reftable_block_source source = { NULL };
	struct reftable_reader rd = { NULL };
	int err;

	block_source_from_strbuf(&source, &buf);
	err = init_reader(&rd, &source, "file.log");
	EXPECT(err == REFTABLE_FORMAT_ERROR);
}

static void test_corrupt_table(void)
{
	uint8_t zeros[1024] = { 0 };
	struct strbuf buf = STRBUF_INIT;
	struct reftable_block_source source = { NULL };
	struct reftable_reader rd = { NULL };
	int err;
	strbuf_add(&buf, zeros, sizeof(zeros));

	block_source_from_strbuf(&source, &buf);
	err = init_reader(&rd, &source, "file.log");
	EXPECT(err == REFTABLE_FORMAT_ERROR);
	strbuf_release(&buf);
}

int readwrite_test_main(int argc, const char *argv[])
{
	RUN_TEST(test_corrupt_table);
	RUN_TEST(test_corrupt_table_empty);
	RUN_TEST(test_log_write_read);
	RUN_TEST(test_write_key_order);
	RUN_TEST(test_table_read_write_seek_linear_sha256);
	RUN_TEST(test_log_buffer_size);
	RUN_TEST(test_table_write_small_table);
	RUN_TEST(test_buffer);
	RUN_TEST(test_table_read_api);
	RUN_TEST(test_table_read_write_sequential);
	RUN_TEST(test_table_read_write_seek_linear);
	RUN_TEST(test_table_read_write_seek_index);
	RUN_TEST(test_table_refs_for_no_index);
	RUN_TEST(test_table_refs_for_obj_index);
	RUN_TEST(test_write_empty_table);
	RUN_TEST(test_log_overflow);
	return 0;
}