| /* |
| Copyright 2020 Google LLC |
| |
| Use of this source code is governed by a BSD-style |
| license that can be found in the LICENSE file or at |
| https://developers.google.com/open-source/licenses/bsd |
| */ |
| |
| #include "block.h" |
| |
| #include "blocksource.h" |
| #include "constants.h" |
| #include "record.h" |
| #include "reftable-error.h" |
| #include "system.h" |
| #include <zlib.h> |
| |
| int header_size(int version) |
| { |
| switch (version) { |
| case 1: |
| return 24; |
| case 2: |
| return 28; |
| } |
| abort(); |
| } |
| |
| int footer_size(int version) |
| { |
| switch (version) { |
| case 1: |
| return 68; |
| case 2: |
| return 72; |
| } |
| abort(); |
| } |
| |
| static int block_writer_register_restart(struct block_writer *w, int n, |
| int is_restart, struct strbuf *key) |
| { |
| int rlen = w->restart_len; |
| if (rlen >= MAX_RESTARTS) { |
| is_restart = 0; |
| } |
| |
| if (is_restart) { |
| rlen++; |
| } |
| if (2 + 3 * rlen + n > w->block_size - w->next) |
| return -1; |
| if (is_restart) { |
| if (w->restart_len == w->restart_cap) { |
| w->restart_cap = w->restart_cap * 2 + 1; |
| w->restarts = reftable_realloc( |
| w->restarts, sizeof(uint32_t) * w->restart_cap); |
| } |
| |
| w->restarts[w->restart_len++] = w->next; |
| } |
| |
| w->next += n; |
| |
| strbuf_reset(&w->last_key); |
| strbuf_addbuf(&w->last_key, key); |
| w->entries++; |
| return 0; |
| } |
| |
| void block_writer_init(struct block_writer *bw, uint8_t typ, uint8_t *buf, |
| uint32_t block_size, uint32_t header_off, int hash_size) |
| { |
| bw->buf = buf; |
| bw->hash_size = hash_size; |
| bw->block_size = block_size; |
| bw->header_off = header_off; |
| bw->buf[header_off] = typ; |
| bw->next = header_off + 4; |
| bw->restart_interval = 16; |
| bw->entries = 0; |
| bw->restart_len = 0; |
| bw->last_key.len = 0; |
| } |
| |
| uint8_t block_writer_type(struct block_writer *bw) |
| { |
| return bw->buf[bw->header_off]; |
| } |
| |
| /* Adds the reftable_record to the block. Returns -1 if it does not fit, 0 on |
| success. Returns REFTABLE_API_ERROR if attempting to write a record with |
| empty key. */ |
| int block_writer_add(struct block_writer *w, struct reftable_record *rec) |
| { |
| struct strbuf empty = STRBUF_INIT; |
| struct strbuf last = |
| w->entries % w->restart_interval == 0 ? empty : w->last_key; |
| struct string_view out = { |
| .buf = w->buf + w->next, |
| .len = w->block_size - w->next, |
| }; |
| |
| struct string_view start = out; |
| |
| int is_restart = 0; |
| struct strbuf key = STRBUF_INIT; |
| int n = 0; |
| int err = -1; |
| |
| reftable_record_key(rec, &key); |
| if (!key.len) { |
| err = REFTABLE_API_ERROR; |
| goto done; |
| } |
| |
| n = reftable_encode_key(&is_restart, out, last, key, |
| reftable_record_val_type(rec)); |
| if (n < 0) |
| goto done; |
| string_view_consume(&out, n); |
| |
| n = reftable_record_encode(rec, out, w->hash_size); |
| if (n < 0) |
| goto done; |
| string_view_consume(&out, n); |
| |
| err = block_writer_register_restart(w, start.len - out.len, is_restart, |
| &key); |
| done: |
| strbuf_release(&key); |
| return err; |
| } |
| |
| int block_writer_finish(struct block_writer *w) |
| { |
| int i; |
| for (i = 0; i < w->restart_len; i++) { |
| put_be24(w->buf + w->next, w->restarts[i]); |
| w->next += 3; |
| } |
| |
| put_be16(w->buf + w->next, w->restart_len); |
| w->next += 2; |
| put_be24(w->buf + 1 + w->header_off, w->next); |
| |
| if (block_writer_type(w) == BLOCK_TYPE_LOG) { |
| int block_header_skip = 4 + w->header_off; |
| uLongf src_len = w->next - block_header_skip; |
| uLongf dest_cap = src_len * 1.001 + 12; |
| |
| uint8_t *compressed = reftable_malloc(dest_cap); |
| while (1) { |
| uLongf out_dest_len = dest_cap; |
| int zresult = compress2(compressed, &out_dest_len, |
| w->buf + block_header_skip, |
| src_len, 9); |
| if (zresult == Z_BUF_ERROR && dest_cap < LONG_MAX) { |
| dest_cap *= 2; |
| compressed = |
| reftable_realloc(compressed, dest_cap); |
| if (compressed) |
| continue; |
| } |
| |
| if (Z_OK != zresult) { |
| reftable_free(compressed); |
| return REFTABLE_ZLIB_ERROR; |
| } |
| |
| memcpy(w->buf + block_header_skip, compressed, |
| out_dest_len); |
| w->next = out_dest_len + block_header_skip; |
| reftable_free(compressed); |
| break; |
| } |
| } |
| return w->next; |
| } |
| |
| uint8_t block_reader_type(struct block_reader *r) |
| { |
| return r->block.data[r->header_off]; |
| } |
| |
| int block_reader_init(struct block_reader *br, struct reftable_block *block, |
| uint32_t header_off, uint32_t table_block_size, |
| int hash_size) |
| { |
| uint32_t full_block_size = table_block_size; |
| uint8_t typ = block->data[header_off]; |
| uint32_t sz = get_be24(block->data + header_off + 1); |
| int err = 0; |
| uint16_t restart_count = 0; |
| uint32_t restart_start = 0; |
| uint8_t *restart_bytes = NULL; |
| uint8_t *uncompressed = NULL; |
| |
| if (!reftable_is_block_type(typ)) { |
| err = REFTABLE_FORMAT_ERROR; |
| goto done; |
| } |
| |
| if (typ == BLOCK_TYPE_LOG) { |
| int block_header_skip = 4 + header_off; |
| uLongf dst_len = sz - block_header_skip; /* total size of dest |
| buffer. */ |
| uLongf src_len = block->len - block_header_skip; |
| /* Log blocks specify the *uncompressed* size in their header. |
| */ |
| uncompressed = reftable_malloc(sz); |
| |
| /* Copy over the block header verbatim. It's not compressed. */ |
| memcpy(uncompressed, block->data, block_header_skip); |
| |
| /* Uncompress */ |
| if (Z_OK != |
| uncompress2(uncompressed + block_header_skip, &dst_len, |
| block->data + block_header_skip, &src_len)) { |
| err = REFTABLE_ZLIB_ERROR; |
| goto done; |
| } |
| |
| if (dst_len + block_header_skip != sz) { |
| err = REFTABLE_FORMAT_ERROR; |
| goto done; |
| } |
| |
| /* We're done with the input data. */ |
| reftable_block_done(block); |
| block->data = uncompressed; |
| uncompressed = NULL; |
| block->len = sz; |
| block->source = malloc_block_source(); |
| full_block_size = src_len + block_header_skip; |
| } else if (full_block_size == 0) { |
| full_block_size = sz; |
| } else if (sz < full_block_size && sz < block->len && |
| block->data[sz] != 0) { |
| /* If the block is smaller than the full block size, it is |
| padded (data followed by '\0') or the next block is |
| unaligned. */ |
| full_block_size = sz; |
| } |
| |
| restart_count = get_be16(block->data + sz - 2); |
| restart_start = sz - 2 - 3 * restart_count; |
| restart_bytes = block->data + restart_start; |
| |
| /* transfer ownership. */ |
| br->block = *block; |
| block->data = NULL; |
| block->len = 0; |
| |
| br->hash_size = hash_size; |
| br->block_len = restart_start; |
| br->full_block_size = full_block_size; |
| br->header_off = header_off; |
| br->restart_count = restart_count; |
| br->restart_bytes = restart_bytes; |
| |
| done: |
| reftable_free(uncompressed); |
| return err; |
| } |
| |
| static uint32_t block_reader_restart_offset(struct block_reader *br, int i) |
| { |
| return get_be24(br->restart_bytes + 3 * i); |
| } |
| |
| void block_reader_start(struct block_reader *br, struct block_iter *it) |
| { |
| it->br = br; |
| strbuf_reset(&it->last_key); |
| it->next_off = br->header_off + 4; |
| } |
| |
| struct restart_find_args { |
| int error; |
| struct strbuf key; |
| struct block_reader *r; |
| }; |
| |
| static int restart_key_less(size_t idx, void *args) |
| { |
| struct restart_find_args *a = args; |
| uint32_t off = block_reader_restart_offset(a->r, idx); |
| struct string_view in = { |
| .buf = a->r->block.data + off, |
| .len = a->r->block_len - off, |
| }; |
| |
| /* the restart key is verbatim in the block, so this could avoid the |
| alloc for decoding the key */ |
| struct strbuf rkey = STRBUF_INIT; |
| struct strbuf last_key = STRBUF_INIT; |
| uint8_t unused_extra; |
| int n = reftable_decode_key(&rkey, &unused_extra, last_key, in); |
| int result; |
| if (n < 0) { |
| a->error = 1; |
| return -1; |
| } |
| |
| result = strbuf_cmp(&a->key, &rkey); |
| strbuf_release(&rkey); |
| return result; |
| } |
| |
| void block_iter_copy_from(struct block_iter *dest, struct block_iter *src) |
| { |
| dest->br = src->br; |
| dest->next_off = src->next_off; |
| strbuf_reset(&dest->last_key); |
| strbuf_addbuf(&dest->last_key, &src->last_key); |
| } |
| |
| int block_iter_next(struct block_iter *it, struct reftable_record *rec) |
| { |
| struct string_view in = { |
| .buf = it->br->block.data + it->next_off, |
| .len = it->br->block_len - it->next_off, |
| }; |
| struct string_view start = in; |
| struct strbuf key = STRBUF_INIT; |
| uint8_t extra = 0; |
| int n = 0; |
| |
| if (it->next_off >= it->br->block_len) |
| return 1; |
| |
| n = reftable_decode_key(&key, &extra, it->last_key, in); |
| if (n < 0) |
| return -1; |
| |
| if (!key.len) |
| return REFTABLE_FORMAT_ERROR; |
| |
| string_view_consume(&in, n); |
| n = reftable_record_decode(rec, key, extra, in, it->br->hash_size); |
| if (n < 0) |
| return -1; |
| string_view_consume(&in, n); |
| |
| strbuf_reset(&it->last_key); |
| strbuf_addbuf(&it->last_key, &key); |
| it->next_off += start.len - in.len; |
| strbuf_release(&key); |
| return 0; |
| } |
| |
| int block_reader_first_key(struct block_reader *br, struct strbuf *key) |
| { |
| struct strbuf empty = STRBUF_INIT; |
| int off = br->header_off + 4; |
| struct string_view in = { |
| .buf = br->block.data + off, |
| .len = br->block_len - off, |
| }; |
| |
| uint8_t extra = 0; |
| int n = reftable_decode_key(key, &extra, empty, in); |
| if (n < 0) |
| return n; |
| if (!key->len) |
| return REFTABLE_FORMAT_ERROR; |
| |
| return 0; |
| } |
| |
| int block_iter_seek(struct block_iter *it, struct strbuf *want) |
| { |
| return block_reader_seek(it->br, it, want); |
| } |
| |
| void block_iter_close(struct block_iter *it) |
| { |
| strbuf_release(&it->last_key); |
| } |
| |
| int block_reader_seek(struct block_reader *br, struct block_iter *it, |
| struct strbuf *want) |
| { |
| struct restart_find_args args = { |
| .key = *want, |
| .r = br, |
| }; |
| struct reftable_record rec = reftable_new_record(block_reader_type(br)); |
| struct strbuf key = STRBUF_INIT; |
| int err = 0; |
| struct block_iter next = { |
| .last_key = STRBUF_INIT, |
| }; |
| |
| int i = binsearch(br->restart_count, &restart_key_less, &args); |
| if (args.error) { |
| err = REFTABLE_FORMAT_ERROR; |
| goto done; |
| } |
| |
| it->br = br; |
| if (i > 0) { |
| i--; |
| it->next_off = block_reader_restart_offset(br, i); |
| } else { |
| it->next_off = br->header_off + 4; |
| } |
| |
| /* We're looking for the last entry less/equal than the wanted key, so |
| we have to go one entry too far and then back up. |
| */ |
| while (1) { |
| block_iter_copy_from(&next, it); |
| err = block_iter_next(&next, &rec); |
| if (err < 0) |
| goto done; |
| |
| reftable_record_key(&rec, &key); |
| if (err > 0 || strbuf_cmp(&key, want) >= 0) { |
| err = 0; |
| goto done; |
| } |
| |
| block_iter_copy_from(it, &next); |
| } |
| |
| done: |
| strbuf_release(&key); |
| strbuf_release(&next.last_key); |
| reftable_record_release(&rec); |
| |
| return err; |
| } |
| |
| void block_writer_release(struct block_writer *bw) |
| { |
| FREE_AND_NULL(bw->restarts); |
| strbuf_release(&bw->last_key); |
| /* the block is not owned. */ |
| } |
| |
| void reftable_block_done(struct reftable_block *blockp) |
| { |
| struct reftable_block_source source = blockp->source; |
| if (blockp && source.ops) |
| source.ops->return_block(source.arg, blockp); |
| blockp->data = NULL; |
| blockp->len = 0; |
| blockp->source.ops = NULL; |
| blockp->source.arg = NULL; |
| } |