blob: b9fdda1d3d2e033ddcfa7c1b31730d4188c75306 [file] [log] [blame]
/**
* Copyright 2013, GitHub, Inc
* Copyright 2009-2013, Daniel Lemire, Cliff Moon,
* David McIntosh, Robert Becho, Google Inc. and Veronika Zenz
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, see <http://www.gnu.org/licenses/>.
*/
#include "git-compat-util.h"
#include "ewok.h"
#include "ewok_rlw.h"
static inline size_t min_size(size_t a, size_t b)
{
return a < b ? a : b;
}
static inline size_t max_size(size_t a, size_t b)
{
return a > b ? a : b;
}
static inline void buffer_grow(struct ewah_bitmap *self, size_t new_size)
{
size_t rlw_offset = (uint8_t *)self->rlw - (uint8_t *)self->buffer;
if (self->alloc_size >= new_size)
return;
self->alloc_size = new_size;
REALLOC_ARRAY(self->buffer, self->alloc_size);
self->rlw = self->buffer + (rlw_offset / sizeof(eword_t));
}
static inline void buffer_push(struct ewah_bitmap *self, eword_t value)
{
if (self->buffer_size + 1 >= self->alloc_size)
buffer_grow(self, self->buffer_size * 3 / 2);
self->buffer[self->buffer_size++] = value;
}
static void buffer_push_rlw(struct ewah_bitmap *self, eword_t value)
{
buffer_push(self, value);
self->rlw = self->buffer + self->buffer_size - 1;
}
static size_t add_empty_words(struct ewah_bitmap *self, int v, size_t number)
{
size_t added = 0;
eword_t runlen, can_add;
if (rlw_get_run_bit(self->rlw) != v && rlw_size(self->rlw) == 0) {
rlw_set_run_bit(self->rlw, v);
} else if (rlw_get_literal_words(self->rlw) != 0 ||
rlw_get_run_bit(self->rlw) != v) {
buffer_push_rlw(self, 0);
if (v) rlw_set_run_bit(self->rlw, v);
added++;
}
runlen = rlw_get_running_len(self->rlw);
can_add = min_size(number, RLW_LARGEST_RUNNING_COUNT - runlen);
rlw_set_running_len(self->rlw, runlen + can_add);
number -= can_add;
while (number >= RLW_LARGEST_RUNNING_COUNT) {
buffer_push_rlw(self, 0);
added++;
if (v) rlw_set_run_bit(self->rlw, v);
rlw_set_running_len(self->rlw, RLW_LARGEST_RUNNING_COUNT);
number -= RLW_LARGEST_RUNNING_COUNT;
}
if (number > 0) {
buffer_push_rlw(self, 0);
added++;
if (v) rlw_set_run_bit(self->rlw, v);
rlw_set_running_len(self->rlw, number);
}
return added;
}
size_t ewah_add_empty_words(struct ewah_bitmap *self, int v, size_t number)
{
if (number == 0)
return 0;
self->bit_size += number * BITS_IN_EWORD;
return add_empty_words(self, v, number);
}
static size_t add_literal(struct ewah_bitmap *self, eword_t new_data)
{
eword_t current_num = rlw_get_literal_words(self->rlw);
if (current_num >= RLW_LARGEST_LITERAL_COUNT) {
buffer_push_rlw(self, 0);
rlw_set_literal_words(self->rlw, 1);
buffer_push(self, new_data);
return 2;
}
rlw_set_literal_words(self->rlw, current_num + 1);
/* sanity check */
assert(rlw_get_literal_words(self->rlw) == current_num + 1);
buffer_push(self, new_data);
return 1;
}
void ewah_add_dirty_words(
struct ewah_bitmap *self, const eword_t *buffer,
size_t number, int negate)
{
size_t literals, can_add;
while (1) {
literals = rlw_get_literal_words(self->rlw);
can_add = min_size(number, RLW_LARGEST_LITERAL_COUNT - literals);
rlw_set_literal_words(self->rlw, literals + can_add);
if (self->buffer_size + can_add >= self->alloc_size)
buffer_grow(self, (self->buffer_size + can_add) * 3 / 2);
if (negate) {
size_t i;
for (i = 0; i < can_add; ++i)
self->buffer[self->buffer_size++] = ~buffer[i];
} else {
memcpy(self->buffer + self->buffer_size,
buffer, can_add * sizeof(eword_t));
self->buffer_size += can_add;
}
self->bit_size += can_add * BITS_IN_EWORD;
if (number - can_add == 0)
break;
buffer_push_rlw(self, 0);
buffer += can_add;
number -= can_add;
}
}
static size_t add_empty_word(struct ewah_bitmap *self, int v)
{
int no_literal = (rlw_get_literal_words(self->rlw) == 0);
eword_t run_len = rlw_get_running_len(self->rlw);
if (no_literal && run_len == 0) {
rlw_set_run_bit(self->rlw, v);
assert(rlw_get_run_bit(self->rlw) == v);
}
if (no_literal && rlw_get_run_bit(self->rlw) == v &&
run_len < RLW_LARGEST_RUNNING_COUNT) {
rlw_set_running_len(self->rlw, run_len + 1);
assert(rlw_get_running_len(self->rlw) == run_len + 1);
return 0;
} else {
buffer_push_rlw(self, 0);
assert(rlw_get_running_len(self->rlw) == 0);
assert(rlw_get_run_bit(self->rlw) == 0);
assert(rlw_get_literal_words(self->rlw) == 0);
rlw_set_run_bit(self->rlw, v);
assert(rlw_get_run_bit(self->rlw) == v);
rlw_set_running_len(self->rlw, 1);
assert(rlw_get_running_len(self->rlw) == 1);
assert(rlw_get_literal_words(self->rlw) == 0);
return 1;
}
}
size_t ewah_add(struct ewah_bitmap *self, eword_t word)
{
self->bit_size += BITS_IN_EWORD;
if (word == 0)
return add_empty_word(self, 0);
if (word == (eword_t)(~0))
return add_empty_word(self, 1);
return add_literal(self, word);
}
void ewah_set(struct ewah_bitmap *self, size_t i)
{
const size_t dist =
DIV_ROUND_UP(i + 1, BITS_IN_EWORD) -
DIV_ROUND_UP(self->bit_size, BITS_IN_EWORD);
assert(i >= self->bit_size);
self->bit_size = i + 1;
if (dist > 0) {
if (dist > 1)
add_empty_words(self, 0, dist - 1);
add_literal(self, (eword_t)1 << (i % BITS_IN_EWORD));
return;
}
if (rlw_get_literal_words(self->rlw) == 0) {
rlw_set_running_len(self->rlw,
rlw_get_running_len(self->rlw) - 1);
add_literal(self, (eword_t)1 << (i % BITS_IN_EWORD));
return;
}
self->buffer[self->buffer_size - 1] |=
((eword_t)1 << (i % BITS_IN_EWORD));
/* check if we just completed a stream of 1s */
if (self->buffer[self->buffer_size - 1] == (eword_t)(~0)) {
self->buffer[--self->buffer_size] = 0;
rlw_set_literal_words(self->rlw,
rlw_get_literal_words(self->rlw) - 1);
add_empty_word(self, 1);
}
}
void ewah_each_bit(struct ewah_bitmap *self, void (*callback)(size_t, void*), void *payload)
{
size_t pos = 0;
size_t pointer = 0;
size_t k;
while (pointer < self->buffer_size) {
eword_t *word = &self->buffer[pointer];
if (rlw_get_run_bit(word)) {
size_t len = rlw_get_running_len(word) * BITS_IN_EWORD;
for (k = 0; k < len; ++k, ++pos)
callback(pos, payload);
} else {
pos += rlw_get_running_len(word) * BITS_IN_EWORD;
}
++pointer;
for (k = 0; k < rlw_get_literal_words(word); ++k) {
int c;
/* todo: zero count optimization */
for (c = 0; c < BITS_IN_EWORD; ++c, ++pos) {
if ((self->buffer[pointer] & ((eword_t)1 << c)) != 0)
callback(pos, payload);
}
++pointer;
}
}
}
struct ewah_bitmap *ewah_new(void)
{
struct ewah_bitmap *self;
self = xmalloc(sizeof(struct ewah_bitmap));
self->alloc_size = 32;
ALLOC_ARRAY(self->buffer, self->alloc_size);
ewah_clear(self);
return self;
}
void ewah_clear(struct ewah_bitmap *self)
{
self->buffer_size = 1;
self->buffer[0] = 0;
self->bit_size = 0;
self->rlw = self->buffer;
}
void ewah_free(struct ewah_bitmap *self)
{
if (!self)
return;
if (self->alloc_size)
free(self->buffer);
free(self);
}
static void read_new_rlw(struct ewah_iterator *it)
{
const eword_t *word = NULL;
it->literals = 0;
it->compressed = 0;
while (1) {
word = &it->buffer[it->pointer];
it->rl = rlw_get_running_len(word);
it->lw = rlw_get_literal_words(word);
it->b = rlw_get_run_bit(word);
if (it->rl || it->lw)
return;
if (it->pointer < it->buffer_size - 1) {
it->pointer++;
} else {
it->pointer = it->buffer_size;
return;
}
}
}
int ewah_iterator_next(eword_t *next, struct ewah_iterator *it)
{
if (it->pointer >= it->buffer_size)
return 0;
if (it->compressed < it->rl) {
it->compressed++;
*next = it->b ? (eword_t)(~0) : 0;
} else {
assert(it->literals < it->lw);
it->literals++;
it->pointer++;
assert(it->pointer < it->buffer_size);
*next = it->buffer[it->pointer];
}
if (it->compressed == it->rl && it->literals == it->lw) {
if (++it->pointer < it->buffer_size)
read_new_rlw(it);
}
return 1;
}
void ewah_iterator_init(struct ewah_iterator *it, struct ewah_bitmap *parent)
{
it->buffer = parent->buffer;
it->buffer_size = parent->buffer_size;
it->pointer = 0;
it->lw = 0;
it->rl = 0;
it->compressed = 0;
it->literals = 0;
it->b = 0;
if (it->pointer < it->buffer_size)
read_new_rlw(it);
}
void ewah_not(struct ewah_bitmap *self)
{
size_t pointer = 0;
while (pointer < self->buffer_size) {
eword_t *word = &self->buffer[pointer];
size_t literals, k;
rlw_xor_run_bit(word);
++pointer;
literals = rlw_get_literal_words(word);
for (k = 0; k < literals; ++k) {
self->buffer[pointer] = ~self->buffer[pointer];
++pointer;
}
}
}
void ewah_xor(
struct ewah_bitmap *ewah_i,
struct ewah_bitmap *ewah_j,
struct ewah_bitmap *out)
{
struct rlw_iterator rlw_i;
struct rlw_iterator rlw_j;
size_t literals;
rlwit_init(&rlw_i, ewah_i);
rlwit_init(&rlw_j, ewah_j);
while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
struct rlw_iterator *prey, *predator;
size_t index;
int negate_words;
if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
prey = &rlw_i;
predator = &rlw_j;
} else {
prey = &rlw_j;
predator = &rlw_i;
}
negate_words = !!predator->rlw.running_bit;
index = rlwit_discharge(prey, out,
predator->rlw.running_len, negate_words);
ewah_add_empty_words(out, negate_words,
predator->rlw.running_len - index);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
}
literals = min_size(
rlw_i.rlw.literal_words,
rlw_j.rlw.literal_words);
if (literals) {
size_t k;
for (k = 0; k < literals; ++k) {
ewah_add(out,
rlw_i.buffer[rlw_i.literal_word_start + k] ^
rlw_j.buffer[rlw_j.literal_word_start + k]
);
}
rlwit_discard_first_words(&rlw_i, literals);
rlwit_discard_first_words(&rlw_j, literals);
}
}
if (rlwit_word_size(&rlw_i) > 0)
rlwit_discharge(&rlw_i, out, ~0, 0);
else
rlwit_discharge(&rlw_j, out, ~0, 0);
out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
}
void ewah_and(
struct ewah_bitmap *ewah_i,
struct ewah_bitmap *ewah_j,
struct ewah_bitmap *out)
{
struct rlw_iterator rlw_i;
struct rlw_iterator rlw_j;
size_t literals;
rlwit_init(&rlw_i, ewah_i);
rlwit_init(&rlw_j, ewah_j);
while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
struct rlw_iterator *prey, *predator;
if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
prey = &rlw_i;
predator = &rlw_j;
} else {
prey = &rlw_j;
predator = &rlw_i;
}
if (predator->rlw.running_bit == 0) {
ewah_add_empty_words(out, 0,
predator->rlw.running_len);
rlwit_discard_first_words(prey,
predator->rlw.running_len);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
} else {
size_t index = rlwit_discharge(prey, out,
predator->rlw.running_len, 0);
ewah_add_empty_words(out, 0,
predator->rlw.running_len - index);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
}
}
literals = min_size(
rlw_i.rlw.literal_words,
rlw_j.rlw.literal_words);
if (literals) {
size_t k;
for (k = 0; k < literals; ++k) {
ewah_add(out,
rlw_i.buffer[rlw_i.literal_word_start + k] &
rlw_j.buffer[rlw_j.literal_word_start + k]
);
}
rlwit_discard_first_words(&rlw_i, literals);
rlwit_discard_first_words(&rlw_j, literals);
}
}
if (rlwit_word_size(&rlw_i) > 0)
rlwit_discharge_empty(&rlw_i, out);
else
rlwit_discharge_empty(&rlw_j, out);
out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
}
void ewah_and_not(
struct ewah_bitmap *ewah_i,
struct ewah_bitmap *ewah_j,
struct ewah_bitmap *out)
{
struct rlw_iterator rlw_i;
struct rlw_iterator rlw_j;
size_t literals;
rlwit_init(&rlw_i, ewah_i);
rlwit_init(&rlw_j, ewah_j);
while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
struct rlw_iterator *prey, *predator;
if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
prey = &rlw_i;
predator = &rlw_j;
} else {
prey = &rlw_j;
predator = &rlw_i;
}
if ((predator->rlw.running_bit && prey == &rlw_i) ||
(!predator->rlw.running_bit && prey != &rlw_i)) {
ewah_add_empty_words(out, 0,
predator->rlw.running_len);
rlwit_discard_first_words(prey,
predator->rlw.running_len);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
} else {
size_t index;
int negate_words;
negate_words = (&rlw_i != prey);
index = rlwit_discharge(prey, out,
predator->rlw.running_len, negate_words);
ewah_add_empty_words(out, negate_words,
predator->rlw.running_len - index);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
}
}
literals = min_size(
rlw_i.rlw.literal_words,
rlw_j.rlw.literal_words);
if (literals) {
size_t k;
for (k = 0; k < literals; ++k) {
ewah_add(out,
rlw_i.buffer[rlw_i.literal_word_start + k] &
~(rlw_j.buffer[rlw_j.literal_word_start + k])
);
}
rlwit_discard_first_words(&rlw_i, literals);
rlwit_discard_first_words(&rlw_j, literals);
}
}
if (rlwit_word_size(&rlw_i) > 0)
rlwit_discharge(&rlw_i, out, ~0, 0);
else
rlwit_discharge_empty(&rlw_j, out);
out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
}
void ewah_or(
struct ewah_bitmap *ewah_i,
struct ewah_bitmap *ewah_j,
struct ewah_bitmap *out)
{
struct rlw_iterator rlw_i;
struct rlw_iterator rlw_j;
size_t literals;
rlwit_init(&rlw_i, ewah_i);
rlwit_init(&rlw_j, ewah_j);
while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
struct rlw_iterator *prey, *predator;
if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
prey = &rlw_i;
predator = &rlw_j;
} else {
prey = &rlw_j;
predator = &rlw_i;
}
if (predator->rlw.running_bit) {
ewah_add_empty_words(out, 0,
predator->rlw.running_len);
rlwit_discard_first_words(prey,
predator->rlw.running_len);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
} else {
size_t index = rlwit_discharge(prey, out,
predator->rlw.running_len, 0);
ewah_add_empty_words(out, 0,
predator->rlw.running_len - index);
rlwit_discard_first_words(predator,
predator->rlw.running_len);
}
}
literals = min_size(
rlw_i.rlw.literal_words,
rlw_j.rlw.literal_words);
if (literals) {
size_t k;
for (k = 0; k < literals; ++k) {
ewah_add(out,
rlw_i.buffer[rlw_i.literal_word_start + k] |
rlw_j.buffer[rlw_j.literal_word_start + k]
);
}
rlwit_discard_first_words(&rlw_i, literals);
rlwit_discard_first_words(&rlw_j, literals);
}
}
if (rlwit_word_size(&rlw_i) > 0)
rlwit_discharge(&rlw_i, out, ~0, 0);
else
rlwit_discharge(&rlw_j, out, ~0, 0);
out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
}
#define BITMAP_POOL_MAX 16
static struct ewah_bitmap *bitmap_pool[BITMAP_POOL_MAX];
static size_t bitmap_pool_size;
struct ewah_bitmap *ewah_pool_new(void)
{
if (bitmap_pool_size)
return bitmap_pool[--bitmap_pool_size];
return ewah_new();
}
void ewah_pool_free(struct ewah_bitmap *self)
{
if (self == NULL)
return;
if (bitmap_pool_size == BITMAP_POOL_MAX ||
self->alloc_size == 0) {
ewah_free(self);
return;
}
ewah_clear(self);
bitmap_pool[bitmap_pool_size++] = self;
}
uint32_t ewah_checksum(struct ewah_bitmap *self)
{
const uint8_t *p = (uint8_t *)self->buffer;
uint32_t crc = (uint32_t)self->bit_size;
size_t size = self->buffer_size * sizeof(eword_t);
while (size--)
crc = (crc << 5) - crc + (uint32_t)*p++;
return crc;
}