Refactor packed_git to prepare for sliding mmap windows.
The idea behind the sliding mmap window pack reader implementation
is to have multiple mmap regions active against the same pack file,
thereby allowing the process to mmap in only the active/hot sections
of the pack and reduce overall virtual address space usage.
To implement this we need to refactor the mmap related data
(pack_base, pack_use_cnt) out of struct packed_git and move them
into a new struct pack_window.
We are refactoring the code to support a single struct pack_window
per packfile, thereby emulating the prior behavior of mmap'ing the
entire pack file.
Signed-off-by: Shawn O. Pearce <spearce@spearce.org>
Signed-off-by: Junio C Hamano <junkio@cox.net>
diff --git a/sha1_file.c b/sha1_file.c
index 4183f59..a9f374e 100644
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -455,21 +455,23 @@
struct packed_git *p, *lru = NULL;
for (p = packed_git; p; p = p->next) {
- if (p->pack_use_cnt || !p->pack_base)
+ if (!p->windows || p->windows->inuse_cnt)
continue;
- if (!lru || p->pack_last_used < lru->pack_last_used)
+ if (!lru || p->windows->last_used < lru->windows->last_used)
lru = p;
}
if (!lru)
return 0;
- munmap(lru->pack_base, lru->pack_size);
- lru->pack_base = NULL;
+ munmap(lru->windows->base, lru->windows->len);
+ free(lru->windows);
+ lru->windows = NULL;
return 1;
}
void unuse_packed_git(struct packed_git *p)
{
- p->pack_use_cnt--;
+ if (p->windows)
+ p->windows->inuse_cnt--;
}
int use_packed_git(struct packed_git *p)
@@ -482,10 +484,10 @@
die("packfile %s not a regular file", p->pack_name);
p->pack_size = st.st_size;
}
- if (!p->pack_base) {
+ if (!p->windows) {
int fd;
struct stat st;
- void *map;
+ struct pack_window *win;
struct pack_header *hdr;
pack_mapped += p->pack_size;
@@ -500,16 +502,18 @@
}
if (st.st_size != p->pack_size)
die("packfile %s size mismatch.", p->pack_name);
- map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ win = xcalloc(1, sizeof(*win));
+ win->len = p->pack_size;
+ win->base = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
close(fd);
- if (map == MAP_FAILED)
+ if (win->base == MAP_FAILED)
die("packfile %s cannot be mapped.", p->pack_name);
- p->pack_base = map;
+ p->windows = win;
/* Check if we understand this pack file. If we don't we're
* likely too old to handle it.
*/
- hdr = map;
+ hdr = (struct pack_header*)win->base;
if (hdr->hdr_signature != htonl(PACK_SIGNATURE))
die("packfile %s isn't actually a pack.", p->pack_name);
if (!pack_version_ok(hdr->hdr_version))
@@ -522,13 +526,13 @@
*/
if (hashcmp((unsigned char *)(p->index_base) +
p->index_size - 40,
- (unsigned char *)p->pack_base +
+ p->windows->base +
p->pack_size - 20)) {
die("packfile %s does not match index.", p->pack_name);
}
}
- p->pack_last_used = pack_used_ctr++;
- p->pack_use_cnt++;
+ p->windows->last_used = pack_used_ctr++;
+ p->windows->inuse_cnt++;
return 0;
}
@@ -558,9 +562,7 @@
p->pack_size = st.st_size;
p->index_base = idx_map;
p->next = NULL;
- p->pack_base = NULL;
- p->pack_last_used = 0;
- p->pack_use_cnt = 0;
+ p->windows = NULL;
p->pack_local = local;
if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1))
hashcpy(p->sha1, sha1);
@@ -591,9 +593,7 @@
p->pack_size = 0;
p->index_base = idx_map;
p->next = NULL;
- p->pack_base = NULL;
- p->pack_last_used = 0;
- p->pack_use_cnt = 0;
+ p->windows = NULL;
hashcpy(p->sha1, sha1);
return p;
}
@@ -882,7 +882,7 @@
unsigned long delta_obj_offset,
unsigned long *base_obj_offset)
{
- unsigned char *base_info = (unsigned char *) p->pack_base + offset;
+ unsigned char *base_info = p->windows->base + offset;
unsigned long base_offset;
/* there must be at least 20 bytes left regardless of delta type */
@@ -949,7 +949,7 @@
memset(&stream, 0, sizeof(stream));
- stream.next_in = (unsigned char *) p->pack_base + offset;
+ stream.next_in = p->windows->base + offset;
stream.avail_in = p->pack_size - offset;
stream.next_out = delta_head;
stream.avail_out = sizeof(delta_head);
@@ -984,8 +984,7 @@
if (p->pack_size <= offset)
die("object offset outside of pack file");
- used = unpack_object_header_gently((unsigned char *)p->pack_base +
- offset,
+ used = unpack_object_header_gently(p->windows->base + offset,
p->pack_size - offset, type, sizep);
if (!used)
die("object offset outside of pack file");
@@ -1031,7 +1030,7 @@
if (p->pack_size <= offset + 20)
die("pack file %s records an incomplete delta base",
p->pack_name);
- next_sha1 = (unsigned char *) p->pack_base + offset;
+ next_sha1 = p->windows->base + offset;
if (*delta_chain_length == 0)
hashcpy(base_sha1, next_sha1);
offset = find_pack_entry_one(next_sha1, p);
@@ -1081,7 +1080,7 @@
buffer = xmalloc(size + 1);
buffer[size] = 0;
memset(&stream, 0, sizeof(stream));
- stream.next_in = (unsigned char*)p->pack_base + offset;
+ stream.next_in = p->windows->base + offset;
stream.avail_in = p->pack_size - offset;
stream.next_out = buffer;
stream.avail_out = size;
