Merge branch 'ab/commit-graph-write-progress'
The codepath to show progress meter while writing out commit-graph
file has been improved.
* ab/commit-graph-write-progress:
commit-graph write: emit a percentage for all progress
commit-graph write: add itermediate progress
commit-graph write: remove empty line for readability
commit-graph write: add more descriptive progress output
commit-graph write: show progress for object search
commit-graph write: more descriptive "writing out" output
commit-graph write: add "Writing out" progress output
commit-graph: don't call write_graph_chunk_extra_edges() unnecessarily
commit-graph: rename "large edges" to "extra edges"
diff --git a/Documentation/technical/commit-graph-format.txt b/Documentation/technical/commit-graph-format.txt
index cc0474b..16452a0 100644
--- a/Documentation/technical/commit-graph-format.txt
+++ b/Documentation/technical/commit-graph-format.txt
@@ -76,7 +76,7 @@
of the ith commit. Stores value 0x7000000 if no parent in that
position. If there are more than two parents, the second value
has its most-significant bit on and the other bits store an array
- position into the Large Edge List chunk.
+ position into the Extra Edge List chunk.
* The next 8 bytes store the generation number of the commit and
the commit time in seconds since EPOCH. The generation number
uses the higher 30 bits of the first 4 bytes, while the commit
@@ -84,7 +84,7 @@
2 bits of the lowest byte, storing the 33rd and 34th bit of the
commit time.
- Large Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional]
+ Extra Edge List (ID: {'E', 'D', 'G', 'E'}) [Optional]
This list of 4-byte values store the second through nth parents for
all octopus merges. The second parent value in the commit data stores
an array position within this list along with the most-significant bit
diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c
index c02a3f1..4ae5027 100644
--- a/builtin/commit-graph.c
+++ b/builtin/commit-graph.c
@@ -110,8 +110,8 @@
printf(" oid_lookup");
if (graph->chunk_commit_data)
printf(" commit_metadata");
- if (graph->chunk_large_edges)
- printf(" large_edges");
+ if (graph->chunk_extra_edges)
+ printf(" extra_edges");
printf("\n");
UNLEAK(graph);
diff --git a/commit-graph.c b/commit-graph.c
index 8d52727..47e9be0 100644
--- a/commit-graph.c
+++ b/commit-graph.c
@@ -21,14 +21,14 @@
#define GRAPH_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
#define GRAPH_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
#define GRAPH_CHUNKID_DATA 0x43444154 /* "CDAT" */
-#define GRAPH_CHUNKID_LARGEEDGES 0x45444745 /* "EDGE" */
+#define GRAPH_CHUNKID_EXTRAEDGES 0x45444745 /* "EDGE" */
#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
#define GRAPH_VERSION_1 0x1
#define GRAPH_VERSION GRAPH_VERSION_1
-#define GRAPH_OCTOPUS_EDGES_NEEDED 0x80000000
+#define GRAPH_EXTRA_EDGES_NEEDED 0x80000000
#define GRAPH_EDGE_LAST_MASK 0x7fffffff
#define GRAPH_PARENT_NONE 0x70000000
@@ -209,11 +209,11 @@
graph->chunk_commit_data = data + chunk_offset;
break;
- case GRAPH_CHUNKID_LARGEEDGES:
- if (graph->chunk_large_edges)
+ case GRAPH_CHUNKID_EXTRAEDGES:
+ if (graph->chunk_extra_edges)
chunk_repeated = 1;
else
- graph->chunk_large_edges = data + chunk_offset;
+ graph->chunk_extra_edges = data + chunk_offset;
break;
}
@@ -374,12 +374,12 @@
edge_value = get_be32(commit_data + g->hash_len + 4);
if (edge_value == GRAPH_PARENT_NONE)
return 1;
- if (!(edge_value & GRAPH_OCTOPUS_EDGES_NEEDED)) {
+ if (!(edge_value & GRAPH_EXTRA_EDGES_NEEDED)) {
pptr = insert_parent_or_die(r, g, edge_value, pptr);
return 1;
}
- parent_data_ptr = (uint32_t*)(g->chunk_large_edges +
+ parent_data_ptr = (uint32_t*)(g->chunk_extra_edges +
4 * (uint64_t)(edge_value & GRAPH_EDGE_LAST_MASK));
do {
edge_value = get_be32(parent_data_ptr);
@@ -466,7 +466,9 @@
static void write_graph_chunk_fanout(struct hashfile *f,
struct commit **commits,
- int nr_commits)
+ int nr_commits,
+ struct progress *progress,
+ uint64_t *progress_cnt)
{
int i, count = 0;
struct commit **list = commits;
@@ -480,6 +482,7 @@
while (count < nr_commits) {
if ((*list)->object.oid.hash[0] != i)
break;
+ display_progress(progress, ++*progress_cnt);
count++;
list++;
}
@@ -489,12 +492,16 @@
}
static void write_graph_chunk_oids(struct hashfile *f, int hash_len,
- struct commit **commits, int nr_commits)
+ struct commit **commits, int nr_commits,
+ struct progress *progress,
+ uint64_t *progress_cnt)
{
struct commit **list = commits;
int count;
- for (count = 0; count < nr_commits; count++, list++)
+ for (count = 0; count < nr_commits; count++, list++) {
+ display_progress(progress, ++*progress_cnt);
hashwrite(f, (*list)->object.oid.hash, (int)hash_len);
+ }
}
static const unsigned char *commit_to_sha1(size_t index, void *table)
@@ -504,7 +511,9 @@
}
static void write_graph_chunk_data(struct hashfile *f, int hash_len,
- struct commit **commits, int nr_commits)
+ struct commit **commits, int nr_commits,
+ struct progress *progress,
+ uint64_t *progress_cnt)
{
struct commit **list = commits;
struct commit **last = commits + nr_commits;
@@ -514,6 +523,7 @@
struct commit_list *parent;
int edge_value;
uint32_t packedDate[2];
+ display_progress(progress, ++*progress_cnt);
parse_commit(*list);
hashwrite(f, get_commit_tree_oid(*list)->hash, hash_len);
@@ -542,7 +552,7 @@
if (!parent)
edge_value = GRAPH_PARENT_NONE;
else if (parent->next)
- edge_value = GRAPH_OCTOPUS_EDGES_NEEDED | num_extra_edges;
+ edge_value = GRAPH_EXTRA_EDGES_NEEDED | num_extra_edges;
else {
edge_value = sha1_pos(parent->item->object.oid.hash,
commits,
@@ -556,7 +566,7 @@
hashwrite_be32(f, edge_value);
- if (edge_value & GRAPH_OCTOPUS_EDGES_NEEDED) {
+ if (edge_value & GRAPH_EXTRA_EDGES_NEEDED) {
do {
num_extra_edges++;
parent = parent->next;
@@ -577,9 +587,11 @@
}
}
-static void write_graph_chunk_large_edges(struct hashfile *f,
+static void write_graph_chunk_extra_edges(struct hashfile *f,
struct commit **commits,
- int nr_commits)
+ int nr_commits,
+ struct progress *progress,
+ uint64_t *progress_cnt)
{
struct commit **list = commits;
struct commit **last = commits + nr_commits;
@@ -587,6 +599,9 @@
while (list < last) {
int num_parents = 0;
+
+ display_progress(progress, ++*progress_cnt);
+
for (parent = (*list)->parents; num_parents < 3 && parent;
parent = parent->next)
num_parents++;
@@ -680,15 +695,15 @@
static void close_reachable(struct packed_oid_list *oids, int report_progress)
{
- int i, j;
+ int i;
struct commit *commit;
struct progress *progress = NULL;
if (report_progress)
progress = start_delayed_progress(
- _("Loading known commits in commit graph"), j = 0);
+ _("Loading known commits in commit graph"), oids->nr);
for (i = 0; i < oids->nr; i++) {
- display_progress(progress, ++j);
+ display_progress(progress, i + 1);
commit = lookup_commit(the_repository, &oids->list[i]);
if (commit)
commit->object.flags |= UNINTERESTING;
@@ -702,9 +717,9 @@
*/
if (report_progress)
progress = start_delayed_progress(
- _("Expanding reachable commits in commit graph"), j = 0);
+ _("Expanding reachable commits in commit graph"), oids->nr);
for (i = 0; i < oids->nr; i++) {
- display_progress(progress, ++j);
+ display_progress(progress, i + 1);
commit = lookup_commit(the_repository, &oids->list[i]);
if (commit && !parse_commit(commit))
@@ -714,9 +729,9 @@
if (report_progress)
progress = start_delayed_progress(
- _("Clearing commit marks in commit graph"), j = 0);
+ _("Clearing commit marks in commit graph"), oids->nr);
for (i = 0; i < oids->nr; i++) {
- display_progress(progress, ++j);
+ display_progress(progress, i + 1);
commit = lookup_commit(the_repository, &oids->list[i]);
if (commit)
@@ -811,12 +826,16 @@
struct commit_list *parent;
struct progress *progress = NULL;
const unsigned hashsz = the_hash_algo->rawsz;
+ uint64_t progress_cnt = 0;
+ struct strbuf progress_title = STRBUF_INIT;
+ unsigned long approx_nr_objects;
if (!commit_graph_compatible(the_repository))
return;
oids.nr = 0;
- oids.alloc = approximate_object_count() / 32;
+ approx_nr_objects = approximate_object_count();
+ oids.alloc = approx_nr_objects / 32;
oids.progress = NULL;
oids.progress_done = 0;
@@ -846,8 +865,12 @@
strbuf_addf(&packname, "%s/pack/", obj_dir);
dirlen = packname.len;
if (report_progress) {
- oids.progress = start_delayed_progress(
- _("Finding commits for commit graph"), 0);
+ strbuf_addf(&progress_title,
+ Q_("Finding commits for commit graph in %d pack",
+ "Finding commits for commit graph in %d packs",
+ pack_indexes->nr),
+ pack_indexes->nr);
+ oids.progress = start_delayed_progress(progress_title.buf, 0);
oids.progress_done = 0;
}
for (i = 0; i < pack_indexes->nr; i++) {
@@ -865,14 +888,20 @@
free(p);
}
stop_progress(&oids.progress);
+ strbuf_reset(&progress_title);
strbuf_release(&packname);
}
if (commit_hex) {
- if (report_progress)
- progress = start_delayed_progress(
- _("Finding commits for commit graph"),
- commit_hex->nr);
+ if (report_progress) {
+ strbuf_addf(&progress_title,
+ Q_("Finding commits for commit graph from %d ref",
+ "Finding commits for commit graph from %d refs",
+ commit_hex->nr),
+ commit_hex->nr);
+ progress = start_delayed_progress(progress_title.buf,
+ commit_hex->nr);
+ }
for (i = 0; i < commit_hex->nr; i++) {
const char *end;
struct object_id oid;
@@ -892,26 +921,36 @@
}
}
stop_progress(&progress);
+ strbuf_reset(&progress_title);
}
if (!pack_indexes && !commit_hex) {
if (report_progress)
oids.progress = start_delayed_progress(
- _("Finding commits for commit graph"), 0);
+ _("Finding commits for commit graph among packed objects"),
+ approx_nr_objects);
for_each_packed_object(add_packed_commits, &oids,
FOR_EACH_OBJECT_PACK_ORDER);
+ if (oids.progress_done < approx_nr_objects)
+ display_progress(oids.progress, approx_nr_objects);
stop_progress(&oids.progress);
}
close_reachable(&oids, report_progress);
+ if (report_progress)
+ progress = start_delayed_progress(
+ _("Counting distinct commits in commit graph"),
+ oids.nr);
+ display_progress(progress, 0); /* TODO: Measure QSORT() progress */
QSORT(oids.list, oids.nr, commit_compare);
-
count_distinct = 1;
for (i = 1; i < oids.nr; i++) {
+ display_progress(progress, i + 1);
if (!oideq(&oids.list[i - 1], &oids.list[i]))
count_distinct++;
}
+ stop_progress(&progress);
if (count_distinct >= GRAPH_EDGE_LAST_MASK)
die(_("the commit graph format cannot write %d commits"), count_distinct);
@@ -921,8 +960,13 @@
ALLOC_ARRAY(commits.list, commits.alloc);
num_extra_edges = 0;
+ if (report_progress)
+ progress = start_delayed_progress(
+ _("Finding extra edges in commit graph"),
+ oids.nr);
for (i = 0; i < oids.nr; i++) {
int num_parents = 0;
+ display_progress(progress, i + 1);
if (i > 0 && oideq(&oids.list[i - 1], &oids.list[i]))
continue;
@@ -939,6 +983,7 @@
commits.nr++;
}
num_chunks = num_extra_edges ? 4 : 3;
+ stop_progress(&progress);
if (commits.nr >= GRAPH_EDGE_LAST_MASK)
die(_("too many commits to write graph"));
@@ -966,7 +1011,7 @@
chunk_ids[1] = GRAPH_CHUNKID_OIDLOOKUP;
chunk_ids[2] = GRAPH_CHUNKID_DATA;
if (num_extra_edges)
- chunk_ids[3] = GRAPH_CHUNKID_LARGEEDGES;
+ chunk_ids[3] = GRAPH_CHUNKID_EXTRAEDGES;
else
chunk_ids[3] = 0;
chunk_ids[4] = 0;
@@ -986,10 +1031,23 @@
hashwrite(f, chunk_write, 12);
}
- write_graph_chunk_fanout(f, commits.list, commits.nr);
- write_graph_chunk_oids(f, hashsz, commits.list, commits.nr);
- write_graph_chunk_data(f, hashsz, commits.list, commits.nr);
- write_graph_chunk_large_edges(f, commits.list, commits.nr);
+ if (report_progress) {
+ strbuf_addf(&progress_title,
+ Q_("Writing out commit graph in %d pass",
+ "Writing out commit graph in %d passes",
+ num_chunks),
+ num_chunks);
+ progress = start_delayed_progress(
+ progress_title.buf,
+ num_chunks * commits.nr);
+ }
+ write_graph_chunk_fanout(f, commits.list, commits.nr, progress, &progress_cnt);
+ write_graph_chunk_oids(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
+ write_graph_chunk_data(f, hashsz, commits.list, commits.nr, progress, &progress_cnt);
+ if (num_extra_edges)
+ write_graph_chunk_extra_edges(f, commits.list, commits.nr, progress, &progress_cnt);
+ stop_progress(&progress);
+ strbuf_release(&progress_title);
close_commit_graph(the_repository);
finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
diff --git a/commit-graph.h b/commit-graph.h
index 813e7c1..096d8ba 100644
--- a/commit-graph.h
+++ b/commit-graph.h
@@ -49,7 +49,7 @@
const uint32_t *chunk_oid_fanout;
const unsigned char *chunk_oid_lookup;
const unsigned char *chunk_commit_data;
- const unsigned char *chunk_large_edges;
+ const unsigned char *chunk_extra_edges;
};
struct commit_graph *load_commit_graph_one(const char *graph_file);
diff --git a/t/t5318-commit-graph.sh b/t/t5318-commit-graph.sh
index 694f260..16d10eb 100755
--- a/t/t5318-commit-graph.sh
+++ b/t/t5318-commit-graph.sh
@@ -122,7 +122,7 @@
cd "$TRASH_DIRECTORY/full" &&
git commit-graph write &&
test_path_is_file $objdir/info/commit-graph &&
- graph_read_expect "10" "large_edges"
+ graph_read_expect "10" "extra_edges"
'
graph_git_behavior 'merge 1 vs 2' full merge/1 merge/2
@@ -157,7 +157,7 @@
cd "$TRASH_DIRECTORY/full" &&
git commit-graph write &&
test_path_is_file $objdir/info/commit-graph &&
- graph_read_expect "11" "large_edges"
+ graph_read_expect "11" "extra_edges"
'
graph_git_behavior 'full graph, commit 8 vs merge 1' full commits/8 merge/1
@@ -167,7 +167,7 @@
cd "$TRASH_DIRECTORY/full" &&
git commit-graph write &&
test_path_is_file $objdir/info/commit-graph &&
- graph_read_expect "11" "large_edges"
+ graph_read_expect "11" "extra_edges"
'
graph_git_behavior 'cleared graph, commit 8 vs merge 1' full commits/8 merge/1
@@ -177,7 +177,7 @@
cd "$TRASH_DIRECTORY/full" &&
cat new-idx | git commit-graph write --stdin-packs &&
test_path_is_file $objdir/info/commit-graph &&
- graph_read_expect "9" "large_edges"
+ graph_read_expect "9" "extra_edges"
'
graph_git_behavior 'graph from pack, commit 8 vs merge 1' full commits/8 merge/1
@@ -200,7 +200,7 @@
cd "$TRASH_DIRECTORY/full" &&
git rev-parse merge/3 | git commit-graph write --stdin-commits --append &&
test_path_is_file $objdir/info/commit-graph &&
- graph_read_expect "10" "large_edges"
+ graph_read_expect "10" "extra_edges"
'
graph_git_behavior 'append graph, commit 8 vs merge 1' full commits/8 merge/1
@@ -210,7 +210,7 @@
cd "$TRASH_DIRECTORY/full" &&
git commit-graph write --reachable &&
test_path_is_file $objdir/info/commit-graph &&
- graph_read_expect "11" "large_edges"
+ graph_read_expect "11" "extra_edges"
'
graph_git_behavior 'append graph, commit 8 vs merge 1' full commits/8 merge/1
@@ -231,7 +231,7 @@
cd "$TRASH_DIRECTORY/bare" &&
git commit-graph write &&
test_path_is_file $baredir/info/commit-graph &&
- graph_read_expect "11" "large_edges"
+ graph_read_expect "11" "extra_edges"
'
graph_git_behavior 'bare repo with graph, commit 8 vs merge 1' bare commits/8 merge/1
