Merge branch 'it/fetch-pack-many-refs' into maint
When "git fetch" encounters repositories with too many references, the
command line of "fetch-pack" that is run by a helper e.g. remote-curl, may
fail to hold all of them. Now such an internal invocation can feed the
references through the standard input of "fetch-pack".
By Ivan Todoroski
* it/fetch-pack-many-refs:
remote-curl: main test case for the OS command line overflow
fetch-pack: test cases for the new --stdin option
remote-curl: send the refs to fetch-pack on stdin
fetch-pack: new --stdin option to read refs from stdin
Conflicts:
t/t5500-fetch-pack.sh
diff --git a/Documentation/git-fetch-pack.txt b/Documentation/git-fetch-pack.txt
index ed1bdaa..474fa30 100644
--- a/Documentation/git-fetch-pack.txt
+++ b/Documentation/git-fetch-pack.txt
@@ -32,6 +32,16 @@
--all::
Fetch all remote refs.
+--stdin::
+ Take the list of refs from stdin, one per line. If there
+ are refs specified on the command line in addition to this
+ option, then the refs from stdin are processed after those
+ on the command line.
++
+If '--stateless-rpc' is specified together with this option then
+the list of refs must be in packet format (pkt-line). Each ref must
+be in a separate packet, and the list must end with a flush packet.
+
-q::
--quiet::
Pass '-q' flag to 'git unpack-objects'; this makes the
diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c
index 7124c4b..10db15b 100644
--- a/builtin/fetch-pack.c
+++ b/builtin/fetch-pack.c
@@ -23,7 +23,9 @@
};
static const char fetch_pack_usage[] =
-"git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
+"git fetch-pack [--all] [--stdin] [--quiet|-q] [--keep|-k] [--thin] "
+"[--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] "
+"[--no-progress] [-v] [<host>:]<directory> [<refs>...]";
#define COMPLETE (1U << 0)
#define COMMON (1U << 1)
@@ -942,6 +944,10 @@
args.fetch_all = 1;
continue;
}
+ if (!strcmp("--stdin", arg)) {
+ args.stdin_refs = 1;
+ continue;
+ }
if (!strcmp("-v", arg)) {
args.verbose = 1;
continue;
@@ -973,6 +979,40 @@
if (!dest)
usage(fetch_pack_usage);
+ if (args.stdin_refs) {
+ /*
+ * Copy refs from cmdline to new growable list, then
+ * append the refs from the standard input.
+ */
+ int alloc_heads = nr_heads;
+ int size = nr_heads * sizeof(*heads);
+ heads = memcpy(xmalloc(size), heads, size);
+ if (args.stateless_rpc) {
+ /* in stateless RPC mode we use pkt-line to read
+ * from stdin, until we get a flush packet
+ */
+ static char line[1000];
+ for (;;) {
+ int n = packet_read_line(0, line, sizeof(line));
+ if (!n)
+ break;
+ if (line[n-1] == '\n')
+ n--;
+ ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+ heads[nr_heads++] = xmemdupz(line, n);
+ }
+ }
+ else {
+ /* read from stdin one ref per line, until EOF */
+ struct strbuf line = STRBUF_INIT;
+ while (strbuf_getline(&line, stdin, '\n') != EOF) {
+ ALLOC_GROW(heads, nr_heads + 1, alloc_heads);
+ heads[nr_heads++] = strbuf_detach(&line, NULL);
+ }
+ strbuf_release(&line);
+ }
+ }
+
if (args.stateless_rpc) {
conn = NULL;
fd[0] = 0;
diff --git a/fetch-pack.h b/fetch-pack.h
index 0608eda..7c2069c 100644
--- a/fetch-pack.h
+++ b/fetch-pack.h
@@ -10,6 +10,7 @@
lock_pack:1,
use_thin_pack:1,
fetch_all:1,
+ stdin_refs:1,
verbose:1,
no_progress:1,
include_tag:1,
diff --git a/remote-curl.c b/remote-curl.c
index d159fe7..0896221 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -290,6 +290,7 @@
struct rpc_state {
const char *service_name;
const char **argv;
+ struct strbuf *stdin_preamble;
char *service_url;
char *hdr_content_type;
char *hdr_accept;
@@ -535,6 +536,7 @@
{
const char *svc = rpc->service_name;
struct strbuf buf = STRBUF_INIT;
+ struct strbuf *preamble = rpc->stdin_preamble;
struct child_process client;
int err = 0;
@@ -545,6 +547,8 @@
client.argv = rpc->argv;
if (start_command(&client))
exit(1);
+ if (preamble)
+ write_or_die(client.in, preamble->buf, preamble->len);
if (heads)
write_or_die(client.in, heads->buf, heads->len);
@@ -626,13 +630,14 @@
int nr_heads, struct ref **to_fetch)
{
struct rpc_state rpc;
+ struct strbuf preamble = STRBUF_INIT;
char *depth_arg = NULL;
- const char **argv;
int argc = 0, i, err;
+ const char *argv[15];
- argv = xmalloc((15 + nr_heads) * sizeof(char*));
argv[argc++] = "fetch-pack";
argv[argc++] = "--stateless-rpc";
+ argv[argc++] = "--stdin";
argv[argc++] = "--lock-pack";
if (options.followtags)
argv[argc++] = "--include-tag";
@@ -651,24 +656,27 @@
argv[argc++] = depth_arg;
}
argv[argc++] = url;
+ argv[argc++] = NULL;
+
for (i = 0; i < nr_heads; i++) {
struct ref *ref = to_fetch[i];
if (!ref->name || !*ref->name)
die("cannot fetch by sha1 over smart http");
- argv[argc++] = ref->name;
+ packet_buf_write(&preamble, "%s\n", ref->name);
}
- argv[argc++] = NULL;
+ packet_buf_flush(&preamble);
memset(&rpc, 0, sizeof(rpc));
rpc.service_name = "git-upload-pack",
rpc.argv = argv;
+ rpc.stdin_preamble = &preamble;
rpc.gzip_request = 1;
err = rpc_service(&rpc, heads);
if (rpc.result.len)
safe_write(1, rpc.result.buf, rpc.result.len);
strbuf_release(&rpc.result);
- free(argv);
+ strbuf_release(&preamble);
free(depth_arg);
return err;
}
diff --git a/t/t5500-fetch-pack.sh b/t/t5500-fetch-pack.sh
index ce51692..1d1ca98 100755
--- a/t/t5500-fetch-pack.sh
+++ b/t/t5500-fetch-pack.sh
@@ -326,4 +326,70 @@
test_cmp count7.expected count7.actual
'
+test_expect_success 'setup tests for the --stdin parameter' '
+ for head in C D E F
+ do
+ add $head
+ done &&
+ for head in A B C D E F
+ do
+ git tag $head $head
+ done &&
+ cat >input <<-\EOF
+ refs/heads/C
+ refs/heads/A
+ refs/heads/D
+ refs/tags/C
+ refs/heads/B
+ refs/tags/A
+ refs/heads/E
+ refs/tags/B
+ refs/tags/E
+ refs/tags/D
+ EOF
+ sort <input >expect &&
+ (
+ echo refs/heads/E &&
+ echo refs/tags/E &&
+ cat input
+ ) >input.dup
+'
+
+test_expect_success 'fetch refs from cmdline' '
+ (
+ cd client &&
+ git fetch-pack --no-progress .. $(cat ../input)
+ ) >output &&
+ cut -d " " -f 2 <output | sort >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'fetch refs from stdin' '
+ (
+ cd client &&
+ git fetch-pack --stdin --no-progress .. <../input
+ ) >output &&
+ cut -d " " -f 2 <output | sort >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'fetch mixed refs from cmdline and stdin' '
+ (
+ cd client &&
+ tail -n +5 ../input |
+ git fetch-pack --stdin --no-progress .. $(head -n 4 ../input)
+ ) >output &&
+ cut -d " " -f 2 <output | sort >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'test duplicate refs from stdin' '
+ (
+ cd client &&
+ test_must_fail git fetch-pack --stdin --no-progress .. <../input.dup
+ ) >output &&
+ cut -d " " -f 2 <output | sort >actual &&
+ test_cmp expect actual
+'
+
test_done
diff --git a/t/t5551-http-fetch.sh b/t/t5551-http-fetch.sh
index 26d3557..be6094b 100755
--- a/t/t5551-http-fetch.sh
+++ b/t/t5551-http-fetch.sh
@@ -109,5 +109,36 @@
git clone $HTTPD_URL/smart-redir-temp/repo.git --quiet repo-t
'
+test -n "$GIT_TEST_LONG" && test_set_prereq EXPENSIVE
+
+test_expect_success EXPENSIVE 'create 50,000 tags in the repo' '
+ (
+ cd "$HTTPD_DOCUMENT_ROOT_PATH/repo.git" &&
+ for i in `seq 50000`
+ do
+ echo "commit refs/heads/too-many-refs"
+ echo "mark :$i"
+ echo "committer git <git@example.com> $i +0000"
+ echo "data 0"
+ echo "M 644 inline bla.txt"
+ echo "data 4"
+ echo "bla"
+ # make every commit dangling by always
+ # rewinding the branch after each commit
+ echo "reset refs/heads/too-many-refs"
+ echo "from :1"
+ done | git fast-import --export-marks=marks &&
+
+ # now assign tags to all the dangling commits we created above
+ tag=$(perl -e "print \"bla\" x 30") &&
+ sed -e "s/^:\(.\+\) \(.\+\)$/\2 refs\/tags\/$tag-\1/" <marks >>packed-refs
+ )
+'
+
+test_expect_success EXPENSIVE 'clone the 50,000 tag repo to check OS command line overflow' '
+ git clone $HTTPD_URL/smart/repo.git too-many-refs 2>err &&
+ test_line_count = 0 err
+'
+
stop_httpd
test_done
