blob: 9bc8114c3bb2f6f87f7f61962a64c4e5b012dcf8 [file] [log] [blame]
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001#include "cache.h"
2#include "commit.h"
Daniel Barkalow30ae7642007-09-10 23:02:45 -04003#include "walker.h"
Nick Hengeveld29508e12005-11-18 11:02:58 -08004#include "http.h"
Johannes Schindelin7baa3e82005-10-15 11:10:46 -07005
Jonathan Nieder9cba13c2011-03-16 02:08:34 -05006struct alt_base {
Gerrit Pape2afea3b2007-03-28 09:46:15 +00007 char *base;
Daniel Barkalowb3661562005-09-14 23:26:08 -04008 int got_indices;
9 struct packed_git *packs;
10 struct alt_base *next;
11};
12
Nick Hengevelde388ab72005-11-18 11:03:04 -080013enum object_request_state {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070014 WAITING,
15 ABORTED,
16 ACTIVE,
Gary V. Vaughan4b055482010-05-14 09:31:35 +000017 COMPLETE
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070018};
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070019
Jonathan Nieder9cba13c2011-03-16 02:08:34 -050020struct object_request {
Daniel Barkalow30ae7642007-09-10 23:02:45 -040021 struct walker *walker;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070022 unsigned char sha1[20];
23 struct alt_base *repo;
Nick Hengevelde388ab72005-11-18 11:03:04 -080024 enum object_request_state state;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080025 struct http_object_request *req;
Nick Hengevelde388ab72005-11-18 11:03:04 -080026 struct object_request *next;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070027};
28
Nick Hengevelde388ab72005-11-18 11:03:04 -080029struct alternates_request {
Daniel Barkalow30ae7642007-09-10 23:02:45 -040030 struct walker *walker;
Petr Baudis8e29f6a2006-07-27 23:56:22 +020031 const char *base;
Nick Hengeveldacc075a2005-11-12 09:11:32 -080032 char *url;
Mike Hommey028c2972007-12-09 20:30:59 +010033 struct strbuf *buffer;
Nick Hengeveldacc075a2005-11-12 09:11:32 -080034 struct active_request_slot *slot;
35 int http_specific;
36};
37
Daniel Barkalow30ae7642007-09-10 23:02:45 -040038struct walker_data {
39 const char *url;
40 int got_alternates;
41 struct alt_base *alt;
Daniel Barkalow30ae7642007-09-10 23:02:45 -040042};
43
David Rientjes96f1e582006-08-15 10:23:48 -070044static struct object_request *object_queue_head;
Sergey Vlasovbc8f2652005-10-13 10:49:53 -070045
Daniel Barkalow30ae7642007-09-10 23:02:45 -040046static void fetch_alternates(struct walker *walker, const char *base);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070047
Nick Hengeveld29508e12005-11-18 11:02:58 -080048static void process_object_response(void *callback_data);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070049
Daniel Barkalow30ae7642007-09-10 23:02:45 -040050static void start_object_request(struct walker *walker,
51 struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070052{
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070053 struct active_request_slot *slot;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080054 struct http_object_request *req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070055
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080056 req = new_http_object_request(obj_req->repo->base, obj_req->sha1);
57 if (req == NULL) {
Nick Hengevelde388ab72005-11-18 11:03:04 -080058 obj_req->state = ABORTED;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070059 return;
60 }
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080061 obj_req->req = req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070062
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080063 slot = req->slot;
Nick Hengeveld29508e12005-11-18 11:02:58 -080064 slot->callback_func = process_object_response;
Nick Hengevelde388ab72005-11-18 11:03:04 -080065 slot->callback_data = obj_req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070066
Nick Hengevelda7a8d372005-10-10 23:22:01 -070067 /* Try to get the request started, abort the request on error */
Nick Hengevelde388ab72005-11-18 11:03:04 -080068 obj_req->state = ACTIVE;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070069 if (!start_active_slot(slot)) {
Nick Hengevelde388ab72005-11-18 11:03:04 -080070 obj_req->state = ABORTED;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080071 release_http_object_request(req);
Nick Hengevelde388ab72005-11-18 11:03:04 -080072 return;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070073 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070074}
75
Nick Hengevelde388ab72005-11-18 11:03:04 -080076static void finish_object_request(struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070077{
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080078 if (finish_http_object_request(obj_req->req))
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070079 return;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070080
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080081 if (obj_req->req->rename == 0)
Daniel Barkalow30ae7642007-09-10 23:02:45 -040082 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070083}
84
Nick Hengeveld29508e12005-11-18 11:02:58 -080085static void process_object_response(void *callback_data)
86{
Nick Hengevelde388ab72005-11-18 11:03:04 -080087 struct object_request *obj_req =
88 (struct object_request *)callback_data;
Daniel Barkalow30ae7642007-09-10 23:02:45 -040089 struct walker *walker = obj_req->walker;
90 struct walker_data *data = walker->data;
91 struct alt_base *alt = data->alt;
Nick Hengeveld29508e12005-11-18 11:02:58 -080092
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080093 process_http_object_request(obj_req->req);
Nick Hengevelde388ab72005-11-18 11:03:04 -080094 obj_req->state = COMPLETE;
Nick Hengeveld29508e12005-11-18 11:02:58 -080095
96 /* Use alternates if necessary */
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080097 if (missing_target(obj_req->req)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -040098 fetch_alternates(walker, alt->base);
Nick Hengevelde388ab72005-11-18 11:03:04 -080099 if (obj_req->repo->next != NULL) {
100 obj_req->repo =
101 obj_req->repo->next;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800102 release_http_object_request(obj_req->req);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400103 start_object_request(walker, obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800104 return;
105 }
106 }
107
Nick Hengevelde388ab72005-11-18 11:03:04 -0800108 finish_object_request(obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800109}
110
Nick Hengevelde388ab72005-11-18 11:03:04 -0800111static void release_object_request(struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700112{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800113 struct object_request *entry = object_queue_head;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700114
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800115 if (obj_req->req !=NULL && obj_req->req->localfile != -1)
116 error("fd leakage in release: %d", obj_req->req->localfile);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800117 if (obj_req == object_queue_head) {
118 object_queue_head = obj_req->next;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700119 } else {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800120 while (entry->next != NULL && entry->next != obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700121 entry = entry->next;
Nick Hengevelde388ab72005-11-18 11:03:04 -0800122 if (entry->next == obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700123 entry->next = entry->next->next;
124 }
125
Nick Hengevelde388ab72005-11-18 11:03:04 -0800126 free(obj_req);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700127}
128
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700129#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400130static int fill_active_slot(struct walker *walker)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700131{
Daniel Barkalow45c17412007-09-10 23:02:28 -0400132 struct object_request *obj_req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700133
Daniel Barkalow45c17412007-09-10 23:02:28 -0400134 for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800135 if (obj_req->state == WAITING) {
136 if (has_sha1_file(obj_req->sha1))
Mark Wooding09db4442006-02-01 11:44:28 +0000137 obj_req->state = COMPLETE;
Daniel Barkalow45c17412007-09-10 23:02:28 -0400138 else {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400139 start_object_request(walker, obj_req);
Daniel Barkalow45c17412007-09-10 23:02:28 -0400140 return 1;
141 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700142 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700143 }
Daniel Barkalow45c17412007-09-10 23:02:28 -0400144 return 0;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700145}
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700146#endif
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700147
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400148static void prefetch(struct walker *walker, unsigned char *sha1)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700149{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800150 struct object_request *newreq;
151 struct object_request *tail;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400152 struct walker_data *data = walker->data;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700153
154 newreq = xmalloc(sizeof(*newreq));
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400155 newreq->walker = walker;
Shawn Pearcee7024962006-08-23 02:49:00 -0400156 hashcpy(newreq->sha1, sha1);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400157 newreq->repo = data->alt;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700158 newreq->state = WAITING;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800159 newreq->req = NULL;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700160 newreq->next = NULL;
161
Tay Ray Chuane9176742009-06-06 16:43:41 +0800162 http_is_verbose = walker->get_verbosely;
163
Nick Hengevelde388ab72005-11-18 11:03:04 -0800164 if (object_queue_head == NULL) {
165 object_queue_head = newreq;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700166 } else {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800167 tail = object_queue_head;
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800168 while (tail->next != NULL)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700169 tail = tail->next;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700170 tail->next = newreq;
171 }
Nick Hengeveld29508e12005-11-18 11:02:58 -0800172
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700173#ifdef USE_CURL_MULTI
Nick Hengeveld29508e12005-11-18 11:02:58 -0800174 fill_active_slots();
175 step_active_slots();
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700176#endif
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700177}
178
Nick Hengevelde388ab72005-11-18 11:03:04 -0800179static void process_alternates_response(void *callback_data)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400180{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800181 struct alternates_request *alt_req =
182 (struct alternates_request *)callback_data;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400183 struct walker *walker = alt_req->walker;
184 struct walker_data *cdata = walker->data;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800185 struct active_request_slot *slot = alt_req->slot;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400186 struct alt_base *tail = cdata->alt;
Petr Baudis8e29f6a2006-07-27 23:56:22 +0200187 const char *base = alt_req->base;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800188 static const char null_byte = '\0';
Daniel Barkalowb3661562005-09-14 23:26:08 -0400189 char *data;
190 int i = 0;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700191
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800192 if (alt_req->http_specific) {
193 if (slot->curl_result != CURLE_OK ||
Mike Hommey028c2972007-12-09 20:30:59 +0100194 !alt_req->buffer->len) {
Junio C Hamanodc1b5ea2005-10-10 23:22:02 -0700195
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800196 /* Try reusing the slot to get non-http alternates */
197 alt_req->http_specific = 0;
198 sprintf(alt_req->url, "%s/objects/info/alternates",
199 base);
200 curl_easy_setopt(slot->curl, CURLOPT_URL,
201 alt_req->url);
202 active_requests++;
203 slot->in_use = 1;
Nick Hengeveldc9826472006-03-15 08:59:52 -0800204 if (slot->finished != NULL)
205 (*slot->finished) = 0;
Mark Woodinga3f583c2006-02-01 11:44:39 +0000206 if (!start_active_slot(slot)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400207 cdata->got_alternates = -1;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800208 slot->in_use = 0;
Nick Hengeveldc9826472006-03-15 08:59:52 -0800209 if (slot->finished != NULL)
210 (*slot->finished) = 1;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700211 }
Mark Woodinga3f583c2006-02-01 11:44:39 +0000212 return;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400213 }
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800214 } else if (slot->curl_result != CURLE_OK) {
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700215 if (!missing_target(slot)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400216 cdata->got_alternates = -1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800217 return;
218 }
Daniel Barkalowb3661562005-09-14 23:26:08 -0400219 }
220
Nick Hengeveld29508e12005-11-18 11:02:58 -0800221 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
Mike Hommey028c2972007-12-09 20:30:59 +0100222 alt_req->buffer->len--;
223 data = alt_req->buffer->buf;
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400224
Mike Hommey028c2972007-12-09 20:30:59 +0100225 while (i < alt_req->buffer->len) {
Daniel Barkalowb3661562005-09-14 23:26:08 -0400226 int posn = i;
Mike Hommey028c2972007-12-09 20:30:59 +0100227 while (posn < alt_req->buffer->len && data[posn] != '\n')
Daniel Barkalowb3661562005-09-14 23:26:08 -0400228 posn++;
229 if (data[posn] == '\n') {
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400230 int okay = 0;
231 int serverlen = 0;
232 struct alt_base *newalt;
233 char *target = NULL;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400234 if (data[i] == '/') {
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800235 /*
236 * This counts
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700237 * http://git.host/pub/scm/linux.git/
238 * -----------here^
239 * so memcpy(dst, base, serverlen) will
240 * copy up to "...git.host".
241 */
242 const char *colon_ss = strstr(base,"://");
243 if (colon_ss) {
244 serverlen = (strchr(colon_ss + 3, '/')
245 - base);
246 okay = 1;
247 }
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400248 } else if (!memcmp(data + i, "../", 3)) {
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800249 /*
250 * Relative URL; chop the corresponding
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700251 * number of subpath from base (and ../
252 * from data), and concatenate the result.
253 *
254 * The code first drops ../ from data, and
255 * then drops one ../ from data and one path
256 * from base. IOW, one extra ../ is dropped
257 * from data than path is dropped from base.
258 *
259 * This is not wrong. The alternate in
260 * http://git.host/pub/scm/linux.git/
261 * to borrow from
262 * http://git.host/pub/scm/linus.git/
263 * is ../../linus.git/objects/. You need
264 * two ../../ to borrow from your direct
265 * neighbour.
266 */
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400267 i += 3;
268 serverlen = strlen(base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800269 while (i + 2 < posn &&
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400270 !memcmp(data + i, "../", 3)) {
271 do {
272 serverlen--;
273 } while (serverlen &&
274 base[serverlen - 1] != '/');
275 i += 3;
276 }
Pavel Roskina9486b02006-07-10 02:57:51 -0400277 /* If the server got removed, give up. */
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800278 okay = strchr(base, ':') - base + 3 <
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800279 serverlen;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800280 } else if (alt_req->http_specific) {
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400281 char *colon = strchr(data + i, ':');
282 char *slash = strchr(data + i, '/');
283 if (colon && slash && colon < data + posn &&
284 slash < data + posn && colon < slash) {
285 okay = 1;
286 }
287 }
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700288 /* skip "objects\n" at end */
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400289 if (okay) {
290 target = xmalloc(serverlen + posn - i - 6);
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700291 memcpy(target, base, serverlen);
292 memcpy(target + serverlen, data + i,
293 posn - i - 7);
294 target[serverlen + posn - i - 7] = 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400295 if (walker->get_verbosely)
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800296 fprintf(stderr,
Daniel Barkalowb3661562005-09-14 23:26:08 -0400297 "Also look at %s\n", target);
298 newalt = xmalloc(sizeof(*newalt));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700299 newalt->next = NULL;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400300 newalt->base = target;
301 newalt->got_indices = 0;
302 newalt->packs = NULL;
Nick Hengeveld8d9fbe52006-04-04 05:33:18 -0700303
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700304 while (tail->next != NULL)
305 tail = tail->next;
306 tail->next = newalt;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400307 }
308 }
309 i = posn + 1;
310 }
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700311
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400312 cdata->got_alternates = 1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800313}
314
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400315static void fetch_alternates(struct walker *walker, const char *base)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800316{
Mike Hommey028c2972007-12-09 20:30:59 +0100317 struct strbuf buffer = STRBUF_INIT;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800318 char *url;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800319 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800320 struct alternates_request alt_req;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400321 struct walker_data *cdata = walker->data;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800322
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800323 /*
324 * If another request has already started fetching alternates,
325 * wait for them to arrive and return to processing this request's
326 * curl message
327 */
Nick Hengeveld29508e12005-11-18 11:02:58 -0800328#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400329 while (cdata->got_alternates == 0) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800330 step_active_slots();
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800331 }
Nick Hengeveld29508e12005-11-18 11:02:58 -0800332#endif
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800333
334 /* Nothing to do if they've already been fetched */
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400335 if (cdata->got_alternates == 1)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800336 return;
337
338 /* Start the fetch */
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400339 cdata->got_alternates = 0;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800340
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400341 if (walker->get_verbosely)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800342 fprintf(stderr, "Getting alternates list for %s\n", base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800343
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800344 url = xmalloc(strlen(base) + 31);
345 sprintf(url, "%s/objects/info/http-alternates", base);
346
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800347 /*
348 * Use a callback to process the result, since another request
349 * may fail and need to have alternates loaded before continuing
350 */
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800351 slot = get_active_slot();
Nick Hengevelde388ab72005-11-18 11:03:04 -0800352 slot->callback_func = process_alternates_response;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400353 alt_req.walker = walker;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800354 slot->callback_data = &alt_req;
355
356 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800357 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800358 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
359
360 alt_req.base = base;
361 alt_req.url = url;
362 alt_req.buffer = &buffer;
363 alt_req.http_specific = 1;
364 alt_req.slot = slot;
365
366 if (start_active_slot(slot))
367 run_active_slot(slot);
368 else
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400369 cdata->got_alternates = -1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800370
Mike Hommey028c2972007-12-09 20:30:59 +0100371 strbuf_release(&buffer);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800372 free(url);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400373}
374
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400375static int fetch_indices(struct walker *walker, struct alt_base *repo)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400376{
Tay Ray Chuanb8caac22009-06-06 16:43:59 +0800377 int ret;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700378
Daniel Barkalowb3661562005-09-14 23:26:08 -0400379 if (repo->got_indices)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400380 return 0;
381
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400382 if (walker->get_verbosely)
Petr Baudis6fd72e32005-11-12 01:49:59 +0100383 fprintf(stderr, "Getting pack list for %s\n", repo->base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800384
Tay Ray Chuanb8caac22009-06-06 16:43:59 +0800385 switch (http_get_info_packs(repo->base, &repo->packs)) {
386 case HTTP_OK:
387 case HTTP_MISSING_TARGET:
388 repo->got_indices = 1;
389 ret = 0;
390 break;
391 default:
Nick Hengeveld5e3a7692005-11-18 11:03:11 -0800392 repo->got_indices = 0;
Tay Ray Chuanb8caac22009-06-06 16:43:59 +0800393 ret = -1;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700394 }
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400395
Mike Hommey3a462bc2007-12-10 22:36:11 +0100396 return ret;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400397}
398
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400399static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400400{
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400401 struct packed_git *target;
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700402 int ret;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800403 struct slot_results results;
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800404 struct http_pack_request *preq;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400405
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400406 if (fetch_indices(walker, repo))
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400407 return -1;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400408 target = find_sha1_pack(sha1, repo->packs);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400409 if (!target)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400410 return -1;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400411
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400412 if (walker->get_verbosely) {
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400413 fprintf(stderr, "Getting pack %s\n",
414 sha1_to_hex(target->sha1));
415 fprintf(stderr, " which contains %s\n",
416 sha1_to_hex(sha1));
417 }
418
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800419 preq = new_http_pack_request(target, repo->base);
420 if (preq == NULL)
421 goto abort;
422 preq->lst = &repo->packs;
423 preq->slot->results = &results;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400424
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800425 if (start_active_slot(preq->slot)) {
426 run_active_slot(preq->slot);
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800427 if (results.curl_result != CURLE_OK) {
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800428 error("Unable to get pack file %s\n%s", preq->url,
429 curl_errorstr);
430 goto abort;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700431 }
432 } else {
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800433 error("Unable to start request");
434 goto abort;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400435 }
436
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800437 ret = finish_http_pack_request(preq);
438 release_http_pack_request(preq);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700439 if (ret)
Junio C Hamanob721e012005-10-10 23:22:01 -0700440 return ret;
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700441
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400442 return 0;
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800443
444abort:
445 return -1;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400446}
447
Mark Wooding53f31382006-02-07 10:07:39 +0000448static void abort_object_request(struct object_request *obj_req)
449{
Mark Wooding53f31382006-02-07 10:07:39 +0000450 release_object_request(obj_req);
451}
452
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400453static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700454{
455 char *hex = sha1_to_hex(sha1);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800456 int ret = 0;
Nick Hengevelde388ab72005-11-18 11:03:04 -0800457 struct object_request *obj_req = object_queue_head;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800458 struct http_object_request *req;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700459
David Rientjesa89fccd2006-08-17 11:54:57 -0700460 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
Nick Hengevelde388ab72005-11-18 11:03:04 -0800461 obj_req = obj_req->next;
462 if (obj_req == NULL)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700463 return error("Couldn't find request for %s in the queue", hex);
Junio C Hamano271421c2005-09-30 00:07:39 -0700464
Nick Hengevelde388ab72005-11-18 11:03:04 -0800465 if (has_sha1_file(obj_req->sha1)) {
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800466 if (obj_req->req != NULL)
467 abort_http_object_request(obj_req->req);
Mark Wooding53f31382006-02-07 10:07:39 +0000468 abort_object_request(obj_req);
Nick Hengeveld11f0daf2005-10-10 23:22:01 -0700469 return 0;
470 }
471
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700472#ifdef USE_CURL_MULTI
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800473 while (obj_req->state == WAITING)
Nick Hengeveld29508e12005-11-18 11:02:58 -0800474 step_active_slots();
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700475#else
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400476 start_object_request(walker, obj_req);
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700477#endif
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700478
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800479 /*
480 * obj_req->req might change when fetching alternates in the callback
481 * process_object_response; therefore, the "shortcut" variable, req,
482 * is used only after we're done with slots.
483 */
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800484 while (obj_req->state == ACTIVE)
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800485 run_active_slot(obj_req->req->slot);
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800486
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800487 req = obj_req->req;
488
489 if (req->localfile != -1) {
490 close(req->localfile);
491 req->localfile = -1;
Petr Baudis313c4712005-11-12 00:55:16 +0100492 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700493
Nick Hengevelde388ab72005-11-18 11:03:04 -0800494 if (obj_req->state == ABORTED) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800495 ret = error("Request for %s aborted", hex);
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800496 } else if (req->curl_result != CURLE_OK &&
497 req->http_code != 416) {
498 if (missing_target(req))
Petr Baudise2029eb2005-10-21 18:18:46 +0200499 ret = -1; /* Be silent, it is probably in a pack. */
500 else
501 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800502 req->errorstr, req->curl_result,
503 req->http_code, hex);
504 } else if (req->zret != Z_STREAM_END) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400505 walker->corrupt_object_found++;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800506 ret = error("File %s (%s) corrupt", hex, req->url);
507 } else if (hashcmp(obj_req->sha1, req->real_sha1)) {
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800508 ret = error("File %s has bad hash", hex);
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800509 } else if (req->rename < 0) {
Mark Wooding7b934ec2006-02-01 11:44:35 +0000510 ret = error("unable to write sha1 filename %s",
Shawn O. Pearce0da8b2e2010-04-17 13:07:38 -0700511 sha1_file_name(req->sha1));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700512 }
513
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800514 release_http_object_request(req);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800515 release_object_request(obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800516 return ret;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700517}
518
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400519static int fetch(struct walker *walker, unsigned char *sha1)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400520{
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400521 struct walker_data *data = walker->data;
522 struct alt_base *altbase = data->alt;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700523
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400524 if (!fetch_object(walker, altbase, sha1))
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700525 return 0;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400526 while (altbase) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400527 if (!fetch_pack(walker, altbase, sha1))
Daniel Barkalowb3661562005-09-14 23:26:08 -0400528 return 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400529 fetch_alternates(walker, data->alt->base);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400530 altbase = altbase->next;
531 }
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800532 return error("Unable to find %s under %s", sha1_to_hex(sha1),
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400533 data->alt->base);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400534}
535
Daniel Barkalowc13b2632008-04-26 15:53:09 -0400536static int fetch_ref(struct walker *walker, struct ref *ref)
Daniel Barkalowcd541a62005-06-06 16:38:26 -0400537{
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400538 struct walker_data *data = walker->data;
Daniel Barkalowc13b2632008-04-26 15:53:09 -0400539 return http_fetch_ref(data->alt->base, ref);
Daniel Barkalowcd541a62005-06-06 16:38:26 -0400540}
541
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400542static void cleanup(struct walker *walker)
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700543{
Tay Ray Chuan09ae9ac2010-03-02 18:49:28 +0800544 struct walker_data *data = walker->data;
545 struct alt_base *alt, *alt_next;
546
547 if (data) {
548 alt = data->alt;
549 while (alt) {
550 alt_next = alt->next;
551
552 free(alt->base);
553 free(alt);
554
555 alt = alt_next;
556 }
557 free(data);
558 walker->data = NULL;
559 }
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400560}
561
Tay Ray Chuan888692b2010-03-02 18:49:29 +0800562struct walker *get_http_walker(const char *url)
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400563{
Gerrit Pape9c880b32007-03-28 09:47:35 +0000564 char *s;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400565 struct walker_data *data = xmalloc(sizeof(struct walker_data));
566 struct walker *walker = xmalloc(sizeof(struct walker));
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700567
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400568 data->alt = xmalloc(sizeof(*data->alt));
569 data->alt->base = xmalloc(strlen(url) + 1);
570 strcpy(data->alt->base, url);
571 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
Gerrit Pape9c880b32007-03-28 09:47:35 +0000572 *s = 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400573
574 data->alt->got_indices = 0;
575 data->alt->packs = NULL;
576 data->alt->next = NULL;
577 data->got_alternates = -1;
578
579 walker->corrupt_object_found = 0;
580 walker->fetch = fetch;
581 walker->fetch_ref = fetch_ref;
582 walker->prefetch = prefetch;
583 walker->cleanup = cleanup;
584 walker->data = data;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700585
Daniel Barkalowfc57b6a2007-09-10 23:02:34 -0400586#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400587 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
Daniel Barkalowfc57b6a2007-09-10 23:02:34 -0400588#endif
589
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400590 return walker;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700591}