blob: 4fb1235cd455464ad863a76097197f1c27827509 [file] [log] [blame]
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001#include "cache.h"
Stefan Bellercf78ae42018-03-23 18:21:10 +01002#include "repository.h"
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07003#include "commit.h"
Daniel Barkalow30ae7642007-09-10 23:02:45 -04004#include "walker.h"
Nick Hengeveld29508e12005-11-18 11:02:58 -08005#include "http.h"
Eric Wong94e99012016-07-11 20:51:31 +00006#include "list.h"
Jeff Kingabcbdc02016-12-14 14:39:55 -08007#include "transport.h"
Jonathan Tand6fe0032017-08-18 15:20:34 -07008#include "packfile.h"
Stefan Bellera80d72d2018-03-23 18:20:59 +01009#include "object-store.h"
Johannes Schindelin7baa3e82005-10-15 11:10:46 -070010
Jonathan Nieder9cba13c2011-03-16 02:08:34 -050011struct alt_base {
Gerrit Pape2afea3b2007-03-28 09:46:15 +000012 char *base;
Daniel Barkalowb3661562005-09-14 23:26:08 -040013 int got_indices;
14 struct packed_git *packs;
15 struct alt_base *next;
16};
17
Nick Hengevelde388ab72005-11-18 11:03:04 -080018enum object_request_state {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070019 WAITING,
20 ABORTED,
21 ACTIVE,
Gary V. Vaughan4b055482010-05-14 09:31:35 +000022 COMPLETE
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070023};
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070024
Jonathan Nieder9cba13c2011-03-16 02:08:34 -050025struct object_request {
Daniel Barkalow30ae7642007-09-10 23:02:45 -040026 struct walker *walker;
brian m. carlson16f07052018-03-12 02:27:31 +000027 struct object_id oid;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070028 struct alt_base *repo;
Nick Hengevelde388ab72005-11-18 11:03:04 -080029 enum object_request_state state;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080030 struct http_object_request *req;
Eric Wong94e99012016-07-11 20:51:31 +000031 struct list_head node;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070032};
33
Nick Hengevelde388ab72005-11-18 11:03:04 -080034struct alternates_request {
Daniel Barkalow30ae7642007-09-10 23:02:45 -040035 struct walker *walker;
Petr Baudis8e29f6a2006-07-27 23:56:22 +020036 const char *base;
Jeff King54ba4c52015-09-24 17:07:31 -040037 struct strbuf *url;
Mike Hommey028c2972007-12-09 20:30:59 +010038 struct strbuf *buffer;
Nick Hengeveldacc075a2005-11-12 09:11:32 -080039 struct active_request_slot *slot;
40 int http_specific;
41};
42
Daniel Barkalow30ae7642007-09-10 23:02:45 -040043struct walker_data {
44 const char *url;
45 int got_alternates;
46 struct alt_base *alt;
Daniel Barkalow30ae7642007-09-10 23:02:45 -040047};
48
Eric Wong94e99012016-07-11 20:51:31 +000049static LIST_HEAD(object_queue_head);
Sergey Vlasovbc8f2652005-10-13 10:49:53 -070050
Daniel Barkalow30ae7642007-09-10 23:02:45 -040051static void fetch_alternates(struct walker *walker, const char *base);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070052
Nick Hengeveld29508e12005-11-18 11:02:58 -080053static void process_object_response(void *callback_data);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070054
Daniel Barkalow30ae7642007-09-10 23:02:45 -040055static void start_object_request(struct walker *walker,
56 struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070057{
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070058 struct active_request_slot *slot;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080059 struct http_object_request *req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070060
Jeff Kingf0be0db2019-01-07 03:34:40 -050061 req = new_http_object_request(obj_req->repo->base, &obj_req->oid);
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080062 if (req == NULL) {
Nick Hengevelde388ab72005-11-18 11:03:04 -080063 obj_req->state = ABORTED;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070064 return;
65 }
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080066 obj_req->req = req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070067
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080068 slot = req->slot;
Nick Hengeveld29508e12005-11-18 11:02:58 -080069 slot->callback_func = process_object_response;
Nick Hengevelde388ab72005-11-18 11:03:04 -080070 slot->callback_data = obj_req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070071
Nick Hengevelda7a8d372005-10-10 23:22:01 -070072 /* Try to get the request started, abort the request on error */
Nick Hengevelde388ab72005-11-18 11:03:04 -080073 obj_req->state = ACTIVE;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070074 if (!start_active_slot(slot)) {
Nick Hengevelde388ab72005-11-18 11:03:04 -080075 obj_req->state = ABORTED;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080076 release_http_object_request(req);
Nick Hengevelde388ab72005-11-18 11:03:04 -080077 return;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070078 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070079}
80
Nick Hengevelde388ab72005-11-18 11:03:04 -080081static void finish_object_request(struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070082{
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080083 if (finish_http_object_request(obj_req->req))
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070084 return;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070085
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080086 if (obj_req->req->rename == 0)
brian m. carlson16f07052018-03-12 02:27:31 +000087 walker_say(obj_req->walker, "got %s\n", oid_to_hex(&obj_req->oid));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070088}
89
Nick Hengeveld29508e12005-11-18 11:02:58 -080090static void process_object_response(void *callback_data)
91{
Nick Hengevelde388ab72005-11-18 11:03:04 -080092 struct object_request *obj_req =
93 (struct object_request *)callback_data;
Daniel Barkalow30ae7642007-09-10 23:02:45 -040094 struct walker *walker = obj_req->walker;
95 struct walker_data *data = walker->data;
96 struct alt_base *alt = data->alt;
Nick Hengeveld29508e12005-11-18 11:02:58 -080097
Tay Ray Chuan5424bc52009-06-06 16:44:02 +080098 process_http_object_request(obj_req->req);
Nick Hengevelde388ab72005-11-18 11:03:04 -080099 obj_req->state = COMPLETE;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800100
Jeff Kingccbbd8b2019-03-24 08:09:46 -0400101 normalize_curl_result(&obj_req->req->curl_result,
102 obj_req->req->http_code,
103 obj_req->req->errorstr,
104 sizeof(obj_req->req->errorstr));
105
Nick Hengeveld29508e12005-11-18 11:02:58 -0800106 /* Use alternates if necessary */
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800107 if (missing_target(obj_req->req)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400108 fetch_alternates(walker, alt->base);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800109 if (obj_req->repo->next != NULL) {
110 obj_req->repo =
111 obj_req->repo->next;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800112 release_http_object_request(obj_req->req);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400113 start_object_request(walker, obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800114 return;
115 }
116 }
117
Nick Hengevelde388ab72005-11-18 11:03:04 -0800118 finish_object_request(obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800119}
120
Nick Hengevelde388ab72005-11-18 11:03:04 -0800121static void release_object_request(struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700122{
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800123 if (obj_req->req !=NULL && obj_req->req->localfile != -1)
124 error("fd leakage in release: %d", obj_req->req->localfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700125
Eric Wong94e99012016-07-11 20:51:31 +0000126 list_del(&obj_req->node);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800127 free(obj_req);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700128}
129
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700130#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400131static int fill_active_slot(struct walker *walker)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700132{
Daniel Barkalow45c17412007-09-10 23:02:28 -0400133 struct object_request *obj_req;
Eric Wong94e99012016-07-11 20:51:31 +0000134 struct list_head *pos, *tmp, *head = &object_queue_head;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700135
Eric Wong94e99012016-07-11 20:51:31 +0000136 list_for_each_safe(pos, tmp, head) {
137 obj_req = list_entry(pos, struct object_request, node);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800138 if (obj_req->state == WAITING) {
Jeff King98374a02019-01-07 03:37:54 -0500139 if (has_object_file(&obj_req->oid))
Mark Wooding09db4442006-02-01 11:44:28 +0000140 obj_req->state = COMPLETE;
Daniel Barkalow45c17412007-09-10 23:02:28 -0400141 else {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400142 start_object_request(walker, obj_req);
Daniel Barkalow45c17412007-09-10 23:02:28 -0400143 return 1;
144 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700145 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700146 }
Daniel Barkalow45c17412007-09-10 23:02:28 -0400147 return 0;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700148}
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700149#endif
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700150
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400151static void prefetch(struct walker *walker, unsigned char *sha1)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700152{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800153 struct object_request *newreq;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400154 struct walker_data *data = walker->data;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700155
156 newreq = xmalloc(sizeof(*newreq));
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400157 newreq->walker = walker;
brian m. carlson16f07052018-03-12 02:27:31 +0000158 hashcpy(newreq->oid.hash, sha1);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400159 newreq->repo = data->alt;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700160 newreq->state = WAITING;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800161 newreq->req = NULL;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700162
Tay Ray Chuane9176742009-06-06 16:43:41 +0800163 http_is_verbose = walker->get_verbosely;
Eric Wong94e99012016-07-11 20:51:31 +0000164 list_add_tail(&newreq->node, &object_queue_head);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800165
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700166#ifdef USE_CURL_MULTI
Nick Hengeveld29508e12005-11-18 11:02:58 -0800167 fill_active_slots();
168 step_active_slots();
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700169#endif
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700170}
171
Jeff Kingabcbdc02016-12-14 14:39:55 -0800172static int is_alternate_allowed(const char *url)
173{
174 const char *protocols[] = {
175 "http", "https", "ftp", "ftps"
176 };
177 int i;
178
Eric Wongde461382017-03-04 08:36:45 +0000179 if (http_follow_config != HTTP_FOLLOW_ALWAYS) {
180 warning("alternate disabled by http.followRedirects: %s", url);
181 return 0;
182 }
183
Jeff Kingabcbdc02016-12-14 14:39:55 -0800184 for (i = 0; i < ARRAY_SIZE(protocols); i++) {
185 const char *end;
186 if (skip_prefix(url, protocols[i], &end) &&
187 starts_with(end, "://"))
188 break;
189 }
190
191 if (i >= ARRAY_SIZE(protocols)) {
192 warning("ignoring alternate with unknown protocol: %s", url);
193 return 0;
194 }
195 if (!is_transport_allowed(protocols[i], 0)) {
196 warning("ignoring alternate with restricted protocol: %s", url);
197 return 0;
198 }
199
200 return 1;
201}
202
Nick Hengevelde388ab72005-11-18 11:03:04 -0800203static void process_alternates_response(void *callback_data)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400204{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800205 struct alternates_request *alt_req =
206 (struct alternates_request *)callback_data;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400207 struct walker *walker = alt_req->walker;
208 struct walker_data *cdata = walker->data;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800209 struct active_request_slot *slot = alt_req->slot;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400210 struct alt_base *tail = cdata->alt;
Petr Baudis8e29f6a2006-07-27 23:56:22 +0200211 const char *base = alt_req->base;
Dan McGeea04ff3e2011-05-03 23:47:27 +0800212 const char null_byte = '\0';
Daniel Barkalowb3661562005-09-14 23:26:08 -0400213 char *data;
214 int i = 0;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700215
Jeff Kingccbbd8b2019-03-24 08:09:46 -0400216 normalize_curl_result(&slot->curl_result, slot->http_code,
217 curl_errorstr, sizeof(curl_errorstr));
218
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800219 if (alt_req->http_specific) {
220 if (slot->curl_result != CURLE_OK ||
Mike Hommey028c2972007-12-09 20:30:59 +0100221 !alt_req->buffer->len) {
Junio C Hamanodc1b5ea2005-10-10 23:22:02 -0700222
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800223 /* Try reusing the slot to get non-http alternates */
224 alt_req->http_specific = 0;
Jeff King54ba4c52015-09-24 17:07:31 -0400225 strbuf_reset(alt_req->url);
226 strbuf_addf(alt_req->url, "%s/objects/info/alternates",
227 base);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800228 curl_easy_setopt(slot->curl, CURLOPT_URL,
Jeff King54ba4c52015-09-24 17:07:31 -0400229 alt_req->url->buf);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800230 active_requests++;
231 slot->in_use = 1;
Nick Hengeveldc9826472006-03-15 08:59:52 -0800232 if (slot->finished != NULL)
233 (*slot->finished) = 0;
Mark Woodinga3f583c2006-02-01 11:44:39 +0000234 if (!start_active_slot(slot)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400235 cdata->got_alternates = -1;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800236 slot->in_use = 0;
Nick Hengeveldc9826472006-03-15 08:59:52 -0800237 if (slot->finished != NULL)
238 (*slot->finished) = 1;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700239 }
Mark Woodinga3f583c2006-02-01 11:44:39 +0000240 return;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400241 }
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800242 } else if (slot->curl_result != CURLE_OK) {
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700243 if (!missing_target(slot)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400244 cdata->got_alternates = -1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800245 return;
246 }
Daniel Barkalowb3661562005-09-14 23:26:08 -0400247 }
248
Dan McGeea04ff3e2011-05-03 23:47:27 +0800249 fwrite_buffer((char *)&null_byte, 1, 1, alt_req->buffer);
Mike Hommey028c2972007-12-09 20:30:59 +0100250 alt_req->buffer->len--;
251 data = alt_req->buffer->buf;
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400252
Mike Hommey028c2972007-12-09 20:30:59 +0100253 while (i < alt_req->buffer->len) {
Daniel Barkalowb3661562005-09-14 23:26:08 -0400254 int posn = i;
Mike Hommey028c2972007-12-09 20:30:59 +0100255 while (posn < alt_req->buffer->len && data[posn] != '\n')
Daniel Barkalowb3661562005-09-14 23:26:08 -0400256 posn++;
257 if (data[posn] == '\n') {
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400258 int okay = 0;
259 int serverlen = 0;
260 struct alt_base *newalt;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400261 if (data[i] == '/') {
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800262 /*
263 * This counts
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700264 * http://git.host/pub/scm/linux.git/
265 * -----------here^
266 * so memcpy(dst, base, serverlen) will
267 * copy up to "...git.host".
268 */
269 const char *colon_ss = strstr(base,"://");
270 if (colon_ss) {
271 serverlen = (strchr(colon_ss + 3, '/')
272 - base);
273 okay = 1;
274 }
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400275 } else if (!memcmp(data + i, "../", 3)) {
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800276 /*
277 * Relative URL; chop the corresponding
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700278 * number of subpath from base (and ../
279 * from data), and concatenate the result.
280 *
281 * The code first drops ../ from data, and
282 * then drops one ../ from data and one path
283 * from base. IOW, one extra ../ is dropped
284 * from data than path is dropped from base.
285 *
286 * This is not wrong. The alternate in
287 * http://git.host/pub/scm/linux.git/
288 * to borrow from
289 * http://git.host/pub/scm/linus.git/
290 * is ../../linus.git/objects/. You need
291 * two ../../ to borrow from your direct
292 * neighbour.
293 */
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400294 i += 3;
295 serverlen = strlen(base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800296 while (i + 2 < posn &&
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400297 !memcmp(data + i, "../", 3)) {
298 do {
299 serverlen--;
300 } while (serverlen &&
301 base[serverlen - 1] != '/');
302 i += 3;
303 }
Pavel Roskina9486b02006-07-10 02:57:51 -0400304 /* If the server got removed, give up. */
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800305 okay = strchr(base, ':') - base + 3 <
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800306 serverlen;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800307 } else if (alt_req->http_specific) {
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400308 char *colon = strchr(data + i, ':');
309 char *slash = strchr(data + i, '/');
310 if (colon && slash && colon < data + posn &&
311 slash < data + posn && colon < slash) {
312 okay = 1;
313 }
314 }
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400315 if (okay) {
René Scharfe59b82632014-08-30 17:55:45 +0200316 struct strbuf target = STRBUF_INIT;
317 strbuf_add(&target, base, serverlen);
Jeff Kingd61434a2017-03-13 10:04:45 -0400318 strbuf_add(&target, data + i, posn - i);
319 if (!strbuf_strip_suffix(&target, "objects")) {
320 warning("ignoring alternate that does"
321 " not end in 'objects': %s",
322 target.buf);
323 strbuf_release(&target);
324 } else if (is_alternate_allowed(target.buf)) {
Jeff Kingabcbdc02016-12-14 14:39:55 -0800325 warning("adding alternate object store: %s",
326 target.buf);
327 newalt = xmalloc(sizeof(*newalt));
328 newalt->next = NULL;
329 newalt->base = strbuf_detach(&target, NULL);
330 newalt->got_indices = 0;
331 newalt->packs = NULL;
332
333 while (tail->next != NULL)
334 tail = tail->next;
335 tail->next = newalt;
Eric Wong5cae73d2017-03-04 01:50:16 +0000336 } else {
337 strbuf_release(&target);
Jeff Kingabcbdc02016-12-14 14:39:55 -0800338 }
Daniel Barkalowb3661562005-09-14 23:26:08 -0400339 }
340 }
341 i = posn + 1;
342 }
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700343
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400344 cdata->got_alternates = 1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800345}
346
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400347static void fetch_alternates(struct walker *walker, const char *base)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800348{
Mike Hommey028c2972007-12-09 20:30:59 +0100349 struct strbuf buffer = STRBUF_INIT;
Jeff King54ba4c52015-09-24 17:07:31 -0400350 struct strbuf url = STRBUF_INIT;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800351 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800352 struct alternates_request alt_req;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400353 struct walker_data *cdata = walker->data;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800354
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800355 /*
356 * If another request has already started fetching alternates,
357 * wait for them to arrive and return to processing this request's
358 * curl message
359 */
Nick Hengeveld29508e12005-11-18 11:02:58 -0800360#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400361 while (cdata->got_alternates == 0) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800362 step_active_slots();
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800363 }
Nick Hengeveld29508e12005-11-18 11:02:58 -0800364#endif
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800365
366 /* Nothing to do if they've already been fetched */
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400367 if (cdata->got_alternates == 1)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800368 return;
369
370 /* Start the fetch */
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400371 cdata->got_alternates = 0;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800372
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400373 if (walker->get_verbosely)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800374 fprintf(stderr, "Getting alternates list for %s\n", base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800375
Jeff King54ba4c52015-09-24 17:07:31 -0400376 strbuf_addf(&url, "%s/objects/info/http-alternates", base);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800377
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800378 /*
379 * Use a callback to process the result, since another request
380 * may fail and need to have alternates loaded before continuing
381 */
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800382 slot = get_active_slot();
Nick Hengevelde388ab72005-11-18 11:03:04 -0800383 slot->callback_func = process_alternates_response;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400384 alt_req.walker = walker;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800385 slot->callback_data = &alt_req;
386
387 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800388 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
Jeff King54ba4c52015-09-24 17:07:31 -0400389 curl_easy_setopt(slot->curl, CURLOPT_URL, url.buf);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800390
391 alt_req.base = base;
Jeff King54ba4c52015-09-24 17:07:31 -0400392 alt_req.url = &url;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800393 alt_req.buffer = &buffer;
394 alt_req.http_specific = 1;
395 alt_req.slot = slot;
396
397 if (start_active_slot(slot))
398 run_active_slot(slot);
399 else
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400400 cdata->got_alternates = -1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800401
Mike Hommey028c2972007-12-09 20:30:59 +0100402 strbuf_release(&buffer);
Jeff King54ba4c52015-09-24 17:07:31 -0400403 strbuf_release(&url);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400404}
405
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400406static int fetch_indices(struct walker *walker, struct alt_base *repo)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400407{
Tay Ray Chuanb8caac22009-06-06 16:43:59 +0800408 int ret;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700409
Daniel Barkalowb3661562005-09-14 23:26:08 -0400410 if (repo->got_indices)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400411 return 0;
412
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400413 if (walker->get_verbosely)
Petr Baudis6fd72e32005-11-12 01:49:59 +0100414 fprintf(stderr, "Getting pack list for %s\n", repo->base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800415
Tay Ray Chuanb8caac22009-06-06 16:43:59 +0800416 switch (http_get_info_packs(repo->base, &repo->packs)) {
417 case HTTP_OK:
418 case HTTP_MISSING_TARGET:
419 repo->got_indices = 1;
420 ret = 0;
421 break;
422 default:
Nick Hengeveld5e3a7692005-11-18 11:03:11 -0800423 repo->got_indices = 0;
Tay Ray Chuanb8caac22009-06-06 16:43:59 +0800424 ret = -1;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700425 }
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400426
Mike Hommey3a462bc2007-12-10 22:36:11 +0100427 return ret;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400428}
429
Michael Haggerty07c19e72012-09-09 08:19:38 +0200430static int http_fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400431{
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400432 struct packed_git *target;
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700433 int ret;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800434 struct slot_results results;
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800435 struct http_pack_request *preq;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400436
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400437 if (fetch_indices(walker, repo))
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400438 return -1;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400439 target = find_sha1_pack(sha1, repo->packs);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400440 if (!target)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400441 return -1;
Jonathan Taneb053492020-06-10 13:57:16 -0700442 close_pack_index(target);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400443
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400444 if (walker->get_verbosely) {
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400445 fprintf(stderr, "Getting pack %s\n",
brian m. carlson2bf1db72019-02-19 00:05:12 +0000446 hash_to_hex(target->hash));
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400447 fprintf(stderr, " which contains %s\n",
brian m. carlson2bf1db72019-02-19 00:05:12 +0000448 hash_to_hex(sha1));
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400449 }
450
Jonathan Taneb053492020-06-10 13:57:16 -0700451 preq = new_http_pack_request(target->hash, repo->base);
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800452 if (preq == NULL)
453 goto abort;
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800454 preq->slot->results = &results;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400455
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800456 if (start_active_slot(preq->slot)) {
457 run_active_slot(preq->slot);
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800458 if (results.curl_result != CURLE_OK) {
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800459 error("Unable to get pack file %s\n%s", preq->url,
460 curl_errorstr);
461 goto abort;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700462 }
463 } else {
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800464 error("Unable to start request");
465 goto abort;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400466 }
467
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800468 ret = finish_http_pack_request(preq);
469 release_http_pack_request(preq);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700470 if (ret)
Junio C Hamanob721e012005-10-10 23:22:01 -0700471 return ret;
Jonathan Taneb053492020-06-10 13:57:16 -0700472 http_install_packfile(target, &repo->packs);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700473
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400474 return 0;
Tay Ray Chuan2264dfa2009-06-06 16:44:01 +0800475
476abort:
477 return -1;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400478}
479
Mark Wooding53f31382006-02-07 10:07:39 +0000480static void abort_object_request(struct object_request *obj_req)
481{
Mark Wooding53f31382006-02-07 10:07:39 +0000482 release_object_request(obj_req);
483}
484
brian m. carlson2bf1db72019-02-19 00:05:12 +0000485static int fetch_object(struct walker *walker, unsigned char *hash)
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700486{
brian m. carlson2bf1db72019-02-19 00:05:12 +0000487 char *hex = hash_to_hex(hash);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800488 int ret = 0;
Eric Wong94e99012016-07-11 20:51:31 +0000489 struct object_request *obj_req = NULL;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800490 struct http_object_request *req;
Eric Wong94e99012016-07-11 20:51:31 +0000491 struct list_head *pos, *head = &object_queue_head;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700492
Eric Wong94e99012016-07-11 20:51:31 +0000493 list_for_each(pos, head) {
494 obj_req = list_entry(pos, struct object_request, node);
brian m. carlson2bf1db72019-02-19 00:05:12 +0000495 if (hasheq(obj_req->oid.hash, hash))
Eric Wong94e99012016-07-11 20:51:31 +0000496 break;
497 }
Nick Hengevelde388ab72005-11-18 11:03:04 -0800498 if (obj_req == NULL)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700499 return error("Couldn't find request for %s in the queue", hex);
Junio C Hamano271421c2005-09-30 00:07:39 -0700500
Jeff King98374a02019-01-07 03:37:54 -0500501 if (has_object_file(&obj_req->oid)) {
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800502 if (obj_req->req != NULL)
503 abort_http_object_request(obj_req->req);
Mark Wooding53f31382006-02-07 10:07:39 +0000504 abort_object_request(obj_req);
Nick Hengeveld11f0daf2005-10-10 23:22:01 -0700505 return 0;
506 }
507
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700508#ifdef USE_CURL_MULTI
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800509 while (obj_req->state == WAITING)
Nick Hengeveld29508e12005-11-18 11:02:58 -0800510 step_active_slots();
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700511#else
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400512 start_object_request(walker, obj_req);
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700513#endif
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700514
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800515 /*
516 * obj_req->req might change when fetching alternates in the callback
517 * process_object_response; therefore, the "shortcut" variable, req,
518 * is used only after we're done with slots.
519 */
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800520 while (obj_req->state == ACTIVE)
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800521 run_active_slot(obj_req->req->slot);
Tay Ray Chuan4c42aa12009-06-06 16:43:33 +0800522
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800523 req = obj_req->req;
524
525 if (req->localfile != -1) {
526 close(req->localfile);
527 req->localfile = -1;
Petr Baudis313c4712005-11-12 00:55:16 +0100528 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700529
Jeff King3d10f722019-03-24 08:13:16 -0400530 normalize_curl_result(&req->curl_result, req->http_code,
531 req->errorstr, sizeof(req->errorstr));
Eric Wong17966c02016-07-11 20:51:30 +0000532
Nick Hengevelde388ab72005-11-18 11:03:04 -0800533 if (obj_req->state == ABORTED) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800534 ret = error("Request for %s aborted", hex);
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800535 } else if (req->curl_result != CURLE_OK &&
536 req->http_code != 416) {
537 if (missing_target(req))
Petr Baudise2029eb2005-10-21 18:18:46 +0200538 ret = -1; /* Be silent, it is probably in a pack. */
539 else
540 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800541 req->errorstr, req->curl_result,
542 req->http_code, hex);
543 } else if (req->zret != Z_STREAM_END) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400544 walker->corrupt_object_found++;
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800545 ret = error("File %s (%s) corrupt", hex, req->url);
Jeff Kingf0be0db2019-01-07 03:34:40 -0500546 } else if (!oideq(&obj_req->oid, &req->real_oid)) {
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800547 ret = error("File %s has bad hash", hex);
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800548 } else if (req->rename < 0) {
Christian Couderea657732018-01-17 18:54:54 +0100549 struct strbuf buf = STRBUF_INIT;
Jeff King514c5fd2019-01-07 03:35:42 -0500550 loose_object_path(the_repository, &buf, &req->oid);
Christian Couderea657732018-01-17 18:54:54 +0100551 ret = error("unable to write sha1 filename %s", buf.buf);
552 strbuf_release(&buf);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700553 }
554
Tay Ray Chuan5424bc52009-06-06 16:44:02 +0800555 release_http_object_request(req);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800556 release_object_request(obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800557 return ret;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700558}
559
brian m. carlson2bf1db72019-02-19 00:05:12 +0000560static int fetch(struct walker *walker, unsigned char *hash)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400561{
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400562 struct walker_data *data = walker->data;
563 struct alt_base *altbase = data->alt;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700564
brian m. carlson2bf1db72019-02-19 00:05:12 +0000565 if (!fetch_object(walker, hash))
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700566 return 0;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400567 while (altbase) {
brian m. carlson2bf1db72019-02-19 00:05:12 +0000568 if (!http_fetch_pack(walker, altbase, hash))
Daniel Barkalowb3661562005-09-14 23:26:08 -0400569 return 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400570 fetch_alternates(walker, data->alt->base);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400571 altbase = altbase->next;
572 }
brian m. carlson2bf1db72019-02-19 00:05:12 +0000573 return error("Unable to find %s under %s", hash_to_hex(hash),
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400574 data->alt->base);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400575}
576
Daniel Barkalowc13b2632008-04-26 15:53:09 -0400577static int fetch_ref(struct walker *walker, struct ref *ref)
Daniel Barkalowcd541a62005-06-06 16:38:26 -0400578{
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400579 struct walker_data *data = walker->data;
Daniel Barkalowc13b2632008-04-26 15:53:09 -0400580 return http_fetch_ref(data->alt->base, ref);
Daniel Barkalowcd541a62005-06-06 16:38:26 -0400581}
582
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400583static void cleanup(struct walker *walker)
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700584{
Tay Ray Chuan09ae9ac2010-03-02 18:49:28 +0800585 struct walker_data *data = walker->data;
586 struct alt_base *alt, *alt_next;
587
588 if (data) {
589 alt = data->alt;
590 while (alt) {
591 alt_next = alt->next;
592
593 free(alt->base);
594 free(alt);
595
596 alt = alt_next;
597 }
598 free(data);
599 walker->data = NULL;
600 }
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400601}
602
Tay Ray Chuan888692b2010-03-02 18:49:29 +0800603struct walker *get_http_walker(const char *url)
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400604{
Gerrit Pape9c880b32007-03-28 09:47:35 +0000605 char *s;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400606 struct walker_data *data = xmalloc(sizeof(struct walker_data));
607 struct walker *walker = xmalloc(sizeof(struct walker));
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700608
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400609 data->alt = xmalloc(sizeof(*data->alt));
Jeff King95244ae2014-06-19 17:19:43 -0400610 data->alt->base = xstrdup(url);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400611 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
Gerrit Pape9c880b32007-03-28 09:47:35 +0000612 *s = 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400613
614 data->alt->got_indices = 0;
615 data->alt->packs = NULL;
616 data->alt->next = NULL;
617 data->got_alternates = -1;
618
619 walker->corrupt_object_found = 0;
620 walker->fetch = fetch;
621 walker->fetch_ref = fetch_ref;
622 walker->prefetch = prefetch;
623 walker->cleanup = cleanup;
624 walker->data = data;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700625
Daniel Barkalowfc57b6a2007-09-10 23:02:34 -0400626#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400627 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
Daniel Barkalowfc57b6a2007-09-10 23:02:34 -0400628#endif
629
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400630 return walker;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700631}