blob: 444aebf5268c982255f73b151ff5c03bdffecdbe [file] [log] [blame]
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001#include "cache.h"
2#include "commit.h"
Junio C Hamano271421c2005-09-30 00:07:39 -07003#include "pack.h"
Daniel Barkalow30ae7642007-09-10 23:02:45 -04004#include "walker.h"
Nick Hengeveld29508e12005-11-18 11:02:58 -08005#include "http.h"
Johannes Schindelin7baa3e82005-10-15 11:10:46 -07006
Nick Hengeveld49a0f242005-09-28 10:14:04 -07007#define PREV_BUF_SIZE 4096
8#define RANGE_HEADER_SIZE 30
9
Daniel Barkalowb3661562005-09-14 23:26:08 -040010struct alt_base
11{
Gerrit Pape2afea3b2007-03-28 09:46:15 +000012 char *base;
Daniel Barkalowb3661562005-09-14 23:26:08 -040013 int got_indices;
14 struct packed_git *packs;
15 struct alt_base *next;
16};
17
Nick Hengevelde388ab72005-11-18 11:03:04 -080018enum object_request_state {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070019 WAITING,
20 ABORTED,
21 ACTIVE,
22 COMPLETE,
23};
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070024
Nick Hengevelde388ab72005-11-18 11:03:04 -080025struct object_request
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070026{
Daniel Barkalow30ae7642007-09-10 23:02:45 -040027 struct walker *walker;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070028 unsigned char sha1[20];
29 struct alt_base *repo;
30 char *url;
31 char filename[PATH_MAX];
32 char tmpfile[PATH_MAX];
33 int local;
Nick Hengevelde388ab72005-11-18 11:03:04 -080034 enum object_request_state state;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070035 CURLcode curl_result;
36 char errorstr[CURL_ERROR_SIZE];
37 long http_code;
38 unsigned char real_sha1[20];
39 SHA_CTX c;
40 z_stream stream;
41 int zret;
42 int rename;
43 struct active_request_slot *slot;
Nick Hengevelde388ab72005-11-18 11:03:04 -080044 struct object_request *next;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070045};
46
Nick Hengevelde388ab72005-11-18 11:03:04 -080047struct alternates_request {
Daniel Barkalow30ae7642007-09-10 23:02:45 -040048 struct walker *walker;
Petr Baudis8e29f6a2006-07-27 23:56:22 +020049 const char *base;
Nick Hengeveldacc075a2005-11-12 09:11:32 -080050 char *url;
51 struct buffer *buffer;
52 struct active_request_slot *slot;
53 int http_specific;
54};
55
Daniel Barkalow30ae7642007-09-10 23:02:45 -040056struct walker_data {
57 const char *url;
58 int got_alternates;
59 struct alt_base *alt;
60 struct curl_slist *no_pragma_header;
61};
62
David Rientjes96f1e582006-08-15 10:23:48 -070063static struct object_request *object_queue_head;
Sergey Vlasovbc8f2652005-10-13 10:49:53 -070064
barkalow@iabervon.org182005b2005-07-31 20:54:17 -040065static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
66 void *data)
67{
Brian Gerstbf0f9102005-05-18 08:14:09 -040068 unsigned char expn[4096];
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070069 size_t size = eltsize * nmemb;
70 int posn = 0;
Nick Hengevelde388ab72005-11-18 11:03:04 -080071 struct object_request *obj_req = (struct object_request *)data;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070072 do {
Andy Whitcroft93822c22007-01-08 15:58:23 +000073 ssize_t retval = xwrite(obj_req->local,
Florian Forster1d7f1712006-06-18 17:18:09 +020074 (char *) ptr + posn, size - posn);
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070075 if (retval < 0)
76 return posn;
77 posn += retval;
78 } while (posn < size);
79
Nick Hengevelde388ab72005-11-18 11:03:04 -080080 obj_req->stream.avail_in = size;
81 obj_req->stream.next_in = ptr;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070082 do {
Nick Hengevelde388ab72005-11-18 11:03:04 -080083 obj_req->stream.next_out = expn;
84 obj_req->stream.avail_out = sizeof(expn);
85 obj_req->zret = inflate(&obj_req->stream, Z_SYNC_FLUSH);
86 SHA1_Update(&obj_req->c, expn,
87 sizeof(expn) - obj_req->stream.avail_out);
88 } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -070089 data_received++;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -070090 return size;
91}
92
Junio C Hamanobe4a0152006-09-16 10:58:20 -070093static int missing__target(int code, int result)
94{
95 return /* file:// URL -- do we ever use one??? */
96 (result == CURLE_FILE_COULDNT_READ_FILE) ||
97 /* http:// and https:// URL */
Junio C Hamano4adffc72006-09-16 11:06:02 -070098 (code == 404 && result == CURLE_HTTP_RETURNED_ERROR) ||
99 /* ftp:// URL */
100 (code == 550 && result == CURLE_FTP_COULDNT_RETR_FILE)
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700101 ;
102}
103
104#define missing_target(a) missing__target((a)->http_code, (a)->curl_result)
105
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400106static void fetch_alternates(struct walker *walker, const char *base);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700107
Nick Hengeveld29508e12005-11-18 11:02:58 -0800108static void process_object_response(void *callback_data);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700109
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400110static void start_object_request(struct walker *walker,
111 struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700112{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800113 char *hex = sha1_to_hex(obj_req->sha1);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700114 char prevfile[PATH_MAX];
115 char *url;
116 char *posn;
117 int prevlocal;
118 unsigned char prev_buf[PREV_BUF_SIZE];
119 ssize_t prev_read = 0;
120 long prev_posn = 0;
121 char range[RANGE_HEADER_SIZE];
122 struct curl_slist *range_header = NULL;
123 struct active_request_slot *slot;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400124 struct walker_data *data = walker->data;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700125
Nick Hengevelde388ab72005-11-18 11:03:04 -0800126 snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700127 unlink(prevfile);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800128 rename(obj_req->tmpfile, prevfile);
129 unlink(obj_req->tmpfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700130
Nick Hengevelde388ab72005-11-18 11:03:04 -0800131 if (obj_req->local != -1)
132 error("fd leakage in start: %d", obj_req->local);
133 obj_req->local = open(obj_req->tmpfile,
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700134 O_WRONLY | O_CREAT | O_EXCL, 0666);
Junio C Hamanob721e012005-10-10 23:22:01 -0700135 /* This could have failed due to the "lazy directory creation";
136 * try to mkdir the last path component.
137 */
Nick Hengevelde388ab72005-11-18 11:03:04 -0800138 if (obj_req->local < 0 && errno == ENOENT) {
139 char *dir = strrchr(obj_req->tmpfile, '/');
Junio C Hamanob721e012005-10-10 23:22:01 -0700140 if (dir) {
141 *dir = 0;
Nick Hengevelde388ab72005-11-18 11:03:04 -0800142 mkdir(obj_req->tmpfile, 0777);
Junio C Hamanob721e012005-10-10 23:22:01 -0700143 *dir = '/';
144 }
Nick Hengevelde388ab72005-11-18 11:03:04 -0800145 obj_req->local = open(obj_req->tmpfile,
Junio C Hamanob721e012005-10-10 23:22:01 -0700146 O_WRONLY | O_CREAT | O_EXCL, 0666);
147 }
148
Nick Hengevelde388ab72005-11-18 11:03:04 -0800149 if (obj_req->local < 0) {
150 obj_req->state = ABORTED;
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800151 error("Couldn't create temporary file %s for %s: %s",
Nick Hengevelde388ab72005-11-18 11:03:04 -0800152 obj_req->tmpfile, obj_req->filename, strerror(errno));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700153 return;
154 }
155
Nick Hengevelde388ab72005-11-18 11:03:04 -0800156 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700157
Nick Hengevelde388ab72005-11-18 11:03:04 -0800158 inflateInit(&obj_req->stream);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700159
Nick Hengevelde388ab72005-11-18 11:03:04 -0800160 SHA1_Init(&obj_req->c);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700161
Gerrit Pape2afea3b2007-03-28 09:46:15 +0000162 url = xmalloc(strlen(obj_req->repo->base) + 51);
163 obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800164 strcpy(url, obj_req->repo->base);
165 posn = url + strlen(obj_req->repo->base);
Gerrit Pape2afea3b2007-03-28 09:46:15 +0000166 strcpy(posn, "/objects/");
167 posn += 9;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700168 memcpy(posn, hex, 2);
169 posn += 2;
170 *(posn++) = '/';
171 strcpy(posn, hex + 2);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800172 strcpy(obj_req->url, url);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700173
174 /* If a previous temp file is present, process what was already
175 fetched. */
176 prevlocal = open(prevfile, O_RDONLY);
177 if (prevlocal != -1) {
178 do {
Andy Whitcroft93d26e42007-01-08 15:58:08 +0000179 prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700180 if (prev_read>0) {
181 if (fwrite_sha1_file(prev_buf,
182 1,
183 prev_read,
Nick Hengevelde388ab72005-11-18 11:03:04 -0800184 obj_req) == prev_read) {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700185 prev_posn += prev_read;
186 } else {
187 prev_read = -1;
188 }
189 }
190 } while (prev_read > 0);
191 close(prevlocal);
192 }
193 unlink(prevfile);
194
195 /* Reset inflate/SHA1 if there was an error reading the previous temp
196 file; also rewind to the beginning of the local file. */
197 if (prev_read == -1) {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800198 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
199 inflateInit(&obj_req->stream);
200 SHA1_Init(&obj_req->c);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700201 if (prev_posn>0) {
202 prev_posn = 0;
Dana Howb5da2462007-04-05 12:05:57 -0700203 lseek(obj_req->local, 0, SEEK_SET);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800204 ftruncate(obj_req->local, 0);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700205 }
206 }
207
208 slot = get_active_slot();
Nick Hengeveld29508e12005-11-18 11:02:58 -0800209 slot->callback_func = process_object_response;
Nick Hengevelde388ab72005-11-18 11:03:04 -0800210 slot->callback_data = obj_req;
211 obj_req->slot = slot;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800212
Nick Hengevelde388ab72005-11-18 11:03:04 -0800213 curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700214 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800215 curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700216 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400217 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700218
219 /* If we have successfully processed data from a previous fetch
220 attempt, only fetch the data we don't already have. */
221 if (prev_posn>0) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400222 if (walker->get_verbosely)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700223 fprintf(stderr,
224 "Resuming fetch of object %s at byte %ld\n",
225 hex, prev_posn);
226 sprintf(range, "Range: bytes=%ld-", prev_posn);
227 range_header = curl_slist_append(range_header, range);
228 curl_easy_setopt(slot->curl,
229 CURLOPT_HTTPHEADER, range_header);
230 }
231
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700232 /* Try to get the request started, abort the request on error */
Nick Hengevelde388ab72005-11-18 11:03:04 -0800233 obj_req->state = ACTIVE;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700234 if (!start_active_slot(slot)) {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800235 obj_req->state = ABORTED;
236 obj_req->slot = NULL;
237 close(obj_req->local); obj_req->local = -1;
238 free(obj_req->url);
239 return;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700240 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700241}
242
Nick Hengevelde388ab72005-11-18 11:03:04 -0800243static void finish_object_request(struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700244{
Nick Hengeveld50496b22005-11-03 17:54:52 -0800245 struct stat st;
246
Nick Hengevelde388ab72005-11-18 11:03:04 -0800247 fchmod(obj_req->local, 0444);
248 close(obj_req->local); obj_req->local = -1;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700249
Nick Hengevelde388ab72005-11-18 11:03:04 -0800250 if (obj_req->http_code == 416) {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700251 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
Nick Hengevelde388ab72005-11-18 11:03:04 -0800252 } else if (obj_req->curl_result != CURLE_OK) {
253 if (stat(obj_req->tmpfile, &st) == 0)
Nick Hengeveld50496b22005-11-03 17:54:52 -0800254 if (st.st_size == 0)
Nick Hengevelde388ab72005-11-18 11:03:04 -0800255 unlink(obj_req->tmpfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700256 return;
257 }
258
Nick Hengevelde388ab72005-11-18 11:03:04 -0800259 inflateEnd(&obj_req->stream);
260 SHA1_Final(obj_req->real_sha1, &obj_req->c);
261 if (obj_req->zret != Z_STREAM_END) {
262 unlink(obj_req->tmpfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700263 return;
264 }
David Rientjesa89fccd2006-08-17 11:54:57 -0700265 if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800266 unlink(obj_req->tmpfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700267 return;
268 }
Nick Hengevelde388ab72005-11-18 11:03:04 -0800269 obj_req->rename =
270 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700271
Nick Hengevelde388ab72005-11-18 11:03:04 -0800272 if (obj_req->rename == 0)
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400273 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700274}
275
Nick Hengeveld29508e12005-11-18 11:02:58 -0800276static void process_object_response(void *callback_data)
277{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800278 struct object_request *obj_req =
279 (struct object_request *)callback_data;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400280 struct walker *walker = obj_req->walker;
281 struct walker_data *data = walker->data;
282 struct alt_base *alt = data->alt;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800283
Nick Hengevelde388ab72005-11-18 11:03:04 -0800284 obj_req->curl_result = obj_req->slot->curl_result;
285 obj_req->http_code = obj_req->slot->http_code;
286 obj_req->slot = NULL;
287 obj_req->state = COMPLETE;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800288
289 /* Use alternates if necessary */
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700290 if (missing_target(obj_req)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400291 fetch_alternates(walker, alt->base);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800292 if (obj_req->repo->next != NULL) {
293 obj_req->repo =
294 obj_req->repo->next;
295 close(obj_req->local);
296 obj_req->local = -1;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400297 start_object_request(walker, obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800298 return;
299 }
300 }
301
Nick Hengevelde388ab72005-11-18 11:03:04 -0800302 finish_object_request(obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800303}
304
Nick Hengevelde388ab72005-11-18 11:03:04 -0800305static void release_object_request(struct object_request *obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700306{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800307 struct object_request *entry = object_queue_head;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700308
Nick Hengevelde388ab72005-11-18 11:03:04 -0800309 if (obj_req->local != -1)
310 error("fd leakage in release: %d", obj_req->local);
311 if (obj_req == object_queue_head) {
312 object_queue_head = obj_req->next;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700313 } else {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800314 while (entry->next != NULL && entry->next != obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700315 entry = entry->next;
Nick Hengevelde388ab72005-11-18 11:03:04 -0800316 if (entry->next == obj_req)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700317 entry->next = entry->next->next;
318 }
319
Nick Hengevelde388ab72005-11-18 11:03:04 -0800320 free(obj_req->url);
321 free(obj_req);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700322}
323
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700324#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400325static int fill_active_slot(struct walker *walker)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700326{
Daniel Barkalow45c17412007-09-10 23:02:28 -0400327 struct object_request *obj_req;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700328
Daniel Barkalow45c17412007-09-10 23:02:28 -0400329 for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800330 if (obj_req->state == WAITING) {
331 if (has_sha1_file(obj_req->sha1))
Mark Wooding09db4442006-02-01 11:44:28 +0000332 obj_req->state = COMPLETE;
Daniel Barkalow45c17412007-09-10 23:02:28 -0400333 else {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400334 start_object_request(walker, obj_req);
Daniel Barkalow45c17412007-09-10 23:02:28 -0400335 return 1;
336 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700337 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700338 }
Daniel Barkalow45c17412007-09-10 23:02:28 -0400339 return 0;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700340}
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700341#endif
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700342
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400343static void prefetch(struct walker *walker, unsigned char *sha1)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700344{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800345 struct object_request *newreq;
346 struct object_request *tail;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400347 struct walker_data *data = walker->data;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700348 char *filename = sha1_file_name(sha1);
349
350 newreq = xmalloc(sizeof(*newreq));
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400351 newreq->walker = walker;
Shawn Pearcee7024962006-08-23 02:49:00 -0400352 hashcpy(newreq->sha1, sha1);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400353 newreq->repo = data->alt;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700354 newreq->url = NULL;
355 newreq->local = -1;
356 newreq->state = WAITING;
357 snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
358 snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
359 "%s.temp", filename);
Nick Hengevelde8dff6b2006-06-06 22:22:35 -0700360 newreq->slot = NULL;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700361 newreq->next = NULL;
362
Nick Hengevelde388ab72005-11-18 11:03:04 -0800363 if (object_queue_head == NULL) {
364 object_queue_head = newreq;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700365 } else {
Nick Hengevelde388ab72005-11-18 11:03:04 -0800366 tail = object_queue_head;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700367 while (tail->next != NULL) {
368 tail = tail->next;
369 }
370 tail->next = newreq;
371 }
Nick Hengeveld29508e12005-11-18 11:02:58 -0800372
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700373#ifdef USE_CURL_MULTI
Nick Hengeveld29508e12005-11-18 11:02:58 -0800374 fill_active_slots();
375 step_active_slots();
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700376#endif
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700377}
378
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400379static int fetch_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400380{
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700381 char *hex = sha1_to_hex(sha1);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400382 char *filename;
383 char *url;
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700384 char tmpfile[PATH_MAX];
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700385 long prev_posn = 0;
386 char range[RANGE_HEADER_SIZE];
387 struct curl_slist *range_header = NULL;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400388 struct walker_data *data = walker->data;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400389
390 FILE *indexfile;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700391 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800392 struct slot_results results;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400393
394 if (has_pack_index(sha1))
395 return 0;
396
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400397 if (walker->get_verbosely)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700398 fprintf(stderr, "Getting index for pack %s\n", hex);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800399
Daniel Barkalowb3661562005-09-14 23:26:08 -0400400 url = xmalloc(strlen(repo->base) + 64);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700401 sprintf(url, "%s/objects/pack/pack-%s.idx", repo->base, hex);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800402
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400403 filename = sha1_pack_index_name(sha1);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700404 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
405 indexfile = fopen(tmpfile, "a");
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400406 if (!indexfile)
407 return error("Unable to open local file %s for pack index",
408 filename);
409
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700410 slot = get_active_slot();
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800411 slot->results = &results;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700412 curl_easy_setopt(slot->curl, CURLOPT_FILE, indexfile);
413 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
414 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400415 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700416 slot->local = indexfile;
417
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700418 /* If there is data present from a previous transfer attempt,
419 resume where it left off */
420 prev_posn = ftell(indexfile);
421 if (prev_posn>0) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400422 if (walker->get_verbosely)
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700423 fprintf(stderr,
424 "Resuming fetch of index for pack %s at byte %ld\n",
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700425 hex, prev_posn);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700426 sprintf(range, "Range: bytes=%ld-", prev_posn);
427 range_header = curl_slist_append(range_header, range);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700428 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700429 }
430
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700431 if (start_active_slot(slot)) {
432 run_active_slot(slot);
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800433 if (results.curl_result != CURLE_OK) {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700434 fclose(indexfile);
435 return error("Unable to get pack index %s\n%s", url,
436 curl_errorstr);
437 }
438 } else {
Petr Baudis313c4712005-11-12 00:55:16 +0100439 fclose(indexfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700440 return error("Unable to start request");
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400441 }
442
443 fclose(indexfile);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700444
Junio C Hamanob721e012005-10-10 23:22:01 -0700445 return move_temp_to_file(tmpfile, filename);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400446}
447
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400448static int setup_index(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400449{
450 struct packed_git *new_pack;
451 if (has_pack_file(sha1))
Pavel Roskina9486b02006-07-10 02:57:51 -0400452 return 0; /* don't list this as something we can get */
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400453
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400454 if (fetch_index(walker, repo, sha1))
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400455 return -1;
456
457 new_pack = parse_pack_index(sha1);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400458 new_pack->next = repo->packs;
459 repo->packs = new_pack;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400460 return 0;
461}
462
Nick Hengevelde388ab72005-11-18 11:03:04 -0800463static void process_alternates_response(void *callback_data)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400464{
Nick Hengevelde388ab72005-11-18 11:03:04 -0800465 struct alternates_request *alt_req =
466 (struct alternates_request *)callback_data;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400467 struct walker *walker = alt_req->walker;
468 struct walker_data *cdata = walker->data;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800469 struct active_request_slot *slot = alt_req->slot;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400470 struct alt_base *tail = cdata->alt;
Petr Baudis8e29f6a2006-07-27 23:56:22 +0200471 const char *base = alt_req->base;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800472 static const char null_byte = '\0';
Daniel Barkalowb3661562005-09-14 23:26:08 -0400473 char *data;
474 int i = 0;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700475
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800476 if (alt_req->http_specific) {
477 if (slot->curl_result != CURLE_OK ||
478 !alt_req->buffer->posn) {
Junio C Hamanodc1b5ea2005-10-10 23:22:02 -0700479
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800480 /* Try reusing the slot to get non-http alternates */
481 alt_req->http_specific = 0;
482 sprintf(alt_req->url, "%s/objects/info/alternates",
483 base);
484 curl_easy_setopt(slot->curl, CURLOPT_URL,
485 alt_req->url);
486 active_requests++;
487 slot->in_use = 1;
Nick Hengeveldc9826472006-03-15 08:59:52 -0800488 if (slot->finished != NULL)
489 (*slot->finished) = 0;
Mark Woodinga3f583c2006-02-01 11:44:39 +0000490 if (!start_active_slot(slot)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400491 cdata->got_alternates = -1;
Nick Hengeveld29508e12005-11-18 11:02:58 -0800492 slot->in_use = 0;
Nick Hengeveldc9826472006-03-15 08:59:52 -0800493 if (slot->finished != NULL)
494 (*slot->finished) = 1;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700495 }
Mark Woodinga3f583c2006-02-01 11:44:39 +0000496 return;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400497 }
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800498 } else if (slot->curl_result != CURLE_OK) {
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700499 if (!missing_target(slot)) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400500 cdata->got_alternates = -1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800501 return;
502 }
Daniel Barkalowb3661562005-09-14 23:26:08 -0400503 }
504
Nick Hengeveld29508e12005-11-18 11:02:58 -0800505 fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800506 alt_req->buffer->posn--;
507 data = alt_req->buffer->buffer;
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400508
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800509 while (i < alt_req->buffer->posn) {
Daniel Barkalowb3661562005-09-14 23:26:08 -0400510 int posn = i;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800511 while (posn < alt_req->buffer->posn && data[posn] != '\n')
Daniel Barkalowb3661562005-09-14 23:26:08 -0400512 posn++;
513 if (data[posn] == '\n') {
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400514 int okay = 0;
515 int serverlen = 0;
516 struct alt_base *newalt;
517 char *target = NULL;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400518 if (data[i] == '/') {
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700519 /* This counts
520 * http://git.host/pub/scm/linux.git/
521 * -----------here^
522 * so memcpy(dst, base, serverlen) will
523 * copy up to "...git.host".
524 */
525 const char *colon_ss = strstr(base,"://");
526 if (colon_ss) {
527 serverlen = (strchr(colon_ss + 3, '/')
528 - base);
529 okay = 1;
530 }
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400531 } else if (!memcmp(data + i, "../", 3)) {
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700532 /* Relative URL; chop the corresponding
533 * number of subpath from base (and ../
534 * from data), and concatenate the result.
535 *
536 * The code first drops ../ from data, and
537 * then drops one ../ from data and one path
538 * from base. IOW, one extra ../ is dropped
539 * from data than path is dropped from base.
540 *
541 * This is not wrong. The alternate in
542 * http://git.host/pub/scm/linux.git/
543 * to borrow from
544 * http://git.host/pub/scm/linus.git/
545 * is ../../linus.git/objects/. You need
546 * two ../../ to borrow from your direct
547 * neighbour.
548 */
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400549 i += 3;
550 serverlen = strlen(base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800551 while (i + 2 < posn &&
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400552 !memcmp(data + i, "../", 3)) {
553 do {
554 serverlen--;
555 } while (serverlen &&
556 base[serverlen - 1] != '/');
557 i += 3;
558 }
Pavel Roskina9486b02006-07-10 02:57:51 -0400559 /* If the server got removed, give up. */
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800560 okay = strchr(base, ':') - base + 3 <
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400561 serverlen;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800562 } else if (alt_req->http_specific) {
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400563 char *colon = strchr(data + i, ':');
564 char *slash = strchr(data + i, '/');
565 if (colon && slash && colon < data + posn &&
566 slash < data + posn && colon < slash) {
567 okay = 1;
568 }
569 }
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700570 /* skip "objects\n" at end */
Daniel Barkalow1b0c1e62005-09-18 14:14:19 -0400571 if (okay) {
572 target = xmalloc(serverlen + posn - i - 6);
Junio C Hamano5df1e0d2006-09-12 23:53:27 -0700573 memcpy(target, base, serverlen);
574 memcpy(target + serverlen, data + i,
575 posn - i - 7);
576 target[serverlen + posn - i - 7] = 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400577 if (walker->get_verbosely)
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800578 fprintf(stderr,
Daniel Barkalowb3661562005-09-14 23:26:08 -0400579 "Also look at %s\n", target);
580 newalt = xmalloc(sizeof(*newalt));
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700581 newalt->next = NULL;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400582 newalt->base = target;
583 newalt->got_indices = 0;
584 newalt->packs = NULL;
Nick Hengeveld8d9fbe52006-04-04 05:33:18 -0700585
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700586 while (tail->next != NULL)
587 tail = tail->next;
588 tail->next = newalt;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400589 }
590 }
591 i = posn + 1;
592 }
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700593
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400594 cdata->got_alternates = 1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800595}
596
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400597static void fetch_alternates(struct walker *walker, const char *base)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800598{
599 struct buffer buffer;
600 char *url;
601 char *data;
602 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800603 struct alternates_request alt_req;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400604 struct walker_data *cdata = walker->data;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800605
606 /* If another request has already started fetching alternates,
607 wait for them to arrive and return to processing this request's
608 curl message */
Nick Hengeveld29508e12005-11-18 11:02:58 -0800609#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400610 while (cdata->got_alternates == 0) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800611 step_active_slots();
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800612 }
Nick Hengeveld29508e12005-11-18 11:02:58 -0800613#endif
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800614
615 /* Nothing to do if they've already been fetched */
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400616 if (cdata->got_alternates == 1)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800617 return;
618
619 /* Start the fetch */
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400620 cdata->got_alternates = 0;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800621
622 data = xmalloc(4096);
623 buffer.size = 4096;
624 buffer.posn = 0;
625 buffer.buffer = data;
626
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400627 if (walker->get_verbosely)
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800628 fprintf(stderr, "Getting alternates list for %s\n", base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800629
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800630 url = xmalloc(strlen(base) + 31);
631 sprintf(url, "%s/objects/info/http-alternates", base);
632
633 /* Use a callback to process the result, since another request
634 may fail and need to have alternates loaded before continuing */
635 slot = get_active_slot();
Nick Hengevelde388ab72005-11-18 11:03:04 -0800636 slot->callback_func = process_alternates_response;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400637 alt_req.walker = walker;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800638 slot->callback_data = &alt_req;
639
640 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800641 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800642 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
643
644 alt_req.base = base;
645 alt_req.url = url;
646 alt_req.buffer = &buffer;
647 alt_req.http_specific = 1;
648 alt_req.slot = slot;
649
650 if (start_active_slot(slot))
651 run_active_slot(slot);
652 else
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400653 cdata->got_alternates = -1;
Nick Hengeveldacc075a2005-11-12 09:11:32 -0800654
655 free(data);
656 free(url);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400657}
658
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400659static int fetch_indices(struct walker *walker, struct alt_base *repo)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400660{
661 unsigned char sha1[20];
662 char *url;
663 struct buffer buffer;
664 char *data;
665 int i = 0;
666
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700667 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800668 struct slot_results results;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700669
Daniel Barkalowb3661562005-09-14 23:26:08 -0400670 if (repo->got_indices)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400671 return 0;
672
673 data = xmalloc(4096);
674 buffer.size = 4096;
675 buffer.posn = 0;
676 buffer.buffer = data;
677
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400678 if (walker->get_verbosely)
Petr Baudis6fd72e32005-11-12 01:49:59 +0100679 fprintf(stderr, "Getting pack list for %s\n", repo->base);
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800680
Daniel Barkalowb3661562005-09-14 23:26:08 -0400681 url = xmalloc(strlen(repo->base) + 21);
682 sprintf(url, "%s/objects/info/packs", repo->base);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400683
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700684 slot = get_active_slot();
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800685 slot->results = &results;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700686 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800687 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700688 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
689 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
690 if (start_active_slot(slot)) {
691 run_active_slot(slot);
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800692 if (results.curl_result != CURLE_OK) {
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700693 if (missing_target(&results)) {
Nick Hengeveld5e3a7692005-11-18 11:03:11 -0800694 repo->got_indices = 1;
695 free(buffer.buffer);
696 return 0;
697 } else {
698 repo->got_indices = 0;
699 free(buffer.buffer);
700 return error("%s", curl_errorstr);
701 }
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700702 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700703 } else {
Nick Hengeveld5e3a7692005-11-18 11:03:11 -0800704 repo->got_indices = 0;
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700705 free(buffer.buffer);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700706 return error("Unable to start request");
707 }
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400708
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700709 data = buffer.buffer;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400710 while (i < buffer.posn) {
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400711 switch (data[i]) {
712 case 'P':
713 i++;
Junio C Hamano455c1612005-12-21 12:10:10 -0800714 if (i + 52 <= buffer.posn &&
Junio C Hamanocc44c762007-02-20 01:53:29 -0800715 !prefixcmp(data + i, " pack-") &&
Junio C Hamano1968d772007-02-20 01:55:07 -0800716 !prefixcmp(data + i + 46, ".pack\n")) {
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400717 get_sha1_hex(data + i + 6, sha1);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400718 setup_index(walker, repo, sha1);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400719 i += 51;
720 break;
721 }
722 default:
Junio C Hamano455c1612005-12-21 12:10:10 -0800723 while (i < buffer.posn && data[i] != '\n')
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400724 i++;
725 }
726 i++;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400727 }
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400728
Sergey Vlasovbc8f2652005-10-13 10:49:53 -0700729 free(buffer.buffer);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400730 repo->got_indices = 1;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400731 return 0;
732}
733
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400734static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400735{
736 char *url;
737 struct packed_git *target;
738 struct packed_git **lst;
739 FILE *packfile;
740 char *filename;
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700741 char tmpfile[PATH_MAX];
742 int ret;
743 long prev_posn = 0;
744 char range[RANGE_HEADER_SIZE];
745 struct curl_slist *range_header = NULL;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400746 struct walker_data *data = walker->data;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700747
748 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800749 struct slot_results results;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400750
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400751 if (fetch_indices(walker, repo))
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400752 return -1;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400753 target = find_sha1_pack(sha1, repo->packs);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400754 if (!target)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400755 return -1;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400756
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400757 if (walker->get_verbosely) {
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400758 fprintf(stderr, "Getting pack %s\n",
759 sha1_to_hex(target->sha1));
760 fprintf(stderr, " which contains %s\n",
761 sha1_to_hex(sha1));
762 }
763
Daniel Barkalowb3661562005-09-14 23:26:08 -0400764 url = xmalloc(strlen(repo->base) + 65);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400765 sprintf(url, "%s/objects/pack/pack-%s.pack",
Daniel Barkalowb3661562005-09-14 23:26:08 -0400766 repo->base, sha1_to_hex(target->sha1));
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400767
768 filename = sha1_pack_name(target->sha1);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700769 snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
770 packfile = fopen(tmpfile, "a");
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400771 if (!packfile)
772 return error("Unable to open local file %s for pack",
773 filename);
774
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700775 slot = get_active_slot();
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800776 slot->results = &results;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700777 curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
778 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
779 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400780 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, data->no_pragma_header);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700781 slot->local = packfile;
Nick Hengeveld1ddea772005-09-26 10:52:11 -0700782
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700783 /* If there is data present from a previous transfer attempt,
784 resume where it left off */
785 prev_posn = ftell(packfile);
786 if (prev_posn>0) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400787 if (walker->get_verbosely)
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700788 fprintf(stderr,
789 "Resuming fetch of pack %s at byte %ld\n",
790 sha1_to_hex(target->sha1), prev_posn);
791 sprintf(range, "Range: bytes=%ld-", prev_posn);
792 range_header = curl_slist_append(range_header, range);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700793 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700794 }
795
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700796 if (start_active_slot(slot)) {
797 run_active_slot(slot);
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800798 if (results.curl_result != CURLE_OK) {
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700799 fclose(packfile);
800 return error("Unable to get pack file %s\n%s", url,
801 curl_errorstr);
802 }
803 } else {
Petr Baudis313c4712005-11-12 00:55:16 +0100804 fclose(packfile);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700805 return error("Unable to start request");
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400806 }
807
Shawn O. Pearce1c23d792007-01-09 20:04:52 -0500808 target->pack_size = ftell(packfile);
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400809 fclose(packfile);
810
Junio C Hamanob721e012005-10-10 23:22:01 -0700811 ret = move_temp_to_file(tmpfile, filename);
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700812 if (ret)
Junio C Hamanob721e012005-10-10 23:22:01 -0700813 return ret;
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700814
Daniel Barkalowb3661562005-09-14 23:26:08 -0400815 lst = &repo->packs;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400816 while (*lst != target)
817 lst = &((*lst)->next);
818 *lst = (*lst)->next;
819
Junio C Hamano271421c2005-09-30 00:07:39 -0700820 if (verify_pack(target, 0))
821 return -1;
barkalow@iabervon.org182005b2005-07-31 20:54:17 -0400822 install_packed_git(target);
823
824 return 0;
825}
826
Mark Wooding53f31382006-02-07 10:07:39 +0000827static void abort_object_request(struct object_request *obj_req)
828{
829 if (obj_req->local >= 0) {
830 close(obj_req->local);
831 obj_req->local = -1;
832 }
833 unlink(obj_req->tmpfile);
834 if (obj_req->slot) {
Junio C Hamanoa6080a02007-06-07 00:04:01 -0700835 release_active_slot(obj_req->slot);
Mark Wooding53f31382006-02-07 10:07:39 +0000836 obj_req->slot = NULL;
837 }
838 release_object_request(obj_req);
839}
840
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400841static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700842{
843 char *hex = sha1_to_hex(sha1);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800844 int ret = 0;
Nick Hengevelde388ab72005-11-18 11:03:04 -0800845 struct object_request *obj_req = object_queue_head;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700846
David Rientjesa89fccd2006-08-17 11:54:57 -0700847 while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
Nick Hengevelde388ab72005-11-18 11:03:04 -0800848 obj_req = obj_req->next;
849 if (obj_req == NULL)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700850 return error("Couldn't find request for %s in the queue", hex);
Junio C Hamano271421c2005-09-30 00:07:39 -0700851
Nick Hengevelde388ab72005-11-18 11:03:04 -0800852 if (has_sha1_file(obj_req->sha1)) {
Mark Wooding53f31382006-02-07 10:07:39 +0000853 abort_object_request(obj_req);
Nick Hengeveld11f0daf2005-10-10 23:22:01 -0700854 return 0;
855 }
856
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700857#ifdef USE_CURL_MULTI
Nick Hengevelde388ab72005-11-18 11:03:04 -0800858 while (obj_req->state == WAITING) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800859 step_active_slots();
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700860 }
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700861#else
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400862 start_object_request(walker, obj_req);
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700863#endif
Nick Hengeveld49a0f242005-09-28 10:14:04 -0700864
Nick Hengevelde388ab72005-11-18 11:03:04 -0800865 while (obj_req->state == ACTIVE) {
866 run_active_slot(obj_req->slot);
Nick Hengevelda7a8d372005-10-10 23:22:01 -0700867 }
Nick Hengevelde388ab72005-11-18 11:03:04 -0800868 if (obj_req->local != -1) {
869 close(obj_req->local); obj_req->local = -1;
Petr Baudis313c4712005-11-12 00:55:16 +0100870 }
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700871
Nick Hengevelde388ab72005-11-18 11:03:04 -0800872 if (obj_req->state == ABORTED) {
Nick Hengeveld29508e12005-11-18 11:02:58 -0800873 ret = error("Request for %s aborted", hex);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800874 } else if (obj_req->curl_result != CURLE_OK &&
875 obj_req->http_code != 416) {
Junio C Hamanobe4a0152006-09-16 10:58:20 -0700876 if (missing_target(obj_req))
Petr Baudise2029eb2005-10-21 18:18:46 +0200877 ret = -1; /* Be silent, it is probably in a pack. */
878 else
879 ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
Nick Hengevelde388ab72005-11-18 11:03:04 -0800880 obj_req->errorstr, obj_req->curl_result,
881 obj_req->http_code, hex);
882 } else if (obj_req->zret != Z_STREAM_END) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400883 walker->corrupt_object_found++;
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800884 ret = error("File %s (%s) corrupt", hex, obj_req->url);
David Rientjesa89fccd2006-08-17 11:54:57 -0700885 } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800886 ret = error("File %s has bad hash", hex);
Nick Hengevelde388ab72005-11-18 11:03:04 -0800887 } else if (obj_req->rename < 0) {
Mark Wooding7b934ec2006-02-01 11:44:35 +0000888 ret = error("unable to write sha1 filename %s",
889 obj_req->filename);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700890 }
891
Nick Hengevelde388ab72005-11-18 11:03:04 -0800892 release_object_request(obj_req);
Nick Hengeveld29508e12005-11-18 11:02:58 -0800893 return ret;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700894}
895
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400896static int fetch(struct walker *walker, unsigned char *sha1)
Daniel Barkalowb3661562005-09-14 23:26:08 -0400897{
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400898 struct walker_data *data = walker->data;
899 struct alt_base *altbase = data->alt;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700900
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400901 if (!fetch_object(walker, altbase, sha1))
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700902 return 0;
Daniel Barkalowb3661562005-09-14 23:26:08 -0400903 while (altbase) {
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400904 if (!fetch_pack(walker, altbase, sha1))
Daniel Barkalowb3661562005-09-14 23:26:08 -0400905 return 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400906 fetch_alternates(walker, data->alt->base);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400907 altbase = altbase->next;
908 }
Junio C Hamanobd2afde2006-02-22 17:47:10 -0800909 return error("Unable to find %s under %s", sha1_to_hex(sha1),
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400910 data->alt->base);
Daniel Barkalowb3661562005-09-14 23:26:08 -0400911}
912
Junio C Hamano94fa4472005-10-10 23:22:02 -0700913static inline int needs_quote(int ch)
914{
Florian Forstercfd432e62006-06-18 17:18:03 +0200915 if (((ch >= 'A') && (ch <= 'Z'))
916 || ((ch >= 'a') && (ch <= 'z'))
917 || ((ch >= '0') && (ch <= '9'))
918 || (ch == '/')
919 || (ch == '-')
920 || (ch == '.'))
Junio C Hamano94fa4472005-10-10 23:22:02 -0700921 return 0;
Florian Forstercfd432e62006-06-18 17:18:03 +0200922 return 1;
Junio C Hamano94fa4472005-10-10 23:22:02 -0700923}
924
925static inline int hex(int v)
926{
927 if (v < 10) return '0' + v;
928 else return 'A' + v - 10;
929}
930
931static char *quote_ref_url(const char *base, const char *ref)
932{
933 const char *cp;
934 char *dp, *qref;
935 int len, baselen, ch;
936
937 baselen = strlen(base);
Gerrit Pape2afea3b2007-03-28 09:46:15 +0000938 len = baselen + 7; /* "/refs/" + NUL */
Junio C Hamano94fa4472005-10-10 23:22:02 -0700939 for (cp = ref; (ch = *cp) != 0; cp++, len++)
940 if (needs_quote(ch))
941 len += 2; /* extra two hex plus replacement % */
942 qref = xmalloc(len);
943 memcpy(qref, base, baselen);
Gerrit Pape2afea3b2007-03-28 09:46:15 +0000944 memcpy(qref + baselen, "/refs/", 6);
945 for (cp = ref, dp = qref + baselen + 6; (ch = *cp) != 0; cp++) {
Junio C Hamano94fa4472005-10-10 23:22:02 -0700946 if (needs_quote(ch)) {
947 *dp++ = '%';
948 *dp++ = hex((ch >> 4) & 0xF);
949 *dp++ = hex(ch & 0xF);
950 }
951 else
952 *dp++ = ch;
953 }
954 *dp = 0;
955
956 return qref;
957}
958
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400959static int fetch_ref(struct walker *walker, char *ref, unsigned char *sha1)
Daniel Barkalowcd541a62005-06-06 16:38:26 -0400960{
Junio C Hamano94fa4472005-10-10 23:22:02 -0700961 char *url;
Daniel Barkalowfa3e0652005-06-21 20:45:49 -0400962 char hex[42];
963 struct buffer buffer;
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400964 struct walker_data *data = walker->data;
965 const char *base = data->alt->base;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700966 struct active_request_slot *slot;
Nick Hengeveldcb754fd2006-01-31 18:00:37 -0800967 struct slot_results results;
Daniel Barkalowfa3e0652005-06-21 20:45:49 -0400968 buffer.size = 41;
969 buffer.posn = 0;
970 buffer.buffer = hex;
971 hex[41] = '\0';
Junio C Hamano8fcf7f92006-01-31 18:15:51 -0800972
Junio C Hamano94fa4472005-10-10 23:22:02 -0700973 url = quote_ref_url(base, ref);
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700974 slot = get_active_slot();
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800975 slot->results = &results;
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700976 curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
977 curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
978 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
979 curl_easy_setopt(slot->curl, CURLOPT_URL, url);
980 if (start_active_slot(slot)) {
981 run_active_slot(slot);
Nick Hengeveldc8568e12006-01-31 11:06:55 -0800982 if (results.curl_result != CURLE_OK)
Nick Hengeveld1d389ab2005-10-10 23:22:01 -0700983 return error("Couldn't get %s for %s\n%s",
984 url, ref, curl_errorstr);
985 } else {
986 return error("Unable to start request");
987 }
Daniel Barkalowfa3e0652005-06-21 20:45:49 -0400988
989 hex[40] = '\0';
990 get_sha1_hex(hex, sha1);
991 return 0;
Daniel Barkalowcd541a62005-06-06 16:38:26 -0400992}
993
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400994static void cleanup(struct walker *walker)
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -0700995{
Daniel Barkalow30ae7642007-09-10 23:02:45 -0400996 struct walker_data *data = walker->data;
997 http_cleanup();
998
999 curl_slist_free_all(data->no_pragma_header);
1000}
1001
1002struct walker *get_http_walker(const char *url)
1003{
Gerrit Pape9c880b32007-03-28 09:47:35 +00001004 char *s;
Daniel Barkalow30ae7642007-09-10 23:02:45 -04001005 struct walker_data *data = xmalloc(sizeof(struct walker_data));
1006 struct walker *walker = xmalloc(sizeof(struct walker));
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001007
Nick Hengeveld29508e12005-11-18 11:02:58 -08001008 http_init();
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001009
Daniel Barkalow30ae7642007-09-10 23:02:45 -04001010 data->no_pragma_header = curl_slist_append(NULL, "Pragma:");
Darrin Thompson3dcb90f2005-07-12 21:12:40 -05001011
Daniel Barkalow30ae7642007-09-10 23:02:45 -04001012 data->alt = xmalloc(sizeof(*data->alt));
1013 data->alt->base = xmalloc(strlen(url) + 1);
1014 strcpy(data->alt->base, url);
1015 for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
Gerrit Pape9c880b32007-03-28 09:47:35 +00001016 *s = 0;
Daniel Barkalow30ae7642007-09-10 23:02:45 -04001017
1018 data->alt->got_indices = 0;
1019 data->alt->packs = NULL;
1020 data->alt->next = NULL;
1021 data->got_alternates = -1;
1022
1023 walker->corrupt_object_found = 0;
1024 walker->fetch = fetch;
1025 walker->fetch_ref = fetch_ref;
1026 walker->prefetch = prefetch;
1027 walker->cleanup = cleanup;
1028 walker->data = data;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001029
Daniel Barkalowfc57b6a2007-09-10 23:02:34 -04001030#ifdef USE_CURL_MULTI
Daniel Barkalow30ae7642007-09-10 23:02:45 -04001031 add_fill_function(walker, (int (*)(void *)) fill_active_slot);
Daniel Barkalowfc57b6a2007-09-10 23:02:34 -04001032#endif
1033
Daniel Barkalow30ae7642007-09-10 23:02:45 -04001034 return walker;
Daniel Barkalow6eb7ed52005-04-23 18:47:23 -07001035}