Merge branch 'sp/maint-dumb-http-pack-reidx'

* sp/maint-dumb-http-pack-reidx:
  http.c::new_http_pack_request: do away with the temp variable filename
  http-fetch: Use temporary files for pack-*.idx until verified
  http-fetch: Use index-pack rather than verify-pack to check packs
  Allow parse_pack_index on temporary files
  Extract verify_pack_index for reuse from verify_pack
  Introduce close_pack_index to permit replacement
  http.c: Remove unnecessary strdup of sha1_to_hex result
  http.c: Don't store destination name in request structures
  http.c: Drop useless != NULL test in finish_http_pack_request
  http.c: Tiny refactoring of finish_http_pack_request
  t5550-http-fetch: Use subshell for repository operations
  http.c: Remove bad free of static block

diff --git a/cache.h b/cache.h
index d00d2be320152cfdd2580468db27d6735dc59748..c96602305f94b50e3f39be124c5e8d5af5a36f51 100644 (file)
--- a/cache.h
+++ b/cache.h
@@ -907,7 +907,7 @@ struct extra_have_objects {
 extern struct ref **get_remote_heads(int in, struct ref **list, int nr_match, char **match, unsigned int flags, struct extra_have_objects *);
 extern int server_supports(const char *feature);
 
-extern struct packed_git *parse_pack_index(unsigned char *sha1);
+extern struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path);
 
 extern void prepare_packed_git(void);
 extern void reprepare_packed_git(void);
@@ -918,6 +918,7 @@ extern struct packed_git *find_sha1_pack(const unsigned char *sha1,
 
 extern void pack_report(void);
 extern int open_pack_index(struct packed_git *);
+extern void close_pack_index(struct packed_git *);
 extern unsigned char *use_pack(struct packed_git *, struct pack_window **, off_t, unsigned int *);
 extern void close_pack_windows(struct packed_git *);
 extern void unuse_pack(struct pack_window **);

diff --git a/http-walker.c b/http-walker.c
index ef99ae647ae02995495c71455eef785bdeca1789..8ca76d0507bdc1d95283e1f5fee3f88180cdce26 100644 (file)
--- a/http-walker.c
+++ b/http-walker.c
@@ -510,7 +510,7 @@ static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned c
                ret = error("File %s has bad hash", hex);
        } else if (req->rename < 0) {
                ret = error("unable to write sha1 filename %s",
-                           req->filename);
+                           sha1_file_name(req->sha1));
        }
 
        release_http_object_request(req);

diff --git a/http.c b/http.c
index 07a03fd79b5475f13d21f2b3a6825df7246188bf..1320c50e32eb7b8715b263bc2af089c3dbce39fa 100644 (file)
--- a/http.c
+++ b/http.c
@@ -1,6 +1,7 @@
 #include "http.h"
 #include "pack.h"
 #include "sideband.h"
+#include "run-command.h"
 
 int data_received;
 int active_requests;
@@ -914,47 +915,67 @@ int http_fetch_ref(const char *base, struct ref *ref)
 }
 
 /* Helpers for fetching packs */
-static int fetch_pack_index(unsigned char *sha1, const char *base_url)
+static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
 {
-       int ret = 0;
-       char *hex = xstrdup(sha1_to_hex(sha1));
-       char *filename;
-       char *url = NULL;
+       char *url, *tmp;
        struct strbuf buf = STRBUF_INIT;
 
-       if (has_pack_index(sha1)) {
-               ret = 0;
-               goto cleanup;
-       }
-
        if (http_is_verbose)
-               fprintf(stderr, "Getting index for pack %s\n", hex);
+               fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
 
        end_url_with_slash(&buf, base_url);
-       strbuf_addf(&buf, "objects/pack/pack-%s.idx", hex);
+       strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
        url = strbuf_detach(&buf, NULL);
 
-       filename = sha1_pack_index_name(sha1);
-       if (http_get_file(url, filename, 0) != HTTP_OK)
-               ret = error("Unable to get pack index %s\n", url);
+       strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
+       tmp = strbuf_detach(&buf, NULL);
+
+       if (http_get_file(url, tmp, 0) != HTTP_OK) {
+               error("Unable to get pack index %s\n", url);
+               free(tmp);
+               tmp = NULL;
+       }
 
-cleanup:
-       free(hex);
        free(url);
-       return ret;
+       return tmp;
 }
 
 static int fetch_and_setup_pack_index(struct packed_git **packs_head,
        unsigned char *sha1, const char *base_url)
 {
        struct packed_git *new_pack;
+       char *tmp_idx = NULL;
+       int ret;
 
-       if (fetch_pack_index(sha1, base_url))
+       if (has_pack_index(sha1)) {
+               new_pack = parse_pack_index(sha1, NULL);
+               if (!new_pack)
+                       return -1; /* parse_pack_index() already issued error message */
+               goto add_pack;
+       }
+
+       tmp_idx = fetch_pack_index(sha1, base_url);
+       if (!tmp_idx)
                return -1;
 
-       new_pack = parse_pack_index(sha1);
-       if (!new_pack)
+       new_pack = parse_pack_index(sha1, tmp_idx);
+       if (!new_pack) {
+               unlink(tmp_idx);
+               free(tmp_idx);
+
                return -1; /* parse_pack_index() already issued error message */
+       }
+
+       ret = verify_pack_index(new_pack);
+       if (!ret) {
+               close_pack_index(new_pack);
+               ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
+       }
+       free(tmp_idx);
+       if (ret)
+               return -1;
+
+add_pack:
        new_pack->next = *packs_head;
        *packs_head = new_pack;
        return 0;
@@ -1018,37 +1039,62 @@ void release_http_pack_request(struct http_pack_request *preq)
 
 int finish_http_pack_request(struct http_pack_request *preq)
 {
-       int ret;
        struct packed_git **lst;
+       struct packed_git *p = preq->target;
+       char *tmp_idx;
+       struct child_process ip;
+       const char *ip_argv[8];
 
-       preq->target->pack_size = ftell(preq->packfile);
+       close_pack_index(p);
 
-       if (preq->packfile != NULL) {
-               fclose(preq->packfile);
-               preq->packfile = NULL;
-               preq->slot->local = NULL;
-       }
-
-       ret = move_temp_to_file(preq->tmpfile, preq->filename);
-       if (ret)
-               return ret;
+       fclose(preq->packfile);
+       preq->packfile = NULL;
+       preq->slot->local = NULL;
 
        lst = preq->lst;
-       while (*lst != preq->target)
+       while (*lst != p)
                lst = &((*lst)->next);
        *lst = (*lst)->next;
 
-       if (verify_pack(preq->target))
+       tmp_idx = xstrdup(preq->tmpfile);
+       strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
+              ".idx.temp");
+
+       ip_argv[0] = "index-pack";
+       ip_argv[1] = "-o";
+       ip_argv[2] = tmp_idx;
+       ip_argv[3] = preq->tmpfile;
+       ip_argv[4] = NULL;
+
+       memset(&ip, 0, sizeof(ip));
+       ip.argv = ip_argv;
+       ip.git_cmd = 1;
+       ip.no_stdin = 1;
+       ip.no_stdout = 1;
+
+       if (run_command(&ip)) {
+               unlink(preq->tmpfile);
+               unlink(tmp_idx);
+               free(tmp_idx);
+               return -1;
+       }
+
+       unlink(sha1_pack_index_name(p->sha1));
+
+       if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
+        || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
+               free(tmp_idx);
                return -1;
-       install_packed_git(preq->target);
+       }
 
+       install_packed_git(p);
+       free(tmp_idx);
        return 0;
 }
 
 struct http_pack_request *new_http_pack_request(
        struct packed_git *target, const char *base_url)
 {
-       char *filename;
        long prev_posn = 0;
        char range[RANGE_HEADER_SIZE];
        struct strbuf buf = STRBUF_INIT;
@@ -1063,9 +1109,8 @@ struct http_pack_request *new_http_pack_request(
                sha1_to_hex(target->sha1));
        preq->url = strbuf_detach(&buf, NULL);
 
-       filename = sha1_pack_name(target->sha1);
-       snprintf(preq->filename, sizeof(preq->filename), "%s", filename);
-       snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp", filename);
+       snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
+               sha1_pack_name(target->sha1));
        preq->packfile = fopen(preq->tmpfile, "a");
        if (!preq->packfile) {
                error("Unable to open local file %s for pack",
@@ -1100,7 +1145,6 @@ struct http_pack_request *new_http_pack_request(
        return preq;
 
 abort:
-       free(filename);
        free(preq->url);
        free(preq);
        return NULL;
@@ -1155,7 +1199,6 @@ struct http_object_request *new_http_object_request(const char *base_url,
        freq->localfile = -1;
 
        filename = sha1_file_name(sha1);
-       snprintf(freq->filename, sizeof(freq->filename), "%s", filename);
        snprintf(freq->tmpfile, sizeof(freq->tmpfile),
                 "%s.temp", filename);
 
@@ -1184,8 +1227,8 @@ struct http_object_request *new_http_object_request(const char *base_url,
        }
 
        if (freq->localfile < 0) {
-               error("Couldn't create temporary file %s for %s: %s",
-                     freq->tmpfile, freq->filename, strerror(errno));
+               error("Couldn't create temporary file %s: %s",
+                     freq->tmpfile, strerror(errno));
                goto abort;
        }
 
@@ -1232,8 +1275,8 @@ struct http_object_request *new_http_object_request(const char *base_url,
                        prev_posn = 0;
                        lseek(freq->localfile, 0, SEEK_SET);
                        if (ftruncate(freq->localfile, 0) < 0) {
-                               error("Couldn't truncate temporary file %s for %s: %s",
-                                         freq->tmpfile, freq->filename, strerror(errno));
+                               error("Couldn't truncate temporary file %s: %s",
+                                         freq->tmpfile, strerror(errno));
                                goto abort;
                        }
                }
@@ -1309,7 +1352,7 @@ int finish_http_object_request(struct http_object_request *freq)
                return -1;
        }
        freq->rename =
-               move_temp_to_file(freq->tmpfile, freq->filename);
+               move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));
 
        return freq->rename;
 }

diff --git a/http.h b/http.h
index 37a6a6a3b0cd0ddbbf395230f04b94fc54191dbe..a0b59015948c3b9736dd6c09fd9137f7ecd4f59a 100644 (file)
--- a/http.h
+++ b/http.h
@@ -155,7 +155,6 @@ struct http_pack_request
        struct packed_git *target;
        struct packed_git **lst;
        FILE *packfile;
-       char filename[PATH_MAX];
        char tmpfile[PATH_MAX];
        struct curl_slist *range_header;
        struct active_request_slot *slot;
@@ -170,7 +169,6 @@ extern void release_http_pack_request(struct http_pack_request *preq);
 struct http_object_request
 {
        char *url;
-       char filename[PATH_MAX];
        char tmpfile[PATH_MAX];
        int localfile;
        CURLcode curl_result;

diff --git a/pack-check.c b/pack-check.c
index 166ca703c10face0d4961da6ceee7a149ebcfac4..395fb9527a3bc6dd8ca648233911a1c35604440d 100644 (file)
--- a/pack-check.c
+++ b/pack-check.c
@@ -133,14 +133,13 @@ static int verify_packfile(struct packed_git *p,
        return err;
 }
 
-int verify_pack(struct packed_git *p)
+int verify_pack_index(struct packed_git *p)
 {
        off_t index_size;
        const unsigned char *index_base;
        git_SHA_CTX ctx;
        unsigned char sha1[20];
        int err = 0;
-       struct pack_window *w_curs = NULL;
 
        if (open_pack_index(p))
                return error("packfile %s index not opened", p->pack_name);
@@ -154,8 +153,18 @@ int verify_pack(struct packed_git *p)
        if (hashcmp(sha1, index_base + index_size - 20))
                err = error("Packfile index for %s SHA1 mismatch",
                            p->pack_name);
+       return err;
+}
+
+int verify_pack(struct packed_git *p)
+{
+       int err = 0;
+       struct pack_window *w_curs = NULL;
+
+       err |= verify_pack_index(p);
+       if (!p->index_data)
+               return -1;
 
-       /* Verify pack file */
        err |= verify_packfile(p, &w_curs);
        unuse_pack(&w_curs);
 

diff --git a/pack.h b/pack.h
index d268c014c9eb7040bd65125b13d68edce670274b..bb275762b7eb6f473f333ae40780821e383db20b 100644 (file)
--- a/pack.h
+++ b/pack.h
@@ -57,6 +57,7 @@ struct pack_idx_entry {
 
 extern const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, int nr_objects, unsigned char *sha1);
 extern int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, off_t offset, off_t len, unsigned int nr);
+extern int verify_pack_index(struct packed_git *);
 extern int verify_pack(struct packed_git *);
 extern void fixup_pack_header_footer(int, unsigned char *, const char *, uint32_t, unsigned char *, off_t);
 extern char *index_pack_lockfile(int fd);

diff --git a/sha1_file.c b/sha1_file.c
index b2074e7a8ee6d55fdb1094b93a2954bd164d5445..d8e61a65d10ed6fa792d207e36fb6b245ef86970 100644 (file)
--- a/sha1_file.c
+++ b/sha1_file.c
@@ -599,6 +599,14 @@ void unuse_pack(struct pack_window **w_cursor)
        }
 }
 
+void close_pack_index(struct packed_git *p)
+{
+       if (p->index_data) {
+               munmap((void *)p->index_data, p->index_size);
+               p->index_data = NULL;
+       }
+}
+
 /*
  * This is used by git-repack in case a newly created pack happens to
  * contain the same set of objects as an existing one.  In that case
@@ -620,8 +628,7 @@ void free_pack_by_name(const char *pack_name)
                        close_pack_windows(p);
                        if (p->pack_fd != -1)
                                close(p->pack_fd);
-                       if (p->index_data)
-                               munmap((void *)p->index_data, p->index_size);
+                       close_pack_index(p);
                        free(p->bad_object_sha1);
                        *pp = p->next;
                        free(p);
@@ -831,9 +838,8 @@ struct packed_git *add_packed_git(const char *path, int path_len, int local)
        return p;
 }
 
-struct packed_git *parse_pack_index(unsigned char *sha1)
+struct packed_git *parse_pack_index(unsigned char *sha1, const char *idx_path)
 {
-       const char *idx_path = sha1_pack_index_name(sha1);
        const char *path = sha1_pack_name(sha1);
        struct packed_git *p = alloc_packed_git(strlen(path) + 1);
 

diff --git a/t/t5550-http-fetch.sh b/t/t5550-http-fetch.sh
index 8cfce969bcdac6e2091e635dad9c58ca616e5c3b..fc675b50adfa945724cd7d17dbb6034df53426d3 100755 (executable)
--- a/t/t5550-http-fetch.sh
+++ b/t/t5550-http-fetch.sh
@@ -55,12 +55,43 @@ test_expect_success 'http remote detects correct HEAD' '
 
 test_expect_success 'fetch packed objects' '
        cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
-       cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
-       git --bare repack &&
-       git --bare prune-packed &&
+       (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git &&
+        git --bare repack &&
+        git --bare prune-packed
+       ) &&
        git clone $HTTPD_URL/dumb/repo_pack.git
 '
 
+test_expect_success 'fetch notices corrupt pack' '
+       cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
+       (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad1.git &&
+        p=`ls objects/pack/pack-*.pack` &&
+        chmod u+w $p &&
+        printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc
+       ) &&
+       mkdir repo_bad1.git &&
+       (cd repo_bad1.git &&
+        git --bare init &&
+        test_must_fail git --bare fetch $HTTPD_URL/dumb/repo_bad1.git &&
+        test 0 = `ls objects/pack/pack-*.pack | wc -l`
+       )
+'
+
+test_expect_success 'fetch notices corrupt idx' '
+       cp -R "$HTTPD_DOCUMENT_ROOT_PATH"/repo_pack.git "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
+       (cd "$HTTPD_DOCUMENT_ROOT_PATH"/repo_bad2.git &&
+        p=`ls objects/pack/pack-*.idx` &&
+        chmod u+w $p &&
+        printf %0256d 0 | dd of=$p bs=256 count=1 seek=1 conv=notrunc
+       ) &&
+       mkdir repo_bad2.git &&
+       (cd repo_bad2.git &&
+        git --bare init &&
+        test_must_fail git --bare fetch $HTTPD_URL/dumb/repo_bad2.git &&
+        test 0 = `ls objects/pack | wc -l`
+       )
+'
+
 test_expect_success 'did not use upload-pack service' '
        grep '/git-upload-pack' <"$HTTPD_ROOT_PATH"/access.log >act
        : >exp