X-Git-Url: https://asedeno.scripts.mit.edu/gitweb/?a=blobdiff_plain;f=builtin-pack-objects.c;h=4c91e944c21df3d5e02535115cdd6098ee49d86d;hb=92396402e2d547f8805dc24db53a4c725796810e;hp=a6adc8c271e20374b86462391016897f1d17a0f3;hpb=a1eb73d917e15cd97314e0a39cbe857329339a96;p=git.git diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index a6adc8c27..4c91e944c 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -22,15 +22,15 @@ #include #endif -static const char pack_usage[] = "\ -git pack-objects [{ -q | --progress | --all-progress }] \n\ - [--max-pack-size=N] [--local] [--incremental] \n\ - [--window=N] [--window-memory=N] [--depth=N] \n\ - [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\ - [--threads=N] [--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\ - [--stdout | base-name] [--include-tag] \n\ - [--keep-unreachable | --unpack-unreachable] \n\ - [idx.sha1, &type, &size); if (!buf) die("unable to read %s", sha1_to_hex(entry->idx.sha1)); /* * make sure no cached delta data remains from a - * previous attempt before a pack split occured. + * previous attempt before a pack split occurred. */ free(entry->delta_data); entry->delta_data = NULL; @@ -367,46 +368,60 @@ static unsigned long write_object(struct sha1file *f, struct revindex_entry *revidx; off_t offset; - if (entry->delta) { + if (entry->delta) type = (allow_ofs_delta && entry->delta->idx.offset) ? OBJ_OFS_DELTA : OBJ_REF_DELTA; - reused_delta++; - } hdrlen = encode_header(type, entry->size, header); + offset = entry->in_pack_offset; revidx = find_pack_revindex(p, offset); datalen = revidx[1].offset - offset; if (!pack_to_stdout && p->index_version > 1 && - check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) - die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { + error("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + offset += entry->in_pack_header_size; datalen -= entry->in_pack_header_size; + if (!pack_to_stdout && p->index_version == 1 && + check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { + error("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + if (type == OBJ_OFS_DELTA) { off_t ofs = entry->idx.offset - entry->delta->idx.offset; unsigned pos = sizeof(dheader) - 1; dheader[pos] = ofs & 127; while (ofs >>= 7) dheader[--pos] = 128 | (--ofs & 127); - if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) + if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, dheader + pos, sizeof(dheader) - pos); hdrlen += sizeof(dheader) - pos; + reused_delta++; } else if (type == OBJ_REF_DELTA) { - if (limit && hdrlen + 20 + datalen + 20 >= limit) + if (limit && hdrlen + 20 + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, entry->delta->idx.sha1, 20); hdrlen += 20; + reused_delta++; } else { - if (limit && hdrlen + datalen + 20 >= limit) + if (limit && hdrlen + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); } - - if (!pack_to_stdout && p->index_version == 1 && - check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) - die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); copy_pack_data(f, p, &w_curs, offset, datalen); unuse_pack(&w_curs); reused++; @@ -473,9 +488,8 @@ static void write_pack_file(void) } else { char tmpname[PATH_MAX]; int fd; - snprintf(tmpname, sizeof(tmpname), - "%s/pack/tmp_pack_XXXXXX", get_object_directory()); - fd = xmkstemp(tmpname); + fd = odb_mkstemp(tmpname, sizeof(tmpname), + "pack/tmp_pack_XXXXXX"); pack_tmp_name = xstrdup(tmpname); f = sha1fd(fd, pack_tmp_name); } @@ -522,11 +536,9 @@ static void write_pack_file(void) base_name, sha1_to_hex(sha1)); free_pack_by_name(tmpname); if (adjust_perm(pack_tmp_name, mode)) - die("unable to make temporary pack file readable: %s", - strerror(errno)); + die_errno("unable to make temporary pack file readable"); if (rename(pack_tmp_name, tmpname)) - die("unable to rename temporary pack file: %s", - strerror(errno)); + die_errno("unable to rename temporary pack file"); /* * Packs are runtime accessed in their mtime @@ -552,11 +564,9 @@ static void write_pack_file(void) snprintf(tmpname, sizeof(tmpname), "%s-%s.idx", base_name, sha1_to_hex(sha1)); if (adjust_perm(idx_tmp_name, mode)) - die("unable to make temporary index file readable: %s", - strerror(errno)); + die_errno("unable to make temporary index file readable"); if (rename(idx_tmp_name, tmpname)) - die("unable to rename temporary index file: %s", - strerror(errno)); + die_errno("unable to rename temporary index file"); free(idx_tmp_name); free(pack_tmp_name); @@ -639,8 +649,7 @@ static void rehash_objects(void) static unsigned name_hash(const char *name) { - unsigned char c; - unsigned hash = 0; + unsigned c, hash = 0; if (!name) return 0; @@ -999,6 +1008,33 @@ static void add_preferred_base(unsigned char *sha1) it->pcache.tree_size = size; } +static void cleanup_preferred_base(void) +{ + struct pbase_tree *it; + unsigned i; + + it = pbase_tree; + pbase_tree = NULL; + while (it) { + struct pbase_tree *this = it; + it = this->next; + free(this->pcache.tree_data); + free(this); + } + + for (i = 0; i < ARRAY_SIZE(pbase_tree_cache); i++) { + if (!pbase_tree_cache[i]) + continue; + free(pbase_tree_cache[i]->tree_data); + free(pbase_tree_cache[i]); + pbase_tree_cache[i] = NULL; + } + + free(done_pbase_paths); + done_pbase_paths = NULL; + done_pbase_paths_num = done_pbase_paths_alloc = 0; +} + static void check_object(struct object_entry *entry) { if (entry->in_pack) { @@ -1017,9 +1053,11 @@ static void check_object(struct object_entry *entry) * We want in_pack_type even if we do not reuse delta * since non-delta representations could still be reused. */ - used = unpack_object_header_gently(buf, avail, + used = unpack_object_header_buffer(buf, avail, &entry->in_pack_type, &entry->size); + if (used == 0) + goto give_up; /* * Determine if this is a delta and if so whether we can @@ -1031,6 +1069,8 @@ static void check_object(struct object_entry *entry) /* Not a delta hence we've already got all we need. */ entry->type = entry->in_pack_type; entry->in_pack_header_size = used; + if (entry->type < OBJ_COMMIT || entry->type > OBJ_BLOB) + goto give_up; unuse_pack(&w_curs); return; case OBJ_REF_DELTA: @@ -1047,19 +1087,25 @@ static void check_object(struct object_entry *entry) ofs = c & 127; while (c & 128) { ofs += 1; - if (!ofs || MSB(ofs, 7)) - die("delta base offset overflow in pack for %s", - sha1_to_hex(entry->idx.sha1)); + if (!ofs || MSB(ofs, 7)) { + error("delta base offset overflow in pack for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } c = buf[used_0++]; ofs = (ofs << 7) + (c & 127); } - if (ofs >= entry->in_pack_offset) - die("delta base offset out of bound for %s", - sha1_to_hex(entry->idx.sha1)); ofs = entry->in_pack_offset - ofs; + if (ofs <= 0 || ofs >= entry->in_pack_offset) { + error("delta base offset out of bound for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } if (reuse_delta && !entry->preferred_base) { struct revindex_entry *revidx; revidx = find_pack_revindex(p, ofs); + if (!revidx) + goto give_up; base_ref = nth_packed_object_sha1(p, revidx->nr); } entry->in_pack_header_size = used + used_0; @@ -1079,6 +1125,7 @@ static void check_object(struct object_entry *entry) */ entry->type = entry->in_pack_type; entry->delta = base_entry; + entry->delta_size = entry->size; entry->delta_sibling = base_entry->delta_child; base_entry->delta_child = entry; unuse_pack(&w_curs); @@ -1093,6 +1140,8 @@ static void check_object(struct object_entry *entry) */ entry->size = get_size_from_delta(p, &w_curs, entry->in_pack_offset + entry->in_pack_header_size); + if (entry->size == 0) + goto give_up; unuse_pack(&w_curs); return; } @@ -1102,6 +1151,7 @@ static void check_object(struct object_entry *entry) * with sha1_object_info() to find about the object type * at this point... */ + give_up: unuse_pack(&w_curs); } @@ -1384,12 +1434,10 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, int window, int depth, unsigned *processed) { uint32_t i, idx = 0, count = 0; - unsigned int array_size = window * sizeof(struct unpacked); struct unpacked *array; unsigned long mem_usage = 0; - array = xmalloc(array_size); - memset(array, 0, array_size); + array = xcalloc(window, sizeof(struct unpacked)); for (;;) { struct object_entry *entry; @@ -1578,18 +1626,28 @@ static void *threaded_find_deltas(void *arg) static void ll_find_deltas(struct object_entry **list, unsigned list_size, int window, int depth, unsigned *processed) { - struct thread_params p[delta_search_threads]; + struct thread_params *p; int i, ret, active_threads = 0; + if (!delta_search_threads) /* --threads=0 means autodetect */ + delta_search_threads = online_cpus(); if (delta_search_threads <= 1) { find_deltas(list, &list_size, window, depth, processed); return; } + if (progress > pack_to_stdout) + fprintf(stderr, "Delta compression using up to %d threads.\n", + delta_search_threads); + p = xcalloc(delta_search_threads, sizeof(*p)); /* Partition the work amongst work threads. */ for (i = 0; i < delta_search_threads; i++) { unsigned sub_size = list_size / (delta_search_threads - i); + /* don't use too small segments or no deltas will be found */ + if (sub_size < 2*window && i+1 < delta_search_threads) + sub_size = 0; + p[i].window = window; p[i].depth = depth; p[i].processed = processed; @@ -1689,6 +1747,7 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, active_threads--; } } + free(p); } #else @@ -1715,6 +1774,16 @@ static void prepare_pack(int window, int depth) get_object_details(); + /* + * If we're locally repacking then we need to be doubly careful + * from now on in order to make sure no stealth corruption gets + * propagated to the new pack. Clients receiving streamed packs + * should validate everything they get anyway so no need to incur + * the additional cost here in that case. + */ + if (!pack_to_stdout) + do_check_packed_object_crc = 1; + if (!nr_objects || !window || !depth) return; @@ -1741,6 +1810,14 @@ static void prepare_pack(int window, int depth) if (entry->type < 0) die("unable to get type of object %s", sha1_to_hex(entry->idx.sha1)); + } else { + if (entry->type < 0) { + /* + * This object is not found, but we + * don't have to include it anyway. + */ + continue; + } } delta_list[n++] = entry; @@ -1762,7 +1839,7 @@ static void prepare_pack(int window, int depth) static int git_pack_config(const char *k, const char *v, void *cb) { - if(!strcmp(k, "pack.window")) { + if (!strcmp(k, "pack.window")) { window = git_config_int(k, v); return 0; } @@ -1829,7 +1906,7 @@ static void read_object_list_from_stdin(void) if (!ferror(stdin)) die("fgets returned NULL, not EOF, not error!"); if (errno != EINTR) - die("fgets: %s", strerror(errno)); + die_errno("fgets"); clearerr(stdin); continue; } @@ -1850,19 +1927,25 @@ static void read_object_list_from_stdin(void) #define OBJECT_ADDED (1u<<20) -static void show_commit(struct commit *commit) +static void show_commit(struct commit *commit, void *data) { add_object_entry(commit->object.sha1, OBJ_COMMIT, NULL, 0); commit->object.flags |= OBJECT_ADDED; } -static void show_object(struct object_array_entry *p) +static void show_object(struct object *obj, const struct name_path *path, const char *last) { - add_preferred_base_object(p->name); - add_object_entry(p->item->sha1, p->item->type, p->name, 0); - p->item->flags |= OBJECT_ADDED; - free((char *)p->name); - p->name = NULL; + char *name = path_name(path, last); + + add_preferred_base_object(name); + add_object_entry(obj->sha1, obj->type, name, 0); + obj->flags |= OBJECT_ADDED; + + /* + * We will have generated the hash from the name, + * but not saved a pointer to it - we can free it + */ + free((char *)name); } static void show_edge(struct commit *commit) @@ -1917,11 +2000,7 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs) const unsigned char *sha1; struct object *o; - for (i = 0; i < revs->num_ignore_packed; i++) { - if (matches_pack_name(p, revs->ignore_packed[i])) - break; - } - if (revs->num_ignore_packed <= i) + if (!p->pack_local || p->pack_keep) continue; if (open_pack_index(p)) die("cannot open pack index"); @@ -1950,6 +2029,29 @@ static void add_objects_in_unpacked_packs(struct rev_info *revs) free(in_pack.array); } +static int has_sha1_pack_kept_or_nonlocal(const unsigned char *sha1) +{ + static struct packed_git *last_found = (void *)1; + struct packed_git *p; + + p = (last_found != (void *)1) ? last_found : packed_git; + + while (p) { + if ((!p->pack_local || p->pack_keep) && + find_pack_entry_one(sha1, p)) { + last_found = p; + return 1; + } + if (p == last_found) + p = packed_git; + else + p = p->next; + if (p == last_found) + p = p->next; + } + return 0; +} + static void loosen_unused_packed_objects(struct rev_info *revs) { struct packed_git *p; @@ -1957,11 +2059,7 @@ static void loosen_unused_packed_objects(struct rev_info *revs) const unsigned char *sha1; for (p = packed_git; p; p = p->next) { - for (i = 0; i < revs->num_ignore_packed; i++) { - if (matches_pack_name(p, revs->ignore_packed[i])) - break; - } - if (revs->num_ignore_packed <= i) + if (!p->pack_local || p->pack_keep) continue; if (open_pack_index(p)) @@ -1969,7 +2067,8 @@ static void loosen_unused_packed_objects(struct rev_info *revs) for (i = 0; i < p->num_objects; i++) { sha1 = nth_packed_object_sha1(p, i); - if (!locate_object_entry(sha1)) + if (!locate_object_entry(sha1) && + !has_sha1_pack_kept_or_nonlocal(sha1)) if (force_object_loose(sha1, p->mtime)) die("unable to force loose object"); } @@ -2006,7 +2105,7 @@ static void get_object_list(int ac, const char **av) if (prepare_revision_walk(&revs)) die("revision walk setup failed"); mark_edges_uninteresting(revs.commits, &revs, show_edge); - traverse_commit_list(&revs, show_commit, show_object); + traverse_commit_list(&revs, show_commit, show_object, NULL); if (keep_unreachable) add_objects_in_unpacked_packs(&revs); @@ -2030,6 +2129,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) int rp_ac_alloc = 64; int rp_ac; + read_replace_refs = 0; + rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av)); rp_av[0] = "pack-objects"; @@ -2159,7 +2260,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) continue; } if (!strcmp("--unpacked", arg) || - !prefixcmp(arg, "--unpacked=") || !strcmp("--reflog", arg) || !strcmp("--all", arg)) { use_internal_rev_list = 1; @@ -2188,6 +2288,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) die("bad %s", arg); continue; } + if (!strcmp(arg, "--keep-true-parents")) { + grafts_replace_parents = 0; + continue; + } usage(pack_usage); } @@ -2222,11 +2326,6 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) if (keep_unreachable && unpack_unreachable) die("--keep-unreachable and --unpack-unreachable are incompatible."); -#ifdef THREADED_DELTA_SEARCH - if (!delta_search_threads) /* --threads=0 means autodetect */ - delta_search_threads = online_cpus(); -#endif - prepare_packed_git(); if (progress) @@ -2237,6 +2336,7 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) rp_av[rp_ac] = NULL; get_object_list(rp_ac, rp_av); } + cleanup_preferred_base(); if (include_tag && nr_result) for_each_ref(add_ref_tag, NULL); stop_progress(&progress_state);