X-Git-Url: https://asedeno.scripts.mit.edu/gitweb/?a=blobdiff_plain;f=builtin-pack-objects.c;h=bcefa52c69481ceb301a96af6402ca2a029144fb;hb=521537476fe99b97bfcdf1b8f0c579061af5fd3e;hp=447d492dbbfb578b8903293fa1d2a988eecfefac;hpb=afce435000030e3ad076ef1dd413e0fa314cdcbb;p=git.git diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c index 447d492db..bcefa52c6 100644 --- a/builtin-pack-objects.c +++ b/builtin-pack-objects.c @@ -23,7 +23,7 @@ #endif static const char pack_usage[] = "\ -git-pack-objects [{ -q | --progress | --all-progress }] \n\ +git pack-objects [{ -q | --progress | --all-progress }] \n\ [--max-pack-size=N] [--local] [--incremental] \n\ [--window=N] [--window-memory=N] [--depth=N] \n\ [--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\ @@ -71,13 +71,14 @@ static int reuse_delta = 1, reuse_object = 1; static int keep_unreachable, unpack_unreachable, include_tag; static int local; static int incremental; +static int ignore_packed_keep; static int allow_ofs_delta; static const char *base_name; static int progress = 1; static int window = 10; static uint32_t pack_size_limit, pack_size_limit_cfg; static int depth = 50; -static int delta_search_threads = 1; +static int delta_search_threads; static int pack_to_stdout; static int num_preferred_base; static struct progress *progress_state; @@ -194,43 +195,21 @@ static int check_pack_inflate(struct packed_git *p, int st; memset(&stream, 0, sizeof(stream)); - inflateInit(&stream); + git_inflate_init(&stream); do { in = use_pack(p, w_curs, offset, &stream.avail_in); stream.next_in = in; stream.next_out = fakebuf; stream.avail_out = sizeof(fakebuf); - st = inflate(&stream, Z_FINISH); + st = git_inflate(&stream, Z_FINISH); offset += stream.next_in - in; } while (st == Z_OK || st == Z_BUF_ERROR); - inflateEnd(&stream); + git_inflate_end(&stream); return (st == Z_STREAM_END && stream.total_out == expect && stream.total_in == len) ? 0 : -1; } -static int check_pack_crc(struct packed_git *p, struct pack_window **w_curs, - off_t offset, off_t len, unsigned int nr) -{ - const uint32_t *index_crc; - uint32_t data_crc = crc32(0, Z_NULL, 0); - - do { - unsigned int avail; - void *data = use_pack(p, w_curs, offset, &avail); - if (avail > len) - avail = len; - data_crc = crc32(data_crc, data, avail); - offset += avail; - len -= avail; - } while (len); - - index_crc = p->index_data; - index_crc += 2 + 256 + p->num_objects * (20/4) + nr; - - return data_crc != ntohl(*index_crc); -} - static void copy_pack_data(struct sha1file *f, struct packed_git *p, struct pack_window **w_curs, @@ -267,8 +246,16 @@ static unsigned long write_object(struct sha1file *f, type = entry->type; /* write limit if limited packsize and not first object */ - limit = pack_size_limit && nr_written ? - pack_size_limit - write_offset : 0; + if (!pack_size_limit || !nr_written) + limit = 0; + else if (pack_size_limit <= write_offset) + /* + * the earlier object did not fit the limit; avoid + * mistaking this with unlimited (i.e. limit = 0). + */ + limit = 1; + else + limit = pack_size_limit - write_offset; if (!entry->delta) usable_delta = 0; /* no delta */ @@ -299,6 +286,7 @@ static unsigned long write_object(struct sha1file *f, */ if (!to_reuse) { + no_reuse: if (!usable_delta) { buf = read_sha1_file(entry->idx.sha1, &type, &size); if (!buf) @@ -380,46 +368,60 @@ static unsigned long write_object(struct sha1file *f, struct revindex_entry *revidx; off_t offset; - if (entry->delta) { + if (entry->delta) type = (allow_ofs_delta && entry->delta->idx.offset) ? OBJ_OFS_DELTA : OBJ_REF_DELTA; - reused_delta++; - } hdrlen = encode_header(type, entry->size, header); + offset = entry->in_pack_offset; revidx = find_pack_revindex(p, offset); datalen = revidx[1].offset - offset; if (!pack_to_stdout && p->index_version > 1 && - check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) - die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + check_pack_crc(p, &w_curs, offset, datalen, revidx->nr)) { + error("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + offset += entry->in_pack_header_size; datalen -= entry->in_pack_header_size; + if (!pack_to_stdout && p->index_version == 1 && + check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) { + error("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); + unuse_pack(&w_curs); + goto no_reuse; + } + if (type == OBJ_OFS_DELTA) { off_t ofs = entry->idx.offset - entry->delta->idx.offset; unsigned pos = sizeof(dheader) - 1; dheader[pos] = ofs & 127; while (ofs >>= 7) dheader[--pos] = 128 | (--ofs & 127); - if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) + if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, dheader + pos, sizeof(dheader) - pos); hdrlen += sizeof(dheader) - pos; + reused_delta++; } else if (type == OBJ_REF_DELTA) { - if (limit && hdrlen + 20 + datalen + 20 >= limit) + if (limit && hdrlen + 20 + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); sha1write(f, entry->delta->idx.sha1, 20); hdrlen += 20; + reused_delta++; } else { - if (limit && hdrlen + datalen + 20 >= limit) + if (limit && hdrlen + datalen + 20 >= limit) { + unuse_pack(&w_curs); return 0; + } sha1write(f, header, hdrlen); } - - if (!pack_to_stdout && p->index_version == 1 && - check_pack_inflate(p, &w_curs, offset, datalen, entry->size)) - die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1)); copy_pack_data(f, p, &w_curs, offset, datalen); unuse_pack(&w_curs); reused++; @@ -432,25 +434,22 @@ static unsigned long write_object(struct sha1file *f, return hdrlen + datalen; } -static off_t write_one(struct sha1file *f, +static int write_one(struct sha1file *f, struct object_entry *e, - off_t offset) + off_t *offset) { unsigned long size; /* offset is non zero if object is written already. */ if (e->idx.offset || e->preferred_base) - return offset; + return 1; /* if we are deltified, write out base object first. */ - if (e->delta) { - offset = write_one(f, e->delta, offset); - if (!offset) - return 0; - } + if (e->delta && !write_one(f, e->delta, offset)) + return 0; - e->idx.offset = offset; - size = write_object(f, e, offset); + e->idx.offset = *offset; + size = write_object(f, e, *offset); if (!size) { e->idx.offset = 0; return 0; @@ -458,9 +457,10 @@ static off_t write_one(struct sha1file *f, written_list[nr_written++] = &e->idx; /* make sure off_t is sufficiently large not to wrap */ - if (offset > offset + size) + if (*offset > *offset + size) die("pack too large for current definition of off_t"); - return offset + size; + *offset += size; + return 1; } /* forward declaration for write_pack_file */ @@ -470,7 +470,7 @@ static void write_pack_file(void) { uint32_t i = 0, j; struct sha1file *f; - off_t offset, offset_one, last_obj_offset = 0; + off_t offset; struct pack_header hdr; uint32_t nr_remaining = nr_result; time_t last_mtime = 0; @@ -488,9 +488,8 @@ static void write_pack_file(void) } else { char tmpname[PATH_MAX]; int fd; - snprintf(tmpname, sizeof(tmpname), - "%s/tmp_pack_XXXXXX", get_object_directory()); - fd = xmkstemp(tmpname); + fd = odb_mkstemp(tmpname, sizeof(tmpname), + "pack/tmp_pack_XXXXXX"); pack_tmp_name = xstrdup(tmpname); f = sha1fd(fd, pack_tmp_name); } @@ -502,11 +501,8 @@ static void write_pack_file(void) offset = sizeof(hdr); nr_written = 0; for (; i < nr_objects; i++) { - last_obj_offset = offset; - offset_one = write_one(f, objects + i, offset); - if (!offset_one) + if (!write_one(f, objects + i, &offset)) break; - offset = offset_one; display_progress(progress_state, written); } @@ -519,9 +515,9 @@ static void write_pack_file(void) } else if (nr_written == nr_remaining) { sha1close(f, sha1, CSUM_FSYNC); } else { - int fd = sha1close(f, NULL, 0); - fixup_pack_header_footer(fd, sha1, pack_tmp_name, nr_written); - fsync_or_die(fd, pack_tmp_name); + int fd = sha1close(f, sha1, 0); + fixup_pack_header_footer(fd, sha1, pack_tmp_name, + nr_written, sha1, offset); close(fd); } @@ -538,6 +534,7 @@ static void write_pack_file(void) snprintf(tmpname, sizeof(tmpname), "%s-%s.pack", base_name, sha1_to_hex(sha1)); + free_pack_by_name(tmpname); if (adjust_perm(pack_tmp_name, mode)) die("unable to make temporary pack file readable: %s", strerror(errno)); @@ -590,7 +587,8 @@ static void write_pack_file(void) free(written_list); stop_progress(&progress_state); if (written != nr_result) - die("wrote %u objects while expecting %u", written, nr_result); + die("wrote %"PRIu32" objects while expecting %"PRIu32, + written, nr_result); /* * We have scanned through [0 ... i). Since we have written * the correct number of objects, the remaining [i ... nr_objects) @@ -602,7 +600,8 @@ static void write_pack_file(void) j += !e->idx.offset && !e->preferred_base; } if (j) - die("wrote %u objects as expected but %u unwritten", written, j); + die("wrote %"PRIu32" objects as expected but %"PRIu32 + " unwritten", written, j); } static int locate_object_entry_hash(const unsigned char *sha1) @@ -715,6 +714,9 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type, return 0; } + if (!exclude && local && has_loose_object_nonlocal(sha1)) + return 0; + for (p = packed_git; p; p = p->next) { off_t offset = find_pack_entry_one(sha1, p); if (offset) { @@ -728,6 +730,8 @@ static int add_object_entry(const unsigned char *sha1, enum object_type type, return 0; if (local && !p->pack_local) return 0; + if (ignore_packed_keep && p->pack_local && p->pack_keep) + return 0; } } @@ -1027,9 +1031,11 @@ static void check_object(struct object_entry *entry) * We want in_pack_type even if we do not reuse delta * since non-delta representations could still be reused. */ - used = unpack_object_header_gently(buf, avail, + used = unpack_object_header_buffer(buf, avail, &entry->in_pack_type, &entry->size); + if (used == 0) + goto give_up; /* * Determine if this is a delta and if so whether we can @@ -1041,6 +1047,8 @@ static void check_object(struct object_entry *entry) /* Not a delta hence we've already got all we need. */ entry->type = entry->in_pack_type; entry->in_pack_header_size = used; + if (entry->type < OBJ_COMMIT || entry->type > OBJ_BLOB) + goto give_up; unuse_pack(&w_curs); return; case OBJ_REF_DELTA: @@ -1057,19 +1065,25 @@ static void check_object(struct object_entry *entry) ofs = c & 127; while (c & 128) { ofs += 1; - if (!ofs || MSB(ofs, 7)) - die("delta base offset overflow in pack for %s", - sha1_to_hex(entry->idx.sha1)); + if (!ofs || MSB(ofs, 7)) { + error("delta base offset overflow in pack for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } c = buf[used_0++]; ofs = (ofs << 7) + (c & 127); } - if (ofs >= entry->in_pack_offset) - die("delta base offset out of bound for %s", - sha1_to_hex(entry->idx.sha1)); ofs = entry->in_pack_offset - ofs; + if (ofs <= 0 || ofs >= entry->in_pack_offset) { + error("delta base offset out of bound for %s", + sha1_to_hex(entry->idx.sha1)); + goto give_up; + } if (reuse_delta && !entry->preferred_base) { struct revindex_entry *revidx; revidx = find_pack_revindex(p, ofs); + if (!revidx) + goto give_up; base_ref = nth_packed_object_sha1(p, revidx->nr); } entry->in_pack_header_size = used + used_0; @@ -1089,6 +1103,7 @@ static void check_object(struct object_entry *entry) */ entry->type = entry->in_pack_type; entry->delta = base_entry; + entry->delta_size = entry->size; entry->delta_sibling = base_entry->delta_child; base_entry->delta_child = entry; unuse_pack(&w_curs); @@ -1103,6 +1118,8 @@ static void check_object(struct object_entry *entry) */ entry->size = get_size_from_delta(p, &w_curs, entry->in_pack_offset + entry->in_pack_header_size); + if (entry->size == 0) + goto give_up; unuse_pack(&w_curs); return; } @@ -1112,13 +1129,17 @@ static void check_object(struct object_entry *entry) * with sha1_object_info() to find about the object type * at this point... */ + give_up: unuse_pack(&w_curs); } entry->type = sha1_object_info(entry->idx.sha1, &entry->size); - if (entry->type < 0) - die("unable to get type of object %s", - sha1_to_hex(entry->idx.sha1)); + /* + * The error condition is checked in prepare_pack(). This is + * to permit a missing preferred base object to be ignored + * as a preferred base. Doing so can result in a larger + * pack file, but the transfer will still take place. + */ } static int pack_offset_sort(const void *_a, const void *_b) @@ -1148,8 +1169,6 @@ static void get_object_details(void) sorted_by_offset[i] = objects + i; qsort(sorted_by_offset, nr_objects, sizeof(*sorted_by_offset), pack_offset_sort); - init_pack_revindex(); - for (i = 0; i < nr_objects; i++) check_object(sorted_by_offset[i]); @@ -1393,15 +1412,13 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, int window, int depth, unsigned *processed) { uint32_t i, idx = 0, count = 0; - unsigned int array_size = window * sizeof(struct unpacked); struct unpacked *array; unsigned long mem_usage = 0; - array = xmalloc(array_size); - memset(array, 0, array_size); + array = xcalloc(window, sizeof(struct unpacked)); for (;;) { - struct object_entry *entry = *list++; + struct object_entry *entry; struct unpacked *n = array + idx; int j, max_depth, best_base = -1; @@ -1410,6 +1427,7 @@ static void find_deltas(struct object_entry **list, unsigned *list_size, progress_unlock(); break; } + entry = *list++; (*list_size)--; if (!entry->preferred_base) { (*processed)++; @@ -1593,11 +1611,18 @@ static void ll_find_deltas(struct object_entry **list, unsigned list_size, find_deltas(list, &list_size, window, depth, processed); return; } + if (progress > pack_to_stdout) + fprintf(stderr, "Delta compression using %d threads.\n", + delta_search_threads); /* Partition the work amongst work threads. */ for (i = 0; i < delta_search_threads; i++) { unsigned sub_size = list_size / (delta_search_threads - i); + /* don't use too small segments or no deltas will be found */ + if (sub_size < 2*window && i+1 < delta_search_threads) + sub_size = 0; + p[i].window = window; p[i].depth = depth; p[i].processed = processed; @@ -1718,10 +1743,21 @@ static int add_ref_tag(const char *path, const unsigned char *sha1, int flag, vo static void prepare_pack(int window, int depth) { struct object_entry **delta_list; - uint32_t i, n, nr_deltas; + uint32_t i, nr_deltas; + unsigned n; get_object_details(); + /* + * If we're locally repacking then we need to be doubly careful + * from now on in order to make sure no stealth corruption gets + * propagated to the new pack. Clients receiving streamed packs + * should validate everything they get anyway so no need to incur + * the additional cost here in that case. + */ + if (!pack_to_stdout) + do_check_packed_object_crc = 1; + if (!nr_objects || !window || !depth) return; @@ -1743,8 +1779,20 @@ static void prepare_pack(int window, int depth) if (entry->no_try_delta) continue; - if (!entry->preferred_base) + if (!entry->preferred_base) { nr_deltas++; + if (entry->type < 0) + die("unable to get type of object %s", + sha1_to_hex(entry->idx.sha1)); + } else { + if (entry->type < 0) { + /* + * This object is not found, but we + * don't have to include it anyway. + */ + continue; + } + } delta_list[n++] = entry; } @@ -1809,7 +1857,8 @@ static int git_pack_config(const char *k, const char *v, void *cb) if (!strcmp(k, "pack.indexversion")) { pack_idx_default_version = git_config_int(k, v); if (pack_idx_default_version > 2) - die("bad pack.indexversion=%d", pack_idx_default_version); + die("bad pack.indexversion=%"PRIu32, + pack_idx_default_version); return 0; } if (!strcmp(k, "pack.packsizelimit")) { @@ -1890,7 +1939,7 @@ static void mark_in_pack_object(struct object *object, struct packed_git *p, str /* * Compare the objects in the offset order, in order to emulate the - * "git-rev-list --objects" output that produced the pack originally. + * "git rev-list --objects" output that produced the pack originally. */ static int ofscmp(const void *a_, const void *b_) { @@ -2059,6 +2108,10 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) incremental = 1; continue; } + if (!strcmp("--honor-pack-keep", arg)) { + ignore_packed_keep = 1; + continue; + } if (!prefixcmp(arg, "--compression=")) { char *end; int level = strtoul(arg+14, &end, 0); @@ -2243,7 +2296,8 @@ int cmd_pack_objects(int argc, const char **argv, const char *prefix) prepare_pack(window, depth); write_pack_file(); if (progress) - fprintf(stderr, "Total %u (delta %u), reused %u (delta %u)\n", + fprintf(stderr, "Total %"PRIu32" (delta %"PRIu32")," + " reused %"PRIu32" (delta %"PRIu32")\n", written, written_delta, reused, reused_delta); return 0; }