Merge branch 'sv/first-parent'

[git.git] / builtin-pack-objects.c

diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 69ac27788b37c89ede1dadf293c13de05fb03e82..e20851e1c9c9ad517314794b7c8d4e3615afa9c5 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
@@ -43,6 +43,7 @@ struct object_entry {
                                             */
        void *delta_data;       /* cached delta (uncompressed) */
        unsigned long delta_size;       /* delta data size (uncompressed) */
+       unsigned long z_delta_size;     /* delta data size (compressed) */
        unsigned int hash;      /* name hint hash */
        enum object_type type;
        enum object_type in_pack_type;  /* could be delta */
@@ -124,6 +125,32 @@ static void *get_delta(struct object_entry *entry)
        return delta_buf;
 }
 
+static unsigned long do_compress(void **pptr, unsigned long size)
+{
+       z_stream stream;
+       void *in, *out;
+       unsigned long maxsize;
+
+       memset(&stream, 0, sizeof(stream));
+       deflateInit(&stream, pack_compression_level);
+       maxsize = deflateBound(&stream, size);
+
+       in = *pptr;
+       out = xmalloc(maxsize);
+       *pptr = out;
+
+       stream.next_in = in;
+       stream.avail_in = size;
+       stream.next_out = out;
+       stream.avail_out = maxsize;
+       while (deflate(&stream, Z_FINISH) == Z_OK)
+               ; /* nothing */
+       deflateEnd(&stream);
+
+       free(in);
+       return stream.total_out;
+}
+
 /*
  * The per-object header is a pretty dense thing, which is
  *  - first byte: low four bits are "size", then three bits of "type",
@@ -226,11 +253,10 @@ static unsigned long write_object(struct sha1file *f,
                                  struct object_entry *entry,
                                  off_t write_offset)
 {
-       unsigned long size, limit;
+       unsigned long size, limit, datalen;
        void *buf;
        unsigned char header[10], dheader[10];
        unsigned hdrlen;
-       off_t datalen;
        enum object_type type;
        int usable_delta, to_reuse;
 
@@ -272,13 +298,17 @@ static unsigned long write_object(struct sha1file *f,
                                 */
 
        if (!to_reuse) {
-               z_stream stream;
-               unsigned long maxsize;
-               void *out;
                if (!usable_delta) {
                        buf = read_sha1_file(entry->idx.sha1, &type, &size);
                        if (!buf)
                                die("unable to read %s", sha1_to_hex(entry->idx.sha1));
+                       /*
+                        * make sure no cached delta data remains from a
+                        * previous attempt before a pack split occured.
+                        */
+                       free(entry->delta_data);
+                       entry->delta_data = NULL;
+                       entry->z_delta_size = 0;
                } else if (entry->delta_data) {
                        size = entry->delta_size;
                        buf = entry->delta_data;
@@ -291,20 +321,11 @@ static unsigned long write_object(struct sha1file *f,
                        type = (allow_ofs_delta && entry->delta->idx.offset) ?
                                OBJ_OFS_DELTA : OBJ_REF_DELTA;
                }
-               /* compress the data to store and put compressed length in datalen */
-               memset(&stream, 0, sizeof(stream));
-               deflateInit(&stream, pack_compression_level);
-               maxsize = deflateBound(&stream, size);
-               out = xmalloc(maxsize);
-               /* Compress it */
-               stream.next_in = buf;
-               stream.avail_in = size;
-               stream.next_out = out;
-               stream.avail_out = maxsize;
-               while (deflate(&stream, Z_FINISH) == Z_OK)
-                       /* nothing */;
-               deflateEnd(&stream);
-               datalen = stream.total_out;
+
+               if (entry->z_delta_size)
+                       datalen = entry->z_delta_size;
+               else
+                       datalen = do_compress(&buf, size);
 
                /*
                 * The object header is a byte of 'type' followed by zero or
@@ -324,7 +345,6 @@ static unsigned long write_object(struct sha1file *f,
                        while (ofs >>= 7)
                                dheader[--pos] = 128 | (--ofs & 127);
                        if (limit && hdrlen + sizeof(dheader) - pos + datalen + 20 >= limit) {
-                               free(out);
                                free(buf);
                                return 0;
                        }
@@ -337,7 +357,6 @@ static unsigned long write_object(struct sha1file *f,
                         * an additional 20 bytes for the base sha1.
                         */
                        if (limit && hdrlen + 20 + datalen + 20 >= limit) {
-                               free(out);
                                free(buf);
                                return 0;
                        }
@@ -346,14 +365,12 @@ static unsigned long write_object(struct sha1file *f,
                        hdrlen += 20;
                } else {
                        if (limit && hdrlen + datalen + 20 >= limit) {
-                               free(out);
                                free(buf);
                                return 0;
                        }
                        sha1write(f, header, hdrlen);
                }
-               sha1write(f, out, datalen);
-               free(out);
+               sha1write(f, buf, datalen);
                free(buf);
        }
        else {
@@ -1442,11 +1459,34 @@ static void find_deltas(struct object_entry **list, unsigned *list_size,
                                best_base = other_idx;
                }
 
+               /*
+                * If we decided to cache the delta data, then it is best
+                * to compress it right away.  First because we have to do
+                * it anyway, and doing it here while we're threaded will
+                * save a lot of time in the non threaded write phase,
+                * as well as allow for caching more deltas within
+                * the same cache size limit.
+                * ...
+                * But only if not writing to stdout, since in that case
+                * the network is most likely throttling writes anyway,
+                * and therefore it is best to go to the write phase ASAP
+                * instead, as we can afford spending more time compressing
+                * between writes at that moment.
+                */
+               if (entry->delta_data && !pack_to_stdout) {
+                       entry->z_delta_size = do_compress(&entry->delta_data,
+                                                         entry->delta_size);
+                       cache_lock();
+                       delta_cache_size -= entry->delta_size;
+                       delta_cache_size += entry->z_delta_size;
+                       cache_unlock();
+               }
+
                /* if we made n a delta, and if n is already at max
                 * depth, leaving it in the window is pointless.  we
                 * should evict it first.
                 */
-               if (entry->delta && depth <= n->depth)
+               if (entry->delta && max_depth <= n->depth)
                        continue;
 
                /*