fast-import.c

   1 /*
   2 Format of STDIN stream:
   3
   4   stream ::= cmd*;
   5
   6   cmd ::= new_blob
   7         | new_branch
   8         | new_commit
   9         | new_tag
  10         ;
  11
  12   new_blob ::= 'blob' lf
  13         mark?
  14     file_content;
  15   file_content ::= data;
  16
  17   new_branch ::= 'branch' sp ref_str lf
  18     ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
  19     lf;
  20
  21   new_commit ::= 'commit' sp ref_str lf
  22         mark?
  23         ('author' sp name '<' email '>' ts tz lf)?
  24         'committer' sp name '<' email '>' ts tz lf
  25         commit_msg
  26     file_change*
  27     lf;
  28   commit_msg ::= data;
  29
  30   file_change ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf
  31                 | 'D' sp path_str lf
  32                 ;
  33   mode ::= '644' | '755';
  34
  35   new_tag ::= 'tag' sp tag_str lf
  36     'from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf
  37         'tagger' sp name '<' email '>' ts tz lf
  38     tag_msg;
  39   tag_msg ::= data;
  40
  41      # note: the first idnum in a stream should be 1 and subsequent
  42      # idnums should not have gaps between values as this will cause
  43      # the stream parser to reserve space for the gapped values.  An
  44          # idnum can be updated in the future to a new object by issuing
  45      # a new mark directive with the old idnum.
  46          #
  47   mark ::= 'mark' sp idnum lf;
  48
  49      # note: declen indicates the length of binary_data in bytes.
  50      # declen does not include the lf preceeding or trailing the
  51      # binary data.
  52      #
  53   data ::= 'data' sp declen lf
  54     binary_data
  55         lf;
  56
  57      # note: quoted strings are C-style quoting supporting \c for
  58      # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
  59          # is the signed byte value in octal.  Note that the only
  60      # characters which must actually be escaped to protect the
  61      # stream formatting is: \, " and LF.  Otherwise these values
  62          # are UTF8.
  63      #
  64   ref_str     ::= ref     | '"' quoted(ref)     '"' ;
  65   sha1exp_str ::= sha1exp | '"' quoted(sha1exp) '"' ;
  66   tag_str     ::= tag     | '"' quoted(tag)     '"' ;
  67   path_str    ::= path    | '"' quoted(path)    '"' ;
  68
  69   declen ::= # unsigned 32 bit value, ascii base10 notation;
  70   binary_data ::= # file content, not interpreted;
  71
  72   sp ::= # ASCII space character;
  73   lf ::= # ASCII newline (LF) character;
  74
  75      # note: a colon (':') must precede the numerical value assigned to
  76          # an idnum.  This is to distinguish it from a ref or tag name as
  77      # GIT does not permit ':' in ref or tag strings.
  78          #
  79   idnum   ::= ':' declen;
  80   path    ::= # GIT style file path, e.g. "a/b/c";
  81   ref     ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
  82   tag     ::= # GIT tag name, e.g. "FIREFOX_1_5";
  83   sha1exp ::= # Any valid GIT SHA1 expression;
  84   hexsha1 ::= # SHA1 in hexadecimal format;
  85
  86      # note: name and email are UTF8 strings, however name must not
  87          # contain '<' or lf and email must not contain any of the
  88      # following: '<', '>', lf.
  89          #
  90   name  ::= # valid GIT author/committer name;
  91   email ::= # valid GIT author/committer email;
  92   ts    ::= # time since the epoch in seconds, ascii base10 notation;
  93   tz    ::= # GIT style timezone;
  94 */
  95
  96 #include "builtin.h"
  97 #include "cache.h"
  98 #include "object.h"
  99 #include "blob.h"
 100 #include "tree.h"
 101 #include "delta.h"
 102 #include "pack.h"
 103 #include "refs.h"
 104 #include "csum-file.h"
 105 #include "strbuf.h"
 106 #include "quote.h"
 107
 108 struct object_entry
 109 {
 110         struct object_entry *next;
 111         enum object_type type;
 112         unsigned long offset;
 113         unsigned char sha1[20];
 114 };
 115
 116 struct object_entry_pool
 117 {
 118         struct object_entry_pool *next_pool;
 119         struct object_entry *next_free;
 120         struct object_entry *end;
 121         struct object_entry entries[FLEX_ARRAY]; /* more */
 122 };
 123
 124 struct last_object
 125 {
 126         void *data;
 127         unsigned int len;
 128         unsigned int depth;
 129         unsigned char sha1[20];
 130 };
 131
 132 struct mem_pool
 133 {
 134         struct mem_pool *next_pool;
 135         char *next_free;
 136         char *end;
 137         char space[FLEX_ARRAY]; /* more */
 138 };
 139
 140 struct atom_str
 141 {
 142         struct atom_str *next_atom;
 143         int str_len;
 144         char str_dat[FLEX_ARRAY]; /* more */
 145 };
 146
 147 struct tree_content;
 148 struct tree_entry
 149 {
 150         struct tree_content *tree;
 151         struct atom_str* name;
 152         unsigned int mode;
 153         unsigned char sha1[20];
 154 };
 155
 156 struct tree_content
 157 {
 158         unsigned int entry_capacity; /* must match avail_tree_content */
 159         unsigned int entry_count;
 160         struct tree_entry *entries[FLEX_ARRAY]; /* more */
 161 };
 162
 163 struct avail_tree_content
 164 {
 165         unsigned int entry_capacity; /* must match tree_content */
 166         struct avail_tree_content *next_avail;
 167 };
 168
 169 struct branch
 170 {
 171         struct branch *table_next_branch;
 172         struct branch *active_next_branch;
 173         const char *name;
 174         unsigned long last_commit;
 175         struct tree_entry branch_tree;
 176         unsigned char sha1[20];
 177 };
 178
 179
 180 /* Stats and misc. counters */
 181 static unsigned long max_depth = 10;
 182 static unsigned long alloc_count;
 183 static unsigned long branch_count;
 184 static unsigned long object_count;
 185 static unsigned long duplicate_count;
 186 static unsigned long object_count_by_type[9];
 187 static unsigned long duplicate_count_by_type[9];
 188
 189 /* Memory pools */
 190 static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
 191 static size_t total_allocd;
 192 static struct mem_pool *mem_pool;
 193
 194 /* Atom management */
 195 static unsigned int atom_table_sz = 4451;
 196 static unsigned int atom_cnt;
 197 static struct atom_str **atom_table;
 198
 199 /* The .pack file being generated */
 200 static int pack_fd;
 201 static unsigned long pack_offset;
 202 static unsigned char pack_sha1[20];
 203
 204 /* Table of objects we've written. */
 205 static unsigned int object_entry_alloc = 1000;
 206 static struct object_entry_pool *blocks;
 207 static struct object_entry *object_table[1 << 16];
 208
 209 /* Our last blob */
 210 static struct last_object last_blob;
 211
 212 /* Tree management */
 213 static unsigned int tree_entry_alloc = 1000;
 214 static void *avail_tree_entry;
 215 static unsigned int avail_tree_table_sz = 100;
 216 static struct avail_tree_content **avail_tree_table;
 217
 218 /* Branch data */
 219 static unsigned long max_active_branches = 5;
 220 static unsigned long cur_active_branches;
 221 static unsigned long branch_table_sz = 1039;
 222 static struct branch **branch_table;
 223 static struct branch *active_branches;
 224
 225 /* Input stream parsing */
 226 static struct strbuf command_buf;
 227 static unsigned long command_mark;
 228
 229
 230 static void alloc_objects(int cnt)
 231 {
 232         struct object_entry_pool *b;
 233
 234         b = xmalloc(sizeof(struct object_entry_pool)
 235                 + cnt * sizeof(struct object_entry));
 236         b->next_pool = blocks;
 237         b->next_free = b->entries;
 238         b->end = b->entries + cnt;
 239         blocks = b;
 240         alloc_count += cnt;
 241 }
 242
 243 static struct object_entry* new_object(unsigned char *sha1)
 244 {
 245         struct object_entry *e;
 246
 247         if (blocks->next_free == blocks->end)
 248                 alloc_objects(object_entry_alloc);
 249
 250         e = blocks->next_free++;
 251         memcpy(e->sha1, sha1, sizeof(e->sha1));
 252         return e;
 253 }
 254
 255 static struct object_entry* find_object(unsigned char *sha1)
 256 {
 257         unsigned int h = sha1[0] << 8 | sha1[1];
 258         struct object_entry *e;
 259         for (e = object_table[h]; e; e = e->next)
 260                 if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
 261                         return e;
 262         return NULL;
 263 }
 264
 265 static struct object_entry* insert_object(unsigned char *sha1)
 266 {
 267         unsigned int h = sha1[0] << 8 | sha1[1];
 268         struct object_entry *e = object_table[h];
 269         struct object_entry *p = NULL;
 270
 271         while (e) {
 272                 if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
 273                         return e;
 274                 p = e;
 275                 e = e->next;
 276         }
 277
 278         e = new_object(sha1);
 279         e->next = NULL;
 280         e->offset = 0;
 281         if (p)
 282                 p->next = e;
 283         else
 284                 object_table[h] = e;
 285         return e;
 286 }
 287
 288 static unsigned int hc_str(const char *s, size_t len)
 289 {
 290         unsigned int r = 0;
 291         while (len-- > 0)
 292                 r = r * 31 + *s++;
 293         return r;
 294 }
 295
 296 static void* pool_alloc(size_t len)
 297 {
 298         struct mem_pool *p;
 299         void *r;
 300
 301         for (p = mem_pool; p; p = p->next_pool)
 302                 if ((p->end - p->next_free >= len))
 303                         break;
 304
 305         if (!p) {
 306                 if (len >= (mem_pool_alloc/2)) {
 307                         total_allocd += len;
 308                         return xmalloc(len);
 309                 }
 310                 total_allocd += sizeof(struct mem_pool) + mem_pool_alloc;
 311                 p = xmalloc(sizeof(struct mem_pool) + mem_pool_alloc);
 312                 p->next_pool = mem_pool;
 313                 p->next_free = p->space;
 314                 p->end = p->next_free + mem_pool_alloc;
 315                 mem_pool = p;
 316         }
 317
 318         r = p->next_free;
 319         p->next_free += len;
 320         return r;
 321 }
 322
 323 static void* pool_calloc(size_t count, size_t size)
 324 {
 325         size_t len = count * size;
 326         void *r = pool_alloc(len);
 327         memset(r, 0, len);
 328         return r;
 329 }
 330
 331 static char* pool_strdup(const char *s)
 332 {
 333         char *r = pool_alloc(strlen(s) + 1);
 334         strcpy(r, s);
 335         return r;
 336 }
 337
 338 static struct atom_str* to_atom(const char *s, size_t len)
 339 {
 340         unsigned int hc = hc_str(s, len) % atom_table_sz;
 341         struct atom_str *c;
 342
 343         for (c = atom_table[hc]; c; c = c->next_atom)
 344                 if (c->str_len == len && !strncmp(s, c->str_dat, len))
 345                         return c;
 346
 347         c = pool_alloc(sizeof(struct atom_str) + len + 1);
 348         c->str_len = len;
 349         strncpy(c->str_dat, s, len);
 350         c->str_dat[len] = 0;
 351         c->next_atom = atom_table[hc];
 352         atom_table[hc] = c;
 353         atom_cnt++;
 354         return c;
 355 }
 356
 357 static struct branch* lookup_branch(const char *name)
 358 {
 359         unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
 360         struct branch *b;
 361
 362         for (b = branch_table[hc]; b; b = b->table_next_branch)
 363                 if (!strcmp(name, b->name))
 364                         return b;
 365         return NULL;
 366 }
 367
 368 static struct branch* new_branch(const char *name)
 369 {
 370         unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
 371         struct branch* b = lookup_branch(name);
 372
 373         if (b)
 374                 die("Invalid attempt to create duplicate branch: %s", name);
 375         if (check_ref_format(name))
 376                 die("Branch name doesn't conform to GIT standards: %s", name);
 377
 378         b = pool_calloc(1, sizeof(struct branch));
 379         b->name = pool_strdup(name);
 380         b->table_next_branch = branch_table[hc];
 381         branch_table[hc] = b;
 382         branch_count++;
 383         return b;
 384 }
 385
 386 static unsigned int hc_entries(unsigned int cnt)
 387 {
 388         cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8;
 389         return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1;
 390 }
 391
 392 static struct tree_content* new_tree_content(unsigned int cnt)
 393 {
 394         struct avail_tree_content *f, *l = NULL;
 395         struct tree_content *t;
 396         unsigned int hc = hc_entries(cnt);
 397
 398         for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail)
 399                 if (f->entry_capacity >= cnt)
 400                         break;
 401
 402         if (f) {
 403                 if (l)
 404                         l->next_avail = f->next_avail;
 405                 else
 406                         avail_tree_table[hc] = f->next_avail;
 407         } else {
 408                 cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
 409                 f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);
 410                 f->entry_capacity = cnt;
 411         }
 412
 413         t = (struct tree_content*)f;
 414         t->entry_count = 0;
 415         return t;
 416 }
 417
 418 static void release_tree_entry(struct tree_entry *e);
 419 static void release_tree_content(struct tree_content *t)
 420 {
 421         struct avail_tree_content *f = (struct avail_tree_content*)t;
 422         unsigned int hc = hc_entries(f->entry_capacity);
 423         f->next_avail = avail_tree_table[hc];
 424         avail_tree_table[hc] = f;
 425 }
 426
 427 static void release_tree_content_recursive(struct tree_content *t)
 428 {
 429         unsigned int i;
 430         for (i = 0; i < t->entry_count; i++)
 431                 release_tree_entry(t->entries[i]);
 432         release_tree_content(t);
 433 }
 434
 435 static struct tree_content* grow_tree_content(
 436         struct tree_content *t,
 437         int amt)
 438 {
 439         struct tree_content *r = new_tree_content(t->entry_count + amt);
 440         r->entry_count = t->entry_count;
 441         memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
 442         release_tree_content(t);
 443         return r;
 444 }
 445
 446 static struct tree_entry* new_tree_entry()
 447 {
 448         struct tree_entry *e;
 449
 450         if (!avail_tree_entry) {
 451                 unsigned int n = tree_entry_alloc;
 452                 avail_tree_entry = e = xmalloc(n * sizeof(struct tree_entry));
 453                 while (n--) {
 454                         *((void**)e) = e + 1;
 455                         e++;
 456                 }
 457         }
 458
 459         e = avail_tree_entry;
 460         avail_tree_entry = *((void**)e);
 461         return e;
 462 }
 463
 464 static void release_tree_entry(struct tree_entry *e)
 465 {
 466         if (e->tree)
 467                 release_tree_content_recursive(e->tree);
 468         *((void**)e) = avail_tree_entry;
 469         avail_tree_entry = e;
 470 }
 471
 472 static void yread(int fd, void *buffer, size_t length)
 473 {
 474         ssize_t ret = 0;
 475         while (ret < length) {
 476                 ssize_t size = xread(fd, (char *) buffer + ret, length - ret);
 477                 if (!size)
 478                         die("Read from descriptor %i: end of stream", fd);
 479                 if (size < 0)
 480                         die("Read from descriptor %i: %s", fd, strerror(errno));
 481                 ret += size;
 482         }
 483 }
 484
 485 static void ywrite(int fd, void *buffer, size_t length)
 486 {
 487         ssize_t ret = 0;
 488         while (ret < length) {
 489                 ssize_t size = xwrite(fd, (char *) buffer + ret, length - ret);
 490                 if (!size)
 491                         die("Write to descriptor %i: end of file", fd);
 492                 if (size < 0)
 493                         die("Write to descriptor %i: %s", fd, strerror(errno));
 494                 ret += size;
 495         }
 496 }
 497
 498 static size_t encode_header(
 499         enum object_type type,
 500         size_t size,
 501         unsigned char *hdr)
 502 {
 503         int n = 1;
 504         unsigned char c;
 505
 506         if (type < OBJ_COMMIT || type > OBJ_DELTA)
 507                 die("bad type %d", type);
 508
 509         c = (type << 4) | (size & 15);
 510         size >>= 4;
 511         while (size) {
 512                 *hdr++ = c | 0x80;
 513                 c = size & 0x7f;
 514                 size >>= 7;
 515                 n++;
 516         }
 517         *hdr = c;
 518         return n;
 519 }
 520
 521 static int store_object(
 522         enum object_type type,
 523         void *dat,
 524         size_t datlen,
 525         struct last_object *last,
 526         unsigned char *sha1out)
 527 {
 528         void *out, *delta;
 529         struct object_entry *e;
 530         unsigned char hdr[96];
 531         unsigned char sha1[20];
 532         unsigned long hdrlen, deltalen;
 533         SHA_CTX c;
 534         z_stream s;
 535
 536         hdrlen = sprintf((char*)hdr,"%s %lu",type_names[type],datlen) + 1;
 537         SHA1_Init(&c);
 538         SHA1_Update(&c, hdr, hdrlen);
 539         SHA1_Update(&c, dat, datlen);
 540         SHA1_Final(sha1, &c);
 541         if (sha1out)
 542                 memcpy(sha1out, sha1, sizeof(sha1));
 543
 544         e = insert_object(sha1);
 545         if (e->offset) {
 546                 duplicate_count++;
 547                 duplicate_count_by_type[type]++;
 548                 return 1;
 549         }
 550         e->type = type;
 551         e->offset = pack_offset;
 552         object_count++;
 553         object_count_by_type[type]++;
 554
 555         if (last && last->data && last->depth < max_depth)
 556                 delta = diff_delta(last->data, last->len,
 557                         dat, datlen,
 558                         &deltalen, 0);
 559         else
 560                 delta = 0;
 561
 562         memset(&s, 0, sizeof(s));
 563         deflateInit(&s, zlib_compression_level);
 564
 565         if (delta) {
 566                 last->depth++;
 567                 s.next_in = delta;
 568                 s.avail_in = deltalen;
 569                 hdrlen = encode_header(OBJ_DELTA, deltalen, hdr);
 570                 ywrite(pack_fd, hdr, hdrlen);
 571                 ywrite(pack_fd, last->sha1, sizeof(sha1));
 572                 pack_offset += hdrlen + sizeof(sha1);
 573         } else {
 574                 if (last)
 575                         last->depth = 0;
 576                 s.next_in = dat;
 577                 s.avail_in = datlen;
 578                 hdrlen = encode_header(type, datlen, hdr);
 579                 ywrite(pack_fd, hdr, hdrlen);
 580                 pack_offset += hdrlen;
 581         }
 582
 583         s.avail_out = deflateBound(&s, s.avail_in);
 584         s.next_out = out = xmalloc(s.avail_out);
 585         while (deflate(&s, Z_FINISH) == Z_OK)
 586                 /* nothing */;
 587         deflateEnd(&s);
 588
 589         ywrite(pack_fd, out, s.total_out);
 590         pack_offset += s.total_out;
 591
 592         free(out);
 593         if (delta)
 594                 free(delta);
 595         if (last) {
 596                 if (last->data)
 597                         free(last->data);
 598                 last->data = dat;
 599                 last->len = datlen;
 600                 memcpy(last->sha1, sha1, sizeof(sha1));
 601         }
 602         return 0;
 603 }
 604
 605 static const char *get_mode(const char *str, unsigned int *modep)
 606 {
 607         unsigned char c;
 608         unsigned int mode = 0;
 609
 610         while ((c = *str++) != ' ') {
 611                 if (c < '0' || c > '7')
 612                         return NULL;
 613                 mode = (mode << 3) + (c - '0');
 614         }
 615         *modep = mode;
 616         return str;
 617 }
 618
 619 static void load_tree(struct tree_entry *root)
 620 {
 621         struct object_entry *myoe;
 622         struct tree_content *t;
 623         unsigned long size;
 624         char *buf;
 625         const char *c;
 626         char type[20];
 627
 628         root->tree = t = new_tree_content(8);
 629         if (!memcmp(root->sha1, null_sha1, 20))
 630                 return;
 631
 632         myoe = find_object(root->sha1);
 633         if (myoe) {
 634                 die("FIXME");
 635         } else {
 636                 buf = read_sha1_file(root->sha1, type, &size);
 637                 if (!buf || strcmp(type, tree_type))
 638                         die("Can't load existing tree %s", sha1_to_hex(root->sha1));
 639         }
 640
 641         c = buf;
 642         while (c != (buf + size)) {
 643                 struct tree_entry *e = new_tree_entry();
 644
 645                 if (t->entry_count == t->entry_capacity)
 646                         root->tree = t = grow_tree_content(t, 8);
 647                 t->entries[t->entry_count++] = e;
 648
 649                 e->tree = NULL;
 650                 c = get_mode(c, &e->mode);
 651                 if (!c)
 652                         die("Corrupt mode in %s", sha1_to_hex(root->sha1));
 653                 e->name = to_atom(c, strlen(c));
 654                 c += e->name->str_len + 1;
 655                 memcpy(e->sha1, c, sizeof(e->sha1));
 656                 c += 20;
 657         }
 658         free(buf);
 659 }
 660
 661 static int tecmp (const void *_a, const void *_b)
 662 {
 663         struct tree_entry *a = *((struct tree_entry**)_a);
 664         struct tree_entry *b = *((struct tree_entry**)_b);
 665         return base_name_compare(
 666                 a->name->str_dat, a->name->str_len, a->mode,
 667                 b->name->str_dat, b->name->str_len, b->mode);
 668 }
 669
 670 static void store_tree(struct tree_entry *root)
 671 {
 672         struct tree_content *t = root->tree;
 673         unsigned int i;
 674         size_t maxlen;
 675         char *buf, *c;
 676
 677         if (memcmp(root->sha1, null_sha1, 20))
 678                 return;
 679
 680         maxlen = 0;
 681         for (i = 0; i < t->entry_count; i++) {
 682                 maxlen += t->entries[i]->name->str_len + 34;
 683                 if (t->entries[i]->tree)
 684                         store_tree(t->entries[i]);
 685         }
 686
 687         qsort(t->entries, t->entry_count, sizeof(t->entries[0]), tecmp);
 688         buf = c = xmalloc(maxlen);
 689         for (i = 0; i < t->entry_count; i++) {
 690                 struct tree_entry *e = t->entries[i];
 691                 c += sprintf(c, "%o", e->mode);
 692                 *c++ = ' ';
 693                 strcpy(c, e->name->str_dat);
 694                 c += e->name->str_len + 1;
 695                 memcpy(c, e->sha1, 20);
 696                 c += 20;
 697         }
 698         store_object(OBJ_TREE, buf, c - buf, NULL, root->sha1);
 699         free(buf);
 700 }
 701
 702 static int tree_content_set(
 703         struct tree_entry *root,
 704         const char *p,
 705         const unsigned char *sha1,
 706         const unsigned int mode)
 707 {
 708         struct tree_content *t = root->tree;
 709         const char *slash1;
 710         unsigned int i, n;
 711         struct tree_entry *e;
 712
 713         slash1 = strchr(p, '/');
 714         if (slash1)
 715                 n = slash1 - p;
 716         else
 717                 n = strlen(p);
 718
 719         for (i = 0; i < t->entry_count; i++) {
 720                 e = t->entries[i];
 721                 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
 722                         if (!slash1) {
 723                                 if (e->mode == mode && !memcmp(e->sha1, sha1, 20))
 724                                         return 0;
 725                                 e->mode = mode;
 726                                 memcpy(e->sha1, sha1, 20);
 727                                 if (e->tree) {
 728                                         release_tree_content_recursive(e->tree);
 729                                         e->tree = NULL;
 730                                 }
 731                                 memcpy(root->sha1, null_sha1, 20);
 732                                 return 1;
 733                         }
 734                         if (!S_ISDIR(e->mode)) {
 735                                 e->tree = new_tree_content(8);
 736                                 e->mode = S_IFDIR;
 737                         }
 738                         if (!e->tree)
 739                                 load_tree(e);
 740                         if (tree_content_set(e, slash1 + 1, sha1, mode)) {
 741                                 memcpy(root->sha1, null_sha1, 20);
 742                                 return 1;
 743                         }
 744                         return 0;
 745                 }
 746         }
 747
 748         if (t->entry_count == t->entry_capacity)
 749                 root->tree = t = grow_tree_content(t, 8);
 750         e = new_tree_entry();
 751         e->name = to_atom(p, n);
 752         t->entries[t->entry_count++] = e;
 753         if (slash1) {
 754                 e->tree = new_tree_content(8);
 755                 e->mode = S_IFDIR;
 756                 tree_content_set(e, slash1 + 1, sha1, mode);
 757         } else {
 758                 e->tree = NULL;
 759                 e->mode = mode;
 760                 memcpy(e->sha1, sha1, 20);
 761         }
 762         memcpy(root->sha1, null_sha1, 20);
 763         return 1;
 764 }
 765
 766 static int tree_content_remove(struct tree_entry *root, const char *p)
 767 {
 768         struct tree_content *t = root->tree;
 769         const char *slash1;
 770         unsigned int i, n;
 771         struct tree_entry *e;
 772
 773         slash1 = strchr(p, '/');
 774         if (slash1)
 775                 n = slash1 - p;
 776         else
 777                 n = strlen(p);
 778
 779         for (i = 0; i < t->entry_count; i++) {
 780                 e = t->entries[i];
 781                 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
 782                         if (!slash1 || !S_ISDIR(e->mode))
 783                                 goto del_entry;
 784                         if (!e->tree)
 785                                 load_tree(e);
 786                         if (tree_content_remove(e, slash1 + 1)) {
 787                                 if (!e->tree->entry_count)
 788                                         goto del_entry;
 789                                 memcpy(root->sha1, null_sha1, 20);
 790                                 return 1;
 791                         }
 792                         return 0;
 793                 }
 794         }
 795         return 0;
 796
 797 del_entry:
 798         for (i++; i < t->entry_count; i++)
 799                 t->entries[i-1] = t->entries[i];
 800         t->entry_count--;
 801         release_tree_entry(e);
 802         memcpy(root->sha1, null_sha1, 20);
 803         return 1;
 804 }
 805
 806 static void init_pack_header()
 807 {
 808         struct pack_header hdr;
 809
 810         hdr.hdr_signature = htonl(PACK_SIGNATURE);
 811         hdr.hdr_version = htonl(2);
 812         hdr.hdr_entries = 0;
 813
 814         ywrite(pack_fd, &hdr, sizeof(hdr));
 815         pack_offset = sizeof(hdr);
 816 }
 817
 818 static void fixup_header_footer()
 819 {
 820         SHA_CTX c;
 821         char hdr[8];
 822         unsigned long cnt;
 823         char *buf;
 824         size_t n;
 825
 826         if (lseek(pack_fd, 0, SEEK_SET) != 0)
 827                 die("Failed seeking to start: %s", strerror(errno));
 828
 829         SHA1_Init(&c);
 830         yread(pack_fd, hdr, 8);
 831         SHA1_Update(&c, hdr, 8);
 832
 833         cnt = htonl(object_count);
 834         SHA1_Update(&c, &cnt, 4);
 835         ywrite(pack_fd, &cnt, 4);
 836
 837         buf = xmalloc(128 * 1024);
 838         for (;;) {
 839                 n = xread(pack_fd, buf, 128 * 1024);
 840                 if (n <= 0)
 841                         break;
 842                 SHA1_Update(&c, buf, n);
 843         }
 844         free(buf);
 845
 846         SHA1_Final(pack_sha1, &c);
 847         ywrite(pack_fd, pack_sha1, sizeof(pack_sha1));
 848 }
 849
 850 static int oecmp (const void *_a, const void *_b)
 851 {
 852         struct object_entry *a = *((struct object_entry**)_a);
 853         struct object_entry *b = *((struct object_entry**)_b);
 854         return memcmp(a->sha1, b->sha1, sizeof(a->sha1));
 855 }
 856
 857 static void write_index(const char *idx_name)
 858 {
 859         struct sha1file *f;
 860         struct object_entry **idx, **c, **last;
 861         struct object_entry *e;
 862         struct object_entry_pool *o;
 863         unsigned int array[256];
 864         int i;
 865
 866         /* Build the sorted table of object IDs. */
 867         idx = xmalloc(object_count * sizeof(struct object_entry*));
 868         c = idx;
 869         for (o = blocks; o; o = o->next_pool)
 870                 for (e = o->entries; e != o->next_free; e++)
 871                         *c++ = e;
 872         last = idx + object_count;
 873         qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
 874
 875         /* Generate the fan-out array. */
 876         c = idx;
 877         for (i = 0; i < 256; i++) {
 878                 struct object_entry **next = c;;
 879                 while (next < last) {
 880                         if ((*next)->sha1[0] != i)
 881                                 break;
 882                         next++;
 883                 }
 884                 array[i] = htonl(next - idx);
 885                 c = next;
 886         }
 887
 888         f = sha1create("%s", idx_name);
 889         sha1write(f, array, 256 * sizeof(int));
 890         for (c = idx; c != last; c++) {
 891                 unsigned int offset = htonl((*c)->offset);
 892                 sha1write(f, &offset, 4);
 893                 sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
 894         }
 895         sha1write(f, pack_sha1, sizeof(pack_sha1));
 896         sha1close(f, NULL, 1);
 897         free(idx);
 898 }
 899
 900 static void dump_branches()
 901 {
 902         static const char *msg = "fast-import";
 903         unsigned int i;
 904         struct branch *b;
 905         struct ref_lock *lock;
 906
 907         for (i = 0; i < branch_table_sz; i++) {
 908                 for (b = branch_table[i]; b; b = b->table_next_branch) {
 909                         lock = lock_any_ref_for_update(b->name, NULL, 0);
 910                         if (!lock || write_ref_sha1(lock, b->sha1, msg) < 0)
 911                                 die("Can't write %s", b->name);
 912                 }
 913         }
 914 }
 915
 916 static void read_next_command()
 917 {
 918         read_line(&command_buf, stdin, '\n');
 919 }
 920
 921 static void cmd_mark()
 922 {
 923         if (!strncmp("mark :", command_buf.buf, 6)) {
 924                 command_mark = strtoul(command_buf.buf + 6, NULL, 10);
 925                 read_next_command();
 926         }
 927         else
 928                 command_mark = 0;
 929 }
 930
 931 static void* cmd_data (size_t *size)
 932 {
 933         size_t n = 0;
 934         void *buffer;
 935         size_t length;
 936
 937         if (strncmp("data ", command_buf.buf, 5))
 938                 die("Expected 'data n' command, found: %s", command_buf.buf);
 939
 940         length = strtoul(command_buf.buf + 5, NULL, 10);
 941         buffer = xmalloc(length);
 942
 943         while (n < length) {
 944                 size_t s = fread((char*)buffer + n, 1, length - n, stdin);
 945                 if (!s && feof(stdin))
 946                         die("EOF in data (%lu bytes remaining)", length - n);
 947                 n += s;
 948         }
 949
 950         if (fgetc(stdin) != '\n')
 951                 die("An lf did not trail the binary data as expected.");
 952
 953         *size = length;
 954         return buffer;
 955 }
 956
 957 static void cmd_new_blob()
 958 {
 959         size_t datlen;
 960         void *dat;
 961         unsigned char sha1[20];
 962
 963         read_next_command();
 964         cmd_mark();
 965         dat = cmd_data(&datlen);
 966
 967         if (store_object(OBJ_BLOB, dat, datlen, &last_blob, sha1))
 968                 free(dat);
 969 }
 970
 971 static void unload_one_branch()
 972 {
 973         while (cur_active_branches >= max_active_branches) {
 974                 unsigned long min_commit = ULONG_MAX;
 975                 struct branch *e, *l = NULL, *p = NULL;
 976
 977                 for (e = active_branches; e; e = e->active_next_branch) {
 978                         if (e->last_commit < min_commit) {
 979                                 p = l;
 980                                 min_commit = e->last_commit;
 981                         }
 982                         l = e;
 983                 }
 984
 985                 if (p) {
 986                         e = p->active_next_branch;
 987                         p->active_next_branch = e->active_next_branch;
 988                 } else {
 989                         e = active_branches;
 990                         active_branches = e->active_next_branch;
 991                 }
 992                 e->active_next_branch = NULL;
 993                 if (e->branch_tree.tree) {
 994                         release_tree_content_recursive(e->branch_tree.tree);
 995                         e->branch_tree.tree = NULL;
 996                 }
 997                 cur_active_branches--;
 998         }
 999 }
1000
1001 static void load_branch(struct branch *b)
1002 {
1003         load_tree(&b->branch_tree);
1004         b->active_next_branch = active_branches;
1005         active_branches = b;
1006         cur_active_branches++;
1007 }
1008
1009 static void file_change_m(struct branch *b)
1010 {
1011         const char *p = command_buf.buf + 2;
1012         char *p_uq;
1013         const char *endp;
1014         struct object_entry *oe;
1015         unsigned char sha1[20];
1016         unsigned int mode;
1017         char type[20];
1018
1019         p = get_mode(p, &mode);
1020         if (!p)
1021                 die("Corrupt mode: %s", command_buf.buf);
1022         switch (mode) {
1023         case S_IFREG | 0644:
1024         case S_IFREG | 0755:
1025         case S_IFLNK:
1026         case 0644:
1027         case 0755:
1028                 /* ok */
1029                 break;
1030         default:
1031                 die("Corrupt mode: %s", command_buf.buf);
1032         }
1033
1034         if (get_sha1_hex(p, sha1))
1035                 die("Invalid SHA1: %s", command_buf.buf);
1036         p += 40;
1037         if (*p++ != ' ')
1038                 die("Missing space after SHA1: %s", command_buf.buf);
1039
1040         p_uq = unquote_c_style(p, &endp);
1041         if (p_uq) {
1042                 if (*endp)
1043                         die("Garbage after path in: %s", command_buf.buf);
1044                 p = p_uq;
1045         }
1046
1047         oe = find_object(sha1);
1048         if (oe) {
1049                 if (oe->type != OBJ_BLOB)
1050                         die("Not a blob (actually a %s): %s",
1051                                 command_buf.buf, type_names[oe->type]);
1052         } else {
1053                 if (sha1_object_info(sha1, type, NULL))
1054                         die("Blob not found: %s", command_buf.buf);
1055                 if (strcmp(blob_type, type))
1056                         die("Not a blob (actually a %s): %s",
1057                                 command_buf.buf, type);
1058         }
1059
1060         tree_content_set(&b->branch_tree, p, sha1, S_IFREG | mode);
1061
1062         if (p_uq)
1063                 free(p_uq);
1064 }
1065
1066 static void file_change_d(struct branch *b)
1067 {
1068         const char *p = command_buf.buf + 2;
1069         char *p_uq;
1070         const char *endp;
1071
1072         p_uq = unquote_c_style(p, &endp);
1073         if (p_uq) {
1074                 if (*endp)
1075                         die("Garbage after path in: %s", command_buf.buf);
1076                 p = p_uq;
1077         }
1078         tree_content_remove(&b->branch_tree, p);
1079         if (p_uq)
1080                 free(p_uq);
1081 }
1082
1083 static void cmd_new_commit()
1084 {
1085         struct branch *b;
1086         void *msg;
1087         size_t msglen;
1088         char *str_uq;
1089         const char *endp;
1090         char *sp;
1091         char *author = NULL;
1092         char *committer = NULL;
1093         char *body;
1094
1095         /* Obtain the branch name from the rest of our command */
1096         sp = strchr(command_buf.buf, ' ') + 1;
1097         str_uq = unquote_c_style(sp, &endp);
1098         if (str_uq) {
1099                 if (*endp)
1100                         die("Garbage after ref in: %s", command_buf.buf);
1101                 sp = str_uq;
1102         }
1103         b = lookup_branch(sp);
1104         if (!b)
1105                 die("Branch not declared: %s", sp);
1106         if (str_uq)
1107                 free(str_uq);
1108
1109         read_next_command();
1110         cmd_mark();
1111         if (!strncmp("author ", command_buf.buf, 7)) {
1112                 author = strdup(command_buf.buf);
1113                 read_next_command();
1114         }
1115         if (!strncmp("committer ", command_buf.buf, 10)) {
1116                 committer = strdup(command_buf.buf);
1117                 read_next_command();
1118         }
1119         if (!committer)
1120                 die("Expected committer but didn't get one");
1121         msg = cmd_data(&msglen);
1122
1123         /* ensure the branch is active/loaded */
1124         if (!b->branch_tree.tree) {
1125                 unload_one_branch();
1126                 load_branch(b);
1127         }
1128
1129         /* file_change* */
1130         for (;;) {
1131                 read_next_command();
1132                 if (1 == command_buf.len)
1133                         break;
1134                 else if (!strncmp("M ", command_buf.buf, 2))
1135                         file_change_m(b);
1136                 else if (!strncmp("D ", command_buf.buf, 2))
1137                         file_change_d(b);
1138                 else
1139                         die("Unsupported file_change: %s", command_buf.buf);
1140         }
1141
1142         /* build the tree and the commit */
1143         store_tree(&b->branch_tree);
1144         body = xmalloc(97 + msglen
1145                 + (author
1146                         ? strlen(author) + strlen(committer)
1147                         : 2 * strlen(committer)));
1148         sp = body;
1149         sp += sprintf(sp, "tree %s\n", sha1_to_hex(b->branch_tree.sha1));
1150         if (memcmp(b->sha1, null_sha1, 20))
1151                 sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
1152         if (author)
1153                 sp += sprintf(sp, "%s\n", author);
1154         else
1155                 sp += sprintf(sp, "author %s\n", committer + 10);
1156         sp += sprintf(sp, "%s\n\n", committer);
1157         memcpy(sp, msg, msglen);
1158         sp += msglen;
1159         if (author)
1160                 free(author);
1161         free(committer);
1162         free(msg);
1163
1164         store_object(OBJ_COMMIT, body, sp - body, NULL, b->sha1);
1165         free(body);
1166         b->last_commit = object_count_by_type[OBJ_COMMIT];
1167 }
1168
1169 static void cmd_new_branch()
1170 {
1171         struct branch *b;
1172         char *str_uq;
1173         const char *endp;
1174         char *sp;
1175
1176         /* Obtain the new branch name from the rest of our command */
1177         sp = strchr(command_buf.buf, ' ') + 1;
1178         str_uq = unquote_c_style(sp, &endp);
1179         if (str_uq) {
1180                 if (*endp)
1181                         die("Garbage after ref in: %s", command_buf.buf);
1182                 sp = str_uq;
1183         }
1184         b = new_branch(sp);
1185         if (str_uq)
1186                 free(str_uq);
1187         read_next_command();
1188
1189         /* from ... */
1190         if (!strncmp("from ", command_buf.buf, 5)) {
1191                 const char *from;
1192                 struct branch *s;
1193
1194                 from = strchr(command_buf.buf, ' ') + 1;
1195                 str_uq = unquote_c_style(from, &endp);
1196                 if (str_uq) {
1197                         if (*endp)
1198                                 die("Garbage after string in: %s", command_buf.buf);
1199                         from = str_uq;
1200                 }
1201
1202                 s = lookup_branch(from);
1203                 if (b == s)
1204                         die("Can't create a branch from itself: %s", b->name);
1205                 else if (s) {
1206                         memcpy(b->sha1, s->sha1, 20);
1207                         memcpy(b->branch_tree.sha1, s->branch_tree.sha1, 20);
1208                 } else if (!get_sha1(from, b->sha1)) {
1209                         if (!memcmp(b->sha1, null_sha1, 20))
1210                                 memcpy(b->branch_tree.sha1, null_sha1, 20);
1211                         else {
1212                                 unsigned long size;
1213                                 char *buf;
1214
1215                                 buf = read_object_with_reference(b->sha1,
1216                                         type_names[OBJ_COMMIT], &size, b->sha1);
1217                                 if (!buf || size < 46)
1218                                         die("Not a valid commit: %s", from);
1219                                 if (memcmp("tree ", buf, 5)
1220                                         || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1221                                         die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1222                                 free(buf);
1223                         }
1224                 } else
1225                         die("Invalid ref name or SHA1 expression: %s", from);
1226
1227                 if (str_uq)
1228                         free(str_uq);
1229                 read_next_command();
1230         } else {
1231                 memcpy(b->sha1, null_sha1, 20);
1232                 memcpy(b->branch_tree.sha1, null_sha1, 20);
1233         }
1234
1235         if (command_buf.eof || command_buf.len > 1)
1236                 die("An lf did not terminate the branch command as expected.");
1237 }
1238
1239 static const char fast_import_usage[] =
1240 "git-fast-import [--objects=n] [--depth=n] [--active-branches=n] temp.pack";
1241
1242 int main(int argc, const char **argv)
1243 {
1244         const char *base_name;
1245         int i;
1246         unsigned long est_obj_cnt = 1000;
1247         char *pack_name;
1248         char *idx_name;
1249         struct stat sb;
1250
1251         setup_ident();
1252         git_config(git_default_config);
1253
1254         for (i = 1; i < argc; i++) {
1255                 const char *a = argv[i];
1256
1257                 if (*a != '-' || !strcmp(a, "--"))
1258                         break;
1259                 else if (!strncmp(a, "--objects=", 10))
1260                         est_obj_cnt = strtoul(a + 10, NULL, 0);
1261                 else if (!strncmp(a, "--depth=", 8))
1262                         max_depth = strtoul(a + 8, NULL, 0);
1263                 else if (!strncmp(a, "--active-branches=", 18))
1264                         max_active_branches = strtoul(a + 18, NULL, 0);
1265                 else
1266                         die("unknown option %s", a);
1267         }
1268         if ((i+1) != argc)
1269                 usage(fast_import_usage);
1270         base_name = argv[i];
1271
1272         pack_name = xmalloc(strlen(base_name) + 6);
1273         sprintf(pack_name, "%s.pack", base_name);
1274         idx_name = xmalloc(strlen(base_name) + 5);
1275         sprintf(idx_name, "%s.idx", base_name);
1276
1277         pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
1278         if (pack_fd < 0)
1279                 die("Can't create %s: %s", pack_name, strerror(errno));
1280
1281         init_pack_header();
1282         alloc_objects(est_obj_cnt);
1283         strbuf_init(&command_buf);
1284
1285         atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
1286         branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
1287         avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
1288
1289         for (;;) {
1290                 read_next_command();
1291                 if (command_buf.eof)
1292                         break;
1293                 else if (!strcmp("blob", command_buf.buf))
1294                         cmd_new_blob();
1295                 else if (!strncmp("branch ", command_buf.buf, 7))
1296                         cmd_new_branch();
1297                 else if (!strncmp("commit ", command_buf.buf, 7))
1298                         cmd_new_commit();
1299                 else
1300                         die("Unsupported command: %s", command_buf.buf);
1301         }
1302
1303         fixup_header_footer();
1304         close(pack_fd);
1305         write_index(idx_name);
1306         dump_branches();
1307
1308         fprintf(stderr, "%s statistics:\n", argv[0]);
1309         fprintf(stderr, "---------------------------------------------------\n");
1310         fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow  )\n", alloc_count, alloc_count - est_obj_cnt);
1311         fprintf(stderr, "Total objects:   %10lu (%10lu duplicates)\n", object_count, duplicate_count);
1312         fprintf(stderr, "      blobs  :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB]);
1313         fprintf(stderr, "      trees  :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE]);
1314         fprintf(stderr, "      commits:   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT]);
1315         fprintf(stderr, "      tags   :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG]);
1316         fprintf(stderr, "Total branches:  %10lu\n", branch_count);
1317         fprintf(stderr, "Total atoms:     %10u\n", atom_cnt);
1318         fprintf(stderr, "Memory total:    %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1319         fprintf(stderr, "       pools:    %10lu KiB\n", total_allocd/1024);
1320         fprintf(stderr, "     objects:    %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1321         fprintf(stderr, "---------------------------------------------------\n");
1322
1323         stat(pack_name, &sb);
1324         fprintf(stderr, "Pack size:       %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1325         stat(idx_name, &sb);
1326         fprintf(stderr, "Index size:      %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1327
1328         fprintf(stderr, "\n");
1329
1330         return 0;
1331 }