3 # This tool is copyright (c) 2005, Martin Langhoff.
4 # It is released under the Gnu Public License, version 2.
6 # The basic idea is to walk the output of tla abrowse,
7 # fetch the changesets and apply them.
12 git-archimport [ -h ] [ -v ] [ -T ] [ -t tempdir ] <archive>/<branch> [ <archive>/<branch> ]
14 Imports a project from one or more Arch repositories. It will follow branches
15 and repositories within the namespaces defined by the <archive/branch>
16 parameters suppplied. If it cannot find the remote branch a merge comes from
17 it will just import it as a regular commit. If it can find it, it will mark it
18 as a merge whenever possible.
20 See man (1) git-archimport for more details.
24 - create tag objects instead of ref tags
25 - audit shell-escaping of filenames
26 - hide our private tags somewhere smarter
27 - find a way to make "cat *patches | patch" safe even when patchfiles are missing newlines
31 Add print in front of the shell commands invoked via backticks.
39 use File::Temp qw(tempfile tempdir);
40 use File::Path qw(mkpath);
41 use File::Basename qw(basename dirname);
42 use String::ShellQuote;
46 use POSIX qw(strftime dup2);
47 use Data::Dumper qw/ Dumper /;
50 $SIG{'PIPE'}="IGNORE";
53 my $git_dir = $ENV{"GIT_DIR"} || ".git";
54 $ENV{"GIT_DIR"} = $git_dir;
56 our($opt_h,$opt_v, $opt_T,
61 Usage: ${\basename $0} # fetch/update GIT from Arch
62 [ -h ] [ -v ] [ -T ] [ -t tempdir ]
63 repository/arch-branch [ repository/arch-branch] ...
68 getopts("Thvt:") or usage();
71 @ARGV >= 1 or usage();
72 my @arch_roots = @ARGV;
74 my ($tmpdir, $tmpdirname) = tempdir('git-archimport-XXXXXX', TMPDIR => 1, CLEANUP => 1);
75 my $tmp = $opt_t || 1;
76 $tmp = tempdir('git-archimport-XXXXXX', TMPDIR => 1, CLEANUP => 1);
77 $opt_v && print "+ Using $tmp as temporary directory\n";
79 my @psets = (); # the collection
80 my %psets = (); # the collection, by name
82 my %rptags = (); # my reverse private tags
83 # to map a SHA1 to a commitid
85 foreach my $root (@arch_roots) {
86 my ($arepo, $abranch) = split(m!/!, $root);
87 open ABROWSE, "tla abrowse -f -A $arepo --desc --merges $abranch |"
88 or die "Problems with tla abrowse: $!";
90 my %ps = (); # the current one
97 # first record padded w 8 spaces
100 # store the record we just captured
102 my %temp = %ps; # break references
103 push (@psets, \%temp);
104 $psets{$temp{id}} = \%temp;
108 my ($id, $type) = split(m/\s{3}/, $_);
113 if ($type =~ m/^\(simple changeset\)/) {
115 } elsif ($type eq '(initial import)') {
117 } elsif ($type =~ m/^\(tag revision of (.+)\)/) {
121 warn "Unknown type $type";
127 # 10 leading spaces or more
128 # indicate commit metadata
131 if ($lastseen eq 'id' && m/^\d{4}-\d{2}-\d{2}/) {
133 my ($date, $authoremail) = split(m/\s{2,}/, $_);
135 $ps{date} =~ s/\bGMT$//; # strip off trailign GMT
136 if ($ps{date} =~ m/\b\w+$/) {
137 warn 'Arch dates not in GMT?! - imported dates will be wrong';
140 $authoremail =~ m/^(.+)\s(\S+)$/;
146 } elsif ($lastseen eq 'date') {
147 # the only hint is position
148 # subject is after date
152 } elsif ($lastseen eq 'subj' && $_ eq 'merges in:') {
154 $lastseen = 'merges';
156 } elsif ($lastseen eq 'merges' && s/^\s{2}//) {
157 push (@{$ps{merges}}, $_);
159 warn 'more metadata after merges!?';
166 my %temp = %ps; # break references
167 push (@psets, \%temp);
168 $psets{ $temp{id} } = \%temp;
172 } # end foreach $root
174 ## Order patches by time
175 @psets = sort {$a->{date}.$b->{id} cmp $b->{date}.$b->{id}} @psets;
177 #print Dumper \@psets;
180 ## TODO cleanup irrelevant patches
181 ## and put an initial import
184 unless (-d $git_dir) { # initial import
185 if ($psets[0]{type} eq 'i' || $psets[0]{type} eq 't') {
186 print "Starting import from $psets[0]{id}\n";
191 die "Need to start from an import or a tag -- cannot use $psets[0]{id}";
193 } else { # progressing an import
195 opendir(DIR, "$git_dir/archimport/tags")
196 || die "can't opendir: $!";
197 while (my $file = readdir(DIR)) {
198 # skip non-interesting-files
199 next unless -f "$git_dir/archimport/tags/$file";
200 next if $file =~ m/--base-0$/; # don't care for base-0
201 my $sha = ptag($file);
203 # reconvert the 3rd '--' sequence from the end
205 # $file = reverse $file;
206 # $file =~ s!^(.+?--.+?--.+?--.+?)--(.+)$!$1/$2!;
207 # $file = reverse $file;
208 $rptags{$sha} = $file;
214 foreach my $ps (@psets) {
216 $ps->{branch} = branchname($ps->{id});
219 # ensure we have a clean state
221 if (`git diff-files`) {
222 die "Unclean tree when about to process $ps->{id} " .
223 " - did we fail to commit cleanly before?";
228 # skip commits already in repo
230 if (ptag($ps->{id})) {
231 $opt_v && print " * Skipping already imported: $ps->{id}\n";
235 print " * Starting to work on $ps->{id}\n";
238 # create the branch if needed
240 if ($ps->{type} eq 'i' && !$import) {
241 die "Should not have more than one 'Initial import' per GIT import: $ps->{id}";
244 unless ($import) { # skip for import
245 if ( -e "$git_dir/refs/heads/$ps->{branch}") {
246 # we know about this branch
247 `git checkout $ps->{branch}`;
249 # new branch! we need to verify a few things
250 die "Branch on a non-tag!" unless $ps->{type} eq 't';
251 my $branchpoint = ptag($ps->{tag});
252 die "Tagging from unknown id unsupported: $ps->{tag}"
255 # find where we are supposed to branch from
256 `git checkout -b $ps->{branch} $branchpoint`;
258 # If we trust Arch with the fact that this is just
259 # a tag, and it does not affect the state of the tree
260 # then we just tag and move on
261 tag($ps->{id}, $branchpoint);
262 ptag($ps->{id}, $branchpoint);
263 print " * Tagged $ps->{id} at $branchpoint\n";
270 # Apply the import/changeset/merge into the working tree
272 if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
273 apply_import($ps) or die $!;
275 } elsif ($ps->{type} eq 's') {
280 # prepare update git's index, based on what arch knows
281 # about the pset, resolve parents, etc
285 my $commitlog = `tla cat-archive-log -A $ps->{repo} $ps->{id}`;
286 die "Error in cat-archive-log: $!" if $?;
288 # parselog will git-add/rm files
289 # and generally prepare things for the commit
290 # NOTE: parselog will shell-quote filenames!
291 my ($sum, $msg, $add, $del, $mod, $ren) = parselog($commitlog);
292 my $logmessage = "$sum\n$msg";
295 # imports don't give us good info
296 # on added files. Shame on them
297 if ($ps->{type} eq 'i' || $ps->{type} eq 't') {
298 `find . -type f -print0 | grep -zv '^./$git_dir' | xargs -0 -l100 git-update-index --add`;
299 `git-ls-files --deleted -z | xargs --no-run-if-empty -0 -l100 git-update-index --remove`;
304 my @slice = splice(@$add, 0, 100);
305 my $slice = join(' ', @slice);
306 `git-update-index --add $slice`;
307 die "Error in git-update-index --add: $!" if $?;
311 foreach my $file (@$del) {
312 unlink $file or die "Problems deleting $file : $!";
315 my @slice = splice(@$del, 0, 100);
316 my $slice = join(' ', @slice);
317 `git-update-index --remove $slice`;
318 die "Error in git-update-index --remove: $!" if $?;
321 if (@$ren) { # renamed
323 die "Odd number of entries in rename!?";
327 my $from = pop @$ren;
330 unless (-d dirname($to)) {
331 mkpath(dirname($to)); # will die on err
333 #print "moving $from $to";
335 die "Error renaming $from $to : $!" if $?;
336 `git-update-index --remove $from`;
337 die "Error in git-update-index --remove: $!" if $?;
338 `git-update-index --add $to`;
339 die "Error in git-update-index --add: $!" if $?;
343 if (@$mod) { # must be _after_ renames
345 my @slice = splice(@$mod, 0, 100);
346 my $slice = join(' ', @slice);
347 `git-update-index $slice`;
348 die "Error in git-update-index: $!" if $?;
352 # warn "errors when running git-update-index! $!";
353 $tree = `git-write-tree`;
354 die "cannot write tree $!" if $?;
362 if ( -e "$git_dir/refs/heads/$ps->{branch}") {
363 if (open HEAD, "<$git_dir/refs/heads/$ps->{branch}") {
369 if ($ps->{type} eq 's') {
370 warn "Could not find the right head for the branch $ps->{branch}";
376 push @par, find_parents($ps);
378 my $par = join (' ', @par);
381 # Commit, tag and clean state
384 $ENV{GIT_AUTHOR_NAME} = $ps->{author};
385 $ENV{GIT_AUTHOR_EMAIL} = $ps->{email};
386 $ENV{GIT_AUTHOR_DATE} = $ps->{date};
387 $ENV{GIT_COMMITTER_NAME} = $ps->{author};
388 $ENV{GIT_COMMITTER_EMAIL} = $ps->{email};
389 $ENV{GIT_COMMITTER_DATE} = $ps->{date};
391 my ($pid, $commit_rh, $commit_wh);
392 $commit_rh = 'commit_rh';
393 $commit_wh = 'commit_wh';
395 $pid = open2(*READER, *WRITER, "git-commit-tree $tree $par")
397 print WRITER $logmessage; # write
399 my $commitid = <READER>; # read
402 waitpid $pid,0; # close;
404 if (length $commitid != 40) {
405 die "Something went wrong with the commit! $! $commitid";
410 open HEAD, ">$git_dir/refs/heads/$ps->{branch}";
411 print HEAD $commitid;
413 system('git-update-ref', 'HEAD', "$ps->{branch}");
416 ptag($ps->{id}, $commitid); # private tag
417 if ($opt_T || $ps->{type} eq 't' || $ps->{type} eq 'i') {
418 tag($ps->{id}, $commitid);
420 print " * Committed $ps->{id}\n";
421 print " + tree $tree\n";
422 print " + commit $commitid\n";
423 $opt_v && print " + commit date is $ps->{date} \n";
424 $opt_v && print " + parents: $par \n";
430 my @parts = split(m/--/, $id);
431 return join('--', @parts[0..1]);
436 my $bname = branchname($ps->{id});
440 `tla get -s --no-pristine -A $ps->{repo} $ps->{id} $tmp/import`;
441 die "Cannot get import: $!" if $?;
442 `rsync -v --archive --delete --exclude '$git_dir' --exclude '.arch-ids' --exclude '{arch}' $tmp/import/* ./`;
443 die "Cannot rsync import:$!" if $?;
445 `rm -fr $tmp/import`;
446 die "Cannot remove tempdir: $!" if $?;
458 `tla get-changeset -A $ps->{repo} $ps->{id} $tmp/changeset`;
459 die "Cannot get changeset: $!" if $?;
462 if (`find $tmp/changeset/patches -type f -name '*.patch'`) {
463 # this can be sped up considerably by doing
464 # (find | xargs cat) | patch
465 # but that cna get mucked up by patches
466 # with missing trailing newlines or the standard
467 # 'missing newline' flag in the patch - possibly
468 # produced with an old/buggy diff.
469 # slow and safe, we invoke patch once per patchfile
470 `find $tmp/changeset/patches -type f -name '*.patch' -print0 | grep -zv '{arch}' | xargs -iFILE -0 --no-run-if-empty patch -p1 --forward -iFILE`;
471 die "Problem applying patches! $!" if $?;
474 # apply changed binary files
475 if (my @modified = `find $tmp/changeset/patches -type f -name '*.modified'`) {
476 foreach my $mod (@modified) {
479 $orig =~ s/\.modified$//; # lazy
480 $orig =~ s!^\Q$tmp\E/changeset/patches/!!;
481 #print "rsync -p '$mod' '$orig'";
482 `rsync -p $mod ./$orig`;
483 die "Problem applying binary changes! $!" if $?;
488 `rsync --archive --exclude '$git_dir' --exclude '.arch-ids' --exclude '{arch}' $tmp/changeset/new-files-archive/* ./`;
490 # deleted files are hinted from the commitlog processing
492 `rm -fr $tmp/changeset`;
497 # A log entry looks like
498 # Revision: moodle-org--moodle--1.3.3--patch-15
499 # Archive: arch-eduforge@catalyst.net.nz--2004
500 # Creator: Penny Leach <penny@catalyst.net.nz>
501 # Date: Wed May 25 14:15:34 NZST 2005
502 # Standard-date: 2005-05-25 02:15:34 GMT
503 # New-files: lang/de/.arch-ids/block_glossary_random.php.id
504 # lang/de/.arch-ids/block_html.php.id
505 # New-directories: lang/de/help/questionnaire
506 # lang/de/help/questionnaire/.arch-ids
507 # Renamed-files: .arch-ids/db_sears.sql.id db/.arch-ids/db_sears.sql.id
508 # db_sears.sql db/db_sears.sql
509 # Removed-files: lang/be/docs/.arch-ids/release.html.id
510 # lang/be/docs/.arch-ids/releaseold.html.id
511 # Modified-files: admin/cron.php admin/delete.php
512 # admin/editor.html backup/lib.php backup/restore.php
513 # New-patches: arch-eduforge@catalyst.net.nz--2004/moodle-org--moodle--1.3.3--patch-15
514 # Summary: Updating to latest from MOODLE_14_STABLE (1.4.5+)
517 # Updating yadda tadda tadda madda
522 my (@add, @del, @mod, @ren, @kw, $sum, $msg );
524 if ($log =~ m/(?:\n|^)New-files:(.*?)(?=\n\w)/s ) {
526 @add = split(m/\s+/s, $files);
529 if ($log =~ m/(?:\n|^)Removed-files:(.*?)(?=\n\w)/s ) {
531 @del = split(m/\s+/s, $files);
534 if ($log =~ m/(?:\n|^)Modified-files:(.*?)(?=\n\w)/s ) {
536 @mod = split(m/\s+/s, $files);
539 if ($log =~ m/(?:\n|^)Renamed-files:(.*?)(?=\n\w)/s ) {
541 @ren = split(m/\s+/s, $files);
545 if ($log =~ m/^Summary:(.+?)$/m ) {
552 if ($log =~ m/\n\n(.+)$/s) {
560 foreach my $ref ( (\@add, \@del, \@mod, \@ren) ) {
562 while (my $t = pop @$ref) {
563 next unless length ($t);
564 next if $t =~ m!\{arch\}/!;
565 next if $t =~ m!\.arch-ids/!;
566 next if $t =~ m!\.arch-inventory$!;
567 # tla cat-archive-log will give us filenames with spaces as file\(sp)name - why?
568 # we can assume that any filename with \ indicates some pika escaping that we want to get rid of.
570 $t = `tla escape --unescaped '$t'`;
572 push (@tmp, shell_quote($t));
577 #print Dumper [$sum, $msg, \@add, \@del, \@mod, \@ren];
578 return ($sum, $msg, \@add, \@del, \@mod, \@ren);
583 my ($tag, $commit) = @_;
585 $tag = shell_quote($tag);
588 open(C,">$git_dir/refs/tags/$tag")
589 or die "Cannot create tag $tag: $!\n";
591 or die "Cannot write tag $tag: $!\n";
593 or die "Cannot write tag $tag: $!\n";
594 print " * Created tag ' $tag' on '$commit'\n" if $opt_v;
596 open(C,"<$git_dir/refs/tags/$tag")
597 or die "Cannot read tag $tag: $!\n";
600 die "Error reading tag $tag: $!\n" unless length $commit == 40;
602 or die "Cannot read tag $tag: $!\n";
607 # write/read a private tag
608 # reads fail softly if the tag isn't there
610 my ($tag, $commit) = @_;
612 $tag = shell_quote($tag);
614 unless (-d "$git_dir/archimport/tags") {
615 mkpath("$git_dir/archimport/tags");
618 if ($commit) { # write
619 open(C,">$git_dir/archimport/tags/$tag")
620 or die "Cannot create tag $tag: $!\n";
622 or die "Cannot write tag $tag: $!\n";
624 or die "Cannot write tag $tag: $!\n";
625 $rptags{$commit} = $tag
626 unless $tag =~ m/--base-0$/;
628 # if the tag isn't there, return 0
629 unless ( -s "$git_dir/archimport/tags/$tag") {
632 open(C,"<$git_dir/archimport/tags/$tag")
633 or die "Cannot read tag $tag: $!\n";
636 die "Error reading tag $tag: $!\n" unless length $commit == 40;
638 or die "Cannot read tag $tag: $!\n";
639 unless (defined $rptags{$commit}) {
640 $rptags{$commit} = $tag;
648 # Identify what branches are merging into me
649 # and whether we are fully merged
650 # git-merge-base <headsha> <headsha> should tell
651 # me what the base of the merge should be
655 my %branches; # holds an arrayref per branch
656 # the arrayref contains a list of
657 # merged patches between the base
658 # of the merge and the current head
660 my @parents; # parents found for this commit
662 # simple loop to split the merges
664 foreach my $merge (@{$ps->{merges}}) {
665 my $branch = branchname($merge);
666 unless (defined $branches{$branch} ){
667 $branches{$branch} = [];
669 push @{$branches{$branch}}, $merge;
673 # foreach branch find a merge base and walk it to the
674 # head where we are, collecting the merged patchsets that
675 # Arch has recorded. Keep that in @have
676 # Compare that with the commits on the other branch
677 # between merge-base and the tip of the branch (@need)
678 # and see if we have a series of consecutive patches
679 # starting from the merge base. The tip of the series
680 # of consecutive patches merged is our new parent for
683 foreach my $branch (keys %branches) {
685 # check that we actually know about the branch
686 next unless -e "$git_dir/refs/heads/$branch";
688 my $mergebase = `git-merge-base $branch $ps->{branch}`;
689 die "Cannot find merge base for $branch and $ps->{branch}" if $?;
692 # now walk up to the mergepoint collecting what patches we have
693 my $branchtip = git_rev_parse($ps->{branch});
694 my @ancestors = `git-rev-list --merge-order $branchtip ^$mergebase`;
695 my %have; # collected merges this branch has
696 foreach my $merge (@{$ps->{merges}}) {
700 foreach my $par (@ancestors) {
701 $par = commitid2pset($par);
702 if (defined $par->{merges}) {
703 foreach my $merge (@{$par->{merges}}) {
704 $ancestorshave{$merge}=1;
708 # print "++++ Merges in $ps->{id} are....\n";
709 # my @have = sort keys %have; print Dumper(\@have);
711 # merge what we have with what ancestors have
712 %have = (%have, %ancestorshave);
714 # see what the remote branch has - these are the merges we
715 # will want to have in a consecutive series from the mergebase
716 my $otherbranchtip = git_rev_parse($branch);
717 my @needraw = `git-rev-list --merge-order $otherbranchtip ^$mergebase`;
719 foreach my $needps (@needraw) { # get the psets
720 $needps = commitid2pset($needps);
721 # git-rev-list will also
722 # list commits merged in via earlier
723 # merges. we are only interested in commits
724 # from the branch we're looking at
725 if ($branch eq $needps->{branch}) {
726 push @need, $needps->{id};
730 # print "++++ Merges from $branch we want are....\n";
731 # print Dumper(\@need);
734 while (my $needed_commit = pop @need) {
735 if ($have{$needed_commit}) {
736 $newparent = $needed_commit;
738 last; # break out of the while
742 push @parents, $newparent;
746 } # end foreach branch
748 # prune redundant parents
750 foreach my $p (@parents) {
753 foreach my $p (@parents) {
754 next unless exists $psets{$p}{merges};
755 next unless ref $psets{$p}{merges};
756 my @merges = @{$psets{$p}{merges}};
757 foreach my $merge (@merges) {
758 if ($parents{$merge}) {
759 delete $parents{$merge};
763 @parents = keys %parents;
764 @parents = map { " -p " . ptag($_) } @parents;
770 my $val = `git-rev-parse $name`;
771 die "Error: git-rev-parse $name" if $?;
776 # resolve a SHA1 to a known patchset
778 my $commitid = shift;
780 my $name = $rptags{$commitid}
781 || die "Cannot find reverse tag mapping for $commitid";
782 # the keys in %rptag are slightly munged; unmunge
783 # reconvert the 3rd '--' sequence from the end
785 $name = reverse $name;
786 $name =~ s!^(.+?--.+?--.+?--.+?)--(.+)$!$1/$2!;
787 $name = reverse $name;
788 my $ps = $psets{$name}
789 || (print Dumper(sort keys %psets)) && die "Cannot find patchset for $name";