contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def diffTreePattern():
  75     # This is a simple generator for the diff tree regex pattern. This could be
  76     # a class variable if this and parseDiffTreeEntry were a part of a class.
  77     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
  78     while True:
  79         yield pattern
  80
  81 def parseDiffTreeEntry(entry):
  82     """Parses a single diff tree entry into its component elements.
  83
  84     See git-diff-tree(1) manpage for details about the format of the diff
  85     output. This method returns a dictionary with the following elements:
  86
  87     src_mode - The mode of the source file
  88     dst_mode - The mode of the destination file
  89     src_sha1 - The sha1 for the source file
  90     dst_sha1 - The sha1 fr the destination file
  91     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
  92     status_score - The score for the status (applicable for 'C' and 'R'
  93                    statuses). This is None if there is no score.
  94     src - The path for the source file.
  95     dst - The path for the destination file. This is only present for
  96           copy or renames. If it is not present, this is None.
  97
  98     If the pattern is not matched, None is returned."""
  99
 100     match = diffTreePattern().next().match(entry)
 101     if match:
 102         return {
 103             'src_mode': match.group(1),
 104             'dst_mode': match.group(2),
 105             'src_sha1': match.group(3),
 106             'dst_sha1': match.group(4),
 107             'status': match.group(5),
 108             'status_score': match.group(6),
 109             'src': match.group(7),
 110             'dst': match.group(10)
 111         }
 112     return None
 113
 114 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 115     cmd = "p4 -G %s" % cmd
 116     if verbose:
 117         sys.stderr.write("Opening pipe: %s\n" % cmd)
 118
 119     # Use a temporary file to avoid deadlocks without
 120     # subprocess.communicate(), which would put another copy
 121     # of stdout into memory.
 122     stdin_file = None
 123     if stdin is not None:
 124         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 125         stdin_file.write(stdin)
 126         stdin_file.flush()
 127         stdin_file.seek(0)
 128
 129     p4 = subprocess.Popen(cmd, shell=True,
 130                           stdin=stdin_file,
 131                           stdout=subprocess.PIPE)
 132
 133     result = []
 134     try:
 135         while True:
 136             entry = marshal.load(p4.stdout)
 137             result.append(entry)
 138     except EOFError:
 139         pass
 140     exitCode = p4.wait()
 141     if exitCode != 0:
 142         entry = {}
 143         entry["p4ExitCode"] = exitCode
 144         result.append(entry)
 145
 146     return result
 147
 148 def p4Cmd(cmd):
 149     list = p4CmdList(cmd)
 150     result = {}
 151     for entry in list:
 152         result.update(entry)
 153     return result;
 154
 155 def p4Where(depotPath):
 156     if not depotPath.endswith("/"):
 157         depotPath += "/"
 158     output = p4Cmd("where %s..." % depotPath)
 159     if output["code"] == "error":
 160         return ""
 161     clientPath = ""
 162     if "path" in output:
 163         clientPath = output.get("path")
 164     elif "data" in output:
 165         data = output.get("data")
 166         lastSpace = data.rfind(" ")
 167         clientPath = data[lastSpace + 1:]
 168
 169     if clientPath.endswith("..."):
 170         clientPath = clientPath[:-3]
 171     return clientPath
 172
 173 def currentGitBranch():
 174     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 175
 176 def isValidGitDir(path):
 177     if (os.path.exists(path + "/HEAD")
 178         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 179         return True;
 180     return False
 181
 182 def parseRevision(ref):
 183     return read_pipe("git rev-parse %s" % ref).strip()
 184
 185 def extractLogMessageFromGitCommit(commit):
 186     logMessage = ""
 187
 188     ## fixme: title is first line of commit, not 1st paragraph.
 189     foundTitle = False
 190     for log in read_pipe_lines("git cat-file commit %s" % commit):
 191        if not foundTitle:
 192            if len(log) == 1:
 193                foundTitle = True
 194            continue
 195
 196        logMessage += log
 197     return logMessage
 198
 199 def extractSettingsGitLog(log):
 200     values = {}
 201     for line in log.split("\n"):
 202         line = line.strip()
 203         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 204         if not m:
 205             continue
 206
 207         assignments = m.group(1).split (':')
 208         for a in assignments:
 209             vals = a.split ('=')
 210             key = vals[0].strip()
 211             val = ('='.join (vals[1:])).strip()
 212             if val.endswith ('\"') and val.startswith('"'):
 213                 val = val[1:-1]
 214
 215             values[key] = val
 216
 217     paths = values.get("depot-paths")
 218     if not paths:
 219         paths = values.get("depot-path")
 220     if paths:
 221         values['depot-paths'] = paths.split(',')
 222     return values
 223
 224 def gitBranchExists(branch):
 225     proc = subprocess.Popen(["git", "rev-parse", branch],
 226                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 227     return proc.wait() == 0;
 228
 229 def gitConfig(key):
 230     return read_pipe("git config %s" % key, ignore_error=True).strip()
 231
 232 def p4BranchesInGit(branchesAreInRemotes = True):
 233     branches = {}
 234
 235     cmdline = "git rev-parse --symbolic "
 236     if branchesAreInRemotes:
 237         cmdline += " --remotes"
 238     else:
 239         cmdline += " --branches"
 240
 241     for line in read_pipe_lines(cmdline):
 242         line = line.strip()
 243
 244         ## only import to p4/
 245         if not line.startswith('p4/') or line == "p4/HEAD":
 246             continue
 247         branch = line
 248
 249         # strip off p4
 250         branch = re.sub ("^p4/", "", line)
 251
 252         branches[branch] = parseRevision(line)
 253     return branches
 254
 255 def findUpstreamBranchPoint(head = "HEAD"):
 256     branches = p4BranchesInGit()
 257     # map from depot-path to branch name
 258     branchByDepotPath = {}
 259     for branch in branches.keys():
 260         tip = branches[branch]
 261         log = extractLogMessageFromGitCommit(tip)
 262         settings = extractSettingsGitLog(log)
 263         if settings.has_key("depot-paths"):
 264             paths = ",".join(settings["depot-paths"])
 265             branchByDepotPath[paths] = "remotes/p4/" + branch
 266
 267     settings = None
 268     parent = 0
 269     while parent < 65535:
 270         commit = head + "~%s" % parent
 271         log = extractLogMessageFromGitCommit(commit)
 272         settings = extractSettingsGitLog(log)
 273         if settings.has_key("depot-paths"):
 274             paths = ",".join(settings["depot-paths"])
 275             if branchByDepotPath.has_key(paths):
 276                 return [branchByDepotPath[paths], settings]
 277
 278         parent = parent + 1
 279
 280     return ["", settings]
 281
 282 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 283     if not silent:
 284         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 285                % localRefPrefix)
 286
 287     originPrefix = "origin/p4/"
 288
 289     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 290         line = line.strip()
 291         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 292             continue
 293
 294         headName = line[len(originPrefix):]
 295         remoteHead = localRefPrefix + headName
 296         originHead = line
 297
 298         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 299         if (not original.has_key('depot-paths')
 300             or not original.has_key('change')):
 301             continue
 302
 303         update = False
 304         if not gitBranchExists(remoteHead):
 305             if verbose:
 306                 print "creating %s" % remoteHead
 307             update = True
 308         else:
 309             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 310             if settings.has_key('change') > 0:
 311                 if settings['depot-paths'] == original['depot-paths']:
 312                     originP4Change = int(original['change'])
 313                     p4Change = int(settings['change'])
 314                     if originP4Change > p4Change:
 315                         print ("%s (%s) is newer than %s (%s). "
 316                                "Updating p4 branch from origin."
 317                                % (originHead, originP4Change,
 318                                   remoteHead, p4Change))
 319                         update = True
 320                 else:
 321                     print ("Ignoring: %s was imported from %s while "
 322                            "%s was imported from %s"
 323                            % (originHead, ','.join(original['depot-paths']),
 324                               remoteHead, ','.join(settings['depot-paths'])))
 325
 326         if update:
 327             system("git update-ref %s %s" % (remoteHead, originHead))
 328
 329 def originP4BranchesExist():
 330         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 331
 332 def p4ChangesForPaths(depotPaths, changeRange):
 333     assert depotPaths
 334     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 335                                                         for p in depotPaths]))
 336
 337     changes = []
 338     for line in output:
 339         changeNum = line.split(" ")[1]
 340         changes.append(int(changeNum))
 341
 342     changes.sort()
 343     return changes
 344
 345 class Command:
 346     def __init__(self):
 347         self.usage = "usage: %prog [options]"
 348         self.needsGit = True
 349
 350 class P4Debug(Command):
 351     def __init__(self):
 352         Command.__init__(self)
 353         self.options = [
 354             optparse.make_option("--verbose", dest="verbose", action="store_true",
 355                                  default=False),
 356             ]
 357         self.description = "A tool to debug the output of p4 -G."
 358         self.needsGit = False
 359         self.verbose = False
 360
 361     def run(self, args):
 362         j = 0
 363         for output in p4CmdList(" ".join(args)):
 364             print 'Element: %d' % j
 365             j += 1
 366             print output
 367         return True
 368
 369 class P4RollBack(Command):
 370     def __init__(self):
 371         Command.__init__(self)
 372         self.options = [
 373             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 374             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 375         ]
 376         self.description = "A tool to debug the multi-branch import. Don't use :)"
 377         self.verbose = False
 378         self.rollbackLocalBranches = False
 379
 380     def run(self, args):
 381         if len(args) != 1:
 382             return False
 383         maxChange = int(args[0])
 384
 385         if "p4ExitCode" in p4Cmd("changes -m 1"):
 386             die("Problems executing p4");
 387
 388         if self.rollbackLocalBranches:
 389             refPrefix = "refs/heads/"
 390             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 391         else:
 392             refPrefix = "refs/remotes/"
 393             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 394
 395         for line in lines:
 396             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 397                 line = line.strip()
 398                 ref = refPrefix + line
 399                 log = extractLogMessageFromGitCommit(ref)
 400                 settings = extractSettingsGitLog(log)
 401
 402                 depotPaths = settings['depot-paths']
 403                 change = settings['change']
 404
 405                 changed = False
 406
 407                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 408                                                            for p in depotPaths]))) == 0:
 409                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 410                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 411                     continue
 412
 413                 while change and int(change) > maxChange:
 414                     changed = True
 415                     if self.verbose:
 416                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 417                     system("git update-ref %s \"%s^\"" % (ref, ref))
 418                     log = extractLogMessageFromGitCommit(ref)
 419                     settings =  extractSettingsGitLog(log)
 420
 421
 422                     depotPaths = settings['depot-paths']
 423                     change = settings['change']
 424
 425                 if changed:
 426                     print "%s rewound to %s" % (ref, change)
 427
 428         return True
 429
 430 class P4Submit(Command):
 431     def __init__(self):
 432         Command.__init__(self)
 433         self.options = [
 434                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 435                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 436                 optparse.make_option("--origin", dest="origin"),
 437                 optparse.make_option("--reset", action="store_true", dest="reset"),
 438                 optparse.make_option("--log-substitutions", dest="substFile"),
 439                 optparse.make_option("--dry-run", action="store_true"),
 440                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 441                 optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"),
 442                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 443         ]
 444         self.description = "Submit changes from git to the perforce depot."
 445         self.usage += " [name of git branch to submit into perforce depot]"
 446         self.firstTime = True
 447         self.reset = False
 448         self.interactive = True
 449         self.dryRun = False
 450         self.substFile = ""
 451         self.firstTime = True
 452         self.origin = ""
 453         self.directSubmit = False
 454         self.trustMeLikeAFool = False
 455         self.detectRename = False
 456         self.verbose = False
 457         self.isWindows = (platform.system() == "Windows")
 458
 459         self.logSubstitutions = {}
 460         self.logSubstitutions["<enter description here>"] = "%log%"
 461         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 462
 463     def check(self):
 464         if len(p4CmdList("opened ...")) > 0:
 465             die("You have files opened with perforce! Close them before starting the sync.")
 466
 467     def start(self):
 468         if len(self.config) > 0 and not self.reset:
 469             die("Cannot start sync. Previous sync config found at %s\n"
 470                 "If you want to start submitting again from scratch "
 471                 "maybe you want to call git-p4 submit --reset" % self.configFile)
 472
 473         commits = []
 474         if self.directSubmit:
 475             commits.append("0")
 476         else:
 477             for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 478                 commits.append(line.strip())
 479             commits.reverse()
 480
 481         self.config["commits"] = commits
 482
 483     def prepareLogMessage(self, template, message):
 484         result = ""
 485
 486         for line in template.split("\n"):
 487             if line.startswith("#"):
 488                 result += line + "\n"
 489                 continue
 490
 491             substituted = False
 492             for key in self.logSubstitutions.keys():
 493                 if line.find(key) != -1:
 494                     value = self.logSubstitutions[key]
 495                     value = value.replace("%log%", message)
 496                     if value != "@remove@":
 497                         result += line.replace(key, value) + "\n"
 498                     substituted = True
 499                     break
 500
 501             if not substituted:
 502                 result += line + "\n"
 503
 504         return result
 505
 506     def prepareSubmitTemplate(self):
 507         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 508         template = ""
 509         inFilesSection = False
 510         for line in read_pipe_lines("p4 change -o"):
 511             if inFilesSection:
 512                 if line.startswith("\t"):
 513                     # path starts and ends with a tab
 514                     path = line[1:]
 515                     lastTab = path.rfind("\t")
 516                     if lastTab != -1:
 517                         path = path[:lastTab]
 518                         if not path.startswith(self.depotPath):
 519                             continue
 520                 else:
 521                     inFilesSection = False
 522             else:
 523                 if line.startswith("Files:"):
 524                     inFilesSection = True
 525
 526             template += line
 527
 528         return template
 529
 530     def applyCommit(self, id):
 531         if self.directSubmit:
 532             print "Applying local change in working directory/index"
 533             diff = self.diffStatus
 534         else:
 535             print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 536             diffOpts = ("", "-M")[self.detectRename]
 537             diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 538         filesToAdd = set()
 539         filesToDelete = set()
 540         editedFiles = set()
 541         for line in diff:
 542             diff = parseDiffTreeEntry(line)
 543             modifier = diff['status']
 544             path = diff['src']
 545             if modifier == "M":
 546                 system("p4 edit \"%s\"" % path)
 547                 editedFiles.add(path)
 548             elif modifier == "A":
 549                 filesToAdd.add(path)
 550                 if path in filesToDelete:
 551                     filesToDelete.remove(path)
 552             elif modifier == "D":
 553                 filesToDelete.add(path)
 554                 if path in filesToAdd:
 555                     filesToAdd.remove(path)
 556             elif modifier == "R":
 557                 src, dest = diff['src'], diff['dst']
 558                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 559                 system("p4 edit \"%s\"" % (dest))
 560                 os.unlink(dest)
 561                 editedFiles.add(dest)
 562                 filesToDelete.add(src)
 563             else:
 564                 die("unknown modifier %s for %s" % (modifier, path))
 565
 566         if self.directSubmit:
 567             diffcmd = "cat \"%s\"" % self.diffFile
 568         else:
 569             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 570         patchcmd = diffcmd + " | git apply "
 571         tryPatchCmd = patchcmd + "--check -"
 572         applyPatchCmd = patchcmd + "--check --apply -"
 573
 574         if os.system(tryPatchCmd) != 0:
 575             print "Unfortunately applying the change failed!"
 576             print "What do you want to do?"
 577             response = "x"
 578             while response != "s" and response != "a" and response != "w":
 579                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 580                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 581             if response == "s":
 582                 print "Skipping! Good luck with the next patches..."
 583                 for f in editedFiles:
 584                     system("p4 revert \"%s\"" % f);
 585                 for f in filesToAdd:
 586                     system("rm %s" %f)
 587                 return
 588             elif response == "a":
 589                 os.system(applyPatchCmd)
 590                 if len(filesToAdd) > 0:
 591                     print "You may also want to call p4 add on the following files:"
 592                     print " ".join(filesToAdd)
 593                 if len(filesToDelete):
 594                     print "The following files should be scheduled for deletion with p4 delete:"
 595                     print " ".join(filesToDelete)
 596                 die("Please resolve and submit the conflict manually and "
 597                     + "continue afterwards with git-p4 submit --continue")
 598             elif response == "w":
 599                 system(diffcmd + " > patch.txt")
 600                 print "Patch saved to patch.txt in %s !" % self.clientPath
 601                 die("Please resolve and submit the conflict manually and "
 602                     "continue afterwards with git-p4 submit --continue")
 603
 604         system(applyPatchCmd)
 605
 606         for f in filesToAdd:
 607             system("p4 add \"%s\"" % f)
 608         for f in filesToDelete:
 609             system("p4 revert \"%s\"" % f)
 610             system("p4 delete \"%s\"" % f)
 611
 612         logMessage = ""
 613         if not self.directSubmit:
 614             logMessage = extractLogMessageFromGitCommit(id)
 615             logMessage = logMessage.replace("\n", "\n\t")
 616             if self.isWindows:
 617                 logMessage = logMessage.replace("\n", "\r\n")
 618             logMessage = logMessage.strip()
 619
 620         template = self.prepareSubmitTemplate()
 621
 622         if self.interactive:
 623             submitTemplate = self.prepareLogMessage(template, logMessage)
 624             diff = read_pipe("p4 diff -du ...")
 625
 626             for newFile in filesToAdd:
 627                 diff += "==== new file ====\n"
 628                 diff += "--- /dev/null\n"
 629                 diff += "+++ %s\n" % newFile
 630                 f = open(newFile, "r")
 631                 for line in f.readlines():
 632                     diff += "+" + line
 633                 f.close()
 634
 635             separatorLine = "######## everything below this line is just the diff #######"
 636             if platform.system() == "Windows":
 637                 separatorLine += "\r"
 638             separatorLine += "\n"
 639
 640             response = "e"
 641             if self.trustMeLikeAFool:
 642                 response = "y"
 643
 644             firstIteration = True
 645             while response == "e":
 646                 if not firstIteration:
 647                     response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 648                 firstIteration = False
 649                 if response == "e":
 650                     [handle, fileName] = tempfile.mkstemp()
 651                     tmpFile = os.fdopen(handle, "w+")
 652                     tmpFile.write(submitTemplate + separatorLine + diff)
 653                     tmpFile.close()
 654                     defaultEditor = "vi"
 655                     if platform.system() == "Windows":
 656                         defaultEditor = "notepad"
 657                     editor = os.environ.get("EDITOR", defaultEditor);
 658                     system(editor + " " + fileName)
 659                     tmpFile = open(fileName, "rb")
 660                     message = tmpFile.read()
 661                     tmpFile.close()
 662                     os.remove(fileName)
 663                     submitTemplate = message[:message.index(separatorLine)]
 664                     if self.isWindows:
 665                         submitTemplate = submitTemplate.replace("\r\n", "\n")
 666
 667             if response == "y" or response == "yes":
 668                if self.dryRun:
 669                    print submitTemplate
 670                    raw_input("Press return to continue...")
 671                else:
 672                    if self.directSubmit:
 673                        print "Submitting to git first"
 674                        os.chdir(self.oldWorkingDirectory)
 675                        write_pipe("git commit -a -F -", submitTemplate)
 676                        os.chdir(self.clientPath)
 677
 678                    write_pipe("p4 submit -i", submitTemplate)
 679             elif response == "s":
 680                 for f in editedFiles:
 681                     system("p4 revert \"%s\"" % f);
 682                 for f in filesToAdd:
 683                     system("p4 revert \"%s\"" % f);
 684                     system("rm %s" %f)
 685                 for f in filesToDelete:
 686                     system("p4 delete \"%s\"" % f);
 687                 return
 688             else:
 689                 print "Not submitting!"
 690                 self.interactive = False
 691         else:
 692             fileName = "submit.txt"
 693             file = open(fileName, "w+")
 694             file.write(self.prepareLogMessage(template, logMessage))
 695             file.close()
 696             print ("Perforce submit template written as %s. "
 697                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 698                    % (fileName, fileName))
 699
 700     def run(self, args):
 701         if len(args) == 0:
 702             self.master = currentGitBranch()
 703             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 704                 die("Detecting current git branch failed!")
 705         elif len(args) == 1:
 706             self.master = args[0]
 707         else:
 708             return False
 709
 710         [upstream, settings] = findUpstreamBranchPoint()
 711         self.depotPath = settings['depot-paths'][0]
 712         if len(self.origin) == 0:
 713             self.origin = upstream
 714
 715         if self.verbose:
 716             print "Origin branch is " + self.origin
 717
 718         if len(self.depotPath) == 0:
 719             print "Internal error: cannot locate perforce depot path from existing branches"
 720             sys.exit(128)
 721
 722         self.clientPath = p4Where(self.depotPath)
 723
 724         if len(self.clientPath) == 0:
 725             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 726             sys.exit(128)
 727
 728         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 729         self.oldWorkingDirectory = os.getcwd()
 730
 731         if self.directSubmit:
 732             self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 733             if len(self.diffStatus) == 0:
 734                 print "No changes in working directory to submit."
 735                 return True
 736             patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 737             self.diffFile = self.gitdir + "/p4-git-diff"
 738             f = open(self.diffFile, "wb")
 739             f.write(patch)
 740             f.close();
 741
 742         os.chdir(self.clientPath)
 743         print "Syncronizing p4 checkout..."
 744         system("p4 sync ...")
 745
 746         if self.reset:
 747             self.firstTime = True
 748
 749         if len(self.substFile) > 0:
 750             for line in open(self.substFile, "r").readlines():
 751                 tokens = line.strip().split("=")
 752                 self.logSubstitutions[tokens[0]] = tokens[1]
 753
 754         self.check()
 755         self.configFile = self.gitdir + "/p4-git-sync.cfg"
 756         self.config = shelve.open(self.configFile, writeback=True)
 757
 758         if self.firstTime:
 759             self.start()
 760
 761         commits = self.config.get("commits", [])
 762
 763         while len(commits) > 0:
 764             self.firstTime = False
 765             commit = commits[0]
 766             commits = commits[1:]
 767             self.config["commits"] = commits
 768             self.applyCommit(commit)
 769             if not self.interactive:
 770                 break
 771
 772         self.config.close()
 773
 774         if self.directSubmit:
 775             os.remove(self.diffFile)
 776
 777         if len(commits) == 0:
 778             if self.firstTime:
 779                 print "No changes found to apply between %s and current HEAD" % self.origin
 780             else:
 781                 print "All changes applied!"
 782                 os.chdir(self.oldWorkingDirectory)
 783
 784                 sync = P4Sync()
 785                 sync.run([])
 786
 787                 response = raw_input("Do you want to rebase current HEAD from Perforce now using git-p4 rebase? [y]es/[n]o ")
 788                 if response == "y" or response == "yes":
 789                     rebase = P4Rebase()
 790                     rebase.rebase()
 791             os.remove(self.configFile)
 792
 793         return True
 794
 795 class P4Sync(Command):
 796     def __init__(self):
 797         Command.__init__(self)
 798         self.options = [
 799                 optparse.make_option("--branch", dest="branch"),
 800                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 801                 optparse.make_option("--changesfile", dest="changesFile"),
 802                 optparse.make_option("--silent", dest="silent", action="store_true"),
 803                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 804                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 805                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 806                                      help="Import into refs/heads/ , not refs/remotes"),
 807                 optparse.make_option("--max-changes", dest="maxChanges"),
 808                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 809                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 810         ]
 811         self.description = """Imports from Perforce into a git repository.\n
 812     example:
 813     //depot/my/project/ -- to import the current head
 814     //depot/my/project/@all -- to import everything
 815     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 816
 817     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 818
 819         self.usage += " //depot/path[@revRange]"
 820         self.silent = False
 821         self.createdBranches = Set()
 822         self.committedChanges = Set()
 823         self.branch = ""
 824         self.detectBranches = False
 825         self.detectLabels = False
 826         self.changesFile = ""
 827         self.syncWithOrigin = True
 828         self.verbose = False
 829         self.importIntoRemotes = True
 830         self.maxChanges = ""
 831         self.isWindows = (platform.system() == "Windows")
 832         self.keepRepoPath = False
 833         self.depotPaths = None
 834         self.p4BranchesInGit = []
 835
 836         if gitConfig("git-p4.syncFromOrigin") == "false":
 837             self.syncWithOrigin = False
 838
 839     def extractFilesFromCommit(self, commit):
 840         files = []
 841         fnum = 0
 842         while commit.has_key("depotFile%s" % fnum):
 843             path =  commit["depotFile%s" % fnum]
 844
 845             found = [p for p in self.depotPaths
 846                      if path.startswith (p)]
 847             if not found:
 848                 fnum = fnum + 1
 849                 continue
 850
 851             file = {}
 852             file["path"] = path
 853             file["rev"] = commit["rev%s" % fnum]
 854             file["action"] = commit["action%s" % fnum]
 855             file["type"] = commit["type%s" % fnum]
 856             files.append(file)
 857             fnum = fnum + 1
 858         return files
 859
 860     def stripRepoPath(self, path, prefixes):
 861         if self.keepRepoPath:
 862             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 863
 864         for p in prefixes:
 865             if path.startswith(p):
 866                 path = path[len(p):]
 867
 868         return path
 869
 870     def splitFilesIntoBranches(self, commit):
 871         branches = {}
 872         fnum = 0
 873         while commit.has_key("depotFile%s" % fnum):
 874             path =  commit["depotFile%s" % fnum]
 875             found = [p for p in self.depotPaths
 876                      if path.startswith (p)]
 877             if not found:
 878                 fnum = fnum + 1
 879                 continue
 880
 881             file = {}
 882             file["path"] = path
 883             file["rev"] = commit["rev%s" % fnum]
 884             file["action"] = commit["action%s" % fnum]
 885             file["type"] = commit["type%s" % fnum]
 886             fnum = fnum + 1
 887
 888             relPath = self.stripRepoPath(path, self.depotPaths)
 889
 890             for branch in self.knownBranches.keys():
 891
 892                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 893                 if relPath.startswith(branch + "/"):
 894                     if branch not in branches:
 895                         branches[branch] = []
 896                     branches[branch].append(file)
 897                     break
 898
 899         return branches
 900
 901     ## Should move this out, doesn't use SELF.
 902     def readP4Files(self, files):
 903         files = [f for f in files
 904                  if f['action'] != 'delete']
 905
 906         if not files:
 907             return
 908
 909         filedata = p4CmdList('-x - print',
 910                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 911                                               for f in files]),
 912                              stdin_mode='w+')
 913         if "p4ExitCode" in filedata[0]:
 914             die("Problems executing p4. Error: [%d]."
 915                 % (filedata[0]['p4ExitCode']));
 916
 917         j = 0;
 918         contents = {}
 919         while j < len(filedata):
 920             stat = filedata[j]
 921             j += 1
 922             text = ''
 923             while j < len(filedata) and filedata[j]['code'] in ('text',
 924                                                                 'binary'):
 925                 text += filedata[j]['data']
 926                 j += 1
 927
 928
 929             if not stat.has_key('depotFile'):
 930                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 931                 continue
 932
 933             contents[stat['depotFile']] = text
 934
 935         for f in files:
 936             assert not f.has_key('data')
 937             f['data'] = contents[f['path']]
 938
 939     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 940         epoch = details["time"]
 941         author = details["user"]
 942
 943         if self.verbose:
 944             print "commit into %s" % branch
 945
 946         # start with reading files; if that fails, we should not
 947         # create a commit.
 948         new_files = []
 949         for f in files:
 950             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 951                 new_files.append (f)
 952             else:
 953                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 954         files = new_files
 955         self.readP4Files(files)
 956
 957
 958
 959
 960         self.gitStream.write("commit %s\n" % branch)
 961 #        gitStream.write("mark :%s\n" % details["change"])
 962         self.committedChanges.add(int(details["change"]))
 963         committer = ""
 964         if author not in self.users:
 965             self.getUserMapFromPerforceServer()
 966         if author in self.users:
 967             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 968         else:
 969             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 970
 971         self.gitStream.write("committer %s\n" % committer)
 972
 973         self.gitStream.write("data <<EOT\n")
 974         self.gitStream.write(details["desc"])
 975         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 976                              % (','.join (branchPrefixes), details["change"]))
 977         if len(details['options']) > 0:
 978             self.gitStream.write(": options = %s" % details['options'])
 979         self.gitStream.write("]\nEOT\n\n")
 980
 981         if len(parent) > 0:
 982             if self.verbose:
 983                 print "parent %s" % parent
 984             self.gitStream.write("from %s\n" % parent)
 985
 986         for file in files:
 987             if file["type"] == "apple":
 988                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 989                 continue
 990
 991             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 992             if file["action"] == "delete":
 993                 self.gitStream.write("D %s\n" % relPath)
 994             else:
 995                 data = file['data']
 996
 997                 mode = "644"
 998                 if isP4Exec(file["type"]):
 999                     mode = "755"
1000                 elif file["type"] == "symlink":
1001                     mode = "120000"
1002                     # p4 print on a symlink contains "target\n", so strip it off
1003                     data = data[:-1]
1004
1005                 if self.isWindows and file["type"].endswith("text"):
1006                     data = data.replace("\r\n", "\n")
1007
1008                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1009                 self.gitStream.write("data %s\n" % len(data))
1010                 self.gitStream.write(data)
1011                 self.gitStream.write("\n")
1012
1013         self.gitStream.write("\n")
1014
1015         change = int(details["change"])
1016
1017         if self.labels.has_key(change):
1018             label = self.labels[change]
1019             labelDetails = label[0]
1020             labelRevisions = label[1]
1021             if self.verbose:
1022                 print "Change %s is labelled %s" % (change, labelDetails)
1023
1024             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1025                                                     for p in branchPrefixes]))
1026
1027             if len(files) == len(labelRevisions):
1028
1029                 cleanedFiles = {}
1030                 for info in files:
1031                     if info["action"] == "delete":
1032                         continue
1033                     cleanedFiles[info["depotFile"]] = info["rev"]
1034
1035                 if cleanedFiles == labelRevisions:
1036                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1037                     self.gitStream.write("from %s\n" % branch)
1038
1039                     owner = labelDetails["Owner"]
1040                     tagger = ""
1041                     if author in self.users:
1042                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1043                     else:
1044                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1045                     self.gitStream.write("tagger %s\n" % tagger)
1046                     self.gitStream.write("data <<EOT\n")
1047                     self.gitStream.write(labelDetails["Description"])
1048                     self.gitStream.write("EOT\n\n")
1049
1050                 else:
1051                     if not self.silent:
1052                         print ("Tag %s does not match with change %s: files do not match."
1053                                % (labelDetails["label"], change))
1054
1055             else:
1056                 if not self.silent:
1057                     print ("Tag %s does not match with change %s: file count is different."
1058                            % (labelDetails["label"], change))
1059
1060     def getUserCacheFilename(self):
1061         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1062         return home + "/.gitp4-usercache.txt"
1063
1064     def getUserMapFromPerforceServer(self):
1065         if self.userMapFromPerforceServer:
1066             return
1067         self.users = {}
1068
1069         for output in p4CmdList("users"):
1070             if not output.has_key("User"):
1071                 continue
1072             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1073
1074
1075         s = ''
1076         for (key, val) in self.users.items():
1077             s += "%s\t%s\n" % (key, val)
1078
1079         open(self.getUserCacheFilename(), "wb").write(s)
1080         self.userMapFromPerforceServer = True
1081
1082     def loadUserMapFromCache(self):
1083         self.users = {}
1084         self.userMapFromPerforceServer = False
1085         try:
1086             cache = open(self.getUserCacheFilename(), "rb")
1087             lines = cache.readlines()
1088             cache.close()
1089             for line in lines:
1090                 entry = line.strip().split("\t")
1091                 self.users[entry[0]] = entry[1]
1092         except IOError:
1093             self.getUserMapFromPerforceServer()
1094
1095     def getLabels(self):
1096         self.labels = {}
1097
1098         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1099         if len(l) > 0 and not self.silent:
1100             print "Finding files belonging to labels in %s" % `self.depotPath`
1101
1102         for output in l:
1103             label = output["label"]
1104             revisions = {}
1105             newestChange = 0
1106             if self.verbose:
1107                 print "Querying files for label %s" % label
1108             for file in p4CmdList("files "
1109                                   +  ' '.join (["%s...@%s" % (p, label)
1110                                                 for p in self.depotPaths])):
1111                 revisions[file["depotFile"]] = file["rev"]
1112                 change = int(file["change"])
1113                 if change > newestChange:
1114                     newestChange = change
1115
1116             self.labels[newestChange] = [output, revisions]
1117
1118         if self.verbose:
1119             print "Label changes: %s" % self.labels.keys()
1120
1121     def guessProjectName(self):
1122         for p in self.depotPaths:
1123             if p.endswith("/"):
1124                 p = p[:-1]
1125             p = p[p.strip().rfind("/") + 1:]
1126             if not p.endswith("/"):
1127                p += "/"
1128             return p
1129
1130     def getBranchMapping(self):
1131         lostAndFoundBranches = set()
1132
1133         for info in p4CmdList("branches"):
1134             details = p4Cmd("branch -o %s" % info["branch"])
1135             viewIdx = 0
1136             while details.has_key("View%s" % viewIdx):
1137                 paths = details["View%s" % viewIdx].split(" ")
1138                 viewIdx = viewIdx + 1
1139                 # require standard //depot/foo/... //depot/bar/... mapping
1140                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1141                     continue
1142                 source = paths[0]
1143                 destination = paths[1]
1144                 ## HACK
1145                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1146                     source = source[len(self.depotPaths[0]):-4]
1147                     destination = destination[len(self.depotPaths[0]):-4]
1148
1149                     if destination in self.knownBranches:
1150                         if not self.silent:
1151                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1152                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1153                         continue
1154
1155                     self.knownBranches[destination] = source
1156
1157                     lostAndFoundBranches.discard(destination)
1158
1159                     if source not in self.knownBranches:
1160                         lostAndFoundBranches.add(source)
1161
1162
1163         for branch in lostAndFoundBranches:
1164             self.knownBranches[branch] = branch
1165
1166     def listExistingP4GitBranches(self):
1167         # branches holds mapping from name to commit
1168         branches = p4BranchesInGit(self.importIntoRemotes)
1169         self.p4BranchesInGit = branches.keys()
1170         for branch in branches.keys():
1171             self.initialParents[self.refPrefix + branch] = branches[branch]
1172
1173     def updateOptionDict(self, d):
1174         option_keys = {}
1175         if self.keepRepoPath:
1176             option_keys['keepRepoPath'] = 1
1177
1178         d["options"] = ' '.join(sorted(option_keys.keys()))
1179
1180     def readOptions(self, d):
1181         self.keepRepoPath = (d.has_key('options')
1182                              and ('keepRepoPath' in d['options']))
1183
1184     def gitRefForBranch(self, branch):
1185         if branch == "main":
1186             return self.refPrefix + "master"
1187
1188         if len(branch) <= 0:
1189             return branch
1190
1191         return self.refPrefix + self.projectName + branch
1192
1193     def gitCommitByP4Change(self, ref, change):
1194         if self.verbose:
1195             print "looking in ref " + ref + " for change %s using bisect..." % change
1196
1197         earliestCommit = ""
1198         latestCommit = parseRevision(ref)
1199
1200         while True:
1201             if self.verbose:
1202                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1203             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1204             if len(next) == 0:
1205                 if self.verbose:
1206                     print "argh"
1207                 return ""
1208             log = extractLogMessageFromGitCommit(next)
1209             settings = extractSettingsGitLog(log)
1210             currentChange = int(settings['change'])
1211             if self.verbose:
1212                 print "current change %s" % currentChange
1213
1214             if currentChange == change:
1215                 if self.verbose:
1216                     print "found %s" % next
1217                 return next
1218
1219             if currentChange < change:
1220                 earliestCommit = "^%s" % next
1221             else:
1222                 latestCommit = "%s" % next
1223
1224         return ""
1225
1226     def importNewBranch(self, branch, maxChange):
1227         # make fast-import flush all changes to disk and update the refs using the checkpoint
1228         # command so that we can try to find the branch parent in the git history
1229         self.gitStream.write("checkpoint\n\n");
1230         self.gitStream.flush();
1231         branchPrefix = self.depotPaths[0] + branch + "/"
1232         range = "@1,%s" % maxChange
1233         #print "prefix" + branchPrefix
1234         changes = p4ChangesForPaths([branchPrefix], range)
1235         if len(changes) <= 0:
1236             return False
1237         firstChange = changes[0]
1238         #print "first change in branch: %s" % firstChange
1239         sourceBranch = self.knownBranches[branch]
1240         sourceDepotPath = self.depotPaths[0] + sourceBranch
1241         sourceRef = self.gitRefForBranch(sourceBranch)
1242         #print "source " + sourceBranch
1243
1244         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1245         #print "branch parent: %s" % branchParentChange
1246         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1247         if len(gitParent) > 0:
1248             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1249             #print "parent git commit: %s" % gitParent
1250
1251         self.importChanges(changes)
1252         return True
1253
1254     def importChanges(self, changes):
1255         cnt = 1
1256         for change in changes:
1257             description = p4Cmd("describe %s" % change)
1258             self.updateOptionDict(description)
1259
1260             if not self.silent:
1261                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1262                 sys.stdout.flush()
1263             cnt = cnt + 1
1264
1265             try:
1266                 if self.detectBranches:
1267                     branches = self.splitFilesIntoBranches(description)
1268                     for branch in branches.keys():
1269                         ## HACK  --hwn
1270                         branchPrefix = self.depotPaths[0] + branch + "/"
1271
1272                         parent = ""
1273
1274                         filesForCommit = branches[branch]
1275
1276                         if self.verbose:
1277                             print "branch is %s" % branch
1278
1279                         self.updatedBranches.add(branch)
1280
1281                         if branch not in self.createdBranches:
1282                             self.createdBranches.add(branch)
1283                             parent = self.knownBranches[branch]
1284                             if parent == branch:
1285                                 parent = ""
1286                             else:
1287                                 fullBranch = self.projectName + branch
1288                                 if fullBranch not in self.p4BranchesInGit:
1289                                     if not self.silent:
1290                                         print("\n    Importing new branch %s" % fullBranch);
1291                                     if self.importNewBranch(branch, change - 1):
1292                                         parent = ""
1293                                         self.p4BranchesInGit.append(fullBranch)
1294                                     if not self.silent:
1295                                         print("\n    Resuming with change %s" % change);
1296
1297                                 if self.verbose:
1298                                     print "parent determined through known branches: %s" % parent
1299
1300                         branch = self.gitRefForBranch(branch)
1301                         parent = self.gitRefForBranch(parent)
1302
1303                         if self.verbose:
1304                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1305
1306                         if len(parent) == 0 and branch in self.initialParents:
1307                             parent = self.initialParents[branch]
1308                             del self.initialParents[branch]
1309
1310                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1311                 else:
1312                     files = self.extractFilesFromCommit(description)
1313                     self.commit(description, files, self.branch, self.depotPaths,
1314                                 self.initialParent)
1315                     self.initialParent = ""
1316             except IOError:
1317                 print self.gitError.read()
1318                 sys.exit(1)
1319
1320     def importHeadRevision(self, revision):
1321         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1322
1323         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1324         details["desc"] = ("Initial import of %s from the state at revision %s"
1325                            % (' '.join(self.depotPaths), revision))
1326         details["change"] = revision
1327         newestRevision = 0
1328
1329         fileCnt = 0
1330         for info in p4CmdList("files "
1331                               +  ' '.join(["%s...%s"
1332                                            % (p, revision)
1333                                            for p in self.depotPaths])):
1334
1335             if info['code'] == 'error':
1336                 sys.stderr.write("p4 returned an error: %s\n"
1337                                  % info['data'])
1338                 sys.exit(1)
1339
1340
1341             change = int(info["change"])
1342             if change > newestRevision:
1343                 newestRevision = change
1344
1345             if info["action"] == "delete":
1346                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1347                 #fileCnt = fileCnt + 1
1348                 continue
1349
1350             for prop in ["depotFile", "rev", "action", "type" ]:
1351                 details["%s%s" % (prop, fileCnt)] = info[prop]
1352
1353             fileCnt = fileCnt + 1
1354
1355         details["change"] = newestRevision
1356         self.updateOptionDict(details)
1357         try:
1358             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1359         except IOError:
1360             print "IO error with git fast-import. Is your git version recent enough?"
1361             print self.gitError.read()
1362
1363
1364     def run(self, args):
1365         self.depotPaths = []
1366         self.changeRange = ""
1367         self.initialParent = ""
1368         self.previousDepotPaths = []
1369
1370         # map from branch depot path to parent branch
1371         self.knownBranches = {}
1372         self.initialParents = {}
1373         self.hasOrigin = originP4BranchesExist()
1374         if not self.syncWithOrigin:
1375             self.hasOrigin = False
1376
1377         if self.importIntoRemotes:
1378             self.refPrefix = "refs/remotes/p4/"
1379         else:
1380             self.refPrefix = "refs/heads/p4/"
1381
1382         if self.syncWithOrigin and self.hasOrigin:
1383             if not self.silent:
1384                 print "Syncing with origin first by calling git fetch origin"
1385             system("git fetch origin")
1386
1387         if len(self.branch) == 0:
1388             self.branch = self.refPrefix + "master"
1389             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1390                 system("git update-ref %s refs/heads/p4" % self.branch)
1391                 system("git branch -D p4");
1392             # create it /after/ importing, when master exists
1393             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1394                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1395
1396         # TODO: should always look at previous commits,
1397         # merge with previous imports, if possible.
1398         if args == []:
1399             if self.hasOrigin:
1400                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1401             self.listExistingP4GitBranches()
1402
1403             if len(self.p4BranchesInGit) > 1:
1404                 if not self.silent:
1405                     print "Importing from/into multiple branches"
1406                 self.detectBranches = True
1407
1408             if self.verbose:
1409                 print "branches: %s" % self.p4BranchesInGit
1410
1411             p4Change = 0
1412             for branch in self.p4BranchesInGit:
1413                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1414
1415                 settings = extractSettingsGitLog(logMsg)
1416
1417                 self.readOptions(settings)
1418                 if (settings.has_key('depot-paths')
1419                     and settings.has_key ('change')):
1420                     change = int(settings['change']) + 1
1421                     p4Change = max(p4Change, change)
1422
1423                     depotPaths = sorted(settings['depot-paths'])
1424                     if self.previousDepotPaths == []:
1425                         self.previousDepotPaths = depotPaths
1426                     else:
1427                         paths = []
1428                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1429                             for i in range(0, min(len(cur), len(prev))):
1430                                 if cur[i] <> prev[i]:
1431                                     i = i - 1
1432                                     break
1433
1434                             paths.append (cur[:i + 1])
1435
1436                         self.previousDepotPaths = paths
1437
1438             if p4Change > 0:
1439                 self.depotPaths = sorted(self.previousDepotPaths)
1440                 self.changeRange = "@%s,#head" % p4Change
1441                 if not self.detectBranches:
1442                     self.initialParent = parseRevision(self.branch)
1443                 if not self.silent and not self.detectBranches:
1444                     print "Performing incremental import into %s git branch" % self.branch
1445
1446         if not self.branch.startswith("refs/"):
1447             self.branch = "refs/heads/" + self.branch
1448
1449         if len(args) == 0 and self.depotPaths:
1450             if not self.silent:
1451                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1452         else:
1453             if self.depotPaths and self.depotPaths != args:
1454                 print ("previous import used depot path %s and now %s was specified. "
1455                        "This doesn't work!" % (' '.join (self.depotPaths),
1456                                                ' '.join (args)))
1457                 sys.exit(1)
1458
1459             self.depotPaths = sorted(args)
1460
1461         revision = ""
1462         self.users = {}
1463
1464         newPaths = []
1465         for p in self.depotPaths:
1466             if p.find("@") != -1:
1467                 atIdx = p.index("@")
1468                 self.changeRange = p[atIdx:]
1469                 if self.changeRange == "@all":
1470                     self.changeRange = ""
1471                 elif ',' not in self.changeRange:
1472                     revision = self.changeRange
1473                     self.changeRange = ""
1474                 p = p[:atIdx]
1475             elif p.find("#") != -1:
1476                 hashIdx = p.index("#")
1477                 revision = p[hashIdx:]
1478                 p = p[:hashIdx]
1479             elif self.previousDepotPaths == []:
1480                 revision = "#head"
1481
1482             p = re.sub ("\.\.\.$", "", p)
1483             if not p.endswith("/"):
1484                 p += "/"
1485
1486             newPaths.append(p)
1487
1488         self.depotPaths = newPaths
1489
1490
1491         self.loadUserMapFromCache()
1492         self.labels = {}
1493         if self.detectLabels:
1494             self.getLabels();
1495
1496         if self.detectBranches:
1497             ## FIXME - what's a P4 projectName ?
1498             self.projectName = self.guessProjectName()
1499
1500             if not self.hasOrigin:
1501                 self.getBranchMapping();
1502             if self.verbose:
1503                 print "p4-git branches: %s" % self.p4BranchesInGit
1504                 print "initial parents: %s" % self.initialParents
1505             for b in self.p4BranchesInGit:
1506                 if b != "master":
1507
1508                     ## FIXME
1509                     b = b[len(self.projectName):]
1510                 self.createdBranches.add(b)
1511
1512         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1513
1514         importProcess = subprocess.Popen(["git", "fast-import"],
1515                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1516                                          stderr=subprocess.PIPE);
1517         self.gitOutput = importProcess.stdout
1518         self.gitStream = importProcess.stdin
1519         self.gitError = importProcess.stderr
1520
1521         if revision:
1522             self.importHeadRevision(revision)
1523         else:
1524             changes = []
1525
1526             if len(self.changesFile) > 0:
1527                 output = open(self.changesFile).readlines()
1528                 changeSet = Set()
1529                 for line in output:
1530                     changeSet.add(int(line))
1531
1532                 for change in changeSet:
1533                     changes.append(change)
1534
1535                 changes.sort()
1536             else:
1537                 if self.verbose:
1538                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1539                                                               self.changeRange)
1540                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1541
1542                 if len(self.maxChanges) > 0:
1543                     changes = changes[:min(int(self.maxChanges), len(changes))]
1544
1545             if len(changes) == 0:
1546                 if not self.silent:
1547                     print "No changes to import!"
1548                 return True
1549
1550             if not self.silent and not self.detectBranches:
1551                 print "Import destination: %s" % self.branch
1552
1553             self.updatedBranches = set()
1554
1555             self.importChanges(changes)
1556
1557             if not self.silent:
1558                 print ""
1559                 if len(self.updatedBranches) > 0:
1560                     sys.stdout.write("Updated branches: ")
1561                     for b in self.updatedBranches:
1562                         sys.stdout.write("%s " % b)
1563                     sys.stdout.write("\n")
1564
1565         self.gitStream.close()
1566         if importProcess.wait() != 0:
1567             die("fast-import failed: %s" % self.gitError.read())
1568         self.gitOutput.close()
1569         self.gitError.close()
1570
1571         return True
1572
1573 class P4Rebase(Command):
1574     def __init__(self):
1575         Command.__init__(self)
1576         self.options = [ ]
1577         self.description = ("Fetches the latest revision from perforce and "
1578                             + "rebases the current work (branch) against it")
1579         self.verbose = False
1580
1581     def run(self, args):
1582         sync = P4Sync()
1583         sync.run([])
1584
1585         return self.rebase()
1586
1587     def rebase(self):
1588         [upstream, settings] = findUpstreamBranchPoint()
1589         if len(upstream) == 0:
1590             die("Cannot find upstream branchpoint for rebase")
1591
1592         # the branchpoint may be p4/foo~3, so strip off the parent
1593         upstream = re.sub("~[0-9]+$", "", upstream)
1594
1595         print "Rebasing the current branch onto %s" % upstream
1596         oldHead = read_pipe("git rev-parse HEAD").strip()
1597         system("git rebase %s" % upstream)
1598         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1599         return True
1600
1601 class P4Clone(P4Sync):
1602     def __init__(self):
1603         P4Sync.__init__(self)
1604         self.description = "Creates a new git repository and imports from Perforce into it"
1605         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1606         self.options.append(
1607             optparse.make_option("--destination", dest="cloneDestination",
1608                                  action='store', default=None,
1609                                  help="where to leave result of the clone"))
1610         self.cloneDestination = None
1611         self.needsGit = False
1612
1613     def defaultDestination(self, args):
1614         ## TODO: use common prefix of args?
1615         depotPath = args[0]
1616         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1617         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1618         depotDir = re.sub(r"\.\.\.$,", "", depotDir)
1619         depotDir = re.sub(r"/$", "", depotDir)
1620         return os.path.split(depotDir)[1]
1621
1622     def run(self, args):
1623         if len(args) < 1:
1624             return False
1625
1626         if self.keepRepoPath and not self.cloneDestination:
1627             sys.stderr.write("Must specify destination for --keep-path\n")
1628             sys.exit(1)
1629
1630         depotPaths = args
1631
1632         if not self.cloneDestination and len(depotPaths) > 1:
1633             self.cloneDestination = depotPaths[-1]
1634             depotPaths = depotPaths[:-1]
1635
1636         for p in depotPaths:
1637             if not p.startswith("//"):
1638                 return False
1639
1640         if not self.cloneDestination:
1641             self.cloneDestination = self.defaultDestination(args)
1642
1643         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1644         if not os.path.exists(self.cloneDestination):
1645             os.makedirs(self.cloneDestination)
1646         os.chdir(self.cloneDestination)
1647         system("git init")
1648         self.gitdir = os.getcwd() + "/.git"
1649         if not P4Sync.run(self, depotPaths):
1650             return False
1651         if self.branch != "master":
1652             if gitBranchExists("refs/remotes/p4/master"):
1653                 system("git branch master refs/remotes/p4/master")
1654                 system("git checkout -f")
1655             else:
1656                 print "Could not detect main branch. No checkout/master branch created."
1657
1658         return True
1659
1660 class P4Branches(Command):
1661     def __init__(self):
1662         Command.__init__(self)
1663         self.options = [ ]
1664         self.description = ("Shows the git branches that hold imports and their "
1665                             + "corresponding perforce depot paths")
1666         self.verbose = False
1667
1668     def run(self, args):
1669         if originP4BranchesExist():
1670             createOrUpdateBranchesFromOrigin()
1671
1672         cmdline = "git rev-parse --symbolic "
1673         cmdline += " --remotes"
1674
1675         for line in read_pipe_lines(cmdline):
1676             line = line.strip()
1677
1678             if not line.startswith('p4/') or line == "p4/HEAD":
1679                 continue
1680             branch = line
1681
1682             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1683             settings = extractSettingsGitLog(log)
1684
1685             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1686         return True
1687
1688 class HelpFormatter(optparse.IndentedHelpFormatter):
1689     def __init__(self):
1690         optparse.IndentedHelpFormatter.__init__(self)
1691
1692     def format_description(self, description):
1693         if description:
1694             return description + "\n"
1695         else:
1696             return ""
1697
1698 def printUsage(commands):
1699     print "usage: %s <command> [options]" % sys.argv[0]
1700     print ""
1701     print "valid commands: %s" % ", ".join(commands)
1702     print ""
1703     print "Try %s <command> --help for command specific help." % sys.argv[0]
1704     print ""
1705
1706 commands = {
1707     "debug" : P4Debug,
1708     "submit" : P4Submit,
1709     "commit" : P4Submit,
1710     "sync" : P4Sync,
1711     "rebase" : P4Rebase,
1712     "clone" : P4Clone,
1713     "rollback" : P4RollBack,
1714     "branches" : P4Branches
1715 }
1716
1717
1718 def main():
1719     if len(sys.argv[1:]) == 0:
1720         printUsage(commands.keys())
1721         sys.exit(2)
1722
1723     cmd = ""
1724     cmdName = sys.argv[1]
1725     try:
1726         klass = commands[cmdName]
1727         cmd = klass()
1728     except KeyError:
1729         print "unknown command %s" % cmdName
1730         print ""
1731         printUsage(commands.keys())
1732         sys.exit(2)
1733
1734     options = cmd.options
1735     cmd.gitdir = os.environ.get("GIT_DIR", None)
1736
1737     args = sys.argv[2:]
1738
1739     if len(options) > 0:
1740         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1741
1742         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1743                                        options,
1744                                        description = cmd.description,
1745                                        formatter = HelpFormatter())
1746
1747         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1748     global verbose
1749     verbose = cmd.verbose
1750     if cmd.needsGit:
1751         if cmd.gitdir == None:
1752             cmd.gitdir = os.path.abspath(".git")
1753             if not isValidGitDir(cmd.gitdir):
1754                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1755                 if os.path.exists(cmd.gitdir):
1756                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1757                     if len(cdup) > 0:
1758                         os.chdir(cdup);
1759
1760         if not isValidGitDir(cmd.gitdir):
1761             if isValidGitDir(cmd.gitdir + "/.git"):
1762                 cmd.gitdir += "/.git"
1763             else:
1764                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1765
1766         os.environ["GIT_DIR"] = cmd.gitdir
1767
1768     if not cmd.run(args):
1769         parser.print_help()
1770
1771
1772 if __name__ == '__main__':
1773     main()