contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <hausmann@kde.org>
   6 # Copyright: 2007 Simon Hausmann <hausmann@kde.org>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10 # TODO: * Consider making --with-origin the default, assuming that the git
  11 #         protocol is always more efficient. (needs manual testing first :)
  12 #
  13
  14 import optparse, sys, os, marshal, popen2, subprocess, shelve
  15 import tempfile, getopt, sha, os.path, time, platform
  16 from sets import Set;
  17
  18 gitdir = os.environ.get("GIT_DIR", "")
  19
  20 def mypopen(command):
  21     return os.popen(command, "rb");
  22
  23 def p4CmdList(cmd):
  24     cmd = "p4 -G %s" % cmd
  25     pipe = os.popen(cmd, "rb")
  26
  27     result = []
  28     try:
  29         while True:
  30             entry = marshal.load(pipe)
  31             result.append(entry)
  32     except EOFError:
  33         pass
  34     exitCode = pipe.close()
  35     if exitCode != None:
  36         entry = {}
  37         entry["p4ExitCode"] = exitCode
  38         result.append(entry)
  39
  40     return result
  41
  42 def p4Cmd(cmd):
  43     list = p4CmdList(cmd)
  44     result = {}
  45     for entry in list:
  46         result.update(entry)
  47     return result;
  48
  49 def p4Where(depotPath):
  50     if not depotPath.endswith("/"):
  51         depotPath += "/"
  52     output = p4Cmd("where %s..." % depotPath)
  53     if output["code"] == "error":
  54         return ""
  55     clientPath = ""
  56     if "path" in output:
  57         clientPath = output.get("path")
  58     elif "data" in output:
  59         data = output.get("data")
  60         lastSpace = data.rfind(" ")
  61         clientPath = data[lastSpace + 1:]
  62
  63     if clientPath.endswith("..."):
  64         clientPath = clientPath[:-3]
  65     return clientPath
  66
  67 def die(msg):
  68     sys.stderr.write(msg + "\n")
  69     sys.exit(1)
  70
  71 def currentGitBranch():
  72     return mypopen("git name-rev HEAD").read().split(" ")[1][:-1]
  73
  74 def isValidGitDir(path):
  75     if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
  76         return True;
  77     return False
  78
  79 def parseRevision(ref):
  80     return mypopen("git rev-parse %s" % ref).read()[:-1]
  81
  82 def system(cmd):
  83     if os.system(cmd) != 0:
  84         die("command failed: %s" % cmd)
  85
  86 def extractLogMessageFromGitCommit(commit):
  87     logMessage = ""
  88     foundTitle = False
  89     for log in mypopen("git cat-file commit %s" % commit).readlines():
  90        if not foundTitle:
  91            if len(log) == 1:
  92                foundTitle = True
  93            continue
  94
  95        logMessage += log
  96     return logMessage
  97
  98 def extractDepotPathAndChangeFromGitLog(log):
  99     values = {}
 100     for line in log.split("\n"):
 101         line = line.strip()
 102         if line.startswith("[git-p4:") and line.endswith("]"):
 103             line = line[8:-1].strip()
 104             for assignment in line.split(":"):
 105                 variable = assignment.strip()
 106                 value = ""
 107                 equalPos = assignment.find("=")
 108                 if equalPos != -1:
 109                     variable = assignment[:equalPos].strip()
 110                     value = assignment[equalPos + 1:].strip()
 111                     if value.startswith("\"") and value.endswith("\""):
 112                         value = value[1:-1]
 113                 values[variable] = value
 114
 115     return values.get("depot-path"), values.get("change")
 116
 117 def gitBranchExists(branch):
 118     proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 119     return proc.wait() == 0;
 120
 121 class Command:
 122     def __init__(self):
 123         self.usage = "usage: %prog [options]"
 124         self.needsGit = True
 125
 126 class P4Debug(Command):
 127     def __init__(self):
 128         Command.__init__(self)
 129         self.options = [
 130         ]
 131         self.description = "A tool to debug the output of p4 -G."
 132         self.needsGit = False
 133
 134     def run(self, args):
 135         for output in p4CmdList(" ".join(args)):
 136             print output
 137         return True
 138
 139 class P4RollBack(Command):
 140     def __init__(self):
 141         Command.__init__(self)
 142         self.options = [
 143             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 144             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 145         ]
 146         self.description = "A tool to debug the multi-branch import. Don't use :)"
 147         self.verbose = False
 148         self.rollbackLocalBranches = False
 149
 150     def run(self, args):
 151         if len(args) != 1:
 152             return False
 153         maxChange = int(args[0])
 154
 155         if self.rollbackLocalBranches:
 156             refPrefix = "refs/heads/"
 157             lines = mypopen("git rev-parse --symbolic --branches").readlines()
 158         else:
 159             refPrefix = "refs/remotes/"
 160             lines = mypopen("git rev-parse --symbolic --remotes").readlines()
 161
 162         for line in lines:
 163             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 164                 ref = refPrefix + line[:-1]
 165                 log = extractLogMessageFromGitCommit(ref)
 166                 depotPath, change = extractDepotPathAndChangeFromGitLog(log)
 167                 changed = False
 168
 169                 if len(p4Cmd("changes -m 1 %s...@%s" % (depotPath, maxChange))) == 0:
 170                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 171                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 172                     continue
 173
 174                 while len(change) > 0 and int(change) > maxChange:
 175                     changed = True
 176                     if self.verbose:
 177                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 178                     system("git update-ref %s \"%s^\"" % (ref, ref))
 179                     log = extractLogMessageFromGitCommit(ref)
 180                     depotPath, change = extractDepotPathAndChangeFromGitLog(log)
 181
 182                 if changed:
 183                     print "%s rewound to %s" % (ref, change)
 184
 185         return True
 186
 187 class P4Submit(Command):
 188     def __init__(self):
 189         Command.__init__(self)
 190         self.options = [
 191                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 192                 optparse.make_option("--origin", dest="origin"),
 193                 optparse.make_option("--reset", action="store_true", dest="reset"),
 194                 optparse.make_option("--log-substitutions", dest="substFile"),
 195                 optparse.make_option("--noninteractive", action="store_false"),
 196                 optparse.make_option("--dry-run", action="store_true"),
 197                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 198         ]
 199         self.description = "Submit changes from git to the perforce depot."
 200         self.usage += " [name of git branch to submit into perforce depot]"
 201         self.firstTime = True
 202         self.reset = False
 203         self.interactive = True
 204         self.dryRun = False
 205         self.substFile = ""
 206         self.firstTime = True
 207         self.origin = ""
 208         self.directSubmit = False
 209
 210         self.logSubstitutions = {}
 211         self.logSubstitutions["<enter description here>"] = "%log%"
 212         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 213
 214     def check(self):
 215         if len(p4CmdList("opened ...")) > 0:
 216             die("You have files opened with perforce! Close them before starting the sync.")
 217
 218     def start(self):
 219         if len(self.config) > 0 and not self.reset:
 220             die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile)
 221
 222         commits = []
 223         if self.directSubmit:
 224             commits.append("0")
 225         else:
 226             for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
 227                 commits.append(line[:-1])
 228             commits.reverse()
 229
 230         self.config["commits"] = commits
 231
 232     def prepareLogMessage(self, template, message):
 233         result = ""
 234
 235         for line in template.split("\n"):
 236             if line.startswith("#"):
 237                 result += line + "\n"
 238                 continue
 239
 240             substituted = False
 241             for key in self.logSubstitutions.keys():
 242                 if line.find(key) != -1:
 243                     value = self.logSubstitutions[key]
 244                     value = value.replace("%log%", message)
 245                     if value != "@remove@":
 246                         result += line.replace(key, value) + "\n"
 247                     substituted = True
 248                     break
 249
 250             if not substituted:
 251                 result += line + "\n"
 252
 253         return result
 254
 255     def apply(self, id):
 256         if self.directSubmit:
 257             print "Applying local change in working directory/index"
 258             diff = self.diffStatus
 259         else:
 260             print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read())
 261             diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
 262         filesToAdd = set()
 263         filesToDelete = set()
 264         editedFiles = set()
 265         for line in diff:
 266             modifier = line[0]
 267             path = line[1:].strip()
 268             if modifier == "M":
 269                 system("p4 edit \"%s\"" % path)
 270                 editedFiles.add(path)
 271             elif modifier == "A":
 272                 filesToAdd.add(path)
 273                 if path in filesToDelete:
 274                     filesToDelete.remove(path)
 275             elif modifier == "D":
 276                 filesToDelete.add(path)
 277                 if path in filesToAdd:
 278                     filesToAdd.remove(path)
 279             else:
 280                 die("unknown modifier %s for %s" % (modifier, path))
 281
 282         if self.directSubmit:
 283             diffcmd = "cat \"%s\"" % self.diffFile
 284         else:
 285             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 286         patchcmd = diffcmd + " | git apply "
 287         tryPatchCmd = patchcmd + "--check -"
 288         applyPatchCmd = patchcmd + "--check --apply -"
 289
 290         if os.system(tryPatchCmd) != 0:
 291             print "Unfortunately applying the change failed!"
 292             print "What do you want to do?"
 293             response = "x"
 294             while response != "s" and response != "a" and response != "w":
 295                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ")
 296             if response == "s":
 297                 print "Skipping! Good luck with the next patches..."
 298                 return
 299             elif response == "a":
 300                 os.system(applyPatchCmd)
 301                 if len(filesToAdd) > 0:
 302                     print "You may also want to call p4 add on the following files:"
 303                     print " ".join(filesToAdd)
 304                 if len(filesToDelete):
 305                     print "The following files should be scheduled for deletion with p4 delete:"
 306                     print " ".join(filesToDelete)
 307                 die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
 308             elif response == "w":
 309                 system(diffcmd + " > patch.txt")
 310                 print "Patch saved to patch.txt in %s !" % self.clientPath
 311                 die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue")
 312
 313         system(applyPatchCmd)
 314
 315         for f in filesToAdd:
 316             system("p4 add %s" % f)
 317         for f in filesToDelete:
 318             system("p4 revert %s" % f)
 319             system("p4 delete %s" % f)
 320
 321         logMessage = ""
 322         if not self.directSubmit:
 323             logMessage = extractLogMessageFromGitCommit(id)
 324             logMessage = logMessage.replace("\n", "\n\t")
 325             logMessage = logMessage[:-1]
 326
 327         template = mypopen("p4 change -o").read()
 328
 329         if self.interactive:
 330             submitTemplate = self.prepareLogMessage(template, logMessage)
 331             diff = mypopen("p4 diff -du ...").read()
 332
 333             for newFile in filesToAdd:
 334                 diff += "==== new file ====\n"
 335                 diff += "--- /dev/null\n"
 336                 diff += "+++ %s\n" % newFile
 337                 f = open(newFile, "r")
 338                 for line in f.readlines():
 339                     diff += "+" + line
 340                 f.close()
 341
 342             separatorLine = "######## everything below this line is just the diff #######"
 343             if platform.system() == "Windows":
 344                 separatorLine += "\r"
 345             separatorLine += "\n"
 346
 347             response = "e"
 348             firstIteration = True
 349             while response == "e":
 350                 if not firstIteration:
 351                     response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 352                 firstIteration = False
 353                 if response == "e":
 354                     [handle, fileName] = tempfile.mkstemp()
 355                     tmpFile = os.fdopen(handle, "w+")
 356                     tmpFile.write(submitTemplate + separatorLine + diff)
 357                     tmpFile.close()
 358                     defaultEditor = "vi"
 359                     if platform.system() == "Windows":
 360                         defaultEditor = "notepad"
 361                     editor = os.environ.get("EDITOR", defaultEditor);
 362                     system(editor + " " + fileName)
 363                     tmpFile = open(fileName, "rb")
 364                     message = tmpFile.read()
 365                     tmpFile.close()
 366                     os.remove(fileName)
 367                     submitTemplate = message[:message.index(separatorLine)]
 368
 369             if response == "y" or response == "yes":
 370                if self.dryRun:
 371                    print submitTemplate
 372                    raw_input("Press return to continue...")
 373                else:
 374                    if self.directSubmit:
 375                        print "Submitting to git first"
 376                        os.chdir(self.oldWorkingDirectory)
 377                        pipe = os.popen("git commit -a -F -", "wb")
 378                        pipe.write(submitTemplate)
 379                        pipe.close()
 380                        os.chdir(self.clientPath)
 381
 382                    pipe = os.popen("p4 submit -i", "wb")
 383                    pipe.write(submitTemplate)
 384                    pipe.close()
 385             elif response == "s":
 386                 for f in editedFiles:
 387                     system("p4 revert \"%s\"" % f);
 388                 for f in filesToAdd:
 389                     system("p4 revert \"%s\"" % f);
 390                     system("rm %s" %f)
 391                 for f in filesToDelete:
 392                     system("p4 delete \"%s\"" % f);
 393                 return
 394             else:
 395                 print "Not submitting!"
 396                 self.interactive = False
 397         else:
 398             fileName = "submit.txt"
 399             file = open(fileName, "w+")
 400             file.write(self.prepareLogMessage(template, logMessage))
 401             file.close()
 402             print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
 403
 404     def run(self, args):
 405         global gitdir
 406         # make gitdir absolute so we can cd out into the perforce checkout
 407         gitdir = os.path.abspath(gitdir)
 408         os.environ["GIT_DIR"] = gitdir
 409
 410         if len(args) == 0:
 411             self.master = currentGitBranch()
 412             if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
 413                 die("Detecting current git branch failed!")
 414         elif len(args) == 1:
 415             self.master = args[0]
 416         else:
 417             return False
 418
 419         depotPath = ""
 420         if gitBranchExists("p4"):
 421             [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
 422         if len(depotPath) == 0 and gitBranchExists("origin"):
 423             [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
 424
 425         if len(depotPath) == 0:
 426             print "Internal error: cannot locate perforce depot path from existing branches"
 427             sys.exit(128)
 428
 429         self.clientPath = p4Where(depotPath)
 430
 431         if len(self.clientPath) == 0:
 432             print "Error: Cannot locate perforce checkout of %s in client view" % depotPath
 433             sys.exit(128)
 434
 435         print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath)
 436         self.oldWorkingDirectory = os.getcwd()
 437
 438         if self.directSubmit:
 439             self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines()
 440             if len(self.diffStatus) == 0:
 441                 print "No changes in working directory to submit."
 442                 return True
 443             patch = mypopen("git diff -p --binary --diff-filter=ACMRTUXB HEAD").read()
 444             self.diffFile = gitdir + "/p4-git-diff"
 445             f = open(self.diffFile, "wb")
 446             f.write(patch)
 447             f.close();
 448
 449         os.chdir(self.clientPath)
 450         response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath)
 451         if response == "y" or response == "yes":
 452             system("p4 sync ...")
 453
 454         if len(self.origin) == 0:
 455             if gitBranchExists("p4"):
 456                 self.origin = "p4"
 457             else:
 458                 self.origin = "origin"
 459
 460         if self.reset:
 461             self.firstTime = True
 462
 463         if len(self.substFile) > 0:
 464             for line in open(self.substFile, "r").readlines():
 465                 tokens = line[:-1].split("=")
 466                 self.logSubstitutions[tokens[0]] = tokens[1]
 467
 468         self.check()
 469         self.configFile = gitdir + "/p4-git-sync.cfg"
 470         self.config = shelve.open(self.configFile, writeback=True)
 471
 472         if self.firstTime:
 473             self.start()
 474
 475         commits = self.config.get("commits", [])
 476
 477         while len(commits) > 0:
 478             self.firstTime = False
 479             commit = commits[0]
 480             commits = commits[1:]
 481             self.config["commits"] = commits
 482             self.apply(commit)
 483             if not self.interactive:
 484                 break
 485
 486         self.config.close()
 487
 488         if self.directSubmit:
 489             os.remove(self.diffFile)
 490
 491         if len(commits) == 0:
 492             if self.firstTime:
 493                 print "No changes found to apply between %s and current HEAD" % self.origin
 494             else:
 495                 print "All changes applied!"
 496                 os.chdir(self.oldWorkingDirectory)
 497                 response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ")
 498                 if response == "y" or response == "yes":
 499                     rebase = P4Rebase()
 500                     rebase.run([])
 501             os.remove(self.configFile)
 502
 503         return True
 504
 505 class P4Sync(Command):
 506     def __init__(self):
 507         Command.__init__(self)
 508         self.options = [
 509                 optparse.make_option("--branch", dest="branch"),
 510                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 511                 optparse.make_option("--changesfile", dest="changesFile"),
 512                 optparse.make_option("--silent", dest="silent", action="store_true"),
 513                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 514                 optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"),
 515                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 516                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"),
 517                 optparse.make_option("--max-changes", dest="maxChanges")
 518         ]
 519         self.description = """Imports from Perforce into a git repository.\n
 520     example:
 521     //depot/my/project/ -- to import the current head
 522     //depot/my/project/@all -- to import everything
 523     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 524
 525     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 526
 527         self.usage += " //depot/path[@revRange]"
 528
 529         self.silent = False
 530         self.createdBranches = Set()
 531         self.committedChanges = Set()
 532         self.branch = ""
 533         self.detectBranches = False
 534         self.detectLabels = False
 535         self.changesFile = ""
 536         self.syncWithOrigin = False
 537         self.verbose = False
 538         self.importIntoRemotes = True
 539         self.maxChanges = ""
 540
 541     def p4File(self, depotPath):
 542         return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 543
 544     def extractFilesFromCommit(self, commit):
 545         files = []
 546         fnum = 0
 547         while commit.has_key("depotFile%s" % fnum):
 548             path =  commit["depotFile%s" % fnum]
 549             if not path.startswith(self.depotPath):
 550     #            if not self.silent:
 551     #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change)
 552                 fnum = fnum + 1
 553                 continue
 554
 555             file = {}
 556             file["path"] = path
 557             file["rev"] = commit["rev%s" % fnum]
 558             file["action"] = commit["action%s" % fnum]
 559             file["type"] = commit["type%s" % fnum]
 560             files.append(file)
 561             fnum = fnum + 1
 562         return files
 563
 564     def splitFilesIntoBranches(self, commit):
 565         branches = {}
 566
 567         fnum = 0
 568         while commit.has_key("depotFile%s" % fnum):
 569             path =  commit["depotFile%s" % fnum]
 570             if not path.startswith(self.depotPath):
 571     #            if not self.silent:
 572     #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change)
 573                 fnum = fnum + 1
 574                 continue
 575
 576             file = {}
 577             file["path"] = path
 578             file["rev"] = commit["rev%s" % fnum]
 579             file["action"] = commit["action%s" % fnum]
 580             file["type"] = commit["type%s" % fnum]
 581             fnum = fnum + 1
 582
 583             relPath = path[len(self.depotPath):]
 584
 585             for branch in self.knownBranches.keys():
 586                 if relPath.startswith(branch + "/"): # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 587                     if branch not in branches:
 588                         branches[branch] = []
 589                     branches[branch].append(file)
 590
 591         return branches
 592
 593     def commit(self, details, files, branch, branchPrefix, parent = ""):
 594         epoch = details["time"]
 595         author = details["user"]
 596
 597         if self.verbose:
 598             print "commit into %s" % branch
 599
 600         self.gitStream.write("commit %s\n" % branch)
 601     #    gitStream.write("mark :%s\n" % details["change"])
 602         self.committedChanges.add(int(details["change"]))
 603         committer = ""
 604         if author not in self.users:
 605             self.getUserMapFromPerforceServer()
 606         if author in self.users:
 607             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 608         else:
 609             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 610
 611         self.gitStream.write("committer %s\n" % committer)
 612
 613         self.gitStream.write("data <<EOT\n")
 614         self.gitStream.write(details["desc"])
 615         self.gitStream.write("\n[git-p4: depot-path = \"%s\": change = %s]\n" % (branchPrefix, details["change"]))
 616         self.gitStream.write("EOT\n\n")
 617
 618         if len(parent) > 0:
 619             if self.verbose:
 620                 print "parent %s" % parent
 621             self.gitStream.write("from %s\n" % parent)
 622
 623         for file in files:
 624             path = file["path"]
 625             if not path.startswith(branchPrefix):
 626     #            if not silent:
 627     #                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 628                 continue
 629             rev = file["rev"]
 630             depotPath = path + "#" + rev
 631             relPath = path[len(branchPrefix):]
 632             action = file["action"]
 633
 634             if file["type"] == "apple":
 635                 print "\nfile %s is a strange apple file that forks. Ignoring!" % path
 636                 continue
 637
 638             if action == "delete":
 639                 self.gitStream.write("D %s\n" % relPath)
 640             else:
 641                 mode = 644
 642                 if file["type"].startswith("x"):
 643                     mode = 755
 644
 645                 data = self.p4File(depotPath)
 646
 647                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 648                 self.gitStream.write("data %s\n" % len(data))
 649                 self.gitStream.write(data)
 650                 self.gitStream.write("\n")
 651
 652         self.gitStream.write("\n")
 653
 654         change = int(details["change"])
 655
 656         if self.labels.has_key(change):
 657             label = self.labels[change]
 658             labelDetails = label[0]
 659             labelRevisions = label[1]
 660             if self.verbose:
 661                 print "Change %s is labelled %s" % (change, labelDetails)
 662
 663             files = p4CmdList("files %s...@%s" % (branchPrefix, change))
 664
 665             if len(files) == len(labelRevisions):
 666
 667                 cleanedFiles = {}
 668                 for info in files:
 669                     if info["action"] == "delete":
 670                         continue
 671                     cleanedFiles[info["depotFile"]] = info["rev"]
 672
 673                 if cleanedFiles == labelRevisions:
 674                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
 675                     self.gitStream.write("from %s\n" % branch)
 676
 677                     owner = labelDetails["Owner"]
 678                     tagger = ""
 679                     if author in self.users:
 680                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
 681                     else:
 682                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
 683                     self.gitStream.write("tagger %s\n" % tagger)
 684                     self.gitStream.write("data <<EOT\n")
 685                     self.gitStream.write(labelDetails["Description"])
 686                     self.gitStream.write("EOT\n\n")
 687
 688                 else:
 689                     if not self.silent:
 690                         print "Tag %s does not match with change %s: files do not match." % (labelDetails["label"], change)
 691
 692             else:
 693                 if not self.silent:
 694                     print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)
 695
 696     def getUserMapFromPerforceServer(self):
 697         self.users = {}
 698
 699         for output in p4CmdList("users"):
 700             if not output.has_key("User"):
 701                 continue
 702             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 703
 704         cache = open(gitdir + "/p4-usercache.txt", "wb")
 705         for user in self.users.keys():
 706             cache.write("%s\t%s\n" % (user, self.users[user]))
 707         cache.close();
 708
 709     def loadUserMapFromCache(self):
 710         self.users = {}
 711         try:
 712             cache = open(gitdir + "/p4-usercache.txt", "rb")
 713             lines = cache.readlines()
 714             cache.close()
 715             for line in lines:
 716                 entry = line[:-1].split("\t")
 717                 self.users[entry[0]] = entry[1]
 718         except IOError:
 719             self.getUserMapFromPerforceServer()
 720
 721     def getLabels(self):
 722         self.labels = {}
 723
 724         l = p4CmdList("labels %s..." % self.depotPath)
 725         if len(l) > 0 and not self.silent:
 726             print "Finding files belonging to labels in %s" % self.depotPath
 727
 728         for output in l:
 729             label = output["label"]
 730             revisions = {}
 731             newestChange = 0
 732             if self.verbose:
 733                 print "Querying files for label %s" % label
 734             for file in p4CmdList("files %s...@%s" % (self.depotPath, label)):
 735                 revisions[file["depotFile"]] = file["rev"]
 736                 change = int(file["change"])
 737                 if change > newestChange:
 738                     newestChange = change
 739
 740             self.labels[newestChange] = [output, revisions]
 741
 742         if self.verbose:
 743             print "Label changes: %s" % self.labels.keys()
 744
 745     def getBranchMapping(self):
 746         self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:]
 747
 748         for info in p4CmdList("branches"):
 749             details = p4Cmd("branch -o %s" % info["branch"])
 750             viewIdx = 0
 751             while details.has_key("View%s" % viewIdx):
 752                 paths = details["View%s" % viewIdx].split(" ")
 753                 viewIdx = viewIdx + 1
 754                 # require standard //depot/foo/... //depot/bar/... mapping
 755                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
 756                     continue
 757                 source = paths[0]
 758                 destination = paths[1]
 759                 if source.startswith(self.depotPath) and destination.startswith(self.depotPath):
 760                     source = source[len(self.depotPath):-4]
 761                     destination = destination[len(self.depotPath):-4]
 762                     if destination not in self.knownBranches:
 763                         self.knownBranches[destination] = source
 764                     if source not in self.knownBranches:
 765                         self.knownBranches[source] = source
 766
 767     def listExistingP4GitBranches(self):
 768         self.p4BranchesInGit = []
 769
 770         cmdline = "git rev-parse --symbolic "
 771         if self.importIntoRemotes:
 772             cmdline += " --remotes"
 773         else:
 774             cmdline += " --branches"
 775
 776         for line in mypopen(cmdline).readlines():
 777             if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"):
 778                 continue
 779             if self.importIntoRemotes:
 780                 # strip off p4
 781                 branch = line[3:-1]
 782             else:
 783                 branch = line[:-1]
 784             self.p4BranchesInGit.append(branch)
 785             self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1])
 786
 787     def run(self, args):
 788         self.depotPath = ""
 789         self.changeRange = ""
 790         self.initialParent = ""
 791         self.previousDepotPath = ""
 792         # map from branch depot path to parent branch
 793         self.knownBranches = {}
 794         self.initialParents = {}
 795
 796         if self.importIntoRemotes:
 797             self.refPrefix = "refs/remotes/p4/"
 798         else:
 799             self.refPrefix = "refs/heads/"
 800
 801         createP4HeadRef = False;
 802
 803         if self.syncWithOrigin and gitBranchExists("origin") and gitBranchExists(self.refPrefix + "master") and not self.detectBranches and self.importIntoRemotes:
 804             ### needs to be ported to multi branch import
 805
 806             print "Syncing with origin first as requested by calling git fetch origin"
 807             system("git fetch origin")
 808             [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
 809             [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
 810             if len(originPreviousDepotPath) > 0 and len(originP4Change) > 0 and len(p4Change) > 0:
 811                 if originPreviousDepotPath == p4PreviousDepotPath:
 812                     originP4Change = int(originP4Change)
 813                     p4Change = int(p4Change)
 814                     if originP4Change > p4Change:
 815                         print "origin (%s) is newer than p4 (%s). Updating p4 branch from origin." % (originP4Change, p4Change)
 816                         system("git update-ref " + self.refPrefix + "master origin");
 817                 else:
 818                     print "Cannot sync with origin. It was imported from %s while remotes/p4 was imported from %s" % (originPreviousDepotPath, p4PreviousDepotPath)
 819
 820         if len(self.branch) == 0:
 821             self.branch = self.refPrefix + "master"
 822             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
 823                 system("git update-ref %s refs/heads/p4" % self.branch)
 824                 system("git branch -D p4");
 825             # create it /after/ importing, when master exists
 826             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes:
 827                 createP4HeadRef = True
 828
 829         # this needs to be called after the conversion from heads/p4 to remotes/p4/master
 830         self.listExistingP4GitBranches()
 831         if len(self.p4BranchesInGit) > 1 and not self.silent:
 832             print "Importing from/into multiple branches"
 833             self.detectBranches = True
 834
 835         if len(args) == 0:
 836             if not gitBranchExists(self.branch) and gitBranchExists("origin") and not self.detectBranches:
 837                 ### needs to be ported to multi branch import
 838                 if not self.silent:
 839                     print "Creating %s branch in git repository based on origin" % self.branch
 840                 branch = self.branch
 841                 if not branch.startswith("refs"):
 842                     branch = "refs/heads/" + branch
 843                 system("git update-ref %s origin" % branch)
 844
 845             if self.verbose:
 846                 print "branches: %s" % self.p4BranchesInGit
 847
 848             p4Change = 0
 849             for branch in self.p4BranchesInGit:
 850                 depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.refPrefix + branch))
 851
 852                 if self.verbose:
 853                     print "path %s change %s" % (depotPath, change)
 854
 855                 if len(depotPath) > 0 and len(change) > 0:
 856                     change = int(change) + 1
 857                     p4Change = max(p4Change, change)
 858
 859                     if len(self.previousDepotPath) == 0:
 860                         self.previousDepotPath = depotPath
 861                     else:
 862                         i = 0
 863                         l = min(len(self.previousDepotPath), len(depotPath))
 864                         while i < l and self.previousDepotPath[i] == depotPath[i]:
 865                             i = i + 1
 866                         self.previousDepotPath = self.previousDepotPath[:i]
 867
 868             if p4Change > 0:
 869                 self.depotPath = self.previousDepotPath
 870                 self.changeRange = "@%s,#head" % p4Change
 871                 self.initialParent = parseRevision(self.branch)
 872                 if not self.silent and not self.detectBranches:
 873                     print "Performing incremental import into %s git branch" % self.branch
 874
 875         if not self.branch.startswith("refs/"):
 876             self.branch = "refs/heads/" + self.branch
 877
 878         if len(self.depotPath) != 0:
 879             self.depotPath = self.depotPath[:-1]
 880
 881         if len(args) == 0 and len(self.depotPath) != 0:
 882             if not self.silent:
 883                 print "Depot path: %s" % self.depotPath
 884         elif len(args) != 1:
 885             return False
 886         else:
 887             if len(self.depotPath) != 0 and self.depotPath != args[0]:
 888                 print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0])
 889                 sys.exit(1)
 890             self.depotPath = args[0]
 891
 892         self.revision = ""
 893         self.users = {}
 894
 895         if self.depotPath.find("@") != -1:
 896             atIdx = self.depotPath.index("@")
 897             self.changeRange = self.depotPath[atIdx:]
 898             if self.changeRange == "@all":
 899                 self.changeRange = ""
 900             elif self.changeRange.find(",") == -1:
 901                 self.revision = self.changeRange
 902                 self.changeRange = ""
 903             self.depotPath = self.depotPath[0:atIdx]
 904         elif self.depotPath.find("#") != -1:
 905             hashIdx = self.depotPath.index("#")
 906             self.revision = self.depotPath[hashIdx:]
 907             self.depotPath = self.depotPath[0:hashIdx]
 908         elif len(self.previousDepotPath) == 0:
 909             self.revision = "#head"
 910
 911         if self.depotPath.endswith("..."):
 912             self.depotPath = self.depotPath[:-3]
 913
 914         if not self.depotPath.endswith("/"):
 915             self.depotPath += "/"
 916
 917         self.loadUserMapFromCache()
 918         self.labels = {}
 919         if self.detectLabels:
 920             self.getLabels();
 921
 922         if self.detectBranches:
 923             self.getBranchMapping();
 924             if self.verbose:
 925                 print "p4-git branches: %s" % self.p4BranchesInGit
 926                 print "initial parents: %s" % self.initialParents
 927             for b in self.p4BranchesInGit:
 928                 if b != "master":
 929                     b = b[len(self.projectName):]
 930                 self.createdBranches.add(b)
 931
 932         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
 933
 934         importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
 935         self.gitOutput = importProcess.stdout
 936         self.gitStream = importProcess.stdin
 937         self.gitError = importProcess.stderr
 938
 939         if len(self.revision) > 0:
 940             print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision)
 941
 942             details = { "user" : "git perforce import user", "time" : int(time.time()) }
 943             details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision)
 944             details["change"] = self.revision
 945             newestRevision = 0
 946
 947             fileCnt = 0
 948             for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)):
 949                 change = int(info["change"])
 950                 if change > newestRevision:
 951                     newestRevision = change
 952
 953                 if info["action"] == "delete":
 954                     # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
 955                     #fileCnt = fileCnt + 1
 956                     continue
 957
 958                 for prop in [ "depotFile", "rev", "action", "type" ]:
 959                     details["%s%s" % (prop, fileCnt)] = info[prop]
 960
 961                 fileCnt = fileCnt + 1
 962
 963             details["change"] = newestRevision
 964
 965             try:
 966                 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath)
 967             except IOError:
 968                 print "IO error with git fast-import. Is your git version recent enough?"
 969                 print self.gitError.read()
 970
 971         else:
 972             changes = []
 973
 974             if len(self.changesFile) > 0:
 975                 output = open(self.changesFile).readlines()
 976                 changeSet = Set()
 977                 for line in output:
 978                     changeSet.add(int(line))
 979
 980                 for change in changeSet:
 981                     changes.append(change)
 982
 983                 changes.sort()
 984             else:
 985                 if self.verbose:
 986                     print "Getting p4 changes for %s...%s" % (self.depotPath, self.changeRange)
 987                 output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines()
 988
 989                 for line in output:
 990                     changeNum = line.split(" ")[1]
 991                     changes.append(changeNum)
 992
 993                 changes.reverse()
 994
 995                 if len(self.maxChanges) > 0:
 996                     changes = changes[0:min(int(self.maxChanges), len(changes))]
 997
 998             if len(changes) == 0:
 999                 if not self.silent:
1000                     print "No changes to import!"
1001                 return True
1002
1003             self.updatedBranches = set()
1004
1005             cnt = 1
1006             for change in changes:
1007                 description = p4Cmd("describe %s" % change)
1008
1009                 if not self.silent:
1010                     sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1011                     sys.stdout.flush()
1012                 cnt = cnt + 1
1013
1014                 try:
1015                     if self.detectBranches:
1016                         branches = self.splitFilesIntoBranches(description)
1017                         for branch in branches.keys():
1018                             branchPrefix = self.depotPath + branch + "/"
1019
1020                             parent = ""
1021
1022                             filesForCommit = branches[branch]
1023
1024                             if self.verbose:
1025                                 print "branch is %s" % branch
1026
1027                             self.updatedBranches.add(branch)
1028
1029                             if branch not in self.createdBranches:
1030                                 self.createdBranches.add(branch)
1031                                 parent = self.knownBranches[branch]
1032                                 if parent == branch:
1033                                     parent = ""
1034                                 elif self.verbose:
1035                                     print "parent determined through known branches: %s" % parent
1036
1037                             # main branch? use master
1038                             if branch == "main":
1039                                 branch = "master"
1040                             else:
1041                                 branch = self.projectName + branch
1042
1043                             if parent == "main":
1044                                 parent = "master"
1045                             elif len(parent) > 0:
1046                                 parent = self.projectName + parent
1047
1048                             branch = self.refPrefix + branch
1049                             if len(parent) > 0:
1050                                 parent = self.refPrefix + parent
1051
1052                             if self.verbose:
1053                                 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1054
1055                             if len(parent) == 0 and branch in self.initialParents:
1056                                 parent = self.initialParents[branch]
1057                                 del self.initialParents[branch]
1058
1059                             self.commit(description, filesForCommit, branch, branchPrefix, parent)
1060                     else:
1061                         files = self.extractFilesFromCommit(description)
1062                         self.commit(description, files, self.branch, self.depotPath, self.initialParent)
1063                         self.initialParent = ""
1064                 except IOError:
1065                     print self.gitError.read()
1066                     sys.exit(1)
1067
1068             if not self.silent:
1069                 print ""
1070                 if len(self.updatedBranches) > 0:
1071                     sys.stdout.write("Updated branches: ")
1072                     for b in self.updatedBranches:
1073                         sys.stdout.write("%s " % b)
1074                     sys.stdout.write("\n")
1075
1076
1077         self.gitStream.close()
1078         if importProcess.wait() != 0:
1079             die("fast-import failed: %s" % self.gitError.read())
1080         self.gitOutput.close()
1081         self.gitError.close()
1082
1083         if createP4HeadRef:
1084             system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1085
1086         return True
1087
1088 class P4Rebase(Command):
1089     def __init__(self):
1090         Command.__init__(self)
1091         self.options = [ optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ]
1092         self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it"
1093         self.syncWithOrigin = False
1094
1095     def run(self, args):
1096         sync = P4Sync()
1097         sync.syncWithOrigin = self.syncWithOrigin
1098         sync.run([])
1099         print "Rebasing the current branch"
1100         oldHead = mypopen("git rev-parse HEAD").read()[:-1]
1101         system("git rebase p4")
1102         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1103         return True
1104
1105 class P4Clone(P4Sync):
1106     def __init__(self):
1107         P4Sync.__init__(self)
1108         self.description = "Creates a new git repository and imports from Perforce into it"
1109         self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]"
1110         self.needsGit = False
1111
1112     def run(self, args):
1113         global gitdir
1114
1115         if len(args) < 1:
1116             return False
1117         depotPath = args[0]
1118         dir = ""
1119         if len(args) == 2:
1120             dir = args[1]
1121         elif len(args) > 2:
1122             return False
1123
1124         if not depotPath.startswith("//"):
1125             return False
1126
1127         if len(dir) == 0:
1128             dir = depotPath
1129             atPos = dir.rfind("@")
1130             if atPos != -1:
1131                 dir = dir[0:atPos]
1132             hashPos = dir.rfind("#")
1133             if hashPos != -1:
1134                 dir = dir[0:hashPos]
1135
1136             if dir.endswith("..."):
1137                 dir = dir[:-3]
1138
1139             if dir.endswith("/"):
1140                dir = dir[:-1]
1141
1142             slashPos = dir.rfind("/")
1143             if slashPos != -1:
1144                 dir = dir[slashPos + 1:]
1145
1146         print "Importing from %s into %s" % (depotPath, dir)
1147         os.makedirs(dir)
1148         os.chdir(dir)
1149         system("git init")
1150         gitdir = os.getcwd() + "/.git"
1151         if not P4Sync.run(self, [depotPath]):
1152             return False
1153         if self.branch != "master":
1154             if gitBranchExists("refs/remotes/p4/master"):
1155                 system("git branch master refs/remotes/p4/master")
1156                 system("git checkout -f")
1157             else:
1158                 print "Could not detect main branch. No checkout/master branch created."
1159         return True
1160
1161 class HelpFormatter(optparse.IndentedHelpFormatter):
1162     def __init__(self):
1163         optparse.IndentedHelpFormatter.__init__(self)
1164
1165     def format_description(self, description):
1166         if description:
1167             return description + "\n"
1168         else:
1169             return ""
1170
1171 def printUsage(commands):
1172     print "usage: %s <command> [options]" % sys.argv[0]
1173     print ""
1174     print "valid commands: %s" % ", ".join(commands)
1175     print ""
1176     print "Try %s <command> --help for command specific help." % sys.argv[0]
1177     print ""
1178
1179 commands = {
1180     "debug" : P4Debug(),
1181     "submit" : P4Submit(),
1182     "sync" : P4Sync(),
1183     "rebase" : P4Rebase(),
1184     "clone" : P4Clone(),
1185     "rollback" : P4RollBack()
1186 }
1187
1188 if len(sys.argv[1:]) == 0:
1189     printUsage(commands.keys())
1190     sys.exit(2)
1191
1192 cmd = ""
1193 cmdName = sys.argv[1]
1194 try:
1195     cmd = commands[cmdName]
1196 except KeyError:
1197     print "unknown command %s" % cmdName
1198     print ""
1199     printUsage(commands.keys())
1200     sys.exit(2)
1201
1202 options = cmd.options
1203 cmd.gitdir = gitdir
1204
1205 args = sys.argv[2:]
1206
1207 if len(options) > 0:
1208     options.append(optparse.make_option("--git-dir", dest="gitdir"))
1209
1210     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1211                                    options,
1212                                    description = cmd.description,
1213                                    formatter = HelpFormatter())
1214
1215     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1216
1217 if cmd.needsGit:
1218     gitdir = cmd.gitdir
1219     if len(gitdir) == 0:
1220         gitdir = ".git"
1221         if not isValidGitDir(gitdir):
1222             gitdir = mypopen("git rev-parse --git-dir").read()[:-1]
1223             if os.path.exists(gitdir):
1224                 cdup = mypopen("git rev-parse --show-cdup").read()[:-1];
1225                 if len(cdup) > 0:
1226                     os.chdir(cdup);
1227
1228     if not isValidGitDir(gitdir):
1229         if isValidGitDir(gitdir + "/.git"):
1230             gitdir += "/.git"
1231         else:
1232             die("fatal: cannot locate git repository at %s" % gitdir)
1233
1234     os.environ["GIT_DIR"] = gitdir
1235
1236 if not cmd.run(args):
1237     parser.print_help()
1238