contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                 optparse.make_option("--origin", dest="origin"),
 470                 optparse.make_option("--reset", action="store_true", dest="reset"),
 471                 optparse.make_option("--log-substitutions", dest="substFile"),
 472                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 473                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 474         ]
 475         self.description = "Submit changes from git to the perforce depot."
 476         self.usage += " [name of git branch to submit into perforce depot]"
 477         self.firstTime = True
 478         self.reset = False
 479         self.interactive = True
 480         self.substFile = ""
 481         self.firstTime = True
 482         self.origin = ""
 483         self.directSubmit = False
 484         self.detectRename = False
 485         self.verbose = False
 486         self.isWindows = (platform.system() == "Windows")
 487
 488         self.logSubstitutions = {}
 489         self.logSubstitutions["<enter description here>"] = "%log%"
 490         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 491
 492     def check(self):
 493         if len(p4CmdList("opened ...")) > 0:
 494             die("You have files opened with perforce! Close them before starting the sync.")
 495
 496     def start(self):
 497         if len(self.config) > 0 and not self.reset:
 498             die("Cannot start sync. Previous sync config found at %s\n"
 499                 "If you want to start submitting again from scratch "
 500                 "maybe you want to call git-p4 submit --reset" % self.configFile)
 501
 502         commits = []
 503         if self.directSubmit:
 504             commits.append("0")
 505         else:
 506             for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 507                 commits.append(line.strip())
 508             commits.reverse()
 509
 510         self.config["commits"] = commits
 511
 512     def prepareLogMessage(self, template, message):
 513         result = ""
 514
 515         for line in template.split("\n"):
 516             if line.startswith("#"):
 517                 result += line + "\n"
 518                 continue
 519
 520             substituted = False
 521             for key in self.logSubstitutions.keys():
 522                 if line.find(key) != -1:
 523                     value = self.logSubstitutions[key]
 524                     value = value.replace("%log%", message)
 525                     if value != "@remove@":
 526                         result += line.replace(key, value) + "\n"
 527                     substituted = True
 528                     break
 529
 530             if not substituted:
 531                 result += line + "\n"
 532
 533         return result
 534
 535     def prepareSubmitTemplate(self):
 536         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 537         template = ""
 538         inFilesSection = False
 539         for line in read_pipe_lines("p4 change -o"):
 540             if inFilesSection:
 541                 if line.startswith("\t"):
 542                     # path starts and ends with a tab
 543                     path = line[1:]
 544                     lastTab = path.rfind("\t")
 545                     if lastTab != -1:
 546                         path = path[:lastTab]
 547                         if not path.startswith(self.depotPath):
 548                             continue
 549                 else:
 550                     inFilesSection = False
 551             else:
 552                 if line.startswith("Files:"):
 553                     inFilesSection = True
 554
 555             template += line
 556
 557         return template
 558
 559     def applyCommit(self, id):
 560         if self.directSubmit:
 561             print "Applying local change in working directory/index"
 562             diff = self.diffStatus
 563         else:
 564             print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 565             diffOpts = ("", "-M")[self.detectRename]
 566             diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 567         filesToAdd = set()
 568         filesToDelete = set()
 569         editedFiles = set()
 570         filesToChangeExecBit = {}
 571         for line in diff:
 572             diff = parseDiffTreeEntry(line)
 573             modifier = diff['status']
 574             path = diff['src']
 575             if modifier == "M":
 576                 system("p4 edit \"%s\"" % path)
 577                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 578                     filesToChangeExecBit[path] = diff['dst_mode']
 579                 editedFiles.add(path)
 580             elif modifier == "A":
 581                 filesToAdd.add(path)
 582                 filesToChangeExecBit[path] = diff['dst_mode']
 583                 if path in filesToDelete:
 584                     filesToDelete.remove(path)
 585             elif modifier == "D":
 586                 filesToDelete.add(path)
 587                 if path in filesToAdd:
 588                     filesToAdd.remove(path)
 589             elif modifier == "R":
 590                 src, dest = diff['src'], diff['dst']
 591                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 592                 system("p4 edit \"%s\"" % (dest))
 593                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 594                     filesToChangeExecBit[dest] = diff['dst_mode']
 595                 os.unlink(dest)
 596                 editedFiles.add(dest)
 597                 filesToDelete.add(src)
 598             else:
 599                 die("unknown modifier %s for %s" % (modifier, path))
 600
 601         if self.directSubmit:
 602             diffcmd = "cat \"%s\"" % self.diffFile
 603         else:
 604             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 605         patchcmd = diffcmd + " | git apply "
 606         tryPatchCmd = patchcmd + "--check -"
 607         applyPatchCmd = patchcmd + "--check --apply -"
 608
 609         if os.system(tryPatchCmd) != 0:
 610             print "Unfortunately applying the change failed!"
 611             print "What do you want to do?"
 612             response = "x"
 613             while response != "s" and response != "a" and response != "w":
 614                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 615                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 616             if response == "s":
 617                 print "Skipping! Good luck with the next patches..."
 618                 for f in editedFiles:
 619                     system("p4 revert \"%s\"" % f);
 620                 for f in filesToAdd:
 621                     system("rm %s" %f)
 622                 return
 623             elif response == "a":
 624                 os.system(applyPatchCmd)
 625                 if len(filesToAdd) > 0:
 626                     print "You may also want to call p4 add on the following files:"
 627                     print " ".join(filesToAdd)
 628                 if len(filesToDelete):
 629                     print "The following files should be scheduled for deletion with p4 delete:"
 630                     print " ".join(filesToDelete)
 631                 die("Please resolve and submit the conflict manually and "
 632                     + "continue afterwards with git-p4 submit --continue")
 633             elif response == "w":
 634                 system(diffcmd + " > patch.txt")
 635                 print "Patch saved to patch.txt in %s !" % self.clientPath
 636                 die("Please resolve and submit the conflict manually and "
 637                     "continue afterwards with git-p4 submit --continue")
 638
 639         system(applyPatchCmd)
 640
 641         for f in filesToAdd:
 642             system("p4 add \"%s\"" % f)
 643         for f in filesToDelete:
 644             system("p4 revert \"%s\"" % f)
 645             system("p4 delete \"%s\"" % f)
 646
 647         # Set/clear executable bits
 648         for f in filesToChangeExecBit.keys():
 649             mode = filesToChangeExecBit[f]
 650             setP4ExecBit(f, mode)
 651
 652         logMessage = ""
 653         if not self.directSubmit:
 654             logMessage = extractLogMessageFromGitCommit(id)
 655             logMessage = logMessage.replace("\n", "\n\t")
 656             if self.isWindows:
 657                 logMessage = logMessage.replace("\n", "\r\n")
 658             logMessage = logMessage.strip()
 659
 660         template = self.prepareSubmitTemplate()
 661
 662         if self.interactive:
 663             submitTemplate = self.prepareLogMessage(template, logMessage)
 664             diff = read_pipe("p4 diff -du ...")
 665
 666             for newFile in filesToAdd:
 667                 diff += "==== new file ====\n"
 668                 diff += "--- /dev/null\n"
 669                 diff += "+++ %s\n" % newFile
 670                 f = open(newFile, "r")
 671                 for line in f.readlines():
 672                     diff += "+" + line
 673                 f.close()
 674
 675             separatorLine = "######## everything below this line is just the diff #######"
 676             if platform.system() == "Windows":
 677                 separatorLine += "\r"
 678             separatorLine += "\n"
 679
 680             [handle, fileName] = tempfile.mkstemp()
 681             tmpFile = os.fdopen(handle, "w+")
 682             tmpFile.write(submitTemplate + separatorLine + diff)
 683             tmpFile.close()
 684             defaultEditor = "vi"
 685             if platform.system() == "Windows":
 686                 defaultEditor = "notepad"
 687             editor = os.environ.get("EDITOR", defaultEditor);
 688             system(editor + " " + fileName)
 689             tmpFile = open(fileName, "rb")
 690             message = tmpFile.read()
 691             tmpFile.close()
 692             os.remove(fileName)
 693             submitTemplate = message[:message.index(separatorLine)]
 694             if self.isWindows:
 695                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 696
 697             if self.directSubmit:
 698                 print "Submitting to git first"
 699                 os.chdir(self.oldWorkingDirectory)
 700                 write_pipe("git commit -a -F -", submitTemplate)
 701                 os.chdir(self.clientPath)
 702
 703             write_pipe("p4 submit -i", submitTemplate)
 704         else:
 705             fileName = "submit.txt"
 706             file = open(fileName, "w+")
 707             file.write(self.prepareLogMessage(template, logMessage))
 708             file.close()
 709             print ("Perforce submit template written as %s. "
 710                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 711                    % (fileName, fileName))
 712
 713     def run(self, args):
 714         if len(args) == 0:
 715             self.master = currentGitBranch()
 716             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 717                 die("Detecting current git branch failed!")
 718         elif len(args) == 1:
 719             self.master = args[0]
 720         else:
 721             return False
 722
 723         [upstream, settings] = findUpstreamBranchPoint()
 724         self.depotPath = settings['depot-paths'][0]
 725         if len(self.origin) == 0:
 726             self.origin = upstream
 727
 728         if self.verbose:
 729             print "Origin branch is " + self.origin
 730
 731         if len(self.depotPath) == 0:
 732             print "Internal error: cannot locate perforce depot path from existing branches"
 733             sys.exit(128)
 734
 735         self.clientPath = p4Where(self.depotPath)
 736
 737         if len(self.clientPath) == 0:
 738             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 739             sys.exit(128)
 740
 741         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 742         self.oldWorkingDirectory = os.getcwd()
 743
 744         if self.directSubmit:
 745             self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 746             if len(self.diffStatus) == 0:
 747                 print "No changes in working directory to submit."
 748                 return True
 749             patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 750             self.diffFile = self.gitdir + "/p4-git-diff"
 751             f = open(self.diffFile, "wb")
 752             f.write(patch)
 753             f.close();
 754
 755         os.chdir(self.clientPath)
 756         print "Syncronizing p4 checkout..."
 757         system("p4 sync ...")
 758
 759         if self.reset:
 760             self.firstTime = True
 761
 762         if len(self.substFile) > 0:
 763             for line in open(self.substFile, "r").readlines():
 764                 tokens = line.strip().split("=")
 765                 self.logSubstitutions[tokens[0]] = tokens[1]
 766
 767         self.check()
 768         self.configFile = self.gitdir + "/p4-git-sync.cfg"
 769         self.config = shelve.open(self.configFile, writeback=True)
 770
 771         if self.firstTime:
 772             self.start()
 773
 774         commits = self.config.get("commits", [])
 775
 776         while len(commits) > 0:
 777             self.firstTime = False
 778             commit = commits[0]
 779             commits = commits[1:]
 780             self.config["commits"] = commits
 781             self.applyCommit(commit)
 782             if not self.interactive:
 783                 break
 784
 785         self.config.close()
 786
 787         if self.directSubmit:
 788             os.remove(self.diffFile)
 789
 790         if len(commits) == 0:
 791             if self.firstTime:
 792                 print "No changes found to apply between %s and current HEAD" % self.origin
 793             else:
 794                 print "All changes applied!"
 795                 os.chdir(self.oldWorkingDirectory)
 796
 797                 sync = P4Sync()
 798                 sync.run([])
 799
 800                 rebase = P4Rebase()
 801                 rebase.rebase()
 802             os.remove(self.configFile)
 803
 804         return True
 805
 806 class P4Sync(Command):
 807     def __init__(self):
 808         Command.__init__(self)
 809         self.options = [
 810                 optparse.make_option("--branch", dest="branch"),
 811                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 812                 optparse.make_option("--changesfile", dest="changesFile"),
 813                 optparse.make_option("--silent", dest="silent", action="store_true"),
 814                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 815                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 816                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 817                                      help="Import into refs/heads/ , not refs/remotes"),
 818                 optparse.make_option("--max-changes", dest="maxChanges"),
 819                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 820                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 821         ]
 822         self.description = """Imports from Perforce into a git repository.\n
 823     example:
 824     //depot/my/project/ -- to import the current head
 825     //depot/my/project/@all -- to import everything
 826     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 827
 828     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 829
 830         self.usage += " //depot/path[@revRange]"
 831         self.silent = False
 832         self.createdBranches = Set()
 833         self.committedChanges = Set()
 834         self.branch = ""
 835         self.detectBranches = False
 836         self.detectLabels = False
 837         self.changesFile = ""
 838         self.syncWithOrigin = True
 839         self.verbose = False
 840         self.importIntoRemotes = True
 841         self.maxChanges = ""
 842         self.isWindows = (platform.system() == "Windows")
 843         self.keepRepoPath = False
 844         self.depotPaths = None
 845         self.p4BranchesInGit = []
 846         self.cloneExclude = []
 847
 848         if gitConfig("git-p4.syncFromOrigin") == "false":
 849             self.syncWithOrigin = False
 850
 851     def extractFilesFromCommit(self, commit):
 852         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 853                              for path in self.cloneExclude]
 854         files = []
 855         fnum = 0
 856         while commit.has_key("depotFile%s" % fnum):
 857             path =  commit["depotFile%s" % fnum]
 858
 859             if [p for p in self.cloneExclude
 860                 if path.startswith (p)]:
 861                 found = False
 862             else:
 863                 found = [p for p in self.depotPaths
 864                          if path.startswith (p)]
 865             if not found:
 866                 fnum = fnum + 1
 867                 continue
 868
 869             file = {}
 870             file["path"] = path
 871             file["rev"] = commit["rev%s" % fnum]
 872             file["action"] = commit["action%s" % fnum]
 873             file["type"] = commit["type%s" % fnum]
 874             files.append(file)
 875             fnum = fnum + 1
 876         return files
 877
 878     def stripRepoPath(self, path, prefixes):
 879         if self.keepRepoPath:
 880             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 881
 882         for p in prefixes:
 883             if path.startswith(p):
 884                 path = path[len(p):]
 885
 886         return path
 887
 888     def splitFilesIntoBranches(self, commit):
 889         branches = {}
 890         fnum = 0
 891         while commit.has_key("depotFile%s" % fnum):
 892             path =  commit["depotFile%s" % fnum]
 893             found = [p for p in self.depotPaths
 894                      if path.startswith (p)]
 895             if not found:
 896                 fnum = fnum + 1
 897                 continue
 898
 899             file = {}
 900             file["path"] = path
 901             file["rev"] = commit["rev%s" % fnum]
 902             file["action"] = commit["action%s" % fnum]
 903             file["type"] = commit["type%s" % fnum]
 904             fnum = fnum + 1
 905
 906             relPath = self.stripRepoPath(path, self.depotPaths)
 907
 908             for branch in self.knownBranches.keys():
 909
 910                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 911                 if relPath.startswith(branch + "/"):
 912                     if branch not in branches:
 913                         branches[branch] = []
 914                     branches[branch].append(file)
 915                     break
 916
 917         return branches
 918
 919     ## Should move this out, doesn't use SELF.
 920     def readP4Files(self, files):
 921         files = [f for f in files
 922                  if f['action'] != 'delete']
 923
 924         if not files:
 925             return
 926
 927         filedata = p4CmdList('-x - print',
 928                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 929                                               for f in files]),
 930                              stdin_mode='w+')
 931         if "p4ExitCode" in filedata[0]:
 932             die("Problems executing p4. Error: [%d]."
 933                 % (filedata[0]['p4ExitCode']));
 934
 935         j = 0;
 936         contents = {}
 937         while j < len(filedata):
 938             stat = filedata[j]
 939             j += 1
 940             text = ''
 941             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 942                 tmp = filedata[j]['data']
 943                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 944                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 945                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 946                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 947                 text += tmp
 948                 j += 1
 949
 950
 951             if not stat.has_key('depotFile'):
 952                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 953                 continue
 954
 955             contents[stat['depotFile']] = text
 956
 957         for f in files:
 958             assert not f.has_key('data')
 959             f['data'] = contents[f['path']]
 960
 961     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 962         epoch = details["time"]
 963         author = details["user"]
 964
 965         if self.verbose:
 966             print "commit into %s" % branch
 967
 968         # start with reading files; if that fails, we should not
 969         # create a commit.
 970         new_files = []
 971         for f in files:
 972             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 973                 new_files.append (f)
 974             else:
 975                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 976         files = new_files
 977         self.readP4Files(files)
 978
 979
 980
 981
 982         self.gitStream.write("commit %s\n" % branch)
 983 #        gitStream.write("mark :%s\n" % details["change"])
 984         self.committedChanges.add(int(details["change"]))
 985         committer = ""
 986         if author not in self.users:
 987             self.getUserMapFromPerforceServer()
 988         if author in self.users:
 989             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 990         else:
 991             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 992
 993         self.gitStream.write("committer %s\n" % committer)
 994
 995         self.gitStream.write("data <<EOT\n")
 996         self.gitStream.write(details["desc"])
 997         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 998                              % (','.join (branchPrefixes), details["change"]))
 999         if len(details['options']) > 0:
1000             self.gitStream.write(": options = %s" % details['options'])
1001         self.gitStream.write("]\nEOT\n\n")
1002
1003         if len(parent) > 0:
1004             if self.verbose:
1005                 print "parent %s" % parent
1006             self.gitStream.write("from %s\n" % parent)
1007
1008         for file in files:
1009             if file["type"] == "apple":
1010                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1011                 continue
1012
1013             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1014             if file["action"] == "delete":
1015                 self.gitStream.write("D %s\n" % relPath)
1016             else:
1017                 data = file['data']
1018
1019                 mode = "644"
1020                 if isP4Exec(file["type"]):
1021                     mode = "755"
1022                 elif file["type"] == "symlink":
1023                     mode = "120000"
1024                     # p4 print on a symlink contains "target\n", so strip it off
1025                     data = data[:-1]
1026
1027                 if self.isWindows and file["type"].endswith("text"):
1028                     data = data.replace("\r\n", "\n")
1029
1030                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1031                 self.gitStream.write("data %s\n" % len(data))
1032                 self.gitStream.write(data)
1033                 self.gitStream.write("\n")
1034
1035         self.gitStream.write("\n")
1036
1037         change = int(details["change"])
1038
1039         if self.labels.has_key(change):
1040             label = self.labels[change]
1041             labelDetails = label[0]
1042             labelRevisions = label[1]
1043             if self.verbose:
1044                 print "Change %s is labelled %s" % (change, labelDetails)
1045
1046             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1047                                                     for p in branchPrefixes]))
1048
1049             if len(files) == len(labelRevisions):
1050
1051                 cleanedFiles = {}
1052                 for info in files:
1053                     if info["action"] == "delete":
1054                         continue
1055                     cleanedFiles[info["depotFile"]] = info["rev"]
1056
1057                 if cleanedFiles == labelRevisions:
1058                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1059                     self.gitStream.write("from %s\n" % branch)
1060
1061                     owner = labelDetails["Owner"]
1062                     tagger = ""
1063                     if author in self.users:
1064                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1065                     else:
1066                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1067                     self.gitStream.write("tagger %s\n" % tagger)
1068                     self.gitStream.write("data <<EOT\n")
1069                     self.gitStream.write(labelDetails["Description"])
1070                     self.gitStream.write("EOT\n\n")
1071
1072                 else:
1073                     if not self.silent:
1074                         print ("Tag %s does not match with change %s: files do not match."
1075                                % (labelDetails["label"], change))
1076
1077             else:
1078                 if not self.silent:
1079                     print ("Tag %s does not match with change %s: file count is different."
1080                            % (labelDetails["label"], change))
1081
1082     def getUserCacheFilename(self):
1083         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1084         return home + "/.gitp4-usercache.txt"
1085
1086     def getUserMapFromPerforceServer(self):
1087         if self.userMapFromPerforceServer:
1088             return
1089         self.users = {}
1090
1091         for output in p4CmdList("users"):
1092             if not output.has_key("User"):
1093                 continue
1094             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1095
1096
1097         s = ''
1098         for (key, val) in self.users.items():
1099             s += "%s\t%s\n" % (key, val)
1100
1101         open(self.getUserCacheFilename(), "wb").write(s)
1102         self.userMapFromPerforceServer = True
1103
1104     def loadUserMapFromCache(self):
1105         self.users = {}
1106         self.userMapFromPerforceServer = False
1107         try:
1108             cache = open(self.getUserCacheFilename(), "rb")
1109             lines = cache.readlines()
1110             cache.close()
1111             for line in lines:
1112                 entry = line.strip().split("\t")
1113                 self.users[entry[0]] = entry[1]
1114         except IOError:
1115             self.getUserMapFromPerforceServer()
1116
1117     def getLabels(self):
1118         self.labels = {}
1119
1120         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1121         if len(l) > 0 and not self.silent:
1122             print "Finding files belonging to labels in %s" % `self.depotPaths`
1123
1124         for output in l:
1125             label = output["label"]
1126             revisions = {}
1127             newestChange = 0
1128             if self.verbose:
1129                 print "Querying files for label %s" % label
1130             for file in p4CmdList("files "
1131                                   +  ' '.join (["%s...@%s" % (p, label)
1132                                                 for p in self.depotPaths])):
1133                 revisions[file["depotFile"]] = file["rev"]
1134                 change = int(file["change"])
1135                 if change > newestChange:
1136                     newestChange = change
1137
1138             self.labels[newestChange] = [output, revisions]
1139
1140         if self.verbose:
1141             print "Label changes: %s" % self.labels.keys()
1142
1143     def guessProjectName(self):
1144         for p in self.depotPaths:
1145             if p.endswith("/"):
1146                 p = p[:-1]
1147             p = p[p.strip().rfind("/") + 1:]
1148             if not p.endswith("/"):
1149                p += "/"
1150             return p
1151
1152     def getBranchMapping(self):
1153         lostAndFoundBranches = set()
1154
1155         for info in p4CmdList("branches"):
1156             details = p4Cmd("branch -o %s" % info["branch"])
1157             viewIdx = 0
1158             while details.has_key("View%s" % viewIdx):
1159                 paths = details["View%s" % viewIdx].split(" ")
1160                 viewIdx = viewIdx + 1
1161                 # require standard //depot/foo/... //depot/bar/... mapping
1162                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1163                     continue
1164                 source = paths[0]
1165                 destination = paths[1]
1166                 ## HACK
1167                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1168                     source = source[len(self.depotPaths[0]):-4]
1169                     destination = destination[len(self.depotPaths[0]):-4]
1170
1171                     if destination in self.knownBranches:
1172                         if not self.silent:
1173                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1174                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1175                         continue
1176
1177                     self.knownBranches[destination] = source
1178
1179                     lostAndFoundBranches.discard(destination)
1180
1181                     if source not in self.knownBranches:
1182                         lostAndFoundBranches.add(source)
1183
1184
1185         for branch in lostAndFoundBranches:
1186             self.knownBranches[branch] = branch
1187
1188     def getBranchMappingFromGitBranches(self):
1189         branches = p4BranchesInGit(self.importIntoRemotes)
1190         for branch in branches.keys():
1191             if branch == "master":
1192                 branch = "main"
1193             else:
1194                 branch = branch[len(self.projectName):]
1195             self.knownBranches[branch] = branch
1196
1197     def listExistingP4GitBranches(self):
1198         # branches holds mapping from name to commit
1199         branches = p4BranchesInGit(self.importIntoRemotes)
1200         self.p4BranchesInGit = branches.keys()
1201         for branch in branches.keys():
1202             self.initialParents[self.refPrefix + branch] = branches[branch]
1203
1204     def updateOptionDict(self, d):
1205         option_keys = {}
1206         if self.keepRepoPath:
1207             option_keys['keepRepoPath'] = 1
1208
1209         d["options"] = ' '.join(sorted(option_keys.keys()))
1210
1211     def readOptions(self, d):
1212         self.keepRepoPath = (d.has_key('options')
1213                              and ('keepRepoPath' in d['options']))
1214
1215     def gitRefForBranch(self, branch):
1216         if branch == "main":
1217             return self.refPrefix + "master"
1218
1219         if len(branch) <= 0:
1220             return branch
1221
1222         return self.refPrefix + self.projectName + branch
1223
1224     def gitCommitByP4Change(self, ref, change):
1225         if self.verbose:
1226             print "looking in ref " + ref + " for change %s using bisect..." % change
1227
1228         earliestCommit = ""
1229         latestCommit = parseRevision(ref)
1230
1231         while True:
1232             if self.verbose:
1233                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1234             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1235             if len(next) == 0:
1236                 if self.verbose:
1237                     print "argh"
1238                 return ""
1239             log = extractLogMessageFromGitCommit(next)
1240             settings = extractSettingsGitLog(log)
1241             currentChange = int(settings['change'])
1242             if self.verbose:
1243                 print "current change %s" % currentChange
1244
1245             if currentChange == change:
1246                 if self.verbose:
1247                     print "found %s" % next
1248                 return next
1249
1250             if currentChange < change:
1251                 earliestCommit = "^%s" % next
1252             else:
1253                 latestCommit = "%s" % next
1254
1255         return ""
1256
1257     def importNewBranch(self, branch, maxChange):
1258         # make fast-import flush all changes to disk and update the refs using the checkpoint
1259         # command so that we can try to find the branch parent in the git history
1260         self.gitStream.write("checkpoint\n\n");
1261         self.gitStream.flush();
1262         branchPrefix = self.depotPaths[0] + branch + "/"
1263         range = "@1,%s" % maxChange
1264         #print "prefix" + branchPrefix
1265         changes = p4ChangesForPaths([branchPrefix], range)
1266         if len(changes) <= 0:
1267             return False
1268         firstChange = changes[0]
1269         #print "first change in branch: %s" % firstChange
1270         sourceBranch = self.knownBranches[branch]
1271         sourceDepotPath = self.depotPaths[0] + sourceBranch
1272         sourceRef = self.gitRefForBranch(sourceBranch)
1273         #print "source " + sourceBranch
1274
1275         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1276         #print "branch parent: %s" % branchParentChange
1277         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1278         if len(gitParent) > 0:
1279             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1280             #print "parent git commit: %s" % gitParent
1281
1282         self.importChanges(changes)
1283         return True
1284
1285     def importChanges(self, changes):
1286         cnt = 1
1287         for change in changes:
1288             description = p4Cmd("describe %s" % change)
1289             self.updateOptionDict(description)
1290
1291             if not self.silent:
1292                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1293                 sys.stdout.flush()
1294             cnt = cnt + 1
1295
1296             try:
1297                 if self.detectBranches:
1298                     branches = self.splitFilesIntoBranches(description)
1299                     for branch in branches.keys():
1300                         ## HACK  --hwn
1301                         branchPrefix = self.depotPaths[0] + branch + "/"
1302
1303                         parent = ""
1304
1305                         filesForCommit = branches[branch]
1306
1307                         if self.verbose:
1308                             print "branch is %s" % branch
1309
1310                         self.updatedBranches.add(branch)
1311
1312                         if branch not in self.createdBranches:
1313                             self.createdBranches.add(branch)
1314                             parent = self.knownBranches[branch]
1315                             if parent == branch:
1316                                 parent = ""
1317                             else:
1318                                 fullBranch = self.projectName + branch
1319                                 if fullBranch not in self.p4BranchesInGit:
1320                                     if not self.silent:
1321                                         print("\n    Importing new branch %s" % fullBranch);
1322                                     if self.importNewBranch(branch, change - 1):
1323                                         parent = ""
1324                                         self.p4BranchesInGit.append(fullBranch)
1325                                     if not self.silent:
1326                                         print("\n    Resuming with change %s" % change);
1327
1328                                 if self.verbose:
1329                                     print "parent determined through known branches: %s" % parent
1330
1331                         branch = self.gitRefForBranch(branch)
1332                         parent = self.gitRefForBranch(parent)
1333
1334                         if self.verbose:
1335                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1336
1337                         if len(parent) == 0 and branch in self.initialParents:
1338                             parent = self.initialParents[branch]
1339                             del self.initialParents[branch]
1340
1341                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1342                 else:
1343                     files = self.extractFilesFromCommit(description)
1344                     self.commit(description, files, self.branch, self.depotPaths,
1345                                 self.initialParent)
1346                     self.initialParent = ""
1347             except IOError:
1348                 print self.gitError.read()
1349                 sys.exit(1)
1350
1351     def importHeadRevision(self, revision):
1352         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1353
1354         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1355         details["desc"] = ("Initial import of %s from the state at revision %s"
1356                            % (' '.join(self.depotPaths), revision))
1357         details["change"] = revision
1358         newestRevision = 0
1359
1360         fileCnt = 0
1361         for info in p4CmdList("files "
1362                               +  ' '.join(["%s...%s"
1363                                            % (p, revision)
1364                                            for p in self.depotPaths])):
1365
1366             if info['code'] == 'error':
1367                 sys.stderr.write("p4 returned an error: %s\n"
1368                                  % info['data'])
1369                 sys.exit(1)
1370
1371
1372             change = int(info["change"])
1373             if change > newestRevision:
1374                 newestRevision = change
1375
1376             if info["action"] == "delete":
1377                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1378                 #fileCnt = fileCnt + 1
1379                 continue
1380
1381             for prop in ["depotFile", "rev", "action", "type" ]:
1382                 details["%s%s" % (prop, fileCnt)] = info[prop]
1383
1384             fileCnt = fileCnt + 1
1385
1386         details["change"] = newestRevision
1387         self.updateOptionDict(details)
1388         try:
1389             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1390         except IOError:
1391             print "IO error with git fast-import. Is your git version recent enough?"
1392             print self.gitError.read()
1393
1394
1395     def run(self, args):
1396         self.depotPaths = []
1397         self.changeRange = ""
1398         self.initialParent = ""
1399         self.previousDepotPaths = []
1400
1401         # map from branch depot path to parent branch
1402         self.knownBranches = {}
1403         self.initialParents = {}
1404         self.hasOrigin = originP4BranchesExist()
1405         if not self.syncWithOrigin:
1406             self.hasOrigin = False
1407
1408         if self.importIntoRemotes:
1409             self.refPrefix = "refs/remotes/p4/"
1410         else:
1411             self.refPrefix = "refs/heads/p4/"
1412
1413         if self.syncWithOrigin and self.hasOrigin:
1414             if not self.silent:
1415                 print "Syncing with origin first by calling git fetch origin"
1416             system("git fetch origin")
1417
1418         if len(self.branch) == 0:
1419             self.branch = self.refPrefix + "master"
1420             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1421                 system("git update-ref %s refs/heads/p4" % self.branch)
1422                 system("git branch -D p4");
1423             # create it /after/ importing, when master exists
1424             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1425                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1426
1427         # TODO: should always look at previous commits,
1428         # merge with previous imports, if possible.
1429         if args == []:
1430             if self.hasOrigin:
1431                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1432             self.listExistingP4GitBranches()
1433
1434             if len(self.p4BranchesInGit) > 1:
1435                 if not self.silent:
1436                     print "Importing from/into multiple branches"
1437                 self.detectBranches = True
1438
1439             if self.verbose:
1440                 print "branches: %s" % self.p4BranchesInGit
1441
1442             p4Change = 0
1443             for branch in self.p4BranchesInGit:
1444                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1445
1446                 settings = extractSettingsGitLog(logMsg)
1447
1448                 self.readOptions(settings)
1449                 if (settings.has_key('depot-paths')
1450                     and settings.has_key ('change')):
1451                     change = int(settings['change']) + 1
1452                     p4Change = max(p4Change, change)
1453
1454                     depotPaths = sorted(settings['depot-paths'])
1455                     if self.previousDepotPaths == []:
1456                         self.previousDepotPaths = depotPaths
1457                     else:
1458                         paths = []
1459                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1460                             for i in range(0, min(len(cur), len(prev))):
1461                                 if cur[i] <> prev[i]:
1462                                     i = i - 1
1463                                     break
1464
1465                             paths.append (cur[:i + 1])
1466
1467                         self.previousDepotPaths = paths
1468
1469             if p4Change > 0:
1470                 self.depotPaths = sorted(self.previousDepotPaths)
1471                 self.changeRange = "@%s,#head" % p4Change
1472                 if not self.detectBranches:
1473                     self.initialParent = parseRevision(self.branch)
1474                 if not self.silent and not self.detectBranches:
1475                     print "Performing incremental import into %s git branch" % self.branch
1476
1477         if not self.branch.startswith("refs/"):
1478             self.branch = "refs/heads/" + self.branch
1479
1480         if len(args) == 0 and self.depotPaths:
1481             if not self.silent:
1482                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1483         else:
1484             if self.depotPaths and self.depotPaths != args:
1485                 print ("previous import used depot path %s and now %s was specified. "
1486                        "This doesn't work!" % (' '.join (self.depotPaths),
1487                                                ' '.join (args)))
1488                 sys.exit(1)
1489
1490             self.depotPaths = sorted(args)
1491
1492         revision = ""
1493         self.users = {}
1494
1495         newPaths = []
1496         for p in self.depotPaths:
1497             if p.find("@") != -1:
1498                 atIdx = p.index("@")
1499                 self.changeRange = p[atIdx:]
1500                 if self.changeRange == "@all":
1501                     self.changeRange = ""
1502                 elif ',' not in self.changeRange:
1503                     revision = self.changeRange
1504                     self.changeRange = ""
1505                 p = p[:atIdx]
1506             elif p.find("#") != -1:
1507                 hashIdx = p.index("#")
1508                 revision = p[hashIdx:]
1509                 p = p[:hashIdx]
1510             elif self.previousDepotPaths == []:
1511                 revision = "#head"
1512
1513             p = re.sub ("\.\.\.$", "", p)
1514             if not p.endswith("/"):
1515                 p += "/"
1516
1517             newPaths.append(p)
1518
1519         self.depotPaths = newPaths
1520
1521
1522         self.loadUserMapFromCache()
1523         self.labels = {}
1524         if self.detectLabels:
1525             self.getLabels();
1526
1527         if self.detectBranches:
1528             ## FIXME - what's a P4 projectName ?
1529             self.projectName = self.guessProjectName()
1530
1531             if self.hasOrigin:
1532                 self.getBranchMappingFromGitBranches()
1533             else:
1534                 self.getBranchMapping()
1535             if self.verbose:
1536                 print "p4-git branches: %s" % self.p4BranchesInGit
1537                 print "initial parents: %s" % self.initialParents
1538             for b in self.p4BranchesInGit:
1539                 if b != "master":
1540
1541                     ## FIXME
1542                     b = b[len(self.projectName):]
1543                 self.createdBranches.add(b)
1544
1545         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1546
1547         importProcess = subprocess.Popen(["git", "fast-import"],
1548                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1549                                          stderr=subprocess.PIPE);
1550         self.gitOutput = importProcess.stdout
1551         self.gitStream = importProcess.stdin
1552         self.gitError = importProcess.stderr
1553
1554         if revision:
1555             self.importHeadRevision(revision)
1556         else:
1557             changes = []
1558
1559             if len(self.changesFile) > 0:
1560                 output = open(self.changesFile).readlines()
1561                 changeSet = Set()
1562                 for line in output:
1563                     changeSet.add(int(line))
1564
1565                 for change in changeSet:
1566                     changes.append(change)
1567
1568                 changes.sort()
1569             else:
1570                 if self.verbose:
1571                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1572                                                               self.changeRange)
1573                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1574
1575                 if len(self.maxChanges) > 0:
1576                     changes = changes[:min(int(self.maxChanges), len(changes))]
1577
1578             if len(changes) == 0:
1579                 if not self.silent:
1580                     print "No changes to import!"
1581                 return True
1582
1583             if not self.silent and not self.detectBranches:
1584                 print "Import destination: %s" % self.branch
1585
1586             self.updatedBranches = set()
1587
1588             self.importChanges(changes)
1589
1590             if not self.silent:
1591                 print ""
1592                 if len(self.updatedBranches) > 0:
1593                     sys.stdout.write("Updated branches: ")
1594                     for b in self.updatedBranches:
1595                         sys.stdout.write("%s " % b)
1596                     sys.stdout.write("\n")
1597
1598         self.gitStream.close()
1599         if importProcess.wait() != 0:
1600             die("fast-import failed: %s" % self.gitError.read())
1601         self.gitOutput.close()
1602         self.gitError.close()
1603
1604         return True
1605
1606 class P4Rebase(Command):
1607     def __init__(self):
1608         Command.__init__(self)
1609         self.options = [ ]
1610         self.description = ("Fetches the latest revision from perforce and "
1611                             + "rebases the current work (branch) against it")
1612         self.verbose = False
1613
1614     def run(self, args):
1615         sync = P4Sync()
1616         sync.run([])
1617
1618         return self.rebase()
1619
1620     def rebase(self):
1621         if os.system("git update-index --refresh") != 0:
1622             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1623         if len(read_pipe("git diff-index HEAD --")) > 0:
1624             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1625
1626         [upstream, settings] = findUpstreamBranchPoint()
1627         if len(upstream) == 0:
1628             die("Cannot find upstream branchpoint for rebase")
1629
1630         # the branchpoint may be p4/foo~3, so strip off the parent
1631         upstream = re.sub("~[0-9]+$", "", upstream)
1632
1633         print "Rebasing the current branch onto %s" % upstream
1634         oldHead = read_pipe("git rev-parse HEAD").strip()
1635         system("git rebase %s" % upstream)
1636         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1637         return True
1638
1639 class P4Clone(P4Sync):
1640     def __init__(self):
1641         P4Sync.__init__(self)
1642         self.description = "Creates a new git repository and imports from Perforce into it"
1643         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1644         self.options += [
1645             optparse.make_option("--destination", dest="cloneDestination",
1646                                  action='store', default=None,
1647                                  help="where to leave result of the clone"),
1648             optparse.make_option("-/", dest="cloneExclude",
1649                                  action="append", type="string",
1650                                  help="exclude depot path")
1651         ]
1652         self.cloneDestination = None
1653         self.needsGit = False
1654
1655     # This is required for the "append" cloneExclude action
1656     def ensure_value(self, attr, value):
1657         if not hasattr(self, attr) or getattr(self, attr) is None:
1658             setattr(self, attr, value)
1659         return getattr(self, attr)
1660
1661     def defaultDestination(self, args):
1662         ## TODO: use common prefix of args?
1663         depotPath = args[0]
1664         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1665         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1666         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1667         depotDir = re.sub(r"/$", "", depotDir)
1668         return os.path.split(depotDir)[1]
1669
1670     def run(self, args):
1671         if len(args) < 1:
1672             return False
1673
1674         if self.keepRepoPath and not self.cloneDestination:
1675             sys.stderr.write("Must specify destination for --keep-path\n")
1676             sys.exit(1)
1677
1678         depotPaths = args
1679
1680         if not self.cloneDestination and len(depotPaths) > 1:
1681             self.cloneDestination = depotPaths[-1]
1682             depotPaths = depotPaths[:-1]
1683
1684         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1685         for p in depotPaths:
1686             if not p.startswith("//"):
1687                 return False
1688
1689         if not self.cloneDestination:
1690             self.cloneDestination = self.defaultDestination(args)
1691
1692         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1693         if not os.path.exists(self.cloneDestination):
1694             os.makedirs(self.cloneDestination)
1695         os.chdir(self.cloneDestination)
1696         system("git init")
1697         self.gitdir = os.getcwd() + "/.git"
1698         if not P4Sync.run(self, depotPaths):
1699             return False
1700         if self.branch != "master":
1701             if gitBranchExists("refs/remotes/p4/master"):
1702                 system("git branch master refs/remotes/p4/master")
1703                 system("git checkout -f")
1704             else:
1705                 print "Could not detect main branch. No checkout/master branch created."
1706
1707         return True
1708
1709 class P4Branches(Command):
1710     def __init__(self):
1711         Command.__init__(self)
1712         self.options = [ ]
1713         self.description = ("Shows the git branches that hold imports and their "
1714                             + "corresponding perforce depot paths")
1715         self.verbose = False
1716
1717     def run(self, args):
1718         if originP4BranchesExist():
1719             createOrUpdateBranchesFromOrigin()
1720
1721         cmdline = "git rev-parse --symbolic "
1722         cmdline += " --remotes"
1723
1724         for line in read_pipe_lines(cmdline):
1725             line = line.strip()
1726
1727             if not line.startswith('p4/') or line == "p4/HEAD":
1728                 continue
1729             branch = line
1730
1731             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1732             settings = extractSettingsGitLog(log)
1733
1734             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1735         return True
1736
1737 class HelpFormatter(optparse.IndentedHelpFormatter):
1738     def __init__(self):
1739         optparse.IndentedHelpFormatter.__init__(self)
1740
1741     def format_description(self, description):
1742         if description:
1743             return description + "\n"
1744         else:
1745             return ""
1746
1747 def printUsage(commands):
1748     print "usage: %s <command> [options]" % sys.argv[0]
1749     print ""
1750     print "valid commands: %s" % ", ".join(commands)
1751     print ""
1752     print "Try %s <command> --help for command specific help." % sys.argv[0]
1753     print ""
1754
1755 commands = {
1756     "debug" : P4Debug,
1757     "submit" : P4Submit,
1758     "commit" : P4Submit,
1759     "sync" : P4Sync,
1760     "rebase" : P4Rebase,
1761     "clone" : P4Clone,
1762     "rollback" : P4RollBack,
1763     "branches" : P4Branches
1764 }
1765
1766
1767 def main():
1768     if len(sys.argv[1:]) == 0:
1769         printUsage(commands.keys())
1770         sys.exit(2)
1771
1772     cmd = ""
1773     cmdName = sys.argv[1]
1774     try:
1775         klass = commands[cmdName]
1776         cmd = klass()
1777     except KeyError:
1778         print "unknown command %s" % cmdName
1779         print ""
1780         printUsage(commands.keys())
1781         sys.exit(2)
1782
1783     options = cmd.options
1784     cmd.gitdir = os.environ.get("GIT_DIR", None)
1785
1786     args = sys.argv[2:]
1787
1788     if len(options) > 0:
1789         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1790
1791         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1792                                        options,
1793                                        description = cmd.description,
1794                                        formatter = HelpFormatter())
1795
1796         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1797     global verbose
1798     verbose = cmd.verbose
1799     if cmd.needsGit:
1800         if cmd.gitdir == None:
1801             cmd.gitdir = os.path.abspath(".git")
1802             if not isValidGitDir(cmd.gitdir):
1803                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1804                 if os.path.exists(cmd.gitdir):
1805                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1806                     if len(cdup) > 0:
1807                         os.chdir(cdup);
1808
1809         if not isValidGitDir(cmd.gitdir):
1810             if isValidGitDir(cmd.gitdir + "/.git"):
1811                 cmd.gitdir += "/.git"
1812             else:
1813                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1814
1815         os.environ["GIT_DIR"] = cmd.gitdir
1816
1817     if not cmd.run(args):
1818         parser.print_help()
1819
1820
1821 if __name__ == '__main__':
1822     main()