contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                 optparse.make_option("--origin", dest="origin"),
 470                 optparse.make_option("--reset", action="store_true", dest="reset"),
 471                 optparse.make_option("--log-substitutions", dest="substFile"),
 472                 optparse.make_option("--dry-run", action="store_true"),
 473                 optparse.make_option("--direct", dest="directSubmit", action="store_true"),
 474                 optparse.make_option("--trust-me-like-a-fool", dest="trustMeLikeAFool", action="store_true"),
 475                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 476         ]
 477         self.description = "Submit changes from git to the perforce depot."
 478         self.usage += " [name of git branch to submit into perforce depot]"
 479         self.firstTime = True
 480         self.reset = False
 481         self.interactive = True
 482         self.dryRun = False
 483         self.substFile = ""
 484         self.firstTime = True
 485         self.origin = ""
 486         self.directSubmit = False
 487         self.trustMeLikeAFool = False
 488         self.detectRename = False
 489         self.verbose = False
 490         self.isWindows = (platform.system() == "Windows")
 491
 492         self.logSubstitutions = {}
 493         self.logSubstitutions["<enter description here>"] = "%log%"
 494         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 495
 496     def check(self):
 497         if len(p4CmdList("opened ...")) > 0:
 498             die("You have files opened with perforce! Close them before starting the sync.")
 499
 500     def start(self):
 501         if len(self.config) > 0 and not self.reset:
 502             die("Cannot start sync. Previous sync config found at %s\n"
 503                 "If you want to start submitting again from scratch "
 504                 "maybe you want to call git-p4 submit --reset" % self.configFile)
 505
 506         commits = []
 507         if self.directSubmit:
 508             commits.append("0")
 509         else:
 510             for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 511                 commits.append(line.strip())
 512             commits.reverse()
 513
 514         self.config["commits"] = commits
 515
 516     def prepareLogMessage(self, template, message):
 517         result = ""
 518
 519         for line in template.split("\n"):
 520             if line.startswith("#"):
 521                 result += line + "\n"
 522                 continue
 523
 524             substituted = False
 525             for key in self.logSubstitutions.keys():
 526                 if line.find(key) != -1:
 527                     value = self.logSubstitutions[key]
 528                     value = value.replace("%log%", message)
 529                     if value != "@remove@":
 530                         result += line.replace(key, value) + "\n"
 531                     substituted = True
 532                     break
 533
 534             if not substituted:
 535                 result += line + "\n"
 536
 537         return result
 538
 539     def prepareSubmitTemplate(self):
 540         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 541         template = ""
 542         inFilesSection = False
 543         for line in read_pipe_lines("p4 change -o"):
 544             if inFilesSection:
 545                 if line.startswith("\t"):
 546                     # path starts and ends with a tab
 547                     path = line[1:]
 548                     lastTab = path.rfind("\t")
 549                     if lastTab != -1:
 550                         path = path[:lastTab]
 551                         if not path.startswith(self.depotPath):
 552                             continue
 553                 else:
 554                     inFilesSection = False
 555             else:
 556                 if line.startswith("Files:"):
 557                     inFilesSection = True
 558
 559             template += line
 560
 561         return template
 562
 563     def applyCommit(self, id):
 564         if self.directSubmit:
 565             print "Applying local change in working directory/index"
 566             diff = self.diffStatus
 567         else:
 568             print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 569             diffOpts = ("", "-M")[self.detectRename]
 570             diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 571         filesToAdd = set()
 572         filesToDelete = set()
 573         editedFiles = set()
 574         filesToChangeExecBit = {}
 575         for line in diff:
 576             diff = parseDiffTreeEntry(line)
 577             modifier = diff['status']
 578             path = diff['src']
 579             if modifier == "M":
 580                 system("p4 edit \"%s\"" % path)
 581                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 582                     filesToChangeExecBit[path] = diff['dst_mode']
 583                 editedFiles.add(path)
 584             elif modifier == "A":
 585                 filesToAdd.add(path)
 586                 filesToChangeExecBit[path] = diff['dst_mode']
 587                 if path in filesToDelete:
 588                     filesToDelete.remove(path)
 589             elif modifier == "D":
 590                 filesToDelete.add(path)
 591                 if path in filesToAdd:
 592                     filesToAdd.remove(path)
 593             elif modifier == "R":
 594                 src, dest = diff['src'], diff['dst']
 595                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 596                 system("p4 edit \"%s\"" % (dest))
 597                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 598                     filesToChangeExecBit[dest] = diff['dst_mode']
 599                 os.unlink(dest)
 600                 editedFiles.add(dest)
 601                 filesToDelete.add(src)
 602             else:
 603                 die("unknown modifier %s for %s" % (modifier, path))
 604
 605         if self.directSubmit:
 606             diffcmd = "cat \"%s\"" % self.diffFile
 607         else:
 608             diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 609         patchcmd = diffcmd + " | git apply "
 610         tryPatchCmd = patchcmd + "--check -"
 611         applyPatchCmd = patchcmd + "--check --apply -"
 612
 613         if os.system(tryPatchCmd) != 0:
 614             print "Unfortunately applying the change failed!"
 615             print "What do you want to do?"
 616             response = "x"
 617             while response != "s" and response != "a" and response != "w":
 618                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 619                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 620             if response == "s":
 621                 print "Skipping! Good luck with the next patches..."
 622                 for f in editedFiles:
 623                     system("p4 revert \"%s\"" % f);
 624                 for f in filesToAdd:
 625                     system("rm %s" %f)
 626                 return
 627             elif response == "a":
 628                 os.system(applyPatchCmd)
 629                 if len(filesToAdd) > 0:
 630                     print "You may also want to call p4 add on the following files:"
 631                     print " ".join(filesToAdd)
 632                 if len(filesToDelete):
 633                     print "The following files should be scheduled for deletion with p4 delete:"
 634                     print " ".join(filesToDelete)
 635                 die("Please resolve and submit the conflict manually and "
 636                     + "continue afterwards with git-p4 submit --continue")
 637             elif response == "w":
 638                 system(diffcmd + " > patch.txt")
 639                 print "Patch saved to patch.txt in %s !" % self.clientPath
 640                 die("Please resolve and submit the conflict manually and "
 641                     "continue afterwards with git-p4 submit --continue")
 642
 643         system(applyPatchCmd)
 644
 645         for f in filesToAdd:
 646             system("p4 add \"%s\"" % f)
 647         for f in filesToDelete:
 648             system("p4 revert \"%s\"" % f)
 649             system("p4 delete \"%s\"" % f)
 650
 651         # Set/clear executable bits
 652         for f in filesToChangeExecBit.keys():
 653             mode = filesToChangeExecBit[f]
 654             setP4ExecBit(f, mode)
 655
 656         logMessage = ""
 657         if not self.directSubmit:
 658             logMessage = extractLogMessageFromGitCommit(id)
 659             logMessage = logMessage.replace("\n", "\n\t")
 660             if self.isWindows:
 661                 logMessage = logMessage.replace("\n", "\r\n")
 662             logMessage = logMessage.strip()
 663
 664         template = self.prepareSubmitTemplate()
 665
 666         if self.interactive:
 667             submitTemplate = self.prepareLogMessage(template, logMessage)
 668             diff = read_pipe("p4 diff -du ...")
 669
 670             for newFile in filesToAdd:
 671                 diff += "==== new file ====\n"
 672                 diff += "--- /dev/null\n"
 673                 diff += "+++ %s\n" % newFile
 674                 f = open(newFile, "r")
 675                 for line in f.readlines():
 676                     diff += "+" + line
 677                 f.close()
 678
 679             separatorLine = "######## everything below this line is just the diff #######"
 680             if platform.system() == "Windows":
 681                 separatorLine += "\r"
 682             separatorLine += "\n"
 683
 684             response = "e"
 685             if self.trustMeLikeAFool:
 686                 response = "y"
 687
 688             firstIteration = True
 689             while response == "e":
 690                 if not firstIteration:
 691                     response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ")
 692                 firstIteration = False
 693                 if response == "e":
 694                     [handle, fileName] = tempfile.mkstemp()
 695                     tmpFile = os.fdopen(handle, "w+")
 696                     tmpFile.write(submitTemplate + separatorLine + diff)
 697                     tmpFile.close()
 698                     defaultEditor = "vi"
 699                     if platform.system() == "Windows":
 700                         defaultEditor = "notepad"
 701                     editor = os.environ.get("EDITOR", defaultEditor);
 702                     system(editor + " " + fileName)
 703                     tmpFile = open(fileName, "rb")
 704                     message = tmpFile.read()
 705                     tmpFile.close()
 706                     os.remove(fileName)
 707                     submitTemplate = message[:message.index(separatorLine)]
 708                     if self.isWindows:
 709                         submitTemplate = submitTemplate.replace("\r\n", "\n")
 710
 711             if response == "y" or response == "yes":
 712                if self.dryRun:
 713                    print submitTemplate
 714                    raw_input("Press return to continue...")
 715                else:
 716                    if self.directSubmit:
 717                        print "Submitting to git first"
 718                        os.chdir(self.oldWorkingDirectory)
 719                        write_pipe("git commit -a -F -", submitTemplate)
 720                        os.chdir(self.clientPath)
 721
 722                    write_pipe("p4 submit -i", submitTemplate)
 723             elif response == "s":
 724                 for f in editedFiles:
 725                     system("p4 revert \"%s\"" % f);
 726                 for f in filesToAdd:
 727                     system("p4 revert \"%s\"" % f);
 728                     system("rm %s" %f)
 729                 for f in filesToDelete:
 730                     system("p4 delete \"%s\"" % f);
 731                 return
 732             else:
 733                 print "Not submitting!"
 734                 self.interactive = False
 735         else:
 736             fileName = "submit.txt"
 737             file = open(fileName, "w+")
 738             file.write(self.prepareLogMessage(template, logMessage))
 739             file.close()
 740             print ("Perforce submit template written as %s. "
 741                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 742                    % (fileName, fileName))
 743
 744     def run(self, args):
 745         if len(args) == 0:
 746             self.master = currentGitBranch()
 747             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 748                 die("Detecting current git branch failed!")
 749         elif len(args) == 1:
 750             self.master = args[0]
 751         else:
 752             return False
 753
 754         [upstream, settings] = findUpstreamBranchPoint()
 755         self.depotPath = settings['depot-paths'][0]
 756         if len(self.origin) == 0:
 757             self.origin = upstream
 758
 759         if self.verbose:
 760             print "Origin branch is " + self.origin
 761
 762         if len(self.depotPath) == 0:
 763             print "Internal error: cannot locate perforce depot path from existing branches"
 764             sys.exit(128)
 765
 766         self.clientPath = p4Where(self.depotPath)
 767
 768         if len(self.clientPath) == 0:
 769             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 770             sys.exit(128)
 771
 772         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 773         self.oldWorkingDirectory = os.getcwd()
 774
 775         if self.directSubmit:
 776             self.diffStatus = read_pipe_lines("git diff -r --name-status HEAD")
 777             if len(self.diffStatus) == 0:
 778                 print "No changes in working directory to submit."
 779                 return True
 780             patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD")
 781             self.diffFile = self.gitdir + "/p4-git-diff"
 782             f = open(self.diffFile, "wb")
 783             f.write(patch)
 784             f.close();
 785
 786         os.chdir(self.clientPath)
 787         print "Syncronizing p4 checkout..."
 788         system("p4 sync ...")
 789
 790         if self.reset:
 791             self.firstTime = True
 792
 793         if len(self.substFile) > 0:
 794             for line in open(self.substFile, "r").readlines():
 795                 tokens = line.strip().split("=")
 796                 self.logSubstitutions[tokens[0]] = tokens[1]
 797
 798         self.check()
 799         self.configFile = self.gitdir + "/p4-git-sync.cfg"
 800         self.config = shelve.open(self.configFile, writeback=True)
 801
 802         if self.firstTime:
 803             self.start()
 804
 805         commits = self.config.get("commits", [])
 806
 807         while len(commits) > 0:
 808             self.firstTime = False
 809             commit = commits[0]
 810             commits = commits[1:]
 811             self.config["commits"] = commits
 812             self.applyCommit(commit)
 813             if not self.interactive:
 814                 break
 815
 816         self.config.close()
 817
 818         if self.directSubmit:
 819             os.remove(self.diffFile)
 820
 821         if len(commits) == 0:
 822             if self.firstTime:
 823                 print "No changes found to apply between %s and current HEAD" % self.origin
 824             else:
 825                 print "All changes applied!"
 826                 os.chdir(self.oldWorkingDirectory)
 827
 828                 sync = P4Sync()
 829                 sync.run([])
 830
 831                 response = raw_input("Do you want to rebase current HEAD from Perforce now using git-p4 rebase? [y]es/[n]o ")
 832                 if response == "y" or response == "yes":
 833                     rebase = P4Rebase()
 834                     rebase.rebase()
 835             os.remove(self.configFile)
 836
 837         return True
 838
 839 class P4Sync(Command):
 840     def __init__(self):
 841         Command.__init__(self)
 842         self.options = [
 843                 optparse.make_option("--branch", dest="branch"),
 844                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 845                 optparse.make_option("--changesfile", dest="changesFile"),
 846                 optparse.make_option("--silent", dest="silent", action="store_true"),
 847                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 848                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 849                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 850                                      help="Import into refs/heads/ , not refs/remotes"),
 851                 optparse.make_option("--max-changes", dest="maxChanges"),
 852                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 853                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 854         ]
 855         self.description = """Imports from Perforce into a git repository.\n
 856     example:
 857     //depot/my/project/ -- to import the current head
 858     //depot/my/project/@all -- to import everything
 859     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 860
 861     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 862
 863         self.usage += " //depot/path[@revRange]"
 864         self.silent = False
 865         self.createdBranches = Set()
 866         self.committedChanges = Set()
 867         self.branch = ""
 868         self.detectBranches = False
 869         self.detectLabels = False
 870         self.changesFile = ""
 871         self.syncWithOrigin = True
 872         self.verbose = False
 873         self.importIntoRemotes = True
 874         self.maxChanges = ""
 875         self.isWindows = (platform.system() == "Windows")
 876         self.keepRepoPath = False
 877         self.depotPaths = None
 878         self.p4BranchesInGit = []
 879
 880         if gitConfig("git-p4.syncFromOrigin") == "false":
 881             self.syncWithOrigin = False
 882
 883     def extractFilesFromCommit(self, commit):
 884         files = []
 885         fnum = 0
 886         while commit.has_key("depotFile%s" % fnum):
 887             path =  commit["depotFile%s" % fnum]
 888
 889             found = [p for p in self.depotPaths
 890                      if path.startswith (p)]
 891             if not found:
 892                 fnum = fnum + 1
 893                 continue
 894
 895             file = {}
 896             file["path"] = path
 897             file["rev"] = commit["rev%s" % fnum]
 898             file["action"] = commit["action%s" % fnum]
 899             file["type"] = commit["type%s" % fnum]
 900             files.append(file)
 901             fnum = fnum + 1
 902         return files
 903
 904     def stripRepoPath(self, path, prefixes):
 905         if self.keepRepoPath:
 906             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 907
 908         for p in prefixes:
 909             if path.startswith(p):
 910                 path = path[len(p):]
 911
 912         return path
 913
 914     def splitFilesIntoBranches(self, commit):
 915         branches = {}
 916         fnum = 0
 917         while commit.has_key("depotFile%s" % fnum):
 918             path =  commit["depotFile%s" % fnum]
 919             found = [p for p in self.depotPaths
 920                      if path.startswith (p)]
 921             if not found:
 922                 fnum = fnum + 1
 923                 continue
 924
 925             file = {}
 926             file["path"] = path
 927             file["rev"] = commit["rev%s" % fnum]
 928             file["action"] = commit["action%s" % fnum]
 929             file["type"] = commit["type%s" % fnum]
 930             fnum = fnum + 1
 931
 932             relPath = self.stripRepoPath(path, self.depotPaths)
 933
 934             for branch in self.knownBranches.keys():
 935
 936                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 937                 if relPath.startswith(branch + "/"):
 938                     if branch not in branches:
 939                         branches[branch] = []
 940                     branches[branch].append(file)
 941                     break
 942
 943         return branches
 944
 945     ## Should move this out, doesn't use SELF.
 946     def readP4Files(self, files):
 947         files = [f for f in files
 948                  if f['action'] != 'delete']
 949
 950         if not files:
 951             return
 952
 953         filedata = p4CmdList('-x - print',
 954                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 955                                               for f in files]),
 956                              stdin_mode='w+')
 957         if "p4ExitCode" in filedata[0]:
 958             die("Problems executing p4. Error: [%d]."
 959                 % (filedata[0]['p4ExitCode']));
 960
 961         j = 0;
 962         contents = {}
 963         while j < len(filedata):
 964             stat = filedata[j]
 965             j += 1
 966             text = ''
 967             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 968                 tmp = filedata[j]['data']
 969                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 970                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 971                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 972                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 973                 text += tmp
 974                 j += 1
 975
 976
 977             if not stat.has_key('depotFile'):
 978                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 979                 continue
 980
 981             contents[stat['depotFile']] = text
 982
 983         for f in files:
 984             assert not f.has_key('data')
 985             f['data'] = contents[f['path']]
 986
 987     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 988         epoch = details["time"]
 989         author = details["user"]
 990
 991         if self.verbose:
 992             print "commit into %s" % branch
 993
 994         # start with reading files; if that fails, we should not
 995         # create a commit.
 996         new_files = []
 997         for f in files:
 998             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 999                 new_files.append (f)
1000             else:
1001                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1002         files = new_files
1003         self.readP4Files(files)
1004
1005
1006
1007
1008         self.gitStream.write("commit %s\n" % branch)
1009 #        gitStream.write("mark :%s\n" % details["change"])
1010         self.committedChanges.add(int(details["change"]))
1011         committer = ""
1012         if author not in self.users:
1013             self.getUserMapFromPerforceServer()
1014         if author in self.users:
1015             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1016         else:
1017             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1018
1019         self.gitStream.write("committer %s\n" % committer)
1020
1021         self.gitStream.write("data <<EOT\n")
1022         self.gitStream.write(details["desc"])
1023         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1024                              % (','.join (branchPrefixes), details["change"]))
1025         if len(details['options']) > 0:
1026             self.gitStream.write(": options = %s" % details['options'])
1027         self.gitStream.write("]\nEOT\n\n")
1028
1029         if len(parent) > 0:
1030             if self.verbose:
1031                 print "parent %s" % parent
1032             self.gitStream.write("from %s\n" % parent)
1033
1034         for file in files:
1035             if file["type"] == "apple":
1036                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1037                 continue
1038
1039             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1040             if file["action"] == "delete":
1041                 self.gitStream.write("D %s\n" % relPath)
1042             else:
1043                 data = file['data']
1044
1045                 mode = "644"
1046                 if isP4Exec(file["type"]):
1047                     mode = "755"
1048                 elif file["type"] == "symlink":
1049                     mode = "120000"
1050                     # p4 print on a symlink contains "target\n", so strip it off
1051                     data = data[:-1]
1052
1053                 if self.isWindows and file["type"].endswith("text"):
1054                     data = data.replace("\r\n", "\n")
1055
1056                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1057                 self.gitStream.write("data %s\n" % len(data))
1058                 self.gitStream.write(data)
1059                 self.gitStream.write("\n")
1060
1061         self.gitStream.write("\n")
1062
1063         change = int(details["change"])
1064
1065         if self.labels.has_key(change):
1066             label = self.labels[change]
1067             labelDetails = label[0]
1068             labelRevisions = label[1]
1069             if self.verbose:
1070                 print "Change %s is labelled %s" % (change, labelDetails)
1071
1072             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1073                                                     for p in branchPrefixes]))
1074
1075             if len(files) == len(labelRevisions):
1076
1077                 cleanedFiles = {}
1078                 for info in files:
1079                     if info["action"] == "delete":
1080                         continue
1081                     cleanedFiles[info["depotFile"]] = info["rev"]
1082
1083                 if cleanedFiles == labelRevisions:
1084                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1085                     self.gitStream.write("from %s\n" % branch)
1086
1087                     owner = labelDetails["Owner"]
1088                     tagger = ""
1089                     if author in self.users:
1090                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1091                     else:
1092                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1093                     self.gitStream.write("tagger %s\n" % tagger)
1094                     self.gitStream.write("data <<EOT\n")
1095                     self.gitStream.write(labelDetails["Description"])
1096                     self.gitStream.write("EOT\n\n")
1097
1098                 else:
1099                     if not self.silent:
1100                         print ("Tag %s does not match with change %s: files do not match."
1101                                % (labelDetails["label"], change))
1102
1103             else:
1104                 if not self.silent:
1105                     print ("Tag %s does not match with change %s: file count is different."
1106                            % (labelDetails["label"], change))
1107
1108     def getUserCacheFilename(self):
1109         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1110         return home + "/.gitp4-usercache.txt"
1111
1112     def getUserMapFromPerforceServer(self):
1113         if self.userMapFromPerforceServer:
1114             return
1115         self.users = {}
1116
1117         for output in p4CmdList("users"):
1118             if not output.has_key("User"):
1119                 continue
1120             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1121
1122
1123         s = ''
1124         for (key, val) in self.users.items():
1125             s += "%s\t%s\n" % (key, val)
1126
1127         open(self.getUserCacheFilename(), "wb").write(s)
1128         self.userMapFromPerforceServer = True
1129
1130     def loadUserMapFromCache(self):
1131         self.users = {}
1132         self.userMapFromPerforceServer = False
1133         try:
1134             cache = open(self.getUserCacheFilename(), "rb")
1135             lines = cache.readlines()
1136             cache.close()
1137             for line in lines:
1138                 entry = line.strip().split("\t")
1139                 self.users[entry[0]] = entry[1]
1140         except IOError:
1141             self.getUserMapFromPerforceServer()
1142
1143     def getLabels(self):
1144         self.labels = {}
1145
1146         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1147         if len(l) > 0 and not self.silent:
1148             print "Finding files belonging to labels in %s" % `self.depotPaths`
1149
1150         for output in l:
1151             label = output["label"]
1152             revisions = {}
1153             newestChange = 0
1154             if self.verbose:
1155                 print "Querying files for label %s" % label
1156             for file in p4CmdList("files "
1157                                   +  ' '.join (["%s...@%s" % (p, label)
1158                                                 for p in self.depotPaths])):
1159                 revisions[file["depotFile"]] = file["rev"]
1160                 change = int(file["change"])
1161                 if change > newestChange:
1162                     newestChange = change
1163
1164             self.labels[newestChange] = [output, revisions]
1165
1166         if self.verbose:
1167             print "Label changes: %s" % self.labels.keys()
1168
1169     def guessProjectName(self):
1170         for p in self.depotPaths:
1171             if p.endswith("/"):
1172                 p = p[:-1]
1173             p = p[p.strip().rfind("/") + 1:]
1174             if not p.endswith("/"):
1175                p += "/"
1176             return p
1177
1178     def getBranchMapping(self):
1179         lostAndFoundBranches = set()
1180
1181         for info in p4CmdList("branches"):
1182             details = p4Cmd("branch -o %s" % info["branch"])
1183             viewIdx = 0
1184             while details.has_key("View%s" % viewIdx):
1185                 paths = details["View%s" % viewIdx].split(" ")
1186                 viewIdx = viewIdx + 1
1187                 # require standard //depot/foo/... //depot/bar/... mapping
1188                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1189                     continue
1190                 source = paths[0]
1191                 destination = paths[1]
1192                 ## HACK
1193                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1194                     source = source[len(self.depotPaths[0]):-4]
1195                     destination = destination[len(self.depotPaths[0]):-4]
1196
1197                     if destination in self.knownBranches:
1198                         if not self.silent:
1199                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1200                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1201                         continue
1202
1203                     self.knownBranches[destination] = source
1204
1205                     lostAndFoundBranches.discard(destination)
1206
1207                     if source not in self.knownBranches:
1208                         lostAndFoundBranches.add(source)
1209
1210
1211         for branch in lostAndFoundBranches:
1212             self.knownBranches[branch] = branch
1213
1214     def getBranchMappingFromGitBranches(self):
1215         branches = p4BranchesInGit(self.importIntoRemotes)
1216         for branch in branches.keys():
1217             if branch == "master":
1218                 branch = "main"
1219             else:
1220                 branch = branch[len(self.projectName):]
1221             self.knownBranches[branch] = branch
1222
1223     def listExistingP4GitBranches(self):
1224         # branches holds mapping from name to commit
1225         branches = p4BranchesInGit(self.importIntoRemotes)
1226         self.p4BranchesInGit = branches.keys()
1227         for branch in branches.keys():
1228             self.initialParents[self.refPrefix + branch] = branches[branch]
1229
1230     def updateOptionDict(self, d):
1231         option_keys = {}
1232         if self.keepRepoPath:
1233             option_keys['keepRepoPath'] = 1
1234
1235         d["options"] = ' '.join(sorted(option_keys.keys()))
1236
1237     def readOptions(self, d):
1238         self.keepRepoPath = (d.has_key('options')
1239                              and ('keepRepoPath' in d['options']))
1240
1241     def gitRefForBranch(self, branch):
1242         if branch == "main":
1243             return self.refPrefix + "master"
1244
1245         if len(branch) <= 0:
1246             return branch
1247
1248         return self.refPrefix + self.projectName + branch
1249
1250     def gitCommitByP4Change(self, ref, change):
1251         if self.verbose:
1252             print "looking in ref " + ref + " for change %s using bisect..." % change
1253
1254         earliestCommit = ""
1255         latestCommit = parseRevision(ref)
1256
1257         while True:
1258             if self.verbose:
1259                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1260             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1261             if len(next) == 0:
1262                 if self.verbose:
1263                     print "argh"
1264                 return ""
1265             log = extractLogMessageFromGitCommit(next)
1266             settings = extractSettingsGitLog(log)
1267             currentChange = int(settings['change'])
1268             if self.verbose:
1269                 print "current change %s" % currentChange
1270
1271             if currentChange == change:
1272                 if self.verbose:
1273                     print "found %s" % next
1274                 return next
1275
1276             if currentChange < change:
1277                 earliestCommit = "^%s" % next
1278             else:
1279                 latestCommit = "%s" % next
1280
1281         return ""
1282
1283     def importNewBranch(self, branch, maxChange):
1284         # make fast-import flush all changes to disk and update the refs using the checkpoint
1285         # command so that we can try to find the branch parent in the git history
1286         self.gitStream.write("checkpoint\n\n");
1287         self.gitStream.flush();
1288         branchPrefix = self.depotPaths[0] + branch + "/"
1289         range = "@1,%s" % maxChange
1290         #print "prefix" + branchPrefix
1291         changes = p4ChangesForPaths([branchPrefix], range)
1292         if len(changes) <= 0:
1293             return False
1294         firstChange = changes[0]
1295         #print "first change in branch: %s" % firstChange
1296         sourceBranch = self.knownBranches[branch]
1297         sourceDepotPath = self.depotPaths[0] + sourceBranch
1298         sourceRef = self.gitRefForBranch(sourceBranch)
1299         #print "source " + sourceBranch
1300
1301         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1302         #print "branch parent: %s" % branchParentChange
1303         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1304         if len(gitParent) > 0:
1305             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1306             #print "parent git commit: %s" % gitParent
1307
1308         self.importChanges(changes)
1309         return True
1310
1311     def importChanges(self, changes):
1312         cnt = 1
1313         for change in changes:
1314             description = p4Cmd("describe %s" % change)
1315             self.updateOptionDict(description)
1316
1317             if not self.silent:
1318                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1319                 sys.stdout.flush()
1320             cnt = cnt + 1
1321
1322             try:
1323                 if self.detectBranches:
1324                     branches = self.splitFilesIntoBranches(description)
1325                     for branch in branches.keys():
1326                         ## HACK  --hwn
1327                         branchPrefix = self.depotPaths[0] + branch + "/"
1328
1329                         parent = ""
1330
1331                         filesForCommit = branches[branch]
1332
1333                         if self.verbose:
1334                             print "branch is %s" % branch
1335
1336                         self.updatedBranches.add(branch)
1337
1338                         if branch not in self.createdBranches:
1339                             self.createdBranches.add(branch)
1340                             parent = self.knownBranches[branch]
1341                             if parent == branch:
1342                                 parent = ""
1343                             else:
1344                                 fullBranch = self.projectName + branch
1345                                 if fullBranch not in self.p4BranchesInGit:
1346                                     if not self.silent:
1347                                         print("\n    Importing new branch %s" % fullBranch);
1348                                     if self.importNewBranch(branch, change - 1):
1349                                         parent = ""
1350                                         self.p4BranchesInGit.append(fullBranch)
1351                                     if not self.silent:
1352                                         print("\n    Resuming with change %s" % change);
1353
1354                                 if self.verbose:
1355                                     print "parent determined through known branches: %s" % parent
1356
1357                         branch = self.gitRefForBranch(branch)
1358                         parent = self.gitRefForBranch(parent)
1359
1360                         if self.verbose:
1361                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1362
1363                         if len(parent) == 0 and branch in self.initialParents:
1364                             parent = self.initialParents[branch]
1365                             del self.initialParents[branch]
1366
1367                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1368                 else:
1369                     files = self.extractFilesFromCommit(description)
1370                     self.commit(description, files, self.branch, self.depotPaths,
1371                                 self.initialParent)
1372                     self.initialParent = ""
1373             except IOError:
1374                 print self.gitError.read()
1375                 sys.exit(1)
1376
1377     def importHeadRevision(self, revision):
1378         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1379
1380         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1381         details["desc"] = ("Initial import of %s from the state at revision %s"
1382                            % (' '.join(self.depotPaths), revision))
1383         details["change"] = revision
1384         newestRevision = 0
1385
1386         fileCnt = 0
1387         for info in p4CmdList("files "
1388                               +  ' '.join(["%s...%s"
1389                                            % (p, revision)
1390                                            for p in self.depotPaths])):
1391
1392             if info['code'] == 'error':
1393                 sys.stderr.write("p4 returned an error: %s\n"
1394                                  % info['data'])
1395                 sys.exit(1)
1396
1397
1398             change = int(info["change"])
1399             if change > newestRevision:
1400                 newestRevision = change
1401
1402             if info["action"] == "delete":
1403                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1404                 #fileCnt = fileCnt + 1
1405                 continue
1406
1407             for prop in ["depotFile", "rev", "action", "type" ]:
1408                 details["%s%s" % (prop, fileCnt)] = info[prop]
1409
1410             fileCnt = fileCnt + 1
1411
1412         details["change"] = newestRevision
1413         self.updateOptionDict(details)
1414         try:
1415             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1416         except IOError:
1417             print "IO error with git fast-import. Is your git version recent enough?"
1418             print self.gitError.read()
1419
1420
1421     def run(self, args):
1422         self.depotPaths = []
1423         self.changeRange = ""
1424         self.initialParent = ""
1425         self.previousDepotPaths = []
1426
1427         # map from branch depot path to parent branch
1428         self.knownBranches = {}
1429         self.initialParents = {}
1430         self.hasOrigin = originP4BranchesExist()
1431         if not self.syncWithOrigin:
1432             self.hasOrigin = False
1433
1434         if self.importIntoRemotes:
1435             self.refPrefix = "refs/remotes/p4/"
1436         else:
1437             self.refPrefix = "refs/heads/p4/"
1438
1439         if self.syncWithOrigin and self.hasOrigin:
1440             if not self.silent:
1441                 print "Syncing with origin first by calling git fetch origin"
1442             system("git fetch origin")
1443
1444         if len(self.branch) == 0:
1445             self.branch = self.refPrefix + "master"
1446             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1447                 system("git update-ref %s refs/heads/p4" % self.branch)
1448                 system("git branch -D p4");
1449             # create it /after/ importing, when master exists
1450             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1451                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1452
1453         # TODO: should always look at previous commits,
1454         # merge with previous imports, if possible.
1455         if args == []:
1456             if self.hasOrigin:
1457                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1458             self.listExistingP4GitBranches()
1459
1460             if len(self.p4BranchesInGit) > 1:
1461                 if not self.silent:
1462                     print "Importing from/into multiple branches"
1463                 self.detectBranches = True
1464
1465             if self.verbose:
1466                 print "branches: %s" % self.p4BranchesInGit
1467
1468             p4Change = 0
1469             for branch in self.p4BranchesInGit:
1470                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1471
1472                 settings = extractSettingsGitLog(logMsg)
1473
1474                 self.readOptions(settings)
1475                 if (settings.has_key('depot-paths')
1476                     and settings.has_key ('change')):
1477                     change = int(settings['change']) + 1
1478                     p4Change = max(p4Change, change)
1479
1480                     depotPaths = sorted(settings['depot-paths'])
1481                     if self.previousDepotPaths == []:
1482                         self.previousDepotPaths = depotPaths
1483                     else:
1484                         paths = []
1485                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1486                             for i in range(0, min(len(cur), len(prev))):
1487                                 if cur[i] <> prev[i]:
1488                                     i = i - 1
1489                                     break
1490
1491                             paths.append (cur[:i + 1])
1492
1493                         self.previousDepotPaths = paths
1494
1495             if p4Change > 0:
1496                 self.depotPaths = sorted(self.previousDepotPaths)
1497                 self.changeRange = "@%s,#head" % p4Change
1498                 if not self.detectBranches:
1499                     self.initialParent = parseRevision(self.branch)
1500                 if not self.silent and not self.detectBranches:
1501                     print "Performing incremental import into %s git branch" % self.branch
1502
1503         if not self.branch.startswith("refs/"):
1504             self.branch = "refs/heads/" + self.branch
1505
1506         if len(args) == 0 and self.depotPaths:
1507             if not self.silent:
1508                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1509         else:
1510             if self.depotPaths and self.depotPaths != args:
1511                 print ("previous import used depot path %s and now %s was specified. "
1512                        "This doesn't work!" % (' '.join (self.depotPaths),
1513                                                ' '.join (args)))
1514                 sys.exit(1)
1515
1516             self.depotPaths = sorted(args)
1517
1518         revision = ""
1519         self.users = {}
1520
1521         newPaths = []
1522         for p in self.depotPaths:
1523             if p.find("@") != -1:
1524                 atIdx = p.index("@")
1525                 self.changeRange = p[atIdx:]
1526                 if self.changeRange == "@all":
1527                     self.changeRange = ""
1528                 elif ',' not in self.changeRange:
1529                     revision = self.changeRange
1530                     self.changeRange = ""
1531                 p = p[:atIdx]
1532             elif p.find("#") != -1:
1533                 hashIdx = p.index("#")
1534                 revision = p[hashIdx:]
1535                 p = p[:hashIdx]
1536             elif self.previousDepotPaths == []:
1537                 revision = "#head"
1538
1539             p = re.sub ("\.\.\.$", "", p)
1540             if not p.endswith("/"):
1541                 p += "/"
1542
1543             newPaths.append(p)
1544
1545         self.depotPaths = newPaths
1546
1547
1548         self.loadUserMapFromCache()
1549         self.labels = {}
1550         if self.detectLabels:
1551             self.getLabels();
1552
1553         if self.detectBranches:
1554             ## FIXME - what's a P4 projectName ?
1555             self.projectName = self.guessProjectName()
1556
1557             if self.hasOrigin:
1558                 self.getBranchMappingFromGitBranches()
1559             else:
1560                 self.getBranchMapping()
1561             if self.verbose:
1562                 print "p4-git branches: %s" % self.p4BranchesInGit
1563                 print "initial parents: %s" % self.initialParents
1564             for b in self.p4BranchesInGit:
1565                 if b != "master":
1566
1567                     ## FIXME
1568                     b = b[len(self.projectName):]
1569                 self.createdBranches.add(b)
1570
1571         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1572
1573         importProcess = subprocess.Popen(["git", "fast-import"],
1574                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1575                                          stderr=subprocess.PIPE);
1576         self.gitOutput = importProcess.stdout
1577         self.gitStream = importProcess.stdin
1578         self.gitError = importProcess.stderr
1579
1580         if revision:
1581             self.importHeadRevision(revision)
1582         else:
1583             changes = []
1584
1585             if len(self.changesFile) > 0:
1586                 output = open(self.changesFile).readlines()
1587                 changeSet = Set()
1588                 for line in output:
1589                     changeSet.add(int(line))
1590
1591                 for change in changeSet:
1592                     changes.append(change)
1593
1594                 changes.sort()
1595             else:
1596                 if self.verbose:
1597                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1598                                                               self.changeRange)
1599                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1600
1601                 if len(self.maxChanges) > 0:
1602                     changes = changes[:min(int(self.maxChanges), len(changes))]
1603
1604             if len(changes) == 0:
1605                 if not self.silent:
1606                     print "No changes to import!"
1607                 return True
1608
1609             if not self.silent and not self.detectBranches:
1610                 print "Import destination: %s" % self.branch
1611
1612             self.updatedBranches = set()
1613
1614             self.importChanges(changes)
1615
1616             if not self.silent:
1617                 print ""
1618                 if len(self.updatedBranches) > 0:
1619                     sys.stdout.write("Updated branches: ")
1620                     for b in self.updatedBranches:
1621                         sys.stdout.write("%s " % b)
1622                     sys.stdout.write("\n")
1623
1624         self.gitStream.close()
1625         if importProcess.wait() != 0:
1626             die("fast-import failed: %s" % self.gitError.read())
1627         self.gitOutput.close()
1628         self.gitError.close()
1629
1630         return True
1631
1632 class P4Rebase(Command):
1633     def __init__(self):
1634         Command.__init__(self)
1635         self.options = [ ]
1636         self.description = ("Fetches the latest revision from perforce and "
1637                             + "rebases the current work (branch) against it")
1638         self.verbose = False
1639
1640     def run(self, args):
1641         sync = P4Sync()
1642         sync.run([])
1643
1644         return self.rebase()
1645
1646     def rebase(self):
1647         [upstream, settings] = findUpstreamBranchPoint()
1648         if len(upstream) == 0:
1649             die("Cannot find upstream branchpoint for rebase")
1650
1651         # the branchpoint may be p4/foo~3, so strip off the parent
1652         upstream = re.sub("~[0-9]+$", "", upstream)
1653
1654         print "Rebasing the current branch onto %s" % upstream
1655         oldHead = read_pipe("git rev-parse HEAD").strip()
1656         system("git rebase %s" % upstream)
1657         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1658         return True
1659
1660 class P4Clone(P4Sync):
1661     def __init__(self):
1662         P4Sync.__init__(self)
1663         self.description = "Creates a new git repository and imports from Perforce into it"
1664         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1665         self.options.append(
1666             optparse.make_option("--destination", dest="cloneDestination",
1667                                  action='store', default=None,
1668                                  help="where to leave result of the clone"))
1669         self.cloneDestination = None
1670         self.needsGit = False
1671
1672     def defaultDestination(self, args):
1673         ## TODO: use common prefix of args?
1674         depotPath = args[0]
1675         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1676         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1677         depotDir = re.sub(r"\.\.\.$,", "", depotDir)
1678         depotDir = re.sub(r"/$", "", depotDir)
1679         return os.path.split(depotDir)[1]
1680
1681     def run(self, args):
1682         if len(args) < 1:
1683             return False
1684
1685         if self.keepRepoPath and not self.cloneDestination:
1686             sys.stderr.write("Must specify destination for --keep-path\n")
1687             sys.exit(1)
1688
1689         depotPaths = args
1690
1691         if not self.cloneDestination and len(depotPaths) > 1:
1692             self.cloneDestination = depotPaths[-1]
1693             depotPaths = depotPaths[:-1]
1694
1695         for p in depotPaths:
1696             if not p.startswith("//"):
1697                 return False
1698
1699         if not self.cloneDestination:
1700             self.cloneDestination = self.defaultDestination(args)
1701
1702         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1703         if not os.path.exists(self.cloneDestination):
1704             os.makedirs(self.cloneDestination)
1705         os.chdir(self.cloneDestination)
1706         system("git init")
1707         self.gitdir = os.getcwd() + "/.git"
1708         if not P4Sync.run(self, depotPaths):
1709             return False
1710         if self.branch != "master":
1711             if gitBranchExists("refs/remotes/p4/master"):
1712                 system("git branch master refs/remotes/p4/master")
1713                 system("git checkout -f")
1714             else:
1715                 print "Could not detect main branch. No checkout/master branch created."
1716
1717         return True
1718
1719 class P4Branches(Command):
1720     def __init__(self):
1721         Command.__init__(self)
1722         self.options = [ ]
1723         self.description = ("Shows the git branches that hold imports and their "
1724                             + "corresponding perforce depot paths")
1725         self.verbose = False
1726
1727     def run(self, args):
1728         if originP4BranchesExist():
1729             createOrUpdateBranchesFromOrigin()
1730
1731         cmdline = "git rev-parse --symbolic "
1732         cmdline += " --remotes"
1733
1734         for line in read_pipe_lines(cmdline):
1735             line = line.strip()
1736
1737             if not line.startswith('p4/') or line == "p4/HEAD":
1738                 continue
1739             branch = line
1740
1741             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1742             settings = extractSettingsGitLog(log)
1743
1744             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1745         return True
1746
1747 class HelpFormatter(optparse.IndentedHelpFormatter):
1748     def __init__(self):
1749         optparse.IndentedHelpFormatter.__init__(self)
1750
1751     def format_description(self, description):
1752         if description:
1753             return description + "\n"
1754         else:
1755             return ""
1756
1757 def printUsage(commands):
1758     print "usage: %s <command> [options]" % sys.argv[0]
1759     print ""
1760     print "valid commands: %s" % ", ".join(commands)
1761     print ""
1762     print "Try %s <command> --help for command specific help." % sys.argv[0]
1763     print ""
1764
1765 commands = {
1766     "debug" : P4Debug,
1767     "submit" : P4Submit,
1768     "commit" : P4Submit,
1769     "sync" : P4Sync,
1770     "rebase" : P4Rebase,
1771     "clone" : P4Clone,
1772     "rollback" : P4RollBack,
1773     "branches" : P4Branches
1774 }
1775
1776
1777 def main():
1778     if len(sys.argv[1:]) == 0:
1779         printUsage(commands.keys())
1780         sys.exit(2)
1781
1782     cmd = ""
1783     cmdName = sys.argv[1]
1784     try:
1785         klass = commands[cmdName]
1786         cmd = klass()
1787     except KeyError:
1788         print "unknown command %s" % cmdName
1789         print ""
1790         printUsage(commands.keys())
1791         sys.exit(2)
1792
1793     options = cmd.options
1794     cmd.gitdir = os.environ.get("GIT_DIR", None)
1795
1796     args = sys.argv[2:]
1797
1798     if len(options) > 0:
1799         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1800
1801         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1802                                        options,
1803                                        description = cmd.description,
1804                                        formatter = HelpFormatter())
1805
1806         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1807     global verbose
1808     verbose = cmd.verbose
1809     if cmd.needsGit:
1810         if cmd.gitdir == None:
1811             cmd.gitdir = os.path.abspath(".git")
1812             if not isValidGitDir(cmd.gitdir):
1813                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1814                 if os.path.exists(cmd.gitdir):
1815                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1816                     if len(cdup) > 0:
1817                         os.chdir(cdup);
1818
1819         if not isValidGitDir(cmd.gitdir):
1820             if isValidGitDir(cmd.gitdir + "/.git"):
1821                 cmd.gitdir += "/.git"
1822             else:
1823                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1824
1825         os.environ["GIT_DIR"] = cmd.gitdir
1826
1827     if not cmd.run(args):
1828         parser.print_help()
1829
1830
1831 if __name__ == '__main__':
1832     main()