contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 468                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 469                 optparse.make_option("--origin", dest="origin"),
 470                 optparse.make_option("--reset", action="store_true", dest="reset"),
 471                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 472         ]
 473         self.description = "Submit changes from git to the perforce depot."
 474         self.usage += " [name of git branch to submit into perforce depot]"
 475         self.firstTime = True
 476         self.reset = False
 477         self.interactive = True
 478         self.firstTime = True
 479         self.origin = ""
 480         self.detectRename = False
 481         self.verbose = False
 482         self.isWindows = (platform.system() == "Windows")
 483
 484     def check(self):
 485         if len(p4CmdList("opened ...")) > 0:
 486             die("You have files opened with perforce! Close them before starting the sync.")
 487
 488     def start(self):
 489         if len(self.config) > 0 and not self.reset:
 490             die("Cannot start sync. Previous sync config found at %s\n"
 491                 "If you want to start submitting again from scratch "
 492                 "maybe you want to call git-p4 submit --reset" % self.configFile)
 493
 494         commits = []
 495         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 496             commits.append(line.strip())
 497         commits.reverse()
 498
 499         self.config["commits"] = commits
 500
 501     # replaces everything between 'Description:' and the next P4 submit template field with the
 502     # commit message
 503     def prepareLogMessage(self, template, message):
 504         result = ""
 505
 506         inDescriptionSection = False
 507
 508         for line in template.split("\n"):
 509             if line.startswith("#"):
 510                 result += line + "\n"
 511                 continue
 512
 513             if inDescriptionSection:
 514                 if line.startswith("Files:"):
 515                     inDescriptionSection = False
 516                 else:
 517                     continue
 518             else:
 519                 if line.startswith("Description:"):
 520                     inDescriptionSection = True
 521                     line += "\n"
 522                     for messageLine in message.split("\n"):
 523                         line += "\t" + messageLine + "\n"
 524
 525             result += line + "\n"
 526
 527         return result
 528
 529     def prepareSubmitTemplate(self):
 530         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 531         template = ""
 532         inFilesSection = False
 533         for line in read_pipe_lines("p4 change -o"):
 534             if inFilesSection:
 535                 if line.startswith("\t"):
 536                     # path starts and ends with a tab
 537                     path = line[1:]
 538                     lastTab = path.rfind("\t")
 539                     if lastTab != -1:
 540                         path = path[:lastTab]
 541                         if not path.startswith(self.depotPath):
 542                             continue
 543                 else:
 544                     inFilesSection = False
 545             else:
 546                 if line.startswith("Files:"):
 547                     inFilesSection = True
 548
 549             template += line
 550
 551         return template
 552
 553     def applyCommit(self, id):
 554         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 555         diffOpts = ("", "-M")[self.detectRename]
 556         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 557         filesToAdd = set()
 558         filesToDelete = set()
 559         editedFiles = set()
 560         filesToChangeExecBit = {}
 561         for line in diff:
 562             diff = parseDiffTreeEntry(line)
 563             modifier = diff['status']
 564             path = diff['src']
 565             if modifier == "M":
 566                 system("p4 edit \"%s\"" % path)
 567                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 568                     filesToChangeExecBit[path] = diff['dst_mode']
 569                 editedFiles.add(path)
 570             elif modifier == "A":
 571                 filesToAdd.add(path)
 572                 filesToChangeExecBit[path] = diff['dst_mode']
 573                 if path in filesToDelete:
 574                     filesToDelete.remove(path)
 575             elif modifier == "D":
 576                 filesToDelete.add(path)
 577                 if path in filesToAdd:
 578                     filesToAdd.remove(path)
 579             elif modifier == "R":
 580                 src, dest = diff['src'], diff['dst']
 581                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 582                 system("p4 edit \"%s\"" % (dest))
 583                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 584                     filesToChangeExecBit[dest] = diff['dst_mode']
 585                 os.unlink(dest)
 586                 editedFiles.add(dest)
 587                 filesToDelete.add(src)
 588             else:
 589                 die("unknown modifier %s for %s" % (modifier, path))
 590
 591         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 592         patchcmd = diffcmd + " | git apply "
 593         tryPatchCmd = patchcmd + "--check -"
 594         applyPatchCmd = patchcmd + "--check --apply -"
 595
 596         if os.system(tryPatchCmd) != 0:
 597             print "Unfortunately applying the change failed!"
 598             print "What do you want to do?"
 599             response = "x"
 600             while response != "s" and response != "a" and response != "w":
 601                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 602                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 603             if response == "s":
 604                 print "Skipping! Good luck with the next patches..."
 605                 for f in editedFiles:
 606                     system("p4 revert \"%s\"" % f);
 607                 for f in filesToAdd:
 608                     system("rm %s" %f)
 609                 return
 610             elif response == "a":
 611                 os.system(applyPatchCmd)
 612                 if len(filesToAdd) > 0:
 613                     print "You may also want to call p4 add on the following files:"
 614                     print " ".join(filesToAdd)
 615                 if len(filesToDelete):
 616                     print "The following files should be scheduled for deletion with p4 delete:"
 617                     print " ".join(filesToDelete)
 618                 die("Please resolve and submit the conflict manually and "
 619                     + "continue afterwards with git-p4 submit --continue")
 620             elif response == "w":
 621                 system(diffcmd + " > patch.txt")
 622                 print "Patch saved to patch.txt in %s !" % self.clientPath
 623                 die("Please resolve and submit the conflict manually and "
 624                     "continue afterwards with git-p4 submit --continue")
 625
 626         system(applyPatchCmd)
 627
 628         for f in filesToAdd:
 629             system("p4 add \"%s\"" % f)
 630         for f in filesToDelete:
 631             system("p4 revert \"%s\"" % f)
 632             system("p4 delete \"%s\"" % f)
 633
 634         # Set/clear executable bits
 635         for f in filesToChangeExecBit.keys():
 636             mode = filesToChangeExecBit[f]
 637             setP4ExecBit(f, mode)
 638
 639         logMessage = extractLogMessageFromGitCommit(id)
 640         if self.isWindows:
 641             logMessage = logMessage.replace("\n", "\r\n")
 642         logMessage = logMessage.strip()
 643
 644         template = self.prepareSubmitTemplate()
 645
 646         if self.interactive:
 647             submitTemplate = self.prepareLogMessage(template, logMessage)
 648             diff = read_pipe("p4 diff -du ...")
 649
 650             for newFile in filesToAdd:
 651                 diff += "==== new file ====\n"
 652                 diff += "--- /dev/null\n"
 653                 diff += "+++ %s\n" % newFile
 654                 f = open(newFile, "r")
 655                 for line in f.readlines():
 656                     diff += "+" + line
 657                 f.close()
 658
 659             separatorLine = "######## everything below this line is just the diff #######"
 660             if platform.system() == "Windows":
 661                 separatorLine += "\r"
 662             separatorLine += "\n"
 663
 664             [handle, fileName] = tempfile.mkstemp()
 665             tmpFile = os.fdopen(handle, "w+")
 666             tmpFile.write(submitTemplate + separatorLine + diff)
 667             tmpFile.close()
 668             defaultEditor = "vi"
 669             if platform.system() == "Windows":
 670                 defaultEditor = "notepad"
 671             editor = os.environ.get("EDITOR", defaultEditor);
 672             system(editor + " " + fileName)
 673             tmpFile = open(fileName, "rb")
 674             message = tmpFile.read()
 675             tmpFile.close()
 676             os.remove(fileName)
 677             submitTemplate = message[:message.index(separatorLine)]
 678             if self.isWindows:
 679                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 680
 681             write_pipe("p4 submit -i", submitTemplate)
 682         else:
 683             fileName = "submit.txt"
 684             file = open(fileName, "w+")
 685             file.write(self.prepareLogMessage(template, logMessage))
 686             file.close()
 687             print ("Perforce submit template written as %s. "
 688                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 689                    % (fileName, fileName))
 690
 691     def run(self, args):
 692         if len(args) == 0:
 693             self.master = currentGitBranch()
 694             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 695                 die("Detecting current git branch failed!")
 696         elif len(args) == 1:
 697             self.master = args[0]
 698         else:
 699             return False
 700
 701         [upstream, settings] = findUpstreamBranchPoint()
 702         self.depotPath = settings['depot-paths'][0]
 703         if len(self.origin) == 0:
 704             self.origin = upstream
 705
 706         if self.verbose:
 707             print "Origin branch is " + self.origin
 708
 709         if len(self.depotPath) == 0:
 710             print "Internal error: cannot locate perforce depot path from existing branches"
 711             sys.exit(128)
 712
 713         self.clientPath = p4Where(self.depotPath)
 714
 715         if len(self.clientPath) == 0:
 716             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 717             sys.exit(128)
 718
 719         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 720         self.oldWorkingDirectory = os.getcwd()
 721
 722         os.chdir(self.clientPath)
 723         print "Syncronizing p4 checkout..."
 724         system("p4 sync ...")
 725
 726         if self.reset:
 727             self.firstTime = True
 728
 729         self.check()
 730         self.configFile = self.gitdir + "/p4-git-sync.cfg"
 731         self.config = shelve.open(self.configFile, writeback=True)
 732
 733         if self.firstTime:
 734             self.start()
 735
 736         commits = self.config.get("commits", [])
 737
 738         while len(commits) > 0:
 739             self.firstTime = False
 740             commit = commits[0]
 741             commits = commits[1:]
 742             self.config["commits"] = commits
 743             self.applyCommit(commit)
 744             if not self.interactive:
 745                 break
 746
 747         self.config.close()
 748
 749         if len(commits) == 0:
 750             if self.firstTime:
 751                 print "No changes found to apply between %s and current HEAD" % self.origin
 752             else:
 753                 print "All changes applied!"
 754                 os.chdir(self.oldWorkingDirectory)
 755
 756                 sync = P4Sync()
 757                 sync.run([])
 758
 759                 rebase = P4Rebase()
 760                 rebase.rebase()
 761             os.remove(self.configFile)
 762
 763         return True
 764
 765 class P4Sync(Command):
 766     def __init__(self):
 767         Command.__init__(self)
 768         self.options = [
 769                 optparse.make_option("--branch", dest="branch"),
 770                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 771                 optparse.make_option("--changesfile", dest="changesFile"),
 772                 optparse.make_option("--silent", dest="silent", action="store_true"),
 773                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 774                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 775                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 776                                      help="Import into refs/heads/ , not refs/remotes"),
 777                 optparse.make_option("--max-changes", dest="maxChanges"),
 778                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 779                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import")
 780         ]
 781         self.description = """Imports from Perforce into a git repository.\n
 782     example:
 783     //depot/my/project/ -- to import the current head
 784     //depot/my/project/@all -- to import everything
 785     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 786
 787     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 788
 789         self.usage += " //depot/path[@revRange]"
 790         self.silent = False
 791         self.createdBranches = Set()
 792         self.committedChanges = Set()
 793         self.branch = ""
 794         self.detectBranches = False
 795         self.detectLabels = False
 796         self.changesFile = ""
 797         self.syncWithOrigin = True
 798         self.verbose = False
 799         self.importIntoRemotes = True
 800         self.maxChanges = ""
 801         self.isWindows = (platform.system() == "Windows")
 802         self.keepRepoPath = False
 803         self.depotPaths = None
 804         self.p4BranchesInGit = []
 805         self.cloneExclude = []
 806
 807         if gitConfig("git-p4.syncFromOrigin") == "false":
 808             self.syncWithOrigin = False
 809
 810     def extractFilesFromCommit(self, commit):
 811         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 812                              for path in self.cloneExclude]
 813         files = []
 814         fnum = 0
 815         while commit.has_key("depotFile%s" % fnum):
 816             path =  commit["depotFile%s" % fnum]
 817
 818             if [p for p in self.cloneExclude
 819                 if path.startswith (p)]:
 820                 found = False
 821             else:
 822                 found = [p for p in self.depotPaths
 823                          if path.startswith (p)]
 824             if not found:
 825                 fnum = fnum + 1
 826                 continue
 827
 828             file = {}
 829             file["path"] = path
 830             file["rev"] = commit["rev%s" % fnum]
 831             file["action"] = commit["action%s" % fnum]
 832             file["type"] = commit["type%s" % fnum]
 833             files.append(file)
 834             fnum = fnum + 1
 835         return files
 836
 837     def stripRepoPath(self, path, prefixes):
 838         if self.keepRepoPath:
 839             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 840
 841         for p in prefixes:
 842             if path.startswith(p):
 843                 path = path[len(p):]
 844
 845         return path
 846
 847     def splitFilesIntoBranches(self, commit):
 848         branches = {}
 849         fnum = 0
 850         while commit.has_key("depotFile%s" % fnum):
 851             path =  commit["depotFile%s" % fnum]
 852             found = [p for p in self.depotPaths
 853                      if path.startswith (p)]
 854             if not found:
 855                 fnum = fnum + 1
 856                 continue
 857
 858             file = {}
 859             file["path"] = path
 860             file["rev"] = commit["rev%s" % fnum]
 861             file["action"] = commit["action%s" % fnum]
 862             file["type"] = commit["type%s" % fnum]
 863             fnum = fnum + 1
 864
 865             relPath = self.stripRepoPath(path, self.depotPaths)
 866
 867             for branch in self.knownBranches.keys():
 868
 869                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 870                 if relPath.startswith(branch + "/"):
 871                     if branch not in branches:
 872                         branches[branch] = []
 873                     branches[branch].append(file)
 874                     break
 875
 876         return branches
 877
 878     ## Should move this out, doesn't use SELF.
 879     def readP4Files(self, files):
 880         files = [f for f in files
 881                  if f['action'] != 'delete']
 882
 883         if not files:
 884             return
 885
 886         filedata = p4CmdList('-x - print',
 887                              stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 888                                               for f in files]),
 889                              stdin_mode='w+')
 890         if "p4ExitCode" in filedata[0]:
 891             die("Problems executing p4. Error: [%d]."
 892                 % (filedata[0]['p4ExitCode']));
 893
 894         j = 0;
 895         contents = {}
 896         while j < len(filedata):
 897             stat = filedata[j]
 898             j += 1
 899             text = ''
 900             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 901                 tmp = filedata[j]['data']
 902                 if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 903                     tmp = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', tmp)
 904                 elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 905                     tmp = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', tmp)
 906                 text += tmp
 907                 j += 1
 908
 909
 910             if not stat.has_key('depotFile'):
 911                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 912                 continue
 913
 914             contents[stat['depotFile']] = text
 915
 916         for f in files:
 917             assert not f.has_key('data')
 918             f['data'] = contents[f['path']]
 919
 920     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 921         epoch = details["time"]
 922         author = details["user"]
 923
 924         if self.verbose:
 925             print "commit into %s" % branch
 926
 927         # start with reading files; if that fails, we should not
 928         # create a commit.
 929         new_files = []
 930         for f in files:
 931             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 932                 new_files.append (f)
 933             else:
 934                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 935         files = new_files
 936         self.readP4Files(files)
 937
 938
 939
 940
 941         self.gitStream.write("commit %s\n" % branch)
 942 #        gitStream.write("mark :%s\n" % details["change"])
 943         self.committedChanges.add(int(details["change"]))
 944         committer = ""
 945         if author not in self.users:
 946             self.getUserMapFromPerforceServer()
 947         if author in self.users:
 948             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 949         else:
 950             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 951
 952         self.gitStream.write("committer %s\n" % committer)
 953
 954         self.gitStream.write("data <<EOT\n")
 955         self.gitStream.write(details["desc"])
 956         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 957                              % (','.join (branchPrefixes), details["change"]))
 958         if len(details['options']) > 0:
 959             self.gitStream.write(": options = %s" % details['options'])
 960         self.gitStream.write("]\nEOT\n\n")
 961
 962         if len(parent) > 0:
 963             if self.verbose:
 964                 print "parent %s" % parent
 965             self.gitStream.write("from %s\n" % parent)
 966
 967         for file in files:
 968             if file["type"] == "apple":
 969                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 970                 continue
 971
 972             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 973             if file["action"] == "delete":
 974                 self.gitStream.write("D %s\n" % relPath)
 975             else:
 976                 data = file['data']
 977
 978                 mode = "644"
 979                 if isP4Exec(file["type"]):
 980                     mode = "755"
 981                 elif file["type"] == "symlink":
 982                     mode = "120000"
 983                     # p4 print on a symlink contains "target\n", so strip it off
 984                     data = data[:-1]
 985
 986                 if self.isWindows and file["type"].endswith("text"):
 987                     data = data.replace("\r\n", "\n")
 988
 989                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 990                 self.gitStream.write("data %s\n" % len(data))
 991                 self.gitStream.write(data)
 992                 self.gitStream.write("\n")
 993
 994         self.gitStream.write("\n")
 995
 996         change = int(details["change"])
 997
 998         if self.labels.has_key(change):
 999             label = self.labels[change]
1000             labelDetails = label[0]
1001             labelRevisions = label[1]
1002             if self.verbose:
1003                 print "Change %s is labelled %s" % (change, labelDetails)
1004
1005             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1006                                                     for p in branchPrefixes]))
1007
1008             if len(files) == len(labelRevisions):
1009
1010                 cleanedFiles = {}
1011                 for info in files:
1012                     if info["action"] == "delete":
1013                         continue
1014                     cleanedFiles[info["depotFile"]] = info["rev"]
1015
1016                 if cleanedFiles == labelRevisions:
1017                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1018                     self.gitStream.write("from %s\n" % branch)
1019
1020                     owner = labelDetails["Owner"]
1021                     tagger = ""
1022                     if author in self.users:
1023                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1024                     else:
1025                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1026                     self.gitStream.write("tagger %s\n" % tagger)
1027                     self.gitStream.write("data <<EOT\n")
1028                     self.gitStream.write(labelDetails["Description"])
1029                     self.gitStream.write("EOT\n\n")
1030
1031                 else:
1032                     if not self.silent:
1033                         print ("Tag %s does not match with change %s: files do not match."
1034                                % (labelDetails["label"], change))
1035
1036             else:
1037                 if not self.silent:
1038                     print ("Tag %s does not match with change %s: file count is different."
1039                            % (labelDetails["label"], change))
1040
1041     def getUserCacheFilename(self):
1042         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1043         return home + "/.gitp4-usercache.txt"
1044
1045     def getUserMapFromPerforceServer(self):
1046         if self.userMapFromPerforceServer:
1047             return
1048         self.users = {}
1049
1050         for output in p4CmdList("users"):
1051             if not output.has_key("User"):
1052                 continue
1053             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1054
1055
1056         s = ''
1057         for (key, val) in self.users.items():
1058             s += "%s\t%s\n" % (key, val)
1059
1060         open(self.getUserCacheFilename(), "wb").write(s)
1061         self.userMapFromPerforceServer = True
1062
1063     def loadUserMapFromCache(self):
1064         self.users = {}
1065         self.userMapFromPerforceServer = False
1066         try:
1067             cache = open(self.getUserCacheFilename(), "rb")
1068             lines = cache.readlines()
1069             cache.close()
1070             for line in lines:
1071                 entry = line.strip().split("\t")
1072                 self.users[entry[0]] = entry[1]
1073         except IOError:
1074             self.getUserMapFromPerforceServer()
1075
1076     def getLabels(self):
1077         self.labels = {}
1078
1079         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1080         if len(l) > 0 and not self.silent:
1081             print "Finding files belonging to labels in %s" % `self.depotPaths`
1082
1083         for output in l:
1084             label = output["label"]
1085             revisions = {}
1086             newestChange = 0
1087             if self.verbose:
1088                 print "Querying files for label %s" % label
1089             for file in p4CmdList("files "
1090                                   +  ' '.join (["%s...@%s" % (p, label)
1091                                                 for p in self.depotPaths])):
1092                 revisions[file["depotFile"]] = file["rev"]
1093                 change = int(file["change"])
1094                 if change > newestChange:
1095                     newestChange = change
1096
1097             self.labels[newestChange] = [output, revisions]
1098
1099         if self.verbose:
1100             print "Label changes: %s" % self.labels.keys()
1101
1102     def guessProjectName(self):
1103         for p in self.depotPaths:
1104             if p.endswith("/"):
1105                 p = p[:-1]
1106             p = p[p.strip().rfind("/") + 1:]
1107             if not p.endswith("/"):
1108                p += "/"
1109             return p
1110
1111     def getBranchMapping(self):
1112         lostAndFoundBranches = set()
1113
1114         for info in p4CmdList("branches"):
1115             details = p4Cmd("branch -o %s" % info["branch"])
1116             viewIdx = 0
1117             while details.has_key("View%s" % viewIdx):
1118                 paths = details["View%s" % viewIdx].split(" ")
1119                 viewIdx = viewIdx + 1
1120                 # require standard //depot/foo/... //depot/bar/... mapping
1121                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1122                     continue
1123                 source = paths[0]
1124                 destination = paths[1]
1125                 ## HACK
1126                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1127                     source = source[len(self.depotPaths[0]):-4]
1128                     destination = destination[len(self.depotPaths[0]):-4]
1129
1130                     if destination in self.knownBranches:
1131                         if not self.silent:
1132                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1133                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1134                         continue
1135
1136                     self.knownBranches[destination] = source
1137
1138                     lostAndFoundBranches.discard(destination)
1139
1140                     if source not in self.knownBranches:
1141                         lostAndFoundBranches.add(source)
1142
1143
1144         for branch in lostAndFoundBranches:
1145             self.knownBranches[branch] = branch
1146
1147     def getBranchMappingFromGitBranches(self):
1148         branches = p4BranchesInGit(self.importIntoRemotes)
1149         for branch in branches.keys():
1150             if branch == "master":
1151                 branch = "main"
1152             else:
1153                 branch = branch[len(self.projectName):]
1154             self.knownBranches[branch] = branch
1155
1156     def listExistingP4GitBranches(self):
1157         # branches holds mapping from name to commit
1158         branches = p4BranchesInGit(self.importIntoRemotes)
1159         self.p4BranchesInGit = branches.keys()
1160         for branch in branches.keys():
1161             self.initialParents[self.refPrefix + branch] = branches[branch]
1162
1163     def updateOptionDict(self, d):
1164         option_keys = {}
1165         if self.keepRepoPath:
1166             option_keys['keepRepoPath'] = 1
1167
1168         d["options"] = ' '.join(sorted(option_keys.keys()))
1169
1170     def readOptions(self, d):
1171         self.keepRepoPath = (d.has_key('options')
1172                              and ('keepRepoPath' in d['options']))
1173
1174     def gitRefForBranch(self, branch):
1175         if branch == "main":
1176             return self.refPrefix + "master"
1177
1178         if len(branch) <= 0:
1179             return branch
1180
1181         return self.refPrefix + self.projectName + branch
1182
1183     def gitCommitByP4Change(self, ref, change):
1184         if self.verbose:
1185             print "looking in ref " + ref + " for change %s using bisect..." % change
1186
1187         earliestCommit = ""
1188         latestCommit = parseRevision(ref)
1189
1190         while True:
1191             if self.verbose:
1192                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1193             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1194             if len(next) == 0:
1195                 if self.verbose:
1196                     print "argh"
1197                 return ""
1198             log = extractLogMessageFromGitCommit(next)
1199             settings = extractSettingsGitLog(log)
1200             currentChange = int(settings['change'])
1201             if self.verbose:
1202                 print "current change %s" % currentChange
1203
1204             if currentChange == change:
1205                 if self.verbose:
1206                     print "found %s" % next
1207                 return next
1208
1209             if currentChange < change:
1210                 earliestCommit = "^%s" % next
1211             else:
1212                 latestCommit = "%s" % next
1213
1214         return ""
1215
1216     def importNewBranch(self, branch, maxChange):
1217         # make fast-import flush all changes to disk and update the refs using the checkpoint
1218         # command so that we can try to find the branch parent in the git history
1219         self.gitStream.write("checkpoint\n\n");
1220         self.gitStream.flush();
1221         branchPrefix = self.depotPaths[0] + branch + "/"
1222         range = "@1,%s" % maxChange
1223         #print "prefix" + branchPrefix
1224         changes = p4ChangesForPaths([branchPrefix], range)
1225         if len(changes) <= 0:
1226             return False
1227         firstChange = changes[0]
1228         #print "first change in branch: %s" % firstChange
1229         sourceBranch = self.knownBranches[branch]
1230         sourceDepotPath = self.depotPaths[0] + sourceBranch
1231         sourceRef = self.gitRefForBranch(sourceBranch)
1232         #print "source " + sourceBranch
1233
1234         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1235         #print "branch parent: %s" % branchParentChange
1236         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1237         if len(gitParent) > 0:
1238             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1239             #print "parent git commit: %s" % gitParent
1240
1241         self.importChanges(changes)
1242         return True
1243
1244     def importChanges(self, changes):
1245         cnt = 1
1246         for change in changes:
1247             description = p4Cmd("describe %s" % change)
1248             self.updateOptionDict(description)
1249
1250             if not self.silent:
1251                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1252                 sys.stdout.flush()
1253             cnt = cnt + 1
1254
1255             try:
1256                 if self.detectBranches:
1257                     branches = self.splitFilesIntoBranches(description)
1258                     for branch in branches.keys():
1259                         ## HACK  --hwn
1260                         branchPrefix = self.depotPaths[0] + branch + "/"
1261
1262                         parent = ""
1263
1264                         filesForCommit = branches[branch]
1265
1266                         if self.verbose:
1267                             print "branch is %s" % branch
1268
1269                         self.updatedBranches.add(branch)
1270
1271                         if branch not in self.createdBranches:
1272                             self.createdBranches.add(branch)
1273                             parent = self.knownBranches[branch]
1274                             if parent == branch:
1275                                 parent = ""
1276                             else:
1277                                 fullBranch = self.projectName + branch
1278                                 if fullBranch not in self.p4BranchesInGit:
1279                                     if not self.silent:
1280                                         print("\n    Importing new branch %s" % fullBranch);
1281                                     if self.importNewBranch(branch, change - 1):
1282                                         parent = ""
1283                                         self.p4BranchesInGit.append(fullBranch)
1284                                     if not self.silent:
1285                                         print("\n    Resuming with change %s" % change);
1286
1287                                 if self.verbose:
1288                                     print "parent determined through known branches: %s" % parent
1289
1290                         branch = self.gitRefForBranch(branch)
1291                         parent = self.gitRefForBranch(parent)
1292
1293                         if self.verbose:
1294                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1295
1296                         if len(parent) == 0 and branch in self.initialParents:
1297                             parent = self.initialParents[branch]
1298                             del self.initialParents[branch]
1299
1300                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1301                 else:
1302                     files = self.extractFilesFromCommit(description)
1303                     self.commit(description, files, self.branch, self.depotPaths,
1304                                 self.initialParent)
1305                     self.initialParent = ""
1306             except IOError:
1307                 print self.gitError.read()
1308                 sys.exit(1)
1309
1310     def importHeadRevision(self, revision):
1311         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1312
1313         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1314         details["desc"] = ("Initial import of %s from the state at revision %s"
1315                            % (' '.join(self.depotPaths), revision))
1316         details["change"] = revision
1317         newestRevision = 0
1318
1319         fileCnt = 0
1320         for info in p4CmdList("files "
1321                               +  ' '.join(["%s...%s"
1322                                            % (p, revision)
1323                                            for p in self.depotPaths])):
1324
1325             if info['code'] == 'error':
1326                 sys.stderr.write("p4 returned an error: %s\n"
1327                                  % info['data'])
1328                 sys.exit(1)
1329
1330
1331             change = int(info["change"])
1332             if change > newestRevision:
1333                 newestRevision = change
1334
1335             if info["action"] == "delete":
1336                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1337                 #fileCnt = fileCnt + 1
1338                 continue
1339
1340             for prop in ["depotFile", "rev", "action", "type" ]:
1341                 details["%s%s" % (prop, fileCnt)] = info[prop]
1342
1343             fileCnt = fileCnt + 1
1344
1345         details["change"] = newestRevision
1346         self.updateOptionDict(details)
1347         try:
1348             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1349         except IOError:
1350             print "IO error with git fast-import. Is your git version recent enough?"
1351             print self.gitError.read()
1352
1353
1354     def run(self, args):
1355         self.depotPaths = []
1356         self.changeRange = ""
1357         self.initialParent = ""
1358         self.previousDepotPaths = []
1359
1360         # map from branch depot path to parent branch
1361         self.knownBranches = {}
1362         self.initialParents = {}
1363         self.hasOrigin = originP4BranchesExist()
1364         if not self.syncWithOrigin:
1365             self.hasOrigin = False
1366
1367         if self.importIntoRemotes:
1368             self.refPrefix = "refs/remotes/p4/"
1369         else:
1370             self.refPrefix = "refs/heads/p4/"
1371
1372         if self.syncWithOrigin and self.hasOrigin:
1373             if not self.silent:
1374                 print "Syncing with origin first by calling git fetch origin"
1375             system("git fetch origin")
1376
1377         if len(self.branch) == 0:
1378             self.branch = self.refPrefix + "master"
1379             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1380                 system("git update-ref %s refs/heads/p4" % self.branch)
1381                 system("git branch -D p4");
1382             # create it /after/ importing, when master exists
1383             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1384                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1385
1386         # TODO: should always look at previous commits,
1387         # merge with previous imports, if possible.
1388         if args == []:
1389             if self.hasOrigin:
1390                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1391             self.listExistingP4GitBranches()
1392
1393             if len(self.p4BranchesInGit) > 1:
1394                 if not self.silent:
1395                     print "Importing from/into multiple branches"
1396                 self.detectBranches = True
1397
1398             if self.verbose:
1399                 print "branches: %s" % self.p4BranchesInGit
1400
1401             p4Change = 0
1402             for branch in self.p4BranchesInGit:
1403                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1404
1405                 settings = extractSettingsGitLog(logMsg)
1406
1407                 self.readOptions(settings)
1408                 if (settings.has_key('depot-paths')
1409                     and settings.has_key ('change')):
1410                     change = int(settings['change']) + 1
1411                     p4Change = max(p4Change, change)
1412
1413                     depotPaths = sorted(settings['depot-paths'])
1414                     if self.previousDepotPaths == []:
1415                         self.previousDepotPaths = depotPaths
1416                     else:
1417                         paths = []
1418                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1419                             for i in range(0, min(len(cur), len(prev))):
1420                                 if cur[i] <> prev[i]:
1421                                     i = i - 1
1422                                     break
1423
1424                             paths.append (cur[:i + 1])
1425
1426                         self.previousDepotPaths = paths
1427
1428             if p4Change > 0:
1429                 self.depotPaths = sorted(self.previousDepotPaths)
1430                 self.changeRange = "@%s,#head" % p4Change
1431                 if not self.detectBranches:
1432                     self.initialParent = parseRevision(self.branch)
1433                 if not self.silent and not self.detectBranches:
1434                     print "Performing incremental import into %s git branch" % self.branch
1435
1436         if not self.branch.startswith("refs/"):
1437             self.branch = "refs/heads/" + self.branch
1438
1439         if len(args) == 0 and self.depotPaths:
1440             if not self.silent:
1441                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1442         else:
1443             if self.depotPaths and self.depotPaths != args:
1444                 print ("previous import used depot path %s and now %s was specified. "
1445                        "This doesn't work!" % (' '.join (self.depotPaths),
1446                                                ' '.join (args)))
1447                 sys.exit(1)
1448
1449             self.depotPaths = sorted(args)
1450
1451         revision = ""
1452         self.users = {}
1453
1454         newPaths = []
1455         for p in self.depotPaths:
1456             if p.find("@") != -1:
1457                 atIdx = p.index("@")
1458                 self.changeRange = p[atIdx:]
1459                 if self.changeRange == "@all":
1460                     self.changeRange = ""
1461                 elif ',' not in self.changeRange:
1462                     revision = self.changeRange
1463                     self.changeRange = ""
1464                 p = p[:atIdx]
1465             elif p.find("#") != -1:
1466                 hashIdx = p.index("#")
1467                 revision = p[hashIdx:]
1468                 p = p[:hashIdx]
1469             elif self.previousDepotPaths == []:
1470                 revision = "#head"
1471
1472             p = re.sub ("\.\.\.$", "", p)
1473             if not p.endswith("/"):
1474                 p += "/"
1475
1476             newPaths.append(p)
1477
1478         self.depotPaths = newPaths
1479
1480
1481         self.loadUserMapFromCache()
1482         self.labels = {}
1483         if self.detectLabels:
1484             self.getLabels();
1485
1486         if self.detectBranches:
1487             ## FIXME - what's a P4 projectName ?
1488             self.projectName = self.guessProjectName()
1489
1490             if self.hasOrigin:
1491                 self.getBranchMappingFromGitBranches()
1492             else:
1493                 self.getBranchMapping()
1494             if self.verbose:
1495                 print "p4-git branches: %s" % self.p4BranchesInGit
1496                 print "initial parents: %s" % self.initialParents
1497             for b in self.p4BranchesInGit:
1498                 if b != "master":
1499
1500                     ## FIXME
1501                     b = b[len(self.projectName):]
1502                 self.createdBranches.add(b)
1503
1504         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1505
1506         importProcess = subprocess.Popen(["git", "fast-import"],
1507                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1508                                          stderr=subprocess.PIPE);
1509         self.gitOutput = importProcess.stdout
1510         self.gitStream = importProcess.stdin
1511         self.gitError = importProcess.stderr
1512
1513         if revision:
1514             self.importHeadRevision(revision)
1515         else:
1516             changes = []
1517
1518             if len(self.changesFile) > 0:
1519                 output = open(self.changesFile).readlines()
1520                 changeSet = Set()
1521                 for line in output:
1522                     changeSet.add(int(line))
1523
1524                 for change in changeSet:
1525                     changes.append(change)
1526
1527                 changes.sort()
1528             else:
1529                 if self.verbose:
1530                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1531                                                               self.changeRange)
1532                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1533
1534                 if len(self.maxChanges) > 0:
1535                     changes = changes[:min(int(self.maxChanges), len(changes))]
1536
1537             if len(changes) == 0:
1538                 if not self.silent:
1539                     print "No changes to import!"
1540                 return True
1541
1542             if not self.silent and not self.detectBranches:
1543                 print "Import destination: %s" % self.branch
1544
1545             self.updatedBranches = set()
1546
1547             self.importChanges(changes)
1548
1549             if not self.silent:
1550                 print ""
1551                 if len(self.updatedBranches) > 0:
1552                     sys.stdout.write("Updated branches: ")
1553                     for b in self.updatedBranches:
1554                         sys.stdout.write("%s " % b)
1555                     sys.stdout.write("\n")
1556
1557         self.gitStream.close()
1558         if importProcess.wait() != 0:
1559             die("fast-import failed: %s" % self.gitError.read())
1560         self.gitOutput.close()
1561         self.gitError.close()
1562
1563         return True
1564
1565 class P4Rebase(Command):
1566     def __init__(self):
1567         Command.__init__(self)
1568         self.options = [ ]
1569         self.description = ("Fetches the latest revision from perforce and "
1570                             + "rebases the current work (branch) against it")
1571         self.verbose = False
1572
1573     def run(self, args):
1574         sync = P4Sync()
1575         sync.run([])
1576
1577         return self.rebase()
1578
1579     def rebase(self):
1580         if os.system("git update-index --refresh") != 0:
1581             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1582         if len(read_pipe("git diff-index HEAD --")) > 0:
1583             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1584
1585         [upstream, settings] = findUpstreamBranchPoint()
1586         if len(upstream) == 0:
1587             die("Cannot find upstream branchpoint for rebase")
1588
1589         # the branchpoint may be p4/foo~3, so strip off the parent
1590         upstream = re.sub("~[0-9]+$", "", upstream)
1591
1592         print "Rebasing the current branch onto %s" % upstream
1593         oldHead = read_pipe("git rev-parse HEAD").strip()
1594         system("git rebase %s" % upstream)
1595         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1596         return True
1597
1598 class P4Clone(P4Sync):
1599     def __init__(self):
1600         P4Sync.__init__(self)
1601         self.description = "Creates a new git repository and imports from Perforce into it"
1602         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1603         self.options += [
1604             optparse.make_option("--destination", dest="cloneDestination",
1605                                  action='store', default=None,
1606                                  help="where to leave result of the clone"),
1607             optparse.make_option("-/", dest="cloneExclude",
1608                                  action="append", type="string",
1609                                  help="exclude depot path")
1610         ]
1611         self.cloneDestination = None
1612         self.needsGit = False
1613
1614     # This is required for the "append" cloneExclude action
1615     def ensure_value(self, attr, value):
1616         if not hasattr(self, attr) or getattr(self, attr) is None:
1617             setattr(self, attr, value)
1618         return getattr(self, attr)
1619
1620     def defaultDestination(self, args):
1621         ## TODO: use common prefix of args?
1622         depotPath = args[0]
1623         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1624         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1625         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1626         depotDir = re.sub(r"/$", "", depotDir)
1627         return os.path.split(depotDir)[1]
1628
1629     def run(self, args):
1630         if len(args) < 1:
1631             return False
1632
1633         if self.keepRepoPath and not self.cloneDestination:
1634             sys.stderr.write("Must specify destination for --keep-path\n")
1635             sys.exit(1)
1636
1637         depotPaths = args
1638
1639         if not self.cloneDestination and len(depotPaths) > 1:
1640             self.cloneDestination = depotPaths[-1]
1641             depotPaths = depotPaths[:-1]
1642
1643         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1644         for p in depotPaths:
1645             if not p.startswith("//"):
1646                 return False
1647
1648         if not self.cloneDestination:
1649             self.cloneDestination = self.defaultDestination(args)
1650
1651         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1652         if not os.path.exists(self.cloneDestination):
1653             os.makedirs(self.cloneDestination)
1654         os.chdir(self.cloneDestination)
1655         system("git init")
1656         self.gitdir = os.getcwd() + "/.git"
1657         if not P4Sync.run(self, depotPaths):
1658             return False
1659         if self.branch != "master":
1660             if gitBranchExists("refs/remotes/p4/master"):
1661                 system("git branch master refs/remotes/p4/master")
1662                 system("git checkout -f")
1663             else:
1664                 print "Could not detect main branch. No checkout/master branch created."
1665
1666         return True
1667
1668 class P4Branches(Command):
1669     def __init__(self):
1670         Command.__init__(self)
1671         self.options = [ ]
1672         self.description = ("Shows the git branches that hold imports and their "
1673                             + "corresponding perforce depot paths")
1674         self.verbose = False
1675
1676     def run(self, args):
1677         if originP4BranchesExist():
1678             createOrUpdateBranchesFromOrigin()
1679
1680         cmdline = "git rev-parse --symbolic "
1681         cmdline += " --remotes"
1682
1683         for line in read_pipe_lines(cmdline):
1684             line = line.strip()
1685
1686             if not line.startswith('p4/') or line == "p4/HEAD":
1687                 continue
1688             branch = line
1689
1690             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1691             settings = extractSettingsGitLog(log)
1692
1693             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1694         return True
1695
1696 class HelpFormatter(optparse.IndentedHelpFormatter):
1697     def __init__(self):
1698         optparse.IndentedHelpFormatter.__init__(self)
1699
1700     def format_description(self, description):
1701         if description:
1702             return description + "\n"
1703         else:
1704             return ""
1705
1706 def printUsage(commands):
1707     print "usage: %s <command> [options]" % sys.argv[0]
1708     print ""
1709     print "valid commands: %s" % ", ".join(commands)
1710     print ""
1711     print "Try %s <command> --help for command specific help." % sys.argv[0]
1712     print ""
1713
1714 commands = {
1715     "debug" : P4Debug,
1716     "submit" : P4Submit,
1717     "commit" : P4Submit,
1718     "sync" : P4Sync,
1719     "rebase" : P4Rebase,
1720     "clone" : P4Clone,
1721     "rollback" : P4RollBack,
1722     "branches" : P4Branches
1723 }
1724
1725
1726 def main():
1727     if len(sys.argv[1:]) == 0:
1728         printUsage(commands.keys())
1729         sys.exit(2)
1730
1731     cmd = ""
1732     cmdName = sys.argv[1]
1733     try:
1734         klass = commands[cmdName]
1735         cmd = klass()
1736     except KeyError:
1737         print "unknown command %s" % cmdName
1738         print ""
1739         printUsage(commands.keys())
1740         sys.exit(2)
1741
1742     options = cmd.options
1743     cmd.gitdir = os.environ.get("GIT_DIR", None)
1744
1745     args = sys.argv[2:]
1746
1747     if len(options) > 0:
1748         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1749
1750         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1751                                        options,
1752                                        description = cmd.description,
1753                                        formatter = HelpFormatter())
1754
1755         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1756     global verbose
1757     verbose = cmd.verbose
1758     if cmd.needsGit:
1759         if cmd.gitdir == None:
1760             cmd.gitdir = os.path.abspath(".git")
1761             if not isValidGitDir(cmd.gitdir):
1762                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1763                 if os.path.exists(cmd.gitdir):
1764                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1765                     if len(cdup) > 0:
1766                         os.chdir(cdup);
1767
1768         if not isValidGitDir(cmd.gitdir):
1769             if isValidGitDir(cmd.gitdir + "/.git"):
1770                 cmd.gitdir += "/.git"
1771             else:
1772                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1773
1774         os.environ["GIT_DIR"] = cmd.gitdir
1775
1776     if not cmd.run(args):
1777         parser.print_help()
1778
1779
1780 if __name__ == '__main__':
1781     main()