contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s" % file)
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                 optparse.make_option("--origin", dest="origin"),
 469                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 470         ]
 471         self.description = "Submit changes from git to the perforce depot."
 472         self.usage += " [name of git branch to submit into perforce depot]"
 473         self.interactive = True
 474         self.origin = ""
 475         self.detectRename = False
 476         self.verbose = False
 477         self.isWindows = (platform.system() == "Windows")
 478
 479     def check(self):
 480         if len(p4CmdList("opened ...")) > 0:
 481             die("You have files opened with perforce! Close them before starting the sync.")
 482
 483     # replaces everything between 'Description:' and the next P4 submit template field with the
 484     # commit message
 485     def prepareLogMessage(self, template, message):
 486         result = ""
 487
 488         inDescriptionSection = False
 489
 490         for line in template.split("\n"):
 491             if line.startswith("#"):
 492                 result += line + "\n"
 493                 continue
 494
 495             if inDescriptionSection:
 496                 if line.startswith("Files:"):
 497                     inDescriptionSection = False
 498                 else:
 499                     continue
 500             else:
 501                 if line.startswith("Description:"):
 502                     inDescriptionSection = True
 503                     line += "\n"
 504                     for messageLine in message.split("\n"):
 505                         line += "\t" + messageLine + "\n"
 506
 507             result += line + "\n"
 508
 509         return result
 510
 511     def prepareSubmitTemplate(self):
 512         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513         template = ""
 514         inFilesSection = False
 515         for line in read_pipe_lines("p4 change -o"):
 516             if inFilesSection:
 517                 if line.startswith("\t"):
 518                     # path starts and ends with a tab
 519                     path = line[1:]
 520                     lastTab = path.rfind("\t")
 521                     if lastTab != -1:
 522                         path = path[:lastTab]
 523                         if not path.startswith(self.depotPath):
 524                             continue
 525                 else:
 526                     inFilesSection = False
 527             else:
 528                 if line.startswith("Files:"):
 529                     inFilesSection = True
 530
 531             template += line
 532
 533         return template
 534
 535     def applyCommit(self, id):
 536         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 537         diffOpts = ("", "-M")[self.detectRename]
 538         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 539         filesToAdd = set()
 540         filesToDelete = set()
 541         editedFiles = set()
 542         filesToChangeExecBit = {}
 543         for line in diff:
 544             diff = parseDiffTreeEntry(line)
 545             modifier = diff['status']
 546             path = diff['src']
 547             if modifier == "M":
 548                 system("p4 edit \"%s\"" % path)
 549                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 550                     filesToChangeExecBit[path] = diff['dst_mode']
 551                 editedFiles.add(path)
 552             elif modifier == "A":
 553                 filesToAdd.add(path)
 554                 filesToChangeExecBit[path] = diff['dst_mode']
 555                 if path in filesToDelete:
 556                     filesToDelete.remove(path)
 557             elif modifier == "D":
 558                 filesToDelete.add(path)
 559                 if path in filesToAdd:
 560                     filesToAdd.remove(path)
 561             elif modifier == "R":
 562                 src, dest = diff['src'], diff['dst']
 563                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 564                 system("p4 edit \"%s\"" % (dest))
 565                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 566                     filesToChangeExecBit[dest] = diff['dst_mode']
 567                 os.unlink(dest)
 568                 editedFiles.add(dest)
 569                 filesToDelete.add(src)
 570             else:
 571                 die("unknown modifier %s for %s" % (modifier, path))
 572
 573         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 574         patchcmd = diffcmd + " | git apply "
 575         tryPatchCmd = patchcmd + "--check -"
 576         applyPatchCmd = patchcmd + "--check --apply -"
 577
 578         if os.system(tryPatchCmd) != 0:
 579             print "Unfortunately applying the change failed!"
 580             print "What do you want to do?"
 581             response = "x"
 582             while response != "s" and response != "a" and response != "w":
 583                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 584                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 585             if response == "s":
 586                 print "Skipping! Good luck with the next patches..."
 587                 for f in editedFiles:
 588                     system("p4 revert \"%s\"" % f);
 589                 for f in filesToAdd:
 590                     system("rm %s" %f)
 591                 return
 592             elif response == "a":
 593                 os.system(applyPatchCmd)
 594                 if len(filesToAdd) > 0:
 595                     print "You may also want to call p4 add on the following files:"
 596                     print " ".join(filesToAdd)
 597                 if len(filesToDelete):
 598                     print "The following files should be scheduled for deletion with p4 delete:"
 599                     print " ".join(filesToDelete)
 600                 die("Please resolve and submit the conflict manually and "
 601                     + "continue afterwards with git-p4 submit --continue")
 602             elif response == "w":
 603                 system(diffcmd + " > patch.txt")
 604                 print "Patch saved to patch.txt in %s !" % self.clientPath
 605                 die("Please resolve and submit the conflict manually and "
 606                     "continue afterwards with git-p4 submit --continue")
 607
 608         system(applyPatchCmd)
 609
 610         for f in filesToAdd:
 611             system("p4 add \"%s\"" % f)
 612         for f in filesToDelete:
 613             system("p4 revert \"%s\"" % f)
 614             system("p4 delete \"%s\"" % f)
 615
 616         # Set/clear executable bits
 617         for f in filesToChangeExecBit.keys():
 618             mode = filesToChangeExecBit[f]
 619             setP4ExecBit(f, mode)
 620
 621         logMessage = extractLogMessageFromGitCommit(id)
 622         if self.isWindows:
 623             logMessage = logMessage.replace("\n", "\r\n")
 624         logMessage = logMessage.strip()
 625
 626         template = self.prepareSubmitTemplate()
 627
 628         if self.interactive:
 629             submitTemplate = self.prepareLogMessage(template, logMessage)
 630             if os.environ.has_key("P4DIFF"):
 631                 del(os.environ["P4DIFF"])
 632             diff = read_pipe("p4 diff -du ...")
 633
 634             for newFile in filesToAdd:
 635                 diff += "==== new file ====\n"
 636                 diff += "--- /dev/null\n"
 637                 diff += "+++ %s\n" % newFile
 638                 f = open(newFile, "r")
 639                 for line in f.readlines():
 640                     diff += "+" + line
 641                 f.close()
 642
 643             separatorLine = "######## everything below this line is just the diff #######"
 644             if platform.system() == "Windows":
 645                 separatorLine += "\r"
 646             separatorLine += "\n"
 647
 648             [handle, fileName] = tempfile.mkstemp()
 649             tmpFile = os.fdopen(handle, "w+")
 650             tmpFile.write(submitTemplate + separatorLine + diff)
 651             tmpFile.close()
 652             defaultEditor = "vi"
 653             if platform.system() == "Windows":
 654                 defaultEditor = "notepad"
 655             if os.environ.has_key("P4EDITOR"):
 656                 editor = os.environ.get("P4EDITOR")
 657             else:
 658                 editor = os.environ.get("EDITOR", defaultEditor);
 659             system(editor + " " + fileName)
 660             tmpFile = open(fileName, "rb")
 661             message = tmpFile.read()
 662             tmpFile.close()
 663             os.remove(fileName)
 664             submitTemplate = message[:message.index(separatorLine)]
 665             if self.isWindows:
 666                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 667
 668             write_pipe("p4 submit -i", submitTemplate)
 669         else:
 670             fileName = "submit.txt"
 671             file = open(fileName, "w+")
 672             file.write(self.prepareLogMessage(template, logMessage))
 673             file.close()
 674             print ("Perforce submit template written as %s. "
 675                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 676                    % (fileName, fileName))
 677
 678     def run(self, args):
 679         if len(args) == 0:
 680             self.master = currentGitBranch()
 681             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 682                 die("Detecting current git branch failed!")
 683         elif len(args) == 1:
 684             self.master = args[0]
 685         else:
 686             return False
 687
 688         [upstream, settings] = findUpstreamBranchPoint()
 689         self.depotPath = settings['depot-paths'][0]
 690         if len(self.origin) == 0:
 691             self.origin = upstream
 692
 693         if self.verbose:
 694             print "Origin branch is " + self.origin
 695
 696         if len(self.depotPath) == 0:
 697             print "Internal error: cannot locate perforce depot path from existing branches"
 698             sys.exit(128)
 699
 700         self.clientPath = p4Where(self.depotPath)
 701
 702         if len(self.clientPath) == 0:
 703             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 704             sys.exit(128)
 705
 706         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 707         self.oldWorkingDirectory = os.getcwd()
 708
 709         os.chdir(self.clientPath)
 710         print "Syncronizing p4 checkout..."
 711         system("p4 sync ...")
 712
 713         self.check()
 714
 715         commits = []
 716         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 717             commits.append(line.strip())
 718         commits.reverse()
 719
 720         while len(commits) > 0:
 721             commit = commits[0]
 722             commits = commits[1:]
 723             self.applyCommit(commit)
 724             if not self.interactive:
 725                 break
 726
 727         if len(commits) == 0:
 728             print "All changes applied!"
 729             os.chdir(self.oldWorkingDirectory)
 730
 731             sync = P4Sync()
 732             sync.run([])
 733
 734             rebase = P4Rebase()
 735             rebase.rebase()
 736
 737         return True
 738
 739 class P4Sync(Command):
 740     def __init__(self):
 741         Command.__init__(self)
 742         self.options = [
 743                 optparse.make_option("--branch", dest="branch"),
 744                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 745                 optparse.make_option("--changesfile", dest="changesFile"),
 746                 optparse.make_option("--silent", dest="silent", action="store_true"),
 747                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 748                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 749                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 750                                      help="Import into refs/heads/ , not refs/remotes"),
 751                 optparse.make_option("--max-changes", dest="maxChanges"),
 752                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 753                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 754                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 755                                      help="Only sync files that are included in the Perforce Client Spec")
 756         ]
 757         self.description = """Imports from Perforce into a git repository.\n
 758     example:
 759     //depot/my/project/ -- to import the current head
 760     //depot/my/project/@all -- to import everything
 761     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 762
 763     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 764
 765         self.usage += " //depot/path[@revRange]"
 766         self.silent = False
 767         self.createdBranches = Set()
 768         self.committedChanges = Set()
 769         self.branch = ""
 770         self.detectBranches = False
 771         self.detectLabels = False
 772         self.changesFile = ""
 773         self.syncWithOrigin = True
 774         self.verbose = False
 775         self.importIntoRemotes = True
 776         self.maxChanges = ""
 777         self.isWindows = (platform.system() == "Windows")
 778         self.keepRepoPath = False
 779         self.depotPaths = None
 780         self.p4BranchesInGit = []
 781         self.cloneExclude = []
 782         self.useClientSpec = False
 783         self.clientSpecDirs = []
 784
 785         if gitConfig("git-p4.syncFromOrigin") == "false":
 786             self.syncWithOrigin = False
 787
 788     def extractFilesFromCommit(self, commit):
 789         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 790                              for path in self.cloneExclude]
 791         files = []
 792         fnum = 0
 793         while commit.has_key("depotFile%s" % fnum):
 794             path =  commit["depotFile%s" % fnum]
 795
 796             if [p for p in self.cloneExclude
 797                 if path.startswith (p)]:
 798                 found = False
 799             else:
 800                 found = [p for p in self.depotPaths
 801                          if path.startswith (p)]
 802             if not found:
 803                 fnum = fnum + 1
 804                 continue
 805
 806             file = {}
 807             file["path"] = path
 808             file["rev"] = commit["rev%s" % fnum]
 809             file["action"] = commit["action%s" % fnum]
 810             file["type"] = commit["type%s" % fnum]
 811             files.append(file)
 812             fnum = fnum + 1
 813         return files
 814
 815     def stripRepoPath(self, path, prefixes):
 816         if self.keepRepoPath:
 817             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 818
 819         for p in prefixes:
 820             if path.startswith(p):
 821                 path = path[len(p):]
 822
 823         return path
 824
 825     def splitFilesIntoBranches(self, commit):
 826         branches = {}
 827         fnum = 0
 828         while commit.has_key("depotFile%s" % fnum):
 829             path =  commit["depotFile%s" % fnum]
 830             found = [p for p in self.depotPaths
 831                      if path.startswith (p)]
 832             if not found:
 833                 fnum = fnum + 1
 834                 continue
 835
 836             file = {}
 837             file["path"] = path
 838             file["rev"] = commit["rev%s" % fnum]
 839             file["action"] = commit["action%s" % fnum]
 840             file["type"] = commit["type%s" % fnum]
 841             fnum = fnum + 1
 842
 843             relPath = self.stripRepoPath(path, self.depotPaths)
 844
 845             for branch in self.knownBranches.keys():
 846
 847                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 848                 if relPath.startswith(branch + "/"):
 849                     if branch not in branches:
 850                         branches[branch] = []
 851                     branches[branch].append(file)
 852                     break
 853
 854         return branches
 855
 856     ## Should move this out, doesn't use SELF.
 857     def readP4Files(self, files):
 858         filesForCommit = []
 859         filesToRead = []
 860
 861         for f in files:
 862             includeFile = True
 863             for val in self.clientSpecDirs:
 864                 if f['path'].startswith(val[0]):
 865                     if val[1] <= 0:
 866                         includeFile = False
 867                     break
 868
 869             if includeFile:
 870                 filesForCommit.append(f)
 871                 if f['action'] != 'delete':
 872                     filesToRead.append(f)
 873
 874         filedata = []
 875         if len(filesToRead) > 0:
 876             filedata = p4CmdList('-x - print',
 877                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 878                                                   for f in filesToRead]),
 879                                  stdin_mode='w+')
 880
 881             if "p4ExitCode" in filedata[0]:
 882                 die("Problems executing p4. Error: [%d]."
 883                     % (filedata[0]['p4ExitCode']));
 884
 885         j = 0;
 886         contents = {}
 887         while j < len(filedata):
 888             stat = filedata[j]
 889             j += 1
 890             text = [];
 891             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 892                 text.append(filedata[j]['data'])
 893                 j += 1
 894             text = ''.join(text)
 895
 896             if not stat.has_key('depotFile'):
 897                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 898                 continue
 899
 900             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 901                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 902             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 903                 text = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 904
 905             contents[stat['depotFile']] = text
 906
 907         for f in filesForCommit:
 908             path = f['path']
 909             if contents.has_key(path):
 910                 f['data'] = contents[path]
 911
 912         return filesForCommit
 913
 914     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 915         epoch = details["time"]
 916         author = details["user"]
 917
 918         if self.verbose:
 919             print "commit into %s" % branch
 920
 921         # start with reading files; if that fails, we should not
 922         # create a commit.
 923         new_files = []
 924         for f in files:
 925             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 926                 new_files.append (f)
 927             else:
 928                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 929         files = self.readP4Files(new_files)
 930
 931         self.gitStream.write("commit %s\n" % branch)
 932 #        gitStream.write("mark :%s\n" % details["change"])
 933         self.committedChanges.add(int(details["change"]))
 934         committer = ""
 935         if author not in self.users:
 936             self.getUserMapFromPerforceServer()
 937         if author in self.users:
 938             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 939         else:
 940             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 941
 942         self.gitStream.write("committer %s\n" % committer)
 943
 944         self.gitStream.write("data <<EOT\n")
 945         self.gitStream.write(details["desc"])
 946         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 947                              % (','.join (branchPrefixes), details["change"]))
 948         if len(details['options']) > 0:
 949             self.gitStream.write(": options = %s" % details['options'])
 950         self.gitStream.write("]\nEOT\n\n")
 951
 952         if len(parent) > 0:
 953             if self.verbose:
 954                 print "parent %s" % parent
 955             self.gitStream.write("from %s\n" % parent)
 956
 957         for file in files:
 958             if file["type"] == "apple":
 959                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 960                 continue
 961
 962             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 963             if file["action"] == "delete":
 964                 self.gitStream.write("D %s\n" % relPath)
 965             else:
 966                 data = file['data']
 967
 968                 mode = "644"
 969                 if isP4Exec(file["type"]):
 970                     mode = "755"
 971                 elif file["type"] == "symlink":
 972                     mode = "120000"
 973                     # p4 print on a symlink contains "target\n", so strip it off
 974                     data = data[:-1]
 975
 976                 if self.isWindows and file["type"].endswith("text"):
 977                     data = data.replace("\r\n", "\n")
 978
 979                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 980                 self.gitStream.write("data %s\n" % len(data))
 981                 self.gitStream.write(data)
 982                 self.gitStream.write("\n")
 983
 984         self.gitStream.write("\n")
 985
 986         change = int(details["change"])
 987
 988         if self.labels.has_key(change):
 989             label = self.labels[change]
 990             labelDetails = label[0]
 991             labelRevisions = label[1]
 992             if self.verbose:
 993                 print "Change %s is labelled %s" % (change, labelDetails)
 994
 995             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 996                                                     for p in branchPrefixes]))
 997
 998             if len(files) == len(labelRevisions):
 999
1000                 cleanedFiles = {}
1001                 for info in files:
1002                     if info["action"] == "delete":
1003                         continue
1004                     cleanedFiles[info["depotFile"]] = info["rev"]
1005
1006                 if cleanedFiles == labelRevisions:
1007                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1008                     self.gitStream.write("from %s\n" % branch)
1009
1010                     owner = labelDetails["Owner"]
1011                     tagger = ""
1012                     if author in self.users:
1013                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1014                     else:
1015                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1016                     self.gitStream.write("tagger %s\n" % tagger)
1017                     self.gitStream.write("data <<EOT\n")
1018                     self.gitStream.write(labelDetails["Description"])
1019                     self.gitStream.write("EOT\n\n")
1020
1021                 else:
1022                     if not self.silent:
1023                         print ("Tag %s does not match with change %s: files do not match."
1024                                % (labelDetails["label"], change))
1025
1026             else:
1027                 if not self.silent:
1028                     print ("Tag %s does not match with change %s: file count is different."
1029                            % (labelDetails["label"], change))
1030
1031     def getUserCacheFilename(self):
1032         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1033         return home + "/.gitp4-usercache.txt"
1034
1035     def getUserMapFromPerforceServer(self):
1036         if self.userMapFromPerforceServer:
1037             return
1038         self.users = {}
1039
1040         for output in p4CmdList("users"):
1041             if not output.has_key("User"):
1042                 continue
1043             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1044
1045
1046         s = ''
1047         for (key, val) in self.users.items():
1048             s += "%s\t%s\n" % (key, val)
1049
1050         open(self.getUserCacheFilename(), "wb").write(s)
1051         self.userMapFromPerforceServer = True
1052
1053     def loadUserMapFromCache(self):
1054         self.users = {}
1055         self.userMapFromPerforceServer = False
1056         try:
1057             cache = open(self.getUserCacheFilename(), "rb")
1058             lines = cache.readlines()
1059             cache.close()
1060             for line in lines:
1061                 entry = line.strip().split("\t")
1062                 self.users[entry[0]] = entry[1]
1063         except IOError:
1064             self.getUserMapFromPerforceServer()
1065
1066     def getLabels(self):
1067         self.labels = {}
1068
1069         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1070         if len(l) > 0 and not self.silent:
1071             print "Finding files belonging to labels in %s" % `self.depotPaths`
1072
1073         for output in l:
1074             label = output["label"]
1075             revisions = {}
1076             newestChange = 0
1077             if self.verbose:
1078                 print "Querying files for label %s" % label
1079             for file in p4CmdList("files "
1080                                   +  ' '.join (["%s...@%s" % (p, label)
1081                                                 for p in self.depotPaths])):
1082                 revisions[file["depotFile"]] = file["rev"]
1083                 change = int(file["change"])
1084                 if change > newestChange:
1085                     newestChange = change
1086
1087             self.labels[newestChange] = [output, revisions]
1088
1089         if self.verbose:
1090             print "Label changes: %s" % self.labels.keys()
1091
1092     def guessProjectName(self):
1093         for p in self.depotPaths:
1094             if p.endswith("/"):
1095                 p = p[:-1]
1096             p = p[p.strip().rfind("/") + 1:]
1097             if not p.endswith("/"):
1098                p += "/"
1099             return p
1100
1101     def getBranchMapping(self):
1102         lostAndFoundBranches = set()
1103
1104         for info in p4CmdList("branches"):
1105             details = p4Cmd("branch -o %s" % info["branch"])
1106             viewIdx = 0
1107             while details.has_key("View%s" % viewIdx):
1108                 paths = details["View%s" % viewIdx].split(" ")
1109                 viewIdx = viewIdx + 1
1110                 # require standard //depot/foo/... //depot/bar/... mapping
1111                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1112                     continue
1113                 source = paths[0]
1114                 destination = paths[1]
1115                 ## HACK
1116                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1117                     source = source[len(self.depotPaths[0]):-4]
1118                     destination = destination[len(self.depotPaths[0]):-4]
1119
1120                     if destination in self.knownBranches:
1121                         if not self.silent:
1122                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1123                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1124                         continue
1125
1126                     self.knownBranches[destination] = source
1127
1128                     lostAndFoundBranches.discard(destination)
1129
1130                     if source not in self.knownBranches:
1131                         lostAndFoundBranches.add(source)
1132
1133
1134         for branch in lostAndFoundBranches:
1135             self.knownBranches[branch] = branch
1136
1137     def getBranchMappingFromGitBranches(self):
1138         branches = p4BranchesInGit(self.importIntoRemotes)
1139         for branch in branches.keys():
1140             if branch == "master":
1141                 branch = "main"
1142             else:
1143                 branch = branch[len(self.projectName):]
1144             self.knownBranches[branch] = branch
1145
1146     def listExistingP4GitBranches(self):
1147         # branches holds mapping from name to commit
1148         branches = p4BranchesInGit(self.importIntoRemotes)
1149         self.p4BranchesInGit = branches.keys()
1150         for branch in branches.keys():
1151             self.initialParents[self.refPrefix + branch] = branches[branch]
1152
1153     def updateOptionDict(self, d):
1154         option_keys = {}
1155         if self.keepRepoPath:
1156             option_keys['keepRepoPath'] = 1
1157
1158         d["options"] = ' '.join(sorted(option_keys.keys()))
1159
1160     def readOptions(self, d):
1161         self.keepRepoPath = (d.has_key('options')
1162                              and ('keepRepoPath' in d['options']))
1163
1164     def gitRefForBranch(self, branch):
1165         if branch == "main":
1166             return self.refPrefix + "master"
1167
1168         if len(branch) <= 0:
1169             return branch
1170
1171         return self.refPrefix + self.projectName + branch
1172
1173     def gitCommitByP4Change(self, ref, change):
1174         if self.verbose:
1175             print "looking in ref " + ref + " for change %s using bisect..." % change
1176
1177         earliestCommit = ""
1178         latestCommit = parseRevision(ref)
1179
1180         while True:
1181             if self.verbose:
1182                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1183             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1184             if len(next) == 0:
1185                 if self.verbose:
1186                     print "argh"
1187                 return ""
1188             log = extractLogMessageFromGitCommit(next)
1189             settings = extractSettingsGitLog(log)
1190             currentChange = int(settings['change'])
1191             if self.verbose:
1192                 print "current change %s" % currentChange
1193
1194             if currentChange == change:
1195                 if self.verbose:
1196                     print "found %s" % next
1197                 return next
1198
1199             if currentChange < change:
1200                 earliestCommit = "^%s" % next
1201             else:
1202                 latestCommit = "%s" % next
1203
1204         return ""
1205
1206     def importNewBranch(self, branch, maxChange):
1207         # make fast-import flush all changes to disk and update the refs using the checkpoint
1208         # command so that we can try to find the branch parent in the git history
1209         self.gitStream.write("checkpoint\n\n");
1210         self.gitStream.flush();
1211         branchPrefix = self.depotPaths[0] + branch + "/"
1212         range = "@1,%s" % maxChange
1213         #print "prefix" + branchPrefix
1214         changes = p4ChangesForPaths([branchPrefix], range)
1215         if len(changes) <= 0:
1216             return False
1217         firstChange = changes[0]
1218         #print "first change in branch: %s" % firstChange
1219         sourceBranch = self.knownBranches[branch]
1220         sourceDepotPath = self.depotPaths[0] + sourceBranch
1221         sourceRef = self.gitRefForBranch(sourceBranch)
1222         #print "source " + sourceBranch
1223
1224         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1225         #print "branch parent: %s" % branchParentChange
1226         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1227         if len(gitParent) > 0:
1228             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1229             #print "parent git commit: %s" % gitParent
1230
1231         self.importChanges(changes)
1232         return True
1233
1234     def importChanges(self, changes):
1235         cnt = 1
1236         for change in changes:
1237             description = p4Cmd("describe %s" % change)
1238             self.updateOptionDict(description)
1239
1240             if not self.silent:
1241                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1242                 sys.stdout.flush()
1243             cnt = cnt + 1
1244
1245             try:
1246                 if self.detectBranches:
1247                     branches = self.splitFilesIntoBranches(description)
1248                     for branch in branches.keys():
1249                         ## HACK  --hwn
1250                         branchPrefix = self.depotPaths[0] + branch + "/"
1251
1252                         parent = ""
1253
1254                         filesForCommit = branches[branch]
1255
1256                         if self.verbose:
1257                             print "branch is %s" % branch
1258
1259                         self.updatedBranches.add(branch)
1260
1261                         if branch not in self.createdBranches:
1262                             self.createdBranches.add(branch)
1263                             parent = self.knownBranches[branch]
1264                             if parent == branch:
1265                                 parent = ""
1266                             else:
1267                                 fullBranch = self.projectName + branch
1268                                 if fullBranch not in self.p4BranchesInGit:
1269                                     if not self.silent:
1270                                         print("\n    Importing new branch %s" % fullBranch);
1271                                     if self.importNewBranch(branch, change - 1):
1272                                         parent = ""
1273                                         self.p4BranchesInGit.append(fullBranch)
1274                                     if not self.silent:
1275                                         print("\n    Resuming with change %s" % change);
1276
1277                                 if self.verbose:
1278                                     print "parent determined through known branches: %s" % parent
1279
1280                         branch = self.gitRefForBranch(branch)
1281                         parent = self.gitRefForBranch(parent)
1282
1283                         if self.verbose:
1284                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1285
1286                         if len(parent) == 0 and branch in self.initialParents:
1287                             parent = self.initialParents[branch]
1288                             del self.initialParents[branch]
1289
1290                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1291                 else:
1292                     files = self.extractFilesFromCommit(description)
1293                     self.commit(description, files, self.branch, self.depotPaths,
1294                                 self.initialParent)
1295                     self.initialParent = ""
1296             except IOError:
1297                 print self.gitError.read()
1298                 sys.exit(1)
1299
1300     def importHeadRevision(self, revision):
1301         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1302
1303         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1304         details["desc"] = ("Initial import of %s from the state at revision %s"
1305                            % (' '.join(self.depotPaths), revision))
1306         details["change"] = revision
1307         newestRevision = 0
1308
1309         fileCnt = 0
1310         for info in p4CmdList("files "
1311                               +  ' '.join(["%s...%s"
1312                                            % (p, revision)
1313                                            for p in self.depotPaths])):
1314
1315             if info['code'] == 'error':
1316                 sys.stderr.write("p4 returned an error: %s\n"
1317                                  % info['data'])
1318                 sys.exit(1)
1319
1320
1321             change = int(info["change"])
1322             if change > newestRevision:
1323                 newestRevision = change
1324
1325             if info["action"] == "delete":
1326                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1327                 #fileCnt = fileCnt + 1
1328                 continue
1329
1330             for prop in ["depotFile", "rev", "action", "type" ]:
1331                 details["%s%s" % (prop, fileCnt)] = info[prop]
1332
1333             fileCnt = fileCnt + 1
1334
1335         details["change"] = newestRevision
1336         self.updateOptionDict(details)
1337         try:
1338             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1339         except IOError:
1340             print "IO error with git fast-import. Is your git version recent enough?"
1341             print self.gitError.read()
1342
1343
1344     def getClientSpec(self):
1345         specList = p4CmdList( "client -o" )
1346         temp = {}
1347         for entry in specList:
1348             for k,v in entry.iteritems():
1349                 if k.startswith("View"):
1350                     if v.startswith('"'):
1351                         start = 1
1352                     else:
1353                         start = 0
1354                     index = v.find("...")
1355                     v = v[start:index]
1356                     if v.startswith("-"):
1357                         v = v[1:]
1358                         temp[v] = -len(v)
1359                     else:
1360                         temp[v] = len(v)
1361         self.clientSpecDirs = temp.items()
1362         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1363
1364     def run(self, args):
1365         self.depotPaths = []
1366         self.changeRange = ""
1367         self.initialParent = ""
1368         self.previousDepotPaths = []
1369
1370         # map from branch depot path to parent branch
1371         self.knownBranches = {}
1372         self.initialParents = {}
1373         self.hasOrigin = originP4BranchesExist()
1374         if not self.syncWithOrigin:
1375             self.hasOrigin = False
1376
1377         if self.importIntoRemotes:
1378             self.refPrefix = "refs/remotes/p4/"
1379         else:
1380             self.refPrefix = "refs/heads/p4/"
1381
1382         if self.syncWithOrigin and self.hasOrigin:
1383             if not self.silent:
1384                 print "Syncing with origin first by calling git fetch origin"
1385             system("git fetch origin")
1386
1387         if len(self.branch) == 0:
1388             self.branch = self.refPrefix + "master"
1389             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1390                 system("git update-ref %s refs/heads/p4" % self.branch)
1391                 system("git branch -D p4");
1392             # create it /after/ importing, when master exists
1393             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1394                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1395
1396         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1397             self.getClientSpec()
1398
1399         # TODO: should always look at previous commits,
1400         # merge with previous imports, if possible.
1401         if args == []:
1402             if self.hasOrigin:
1403                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1404             self.listExistingP4GitBranches()
1405
1406             if len(self.p4BranchesInGit) > 1:
1407                 if not self.silent:
1408                     print "Importing from/into multiple branches"
1409                 self.detectBranches = True
1410
1411             if self.verbose:
1412                 print "branches: %s" % self.p4BranchesInGit
1413
1414             p4Change = 0
1415             for branch in self.p4BranchesInGit:
1416                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1417
1418                 settings = extractSettingsGitLog(logMsg)
1419
1420                 self.readOptions(settings)
1421                 if (settings.has_key('depot-paths')
1422                     and settings.has_key ('change')):
1423                     change = int(settings['change']) + 1
1424                     p4Change = max(p4Change, change)
1425
1426                     depotPaths = sorted(settings['depot-paths'])
1427                     if self.previousDepotPaths == []:
1428                         self.previousDepotPaths = depotPaths
1429                     else:
1430                         paths = []
1431                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1432                             for i in range(0, min(len(cur), len(prev))):
1433                                 if cur[i] <> prev[i]:
1434                                     i = i - 1
1435                                     break
1436
1437                             paths.append (cur[:i + 1])
1438
1439                         self.previousDepotPaths = paths
1440
1441             if p4Change > 0:
1442                 self.depotPaths = sorted(self.previousDepotPaths)
1443                 self.changeRange = "@%s,#head" % p4Change
1444                 if not self.detectBranches:
1445                     self.initialParent = parseRevision(self.branch)
1446                 if not self.silent and not self.detectBranches:
1447                     print "Performing incremental import into %s git branch" % self.branch
1448
1449         if not self.branch.startswith("refs/"):
1450             self.branch = "refs/heads/" + self.branch
1451
1452         if len(args) == 0 and self.depotPaths:
1453             if not self.silent:
1454                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1455         else:
1456             if self.depotPaths and self.depotPaths != args:
1457                 print ("previous import used depot path %s and now %s was specified. "
1458                        "This doesn't work!" % (' '.join (self.depotPaths),
1459                                                ' '.join (args)))
1460                 sys.exit(1)
1461
1462             self.depotPaths = sorted(args)
1463
1464         revision = ""
1465         self.users = {}
1466
1467         newPaths = []
1468         for p in self.depotPaths:
1469             if p.find("@") != -1:
1470                 atIdx = p.index("@")
1471                 self.changeRange = p[atIdx:]
1472                 if self.changeRange == "@all":
1473                     self.changeRange = ""
1474                 elif ',' not in self.changeRange:
1475                     revision = self.changeRange
1476                     self.changeRange = ""
1477                 p = p[:atIdx]
1478             elif p.find("#") != -1:
1479                 hashIdx = p.index("#")
1480                 revision = p[hashIdx:]
1481                 p = p[:hashIdx]
1482             elif self.previousDepotPaths == []:
1483                 revision = "#head"
1484
1485             p = re.sub ("\.\.\.$", "", p)
1486             if not p.endswith("/"):
1487                 p += "/"
1488
1489             newPaths.append(p)
1490
1491         self.depotPaths = newPaths
1492
1493
1494         self.loadUserMapFromCache()
1495         self.labels = {}
1496         if self.detectLabels:
1497             self.getLabels();
1498
1499         if self.detectBranches:
1500             ## FIXME - what's a P4 projectName ?
1501             self.projectName = self.guessProjectName()
1502
1503             if self.hasOrigin:
1504                 self.getBranchMappingFromGitBranches()
1505             else:
1506                 self.getBranchMapping()
1507             if self.verbose:
1508                 print "p4-git branches: %s" % self.p4BranchesInGit
1509                 print "initial parents: %s" % self.initialParents
1510             for b in self.p4BranchesInGit:
1511                 if b != "master":
1512
1513                     ## FIXME
1514                     b = b[len(self.projectName):]
1515                 self.createdBranches.add(b)
1516
1517         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1518
1519         importProcess = subprocess.Popen(["git", "fast-import"],
1520                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1521                                          stderr=subprocess.PIPE);
1522         self.gitOutput = importProcess.stdout
1523         self.gitStream = importProcess.stdin
1524         self.gitError = importProcess.stderr
1525
1526         if revision:
1527             self.importHeadRevision(revision)
1528         else:
1529             changes = []
1530
1531             if len(self.changesFile) > 0:
1532                 output = open(self.changesFile).readlines()
1533                 changeSet = Set()
1534                 for line in output:
1535                     changeSet.add(int(line))
1536
1537                 for change in changeSet:
1538                     changes.append(change)
1539
1540                 changes.sort()
1541             else:
1542                 if self.verbose:
1543                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1544                                                               self.changeRange)
1545                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1546
1547                 if len(self.maxChanges) > 0:
1548                     changes = changes[:min(int(self.maxChanges), len(changes))]
1549
1550             if len(changes) == 0:
1551                 if not self.silent:
1552                     print "No changes to import!"
1553                 return True
1554
1555             if not self.silent and not self.detectBranches:
1556                 print "Import destination: %s" % self.branch
1557
1558             self.updatedBranches = set()
1559
1560             self.importChanges(changes)
1561
1562             if not self.silent:
1563                 print ""
1564                 if len(self.updatedBranches) > 0:
1565                     sys.stdout.write("Updated branches: ")
1566                     for b in self.updatedBranches:
1567                         sys.stdout.write("%s " % b)
1568                     sys.stdout.write("\n")
1569
1570         self.gitStream.close()
1571         if importProcess.wait() != 0:
1572             die("fast-import failed: %s" % self.gitError.read())
1573         self.gitOutput.close()
1574         self.gitError.close()
1575
1576         return True
1577
1578 class P4Rebase(Command):
1579     def __init__(self):
1580         Command.__init__(self)
1581         self.options = [ ]
1582         self.description = ("Fetches the latest revision from perforce and "
1583                             + "rebases the current work (branch) against it")
1584         self.verbose = False
1585
1586     def run(self, args):
1587         sync = P4Sync()
1588         sync.run([])
1589
1590         return self.rebase()
1591
1592     def rebase(self):
1593         if os.system("git update-index --refresh") != 0:
1594             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1595         if len(read_pipe("git diff-index HEAD --")) > 0:
1596             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1597
1598         [upstream, settings] = findUpstreamBranchPoint()
1599         if len(upstream) == 0:
1600             die("Cannot find upstream branchpoint for rebase")
1601
1602         # the branchpoint may be p4/foo~3, so strip off the parent
1603         upstream = re.sub("~[0-9]+$", "", upstream)
1604
1605         print "Rebasing the current branch onto %s" % upstream
1606         oldHead = read_pipe("git rev-parse HEAD").strip()
1607         system("git rebase %s" % upstream)
1608         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1609         return True
1610
1611 class P4Clone(P4Sync):
1612     def __init__(self):
1613         P4Sync.__init__(self)
1614         self.description = "Creates a new git repository and imports from Perforce into it"
1615         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1616         self.options += [
1617             optparse.make_option("--destination", dest="cloneDestination",
1618                                  action='store', default=None,
1619                                  help="where to leave result of the clone"),
1620             optparse.make_option("-/", dest="cloneExclude",
1621                                  action="append", type="string",
1622                                  help="exclude depot path")
1623         ]
1624         self.cloneDestination = None
1625         self.needsGit = False
1626
1627     # This is required for the "append" cloneExclude action
1628     def ensure_value(self, attr, value):
1629         if not hasattr(self, attr) or getattr(self, attr) is None:
1630             setattr(self, attr, value)
1631         return getattr(self, attr)
1632
1633     def defaultDestination(self, args):
1634         ## TODO: use common prefix of args?
1635         depotPath = args[0]
1636         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1637         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1638         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1639         depotDir = re.sub(r"/$", "", depotDir)
1640         return os.path.split(depotDir)[1]
1641
1642     def run(self, args):
1643         if len(args) < 1:
1644             return False
1645
1646         if self.keepRepoPath and not self.cloneDestination:
1647             sys.stderr.write("Must specify destination for --keep-path\n")
1648             sys.exit(1)
1649
1650         depotPaths = args
1651
1652         if not self.cloneDestination and len(depotPaths) > 1:
1653             self.cloneDestination = depotPaths[-1]
1654             depotPaths = depotPaths[:-1]
1655
1656         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1657         for p in depotPaths:
1658             if not p.startswith("//"):
1659                 return False
1660
1661         if not self.cloneDestination:
1662             self.cloneDestination = self.defaultDestination(args)
1663
1664         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1665         if not os.path.exists(self.cloneDestination):
1666             os.makedirs(self.cloneDestination)
1667         os.chdir(self.cloneDestination)
1668         system("git init")
1669         self.gitdir = os.getcwd() + "/.git"
1670         if not P4Sync.run(self, depotPaths):
1671             return False
1672         if self.branch != "master":
1673             if gitBranchExists("refs/remotes/p4/master"):
1674                 system("git branch master refs/remotes/p4/master")
1675                 system("git checkout -f")
1676             else:
1677                 print "Could not detect main branch. No checkout/master branch created."
1678
1679         return True
1680
1681 class P4Branches(Command):
1682     def __init__(self):
1683         Command.__init__(self)
1684         self.options = [ ]
1685         self.description = ("Shows the git branches that hold imports and their "
1686                             + "corresponding perforce depot paths")
1687         self.verbose = False
1688
1689     def run(self, args):
1690         if originP4BranchesExist():
1691             createOrUpdateBranchesFromOrigin()
1692
1693         cmdline = "git rev-parse --symbolic "
1694         cmdline += " --remotes"
1695
1696         for line in read_pipe_lines(cmdline):
1697             line = line.strip()
1698
1699             if not line.startswith('p4/') or line == "p4/HEAD":
1700                 continue
1701             branch = line
1702
1703             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1704             settings = extractSettingsGitLog(log)
1705
1706             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1707         return True
1708
1709 class HelpFormatter(optparse.IndentedHelpFormatter):
1710     def __init__(self):
1711         optparse.IndentedHelpFormatter.__init__(self)
1712
1713     def format_description(self, description):
1714         if description:
1715             return description + "\n"
1716         else:
1717             return ""
1718
1719 def printUsage(commands):
1720     print "usage: %s <command> [options]" % sys.argv[0]
1721     print ""
1722     print "valid commands: %s" % ", ".join(commands)
1723     print ""
1724     print "Try %s <command> --help for command specific help." % sys.argv[0]
1725     print ""
1726
1727 commands = {
1728     "debug" : P4Debug,
1729     "submit" : P4Submit,
1730     "commit" : P4Submit,
1731     "sync" : P4Sync,
1732     "rebase" : P4Rebase,
1733     "clone" : P4Clone,
1734     "rollback" : P4RollBack,
1735     "branches" : P4Branches
1736 }
1737
1738
1739 def main():
1740     if len(sys.argv[1:]) == 0:
1741         printUsage(commands.keys())
1742         sys.exit(2)
1743
1744     cmd = ""
1745     cmdName = sys.argv[1]
1746     try:
1747         klass = commands[cmdName]
1748         cmd = klass()
1749     except KeyError:
1750         print "unknown command %s" % cmdName
1751         print ""
1752         printUsage(commands.keys())
1753         sys.exit(2)
1754
1755     options = cmd.options
1756     cmd.gitdir = os.environ.get("GIT_DIR", None)
1757
1758     args = sys.argv[2:]
1759
1760     if len(options) > 0:
1761         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1762
1763         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1764                                        options,
1765                                        description = cmd.description,
1766                                        formatter = HelpFormatter())
1767
1768         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1769     global verbose
1770     verbose = cmd.verbose
1771     if cmd.needsGit:
1772         if cmd.gitdir == None:
1773             cmd.gitdir = os.path.abspath(".git")
1774             if not isValidGitDir(cmd.gitdir):
1775                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1776                 if os.path.exists(cmd.gitdir):
1777                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1778                     if len(cdup) > 0:
1779                         os.chdir(cdup);
1780
1781         if not isValidGitDir(cmd.gitdir):
1782             if isValidGitDir(cmd.gitdir + "/.git"):
1783                 cmd.gitdir += "/.git"
1784             else:
1785                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1786
1787         os.environ["GIT_DIR"] = cmd.gitdir
1788
1789     if not cmd.run(args):
1790         parser.print_help()
1791
1792
1793 if __name__ == '__main__':
1794     main()