blob: 2fa581789c5d9d4dd967e1783c513d64660f8a9c [file] [log] [blame]
Simon Hausmann86949ee2007-03-19 20:59:12 +01001#!/usr/bin/env python
2#
3# git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4#
Simon Hausmannc8cbbee2007-05-28 14:43:25 +02005# Author: Simon Hausmann <simon@lst.de>
6# Copyright: 2007 Simon Hausmann <simon@lst.de>
Simon Hausmann83dce552007-03-19 22:26:36 +01007# 2007 Trolltech ASA
Simon Hausmann86949ee2007-03-19 20:59:12 +01008# License: MIT <http://www.opensource.org/licenses/mit-license.php>
9#
Eric S. Raymonda33faf22012-12-28 11:40:59 -050010import sys
11if sys.hexversion < 0x02040000:
12 # The limiter is the subprocess module
13 sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14 sys.exit(1)
Pete Wyckofff629fa52013-01-26 22:11:05 -050015import os
16import optparse
17import marshal
18import subprocess
19import tempfile
20import time
21import platform
22import re
23import shutil
Pete Wyckoffd20f0f82013-01-26 22:11:19 -050024import stat
Lars Schneidera5db4b12015-09-26 09:55:03 +020025import zipfile
26import zlib
Dennis Kaarsemaker4b07cd22015-10-20 21:31:46 +020027import ctypes
Luke Diamanddf8a9e82016-12-17 01:00:40 +000028import errno
Han-Wen Nienhuys8b41a972007-05-23 18:20:53 -030029
Brandon Caseya235e852013-01-26 11:14:33 -080030try:
31 from subprocess import CalledProcessError
32except ImportError:
33 # from python2.7:subprocess.py
34 # Exception classes used by this module.
35 class CalledProcessError(Exception):
36 """This exception is raised when a process run by check_call() returns
37 a non-zero exit status. The exit status will be stored in the
38 returncode attribute."""
39 def __init__(self, returncode, cmd):
40 self.returncode = returncode
41 self.cmd = cmd
42 def __str__(self):
43 return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
44
Han-Wen Nienhuys4addad22007-05-23 18:49:35 -030045verbose = False
Simon Hausmann86949ee2007-03-19 20:59:12 +010046
Luke Diamand06804c72012-04-11 17:21:24 +020047# Only labels/tags matching this will be imported/exported
Luke Diamandc8942a22012-04-11 17:21:24 +020048defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
Anand Kumria21a50752008-08-10 19:26:28 +010049
Luke Diamand1051ef02015-06-10 08:30:59 +010050# Grab changes in blocks of this many revisions, unless otherwise requested
51defaultBlockSize = 512
52
Anand Kumria21a50752008-08-10 19:26:28 +010053def p4_build_cmd(cmd):
54 """Build a suitable p4 command line.
55
56 This consolidates building and returning a p4 command line into one
57 location. It means that hooking into the environment, or other configuration
58 can be done more easily.
59 """
Luke Diamand6de040d2011-10-16 10:47:52 -040060 real_cmd = ["p4"]
Anand Kumriaabcaf072008-08-10 19:26:31 +010061
62 user = gitConfig("git-p4.user")
63 if len(user) > 0:
Luke Diamand6de040d2011-10-16 10:47:52 -040064 real_cmd += ["-u",user]
Anand Kumriaabcaf072008-08-10 19:26:31 +010065
66 password = gitConfig("git-p4.password")
67 if len(password) > 0:
Luke Diamand6de040d2011-10-16 10:47:52 -040068 real_cmd += ["-P", password]
Anand Kumriaabcaf072008-08-10 19:26:31 +010069
70 port = gitConfig("git-p4.port")
71 if len(port) > 0:
Luke Diamand6de040d2011-10-16 10:47:52 -040072 real_cmd += ["-p", port]
Anand Kumriaabcaf072008-08-10 19:26:31 +010073
74 host = gitConfig("git-p4.host")
75 if len(host) > 0:
Russell Myers41799aa2012-02-22 11:16:05 -080076 real_cmd += ["-H", host]
Anand Kumriaabcaf072008-08-10 19:26:31 +010077
78 client = gitConfig("git-p4.client")
79 if len(client) > 0:
Luke Diamand6de040d2011-10-16 10:47:52 -040080 real_cmd += ["-c", client]
Anand Kumriaabcaf072008-08-10 19:26:31 +010081
Lars Schneider89a6ecc2016-12-04 15:03:11 +010082 retries = gitConfigInt("git-p4.retries")
83 if retries is None:
84 # Perform 3 retries by default
85 retries = 3
Igor Kushnirbc233522016-12-29 12:22:23 +020086 if retries > 0:
87 # Provide a way to not pass this option by setting git-p4.retries to 0
88 real_cmd += ["-r", str(retries)]
Luke Diamand6de040d2011-10-16 10:47:52 -040089
90 if isinstance(cmd,basestring):
91 real_cmd = ' '.join(real_cmd) + ' ' + cmd
92 else:
93 real_cmd += cmd
Anand Kumria21a50752008-08-10 19:26:28 +010094 return real_cmd
95
Luke Diamand378f7be2016-12-13 21:51:28 +000096def git_dir(path):
97 """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
98 This won't automatically add ".git" to a directory.
99 """
100 d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
101 if not d or len(d) == 0:
102 return None
103 else:
104 return d
105
Miklós Fazekasbbd84862013-03-11 17:45:29 -0400106def chdir(path, is_client_path=False):
107 """Do chdir to the given path, and set the PWD environment
108 variable for use by P4. It does not look at getcwd() output.
109 Since we're not using the shell, it is necessary to set the
110 PWD environment variable explicitly.
111
112 Normally, expand the path to force it to be absolute. This
113 addresses the use of relative path names inside P4 settings,
114 e.g. P4CONFIG=.p4config. P4 does not simply open the filename
115 as given; it looks for .p4config using PWD.
116
117 If is_client_path, the path was handed to us directly by p4,
118 and may be a symbolic link. Do not call os.getcwd() in this
119 case, because it will cause p4 to think that PWD is not inside
120 the client path.
121 """
122
123 os.chdir(path)
124 if not is_client_path:
125 path = os.getcwd()
126 os.environ['PWD'] = path
Robert Blum053fd0c2008-08-01 12:50:03 -0700127
Lars Schneider4d25dc42015-09-26 09:55:02 +0200128def calcDiskFree():
129 """Return free space in bytes on the disk of the given dirname."""
130 if platform.system() == 'Windows':
131 free_bytes = ctypes.c_ulonglong(0)
132 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
133 return free_bytes.value
134 else:
135 st = os.statvfs(os.getcwd())
136 return st.f_bavail * st.f_frsize
137
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -0300138def die(msg):
139 if verbose:
140 raise Exception(msg)
141 else:
142 sys.stderr.write(msg + "\n")
143 sys.exit(1)
144
Luke Diamand6de040d2011-10-16 10:47:52 -0400145def write_pipe(c, stdin):
Han-Wen Nienhuys4addad22007-05-23 18:49:35 -0300146 if verbose:
Luke Diamand6de040d2011-10-16 10:47:52 -0400147 sys.stderr.write('Writing pipe: %s\n' % str(c))
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300148
Luke Diamand6de040d2011-10-16 10:47:52 -0400149 expand = isinstance(c,basestring)
150 p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
151 pipe = p.stdin
152 val = pipe.write(stdin)
153 pipe.close()
154 if p.wait():
155 die('Command failed: %s' % str(c))
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300156
157 return val
158
Luke Diamand6de040d2011-10-16 10:47:52 -0400159def p4_write_pipe(c, stdin):
Anand Kumriad9429192008-08-14 23:40:38 +0100160 real_cmd = p4_build_cmd(c)
Luke Diamand6de040d2011-10-16 10:47:52 -0400161 return write_pipe(real_cmd, stdin)
Anand Kumriad9429192008-08-14 23:40:38 +0100162
Luke Diamand78871bf2017-04-15 11:36:08 +0100163def read_pipe_full(c):
164 """ Read output from command. Returns a tuple
165 of the return status, stdout text and stderr
166 text.
167 """
Han-Wen Nienhuys4addad22007-05-23 18:49:35 -0300168 if verbose:
Luke Diamand6de040d2011-10-16 10:47:52 -0400169 sys.stderr.write('Reading pipe: %s\n' % str(c))
Han-Wen Nienhuys8b41a972007-05-23 18:20:53 -0300170
Luke Diamand6de040d2011-10-16 10:47:52 -0400171 expand = isinstance(c,basestring)
Lars Schneider1f5f3902015-09-21 12:01:41 +0200172 p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
173 (out, err) = p.communicate()
Luke Diamand78871bf2017-04-15 11:36:08 +0100174 return (p.returncode, out, err)
175
176def read_pipe(c, ignore_error=False):
177 """ Read output from command. Returns the output text on
178 success. On failure, terminates execution, unless
179 ignore_error is True, when it returns an empty string.
180 """
181 (retcode, out, err) = read_pipe_full(c)
182 if retcode != 0:
183 if ignore_error:
184 out = ""
185 else:
186 die('Command failed: %s\nError: %s' % (str(c), err))
Lars Schneider1f5f3902015-09-21 12:01:41 +0200187 return out
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300188
Luke Diamand78871bf2017-04-15 11:36:08 +0100189def read_pipe_text(c):
190 """ Read output from a command with trailing whitespace stripped.
191 On error, returns None.
192 """
193 (retcode, out, err) = read_pipe_full(c)
194 if retcode != 0:
195 return None
196 else:
197 return out.rstrip()
198
Anand Kumriad9429192008-08-14 23:40:38 +0100199def p4_read_pipe(c, ignore_error=False):
200 real_cmd = p4_build_cmd(c)
201 return read_pipe(real_cmd, ignore_error)
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300202
Han-Wen Nienhuysbce4c5f2007-05-23 17:14:33 -0300203def read_pipe_lines(c):
Han-Wen Nienhuys4addad22007-05-23 18:49:35 -0300204 if verbose:
Luke Diamand6de040d2011-10-16 10:47:52 -0400205 sys.stderr.write('Reading pipe: %s\n' % str(c))
206
207 expand = isinstance(c, basestring)
208 p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
209 pipe = p.stdout
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300210 val = pipe.readlines()
Luke Diamand6de040d2011-10-16 10:47:52 -0400211 if pipe.close() or p.wait():
212 die('Command failed: %s' % str(c))
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300213
214 return val
Simon Hausmanncaace112007-05-15 14:57:57 +0200215
Anand Kumria23181212008-08-10 19:26:24 +0100216def p4_read_pipe_lines(c):
217 """Specifically invoke p4 on the command supplied. """
Anand Kumria155af832008-08-10 19:26:30 +0100218 real_cmd = p4_build_cmd(c)
Anand Kumria23181212008-08-10 19:26:24 +0100219 return read_pipe_lines(real_cmd)
220
Gary Gibbons8e9497c2012-07-12 19:29:00 -0400221def p4_has_command(cmd):
222 """Ask p4 for help on this command. If it returns an error, the
223 command does not exist in this version of p4."""
224 real_cmd = p4_build_cmd(["help", cmd])
225 p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
226 stderr=subprocess.PIPE)
227 p.communicate()
228 return p.returncode == 0
229
Pete Wyckoff249da4c2012-11-23 17:35:35 -0500230def p4_has_move_command():
231 """See if the move command exists, that it supports -k, and that
232 it has not been administratively disabled. The arguments
233 must be correct, but the filenames do not have to exist. Use
234 ones with wildcards so even if they exist, it will fail."""
235
236 if not p4_has_command("move"):
237 return False
238 cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
239 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
240 (out, err) = p.communicate()
241 # return code will be 1 in either case
242 if err.find("Invalid option") >= 0:
243 return False
244 if err.find("disabled") >= 0:
245 return False
246 # assume it failed because @... was invalid changelist
247 return True
248
Luke Diamandcbff4b22015-11-21 09:54:40 +0000249def system(cmd, ignore_error=False):
Luke Diamand6de040d2011-10-16 10:47:52 -0400250 expand = isinstance(cmd,basestring)
Han-Wen Nienhuys4addad22007-05-23 18:49:35 -0300251 if verbose:
Luke Diamand6de040d2011-10-16 10:47:52 -0400252 sys.stderr.write("executing %s\n" % str(cmd))
Brandon Caseya235e852013-01-26 11:14:33 -0800253 retcode = subprocess.call(cmd, shell=expand)
Luke Diamandcbff4b22015-11-21 09:54:40 +0000254 if retcode and not ignore_error:
Brandon Caseya235e852013-01-26 11:14:33 -0800255 raise CalledProcessError(retcode, cmd)
Han-Wen Nienhuys6754a292007-05-23 17:41:50 -0300256
Luke Diamandcbff4b22015-11-21 09:54:40 +0000257 return retcode
258
Anand Kumriabf9320f2008-08-10 19:26:26 +0100259def p4_system(cmd):
260 """Specifically invoke p4 as the system command. """
Anand Kumria155af832008-08-10 19:26:30 +0100261 real_cmd = p4_build_cmd(cmd)
Luke Diamand6de040d2011-10-16 10:47:52 -0400262 expand = isinstance(real_cmd, basestring)
Brandon Caseya235e852013-01-26 11:14:33 -0800263 retcode = subprocess.call(real_cmd, shell=expand)
264 if retcode:
265 raise CalledProcessError(retcode, real_cmd)
Luke Diamand6de040d2011-10-16 10:47:52 -0400266
Pete Wyckoff7f0e5962013-01-26 22:11:13 -0500267_p4_version_string = None
268def p4_version_string():
269 """Read the version string, showing just the last line, which
270 hopefully is the interesting version bit.
271
272 $ p4 -V
273 Perforce - The Fast Software Configuration Management System.
274 Copyright 1995-2011 Perforce Software. All rights reserved.
275 Rev. P4/NTX86/2011.1/393975 (2011/12/16).
276 """
277 global _p4_version_string
278 if not _p4_version_string:
279 a = p4_read_pipe_lines(["-V"])
280 _p4_version_string = a[-1].rstrip()
281 return _p4_version_string
282
Luke Diamand6de040d2011-10-16 10:47:52 -0400283def p4_integrate(src, dest):
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400284 p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
Luke Diamand6de040d2011-10-16 10:47:52 -0400285
Pete Wyckoff8d7ec362012-04-29 20:57:14 -0400286def p4_sync(f, *options):
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400287 p4_system(["sync"] + list(options) + [wildcard_encode(f)])
Luke Diamand6de040d2011-10-16 10:47:52 -0400288
289def p4_add(f):
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400290 # forcibly add file names with wildcards
291 if wildcard_present(f):
292 p4_system(["add", "-f", f])
293 else:
294 p4_system(["add", f])
Luke Diamand6de040d2011-10-16 10:47:52 -0400295
296def p4_delete(f):
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400297 p4_system(["delete", wildcard_encode(f)])
Luke Diamand6de040d2011-10-16 10:47:52 -0400298
Romain Picarda02b8bc2016-01-12 13:43:47 +0100299def p4_edit(f, *options):
300 p4_system(["edit"] + list(options) + [wildcard_encode(f)])
Luke Diamand6de040d2011-10-16 10:47:52 -0400301
302def p4_revert(f):
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400303 p4_system(["revert", wildcard_encode(f)])
Luke Diamand6de040d2011-10-16 10:47:52 -0400304
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400305def p4_reopen(type, f):
306 p4_system(["reopen", "-t", type, wildcard_encode(f)])
Anand Kumriabf9320f2008-08-10 19:26:26 +0100307
Luke Diamand46c609e2016-12-02 22:43:19 +0000308def p4_reopen_in_change(changelist, files):
309 cmd = ["reopen", "-c", str(changelist)] + files
310 p4_system(cmd)
311
Gary Gibbons8e9497c2012-07-12 19:29:00 -0400312def p4_move(src, dest):
313 p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
314
Luke Diamand1051ef02015-06-10 08:30:59 +0100315def p4_last_change():
Miguel Torroja1997e912017-07-13 09:00:35 +0200316 results = p4CmdList(["changes", "-m", "1"], skip_info=True)
Luke Diamand1051ef02015-06-10 08:30:59 +0100317 return int(results[0]['change'])
318
Pete Wyckoff18fa13d2012-11-23 17:35:34 -0500319def p4_describe(change):
320 """Make sure it returns a valid result by checking for
321 the presence of field "time". Return a dict of the
322 results."""
323
Miguel Torroja1997e912017-07-13 09:00:35 +0200324 ds = p4CmdList(["describe", "-s", str(change)], skip_info=True)
Pete Wyckoff18fa13d2012-11-23 17:35:34 -0500325 if len(ds) != 1:
326 die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
327
328 d = ds[0]
329
330 if "p4ExitCode" in d:
331 die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
332 str(d)))
333 if "code" in d:
334 if d["code"] == "error":
335 die("p4 describe -s %d returned error code: %s" % (change, str(d)))
336
337 if "time" not in d:
338 die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
339
340 return d
341
Pete Wyckoff9cffb8c2011-10-16 10:45:01 -0400342#
343# Canonicalize the p4 type and return a tuple of the
344# base type, plus any modifiers. See "p4 help filetypes"
345# for a list and explanation.
346#
347def split_p4_type(p4type):
David Brownb9fc6ea2007-09-19 13:12:48 -0700348
Pete Wyckoff9cffb8c2011-10-16 10:45:01 -0400349 p4_filetypes_historical = {
350 "ctempobj": "binary+Sw",
351 "ctext": "text+C",
352 "cxtext": "text+Cx",
353 "ktext": "text+k",
354 "kxtext": "text+kx",
355 "ltext": "text+F",
356 "tempobj": "binary+FSw",
357 "ubinary": "binary+F",
358 "uresource": "resource+F",
359 "uxbinary": "binary+Fx",
360 "xbinary": "binary+x",
361 "xltext": "text+Fx",
362 "xtempobj": "binary+Swx",
363 "xtext": "text+x",
364 "xunicode": "unicode+x",
365 "xutf16": "utf16+x",
366 }
367 if p4type in p4_filetypes_historical:
368 p4type = p4_filetypes_historical[p4type]
369 mods = ""
370 s = p4type.split("+")
371 base = s[0]
372 mods = ""
373 if len(s) > 1:
374 mods = s[1]
375 return (base, mods)
376
Luke Diamand60df0712012-02-23 07:51:30 +0000377#
378# return the raw p4 type of a file (text, text+ko, etc)
379#
Pete Wyckoff79467e62014-01-21 18:16:45 -0500380def p4_type(f):
381 results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
Luke Diamand60df0712012-02-23 07:51:30 +0000382 return results[0]['headType']
383
384#
385# Given a type base and modifier, return a regexp matching
386# the keywords that can be expanded in the file
387#
388def p4_keywords_regexp_for_type(base, type_mods):
389 if base in ("text", "unicode", "binary"):
390 kwords = None
391 if "ko" in type_mods:
392 kwords = 'Id|Header'
393 elif "k" in type_mods:
394 kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
395 else:
396 return None
397 pattern = r"""
398 \$ # Starts with a dollar, followed by...
399 (%s) # one of the keywords, followed by...
Pete Wyckoff6b2bf412012-11-04 17:04:02 -0500400 (:[^$\n]+)? # possibly an old expansion, followed by...
Luke Diamand60df0712012-02-23 07:51:30 +0000401 \$ # another dollar
402 """ % kwords
403 return pattern
404 else:
405 return None
406
407#
408# Given a file, return a regexp matching the possible
409# RCS keywords that will be expanded, or None for files
410# with kw expansion turned off.
411#
412def p4_keywords_regexp_for_file(file):
413 if not os.path.exists(file):
414 return None
415 else:
416 (type_base, type_mods) = split_p4_type(p4_type(file))
417 return p4_keywords_regexp_for_type(type_base, type_mods)
David Brownb9fc6ea2007-09-19 13:12:48 -0700418
Chris Pettittc65b6702007-11-01 20:43:14 -0700419def setP4ExecBit(file, mode):
420 # Reopens an already open file and changes the execute bit to match
421 # the execute bit setting in the passed in mode.
422
423 p4Type = "+x"
424
425 if not isModeExec(mode):
426 p4Type = getP4OpenedType(file)
427 p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
428 p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
429 if p4Type[-1] == "+":
430 p4Type = p4Type[0:-1]
431
Luke Diamand6de040d2011-10-16 10:47:52 -0400432 p4_reopen(p4Type, file)
Chris Pettittc65b6702007-11-01 20:43:14 -0700433
434def getP4OpenedType(file):
435 # Returns the perforce file type for the given file.
436
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400437 result = p4_read_pipe(["opened", wildcard_encode(file)])
Blair Holloway34a0dbf2015-04-04 09:46:03 +0100438 match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
Chris Pettittc65b6702007-11-01 20:43:14 -0700439 if match:
440 return match.group(1)
441 else:
Marius Storm-Olsenf3e5ae42008-03-28 15:40:40 +0100442 die("Could not determine file type for %s (result: '%s')" % (file, result))
Chris Pettittc65b6702007-11-01 20:43:14 -0700443
Luke Diamand06804c72012-04-11 17:21:24 +0200444# Return the set of all p4 labels
445def getP4Labels(depotPaths):
446 labels = set()
447 if isinstance(depotPaths,basestring):
448 depotPaths = [depotPaths]
449
450 for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
451 label = l['label']
452 labels.add(label)
453
454 return labels
455
456# Return the set of all git tags
457def getGitTags():
458 gitTags = set()
459 for line in read_pipe_lines(["git", "tag"]):
460 tag = line.strip()
461 gitTags.add(tag)
462 return gitTags
463
Chris Pettittb43b0a32007-11-01 20:43:13 -0700464def diffTreePattern():
465 # This is a simple generator for the diff tree regex pattern. This could be
466 # a class variable if this and parseDiffTreeEntry were a part of a class.
467 pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
468 while True:
469 yield pattern
470
471def parseDiffTreeEntry(entry):
472 """Parses a single diff tree entry into its component elements.
473
474 See git-diff-tree(1) manpage for details about the format of the diff
475 output. This method returns a dictionary with the following elements:
476
477 src_mode - The mode of the source file
478 dst_mode - The mode of the destination file
479 src_sha1 - The sha1 for the source file
480 dst_sha1 - The sha1 fr the destination file
481 status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
482 status_score - The score for the status (applicable for 'C' and 'R'
483 statuses). This is None if there is no score.
484 src - The path for the source file.
485 dst - The path for the destination file. This is only present for
486 copy or renames. If it is not present, this is None.
487
488 If the pattern is not matched, None is returned."""
489
490 match = diffTreePattern().next().match(entry)
491 if match:
492 return {
493 'src_mode': match.group(1),
494 'dst_mode': match.group(2),
495 'src_sha1': match.group(3),
496 'dst_sha1': match.group(4),
497 'status': match.group(5),
498 'status_score': match.group(6),
499 'src': match.group(7),
500 'dst': match.group(10)
501 }
502 return None
503
Chris Pettittc65b6702007-11-01 20:43:14 -0700504def isModeExec(mode):
505 # Returns True if the given git mode represents an executable file,
506 # otherwise False.
507 return mode[-3:] == "755"
508
509def isModeExecChanged(src_mode, dst_mode):
510 return isModeExec(src_mode) != isModeExec(dst_mode)
511
Miguel Torroja1997e912017-07-13 09:00:35 +0200512def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None, skip_info=False):
Luke Diamand6de040d2011-10-16 10:47:52 -0400513
514 if isinstance(cmd,basestring):
515 cmd = "-G " + cmd
516 expand = True
517 else:
518 cmd = ["-G"] + cmd
519 expand = False
520
521 cmd = p4_build_cmd(cmd)
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -0300522 if verbose:
Luke Diamand6de040d2011-10-16 10:47:52 -0400523 sys.stderr.write("Opening pipe: %s\n" % str(cmd))
Scott Lamb9f90c732007-07-15 20:58:10 -0700524
525 # Use a temporary file to avoid deadlocks without
526 # subprocess.communicate(), which would put another copy
527 # of stdout into memory.
528 stdin_file = None
529 if stdin is not None:
530 stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
Luke Diamand6de040d2011-10-16 10:47:52 -0400531 if isinstance(stdin,basestring):
532 stdin_file.write(stdin)
533 else:
534 for i in stdin:
535 stdin_file.write(i + '\n')
Scott Lamb9f90c732007-07-15 20:58:10 -0700536 stdin_file.flush()
537 stdin_file.seek(0)
538
Luke Diamand6de040d2011-10-16 10:47:52 -0400539 p4 = subprocess.Popen(cmd,
540 shell=expand,
Scott Lamb9f90c732007-07-15 20:58:10 -0700541 stdin=stdin_file,
542 stdout=subprocess.PIPE)
Simon Hausmann86949ee2007-03-19 20:59:12 +0100543
544 result = []
545 try:
546 while True:
Scott Lamb9f90c732007-07-15 20:58:10 -0700547 entry = marshal.load(p4.stdout)
Miguel Torroja1997e912017-07-13 09:00:35 +0200548 if skip_info:
549 if 'code' in entry and entry['code'] == 'info':
550 continue
Andrew Garberc3f61632011-04-07 02:01:21 -0400551 if cb is not None:
552 cb(entry)
553 else:
554 result.append(entry)
Simon Hausmann86949ee2007-03-19 20:59:12 +0100555 except EOFError:
556 pass
Scott Lamb9f90c732007-07-15 20:58:10 -0700557 exitCode = p4.wait()
558 if exitCode != 0:
Simon Hausmannac3e0d72007-05-23 23:32:32 +0200559 entry = {}
560 entry["p4ExitCode"] = exitCode
561 result.append(entry)
Simon Hausmann86949ee2007-03-19 20:59:12 +0100562
563 return result
564
565def p4Cmd(cmd):
566 list = p4CmdList(cmd)
567 result = {}
568 for entry in list:
569 result.update(entry)
570 return result;
571
Simon Hausmanncb2c9db2007-03-24 09:15:11 +0100572def p4Where(depotPath):
573 if not depotPath.endswith("/"):
574 depotPath += "/"
Vitor Antunescd884102015-04-21 23:49:30 +0100575 depotPathLong = depotPath + "..."
576 outputList = p4CmdList(["where", depotPathLong])
Tor Arvid Lund7f705dc2008-12-04 14:37:33 +0100577 output = None
578 for entry in outputList:
Tor Arvid Lund75bc9572008-12-09 16:41:50 +0100579 if "depotFile" in entry:
Vitor Antunescd884102015-04-21 23:49:30 +0100580 # Search for the base client side depot path, as long as it starts with the branch's P4 path.
581 # The base path always ends with "/...".
582 if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
Tor Arvid Lund75bc9572008-12-09 16:41:50 +0100583 output = entry
584 break
585 elif "data" in entry:
586 data = entry.get("data")
587 space = data.find(" ")
588 if data[:space] == depotPath:
589 output = entry
590 break
Tor Arvid Lund7f705dc2008-12-04 14:37:33 +0100591 if output == None:
592 return ""
Simon Hausmanndc524032007-05-21 09:34:56 +0200593 if output["code"] == "error":
594 return ""
Simon Hausmanncb2c9db2007-03-24 09:15:11 +0100595 clientPath = ""
596 if "path" in output:
597 clientPath = output.get("path")
598 elif "data" in output:
599 data = output.get("data")
600 lastSpace = data.rfind(" ")
601 clientPath = data[lastSpace + 1:]
602
603 if clientPath.endswith("..."):
604 clientPath = clientPath[:-3]
605 return clientPath
606
Simon Hausmann86949ee2007-03-19 20:59:12 +0100607def currentGitBranch():
Luke Diamandeff45112017-04-15 11:36:09 +0100608 return read_pipe_text(["git", "symbolic-ref", "--short", "-q", "HEAD"])
Simon Hausmann86949ee2007-03-19 20:59:12 +0100609
Simon Hausmann4f5cf762007-03-19 22:25:17 +0100610def isValidGitDir(path):
Luke Diamand378f7be2016-12-13 21:51:28 +0000611 return git_dir(path) != None
Simon Hausmann4f5cf762007-03-19 22:25:17 +0100612
Simon Hausmann463e8af2007-05-17 09:13:54 +0200613def parseRevision(ref):
Han-Wen Nienhuysb25b2062007-05-23 18:49:35 -0300614 return read_pipe("git rev-parse %s" % ref).strip()
Simon Hausmann463e8af2007-05-17 09:13:54 +0200615
Pete Wyckoff28755db2011-12-24 21:07:40 -0500616def branchExists(ref):
617 rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
618 ignore_error=True)
619 return len(rev) > 0
620
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100621def extractLogMessageFromGitCommit(commit):
622 logMessage = ""
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300623
624 ## fixme: title is first line of commit, not 1st paragraph.
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100625 foundTitle = False
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -0300626 for log in read_pipe_lines("git cat-file commit %s" % commit):
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100627 if not foundTitle:
628 if len(log) == 1:
Simon Hausmann1c094182007-05-01 23:15:48 +0200629 foundTitle = True
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100630 continue
631
632 logMessage += log
633 return logMessage
634
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -0300635def extractSettingsGitLog(log):
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100636 values = {}
637 for line in log.split("\n"):
638 line = line.strip()
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -0300639 m = re.search (r"^ *\[git-p4: (.*)\]$", line)
640 if not m:
641 continue
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100642
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -0300643 assignments = m.group(1).split (':')
644 for a in assignments:
645 vals = a.split ('=')
646 key = vals[0].strip()
647 val = ('='.join (vals[1:])).strip()
648 if val.endswith ('\"') and val.startswith('"'):
649 val = val[1:-1]
650
651 values[key] = val
652
Simon Hausmann845b42c2007-06-07 09:19:34 +0200653 paths = values.get("depot-paths")
654 if not paths:
655 paths = values.get("depot-path")
Simon Hausmanna3fdd572007-06-07 22:54:32 +0200656 if paths:
657 values['depot-paths'] = paths.split(',')
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -0300658 return values
Simon Hausmann6ae8de82007-03-22 21:10:25 +0100659
Simon Hausmann8136a632007-03-22 21:27:14 +0100660def gitBranchExists(branch):
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -0300661 proc = subprocess.Popen(["git", "rev-parse", branch],
662 stderr=subprocess.PIPE, stdout=subprocess.PIPE);
Simon Hausmanncaace112007-05-15 14:57:57 +0200663 return proc.wait() == 0;
Simon Hausmann8136a632007-03-22 21:27:14 +0100664
John Chapman36bd8442008-11-08 14:22:49 +1100665_gitConfig = {}
Pete Wyckoffb345d6c2013-01-26 22:11:23 -0500666
Lars Schneider692e1792015-09-26 09:54:58 +0200667def gitConfig(key, typeSpecifier=None):
John Chapman36bd8442008-11-08 14:22:49 +1100668 if not _gitConfig.has_key(key):
Lars Schneider692e1792015-09-26 09:54:58 +0200669 cmd = [ "git", "config" ]
670 if typeSpecifier:
671 cmd += [ typeSpecifier ]
672 cmd += [ key ]
Pete Wyckoffb345d6c2013-01-26 22:11:23 -0500673 s = read_pipe(cmd, ignore_error=True)
674 _gitConfig[key] = s.strip()
John Chapman36bd8442008-11-08 14:22:49 +1100675 return _gitConfig[key]
Simon Hausmann01265102007-05-25 10:36:10 +0200676
Pete Wyckoff0d609032013-01-26 22:11:24 -0500677def gitConfigBool(key):
678 """Return a bool, using git config --bool. It is True only if the
679 variable is set to true, and False if set to false or not present
680 in the config."""
681
682 if not _gitConfig.has_key(key):
Lars Schneider692e1792015-09-26 09:54:58 +0200683 _gitConfig[key] = gitConfig(key, '--bool') == "true"
Simon Hausmann062410b2007-07-18 10:56:31 +0200684 return _gitConfig[key]
685
Lars Schneidercb1dafd2015-09-26 09:54:59 +0200686def gitConfigInt(key):
687 if not _gitConfig.has_key(key):
688 cmd = [ "git", "config", "--int", key ]
Simon Hausmannb9847332007-03-20 20:54:23 +0100689 s = read_pipe(cmd, ignore_error=True)
Simon Hausmann062410b2007-07-18 10:56:31 +0200690 v = s.strip()
Lars Schneidercb1dafd2015-09-26 09:54:59 +0200691 try:
692 _gitConfig[key] = int(gitConfig(key, '--int'))
693 except ValueError:
694 _gitConfig[key] = None
Simon Hausmann062410b2007-07-18 10:56:31 +0200695 return _gitConfig[key]
696
Vitor Antunes7199cf12011-08-19 00:44:05 +0100697def gitConfigList(key):
698 if not _gitConfig.has_key(key):
Pete Wyckoff2abba302013-01-26 22:11:22 -0500699 s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
George Vanburghc3c2b052017-01-25 09:17:29 +0000700 _gitConfig[key] = s.strip().splitlines()
Lars Schneider7960e702015-09-26 09:55:00 +0200701 if _gitConfig[key] == ['']:
702 _gitConfig[key] = []
Vitor Antunes7199cf12011-08-19 00:44:05 +0100703 return _gitConfig[key]
704
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500705def p4BranchesInGit(branchesAreInRemotes=True):
706 """Find all the branches whose names start with "p4/", looking
707 in remotes or heads as specified by the argument. Return
708 a dictionary of { branch: revision } for each one found.
709 The branch names are the short names, without any
710 "p4/" prefix."""
711
Simon Hausmann062410b2007-07-18 10:56:31 +0200712 branches = {}
713
714 cmdline = "git rev-parse --symbolic "
715 if branchesAreInRemotes:
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500716 cmdline += "--remotes"
Simon Hausmann062410b2007-07-18 10:56:31 +0200717 else:
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500718 cmdline += "--branches"
Simon Hausmann062410b2007-07-18 10:56:31 +0200719
720 for line in read_pipe_lines(cmdline):
721 line = line.strip()
722
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500723 # only import to p4/
724 if not line.startswith('p4/'):
Simon Hausmann062410b2007-07-18 10:56:31 +0200725 continue
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500726 # special symbolic ref to p4/master
727 if line == "p4/HEAD":
728 continue
Simon Hausmann062410b2007-07-18 10:56:31 +0200729
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500730 # strip off p4/ prefix
731 branch = line[len("p4/"):]
Simon Hausmann062410b2007-07-18 10:56:31 +0200732
733 branches[branch] = parseRevision(line)
Pete Wyckoff2c8037e2013-01-14 19:46:57 -0500734
Simon Hausmann062410b2007-07-18 10:56:31 +0200735 return branches
736
Pete Wyckoff5a8e84c2013-01-14 19:47:05 -0500737def branch_exists(branch):
738 """Make sure that the given ref name really exists."""
739
740 cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
741 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
742 out, _ = p.communicate()
743 if p.returncode:
744 return False
745 # expect exactly one line of output: the branch name
746 return out.rstrip() == branch
747
Simon Hausmann9ceab362007-06-22 00:01:57 +0200748def findUpstreamBranchPoint(head = "HEAD"):
Simon Hausmann86506fe2007-07-18 12:40:12 +0200749 branches = p4BranchesInGit()
750 # map from depot-path to branch name
751 branchByDepotPath = {}
752 for branch in branches.keys():
753 tip = branches[branch]
754 log = extractLogMessageFromGitCommit(tip)
755 settings = extractSettingsGitLog(log)
756 if settings.has_key("depot-paths"):
757 paths = ",".join(settings["depot-paths"])
758 branchByDepotPath[paths] = "remotes/p4/" + branch
759
Simon Hausmann27d2d812007-06-12 14:31:59 +0200760 settings = None
Simon Hausmann27d2d812007-06-12 14:31:59 +0200761 parent = 0
762 while parent < 65535:
Simon Hausmann9ceab362007-06-22 00:01:57 +0200763 commit = head + "~%s" % parent
Simon Hausmann27d2d812007-06-12 14:31:59 +0200764 log = extractLogMessageFromGitCommit(commit)
765 settings = extractSettingsGitLog(log)
Simon Hausmann86506fe2007-07-18 12:40:12 +0200766 if settings.has_key("depot-paths"):
767 paths = ",".join(settings["depot-paths"])
768 if branchByDepotPath.has_key(paths):
769 return [branchByDepotPath[paths], settings]
Simon Hausmann27d2d812007-06-12 14:31:59 +0200770
Simon Hausmann86506fe2007-07-18 12:40:12 +0200771 parent = parent + 1
Simon Hausmann27d2d812007-06-12 14:31:59 +0200772
Simon Hausmann86506fe2007-07-18 12:40:12 +0200773 return ["", settings]
Simon Hausmann27d2d812007-06-12 14:31:59 +0200774
Simon Hausmann5ca44612007-08-24 17:44:16 +0200775def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
776 if not silent:
777 print ("Creating/updating branch(es) in %s based on origin branch(es)"
778 % localRefPrefix)
779
780 originPrefix = "origin/p4/"
781
782 for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
783 line = line.strip()
784 if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
785 continue
786
787 headName = line[len(originPrefix):]
788 remoteHead = localRefPrefix + headName
789 originHead = line
790
791 original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
792 if (not original.has_key('depot-paths')
793 or not original.has_key('change')):
794 continue
795
796 update = False
797 if not gitBranchExists(remoteHead):
798 if verbose:
799 print "creating %s" % remoteHead
800 update = True
801 else:
802 settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
803 if settings.has_key('change') > 0:
804 if settings['depot-paths'] == original['depot-paths']:
805 originP4Change = int(original['change'])
806 p4Change = int(settings['change'])
807 if originP4Change > p4Change:
808 print ("%s (%s) is newer than %s (%s). "
809 "Updating p4 branch from origin."
810 % (originHead, originP4Change,
811 remoteHead, p4Change))
812 update = True
813 else:
814 print ("Ignoring: %s was imported from %s while "
815 "%s was imported from %s"
816 % (originHead, ','.join(original['depot-paths']),
817 remoteHead, ','.join(settings['depot-paths'])))
818
819 if update:
820 system("git update-ref %s %s" % (remoteHead, originHead))
821
822def originP4BranchesExist():
823 return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
824
Simon Hausmann4f6432d2007-08-26 15:56:36 +0200825
Luke Diamand1051ef02015-06-10 08:30:59 +0100826def p4ParseNumericChangeRange(parts):
827 changeStart = int(parts[0][1:])
828 if parts[1] == '#head':
829 changeEnd = p4_last_change()
830 else:
831 changeEnd = int(parts[1])
832
833 return (changeStart, changeEnd)
834
835def chooseBlockSize(blockSize):
836 if blockSize:
837 return blockSize
838 else:
839 return defaultBlockSize
840
841def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
842 assert depotPaths
843
844 # Parse the change range into start and end. Try to find integer
845 # revision ranges as these can be broken up into blocks to avoid
846 # hitting server-side limits (maxrows, maxscanresults). But if
847 # that doesn't work, fall back to using the raw revision specifier
848 # strings, without using block mode.
849
Lex Spoon96b2d542015-04-20 11:00:20 -0400850 if changeRange is None or changeRange == '':
Luke Diamand1051ef02015-06-10 08:30:59 +0100851 changeStart = 1
852 changeEnd = p4_last_change()
853 block_size = chooseBlockSize(requestedBlockSize)
Lex Spoon96b2d542015-04-20 11:00:20 -0400854 else:
855 parts = changeRange.split(',')
856 assert len(parts) == 2
Luke Diamand1051ef02015-06-10 08:30:59 +0100857 try:
858 (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
859 block_size = chooseBlockSize(requestedBlockSize)
860 except:
861 changeStart = parts[0][1:]
862 changeEnd = parts[1]
863 if requestedBlockSize:
864 die("cannot use --changes-block-size with non-numeric revisions")
865 block_size = None
Lex Spoon96b2d542015-04-20 11:00:20 -0400866
George Vanburgh9943e5b2016-12-17 22:11:23 +0000867 changes = set()
Lex Spoon96b2d542015-04-20 11:00:20 -0400868
Sam Hocevar1f90a642015-12-19 09:39:40 +0000869 # Retrieve changes a block at a time, to prevent running
870 # into a MaxResults/MaxScanRows error from the server.
Luke Diamand1051ef02015-06-10 08:30:59 +0100871
Sam Hocevar1f90a642015-12-19 09:39:40 +0000872 while True:
873 cmd = ['changes']
Luke Diamand1051ef02015-06-10 08:30:59 +0100874
Sam Hocevar1f90a642015-12-19 09:39:40 +0000875 if block_size:
876 end = min(changeEnd, changeStart + block_size)
877 revisionRange = "%d,%d" % (changeStart, end)
878 else:
879 revisionRange = "%s,%s" % (changeStart, changeEnd)
Luke Diamand1051ef02015-06-10 08:30:59 +0100880
Sam Hocevar1f90a642015-12-19 09:39:40 +0000881 for p in depotPaths:
Luke Diamand1051ef02015-06-10 08:30:59 +0100882 cmd += ["%s...@%s" % (p, revisionRange)]
883
Sam Hocevar1f90a642015-12-19 09:39:40 +0000884 # Insert changes in chronological order
Miguel Torrojab596b3b2017-07-13 09:00:34 +0200885 for entry in reversed(p4CmdList(cmd)):
886 if entry.has_key('p4ExitCode'):
887 die('Error retrieving changes descriptions ({})'.format(entry['p4ExitCode']))
888 if not entry.has_key('change'):
889 continue
890 changes.add(int(entry['change']))
Luke Diamand1051ef02015-06-10 08:30:59 +0100891
Sam Hocevar1f90a642015-12-19 09:39:40 +0000892 if not block_size:
893 break
Luke Diamand1051ef02015-06-10 08:30:59 +0100894
Sam Hocevar1f90a642015-12-19 09:39:40 +0000895 if end >= changeEnd:
896 break
Luke Diamand1051ef02015-06-10 08:30:59 +0100897
Sam Hocevar1f90a642015-12-19 09:39:40 +0000898 changeStart = end + 1
Simon Hausmann4f6432d2007-08-26 15:56:36 +0200899
Sam Hocevar1f90a642015-12-19 09:39:40 +0000900 changes = sorted(changes)
901 return changes
Simon Hausmann4f6432d2007-08-26 15:56:36 +0200902
Tor Arvid Lundd53de8b2011-03-15 13:08:02 +0100903def p4PathStartsWith(path, prefix):
904 # This method tries to remedy a potential mixed-case issue:
905 #
906 # If UserA adds //depot/DirA/file1
907 # and UserB adds //depot/dira/file2
908 #
909 # we may or may not have a problem. If you have core.ignorecase=true,
910 # we treat DirA and dira as the same directory
Pete Wyckoff0d609032013-01-26 22:11:24 -0500911 if gitConfigBool("core.ignorecase"):
Tor Arvid Lundd53de8b2011-03-15 13:08:02 +0100912 return path.lower().startswith(prefix.lower())
913 return path.startswith(prefix)
914
Pete Wyckoff543987b2012-02-25 20:06:25 -0500915def getClientSpec():
916 """Look at the p4 client spec, create a View() object that contains
917 all the mappings, and return it."""
918
919 specList = p4CmdList("client -o")
920 if len(specList) != 1:
921 die('Output from "client -o" is %d lines, expecting 1' %
922 len(specList))
923
924 # dictionary of all client parameters
925 entry = specList[0]
926
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +0900927 # the //client/ name
928 client_name = entry["Client"]
929
Pete Wyckoff543987b2012-02-25 20:06:25 -0500930 # just the keys that start with "View"
931 view_keys = [ k for k in entry.keys() if k.startswith("View") ]
932
933 # hold this new View
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +0900934 view = View(client_name)
Pete Wyckoff543987b2012-02-25 20:06:25 -0500935
936 # append the lines, in order, to the view
937 for view_num in range(len(view_keys)):
938 k = "View%d" % view_num
939 if k not in view_keys:
940 die("Expected view key %s missing" % k)
941 view.append(entry[k])
942
943 return view
944
945def getClientRoot():
946 """Grab the client directory."""
947
948 output = p4CmdList("client -o")
949 if len(output) != 1:
950 die('Output from "client -o" is %d lines, expecting 1' % len(output))
951
952 entry = output[0]
953 if "Root" not in entry:
954 die('Client has no "Root"')
955
956 return entry["Root"]
957
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400958#
959# P4 wildcards are not allowed in filenames. P4 complains
960# if you simply add them, but you can force it with "-f", in
961# which case it translates them into %xx encoding internally.
962#
963def wildcard_decode(path):
964 # Search for and fix just these four characters. Do % last so
965 # that fixing it does not inadvertently create new %-escapes.
966 # Cannot have * in a filename in windows; untested as to
967 # what p4 would do in such a case.
968 if not platform.system() == "Windows":
969 path = path.replace("%2A", "*")
970 path = path.replace("%23", "#") \
971 .replace("%40", "@") \
972 .replace("%25", "%")
973 return path
974
975def wildcard_encode(path):
976 # do % first to avoid double-encoding the %s introduced here
977 path = path.replace("%", "%25") \
978 .replace("*", "%2A") \
979 .replace("#", "%23") \
980 .replace("@", "%40")
981 return path
982
983def wildcard_present(path):
Brandon Casey598354c2013-01-26 11:14:32 -0800984 m = re.search("[*#@%]", path)
985 return m is not None
Pete Wyckoff9d7d4462012-04-29 20:57:17 -0400986
Lars Schneidera5db4b12015-09-26 09:55:03 +0200987class LargeFileSystem(object):
988 """Base class for large file system support."""
989
990 def __init__(self, writeToGitStream):
991 self.largeFiles = set()
992 self.writeToGitStream = writeToGitStream
993
994 def generatePointer(self, cloneDestination, contentFile):
995 """Return the content of a pointer file that is stored in Git instead of
996 the actual content."""
997 assert False, "Method 'generatePointer' required in " + self.__class__.__name__
998
999 def pushFile(self, localLargeFile):
1000 """Push the actual content which is not stored in the Git repository to
1001 a server."""
1002 assert False, "Method 'pushFile' required in " + self.__class__.__name__
1003
1004 def hasLargeFileExtension(self, relPath):
1005 return reduce(
1006 lambda a, b: a or b,
1007 [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1008 False
1009 )
1010
1011 def generateTempFile(self, contents):
1012 contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1013 for d in contents:
1014 contentFile.write(d)
1015 contentFile.close()
1016 return contentFile.name
1017
1018 def exceedsLargeFileThreshold(self, relPath, contents):
1019 if gitConfigInt('git-p4.largeFileThreshold'):
1020 contentsSize = sum(len(d) for d in contents)
1021 if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1022 return True
1023 if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024 contentsSize = sum(len(d) for d in contents)
1025 if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1026 return False
1027 contentTempFile = self.generateTempFile(contents)
1028 compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1029 zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1030 zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1031 zf.close()
1032 compressedContentsSize = zf.infolist()[0].compress_size
1033 os.remove(contentTempFile)
1034 os.remove(compressedContentFile.name)
1035 if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1036 return True
1037 return False
1038
1039 def addLargeFile(self, relPath):
1040 self.largeFiles.add(relPath)
1041
1042 def removeLargeFile(self, relPath):
1043 self.largeFiles.remove(relPath)
1044
1045 def isLargeFile(self, relPath):
1046 return relPath in self.largeFiles
1047
1048 def processContent(self, git_mode, relPath, contents):
1049 """Processes the content of git fast import. This method decides if a
1050 file is stored in the large file system and handles all necessary
1051 steps."""
1052 if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1053 contentTempFile = self.generateTempFile(contents)
Lars Schneiderd5eb3cf2016-12-04 17:03:37 +01001054 (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1055 if pointer_git_mode:
1056 git_mode = pointer_git_mode
1057 if localLargeFile:
1058 # Move temp file to final location in large file system
1059 largeFileDir = os.path.dirname(localLargeFile)
1060 if not os.path.isdir(largeFileDir):
1061 os.makedirs(largeFileDir)
1062 shutil.move(contentTempFile, localLargeFile)
1063 self.addLargeFile(relPath)
1064 if gitConfigBool('git-p4.largeFilePush'):
1065 self.pushFile(localLargeFile)
1066 if verbose:
1067 sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
Lars Schneidera5db4b12015-09-26 09:55:03 +02001068 return (git_mode, contents)
1069
1070class MockLFS(LargeFileSystem):
1071 """Mock large file system for testing."""
1072
1073 def generatePointer(self, contentFile):
1074 """The pointer content is the original content prefixed with "pointer-".
1075 The local filename of the large file storage is derived from the file content.
1076 """
1077 with open(contentFile, 'r') as f:
1078 content = next(f)
1079 gitMode = '100644'
1080 pointerContents = 'pointer-' + content
1081 localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1082 return (gitMode, pointerContents, localLargeFile)
1083
1084 def pushFile(self, localLargeFile):
1085 """The remote filename of the large file storage is the same as the local
1086 one but in a different directory.
1087 """
1088 remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1089 if not os.path.exists(remotePath):
1090 os.makedirs(remotePath)
1091 shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1092
Lars Schneiderb47d8072015-09-26 09:55:04 +02001093class GitLFS(LargeFileSystem):
1094 """Git LFS as backend for the git-p4 large file system.
1095 See https://git-lfs.github.com/ for details."""
1096
1097 def __init__(self, *args):
1098 LargeFileSystem.__init__(self, *args)
1099 self.baseGitAttributes = []
1100
1101 def generatePointer(self, contentFile):
1102 """Generate a Git LFS pointer for the content. Return LFS Pointer file
1103 mode and content which is stored in the Git repository instead of
1104 the actual content. Return also the new location of the actual
1105 content.
1106 """
Lars Schneiderd5eb3cf2016-12-04 17:03:37 +01001107 if os.path.getsize(contentFile) == 0:
1108 return (None, '', None)
1109
Lars Schneiderb47d8072015-09-26 09:55:04 +02001110 pointerProcess = subprocess.Popen(
1111 ['git', 'lfs', 'pointer', '--file=' + contentFile],
1112 stdout=subprocess.PIPE
1113 )
1114 pointerFile = pointerProcess.stdout.read()
1115 if pointerProcess.wait():
1116 os.remove(contentFile)
1117 die('git-lfs pointer command failed. Did you install the extension?')
Lars Schneider82f25672016-04-28 08:26:33 +02001118
1119 # Git LFS removed the preamble in the output of the 'pointer' command
1120 # starting from version 1.2.0. Check for the preamble here to support
1121 # earlier versions.
1122 # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1123 if pointerFile.startswith('Git LFS pointer for'):
1124 pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1125
1126 oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
Lars Schneiderb47d8072015-09-26 09:55:04 +02001127 localLargeFile = os.path.join(
1128 os.getcwd(),
1129 '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1130 oid,
1131 )
1132 # LFS Spec states that pointer files should not have the executable bit set.
1133 gitMode = '100644'
Lars Schneider82f25672016-04-28 08:26:33 +02001134 return (gitMode, pointerFile, localLargeFile)
Lars Schneiderb47d8072015-09-26 09:55:04 +02001135
1136 def pushFile(self, localLargeFile):
1137 uploadProcess = subprocess.Popen(
1138 ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1139 )
1140 if uploadProcess.wait():
1141 die('git-lfs push command failed. Did you define a remote?')
1142
1143 def generateGitAttributes(self):
1144 return (
1145 self.baseGitAttributes +
1146 [
1147 '\n',
1148 '#\n',
1149 '# Git LFS (see https://git-lfs.github.com/)\n',
1150 '#\n',
1151 ] +
Lars Schneider862f9312016-12-18 20:01:40 +01001152 ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
Lars Schneiderb47d8072015-09-26 09:55:04 +02001153 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1154 ] +
Lars Schneider862f9312016-12-18 20:01:40 +01001155 ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
Lars Schneiderb47d8072015-09-26 09:55:04 +02001156 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1157 ]
1158 )
1159
1160 def addLargeFile(self, relPath):
1161 LargeFileSystem.addLargeFile(self, relPath)
1162 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1163
1164 def removeLargeFile(self, relPath):
1165 LargeFileSystem.removeLargeFile(self, relPath)
1166 self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1167
1168 def processContent(self, git_mode, relPath, contents):
1169 if relPath == '.gitattributes':
1170 self.baseGitAttributes = contents
1171 return (git_mode, self.generateGitAttributes())
1172 else:
1173 return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1174
Simon Hausmannb9847332007-03-20 20:54:23 +01001175class Command:
1176 def __init__(self):
1177 self.usage = "usage: %prog [options]"
Simon Hausmann8910ac02007-03-26 08:18:55 +02001178 self.needsGit = True
Luke Diamand6a10b6a2012-04-24 09:33:23 +01001179 self.verbose = False
Simon Hausmannb9847332007-03-20 20:54:23 +01001180
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001181class P4UserMap:
1182 def __init__(self):
1183 self.userMapFromPerforceServer = False
Luke Diamandaffb4742012-01-19 09:52:27 +00001184 self.myP4UserId = None
1185
1186 def p4UserId(self):
1187 if self.myP4UserId:
1188 return self.myP4UserId
1189
1190 results = p4CmdList("user -o")
1191 for r in results:
1192 if r.has_key('User'):
1193 self.myP4UserId = r['User']
1194 return r['User']
1195 die("Could not find your p4 user id")
1196
1197 def p4UserIsMe(self, p4User):
1198 # return True if the given p4 user is actually me
1199 me = self.p4UserId()
1200 if not p4User or p4User != me:
1201 return False
1202 else:
1203 return True
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001204
1205 def getUserCacheFilename(self):
1206 home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1207 return home + "/.gitp4-usercache.txt"
1208
1209 def getUserMapFromPerforceServer(self):
1210 if self.userMapFromPerforceServer:
1211 return
1212 self.users = {}
1213 self.emails = {}
1214
1215 for output in p4CmdList("users"):
1216 if not output.has_key("User"):
1217 continue
1218 self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1219 self.emails[output["Email"]] = output["User"]
1220
Lars Schneider10d08a12016-03-01 11:49:56 +01001221 mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1222 for mapUserConfig in gitConfigList("git-p4.mapUser"):
1223 mapUser = mapUserConfigRegex.findall(mapUserConfig)
1224 if mapUser and len(mapUser[0]) == 3:
1225 user = mapUser[0][0]
1226 fullname = mapUser[0][1]
1227 email = mapUser[0][2]
1228 self.users[user] = fullname + " <" + email + ">"
1229 self.emails[email] = user
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001230
1231 s = ''
1232 for (key, val) in self.users.items():
1233 s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1234
1235 open(self.getUserCacheFilename(), "wb").write(s)
1236 self.userMapFromPerforceServer = True
1237
1238 def loadUserMapFromCache(self):
1239 self.users = {}
1240 self.userMapFromPerforceServer = False
1241 try:
1242 cache = open(self.getUserCacheFilename(), "rb")
1243 lines = cache.readlines()
1244 cache.close()
1245 for line in lines:
1246 entry = line.strip().split("\t")
1247 self.users[entry[0]] = entry[1]
1248 except IOError:
1249 self.getUserMapFromPerforceServer()
1250
Simon Hausmannb9847332007-03-20 20:54:23 +01001251class P4Debug(Command):
Simon Hausmann86949ee2007-03-19 20:59:12 +01001252 def __init__(self):
Simon Hausmann6ae8de82007-03-22 21:10:25 +01001253 Command.__init__(self)
Luke Diamand6a10b6a2012-04-24 09:33:23 +01001254 self.options = []
Simon Hausmannc8c39112007-03-19 21:02:30 +01001255 self.description = "A tool to debug the output of p4 -G."
Simon Hausmann8910ac02007-03-26 08:18:55 +02001256 self.needsGit = False
Simon Hausmann86949ee2007-03-19 20:59:12 +01001257
1258 def run(self, args):
Han-Wen Nienhuysb1ce9442007-05-23 18:49:35 -03001259 j = 0
Luke Diamand6de040d2011-10-16 10:47:52 -04001260 for output in p4CmdList(args):
Han-Wen Nienhuysb1ce9442007-05-23 18:49:35 -03001261 print 'Element: %d' % j
1262 j += 1
Simon Hausmann86949ee2007-03-19 20:59:12 +01001263 print output
Simon Hausmannb9847332007-03-20 20:54:23 +01001264 return True
Simon Hausmann86949ee2007-03-19 20:59:12 +01001265
Simon Hausmann58346842007-05-21 22:57:06 +02001266class P4RollBack(Command):
1267 def __init__(self):
1268 Command.__init__(self)
1269 self.options = [
Simon Hausmann0c66a782007-05-23 20:07:57 +02001270 optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
Simon Hausmann58346842007-05-21 22:57:06 +02001271 ]
1272 self.description = "A tool to debug the multi-branch import. Don't use :)"
Simon Hausmann0c66a782007-05-23 20:07:57 +02001273 self.rollbackLocalBranches = False
Simon Hausmann58346842007-05-21 22:57:06 +02001274
1275 def run(self, args):
1276 if len(args) != 1:
1277 return False
1278 maxChange = int(args[0])
Simon Hausmann0c66a782007-05-23 20:07:57 +02001279
Simon Hausmannad192f22007-05-23 23:44:19 +02001280 if "p4ExitCode" in p4Cmd("changes -m 1"):
Simon Hausmann66a2f522007-05-23 23:40:48 +02001281 die("Problems executing p4");
1282
Simon Hausmann0c66a782007-05-23 20:07:57 +02001283 if self.rollbackLocalBranches:
1284 refPrefix = "refs/heads/"
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -03001285 lines = read_pipe_lines("git rev-parse --symbolic --branches")
Simon Hausmann0c66a782007-05-23 20:07:57 +02001286 else:
1287 refPrefix = "refs/remotes/"
Han-Wen Nienhuysb016d392007-05-23 17:10:46 -03001288 lines = read_pipe_lines("git rev-parse --symbolic --remotes")
Simon Hausmann0c66a782007-05-23 20:07:57 +02001289
1290 for line in lines:
1291 if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
Han-Wen Nienhuysb25b2062007-05-23 18:49:35 -03001292 line = line.strip()
1293 ref = refPrefix + line
Simon Hausmann58346842007-05-21 22:57:06 +02001294 log = extractLogMessageFromGitCommit(ref)
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03001295 settings = extractSettingsGitLog(log)
1296
1297 depotPaths = settings['depot-paths']
1298 change = settings['change']
1299
Simon Hausmann58346842007-05-21 22:57:06 +02001300 changed = False
Simon Hausmann52102d42007-05-21 23:44:24 +02001301
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03001302 if len(p4Cmd("changes -m 1 " + ' '.join (['%s...@%s' % (p, maxChange)
1303 for p in depotPaths]))) == 0:
Simon Hausmann52102d42007-05-21 23:44:24 +02001304 print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1305 system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1306 continue
1307
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03001308 while change and int(change) > maxChange:
Simon Hausmann58346842007-05-21 22:57:06 +02001309 changed = True
Simon Hausmann52102d42007-05-21 23:44:24 +02001310 if self.verbose:
1311 print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
Simon Hausmann58346842007-05-21 22:57:06 +02001312 system("git update-ref %s \"%s^\"" % (ref, ref))
1313 log = extractLogMessageFromGitCommit(ref)
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03001314 settings = extractSettingsGitLog(log)
1315
1316
1317 depotPaths = settings['depot-paths']
1318 change = settings['change']
Simon Hausmann58346842007-05-21 22:57:06 +02001319
1320 if changed:
Simon Hausmann52102d42007-05-21 23:44:24 +02001321 print "%s rewound to %s" % (ref, change)
Simon Hausmann58346842007-05-21 22:57:06 +02001322
1323 return True
1324
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001325class P4Submit(Command, P4UserMap):
Pete Wyckoff6bbfd132012-09-09 16:16:13 -04001326
1327 conflict_behavior_choices = ("ask", "skip", "quit")
1328
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001329 def __init__(self):
Simon Hausmannb9847332007-03-20 20:54:23 +01001330 Command.__init__(self)
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001331 P4UserMap.__init__(self)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001332 self.options = [
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001333 optparse.make_option("--origin", dest="origin"),
Vitor Antunesae901092011-02-20 01:18:24 +00001334 optparse.make_option("-M", dest="detectRenames", action="store_true"),
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001335 # preserve the user, requires relevant p4 permissions
1336 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
Luke Diamand06804c72012-04-11 17:21:24 +02001337 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
Pete Wyckoffef739f02012-09-09 16:16:11 -04001338 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04001339 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
Pete Wyckoff6bbfd132012-09-09 16:16:13 -04001340 optparse.make_option("--conflict", dest="conflict_behavior",
Pete Wyckoff44e8d262013-01-14 19:47:08 -05001341 choices=self.conflict_behavior_choices),
1342 optparse.make_option("--branch", dest="branch"),
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00001343 optparse.make_option("--shelve", dest="shelve", action="store_true",
1344 help="Shelve instead of submit. Shelved files are reverted, "
1345 "restoring the workspace to the state before the shelve"),
Luke Diamand46c609e2016-12-02 22:43:19 +00001346 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1347 metavar="CHANGELIST",
1348 help="update an existing shelved changelist, implies --shelve")
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001349 ]
1350 self.description = "Submit changes from git to the perforce depot."
Simon Hausmannc9b50e62007-03-29 19:15:24 +02001351 self.usage += " [name of git branch to submit into perforce depot]"
Simon Hausmann95124972007-03-23 09:16:07 +01001352 self.origin = ""
Vitor Antunesae901092011-02-20 01:18:24 +00001353 self.detectRenames = False
Pete Wyckoff0d609032013-01-26 22:11:24 -05001354 self.preserveUser = gitConfigBool("git-p4.preserveUser")
Pete Wyckoffef739f02012-09-09 16:16:11 -04001355 self.dry_run = False
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00001356 self.shelve = False
Luke Diamand46c609e2016-12-02 22:43:19 +00001357 self.update_shelve = None
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04001358 self.prepare_p4_only = False
Pete Wyckoff6bbfd132012-09-09 16:16:13 -04001359 self.conflict_behavior = None
Marius Storm-Olsenf7baba82007-06-07 14:07:01 +02001360 self.isWindows = (platform.system() == "Windows")
Luke Diamand06804c72012-04-11 17:21:24 +02001361 self.exportLabels = False
Pete Wyckoff249da4c2012-11-23 17:35:35 -05001362 self.p4HasMoveCommand = p4_has_move_command()
Pete Wyckoff44e8d262013-01-14 19:47:08 -05001363 self.branch = None
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001364
Lars Schneidera5db4b12015-09-26 09:55:03 +02001365 if gitConfig('git-p4.largeFileSystem'):
1366 die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1367
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001368 def check(self):
1369 if len(p4CmdList("opened ...")) > 0:
1370 die("You have files opened with perforce! Close them before starting the sync.")
1371
Pete Wyckofff19cb0a2012-07-04 09:34:20 -04001372 def separate_jobs_from_description(self, message):
1373 """Extract and return a possible Jobs field in the commit
1374 message. It goes into a separate section in the p4 change
1375 specification.
1376
1377 A jobs line starts with "Jobs:" and looks like a new field
1378 in a form. Values are white-space separated on the same
1379 line or on following lines that start with a tab.
1380
1381 This does not parse and extract the full git commit message
1382 like a p4 form. It just sees the Jobs: line as a marker
1383 to pass everything from then on directly into the p4 form,
1384 but outside the description section.
1385
1386 Return a tuple (stripped log message, jobs string)."""
1387
1388 m = re.search(r'^Jobs:', message, re.MULTILINE)
1389 if m is None:
1390 return (message, None)
1391
1392 jobtext = message[m.start():]
1393 stripped_message = message[:m.start()].rstrip()
1394 return (stripped_message, jobtext)
1395
1396 def prepareLogMessage(self, template, message, jobs):
1397 """Edits the template returned from "p4 change -o" to insert
1398 the message in the Description field, and the jobs text in
1399 the Jobs field."""
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001400 result = ""
1401
Simon Hausmannedae1e22008-02-19 09:29:06 +01001402 inDescriptionSection = False
1403
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001404 for line in template.split("\n"):
1405 if line.startswith("#"):
1406 result += line + "\n"
1407 continue
1408
Simon Hausmannedae1e22008-02-19 09:29:06 +01001409 if inDescriptionSection:
Michael Horowitzc9dbab02011-02-25 21:31:13 -05001410 if line.startswith("Files:") or line.startswith("Jobs:"):
Simon Hausmannedae1e22008-02-19 09:29:06 +01001411 inDescriptionSection = False
Pete Wyckofff19cb0a2012-07-04 09:34:20 -04001412 # insert Jobs section
1413 if jobs:
1414 result += jobs + "\n"
Simon Hausmannedae1e22008-02-19 09:29:06 +01001415 else:
1416 continue
1417 else:
1418 if line.startswith("Description:"):
1419 inDescriptionSection = True
1420 line += "\n"
1421 for messageLine in message.split("\n"):
1422 line += "\t" + messageLine + "\n"
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001423
Simon Hausmannedae1e22008-02-19 09:29:06 +01001424 result += line + "\n"
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001425
1426 return result
1427
Luke Diamand60df0712012-02-23 07:51:30 +00001428 def patchRCSKeywords(self, file, pattern):
1429 # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1430 (handle, outFileName) = tempfile.mkstemp(dir='.')
1431 try:
1432 outFile = os.fdopen(handle, "w+")
1433 inFile = open(file, "r")
1434 regexp = re.compile(pattern, re.VERBOSE)
1435 for line in inFile.readlines():
1436 line = regexp.sub(r'$\1$', line)
1437 outFile.write(line)
1438 inFile.close()
1439 outFile.close()
1440 # Forcibly overwrite the original file
1441 os.unlink(file)
1442 shutil.move(outFileName, file)
1443 except:
1444 # cleanup our temporary file
1445 os.unlink(outFileName)
1446 print "Failed to strip RCS keywords in %s" % file
1447 raise
1448
1449 print "Patched up RCS keywords in %s" % file
1450
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001451 def p4UserForCommit(self,id):
1452 # Return the tuple (perforce user,git email) for a given git commit id
1453 self.getUserMapFromPerforceServer()
Pete Wyckoff9bf28852013-01-26 22:11:20 -05001454 gitEmail = read_pipe(["git", "log", "--max-count=1",
1455 "--format=%ae", id])
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001456 gitEmail = gitEmail.strip()
1457 if not self.emails.has_key(gitEmail):
1458 return (None,gitEmail)
1459 else:
1460 return (self.emails[gitEmail],gitEmail)
1461
1462 def checkValidP4Users(self,commits):
1463 # check if any git authors cannot be mapped to p4 users
1464 for id in commits:
1465 (user,email) = self.p4UserForCommit(id)
1466 if not user:
1467 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
Pete Wyckoff0d609032013-01-26 22:11:24 -05001468 if gitConfigBool("git-p4.allowMissingP4Users"):
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001469 print "%s" % msg
1470 else:
1471 die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1472
1473 def lastP4Changelist(self):
1474 # Get back the last changelist number submitted in this client spec. This
1475 # then gets used to patch up the username in the change. If the same
1476 # client spec is being used by multiple processes then this might go
1477 # wrong.
1478 results = p4CmdList("client -o") # find the current client
1479 client = None
1480 for r in results:
1481 if r.has_key('Client'):
1482 client = r['Client']
1483 break
1484 if not client:
1485 die("could not get client spec")
Luke Diamand6de040d2011-10-16 10:47:52 -04001486 results = p4CmdList(["changes", "-c", client, "-m", "1"])
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001487 for r in results:
1488 if r.has_key('change'):
1489 return r['change']
1490 die("Could not get changelist number for last submit - cannot patch up user details")
1491
1492 def modifyChangelistUser(self, changelist, newUser):
1493 # fixup the user field of a changelist after it has been submitted.
1494 changes = p4CmdList("change -o %s" % changelist)
Luke Diamandecdba362011-05-07 11:19:43 +01001495 if len(changes) != 1:
1496 die("Bad output from p4 change modifying %s to user %s" %
1497 (changelist, newUser))
1498
1499 c = changes[0]
1500 if c['User'] == newUser: return # nothing to do
1501 c['User'] = newUser
1502 input = marshal.dumps(c)
1503
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001504 result = p4CmdList("change -f -i", stdin=input)
1505 for r in result:
1506 if r.has_key('code'):
1507 if r['code'] == 'error':
1508 die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1509 if r.has_key('data'):
1510 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1511 return
1512 die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1513
1514 def canChangeChangelists(self):
1515 # check to see if we have p4 admin or super-user permissions, either of
1516 # which are required to modify changelists.
Luke Diamand52a48802012-01-19 09:52:25 +00001517 results = p4CmdList(["protects", self.depotPath])
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001518 for r in results:
1519 if r.has_key('perm'):
1520 if r['perm'] == 'admin':
1521 return 1
1522 if r['perm'] == 'super':
1523 return 1
1524 return 0
1525
Luke Diamand46c609e2016-12-02 22:43:19 +00001526 def prepareSubmitTemplate(self, changelist=None):
Pete Wyckofff19cb0a2012-07-04 09:34:20 -04001527 """Run "p4 change -o" to grab a change specification template.
1528 This does not use "p4 -G", as it is nice to keep the submission
1529 template in original order, since a human might edit it.
1530
1531 Remove lines in the Files section that show changes to files
1532 outside the depot path we're committing into."""
1533
Sam Hocevarcbc69242015-12-19 09:39:39 +00001534 [upstream, settings] = findUpstreamBranchPoint()
1535
Miguel Torrojab596b3b2017-07-13 09:00:34 +02001536 template = """\
1537# A Perforce Change Specification.
1538#
1539# Change: The change number. 'new' on a new changelist.
1540# Date: The date this specification was last modified.
1541# Client: The client on which the changelist was created. Read-only.
1542# User: The user who created the changelist.
1543# Status: Either 'pending' or 'submitted'. Read-only.
1544# Type: Either 'public' or 'restricted'. Default is 'public'.
1545# Description: Comments about the changelist. Required.
1546# Jobs: What opened jobs are to be closed by this changelist.
1547# You may delete jobs from this list. (New changelists only.)
1548# Files: What opened files from the default changelist are to be added
1549# to this changelist. You may delete files from this list.
1550# (New changelists only.)
1551"""
1552 files_list = []
Simon Hausmannea99c3a2007-08-08 17:06:55 +02001553 inFilesSection = False
Miguel Torrojab596b3b2017-07-13 09:00:34 +02001554 change_entry = None
Luke Diamand46c609e2016-12-02 22:43:19 +00001555 args = ['change', '-o']
1556 if changelist:
1557 args.append(str(changelist))
Miguel Torrojab596b3b2017-07-13 09:00:34 +02001558 for entry in p4CmdList(args):
1559 if not entry.has_key('code'):
1560 continue
1561 if entry['code'] == 'stat':
1562 change_entry = entry
1563 break
1564 if not change_entry:
1565 die('Failed to decode output of p4 change -o')
1566 for key, value in change_entry.iteritems():
1567 if key.startswith('File'):
1568 if settings.has_key('depot-paths'):
1569 if not [p for p in settings['depot-paths']
1570 if p4PathStartsWith(value, p)]:
1571 continue
Simon Hausmannea99c3a2007-08-08 17:06:55 +02001572 else:
Miguel Torrojab596b3b2017-07-13 09:00:34 +02001573 if not p4PathStartsWith(value, self.depotPath):
1574 continue
1575 files_list.append(value)
1576 continue
1577 # Output in the order expected by prepareLogMessage
1578 for key in ['Change', 'Client', 'User', 'Status', 'Description', 'Jobs']:
1579 if not change_entry.has_key(key):
1580 continue
1581 template += '\n'
1582 template += key + ':'
1583 if key == 'Description':
1584 template += '\n'
1585 for field_line in change_entry[key].splitlines():
1586 template += '\t'+field_line+'\n'
1587 if len(files_list) > 0:
1588 template += '\n'
1589 template += 'Files:\n'
1590 for path in files_list:
1591 template += '\t'+path+'\n'
Simon Hausmannea99c3a2007-08-08 17:06:55 +02001592 return template
1593
Pete Wyckoff7c766e52011-12-04 19:22:45 -05001594 def edit_template(self, template_file):
1595 """Invoke the editor to let the user change the submission
1596 message. Return true if okay to continue with the submit."""
1597
1598 # if configured to skip the editing part, just submit
Pete Wyckoff0d609032013-01-26 22:11:24 -05001599 if gitConfigBool("git-p4.skipSubmitEdit"):
Pete Wyckoff7c766e52011-12-04 19:22:45 -05001600 return True
1601
1602 # look at the modification time, to check later if the user saved
1603 # the file
1604 mtime = os.stat(template_file).st_mtime
1605
1606 # invoke the editor
Luke Diamandf95ceaf2012-04-24 09:33:21 +01001607 if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
Pete Wyckoff7c766e52011-12-04 19:22:45 -05001608 editor = os.environ.get("P4EDITOR")
1609 else:
1610 editor = read_pipe("git var GIT_EDITOR").strip()
Luke Diamand2dade7a2015-05-19 23:23:17 +01001611 system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
Pete Wyckoff7c766e52011-12-04 19:22:45 -05001612
1613 # If the file was not saved, prompt to see if this patch should
1614 # be skipped. But skip this verification step if configured so.
Pete Wyckoff0d609032013-01-26 22:11:24 -05001615 if gitConfigBool("git-p4.skipSubmitEditCheck"):
Pete Wyckoff7c766e52011-12-04 19:22:45 -05001616 return True
1617
Pete Wyckoffd1652042011-12-17 12:39:03 -05001618 # modification time updated means user saved the file
1619 if os.stat(template_file).st_mtime > mtime:
1620 return True
1621
1622 while True:
1623 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1624 if response == 'y':
1625 return True
1626 if response == 'n':
1627 return False
Pete Wyckoff7c766e52011-12-04 19:22:45 -05001628
Luke Diamanddf8a9e82016-12-17 01:00:40 +00001629 def get_diff_description(self, editedFiles, filesToAdd, symlinks):
Maxime Costeb4073bb2014-05-24 18:40:35 +01001630 # diff
1631 if os.environ.has_key("P4DIFF"):
1632 del(os.environ["P4DIFF"])
1633 diff = ""
1634 for editedFile in editedFiles:
1635 diff += p4_read_pipe(['diff', '-du',
1636 wildcard_encode(editedFile)])
1637
1638 # new file diff
1639 newdiff = ""
1640 for newFile in filesToAdd:
1641 newdiff += "==== new file ====\n"
1642 newdiff += "--- /dev/null\n"
1643 newdiff += "+++ %s\n" % newFile
Luke Diamanddf8a9e82016-12-17 01:00:40 +00001644
1645 is_link = os.path.islink(newFile)
1646 expect_link = newFile in symlinks
1647
1648 if is_link and expect_link:
1649 newdiff += "+%s\n" % os.readlink(newFile)
1650 else:
1651 f = open(newFile, "r")
1652 for line in f.readlines():
1653 newdiff += "+" + line
1654 f.close()
Maxime Costeb4073bb2014-05-24 18:40:35 +01001655
Maxime Costee2a892e2014-06-11 14:09:59 +01001656 return (diff + newdiff).replace('\r\n', '\n')
Maxime Costeb4073bb2014-05-24 18:40:35 +01001657
Han-Wen Nienhuys7cb5cbe2007-05-23 16:55:48 -03001658 def applyCommit(self, id):
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04001659 """Apply one commit, return True if it succeeded."""
1660
1661 print "Applying", read_pipe(["git", "show", "-s",
1662 "--format=format:%h %s", id])
Vitor Antunesae901092011-02-20 01:18:24 +00001663
Luke Diamand848de9c2011-05-13 20:46:00 +01001664 (p4User, gitEmail) = self.p4UserForCommit(id)
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01001665
Gary Gibbons84cb0002012-07-04 09:40:19 -04001666 diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001667 filesToAdd = set()
Romain Picarda02b8bc2016-01-12 13:43:47 +01001668 filesToChangeType = set()
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001669 filesToDelete = set()
Simon Hausmannd336c152007-05-16 09:41:26 +02001670 editedFiles = set()
Pete Wyckoffb6ad6dc2012-04-29 20:57:16 -04001671 pureRenameCopy = set()
Luke Diamanddf8a9e82016-12-17 01:00:40 +00001672 symlinks = set()
Chris Pettittc65b6702007-11-01 20:43:14 -07001673 filesToChangeExecBit = {}
Luke Diamand46c609e2016-12-02 22:43:19 +00001674 all_files = list()
Luke Diamand60df0712012-02-23 07:51:30 +00001675
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001676 for line in diff:
Chris Pettittb43b0a32007-11-01 20:43:13 -07001677 diff = parseDiffTreeEntry(line)
1678 modifier = diff['status']
1679 path = diff['src']
Luke Diamand46c609e2016-12-02 22:43:19 +00001680 all_files.append(path)
1681
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001682 if modifier == "M":
Luke Diamand6de040d2011-10-16 10:47:52 -04001683 p4_edit(path)
Chris Pettittc65b6702007-11-01 20:43:14 -07001684 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1685 filesToChangeExecBit[path] = diff['dst_mode']
Simon Hausmannd336c152007-05-16 09:41:26 +02001686 editedFiles.add(path)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001687 elif modifier == "A":
1688 filesToAdd.add(path)
Chris Pettittc65b6702007-11-01 20:43:14 -07001689 filesToChangeExecBit[path] = diff['dst_mode']
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001690 if path in filesToDelete:
1691 filesToDelete.remove(path)
Luke Diamanddf8a9e82016-12-17 01:00:40 +00001692
1693 dst_mode = int(diff['dst_mode'], 8)
1694 if dst_mode == 0120000:
1695 symlinks.add(path)
1696
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001697 elif modifier == "D":
1698 filesToDelete.add(path)
1699 if path in filesToAdd:
1700 filesToAdd.remove(path)
Vitor Antunes4fddb412011-02-20 01:18:25 +00001701 elif modifier == "C":
1702 src, dest = diff['src'], diff['dst']
Luke Diamand6de040d2011-10-16 10:47:52 -04001703 p4_integrate(src, dest)
Pete Wyckoffb6ad6dc2012-04-29 20:57:16 -04001704 pureRenameCopy.add(dest)
Vitor Antunes4fddb412011-02-20 01:18:25 +00001705 if diff['src_sha1'] != diff['dst_sha1']:
Luke Diamand6de040d2011-10-16 10:47:52 -04001706 p4_edit(dest)
Pete Wyckoffb6ad6dc2012-04-29 20:57:16 -04001707 pureRenameCopy.discard(dest)
Vitor Antunes4fddb412011-02-20 01:18:25 +00001708 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
Luke Diamand6de040d2011-10-16 10:47:52 -04001709 p4_edit(dest)
Pete Wyckoffb6ad6dc2012-04-29 20:57:16 -04001710 pureRenameCopy.discard(dest)
Vitor Antunes4fddb412011-02-20 01:18:25 +00001711 filesToChangeExecBit[dest] = diff['dst_mode']
Pete Wyckoffd20f0f82013-01-26 22:11:19 -05001712 if self.isWindows:
1713 # turn off read-only attribute
1714 os.chmod(dest, stat.S_IWRITE)
Vitor Antunes4fddb412011-02-20 01:18:25 +00001715 os.unlink(dest)
1716 editedFiles.add(dest)
Chris Pettittd9a5f252007-10-15 22:15:06 -07001717 elif modifier == "R":
Chris Pettittb43b0a32007-11-01 20:43:13 -07001718 src, dest = diff['src'], diff['dst']
Gary Gibbons8e9497c2012-07-12 19:29:00 -04001719 if self.p4HasMoveCommand:
1720 p4_edit(src) # src must be open before move
1721 p4_move(src, dest) # opens for (move/delete, move/add)
Pete Wyckoffb6ad6dc2012-04-29 20:57:16 -04001722 else:
Gary Gibbons8e9497c2012-07-12 19:29:00 -04001723 p4_integrate(src, dest)
1724 if diff['src_sha1'] != diff['dst_sha1']:
1725 p4_edit(dest)
1726 else:
1727 pureRenameCopy.add(dest)
Chris Pettittc65b6702007-11-01 20:43:14 -07001728 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
Gary Gibbons8e9497c2012-07-12 19:29:00 -04001729 if not self.p4HasMoveCommand:
1730 p4_edit(dest) # with move: already open, writable
Chris Pettittc65b6702007-11-01 20:43:14 -07001731 filesToChangeExecBit[dest] = diff['dst_mode']
Gary Gibbons8e9497c2012-07-12 19:29:00 -04001732 if not self.p4HasMoveCommand:
Pete Wyckoffd20f0f82013-01-26 22:11:19 -05001733 if self.isWindows:
1734 os.chmod(dest, stat.S_IWRITE)
Gary Gibbons8e9497c2012-07-12 19:29:00 -04001735 os.unlink(dest)
1736 filesToDelete.add(src)
Chris Pettittd9a5f252007-10-15 22:15:06 -07001737 editedFiles.add(dest)
Romain Picarda02b8bc2016-01-12 13:43:47 +01001738 elif modifier == "T":
1739 filesToChangeType.add(path)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001740 else:
1741 die("unknown modifier %s for %s" % (modifier, path))
1742
Tolga Ceylan749b6682014-05-06 22:48:54 -07001743 diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
Simon Hausmann47a130b2007-05-20 16:33:21 +02001744 patchcmd = diffcmd + " | git apply "
Simon Hausmannc1b296b2007-05-20 16:55:05 +02001745 tryPatchCmd = patchcmd + "--check -"
1746 applyPatchCmd = patchcmd + "--check --apply -"
Luke Diamand60df0712012-02-23 07:51:30 +00001747 patch_succeeded = True
Simon Hausmann51a26402007-04-15 09:59:56 +02001748
Simon Hausmann47a130b2007-05-20 16:33:21 +02001749 if os.system(tryPatchCmd) != 0:
Luke Diamand60df0712012-02-23 07:51:30 +00001750 fixed_rcs_keywords = False
1751 patch_succeeded = False
Simon Hausmann51a26402007-04-15 09:59:56 +02001752 print "Unfortunately applying the change failed!"
Luke Diamand60df0712012-02-23 07:51:30 +00001753
1754 # Patch failed, maybe it's just RCS keyword woes. Look through
1755 # the patch to see if that's possible.
Pete Wyckoff0d609032013-01-26 22:11:24 -05001756 if gitConfigBool("git-p4.attemptRCSCleanup"):
Luke Diamand60df0712012-02-23 07:51:30 +00001757 file = None
1758 pattern = None
1759 kwfiles = {}
1760 for file in editedFiles | filesToDelete:
1761 # did this file's delta contain RCS keywords?
1762 pattern = p4_keywords_regexp_for_file(file)
1763
1764 if pattern:
1765 # this file is a possibility...look for RCS keywords.
1766 regexp = re.compile(pattern, re.VERBOSE)
1767 for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1768 if regexp.search(line):
1769 if verbose:
1770 print "got keyword match on %s in %s in %s" % (pattern, line, file)
1771 kwfiles[file] = pattern
1772 break
1773
1774 for file in kwfiles:
1775 if verbose:
1776 print "zapping %s with %s" % (line,pattern)
Pete Wyckoffd20f0f82013-01-26 22:11:19 -05001777 # File is being deleted, so not open in p4. Must
1778 # disable the read-only bit on windows.
1779 if self.isWindows and file not in editedFiles:
1780 os.chmod(file, stat.S_IWRITE)
Luke Diamand60df0712012-02-23 07:51:30 +00001781 self.patchRCSKeywords(file, kwfiles[file])
1782 fixed_rcs_keywords = True
1783
1784 if fixed_rcs_keywords:
1785 print "Retrying the patch with RCS keywords cleaned up"
1786 if os.system(tryPatchCmd) == 0:
1787 patch_succeeded = True
1788
1789 if not patch_succeeded:
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04001790 for f in editedFiles:
1791 p4_revert(f)
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04001792 return False
Simon Hausmann51a26402007-04-15 09:59:56 +02001793
Pete Wyckoff55ac2ed2012-09-09 16:16:08 -04001794 #
1795 # Apply the patch for real, and do add/delete/+x handling.
1796 #
Simon Hausmann47a130b2007-05-20 16:33:21 +02001797 system(applyPatchCmd)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001798
Romain Picarda02b8bc2016-01-12 13:43:47 +01001799 for f in filesToChangeType:
1800 p4_edit(f, "-t", "auto")
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001801 for f in filesToAdd:
Luke Diamand6de040d2011-10-16 10:47:52 -04001802 p4_add(f)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001803 for f in filesToDelete:
Luke Diamand6de040d2011-10-16 10:47:52 -04001804 p4_revert(f)
1805 p4_delete(f)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001806
Chris Pettittc65b6702007-11-01 20:43:14 -07001807 # Set/clear executable bits
1808 for f in filesToChangeExecBit.keys():
1809 mode = filesToChangeExecBit[f]
1810 setP4ExecBit(f, mode)
1811
Luke Diamand46c609e2016-12-02 22:43:19 +00001812 if self.update_shelve:
1813 print("all_files = %s" % str(all_files))
1814 p4_reopen_in_change(self.update_shelve, all_files)
1815
Pete Wyckoff55ac2ed2012-09-09 16:16:08 -04001816 #
1817 # Build p4 change description, starting with the contents
1818 # of the git commit message.
1819 #
Simon Hausmann0e36f2d2008-02-19 09:33:08 +01001820 logMessage = extractLogMessageFromGitCommit(id)
Simon Hausmann0e36f2d2008-02-19 09:33:08 +01001821 logMessage = logMessage.strip()
Pete Wyckofff19cb0a2012-07-04 09:34:20 -04001822 (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001823
Luke Diamand46c609e2016-12-02 22:43:19 +00001824 template = self.prepareSubmitTemplate(self.update_shelve)
Pete Wyckofff19cb0a2012-07-04 09:34:20 -04001825 submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001826
1827 if self.preserveUser:
Pete Wyckoff55ac2ed2012-09-09 16:16:08 -04001828 submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001829
Pete Wyckoff55ac2ed2012-09-09 16:16:08 -04001830 if self.checkAuthorship and not self.p4UserIsMe(p4User):
1831 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1832 submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1833 submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1834
1835 separatorLine = "######## everything below this line is just the diff #######\n"
Maxime Costeb4073bb2014-05-24 18:40:35 +01001836 if not self.prepare_p4_only:
1837 submitTemplate += separatorLine
Luke Diamanddf8a9e82016-12-17 01:00:40 +00001838 submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
Pete Wyckoff55ac2ed2012-09-09 16:16:08 -04001839
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001840 (handle, fileName) = tempfile.mkstemp()
Maxime Costee2a892e2014-06-11 14:09:59 +01001841 tmpFile = os.fdopen(handle, "w+b")
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001842 if self.isWindows:
1843 submitTemplate = submitTemplate.replace("\n", "\r\n")
Maxime Costeb4073bb2014-05-24 18:40:35 +01001844 tmpFile.write(submitTemplate)
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001845 tmpFile.close()
1846
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04001847 if self.prepare_p4_only:
1848 #
1849 # Leave the p4 tree prepared, and the submit template around
1850 # and let the user decide what to do next
1851 #
1852 print
1853 print "P4 workspace prepared for submission."
1854 print "To submit or revert, go to client workspace"
1855 print " " + self.clientPath
1856 print
1857 print "To submit, use \"p4 submit\" to write a new description,"
Luke Diamand10de86d2015-01-23 09:15:12 +00001858 print "or \"p4 submit -i <%s\" to use the one prepared by" \
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04001859 " \"git p4\"." % fileName
1860 print "You can delete the file \"%s\" when finished." % fileName
1861
1862 if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1863 print "To preserve change ownership by user %s, you must\n" \
1864 "do \"p4 change -f <change>\" after submitting and\n" \
1865 "edit the User field."
1866 if pureRenameCopy:
1867 print "After submitting, renamed files must be re-synced."
1868 print "Invoke \"p4 sync -f\" on each of these files:"
1869 for f in pureRenameCopy:
1870 print " " + f
1871
1872 print
1873 print "To revert the changes, use \"p4 revert ...\", and delete"
1874 print "the submit template file \"%s\"" % fileName
1875 if filesToAdd:
1876 print "Since the commit adds new files, they must be deleted:"
1877 for f in filesToAdd:
1878 print " " + f
1879 print
1880 return True
1881
Pete Wyckoff55ac2ed2012-09-09 16:16:08 -04001882 #
1883 # Let the user edit the change description, then submit it.
1884 #
GIRARD Etienneb7638fe2015-11-24 07:43:59 +00001885 submitted = False
Luke Diamandecdba362011-05-07 11:19:43 +01001886
GIRARD Etienneb7638fe2015-11-24 07:43:59 +00001887 try:
1888 if self.edit_template(fileName):
1889 # read the edited message and submit
1890 tmpFile = open(fileName, "rb")
1891 message = tmpFile.read()
1892 tmpFile.close()
1893 if self.isWindows:
1894 message = message.replace("\r\n", "\n")
1895 submitTemplate = message[:message.index(separatorLine)]
Luke Diamand46c609e2016-12-02 22:43:19 +00001896
1897 if self.update_shelve:
1898 p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1899 elif self.shelve:
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00001900 p4_write_pipe(['shelve', '-i'], submitTemplate)
1901 else:
1902 p4_write_pipe(['submit', '-i'], submitTemplate)
1903 # The rename/copy happened by applying a patch that created a
1904 # new file. This leaves it writable, which confuses p4.
1905 for f in pureRenameCopy:
1906 p4_sync(f, "-f")
Luke Diamandecdba362011-05-07 11:19:43 +01001907
GIRARD Etienneb7638fe2015-11-24 07:43:59 +00001908 if self.preserveUser:
1909 if p4User:
1910 # Get last changelist number. Cannot easily get it from
1911 # the submit command output as the output is
1912 # unmarshalled.
1913 changelist = self.lastP4Changelist()
1914 self.modifyChangelistUser(changelist, p4User)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001915
GIRARD Etienneb7638fe2015-11-24 07:43:59 +00001916 submitted = True
1917
1918 finally:
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001919 # skip this patch
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00001920 if not submitted or self.shelve:
1921 if self.shelve:
1922 print ("Reverting shelved files.")
1923 else:
1924 print ("Submission cancelled, undoing p4 changes.")
1925 for f in editedFiles | filesToDelete:
GIRARD Etienneb7638fe2015-11-24 07:43:59 +00001926 p4_revert(f)
1927 for f in filesToAdd:
1928 p4_revert(f)
1929 os.remove(f)
Pete Wyckoffc47178d2012-07-04 09:34:18 -04001930
1931 os.remove(fileName)
GIRARD Etienneb7638fe2015-11-24 07:43:59 +00001932 return submitted
Simon Hausmann4f5cf762007-03-19 22:25:17 +01001933
Luke Diamand06804c72012-04-11 17:21:24 +02001934 # Export git tags as p4 labels. Create a p4 label and then tag
1935 # with that.
1936 def exportGitTags(self, gitTags):
Luke Diamandc8942a22012-04-11 17:21:24 +02001937 validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1938 if len(validLabelRegexp) == 0:
1939 validLabelRegexp = defaultLabelRegexp
1940 m = re.compile(validLabelRegexp)
Luke Diamand06804c72012-04-11 17:21:24 +02001941
1942 for name in gitTags:
1943
1944 if not m.match(name):
1945 if verbose:
Luke Diamand05a3cec2012-05-11 07:25:17 +01001946 print "tag %s does not match regexp %s" % (name, validLabelRegexp)
Luke Diamand06804c72012-04-11 17:21:24 +02001947 continue
1948
1949 # Get the p4 commit this corresponds to
Luke Diamandc8942a22012-04-11 17:21:24 +02001950 logMessage = extractLogMessageFromGitCommit(name)
1951 values = extractSettingsGitLog(logMessage)
Luke Diamand06804c72012-04-11 17:21:24 +02001952
Luke Diamandc8942a22012-04-11 17:21:24 +02001953 if not values.has_key('change'):
Luke Diamand06804c72012-04-11 17:21:24 +02001954 # a tag pointing to something not sent to p4; ignore
1955 if verbose:
1956 print "git tag %s does not give a p4 commit" % name
1957 continue
Luke Diamandc8942a22012-04-11 17:21:24 +02001958 else:
1959 changelist = values['change']
Luke Diamand06804c72012-04-11 17:21:24 +02001960
1961 # Get the tag details.
1962 inHeader = True
1963 isAnnotated = False
1964 body = []
1965 for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1966 l = l.strip()
1967 if inHeader:
1968 if re.match(r'tag\s+', l):
1969 isAnnotated = True
1970 elif re.match(r'\s*$', l):
1971 inHeader = False
1972 continue
1973 else:
1974 body.append(l)
1975
1976 if not isAnnotated:
1977 body = ["lightweight tag imported by git p4\n"]
1978
1979 # Create the label - use the same view as the client spec we are using
1980 clientSpec = getClientSpec()
1981
1982 labelTemplate = "Label: %s\n" % name
1983 labelTemplate += "Description:\n"
1984 for b in body:
1985 labelTemplate += "\t" + b + "\n"
1986 labelTemplate += "View:\n"
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09001987 for depot_side in clientSpec.mappings:
1988 labelTemplate += "\t%s\n" % depot_side
Luke Diamand06804c72012-04-11 17:21:24 +02001989
Pete Wyckoffef739f02012-09-09 16:16:11 -04001990 if self.dry_run:
1991 print "Would create p4 label %s for tag" % name
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04001992 elif self.prepare_p4_only:
1993 print "Not creating p4 label %s for tag due to option" \
1994 " --prepare-p4-only" % name
Pete Wyckoffef739f02012-09-09 16:16:11 -04001995 else:
1996 p4_write_pipe(["label", "-i"], labelTemplate)
Luke Diamand06804c72012-04-11 17:21:24 +02001997
Pete Wyckoffef739f02012-09-09 16:16:11 -04001998 # Use the label
1999 p4_system(["tag", "-l", name] +
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002000 ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
Luke Diamand06804c72012-04-11 17:21:24 +02002001
Pete Wyckoffef739f02012-09-09 16:16:11 -04002002 if verbose:
2003 print "created p4 label for tag %s" % name
Luke Diamand06804c72012-04-11 17:21:24 +02002004
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002005 def run(self, args):
Simon Hausmannc9b50e62007-03-29 19:15:24 +02002006 if len(args) == 0:
2007 self.master = currentGitBranch()
Simon Hausmannc9b50e62007-03-29 19:15:24 +02002008 elif len(args) == 1:
2009 self.master = args[0]
Pete Wyckoff28755db2011-12-24 21:07:40 -05002010 if not branchExists(self.master):
2011 die("Branch %s does not exist" % self.master)
Simon Hausmannc9b50e62007-03-29 19:15:24 +02002012 else:
2013 return False
2014
Luke Diamand00ad6e32015-11-21 09:54:41 +00002015 if self.master:
2016 allowSubmit = gitConfig("git-p4.allowSubmit")
2017 if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
2018 die("%s is not in git-p4.allowSubmit" % self.master)
Jing Xue4c2d5d72008-06-22 14:12:39 -04002019
Simon Hausmann27d2d812007-06-12 14:31:59 +02002020 [upstream, settings] = findUpstreamBranchPoint()
Simon Hausmannea99c3a2007-08-08 17:06:55 +02002021 self.depotPath = settings['depot-paths'][0]
Simon Hausmann27d2d812007-06-12 14:31:59 +02002022 if len(self.origin) == 0:
2023 self.origin = upstream
Simon Hausmanna3fdd572007-06-07 22:54:32 +02002024
Luke Diamand46c609e2016-12-02 22:43:19 +00002025 if self.update_shelve:
2026 self.shelve = True
2027
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01002028 if self.preserveUser:
2029 if not self.canChangeChangelists():
2030 die("Cannot preserve user names without p4 super-user or admin permissions")
2031
Pete Wyckoff6bbfd132012-09-09 16:16:13 -04002032 # if not set from the command line, try the config file
2033 if self.conflict_behavior is None:
2034 val = gitConfig("git-p4.conflict")
2035 if val:
2036 if val not in self.conflict_behavior_choices:
2037 die("Invalid value '%s' for config git-p4.conflict" % val)
2038 else:
2039 val = "ask"
2040 self.conflict_behavior = val
2041
Simon Hausmanna3fdd572007-06-07 22:54:32 +02002042 if self.verbose:
2043 print "Origin branch is " + self.origin
Simon Hausmann95124972007-03-23 09:16:07 +01002044
Simon Hausmannea99c3a2007-08-08 17:06:55 +02002045 if len(self.depotPath) == 0:
Simon Hausmann95124972007-03-23 09:16:07 +01002046 print "Internal error: cannot locate perforce depot path from existing branches"
2047 sys.exit(128)
2048
Pete Wyckoff543987b2012-02-25 20:06:25 -05002049 self.useClientSpec = False
Pete Wyckoff0d609032013-01-26 22:11:24 -05002050 if gitConfigBool("git-p4.useclientspec"):
Pete Wyckoff543987b2012-02-25 20:06:25 -05002051 self.useClientSpec = True
2052 if self.useClientSpec:
2053 self.clientSpecDirs = getClientSpec()
Simon Hausmann95124972007-03-23 09:16:07 +01002054
Ville Skyttä2e3a16b2016-08-09 11:53:38 +03002055 # Check for the existence of P4 branches
Vitor Antunescd884102015-04-21 23:49:30 +01002056 branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2057
2058 if self.useClientSpec and not branchesDetected:
Pete Wyckoff543987b2012-02-25 20:06:25 -05002059 # all files are relative to the client spec
2060 self.clientPath = getClientRoot()
2061 else:
2062 self.clientPath = p4Where(self.depotPath)
2063
2064 if self.clientPath == "":
2065 die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
Simon Hausmann95124972007-03-23 09:16:07 +01002066
Simon Hausmannea99c3a2007-08-08 17:06:55 +02002067 print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
Simon Hausmann7944f142007-05-21 11:04:26 +02002068 self.oldWorkingDirectory = os.getcwd()
Simon Hausmannc1b296b2007-05-20 16:55:05 +02002069
Gary Gibbons0591cfa2011-12-09 18:48:14 -05002070 # ensure the clientPath exists
Pete Wyckoff8d7ec362012-04-29 20:57:14 -04002071 new_client_dir = False
Gary Gibbons0591cfa2011-12-09 18:48:14 -05002072 if not os.path.exists(self.clientPath):
Pete Wyckoff8d7ec362012-04-29 20:57:14 -04002073 new_client_dir = True
Gary Gibbons0591cfa2011-12-09 18:48:14 -05002074 os.makedirs(self.clientPath)
2075
Miklós Fazekasbbd84862013-03-11 17:45:29 -04002076 chdir(self.clientPath, is_client_path=True)
Pete Wyckoffef739f02012-09-09 16:16:11 -04002077 if self.dry_run:
2078 print "Would synchronize p4 checkout in %s" % self.clientPath
Pete Wyckoff8d7ec362012-04-29 20:57:14 -04002079 else:
Pete Wyckoffef739f02012-09-09 16:16:11 -04002080 print "Synchronizing p4 checkout..."
2081 if new_client_dir:
2082 # old one was destroyed, and maybe nobody told p4
2083 p4_sync("...", "-f")
2084 else:
2085 p4_sync("...")
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002086 self.check()
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002087
Simon Hausmann4c750c02008-02-19 09:37:16 +01002088 commits = []
Luke Diamand00ad6e32015-11-21 09:54:41 +00002089 if self.master:
2090 commitish = self.master
2091 else:
2092 commitish = 'HEAD'
2093
2094 for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
Simon Hausmann4c750c02008-02-19 09:37:16 +01002095 commits.append(line.strip())
2096 commits.reverse()
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002097
Pete Wyckoff0d609032013-01-26 22:11:24 -05002098 if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
Luke Diamand848de9c2011-05-13 20:46:00 +01002099 self.checkAuthorship = False
2100 else:
2101 self.checkAuthorship = True
2102
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01002103 if self.preserveUser:
2104 self.checkValidP4Users(commits)
2105
Gary Gibbons84cb0002012-07-04 09:40:19 -04002106 #
2107 # Build up a set of options to be passed to diff when
2108 # submitting each commit to p4.
2109 #
2110 if self.detectRenames:
2111 # command-line -M arg
2112 self.diffOpts = "-M"
2113 else:
2114 # If not explicitly set check the config variable
2115 detectRenames = gitConfig("git-p4.detectRenames")
2116
2117 if detectRenames.lower() == "false" or detectRenames == "":
2118 self.diffOpts = ""
2119 elif detectRenames.lower() == "true":
2120 self.diffOpts = "-M"
2121 else:
2122 self.diffOpts = "-M%s" % detectRenames
2123
2124 # no command-line arg for -C or --find-copies-harder, just
2125 # config variables
2126 detectCopies = gitConfig("git-p4.detectCopies")
2127 if detectCopies.lower() == "false" or detectCopies == "":
2128 pass
2129 elif detectCopies.lower() == "true":
2130 self.diffOpts += " -C"
2131 else:
2132 self.diffOpts += " -C%s" % detectCopies
2133
Pete Wyckoff0d609032013-01-26 22:11:24 -05002134 if gitConfigBool("git-p4.detectCopiesHarder"):
Gary Gibbons84cb0002012-07-04 09:40:19 -04002135 self.diffOpts += " --find-copies-harder"
2136
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04002137 #
2138 # Apply the commits, one at a time. On failure, ask if should
2139 # continue to try the rest of the patches, or quit.
2140 #
Pete Wyckoffef739f02012-09-09 16:16:11 -04002141 if self.dry_run:
2142 print "Would apply"
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002143 applied = []
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04002144 last = len(commits) - 1
2145 for i, commit in enumerate(commits):
Pete Wyckoffef739f02012-09-09 16:16:11 -04002146 if self.dry_run:
2147 print " ", read_pipe(["git", "show", "-s",
2148 "--format=format:%h %s", commit])
2149 ok = True
2150 else:
2151 ok = self.applyCommit(commit)
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002152 if ok:
2153 applied.append(commit)
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04002154 else:
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04002155 if self.prepare_p4_only and i < last:
2156 print "Processing only the first commit due to option" \
2157 " --prepare-p4-only"
2158 break
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04002159 if i < last:
2160 quit = False
2161 while True:
Pete Wyckoff6bbfd132012-09-09 16:16:13 -04002162 # prompt for what to do, or use the option/variable
2163 if self.conflict_behavior == "ask":
2164 print "What do you want to do?"
2165 response = raw_input("[s]kip this commit but apply"
2166 " the rest, or [q]uit? ")
2167 if not response:
2168 continue
2169 elif self.conflict_behavior == "skip":
2170 response = "s"
2171 elif self.conflict_behavior == "quit":
2172 response = "q"
2173 else:
2174 die("Unknown conflict_behavior '%s'" %
2175 self.conflict_behavior)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002176
Pete Wyckoff7e5dd9f2012-09-09 16:16:05 -04002177 if response[0] == "s":
2178 print "Skipping this commit, but applying the rest"
2179 break
2180 if response[0] == "q":
2181 print "Quitting"
2182 quit = True
2183 break
2184 if quit:
2185 break
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002186
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002187 chdir(self.oldWorkingDirectory)
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00002188 shelved_applied = "shelved" if self.shelve else "applied"
Pete Wyckoffef739f02012-09-09 16:16:11 -04002189 if self.dry_run:
2190 pass
Pete Wyckoff728b7ad2012-09-09 16:16:12 -04002191 elif self.prepare_p4_only:
2192 pass
Pete Wyckoffef739f02012-09-09 16:16:11 -04002193 elif len(commits) == len(applied):
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00002194 print ("All commits {0}!".format(shelved_applied))
Simon Hausmann14594f42007-08-22 09:07:15 +02002195
Simon Hausmann4c750c02008-02-19 09:37:16 +01002196 sync = P4Sync()
Pete Wyckoff44e8d262013-01-14 19:47:08 -05002197 if self.branch:
2198 sync.branch = self.branch
Simon Hausmann4c750c02008-02-19 09:37:16 +01002199 sync.run([])
Simon Hausmann14594f42007-08-22 09:07:15 +02002200
Simon Hausmann4c750c02008-02-19 09:37:16 +01002201 rebase = P4Rebase()
2202 rebase.rebase()
Simon Hausmann4f5cf762007-03-19 22:25:17 +01002203
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002204 else:
2205 if len(applied) == 0:
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00002206 print ("No commits {0}.".format(shelved_applied))
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002207 else:
Vinicius Kursancewb34fa572016-11-28 09:33:18 +00002208 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002209 for c in commits:
2210 if c in applied:
2211 star = "*"
2212 else:
2213 star = " "
2214 print star, read_pipe(["git", "show", "-s",
2215 "--format=format:%h %s", c])
2216 print "You will have to do 'git p4 sync' and rebase."
2217
Pete Wyckoff0d609032013-01-26 22:11:24 -05002218 if gitConfigBool("git-p4.exportLabels"):
Luke Diamand06dcd152012-05-11 07:25:18 +01002219 self.exportLabels = True
Luke Diamand06804c72012-04-11 17:21:24 +02002220
2221 if self.exportLabels:
2222 p4Labels = getP4Labels(self.depotPath)
2223 gitTags = getGitTags()
2224
2225 missingGitTags = gitTags - p4Labels
2226 self.exportGitTags(missingGitTags)
2227
Ondřej Bílka98e023d2013-07-29 10:18:21 +02002228 # exit with error unless everything applied perfectly
Pete Wyckoff67b0fe22012-09-09 16:16:03 -04002229 if len(commits) != len(applied):
2230 sys.exit(1)
2231
Simon Hausmannb9847332007-03-20 20:54:23 +01002232 return True
2233
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002234class View(object):
2235 """Represent a p4 view ("p4 help views"), and map files in a
2236 repo according to the view."""
2237
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002238 def __init__(self, client_name):
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002239 self.mappings = []
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002240 self.client_prefix = "//%s/" % client_name
2241 # cache results of "p4 where" to lookup client file locations
2242 self.client_spec_path_cache = {}
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002243
2244 def append(self, view_line):
2245 """Parse a view line, splitting it into depot and client
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002246 sides. Append to self.mappings, preserving order. This
2247 is only needed for tag creation."""
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002248
2249 # Split the view line into exactly two words. P4 enforces
2250 # structure on these lines that simplifies this quite a bit.
2251 #
2252 # Either or both words may be double-quoted.
2253 # Single quotes do not matter.
2254 # Double-quote marks cannot occur inside the words.
2255 # A + or - prefix is also inside the quotes.
2256 # There are no quotes unless they contain a space.
2257 # The line is already white-space stripped.
2258 # The two words are separated by a single space.
2259 #
2260 if view_line[0] == '"':
2261 # First word is double quoted. Find its end.
2262 close_quote_index = view_line.find('"', 1)
2263 if close_quote_index <= 0:
2264 die("No first-word closing quote found: %s" % view_line)
2265 depot_side = view_line[1:close_quote_index]
2266 # skip closing quote and space
2267 rhs_index = close_quote_index + 1 + 1
2268 else:
2269 space_index = view_line.find(" ")
2270 if space_index <= 0:
2271 die("No word-splitting space found: %s" % view_line)
2272 depot_side = view_line[0:space_index]
2273 rhs_index = space_index + 1
2274
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002275 # prefix + means overlay on previous mapping
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002276 if depot_side.startswith("+"):
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002277 depot_side = depot_side[1:]
2278
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002279 # prefix - means exclude this path, leave out of mappings
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002280 exclude = False
2281 if depot_side.startswith("-"):
2282 exclude = True
2283 depot_side = depot_side[1:]
2284
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002285 if not exclude:
2286 self.mappings.append(depot_side)
2287
2288 def convert_client_path(self, clientFile):
2289 # chop off //client/ part to make it relative
2290 if not clientFile.startswith(self.client_prefix):
2291 die("No prefix '%s' on clientFile '%s'" %
2292 (self.client_prefix, clientFile))
2293 return clientFile[len(self.client_prefix):]
2294
2295 def update_client_spec_path_cache(self, files):
2296 """ Caching file paths by "p4 where" batch query """
2297
2298 # List depot file paths exclude that already cached
2299 fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2300
2301 if len(fileArgs) == 0:
2302 return # All files in cache
2303
2304 where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2305 for res in where_result:
2306 if "code" in res and res["code"] == "error":
2307 # assume error is "... file(s) not in client view"
2308 continue
2309 if "clientFile" not in res:
Pete Wyckoff20005442014-01-21 18:16:46 -05002310 die("No clientFile in 'p4 where' output")
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002311 if "unmap" in res:
2312 # it will list all of them, but only one not unmap-ped
2313 continue
Lars Schneidera0a50d82015-08-28 14:00:34 +02002314 if gitConfigBool("core.ignorecase"):
2315 res['depotFile'] = res['depotFile'].lower()
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002316 self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2317
2318 # not found files or unmap files set to ""
2319 for depotFile in fileArgs:
Lars Schneidera0a50d82015-08-28 14:00:34 +02002320 if gitConfigBool("core.ignorecase"):
2321 depotFile = depotFile.lower()
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002322 if depotFile not in self.client_spec_path_cache:
2323 self.client_spec_path_cache[depotFile] = ""
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002324
2325 def map_in_client(self, depot_path):
2326 """Return the relative location in the client where this
2327 depot file should live. Returns "" if the file should
2328 not be mapped in the client."""
2329
Lars Schneidera0a50d82015-08-28 14:00:34 +02002330 if gitConfigBool("core.ignorecase"):
2331 depot_path = depot_path.lower()
2332
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002333 if depot_path in self.client_spec_path_cache:
2334 return self.client_spec_path_cache[depot_path]
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002335
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002336 die( "Error: %s is not found in client spec path" % depot_path )
2337 return ""
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002338
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01002339class P4Sync(Command, P4UserMap):
Pete Wyckoff56c09342011-02-19 08:17:57 -05002340 delete_actions = ( "delete", "move/delete", "purge" )
2341
Simon Hausmannb9847332007-03-20 20:54:23 +01002342 def __init__(self):
2343 Command.__init__(self)
Luke Diamand3ea2cfd2011-04-21 20:50:23 +01002344 P4UserMap.__init__(self)
Simon Hausmannb9847332007-03-20 20:54:23 +01002345 self.options = [
2346 optparse.make_option("--branch", dest="branch"),
2347 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2348 optparse.make_option("--changesfile", dest="changesFile"),
2349 optparse.make_option("--silent", dest="silent", action="store_true"),
Simon Hausmannef48f902007-05-17 22:17:49 +02002350 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
Luke Diamand06804c72012-04-11 17:21:24 +02002351 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
Han-Wen Nienhuysd2c6dd32007-05-23 18:49:35 -03002352 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2353 help="Import into refs/heads/ , not refs/remotes"),
Lex Spoon96b2d542015-04-20 11:00:20 -04002354 optparse.make_option("--max-changes", dest="maxChanges",
2355 help="Maximum number of changes to import"),
2356 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2357 help="Internal block size to use when iteratively calling p4 changes"),
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -03002358 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
Tor Arvid Lund3a70cdf2008-02-18 15:22:08 +01002359 help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2360 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
Luke Diamand51334bb2015-01-17 20:56:38 +00002361 help="Only sync files that are included in the Perforce Client Spec"),
2362 optparse.make_option("-/", dest="cloneExclude",
2363 action="append", type="string",
2364 help="exclude depot path"),
Simon Hausmannb9847332007-03-20 20:54:23 +01002365 ]
2366 self.description = """Imports from Perforce into a git repository.\n
2367 example:
2368 //depot/my/project/ -- to import the current head
2369 //depot/my/project/@all -- to import everything
2370 //depot/my/project/@1,6 -- to import only from revision 1 to 6
2371
2372 (a ... is not needed in the path p4 specification, it's added implicitly)"""
2373
2374 self.usage += " //depot/path[@revRange]"
Simon Hausmannb9847332007-03-20 20:54:23 +01002375 self.silent = False
Reilly Grant1d7367d2009-09-10 00:02:38 -07002376 self.createdBranches = set()
2377 self.committedChanges = set()
Simon Hausmann569d1bd2007-03-22 21:34:16 +01002378 self.branch = ""
Simon Hausmannb9847332007-03-20 20:54:23 +01002379 self.detectBranches = False
Simon Hausmanncb53e1f2007-04-08 00:12:02 +02002380 self.detectLabels = False
Luke Diamand06804c72012-04-11 17:21:24 +02002381 self.importLabels = False
Simon Hausmannb9847332007-03-20 20:54:23 +01002382 self.changesFile = ""
Simon Hausmann01265102007-05-25 10:36:10 +02002383 self.syncWithOrigin = True
Simon Hausmanna028a982007-05-23 00:03:08 +02002384 self.importIntoRemotes = True
Simon Hausmann01a9c9c2007-05-23 00:07:35 +02002385 self.maxChanges = ""
Luke Diamand1051ef02015-06-10 08:30:59 +01002386 self.changes_block_size = None
Han-Wen Nienhuys8b41a972007-05-23 18:20:53 -03002387 self.keepRepoPath = False
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002388 self.depotPaths = None
Simon Hausmann3c699642007-06-16 13:09:21 +02002389 self.p4BranchesInGit = []
Tommy Thorn354081d2008-02-03 10:38:51 -08002390 self.cloneExclude = []
Tor Arvid Lund3a70cdf2008-02-18 15:22:08 +01002391 self.useClientSpec = False
Pete Wyckoffa93d33e2012-02-25 20:06:24 -05002392 self.useClientSpec_from_options = False
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002393 self.clientSpecDirs = None
Vitor Antunesfed23692012-01-25 23:48:22 +00002394 self.tempBranches = []
Lars Schneiderd6041762016-06-29 09:35:27 +02002395 self.tempBranchLocation = "refs/git-p4-tmp"
Lars Schneidera5db4b12015-09-26 09:55:03 +02002396 self.largeFileSystem = None
2397
2398 if gitConfig('git-p4.largeFileSystem'):
2399 largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2400 self.largeFileSystem = largeFileSystemConstructor(
2401 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2402 )
Simon Hausmannb9847332007-03-20 20:54:23 +01002403
Simon Hausmann01265102007-05-25 10:36:10 +02002404 if gitConfig("git-p4.syncFromOrigin") == "false":
2405 self.syncWithOrigin = False
2406
Luke Diamand51334bb2015-01-17 20:56:38 +00002407 # This is required for the "append" cloneExclude action
2408 def ensure_value(self, attr, value):
2409 if not hasattr(self, attr) or getattr(self, attr) is None:
2410 setattr(self, attr, value)
2411 return getattr(self, attr)
2412
Vitor Antunesfed23692012-01-25 23:48:22 +00002413 # Force a checkpoint in fast-import and wait for it to finish
2414 def checkpoint(self):
2415 self.gitStream.write("checkpoint\n\n")
2416 self.gitStream.write("progress checkpoint\n\n")
2417 out = self.gitOutput.readline()
2418 if self.verbose:
2419 print "checkpoint finished: " + out
2420
Simon Hausmannb9847332007-03-20 20:54:23 +01002421 def extractFilesFromCommit(self, commit):
Tommy Thorn354081d2008-02-03 10:38:51 -08002422 self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2423 for path in self.cloneExclude]
Simon Hausmannb9847332007-03-20 20:54:23 +01002424 files = []
2425 fnum = 0
2426 while commit.has_key("depotFile%s" % fnum):
2427 path = commit["depotFile%s" % fnum]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002428
Tommy Thorn354081d2008-02-03 10:38:51 -08002429 if [p for p in self.cloneExclude
Tor Arvid Lundd53de8b2011-03-15 13:08:02 +01002430 if p4PathStartsWith(path, p)]:
Tommy Thorn354081d2008-02-03 10:38:51 -08002431 found = False
2432 else:
2433 found = [p for p in self.depotPaths
Tor Arvid Lundd53de8b2011-03-15 13:08:02 +01002434 if p4PathStartsWith(path, p)]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002435 if not found:
Simon Hausmannb9847332007-03-20 20:54:23 +01002436 fnum = fnum + 1
2437 continue
2438
2439 file = {}
2440 file["path"] = path
2441 file["rev"] = commit["rev%s" % fnum]
2442 file["action"] = commit["action%s" % fnum]
2443 file["type"] = commit["type%s" % fnum]
2444 files.append(file)
2445 fnum = fnum + 1
2446 return files
2447
Jan Durovec26e6a272016-04-19 19:49:41 +00002448 def extractJobsFromCommit(self, commit):
2449 jobs = []
2450 jnum = 0
2451 while commit.has_key("job%s" % jnum):
2452 job = commit["job%s" % jnum]
2453 jobs.append(job)
2454 jnum = jnum + 1
2455 return jobs
2456
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002457 def stripRepoPath(self, path, prefixes):
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002458 """When streaming files, this is called to map a p4 depot path
2459 to where it should go in git. The prefixes are either
2460 self.depotPaths, or self.branchPrefixes in the case of
2461 branch detection."""
2462
Ian Wienand39527102011-02-11 16:33:48 -08002463 if self.useClientSpec:
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002464 # branch detection moves files up a level (the branch name)
2465 # from what client spec interpretation gives
Pete Wyckoff0d1696e2012-08-11 12:55:03 -04002466 path = self.clientSpecDirs.map_in_client(path)
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002467 if self.detectBranches:
2468 for b in self.knownBranches:
2469 if path.startswith(b + "/"):
2470 path = path[len(b)+1:]
2471
2472 elif self.keepRepoPath:
2473 # Preserve everything in relative path name except leading
2474 # //depot/; just look at first prefix as they all should
2475 # be in the same depot.
2476 depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2477 if p4PathStartsWith(path, depot):
2478 path = path[len(depot):]
Ian Wienand39527102011-02-11 16:33:48 -08002479
Pete Wyckoff0d1696e2012-08-11 12:55:03 -04002480 else:
Pete Wyckoff0d1696e2012-08-11 12:55:03 -04002481 for p in prefixes:
2482 if p4PathStartsWith(path, p):
2483 path = path[len(p):]
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002484 break
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002485
Pete Wyckoff0d1696e2012-08-11 12:55:03 -04002486 path = wildcard_decode(path)
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002487 return path
Han-Wen Nienhuys6754a292007-05-23 17:41:50 -03002488
Simon Hausmann71b112d2007-05-19 11:54:11 +02002489 def splitFilesIntoBranches(self, commit):
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002490 """Look at each depotFile in the commit to figure out to what
2491 branch it belongs."""
2492
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002493 if self.clientSpecDirs:
2494 files = self.extractFilesFromCommit(commit)
2495 self.clientSpecDirs.update_client_spec_path_cache(files)
2496
Simon Hausmannd5904672007-05-19 11:07:32 +02002497 branches = {}
Simon Hausmann71b112d2007-05-19 11:54:11 +02002498 fnum = 0
2499 while commit.has_key("depotFile%s" % fnum):
2500 path = commit["depotFile%s" % fnum]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002501 found = [p for p in self.depotPaths
Tor Arvid Lundd53de8b2011-03-15 13:08:02 +01002502 if p4PathStartsWith(path, p)]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03002503 if not found:
Simon Hausmann71b112d2007-05-19 11:54:11 +02002504 fnum = fnum + 1
2505 continue
2506
2507 file = {}
2508 file["path"] = path
2509 file["rev"] = commit["rev%s" % fnum]
2510 file["action"] = commit["action%s" % fnum]
2511 file["type"] = commit["type%s" % fnum]
2512 fnum = fnum + 1
2513
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002514 # start with the full relative path where this file would
2515 # go in a p4 client
2516 if self.useClientSpec:
2517 relPath = self.clientSpecDirs.map_in_client(path)
2518 else:
2519 relPath = self.stripRepoPath(path, self.depotPaths)
Simon Hausmannb9847332007-03-20 20:54:23 +01002520
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02002521 for branch in self.knownBranches.keys():
Pete Wyckoff21ef5df2012-08-11 12:55:04 -04002522 # add a trailing slash so that a commit into qt/4.2foo
2523 # doesn't end up in qt/4.2, e.g.
Han-Wen Nienhuys6754a292007-05-23 17:41:50 -03002524 if relPath.startswith(branch + "/"):
Simon Hausmannd5904672007-05-19 11:07:32 +02002525 if branch not in branches:
2526 branches[branch] = []
Simon Hausmann71b112d2007-05-19 11:54:11 +02002527 branches[branch].append(file)
Simon Hausmann6555b2c2007-06-17 11:25:34 +02002528 break
Simon Hausmannb9847332007-03-20 20:54:23 +01002529
2530 return branches
2531
Lars Schneidera5db4b12015-09-26 09:55:03 +02002532 def writeToGitStream(self, gitMode, relPath, contents):
2533 self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2534 self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2535 for d in contents:
2536 self.gitStream.write(d)
2537 self.gitStream.write('\n')
2538
Lars Schneidera8b05162017-02-09 16:06:56 +01002539 def encodeWithUTF8(self, path):
2540 try:
2541 path.decode('ascii')
2542 except:
2543 encoding = 'utf8'
2544 if gitConfig('git-p4.pathEncoding'):
2545 encoding = gitConfig('git-p4.pathEncoding')
2546 path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2547 if self.verbose:
2548 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2549 return path
2550
Luke Diamandb9327052009-07-30 00:13:46 +01002551 # output one file from the P4 stream
2552 # - helper for streamP4Files
2553
2554 def streamOneP4File(self, file, contents):
Luke Diamandb9327052009-07-30 00:13:46 +01002555 relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
Lars Schneidera8b05162017-02-09 16:06:56 +01002556 relPath = self.encodeWithUTF8(relPath)
Luke Diamandb9327052009-07-30 00:13:46 +01002557 if verbose:
Lars Schneiderd2176a52015-09-26 09:55:01 +02002558 size = int(self.stream_file['fileSize'])
2559 sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2560 sys.stdout.flush()
Luke Diamandb9327052009-07-30 00:13:46 +01002561
Pete Wyckoff9cffb8c2011-10-16 10:45:01 -04002562 (type_base, type_mods) = split_p4_type(file["type"])
2563
2564 git_mode = "100644"
2565 if "x" in type_mods:
2566 git_mode = "100755"
2567 if type_base == "symlink":
2568 git_mode = "120000"
Alexandru Juncu1292df12013-08-08 16:17:38 +03002569 # p4 print on a symlink sometimes contains "target\n";
2570 # if it does, remove the newline
Evan Powersb39c3612010-02-16 00:44:08 -08002571 data = ''.join(contents)
Pete Wyckoff40f846c2014-01-21 18:16:40 -05002572 if not data:
2573 # Some version of p4 allowed creating a symlink that pointed
2574 # to nothing. This causes p4 errors when checking out such
2575 # a change, and errors here too. Work around it by ignoring
2576 # the bad symlink; hopefully a future change fixes it.
2577 print "\nIgnoring empty symlink in %s" % file['depotFile']
2578 return
2579 elif data[-1] == '\n':
Alexandru Juncu1292df12013-08-08 16:17:38 +03002580 contents = [data[:-1]]
2581 else:
2582 contents = [data]
Luke Diamandb9327052009-07-30 00:13:46 +01002583
Pete Wyckoff9cffb8c2011-10-16 10:45:01 -04002584 if type_base == "utf16":
Pete Wyckoff55aa5712011-09-17 19:16:14 -04002585 # p4 delivers different text in the python output to -G
2586 # than it does when using "print -o", or normal p4 client
2587 # operations. utf16 is converted to ascii or utf8, perhaps.
2588 # But ascii text saved as -t utf16 is completely mangled.
2589 # Invoke print -o to get the real contents.
Pete Wyckoff7f0e5962013-01-26 22:11:13 -05002590 #
2591 # On windows, the newlines will always be mangled by print, so put
2592 # them back too. This is not needed to the cygwin windows version,
2593 # just the native "NT" type.
2594 #
Lars Schneider1f5f3902015-09-21 12:01:41 +02002595 try:
2596 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2597 except Exception as e:
2598 if 'Translation of file content failed' in str(e):
2599 type_base = 'binary'
2600 else:
2601 raise e
2602 else:
2603 if p4_version_string().find('/NT') >= 0:
2604 text = text.replace('\r\n', '\n')
2605 contents = [ text ]
Pete Wyckoff55aa5712011-09-17 19:16:14 -04002606
Pete Wyckoff9f7ef0e2011-11-05 13:36:07 -04002607 if type_base == "apple":
2608 # Apple filetype files will be streamed as a concatenation of
2609 # its appledouble header and the contents. This is useless
2610 # on both macs and non-macs. If using "print -q -o xx", it
2611 # will create "xx" with the data, and "%xx" with the header.
2612 # This is also not very useful.
2613 #
2614 # Ideally, someday, this script can learn how to generate
2615 # appledouble files directly and import those to git, but
2616 # non-mac machines can never find a use for apple filetype.
2617 print "\nIgnoring apple filetype file %s" % file['depotFile']
2618 return
2619
Pete Wyckoff55aa5712011-09-17 19:16:14 -04002620 # Note that we do not try to de-mangle keywords on utf16 files,
2621 # even though in theory somebody may want that.
Luke Diamand60df0712012-02-23 07:51:30 +00002622 pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2623 if pattern:
2624 regexp = re.compile(pattern, re.VERBOSE)
2625 text = ''.join(contents)
2626 text = regexp.sub(r'$\1$', text)
2627 contents = [ text ]
Luke Diamandb9327052009-07-30 00:13:46 +01002628
Lars Schneidera5db4b12015-09-26 09:55:03 +02002629 if self.largeFileSystem:
2630 (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
Luke Diamandb9327052009-07-30 00:13:46 +01002631
Lars Schneidera5db4b12015-09-26 09:55:03 +02002632 self.writeToGitStream(git_mode, relPath, contents)
Luke Diamandb9327052009-07-30 00:13:46 +01002633
2634 def streamOneP4Deletion(self, file):
2635 relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
Lars Schneidera8b05162017-02-09 16:06:56 +01002636 relPath = self.encodeWithUTF8(relPath)
Luke Diamandb9327052009-07-30 00:13:46 +01002637 if verbose:
Lars Schneiderd2176a52015-09-26 09:55:01 +02002638 sys.stdout.write("delete %s\n" % relPath)
2639 sys.stdout.flush()
Luke Diamandb9327052009-07-30 00:13:46 +01002640 self.gitStream.write("D %s\n" % relPath)
2641
Lars Schneidera5db4b12015-09-26 09:55:03 +02002642 if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2643 self.largeFileSystem.removeLargeFile(relPath)
2644
Luke Diamandb9327052009-07-30 00:13:46 +01002645 # handle another chunk of streaming data
2646 def streamP4FilesCb(self, marshalled):
2647
Pete Wyckoff78189be2012-11-23 17:35:36 -05002648 # catch p4 errors and complain
2649 err = None
2650 if "code" in marshalled:
2651 if marshalled["code"] == "error":
2652 if "data" in marshalled:
2653 err = marshalled["data"].rstrip()
Lars Schneider4d25dc42015-09-26 09:55:02 +02002654
2655 if not err and 'fileSize' in self.stream_file:
2656 required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2657 if required_bytes > 0:
2658 err = 'Not enough space left on %s! Free at least %i MB.' % (
2659 os.getcwd(), required_bytes/1024/1024
2660 )
2661
Pete Wyckoff78189be2012-11-23 17:35:36 -05002662 if err:
2663 f = None
2664 if self.stream_have_file_info:
2665 if "depotFile" in self.stream_file:
2666 f = self.stream_file["depotFile"]
2667 # force a failure in fast-import, else an empty
2668 # commit will be made
2669 self.gitStream.write("\n")
2670 self.gitStream.write("die-now\n")
2671 self.gitStream.close()
2672 # ignore errors, but make sure it exits first
2673 self.importProcess.wait()
2674 if f:
2675 die("Error from p4 print for %s: %s" % (f, err))
2676 else:
2677 die("Error from p4 print: %s" % err)
2678
Andrew Garberc3f61632011-04-07 02:01:21 -04002679 if marshalled.has_key('depotFile') and self.stream_have_file_info:
2680 # start of a new file - output the old one first
2681 self.streamOneP4File(self.stream_file, self.stream_contents)
2682 self.stream_file = {}
2683 self.stream_contents = []
2684 self.stream_have_file_info = False
Luke Diamandb9327052009-07-30 00:13:46 +01002685
Andrew Garberc3f61632011-04-07 02:01:21 -04002686 # pick up the new file information... for the
2687 # 'data' field we need to append to our array
2688 for k in marshalled.keys():
2689 if k == 'data':
Lars Schneiderd2176a52015-09-26 09:55:01 +02002690 if 'streamContentSize' not in self.stream_file:
2691 self.stream_file['streamContentSize'] = 0
2692 self.stream_file['streamContentSize'] += len(marshalled['data'])
Andrew Garberc3f61632011-04-07 02:01:21 -04002693 self.stream_contents.append(marshalled['data'])
2694 else:
2695 self.stream_file[k] = marshalled[k]
Luke Diamandb9327052009-07-30 00:13:46 +01002696
Lars Schneiderd2176a52015-09-26 09:55:01 +02002697 if (verbose and
2698 'streamContentSize' in self.stream_file and
2699 'fileSize' in self.stream_file and
2700 'depotFile' in self.stream_file):
2701 size = int(self.stream_file["fileSize"])
2702 if size > 0:
2703 progress = 100*self.stream_file['streamContentSize']/size
2704 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2705 sys.stdout.flush()
2706
Andrew Garberc3f61632011-04-07 02:01:21 -04002707 self.stream_have_file_info = True
Luke Diamandb9327052009-07-30 00:13:46 +01002708
2709 # Stream directly from "p4 files" into "git fast-import"
2710 def streamP4Files(self, files):
Simon Hausmann30b59402008-03-03 11:55:48 +01002711 filesForCommit = []
2712 filesToRead = []
Luke Diamandb9327052009-07-30 00:13:46 +01002713 filesToDelete = []
Simon Hausmann30b59402008-03-03 11:55:48 +01002714
Tor Arvid Lund3a70cdf2008-02-18 15:22:08 +01002715 for f in files:
Pete Wyckoffecb7cf92012-01-02 18:05:53 -05002716 filesForCommit.append(f)
2717 if f['action'] in self.delete_actions:
2718 filesToDelete.append(f)
2719 else:
2720 filesToRead.append(f)
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -03002721
Luke Diamandb9327052009-07-30 00:13:46 +01002722 # deleted files...
2723 for f in filesToDelete:
2724 self.streamOneP4Deletion(f)
2725
Simon Hausmann30b59402008-03-03 11:55:48 +01002726 if len(filesToRead) > 0:
Luke Diamandb9327052009-07-30 00:13:46 +01002727 self.stream_file = {}
2728 self.stream_contents = []
2729 self.stream_have_file_info = False
2730
Andrew Garberc3f61632011-04-07 02:01:21 -04002731 # curry self argument
2732 def streamP4FilesCbSelf(entry):
2733 self.streamP4FilesCb(entry)
Luke Diamandb9327052009-07-30 00:13:46 +01002734
Luke Diamand6de040d2011-10-16 10:47:52 -04002735 fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2736
2737 p4CmdList(["-x", "-", "print"],
2738 stdin=fileArgs,
2739 cb=streamP4FilesCbSelf)
Han-Wen Nienhuysf2eda792007-05-23 18:49:35 -03002740
Luke Diamandb9327052009-07-30 00:13:46 +01002741 # do the last chunk
2742 if self.stream_file.has_key('depotFile'):
2743 self.streamOneP4File(self.stream_file, self.stream_contents)
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -03002744
Luke Diamandaffb4742012-01-19 09:52:27 +00002745 def make_email(self, userid):
2746 if userid in self.users:
2747 return self.users[userid]
2748 else:
2749 return "%s <a@b>" % userid
2750
Luke Diamand06804c72012-04-11 17:21:24 +02002751 def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
Luke Diamandb43702a2015-08-27 08:18:58 +01002752 """ Stream a p4 tag.
2753 commit is either a git commit, or a fast-import mark, ":<p4commit>"
2754 """
2755
Luke Diamand06804c72012-04-11 17:21:24 +02002756 if verbose:
2757 print "writing tag %s for commit %s" % (labelName, commit)
2758 gitStream.write("tag %s\n" % labelName)
2759 gitStream.write("from %s\n" % commit)
2760
2761 if labelDetails.has_key('Owner'):
2762 owner = labelDetails["Owner"]
2763 else:
2764 owner = None
2765
2766 # Try to use the owner of the p4 label, or failing that,
2767 # the current p4 user id.
2768 if owner:
2769 email = self.make_email(owner)
2770 else:
2771 email = self.make_email(self.p4UserId())
2772 tagger = "%s %s %s" % (email, epoch, self.tz)
2773
2774 gitStream.write("tagger %s\n" % tagger)
2775
2776 print "labelDetails=",labelDetails
2777 if labelDetails.has_key('Description'):
2778 description = labelDetails['Description']
2779 else:
2780 description = 'Label from git p4'
2781
2782 gitStream.write("data %d\n" % len(description))
2783 gitStream.write(description)
2784 gitStream.write("\n")
2785
Lars Schneider4ae048e2015-12-08 10:36:22 +01002786 def inClientSpec(self, path):
2787 if not self.clientSpecDirs:
2788 return True
2789 inClientSpec = self.clientSpecDirs.map_in_client(path)
2790 if not inClientSpec and self.verbose:
2791 print('Ignoring file outside of client spec: {0}'.format(path))
2792 return inClientSpec
2793
2794 def hasBranchPrefix(self, path):
2795 if not self.branchPrefixes:
2796 return True
2797 hasPrefix = [p for p in self.branchPrefixes
2798 if p4PathStartsWith(path, p)]
Andrew Oakley09667d02016-06-22 10:26:11 +01002799 if not hasPrefix and self.verbose:
Lars Schneider4ae048e2015-12-08 10:36:22 +01002800 print('Ignoring file outside of prefix: {0}'.format(path))
2801 return hasPrefix
2802
Pete Wyckoffe63231e2012-08-11 12:55:02 -04002803 def commit(self, details, files, branch, parent = ""):
Simon Hausmannb9847332007-03-20 20:54:23 +01002804 epoch = details["time"]
2805 author = details["user"]
Jan Durovec26e6a272016-04-19 19:49:41 +00002806 jobs = self.extractJobsFromCommit(details)
Simon Hausmannb9847332007-03-20 20:54:23 +01002807
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02002808 if self.verbose:
Lars Schneider4ae048e2015-12-08 10:36:22 +01002809 print('commit into {0}'.format(branch))
Han-Wen Nienhuys96e07dd2007-05-23 18:49:35 -03002810
Kazuki Saitoh9d57c4a2013-08-30 19:02:06 +09002811 if self.clientSpecDirs:
2812 self.clientSpecDirs.update_client_spec_path_cache(files)
2813
Lars Schneider4ae048e2015-12-08 10:36:22 +01002814 files = [f for f in files
2815 if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2816
2817 if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2818 print('Ignoring revision {0} as it would produce an empty commit.'
2819 .format(details['change']))
2820 return
2821
Simon Hausmannb9847332007-03-20 20:54:23 +01002822 self.gitStream.write("commit %s\n" % branch)
Luke Diamandb43702a2015-08-27 08:18:58 +01002823 self.gitStream.write("mark :%s\n" % details["change"])
Simon Hausmannb9847332007-03-20 20:54:23 +01002824 self.committedChanges.add(int(details["change"]))
2825 committer = ""
Simon Hausmannb607e712007-05-20 10:55:54 +02002826 if author not in self.users:
2827 self.getUserMapFromPerforceServer()
Luke Diamandaffb4742012-01-19 09:52:27 +00002828 committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
Simon Hausmannb9847332007-03-20 20:54:23 +01002829
2830 self.gitStream.write("committer %s\n" % committer)
2831
2832 self.gitStream.write("data <<EOT\n")
2833 self.gitStream.write(details["desc"])
Jan Durovec26e6a272016-04-19 19:49:41 +00002834 if len(jobs) > 0:
2835 self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
Pete Wyckoffe63231e2012-08-11 12:55:02 -04002836 self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2837 (','.join(self.branchPrefixes), details["change"]))
Simon Hausmann6581de02007-06-11 10:01:58 +02002838 if len(details['options']) > 0:
2839 self.gitStream.write(": options = %s" % details['options'])
2840 self.gitStream.write("]\nEOT\n\n")
Simon Hausmannb9847332007-03-20 20:54:23 +01002841
2842 if len(parent) > 0:
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02002843 if self.verbose:
2844 print "parent %s" % parent
Simon Hausmannb9847332007-03-20 20:54:23 +01002845 self.gitStream.write("from %s\n" % parent)
2846
Lars Schneider4ae048e2015-12-08 10:36:22 +01002847 self.streamP4Files(files)
Simon Hausmannb9847332007-03-20 20:54:23 +01002848 self.gitStream.write("\n")
2849
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002850 change = int(details["change"])
2851
Simon Hausmann9bda3a82007-05-19 12:05:40 +02002852 if self.labels.has_key(change):
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002853 label = self.labels[change]
2854 labelDetails = label[0]
2855 labelRevisions = label[1]
Simon Hausmann71b112d2007-05-19 11:54:11 +02002856 if self.verbose:
2857 print "Change %s is labelled %s" % (change, labelDetails)
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002858
Luke Diamand6de040d2011-10-16 10:47:52 -04002859 files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
Pete Wyckoffe63231e2012-08-11 12:55:02 -04002860 for p in self.branchPrefixes])
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002861
2862 if len(files) == len(labelRevisions):
2863
2864 cleanedFiles = {}
2865 for info in files:
Pete Wyckoff56c09342011-02-19 08:17:57 -05002866 if info["action"] in self.delete_actions:
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002867 continue
2868 cleanedFiles[info["depotFile"]] = info["rev"]
2869
2870 if cleanedFiles == labelRevisions:
Luke Diamand06804c72012-04-11 17:21:24 +02002871 self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002872
2873 else:
Simon Hausmanna46668f2007-03-28 17:05:38 +02002874 if not self.silent:
Han-Wen Nienhuyscebdf5a2007-05-23 16:53:11 -03002875 print ("Tag %s does not match with change %s: files do not match."
2876 % (labelDetails["label"], change))
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002877
2878 else:
Simon Hausmanna46668f2007-03-28 17:05:38 +02002879 if not self.silent:
Han-Wen Nienhuyscebdf5a2007-05-23 16:53:11 -03002880 print ("Tag %s does not match with change %s: file count is different."
2881 % (labelDetails["label"], change))
Simon Hausmannb9847332007-03-20 20:54:23 +01002882
Luke Diamand06804c72012-04-11 17:21:24 +02002883 # Build a dictionary of changelists and labels, for "detect-labels" option.
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002884 def getLabels(self):
2885 self.labels = {}
2886
Luke Diamand52a48802012-01-19 09:52:25 +00002887 l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
Simon Hausmann10c32112007-04-08 10:15:47 +02002888 if len(l) > 0 and not self.silent:
Shun Kei Leung183f8432007-11-21 11:01:19 +08002889 print "Finding files belonging to labels in %s" % `self.depotPaths`
Simon Hausmann01ce1fe2007-04-07 23:46:50 +02002890
2891 for output in l:
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002892 label = output["label"]
2893 revisions = {}
2894 newestChange = 0
Simon Hausmann71b112d2007-05-19 11:54:11 +02002895 if self.verbose:
2896 print "Querying files for label %s" % label
Luke Diamand6de040d2011-10-16 10:47:52 -04002897 for file in p4CmdList(["files"] +
2898 ["%s...@%s" % (p, label)
2899 for p in self.depotPaths]):
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002900 revisions[file["depotFile"]] = file["rev"]
2901 change = int(file["change"])
2902 if change > newestChange:
2903 newestChange = change
2904
Simon Hausmann9bda3a82007-05-19 12:05:40 +02002905 self.labels[newestChange] = [output, revisions]
2906
2907 if self.verbose:
2908 print "Label changes: %s" % self.labels.keys()
Simon Hausmann1f4ba1c2007-03-26 22:34:34 +02002909
Luke Diamand06804c72012-04-11 17:21:24 +02002910 # Import p4 labels as git tags. A direct mapping does not
2911 # exist, so assume that if all the files are at the same revision
2912 # then we can use that, or it's something more complicated we should
2913 # just ignore.
2914 def importP4Labels(self, stream, p4Labels):
2915 if verbose:
2916 print "import p4 labels: " + ' '.join(p4Labels)
2917
2918 ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
Luke Diamandc8942a22012-04-11 17:21:24 +02002919 validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
Luke Diamand06804c72012-04-11 17:21:24 +02002920 if len(validLabelRegexp) == 0:
2921 validLabelRegexp = defaultLabelRegexp
2922 m = re.compile(validLabelRegexp)
2923
2924 for name in p4Labels:
2925 commitFound = False
2926
2927 if not m.match(name):
2928 if verbose:
2929 print "label %s does not match regexp %s" % (name,validLabelRegexp)
2930 continue
2931
2932 if name in ignoredP4Labels:
2933 continue
2934
2935 labelDetails = p4CmdList(['label', "-o", name])[0]
2936
2937 # get the most recent changelist for each file in this label
2938 change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2939 for p in self.depotPaths])
2940
2941 if change.has_key('change'):
2942 # find the corresponding git commit; take the oldest commit
2943 changelist = int(change['change'])
Luke Diamandb43702a2015-08-27 08:18:58 +01002944 if changelist in self.committedChanges:
2945 gitCommit = ":%d" % changelist # use a fast-import mark
Luke Diamand06804c72012-04-11 17:21:24 +02002946 commitFound = True
Luke Diamandb43702a2015-08-27 08:18:58 +01002947 else:
2948 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2949 "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2950 if len(gitCommit) == 0:
2951 print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2952 else:
2953 commitFound = True
2954 gitCommit = gitCommit.strip()
2955
2956 if commitFound:
Luke Diamand06804c72012-04-11 17:21:24 +02002957 # Convert from p4 time format
2958 try:
2959 tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2960 except ValueError:
Pete Wyckoffa4e90542012-11-23 17:35:38 -05002961 print "Could not convert label time %s" % labelDetails['Update']
Luke Diamand06804c72012-04-11 17:21:24 +02002962 tmwhen = 1
2963
2964 when = int(time.mktime(tmwhen))
2965 self.streamTag(stream, name, labelDetails, gitCommit, when)
2966 if verbose:
2967 print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2968 else:
2969 if verbose:
2970 print "Label %s has no changelists - possibly deleted?" % name
2971
2972 if not commitFound:
2973 # We can't import this label; don't try again as it will get very
2974 # expensive repeatedly fetching all the files for labels that will
2975 # never be imported. If the label is moved in the future, the
2976 # ignore will need to be removed manually.
2977 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2978
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -03002979 def guessProjectName(self):
2980 for p in self.depotPaths:
Simon Hausmann6e5295c2007-06-11 08:50:57 +02002981 if p.endswith("/"):
2982 p = p[:-1]
2983 p = p[p.strip().rfind("/") + 1:]
2984 if not p.endswith("/"):
2985 p += "/"
2986 return p
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -03002987
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02002988 def getBranchMapping(self):
Simon Hausmann6555b2c2007-06-17 11:25:34 +02002989 lostAndFoundBranches = set()
2990
Vitor Antunes8ace74c2011-08-19 00:44:04 +01002991 user = gitConfig("git-p4.branchUser")
2992 if len(user) > 0:
2993 command = "branches -u %s" % user
2994 else:
2995 command = "branches"
2996
2997 for info in p4CmdList(command):
Luke Diamand52a48802012-01-19 09:52:25 +00002998 details = p4Cmd(["branch", "-o", info["branch"]])
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02002999 viewIdx = 0
3000 while details.has_key("View%s" % viewIdx):
3001 paths = details["View%s" % viewIdx].split(" ")
3002 viewIdx = viewIdx + 1
3003 # require standard //depot/foo/... //depot/bar/... mapping
3004 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
3005 continue
3006 source = paths[0]
3007 destination = paths[1]
Simon Hausmann6509e192007-06-07 09:41:53 +02003008 ## HACK
Tor Arvid Lundd53de8b2011-03-15 13:08:02 +01003009 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
Simon Hausmann6509e192007-06-07 09:41:53 +02003010 source = source[len(self.depotPaths[0]):-4]
3011 destination = destination[len(self.depotPaths[0]):-4]
Simon Hausmann6555b2c2007-06-17 11:25:34 +02003012
Simon Hausmann1a2edf42007-06-17 15:10:24 +02003013 if destination in self.knownBranches:
3014 if not self.silent:
3015 print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
3016 print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
3017 continue
3018
Simon Hausmann6555b2c2007-06-17 11:25:34 +02003019 self.knownBranches[destination] = source
3020
3021 lostAndFoundBranches.discard(destination)
3022
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003023 if source not in self.knownBranches:
Simon Hausmann6555b2c2007-06-17 11:25:34 +02003024 lostAndFoundBranches.add(source)
3025
Vitor Antunes7199cf12011-08-19 00:44:05 +01003026 # Perforce does not strictly require branches to be defined, so we also
3027 # check git config for a branch list.
3028 #
3029 # Example of branch definition in git config file:
3030 # [git-p4]
3031 # branchList=main:branchA
3032 # branchList=main:branchB
3033 # branchList=branchA:branchC
3034 configBranches = gitConfigList("git-p4.branchList")
3035 for branch in configBranches:
3036 if branch:
3037 (source, destination) = branch.split(":")
3038 self.knownBranches[destination] = source
3039
3040 lostAndFoundBranches.discard(destination)
3041
3042 if source not in self.knownBranches:
3043 lostAndFoundBranches.add(source)
3044
Simon Hausmann6555b2c2007-06-17 11:25:34 +02003045
3046 for branch in lostAndFoundBranches:
3047 self.knownBranches[branch] = branch
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003048
Simon Hausmann38f9f5e2007-11-15 10:38:45 +01003049 def getBranchMappingFromGitBranches(self):
3050 branches = p4BranchesInGit(self.importIntoRemotes)
3051 for branch in branches.keys():
3052 if branch == "master":
3053 branch = "main"
3054 else:
3055 branch = branch[len(self.projectName):]
3056 self.knownBranches[branch] = branch
3057
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003058 def updateOptionDict(self, d):
3059 option_keys = {}
3060 if self.keepRepoPath:
3061 option_keys['keepRepoPath'] = 1
3062
3063 d["options"] = ' '.join(sorted(option_keys.keys()))
3064
3065 def readOptions(self, d):
3066 self.keepRepoPath = (d.has_key('options')
3067 and ('keepRepoPath' in d['options']))
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003068
Simon Hausmann8134f692007-08-26 16:44:55 +02003069 def gitRefForBranch(self, branch):
3070 if branch == "main":
3071 return self.refPrefix + "master"
3072
3073 if len(branch) <= 0:
3074 return branch
3075
3076 return self.refPrefix + self.projectName + branch
3077
Simon Hausmann1ca3d712007-08-26 17:36:55 +02003078 def gitCommitByP4Change(self, ref, change):
3079 if self.verbose:
3080 print "looking in ref " + ref + " for change %s using bisect..." % change
3081
3082 earliestCommit = ""
3083 latestCommit = parseRevision(ref)
3084
3085 while True:
3086 if self.verbose:
3087 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3088 next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3089 if len(next) == 0:
3090 if self.verbose:
3091 print "argh"
3092 return ""
3093 log = extractLogMessageFromGitCommit(next)
3094 settings = extractSettingsGitLog(log)
3095 currentChange = int(settings['change'])
3096 if self.verbose:
3097 print "current change %s" % currentChange
3098
3099 if currentChange == change:
3100 if self.verbose:
3101 print "found %s" % next
3102 return next
3103
3104 if currentChange < change:
3105 earliestCommit = "^%s" % next
3106 else:
3107 latestCommit = "%s" % next
3108
3109 return ""
3110
3111 def importNewBranch(self, branch, maxChange):
3112 # make fast-import flush all changes to disk and update the refs using the checkpoint
3113 # command so that we can try to find the branch parent in the git history
3114 self.gitStream.write("checkpoint\n\n");
3115 self.gitStream.flush();
3116 branchPrefix = self.depotPaths[0] + branch + "/"
3117 range = "@1,%s" % maxChange
3118 #print "prefix" + branchPrefix
Lex Spoon96b2d542015-04-20 11:00:20 -04003119 changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
Simon Hausmann1ca3d712007-08-26 17:36:55 +02003120 if len(changes) <= 0:
3121 return False
3122 firstChange = changes[0]
3123 #print "first change in branch: %s" % firstChange
3124 sourceBranch = self.knownBranches[branch]
3125 sourceDepotPath = self.depotPaths[0] + sourceBranch
3126 sourceRef = self.gitRefForBranch(sourceBranch)
3127 #print "source " + sourceBranch
3128
Luke Diamand52a48802012-01-19 09:52:25 +00003129 branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
Simon Hausmann1ca3d712007-08-26 17:36:55 +02003130 #print "branch parent: %s" % branchParentChange
3131 gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3132 if len(gitParent) > 0:
3133 self.initialParents[self.gitRefForBranch(branch)] = gitParent
3134 #print "parent git commit: %s" % gitParent
3135
3136 self.importChanges(changes)
3137 return True
3138
Vitor Antunesfed23692012-01-25 23:48:22 +00003139 def searchParent(self, parent, branch, target):
3140 parentFound = False
Pete Wyckoffc7d34882013-01-26 22:11:21 -05003141 for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3142 "--no-merges", parent]):
Vitor Antunesfed23692012-01-25 23:48:22 +00003143 blob = blob.strip()
3144 if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3145 parentFound = True
3146 if self.verbose:
3147 print "Found parent of %s in commit %s" % (branch, blob)
3148 break
3149 if parentFound:
3150 return blob
3151 else:
3152 return None
3153
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003154 def importChanges(self, changes):
3155 cnt = 1
3156 for change in changes:
Pete Wyckoff18fa13d2012-11-23 17:35:34 -05003157 description = p4_describe(change)
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003158 self.updateOptionDict(description)
3159
3160 if not self.silent:
3161 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3162 sys.stdout.flush()
3163 cnt = cnt + 1
3164
3165 try:
3166 if self.detectBranches:
3167 branches = self.splitFilesIntoBranches(description)
3168 for branch in branches.keys():
3169 ## HACK --hwn
3170 branchPrefix = self.depotPaths[0] + branch + "/"
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003171 self.branchPrefixes = [ branchPrefix ]
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003172
3173 parent = ""
3174
3175 filesForCommit = branches[branch]
3176
3177 if self.verbose:
3178 print "branch is %s" % branch
3179
3180 self.updatedBranches.add(branch)
3181
3182 if branch not in self.createdBranches:
3183 self.createdBranches.add(branch)
3184 parent = self.knownBranches[branch]
3185 if parent == branch:
3186 parent = ""
Simon Hausmann1ca3d712007-08-26 17:36:55 +02003187 else:
3188 fullBranch = self.projectName + branch
3189 if fullBranch not in self.p4BranchesInGit:
3190 if not self.silent:
3191 print("\n Importing new branch %s" % fullBranch);
3192 if self.importNewBranch(branch, change - 1):
3193 parent = ""
3194 self.p4BranchesInGit.append(fullBranch)
3195 if not self.silent:
3196 print("\n Resuming with change %s" % change);
3197
3198 if self.verbose:
3199 print "parent determined through known branches: %s" % parent
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003200
Simon Hausmann8134f692007-08-26 16:44:55 +02003201 branch = self.gitRefForBranch(branch)
3202 parent = self.gitRefForBranch(parent)
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003203
3204 if self.verbose:
3205 print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3206
3207 if len(parent) == 0 and branch in self.initialParents:
3208 parent = self.initialParents[branch]
3209 del self.initialParents[branch]
3210
Vitor Antunesfed23692012-01-25 23:48:22 +00003211 blob = None
3212 if len(parent) > 0:
Pete Wyckoff4f9273d2013-01-26 22:11:04 -05003213 tempBranch = "%s/%d" % (self.tempBranchLocation, change)
Vitor Antunesfed23692012-01-25 23:48:22 +00003214 if self.verbose:
3215 print "Creating temporary branch: " + tempBranch
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003216 self.commit(description, filesForCommit, tempBranch)
Vitor Antunesfed23692012-01-25 23:48:22 +00003217 self.tempBranches.append(tempBranch)
3218 self.checkpoint()
3219 blob = self.searchParent(parent, branch, tempBranch)
3220 if blob:
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003221 self.commit(description, filesForCommit, branch, blob)
Vitor Antunesfed23692012-01-25 23:48:22 +00003222 else:
3223 if self.verbose:
3224 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003225 self.commit(description, filesForCommit, branch, parent)
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003226 else:
3227 files = self.extractFilesFromCommit(description)
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003228 self.commit(description, files, self.branch,
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003229 self.initialParent)
Pete Wyckoff47497842013-01-14 19:47:04 -05003230 # only needed once, to connect to the previous commit
Simon Hausmanne87f37a2007-08-26 16:00:52 +02003231 self.initialParent = ""
3232 except IOError:
3233 print self.gitError.read()
3234 sys.exit(1)
3235
Simon Hausmannc208a242007-08-26 16:07:18 +02003236 def importHeadRevision(self, revision):
3237 print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3238
Pete Wyckoff4e2e6ce2011-07-31 09:45:55 -04003239 details = {}
3240 details["user"] = "git perforce import user"
Pete Wyckoff1494fcb2011-02-19 08:17:56 -05003241 details["desc"] = ("Initial import of %s from the state at revision %s\n"
Simon Hausmannc208a242007-08-26 16:07:18 +02003242 % (' '.join(self.depotPaths), revision))
3243 details["change"] = revision
3244 newestRevision = 0
3245
3246 fileCnt = 0
Luke Diamand6de040d2011-10-16 10:47:52 -04003247 fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3248
3249 for info in p4CmdList(["files"] + fileArgs):
Simon Hausmannc208a242007-08-26 16:07:18 +02003250
Pete Wyckoff68b28592011-02-19 08:17:55 -05003251 if 'code' in info and info['code'] == 'error':
Simon Hausmannc208a242007-08-26 16:07:18 +02003252 sys.stderr.write("p4 returned an error: %s\n"
3253 % info['data'])
Pete Wyckoffd88e7072011-02-19 08:17:58 -05003254 if info['data'].find("must refer to client") >= 0:
3255 sys.stderr.write("This particular p4 error is misleading.\n")
3256 sys.stderr.write("Perhaps the depot path was misspelled.\n");
3257 sys.stderr.write("Depot path: %s\n" % " ".join(self.depotPaths))
Simon Hausmannc208a242007-08-26 16:07:18 +02003258 sys.exit(1)
Pete Wyckoff68b28592011-02-19 08:17:55 -05003259 if 'p4ExitCode' in info:
3260 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
Simon Hausmannc208a242007-08-26 16:07:18 +02003261 sys.exit(1)
3262
3263
3264 change = int(info["change"])
3265 if change > newestRevision:
3266 newestRevision = change
3267
Pete Wyckoff56c09342011-02-19 08:17:57 -05003268 if info["action"] in self.delete_actions:
Simon Hausmannc208a242007-08-26 16:07:18 +02003269 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3270 #fileCnt = fileCnt + 1
3271 continue
3272
3273 for prop in ["depotFile", "rev", "action", "type" ]:
3274 details["%s%s" % (prop, fileCnt)] = info[prop]
3275
3276 fileCnt = fileCnt + 1
3277
3278 details["change"] = newestRevision
Pete Wyckoff4e2e6ce2011-07-31 09:45:55 -04003279
Pete Wyckoff9dcb9f22012-04-08 20:18:01 -04003280 # Use time from top-most change so that all git p4 clones of
Pete Wyckoff4e2e6ce2011-07-31 09:45:55 -04003281 # the same p4 repo have the same commit SHA1s.
Pete Wyckoff18fa13d2012-11-23 17:35:34 -05003282 res = p4_describe(newestRevision)
3283 details["time"] = res["time"]
Pete Wyckoff4e2e6ce2011-07-31 09:45:55 -04003284
Simon Hausmannc208a242007-08-26 16:07:18 +02003285 self.updateOptionDict(details)
3286 try:
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003287 self.commit(details, self.extractFilesFromCommit(details), self.branch)
Simon Hausmannc208a242007-08-26 16:07:18 +02003288 except IOError:
3289 print "IO error with git fast-import. Is your git version recent enough?"
3290 print self.gitError.read()
3291
3292
Simon Hausmannb9847332007-03-20 20:54:23 +01003293 def run(self, args):
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003294 self.depotPaths = []
Simon Hausmann179caeb2007-03-22 22:17:42 +01003295 self.changeRange = ""
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003296 self.previousDepotPaths = []
Pete Wyckoff991a2de2013-01-14 19:46:56 -05003297 self.hasOrigin = False
Han-Wen Nienhuysce6f33c2007-05-23 16:46:29 -03003298
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003299 # map from branch depot path to parent branch
3300 self.knownBranches = {}
3301 self.initialParents = {}
Simon Hausmann179caeb2007-03-22 22:17:42 +01003302
Simon Hausmanna028a982007-05-23 00:03:08 +02003303 if self.importIntoRemotes:
3304 self.refPrefix = "refs/remotes/p4/"
3305 else:
Marius Storm-Olsendb775552007-06-07 15:13:59 +02003306 self.refPrefix = "refs/heads/p4/"
Simon Hausmanna028a982007-05-23 00:03:08 +02003307
Pete Wyckoff991a2de2013-01-14 19:46:56 -05003308 if self.syncWithOrigin:
3309 self.hasOrigin = originP4BranchesExist()
3310 if self.hasOrigin:
3311 if not self.silent:
3312 print 'Syncing with origin first, using "git fetch origin"'
3313 system("git fetch origin")
Simon Hausmann10f880f2007-05-24 22:28:28 +02003314
Pete Wyckoff5a8e84c2013-01-14 19:47:05 -05003315 branch_arg_given = bool(self.branch)
Simon Hausmann569d1bd2007-03-22 21:34:16 +01003316 if len(self.branch) == 0:
Marius Storm-Olsendb775552007-06-07 15:13:59 +02003317 self.branch = self.refPrefix + "master"
Simon Hausmanna028a982007-05-23 00:03:08 +02003318 if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
Simon Hausmann48df6fd2007-05-17 21:18:53 +02003319 system("git update-ref %s refs/heads/p4" % self.branch)
Pete Wyckoff55d12432013-01-14 19:46:59 -05003320 system("git branch -D p4")
Simon Hausmann179caeb2007-03-22 22:17:42 +01003321
Pete Wyckoffa93d33e2012-02-25 20:06:24 -05003322 # accept either the command-line option, or the configuration variable
3323 if self.useClientSpec:
3324 # will use this after clone to set the variable
3325 self.useClientSpec_from_options = True
3326 else:
Pete Wyckoff0d609032013-01-26 22:11:24 -05003327 if gitConfigBool("git-p4.useclientspec"):
Pete Wyckoff09fca772011-12-24 21:07:39 -05003328 self.useClientSpec = True
3329 if self.useClientSpec:
Pete Wyckoff543987b2012-02-25 20:06:25 -05003330 self.clientSpecDirs = getClientSpec()
Tor Arvid Lund3a70cdf2008-02-18 15:22:08 +01003331
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -03003332 # TODO: should always look at previous commits,
3333 # merge with previous imports, if possible.
3334 if args == []:
Simon Hausmannd414c742007-05-25 11:36:42 +02003335 if self.hasOrigin:
Simon Hausmann5ca44612007-08-24 17:44:16 +02003336 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
Pete Wyckoff3b650fc2013-01-14 19:46:58 -05003337
3338 # branches holds mapping from branch name to sha1
3339 branches = p4BranchesInGit(self.importIntoRemotes)
Pete Wyckoff8c9e8b62013-01-14 19:47:06 -05003340
3341 # restrict to just this one, disabling detect-branches
3342 if branch_arg_given:
3343 short = self.branch.split("/")[-1]
3344 if short in branches:
3345 self.p4BranchesInGit = [ short ]
3346 else:
3347 self.p4BranchesInGit = branches.keys()
Simon Hausmannabcd7902007-05-24 22:25:36 +02003348
3349 if len(self.p4BranchesInGit) > 1:
3350 if not self.silent:
3351 print "Importing from/into multiple branches"
3352 self.detectBranches = True
Pete Wyckoff8c9e8b62013-01-14 19:47:06 -05003353 for branch in branches.keys():
3354 self.initialParents[self.refPrefix + branch] = \
3355 branches[branch]
Simon Hausmann967f72e2007-03-23 09:30:41 +01003356
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003357 if self.verbose:
3358 print "branches: %s" % self.p4BranchesInGit
3359
3360 p4Change = 0
3361 for branch in self.p4BranchesInGit:
Han-Wen Nienhuyscebdf5a2007-05-23 16:53:11 -03003362 logMsg = extractLogMessageFromGitCommit(self.refPrefix + branch)
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003363
3364 settings = extractSettingsGitLog(logMsg)
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003365
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003366 self.readOptions(settings)
3367 if (settings.has_key('depot-paths')
3368 and settings.has_key ('change')):
3369 change = int(settings['change']) + 1
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003370 p4Change = max(p4Change, change)
3371
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003372 depotPaths = sorted(settings['depot-paths'])
3373 if self.previousDepotPaths == []:
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003374 self.previousDepotPaths = depotPaths
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003375 else:
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003376 paths = []
3377 for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
Vitor Antunes04d277b2011-08-19 00:44:03 +01003378 prev_list = prev.split("/")
3379 cur_list = cur.split("/")
3380 for i in range(0, min(len(cur_list), len(prev_list))):
3381 if cur_list[i] <> prev_list[i]:
Simon Hausmann583e1702007-06-07 09:37:13 +02003382 i = i - 1
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003383 break
3384
Vitor Antunes04d277b2011-08-19 00:44:03 +01003385 paths.append ("/".join(cur_list[:i + 1]))
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003386
3387 self.previousDepotPaths = paths
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003388
3389 if p4Change > 0:
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003390 self.depotPaths = sorted(self.previousDepotPaths)
Simon Hausmannd5904672007-05-19 11:07:32 +02003391 self.changeRange = "@%s,#head" % p4Change
Simon Hausmann341dc1c2007-05-21 00:39:16 +02003392 if not self.silent and not self.detectBranches:
Simon Hausmann967f72e2007-03-23 09:30:41 +01003393 print "Performing incremental import into %s git branch" % self.branch
Simon Hausmann569d1bd2007-03-22 21:34:16 +01003394
Pete Wyckoff40d69ac2013-01-14 19:47:03 -05003395 # accept multiple ref name abbreviations:
3396 # refs/foo/bar/branch -> use it exactly
3397 # p4/branch -> prepend refs/remotes/ or refs/heads/
3398 # branch -> prepend refs/remotes/p4/ or refs/heads/p4/
Simon Hausmannf9162f62007-05-17 09:02:45 +02003399 if not self.branch.startswith("refs/"):
Pete Wyckoff40d69ac2013-01-14 19:47:03 -05003400 if self.importIntoRemotes:
3401 prepend = "refs/remotes/"
3402 else:
3403 prepend = "refs/heads/"
3404 if not self.branch.startswith("p4/"):
3405 prepend += "p4/"
3406 self.branch = prepend + self.branch
Simon Hausmann179caeb2007-03-22 22:17:42 +01003407
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003408 if len(args) == 0 and self.depotPaths:
Simon Hausmannb9847332007-03-20 20:54:23 +01003409 if not self.silent:
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003410 print "Depot paths: %s" % ' '.join(self.depotPaths)
Simon Hausmannb9847332007-03-20 20:54:23 +01003411 else:
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003412 if self.depotPaths and self.depotPaths != args:
Han-Wen Nienhuyscebdf5a2007-05-23 16:53:11 -03003413 print ("previous import used depot path %s and now %s was specified. "
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003414 "This doesn't work!" % (' '.join (self.depotPaths),
3415 ' '.join (args)))
Simon Hausmannb9847332007-03-20 20:54:23 +01003416 sys.exit(1)
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003417
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003418 self.depotPaths = sorted(args)
Simon Hausmannb9847332007-03-20 20:54:23 +01003419
Simon Hausmann1c49fc12007-08-26 16:04:34 +02003420 revision = ""
Simon Hausmannb9847332007-03-20 20:54:23 +01003421 self.users = {}
Simon Hausmannb9847332007-03-20 20:54:23 +01003422
Pete Wyckoff58c8bc72011-12-24 21:07:35 -05003423 # Make sure no revision specifiers are used when --changesfile
3424 # is specified.
3425 bad_changesfile = False
3426 if len(self.changesFile) > 0:
3427 for p in self.depotPaths:
3428 if p.find("@") >= 0 or p.find("#") >= 0:
3429 bad_changesfile = True
3430 break
3431 if bad_changesfile:
3432 die("Option --changesfile is incompatible with revision specifiers")
3433
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003434 newPaths = []
3435 for p in self.depotPaths:
3436 if p.find("@") != -1:
3437 atIdx = p.index("@")
3438 self.changeRange = p[atIdx:]
3439 if self.changeRange == "@all":
3440 self.changeRange = ""
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -03003441 elif ',' not in self.changeRange:
Simon Hausmann1c49fc12007-08-26 16:04:34 +02003442 revision = self.changeRange
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003443 self.changeRange = ""
Han-Wen Nienhuys7fcff9d2007-07-23 15:56:37 -07003444 p = p[:atIdx]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003445 elif p.find("#") != -1:
3446 hashIdx = p.index("#")
Simon Hausmann1c49fc12007-08-26 16:04:34 +02003447 revision = p[hashIdx:]
Han-Wen Nienhuys7fcff9d2007-07-23 15:56:37 -07003448 p = p[:hashIdx]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003449 elif self.previousDepotPaths == []:
Pete Wyckoff58c8bc72011-12-24 21:07:35 -05003450 # pay attention to changesfile, if given, else import
3451 # the entire p4 tree at the head revision
3452 if len(self.changesFile) == 0:
3453 revision = "#head"
Simon Hausmannb9847332007-03-20 20:54:23 +01003454
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003455 p = re.sub ("\.\.\.$", "", p)
3456 if not p.endswith("/"):
3457 p += "/"
3458
3459 newPaths.append(p)
3460
3461 self.depotPaths = newPaths
3462
Pete Wyckoffe63231e2012-08-11 12:55:02 -04003463 # --detect-branches may change this for each branch
3464 self.branchPrefixes = self.depotPaths
3465
Simon Hausmannb607e712007-05-20 10:55:54 +02003466 self.loadUserMapFromCache()
Simon Hausmanncb53e1f2007-04-08 00:12:02 +02003467 self.labels = {}
3468 if self.detectLabels:
3469 self.getLabels();
Simon Hausmannb9847332007-03-20 20:54:23 +01003470
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02003471 if self.detectBranches:
Simon Hausmanndf450922007-06-08 08:49:22 +02003472 ## FIXME - what's a P4 projectName ?
3473 self.projectName = self.guessProjectName()
3474
Simon Hausmann38f9f5e2007-11-15 10:38:45 +01003475 if self.hasOrigin:
3476 self.getBranchMappingFromGitBranches()
3477 else:
3478 self.getBranchMapping()
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003479 if self.verbose:
3480 print "p4-git branches: %s" % self.p4BranchesInGit
3481 print "initial parents: %s" % self.initialParents
3482 for b in self.p4BranchesInGit:
3483 if b != "master":
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003484
3485 ## FIXME
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003486 b = b[len(self.projectName):]
3487 self.createdBranches.add(b)
Simon Hausmann4b97ffb2007-05-18 21:45:23 +02003488
Simon Hausmannf291b4e2007-04-14 11:21:50 +02003489 self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
Simon Hausmannb9847332007-03-20 20:54:23 +01003490
Pete Wyckoff78189be2012-11-23 17:35:36 -05003491 self.importProcess = subprocess.Popen(["git", "fast-import"],
3492 stdin=subprocess.PIPE,
3493 stdout=subprocess.PIPE,
3494 stderr=subprocess.PIPE);
3495 self.gitOutput = self.importProcess.stdout
3496 self.gitStream = self.importProcess.stdin
3497 self.gitError = self.importProcess.stderr
Simon Hausmannb9847332007-03-20 20:54:23 +01003498
Simon Hausmann1c49fc12007-08-26 16:04:34 +02003499 if revision:
Simon Hausmannc208a242007-08-26 16:07:18 +02003500 self.importHeadRevision(revision)
Simon Hausmannb9847332007-03-20 20:54:23 +01003501 else:
3502 changes = []
3503
Simon Hausmann0828ab12007-03-20 20:59:30 +01003504 if len(self.changesFile) > 0:
Simon Hausmannb9847332007-03-20 20:54:23 +01003505 output = open(self.changesFile).readlines()
Reilly Grant1d7367d2009-09-10 00:02:38 -07003506 changeSet = set()
Simon Hausmannb9847332007-03-20 20:54:23 +01003507 for line in output:
3508 changeSet.add(int(line))
3509
3510 for change in changeSet:
3511 changes.append(change)
3512
3513 changes.sort()
3514 else:
Pete Wyckoff9dcb9f22012-04-08 20:18:01 -04003515 # catch "git p4 sync" with no new branches, in a repo that
3516 # does not have any existing p4 branches
Pete Wyckoff5a8e84c2013-01-14 19:47:05 -05003517 if len(args) == 0:
3518 if not self.p4BranchesInGit:
3519 die("No remote p4 branches. Perhaps you never did \"git p4 clone\" in here.")
3520
3521 # The default branch is master, unless --branch is used to
3522 # specify something else. Make sure it exists, or complain
3523 # nicely about how to use --branch.
3524 if not self.detectBranches:
3525 if not branch_exists(self.branch):
3526 if branch_arg_given:
3527 die("Error: branch %s does not exist." % self.branch)
3528 else:
3529 die("Error: no branch %s; perhaps specify one with --branch." %
3530 self.branch)
3531
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003532 if self.verbose:
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -03003533 print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003534 self.changeRange)
Lex Spoon96b2d542015-04-20 11:00:20 -04003535 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
Simon Hausmannb9847332007-03-20 20:54:23 +01003536
Simon Hausmann01a9c9c2007-05-23 00:07:35 +02003537 if len(self.maxChanges) > 0:
Han-Wen Nienhuys7fcff9d2007-07-23 15:56:37 -07003538 changes = changes[:min(int(self.maxChanges), len(changes))]
Simon Hausmann01a9c9c2007-05-23 00:07:35 +02003539
Simon Hausmannb9847332007-03-20 20:54:23 +01003540 if len(changes) == 0:
Simon Hausmann0828ab12007-03-20 20:59:30 +01003541 if not self.silent:
Simon Hausmann341dc1c2007-05-21 00:39:16 +02003542 print "No changes to import!"
Luke Diamand06804c72012-04-11 17:21:24 +02003543 else:
3544 if not self.silent and not self.detectBranches:
3545 print "Import destination: %s" % self.branch
Simon Hausmannb9847332007-03-20 20:54:23 +01003546
Luke Diamand06804c72012-04-11 17:21:24 +02003547 self.updatedBranches = set()
Simon Hausmanna9d1a272007-06-11 23:28:03 +02003548
Pete Wyckoff47497842013-01-14 19:47:04 -05003549 if not self.detectBranches:
3550 if args:
3551 # start a new branch
3552 self.initialParent = ""
3553 else:
3554 # build on a previous revision
3555 self.initialParent = parseRevision(self.branch)
3556
Luke Diamand06804c72012-04-11 17:21:24 +02003557 self.importChanges(changes)
Simon Hausmann341dc1c2007-05-21 00:39:16 +02003558
Luke Diamand06804c72012-04-11 17:21:24 +02003559 if not self.silent:
3560 print ""
3561 if len(self.updatedBranches) > 0:
3562 sys.stdout.write("Updated branches: ")
3563 for b in self.updatedBranches:
3564 sys.stdout.write("%s " % b)
3565 sys.stdout.write("\n")
Simon Hausmannb9847332007-03-20 20:54:23 +01003566
Pete Wyckoff0d609032013-01-26 22:11:24 -05003567 if gitConfigBool("git-p4.importLabels"):
Luke Diamand06dcd152012-05-11 07:25:18 +01003568 self.importLabels = True
Luke Diamand06804c72012-04-11 17:21:24 +02003569
3570 if self.importLabels:
3571 p4Labels = getP4Labels(self.depotPaths)
3572 gitTags = getGitTags()
3573
3574 missingP4Labels = p4Labels - gitTags
3575 self.importP4Labels(self.gitStream, missingP4Labels)
Simon Hausmannb9847332007-03-20 20:54:23 +01003576
Simon Hausmannb9847332007-03-20 20:54:23 +01003577 self.gitStream.close()
Pete Wyckoff78189be2012-11-23 17:35:36 -05003578 if self.importProcess.wait() != 0:
Simon Hausmann29bdbac2007-05-19 10:23:12 +02003579 die("fast-import failed: %s" % self.gitError.read())
Simon Hausmannb9847332007-03-20 20:54:23 +01003580 self.gitOutput.close()
3581 self.gitError.close()
3582
Vitor Antunesfed23692012-01-25 23:48:22 +00003583 # Cleanup temporary branches created during import
3584 if self.tempBranches != []:
3585 for branch in self.tempBranches:
3586 read_pipe("git update-ref -d %s" % branch)
3587 os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3588
Pete Wyckoff55d12432013-01-14 19:46:59 -05003589 # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3590 # a convenient shortcut refname "p4".
3591 if self.importIntoRemotes:
3592 head_ref = self.refPrefix + "HEAD"
3593 if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3594 system(["git", "symbolic-ref", head_ref, self.branch])
3595
Simon Hausmannb9847332007-03-20 20:54:23 +01003596 return True
3597
Simon Hausmann01ce1fe2007-04-07 23:46:50 +02003598class P4Rebase(Command):
3599 def __init__(self):
3600 Command.__init__(self)
Luke Diamand06804c72012-04-11 17:21:24 +02003601 self.options = [
3602 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
Luke Diamand06804c72012-04-11 17:21:24 +02003603 ]
Luke Diamand06804c72012-04-11 17:21:24 +02003604 self.importLabels = False
Han-Wen Nienhuyscebdf5a2007-05-23 16:53:11 -03003605 self.description = ("Fetches the latest revision from perforce and "
3606 + "rebases the current work (branch) against it")
Simon Hausmann01ce1fe2007-04-07 23:46:50 +02003607
3608 def run(self, args):
3609 sync = P4Sync()
Luke Diamand06804c72012-04-11 17:21:24 +02003610 sync.importLabels = self.importLabels
Simon Hausmann01ce1fe2007-04-07 23:46:50 +02003611 sync.run([])
Simon Hausmannd7e38682007-06-12 14:34:46 +02003612
Simon Hausmann14594f42007-08-22 09:07:15 +02003613 return self.rebase()
3614
3615 def rebase(self):
Simon Hausmann36ee4ee2008-01-07 14:21:45 +01003616 if os.system("git update-index --refresh") != 0:
3617 die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3618 if len(read_pipe("git diff-index HEAD --")) > 0:
Veres Lajosf7e604e2013-06-19 07:37:24 +02003619 die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
Simon Hausmann36ee4ee2008-01-07 14:21:45 +01003620
Simon Hausmannd7e38682007-06-12 14:34:46 +02003621 [upstream, settings] = findUpstreamBranchPoint()
3622 if len(upstream) == 0:
3623 die("Cannot find upstream branchpoint for rebase")
3624
3625 # the branchpoint may be p4/foo~3, so strip off the parent
3626 upstream = re.sub("~[0-9]+$", "", upstream)
3627
3628 print "Rebasing the current branch onto %s" % upstream
Han-Wen Nienhuysb25b2062007-05-23 18:49:35 -03003629 oldHead = read_pipe("git rev-parse HEAD").strip()
Simon Hausmannd7e38682007-06-12 14:34:46 +02003630 system("git rebase %s" % upstream)
Vlad Dogaru4e49d952014-04-07 16:19:11 +03003631 system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
Simon Hausmann01ce1fe2007-04-07 23:46:50 +02003632 return True
3633
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003634class P4Clone(P4Sync):
3635 def __init__(self):
3636 P4Sync.__init__(self)
3637 self.description = "Creates a new git repository and imports from Perforce into it"
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003638 self.usage = "usage: %prog [options] //depot/path[@revRange]"
Tommy Thorn354081d2008-02-03 10:38:51 -08003639 self.options += [
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003640 optparse.make_option("--destination", dest="cloneDestination",
3641 action='store', default=None,
Tommy Thorn354081d2008-02-03 10:38:51 -08003642 help="where to leave result of the clone"),
Pete Wyckoff38200072011-02-19 08:18:01 -05003643 optparse.make_option("--bare", dest="cloneBare",
3644 action="store_true", default=False),
Tommy Thorn354081d2008-02-03 10:38:51 -08003645 ]
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003646 self.cloneDestination = None
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003647 self.needsGit = False
Pete Wyckoff38200072011-02-19 08:18:01 -05003648 self.cloneBare = False
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003649
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -03003650 def defaultDestination(self, args):
3651 ## TODO: use common prefix of args?
3652 depotPath = args[0]
3653 depotDir = re.sub("(@[^@]*)$", "", depotPath)
3654 depotDir = re.sub("(#[^#]*)$", "", depotDir)
Toby Allsopp053d9e42008-02-05 09:41:43 +13003655 depotDir = re.sub(r"\.\.\.$", "", depotDir)
Han-Wen Nienhuys6a49f8e2007-05-23 18:49:35 -03003656 depotDir = re.sub(r"/$", "", depotDir)
3657 return os.path.split(depotDir)[1]
3658
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003659 def run(self, args):
3660 if len(args) < 1:
3661 return False
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003662
3663 if self.keepRepoPath and not self.cloneDestination:
3664 sys.stderr.write("Must specify destination for --keep-path\n")
3665 sys.exit(1)
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003666
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003667 depotPaths = args
Simon Hausmann5e100b52007-06-07 21:12:25 +02003668
3669 if not self.cloneDestination and len(depotPaths) > 1:
3670 self.cloneDestination = depotPaths[-1]
3671 depotPaths = depotPaths[:-1]
3672
Tommy Thorn354081d2008-02-03 10:38:51 -08003673 self.cloneExclude = ["/"+p for p in self.cloneExclude]
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003674 for p in depotPaths:
3675 if not p.startswith("//"):
Pete Wyckoff0f487d32013-01-26 22:11:06 -05003676 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003677 return False
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003678
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003679 if not self.cloneDestination:
Marius Storm-Olsen98ad4fa2007-06-07 15:08:33 +02003680 self.cloneDestination = self.defaultDestination(args)
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003681
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -03003682 print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
Pete Wyckoff38200072011-02-19 08:18:01 -05003683
Kevin Greenc3bf3f12007-06-11 16:48:07 -04003684 if not os.path.exists(self.cloneDestination):
3685 os.makedirs(self.cloneDestination)
Robert Blum053fd0c2008-08-01 12:50:03 -07003686 chdir(self.cloneDestination)
Pete Wyckoff38200072011-02-19 08:18:01 -05003687
3688 init_cmd = [ "git", "init" ]
3689 if self.cloneBare:
3690 init_cmd.append("--bare")
Brandon Caseya235e852013-01-26 11:14:33 -08003691 retcode = subprocess.call(init_cmd)
3692 if retcode:
3693 raise CalledProcessError(retcode, init_cmd)
Pete Wyckoff38200072011-02-19 08:18:01 -05003694
Han-Wen Nienhuys6326aa52007-05-23 18:49:35 -03003695 if not P4Sync.run(self, depotPaths):
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003696 return False
Pete Wyckoffc5959562013-01-14 19:47:01 -05003697
3698 # create a master branch and check out a work tree
3699 if gitBranchExists(self.branch):
3700 system([ "git", "branch", "master", self.branch ])
3701 if not self.cloneBare:
3702 system([ "git", "checkout", "-f" ])
3703 else:
3704 print 'Not checking out any branch, use ' \
3705 '"git checkout -q -b master <branch>"'
Han-Wen Nienhuys86dff6b2007-05-23 18:49:35 -03003706
Pete Wyckoffa93d33e2012-02-25 20:06:24 -05003707 # auto-set this variable if invoked with --use-client-spec
3708 if self.useClientSpec_from_options:
3709 system("git config --bool git-p4.useclientspec true")
3710
Simon Hausmannf9a3a4f2007-04-08 10:08:26 +02003711 return True
3712
Simon Hausmann09d89de2007-06-20 23:10:28 +02003713class P4Branches(Command):
3714 def __init__(self):
3715 Command.__init__(self)
3716 self.options = [ ]
3717 self.description = ("Shows the git branches that hold imports and their "
3718 + "corresponding perforce depot paths")
3719 self.verbose = False
3720
3721 def run(self, args):
Simon Hausmann5ca44612007-08-24 17:44:16 +02003722 if originP4BranchesExist():
3723 createOrUpdateBranchesFromOrigin()
3724
Simon Hausmann09d89de2007-06-20 23:10:28 +02003725 cmdline = "git rev-parse --symbolic "
3726 cmdline += " --remotes"
3727
3728 for line in read_pipe_lines(cmdline):
3729 line = line.strip()
3730
3731 if not line.startswith('p4/') or line == "p4/HEAD":
3732 continue
3733 branch = line
3734
3735 log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3736 settings = extractSettingsGitLog(log)
3737
3738 print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3739 return True
3740
Simon Hausmannb9847332007-03-20 20:54:23 +01003741class HelpFormatter(optparse.IndentedHelpFormatter):
3742 def __init__(self):
3743 optparse.IndentedHelpFormatter.__init__(self)
3744
3745 def format_description(self, description):
3746 if description:
3747 return description + "\n"
3748 else:
3749 return ""
Simon Hausmann4f5cf762007-03-19 22:25:17 +01003750
Simon Hausmann86949ee2007-03-19 20:59:12 +01003751def printUsage(commands):
3752 print "usage: %s <command> [options]" % sys.argv[0]
3753 print ""
3754 print "valid commands: %s" % ", ".join(commands)
3755 print ""
3756 print "Try %s <command> --help for command specific help." % sys.argv[0]
3757 print ""
3758
3759commands = {
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003760 "debug" : P4Debug,
3761 "submit" : P4Submit,
Marius Storm-Olsena9834f52007-10-09 16:16:09 +02003762 "commit" : P4Submit,
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003763 "sync" : P4Sync,
3764 "rebase" : P4Rebase,
3765 "clone" : P4Clone,
Simon Hausmann09d89de2007-06-20 23:10:28 +02003766 "rollback" : P4RollBack,
3767 "branches" : P4Branches
Simon Hausmann86949ee2007-03-19 20:59:12 +01003768}
3769
Simon Hausmann86949ee2007-03-19 20:59:12 +01003770
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003771def main():
3772 if len(sys.argv[1:]) == 0:
3773 printUsage(commands.keys())
3774 sys.exit(2)
Simon Hausmann86949ee2007-03-19 20:59:12 +01003775
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003776 cmdName = sys.argv[1]
3777 try:
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003778 klass = commands[cmdName]
3779 cmd = klass()
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003780 except KeyError:
3781 print "unknown command %s" % cmdName
3782 print ""
3783 printUsage(commands.keys())
3784 sys.exit(2)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01003785
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003786 options = cmd.options
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003787 cmd.gitdir = os.environ.get("GIT_DIR", None)
Simon Hausmann86949ee2007-03-19 20:59:12 +01003788
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003789 args = sys.argv[2:]
Simon Hausmanne20a9e52007-03-26 00:13:51 +02003790
Pete Wyckoffb0ccc802012-09-09 16:16:10 -04003791 options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
Luke Diamand6a10b6a2012-04-24 09:33:23 +01003792 if cmd.needsGit:
3793 options.append(optparse.make_option("--git-dir", dest="gitdir"))
Simon Hausmanne20a9e52007-03-26 00:13:51 +02003794
Luke Diamand6a10b6a2012-04-24 09:33:23 +01003795 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3796 options,
3797 description = cmd.description,
3798 formatter = HelpFormatter())
Simon Hausmann86949ee2007-03-19 20:59:12 +01003799
Luke Diamand6a10b6a2012-04-24 09:33:23 +01003800 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003801 global verbose
3802 verbose = cmd.verbose
3803 if cmd.needsGit:
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003804 if cmd.gitdir == None:
3805 cmd.gitdir = os.path.abspath(".git")
3806 if not isValidGitDir(cmd.gitdir):
Luke Diamand378f7be2016-12-13 21:51:28 +00003807 # "rev-parse --git-dir" without arguments will try $PWD/.git
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003808 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3809 if os.path.exists(cmd.gitdir):
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003810 cdup = read_pipe("git rev-parse --show-cdup").strip()
3811 if len(cdup) > 0:
Robert Blum053fd0c2008-08-01 12:50:03 -07003812 chdir(cdup);
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003813
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003814 if not isValidGitDir(cmd.gitdir):
3815 if isValidGitDir(cmd.gitdir + "/.git"):
3816 cmd.gitdir += "/.git"
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003817 else:
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003818 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
Simon Hausmann8910ac02007-03-26 08:18:55 +02003819
Luke Diamand378f7be2016-12-13 21:51:28 +00003820 # so git commands invoked from the P4 workspace will succeed
Han-Wen Nienhuysb86f7372007-05-23 18:49:35 -03003821 os.environ["GIT_DIR"] = cmd.gitdir
Simon Hausmann4f5cf762007-03-19 22:25:17 +01003822
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003823 if not cmd.run(args):
3824 parser.print_help()
Pete Wyckoff09fca772011-12-24 21:07:39 -05003825 sys.exit(2)
Simon Hausmann4f5cf762007-03-19 22:25:17 +01003826
Han-Wen Nienhuysbb6e09b2007-05-23 18:49:35 -03003827
3828if __name__ == '__main__':
3829 main()