PatchCheck.py@ 80924

Last change on this file since 80924 was 80721, checked in by vboxsync, 6 years ago
Devices/EFI/FirmwareNew: Start upgrade process to edk2-stable201908 (compiles on Windows and works to some extent), bugref:4643
Property svn:eol-style set to `native` Property svn:executable set to ``*
File size: 20.7 KB

Line
1	## @file
2	# Check a patch for various format issues
3	#
4	# Copyright (c) 2015 - 2019, Intel Corporation. All rights reserved.<BR>
5	#
6	# SPDX-License-Identifier: BSD-2-Clause-Patent
7	#
8
9	from __future__ import print_function
10
11	VersionNumber = '0.1'
12	__copyright__ = "Copyright (c) 2015 - 2016, Intel Corporation All rights reserved."
13
14	import email
15	import argparse
16	import os
17	import re
18	import subprocess
19	import sys
20
21	class Verbose:
22	SILENT, ONELINE, NORMAL = range(3)
23	level = NORMAL
24
25	class CommitMessageCheck:
26	"""Checks the contents of a git commit message."""
27
28	def __init__(self, subject, message):
29	self.ok = True
30
31	if subject is None and message is None:
32	self.error('Commit message is missing!')
33	return
34
35	self.subject = subject
36	self.msg = message
37
38	self.check_contributed_under()
39	self.check_signed_off_by()
40	self.check_misc_signatures()
41	self.check_overall_format()
42	self.report_message_result()
43
44	url = 'https://github.com/tianocore/tianocore.github.io/wiki/Commit-Message-Format'
45
46	def report_message_result(self):
47	if Verbose.level < Verbose.NORMAL:
48	return
49	if self.ok:
50	# All checks passed
51	return_code = 0
52	print('The commit message format passed all checks.')
53	else:
54	return_code = 1
55	if not self.ok:
56	print(self.url)
57
58	def error(self, *err):
59	if self.ok and Verbose.level > Verbose.ONELINE:
60	print('The commit message format is not valid:')
61	self.ok = False
62	if Verbose.level < Verbose.NORMAL:
63	return
64	count = 0
65	for line in err:
66	prefix = (' *', ' ')[count > 0]
67	print(prefix, line)
68	count += 1
69
70	# Find 'contributed-under:' at the start of a line ignoring case and
71	# requires ':' to be present. Matches if there is white space before
72	# the tag or between the tag and the ':'.
73	contributed_under_re = \
74	re.compile(r'^\scontributed-under\s:', re.MULTILINE\|re.IGNORECASE)
75
76	def check_contributed_under(self):
77	match = self.contributed_under_re.search(self.msg)
78	if match is not None:
79	self.error('Contributed-under! (Note: this must be ' +
80	'removed by the code contributor!)')
81
82	@staticmethod
83	def make_signature_re(sig, re_input=False):
84	if re_input:
85	sub_re = sig
86	else:
87	sub_re = sig.replace('-', r'[-\s]+')
88	re_str = (r'^(?P<tag>' + sub_re +
89	r')(\s):(\s)(?P<value>\S.?)(?:\s)$')
90	try:
91	return re.compile(re_str, re.MULTILINE\|re.IGNORECASE)
92	except Exception:
93	print("Tried to compile re:", re_str)
94	raise
95
96	sig_block_re = \
97	re.compile(r'''^
98	(?: (?P<tag>[^:]+) \s* : \s*
99	(?P<value>\S.*?) )
100	\|
101	(?: \[ (?P<updater>[^:]+) \s* : \s*
102	(?P<note>.+?) \s* \] )
103	\s* $''',
104	re.VERBOSE \| re.MULTILINE)
105
106	def find_signatures(self, sig):
107	if not sig.endswith('-by') and sig != 'Cc':
108	sig += '-by'
109	regex = self.make_signature_re(sig)
110
111	sigs = regex.findall(self.msg)
112
113	bad_case_sigs = filter(lambda m: m[0] != sig, sigs)
114	for s in bad_case_sigs:
115	self.error("'" +s[0] + "' should be '" + sig + "'")
116
117	for s in sigs:
118	if s[1] != '':
119	self.error('There should be no spaces between ' + sig +
120	" and the ':'")
121	if s[2] != ' ':
122	self.error("There should be a space after '" + sig + ":'")
123
124	self.check_email_address(s[3])
125
126	return sigs
127
128	email_re1 = re.compile(r'(?:\s)(.?)(\s)<(.+)>\s$',
129	re.MULTILINE\|re.IGNORECASE)
130
131	def check_email_address(self, email):
132	email = email.strip()
133	mo = self.email_re1.match(email)
134	if mo is None:
135	self.error("Email format is invalid: " + email.strip())
136	return
137
138	name = mo.group(1).strip()
139	if name == '':
140	self.error("Name is not provided with email address: " +
141	email)
142	else:
143	quoted = len(name) > 2 and name[0] == '"' and name[-1] == '"'
144	if name.find(',') >= 0 and not quoted:
145	self.error('Add quotes (") around name with a comma: ' +
146	name)
147
148	if mo.group(2) == '':
149	self.error("There should be a space between the name and " +
150	"email address: " + email)
151
152	if mo.group(3).find(' ') >= 0:
153	self.error("The email address cannot contain a space: " +
154	mo.group(3))
155
156	def check_signed_off_by(self):
157	sob='Signed-off-by'
158	if self.msg.find(sob) < 0:
159	self.error('Missing Signed-off-by! (Note: this must be ' +
160	'added by the code contributor!)')
161	return
162
163	sobs = self.find_signatures('Signed-off')
164
165	if len(sobs) == 0:
166	self.error('Invalid Signed-off-by format!')
167	return
168
169	sig_types = (
170	'Reviewed',
171	'Reported',
172	'Tested',
173	'Suggested',
174	'Acked',
175	'Cc'
176	)
177
178	def check_misc_signatures(self):
179	for sig in self.sig_types:
180	self.find_signatures(sig)
181
182	def check_overall_format(self):
183	lines = self.msg.splitlines()
184
185	if len(lines) >= 1 and lines[0].endswith('\r\n'):
186	empty_line = '\r\n'
187	else:
188	empty_line = '\n'
189
190	lines.insert(0, empty_line)
191	lines.insert(0, self.subject + empty_line)
192
193	count = len(lines)
194
195	if count <= 0:
196	self.error('Empty commit message!')
197	return
198
199	if count >= 1 and len(lines[0]) >= 72:
200	self.error('First line of commit message (subject line) ' +
201	'is too long.')
202
203	if count >= 1 and len(lines[0].strip()) == 0:
204	self.error('First line of commit message (subject line) ' +
205	'is empty.')
206
207	if count >= 2 and lines[1].strip() != '':
208	self.error('Second line of commit message should be ' +
209	'empty.')
210
211	for i in range(2, count):
212	if (len(lines[i]) >= 76 and
213	len(lines[i].split()) > 1 and
214	not lines[i].startswith('git-svn-id:')):
215	self.error('Line %d of commit message is too long.' % (i + 1))
216
217	last_sig_line = None
218	for i in range(count - 1, 0, -1):
219	line = lines[i]
220	mo = self.sig_block_re.match(line)
221	if mo is None:
222	if line.strip() == '':
223	break
224	elif last_sig_line is not None:
225	err2 = 'Add empty line before "%s"?' % last_sig_line
226	self.error('The line before the signature block ' +
227	'should be empty', err2)
228	else:
229	self.error('The signature block was not found')
230	break
231	last_sig_line = line.strip()
232
233	(START, PRE_PATCH, PATCH) = range(3)
234
235	class GitDiffCheck:
236	"""Checks the contents of a git diff."""
237
238	def __init__(self, diff):
239	self.ok = True
240	self.format_ok = True
241	self.lines = diff.splitlines(True)
242	self.count = len(self.lines)
243	self.line_num = 0
244	self.state = START
245	self.new_bin = []
246	while self.line_num < self.count and self.format_ok:
247	line_num = self.line_num
248	self.run()
249	assert(self.line_num > line_num)
250	self.report_message_result()
251
252	def report_message_result(self):
253	if Verbose.level < Verbose.NORMAL:
254	return
255	if self.ok:
256	print('The code passed all checks.')
257	if self.new_bin:
258	print('\nWARNING - The following binary files will be added ' +
259	'into the repository:')
260	for binary in self.new_bin:
261	print(' ' + binary)
262
263	def run(self):
264	line = self.lines[self.line_num]
265
266	if self.state in (PRE_PATCH, PATCH):
267	if line.startswith('diff --git'):
268	self.state = START
269	if self.state == PATCH:
270	if line.startswith('@@ '):
271	self.state = PRE_PATCH
272	elif len(line) >= 1 and line[0] not in ' -+' and \
273	not line.startswith('\r\n') and \
274	not line.startswith(r'\ No newline ') and not self.binary:
275	for line in self.lines[self.line_num + 1:]:
276	if line.startswith('diff --git'):
277	self.format_error('diff found after end of patch')
278	break
279	self.line_num = self.count
280	return
281
282	if self.state == START:
283	if line.startswith('diff --git'):
284	self.state = PRE_PATCH
285	self.filename = line[13:].split(' ', 1)[0]
286	self.is_newfile = False
287	self.force_crlf = not self.filename.endswith('.sh')
288	elif len(line.rstrip()) != 0:
289	self.format_error("didn't find diff command")
290	self.line_num += 1
291	elif self.state == PRE_PATCH:
292	if line.startswith('@@ '):
293	self.state = PATCH
294	self.binary = False
295	elif line.startswith('GIT binary patch') or \
296	line.startswith('Binary files'):
297	self.state = PATCH
298	self.binary = True
299	if self.is_newfile:
300	self.new_bin.append(self.filename)
301	else:
302	ok = False
303	self.is_newfile = self.newfile_prefix_re.match(line)
304	for pfx in self.pre_patch_prefixes:
305	if line.startswith(pfx):
306	ok = True
307	if not ok:
308	self.format_error("didn't find diff hunk marker (@@)")
309	self.line_num += 1
310	elif self.state == PATCH:
311	if self.binary:
312	pass
313	elif line.startswith('-'):
314	pass
315	elif line.startswith('+'):
316	self.check_added_line(line[1:])
317	elif line.startswith('\r\n'):
318	pass
319	elif line.startswith(r'\ No newline '):
320	pass
321	elif not line.startswith(' '):
322	self.format_error("unexpected patch line")
323	self.line_num += 1
324
325	pre_patch_prefixes = (
326	'--- ',
327	'+++ ',
328	'index ',
329	'new file ',
330	'deleted file ',
331	'old mode ',
332	'new mode ',
333	'similarity index ',
334	'copy from ',
335	'copy to ',
336	'rename ',
337	)
338
339	line_endings = ('\r\n', '\n\r', '\n', '\r')
340
341	newfile_prefix_re = \
342	re.compile(r'''^
343	index\ 0+\.\.
344	''',
345	re.VERBOSE)
346
347	def added_line_error(self, msg, line):
348	lines = [ msg ]
349	if self.filename is not None:
350	lines.append('File: ' + self.filename)
351	lines.append('Line: ' + line)
352
353	self.error(*lines)
354
355	old_debug_re = \
356	re.compile(r'''
357	DEBUG \s* \( \s* \( \s*
358	(?: DEBUG_[A-Z_]+ \s* \\| \s)
359	EFI_D_ ([A-Z_]+)
360	''',
361	re.VERBOSE)
362
363	def check_added_line(self, line):
364	eol = ''
365	for an_eol in self.line_endings:
366	if line.endswith(an_eol):
367	eol = an_eol
368	line = line[:-len(eol)]
369
370	stripped = line.rstrip()
371
372	if self.force_crlf and eol != '\r\n':
373	self.added_line_error('Line ending (%s) is not CRLF' % repr(eol),
374	line)
375	if '\t' in line:
376	self.added_line_error('Tab character used', line)
377	if len(stripped) < len(line):
378	self.added_line_error('Trailing whitespace found', line)
379
380	mo = self.old_debug_re.search(line)
381	if mo is not None:
382	self.added_line_error('EFI_D_' + mo.group(1) + ' was used, '
383	'but DEBUG_' + mo.group(1) +
384	' is now recommended', line)
385
386	split_diff_re = re.compile(r'''
387	(?P<cmd>
388	^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
389	)
390	(?P<index>
391	^ index \s+ .+ $
392	)
393	''',
394	re.IGNORECASE \| re.VERBOSE \| re.MULTILINE)
395
396	def format_error(self, err):
397	self.format_ok = False
398	err = 'Patch format error: ' + err
399	err2 = 'Line: ' + self.lines[self.line_num].rstrip()
400	self.error(err, err2)
401
402	def error(self, *err):
403	if self.ok and Verbose.level > Verbose.ONELINE:
404	print('Code format is not valid:')
405	self.ok = False
406	if Verbose.level < Verbose.NORMAL:
407	return
408	count = 0
409	for line in err:
410	prefix = (' *', ' ')[count > 0]
411	print(prefix, line)
412	count += 1
413
414	class CheckOnePatch:
415	"""Checks the contents of a git email formatted patch.
416
417	Various checks are performed on both the commit message and the
418	patch content.
419	"""
420
421	def __init__(self, name, patch):
422	self.patch = patch
423	self.find_patch_pieces()
424
425	msg_check = CommitMessageCheck(self.commit_subject, self.commit_msg)
426	msg_ok = msg_check.ok
427
428	diff_ok = True
429	if self.diff is not None:
430	diff_check = GitDiffCheck(self.diff)
431	diff_ok = diff_check.ok
432
433	self.ok = msg_ok and diff_ok
434
435	if Verbose.level == Verbose.ONELINE:
436	if self.ok:
437	result = 'ok'
438	else:
439	result = list()
440	if not msg_ok:
441	result.append('commit message')
442	if not diff_ok:
443	result.append('diff content')
444	result = 'bad ' + ' and '.join(result)
445	print(name, result)
446
447
448	git_diff_re = re.compile(r'''
449	^ diff \s+ --git \s+ a/.+ \s+ b/.+ $
450	''',
451	re.IGNORECASE \| re.VERBOSE \| re.MULTILINE)
452
453	stat_re = \
454	re.compile(r'''
455	(?P<commit_message> [\s\S\r\n]* )
456	(?P<stat>
457	^ --- $ [\r\n]+
458	(?: ^ \s+ .+ \s+ \\| \s+ \d+ \s+ \+* \-*
459	$ [\r\n]+ )+
460	[\s\S\r\n]+
461	)
462	''',
463	re.IGNORECASE \| re.VERBOSE \| re.MULTILINE)
464
465	subject_prefix_re = \
466	re.compile(r'''^
467	\s* (\[
468	[^\[\]]* # Allow all non-brackets
469	\])* \s*
470	''',
471	re.VERBOSE)
472
473	def find_patch_pieces(self):
474	if sys.version_info < (3, 0):
475	patch = self.patch.encode('ascii', 'ignore')
476	else:
477	patch = self.patch
478
479	self.commit_msg = None
480	self.stat = None
481	self.commit_subject = None
482	self.commit_prefix = None
483	self.diff = None
484
485	if patch.startswith('diff --git'):
486	self.diff = patch
487	return
488
489	pmail = email.message_from_string(patch)
490	parts = list(pmail.walk())
491	assert(len(parts) == 1)
492	assert(parts[0].get_content_type() == 'text/plain')
493	content = parts[0].get_payload(decode=True).decode('utf-8', 'ignore')
494
495	mo = self.git_diff_re.search(content)
496	if mo is not None:
497	self.diff = content[mo.start():]
498	content = content[:mo.start()]
499
500	mo = self.stat_re.search(content)
501	if mo is None:
502	self.commit_msg = content
503	else:
504	self.stat = mo.group('stat')
505	self.commit_msg = mo.group('commit_message')
506
507	self.commit_subject = pmail['subject'].replace('\r\n', '')
508	self.commit_subject = self.commit_subject.replace('\n', '')
509	self.commit_subject = self.subject_prefix_re.sub('', self.commit_subject, 1)
510
511	class CheckGitCommits:
512	"""Reads patches from git based on the specified git revision range.
513
514	The patches are read from git, and then checked.
515	"""
516
517	def __init__(self, rev_spec, max_count):
518	commits = self.read_commit_list_from_git(rev_spec, max_count)
519	if len(commits) == 1 and Verbose.level > Verbose.ONELINE:
520	commits = [ rev_spec ]
521	self.ok = True
522	blank_line = False
523	for commit in commits:
524	if Verbose.level > Verbose.ONELINE:
525	if blank_line:
526	print()
527	else:
528	blank_line = True
529	print('Checking git commit:', commit)
530	patch = self.read_patch_from_git(commit)
531	self.ok &= CheckOnePatch(commit, patch).ok
532	if not commits:
533	print("Couldn't find commit matching: '{}'".format(rev_spec))
534
535	def read_commit_list_from_git(self, rev_spec, max_count):
536	# Run git to get the commit patch
537	cmd = [ 'rev-list', '--abbrev-commit', '--no-walk' ]
538	if max_count is not None:
539	cmd.append('--max-count=' + str(max_count))
540	cmd.append(rev_spec)
541	out = self.run_git(*cmd)
542	return out.split() if out else []
543
544	def read_patch_from_git(self, commit):
545	# Run git to get the commit patch
546	return self.run_git('show', '--pretty=email', '--no-textconv', commit)
547
548	def run_git(self, *args):
549	cmd = [ 'git' ]
550	cmd += args
551	p = subprocess.Popen(cmd,
552	stdout=subprocess.PIPE,
553	stderr=subprocess.STDOUT)
554	Result = p.communicate()
555	return Result[0].decode('utf-8', 'ignore') if Result[0] and Result[0].find(b"fatal")!=0 else None
556
557	class CheckOnePatchFile:
558	"""Performs a patch check for a single file.
559
560	stdin is used when the filename is '-'.
561	"""
562
563	def __init__(self, patch_filename):
564	if patch_filename == '-':
565	patch = sys.stdin.read()
566	patch_filename = 'stdin'
567	else:
568	f = open(patch_filename, 'rb')
569	patch = f.read().decode('utf-8', 'ignore')
570	f.close()
571	if Verbose.level > Verbose.ONELINE:
572	print('Checking patch file:', patch_filename)
573	self.ok = CheckOnePatch(patch_filename, patch).ok
574
575	class CheckOneArg:
576	"""Performs a patch check for a single command line argument.
577
578	The argument will be handed off to a file or git-commit based
579	checker.
580	"""
581
582	def __init__(self, param, max_count=None):
583	self.ok = True
584	if param == '-' or os.path.exists(param):
585	checker = CheckOnePatchFile(param)
586	else:
587	checker = CheckGitCommits(param, max_count)
588	self.ok = checker.ok
589
590	class PatchCheckApp:
591	"""Checks patches based on the command line arguments."""
592
593	def __init__(self):
594	self.parse_options()
595	patches = self.args.patches
596
597	if len(patches) == 0:
598	patches = [ 'HEAD' ]
599
600	self.ok = True
601	self.count = None
602	for patch in patches:
603	self.process_one_arg(patch)
604
605	if self.count is not None:
606	self.process_one_arg('HEAD')
607
608	if self.ok:
609	self.retval = 0
610	else:
611	self.retval = -1
612
613	def process_one_arg(self, arg):
614	if len(arg) >= 2 and arg[0] == '-':
615	try:
616	self.count = int(arg[1:])
617	return
618	except ValueError:
619	pass
620	self.ok &= CheckOneArg(arg, self.count).ok
621	self.count = None
622
623	def parse_options(self):
624	parser = argparse.ArgumentParser(description=__copyright__)
625	parser.add_argument('--version', action='version',
626	version='%(prog)s ' + VersionNumber)
627	parser.add_argument('patches', nargs='*',
628	help='[patch file \| git rev list]')
629	group = parser.add_mutually_exclusive_group()
630	group.add_argument("--oneline",
631	action="store_true",
632	help="Print one result per line")
633	group.add_argument("--silent",
634	action="store_true",
635	help="Print nothing")
636	self.args = parser.parse_args()
637	if self.args.oneline:
638	Verbose.level = Verbose.ONELINE
639	if self.args.silent:
640	Verbose.level = Verbose.SILENT
641
642	if __name__ == "__main__":
643	sys.exit(PatchCheckApp().retval)

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Devices/EFI/Firmware/BaseTools/Scripts/PatchCheck.py@ 80924

Download in other formats: