utils.py@ 86208

Last change on this file since 86208 was 84568, checked in by vboxsync, 5 years ago
common/utils.py: add OS type for macOS 10.15 and new placeholders to avoid running into testboxscript start failures on future macOS releases
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 83.4 KB

Line
1	# -- coding: utf-8 --
2	# $Id: utils.py 84568 2020-05-27 15:06:29Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Common Utility Functions.
7	"""
8
9	from __future__ import print_function;
10
11	__copyright__ = \
12	"""
13	Copyright (C) 2012-2020 Oracle Corporation
14
15	This file is part of VirtualBox Open Source Edition (OSE), as
16	available from http://www.virtualbox.org. This file is free software;
17	you can redistribute it and/or modify it under the terms of the GNU
18	General Public License (GPL) as published by the Free Software
19	Foundation, in version 2 as it comes in the "COPYING" file of the
20	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23	The contents of this file may alternatively be used under the terms
24	of the Common Development and Distribution License Version 1.0
25	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26	VirtualBox OSE distribution, in which case the provisions of the
27	CDDL are applicable instead of those of the GPL.
28
29	You may elect to license modified versions of this file under the
30	terms and conditions of either the GPL or the CDDL or both.
31	"""
32	__version__ = "$Revision: 84568 $"
33
34
35	# Standard Python imports.
36	import datetime;
37	import os;
38	import platform;
39	import re;
40	import stat;
41	import subprocess;
42	import sys;
43	import time;
44	import traceback;
45	import unittest;
46
47	if sys.platform == 'win32':
48	import ctypes;
49	import msvcrt; # pylint: disable=import-error
50	import win32api; # pylint: disable=import-error
51	import win32con; # pylint: disable=import-error
52	import win32console; # pylint: disable=import-error
53	import win32file; # pylint: disable=import-error
54	import win32process; # pylint: disable=import-error
55	import winerror; # pylint: disable=import-error
56	import pywintypes; # pylint: disable=import-error
57	else:
58	import signal;
59
60	# Python 3 hacks:
61	if sys.version_info[0] >= 3:
62	unicode = str; # pylint: disable=redefined-builtin,invalid-name
63	xrange = range; # pylint: disable=redefined-builtin,invalid-name
64	long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67	#
68	# Strings.
69	#
70
71	def toUnicode(sString, encoding = None, errors = 'strict'):
72	"""
73	A little like the python 2 unicode() function.
74	"""
75	if sys.version_info[0] >= 3:
76	if isinstance(sString, bytes):
77	return str(sString, encoding if encoding else 'utf-8', errors);
78	else:
79	if not isinstance(sString, unicode):
80	return unicode(sString, encoding if encoding else 'utf-8', errors);
81	return sString;
82
83
84
85	#
86	# Output.
87	#
88
89	def printOut(sString):
90	"""
91	Outputs a string to standard output, dealing with python 2.x encoding stupidity.
92	"""
93	sStreamEncoding = sys.stdout.encoding;
94	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
95	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
96	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
97	print(sString);
98	else:
99	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
100
101	def printErr(sString):
102	"""
103	Outputs a string to standard error, dealing with python 2.x encoding stupidity.
104	"""
105	sStreamEncoding = sys.stderr.encoding;
106	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
107	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
108	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
109	print(sString, file = sys.stderr);
110	else:
111	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
112
113
114	#
115	# Host OS and CPU.
116	#
117
118	def getHostOs():
119	"""
120	Gets the host OS name (short).
121
122	See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
123	"""
124	sPlatform = platform.system();
125	if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
126	sPlatform = sPlatform.lower();
127	elif sPlatform == 'Windows':
128	sPlatform = 'win';
129	elif sPlatform == 'SunOS':
130	sPlatform = 'solaris';
131	else:
132	raise Exception('Unsupported platform "%s"' % (sPlatform,));
133	return sPlatform;
134
135	g_sHostArch = None;
136
137	def getHostArch():
138	"""
139	Gets the host CPU architecture.
140
141	See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
142	"""
143	global g_sHostArch;
144	if g_sHostArch is None:
145	sArch = platform.machine();
146	if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
147	sArch = 'x86';
148	elif sArch in ('AMD64', 'amd64', 'x86_64'):
149	sArch = 'amd64';
150	elif sArch == 'i86pc': # SunOS
151	if platform.architecture()[0] == '64bit':
152	sArch = 'amd64';
153	else:
154	try:
155	sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
156	except:
157	pass;
158	sArch = sArch.strip();
159	if sArch != 'amd64':
160	sArch = 'x86';
161	else:
162	raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
163	g_sHostArch = sArch;
164	return g_sHostArch;
165
166
167	def getHostOsDotArch():
168	"""
169	Gets the 'os.arch' for the host.
170	"""
171	return '%s.%s' % (getHostOs(), getHostArch());
172
173
174	def isValidOs(sOs):
175	"""
176	Validates the OS name.
177	"""
178	if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
179	'os2', 'solaris', 'win', 'os-agnostic'):
180	return True;
181	return False;
182
183
184	def isValidArch(sArch):
185	"""
186	Validates the CPU architecture name.
187	"""
188	if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
189	'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
190	return True;
191	return False;
192
193	def isValidOsDotArch(sOsDotArch):
194	"""
195	Validates the 'os.arch' string.
196	"""
197
198	asParts = sOsDotArch.split('.');
199	if asParts.length() != 2:
200	return False;
201	return isValidOs(asParts[0]) \
202	and isValidArch(asParts[1]);
203
204	def getHostOsVersion():
205	"""
206	Returns the host OS version. This is platform.release with additional
207	distro indicator on linux.
208	"""
209	sVersion = platform.release();
210	sOs = getHostOs();
211	if sOs == 'linux':
212	sDist = '';
213	try:
214	# try /etc/lsb-release first to distinguish between Debian and Ubuntu
215	oFile = open('/etc/lsb-release');
216	for sLine in oFile:
217	oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s=)\s"(.)"', sLine);
218	if oMatch is not None:
219	sDist = oMatch.group(1).strip();
220	except:
221	pass;
222	if sDist:
223	sVersion += ' / ' + sDist;
224	else:
225	asFiles = \
226	[
227	[ '/etc/debian_version', 'Debian v'],
228	[ '/etc/gentoo-release', '' ],
229	[ '/etc/oracle-release', '' ],
230	[ '/etc/redhat-release', '' ],
231	[ '/etc/SuSE-release', '' ],
232	];
233	for sFile, sPrefix in asFiles:
234	if os.path.isfile(sFile):
235	try:
236	oFile = open(sFile);
237	sLine = oFile.readline();
238	oFile.close();
239	except:
240	continue;
241	sLine = sLine.strip()
242	if sLine:
243	sVersion += ' / ' + sPrefix + sLine;
244	break;
245
246	elif sOs == 'solaris':
247	sVersion = platform.version();
248	if os.path.isfile('/etc/release'):
249	try:
250	oFile = open('/etc/release');
251	sLast = oFile.readlines()[-1];
252	oFile.close();
253	sLast = sLast.strip();
254	if sLast:
255	sVersion += ' (' + sLast + ')';
256	except:
257	pass;
258
259	elif sOs == 'darwin':
260	sOsxVersion = platform.mac_ver()[0];
261	codenames = {"4": "Tiger",
262	"5": "Leopard",
263	"6": "Snow Leopard",
264	"7": "Lion",
265	"8": "Mountain Lion",
266	"9": "Mavericks",
267	"10": "Yosemite",
268	"11": "El Capitan",
269	"12": "Sierra",
270	"13": "High Sierra",
271	"14": "Mojave",
272	"15": "Catalina",
273	"16": "Unknown 16",
274	"17": "Unknown 17",
275	"18": "Unknown 18",
276	"19": "Unknown 19", }
277	sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
278
279	elif sOs == 'win':
280	class OSVersionInfoEx(ctypes.Structure):
281	""" OSVERSIONEX """
282	kaFields = [
283	('dwOSVersionInfoSize', ctypes.c_ulong),
284	('dwMajorVersion', ctypes.c_ulong),
285	('dwMinorVersion', ctypes.c_ulong),
286	('dwBuildNumber', ctypes.c_ulong),
287	('dwPlatformId', ctypes.c_ulong),
288	('szCSDVersion', ctypes.c_wchar*128),
289	('wServicePackMajor', ctypes.c_ushort),
290	('wServicePackMinor', ctypes.c_ushort),
291	('wSuiteMask', ctypes.c_ushort),
292	('wProductType', ctypes.c_byte),
293	('wReserved', ctypes.c_byte)]
294	_fields_ = kaFields # pylint: disable=invalid-name
295
296	def __init__(self):
297	super(OSVersionInfoEx, self).__init__()
298	self.dwOSVersionInfoSize = ctypes.sizeof(self)
299
300	oOsVersion = OSVersionInfoEx()
301	rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
302	if rc == 0:
303	# Python platform.release() is not reliable for newer server releases
304	if oOsVersion.wProductType != 1:
305	if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
306	sVersion = '2016Server';
307	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
308	sVersion = '2012ServerR2';
309	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
310	sVersion = '2012Server';
311	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
312	sVersion = '2008ServerR2';
313	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
314	sVersion = '2008Server';
315	elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
316	sVersion = '2003Server';
317	sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
318	if oOsVersion.wServicePackMajor:
319	sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
320	if oOsVersion.wServicePackMinor:
321	sVersion += '.' + str(oOsVersion.wServicePackMinor)
322
323	return sVersion;
324
325	def getPresentCpuCount():
326	"""
327	Gets the number of CPUs present in the system.
328
329	This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
330	that we return the active count rather than the maximum count. If we don't,
331	we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
332	never will have.
333
334	@todo This is probably not exactly what we get on non-windows...
335	"""
336
337	if getHostOs() == 'win':
338	fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
339	if fnGetActiveProcessorCount:
340	cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
341	if cCpus > 0:
342	return cCpus;
343
344	import multiprocessing
345	return multiprocessing.cpu_count();
346
347
348	#
349	# File system.
350	#
351
352	def openNoInherit(sFile, sMode = 'r'):
353	"""
354	Wrapper around open() that tries it's best to make sure the file isn't
355	inherited by child processes.
356
357	This is a best effort thing at the moment as it doesn't synchronizes with
358	child process spawning in any way. Thus it can be subject to races in
359	multithreaded programs.
360	"""
361
362	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
363	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
364	if uPythonVer >= ((3 << 16) \| 4):
365	oFile = open(sFile, sMode);
366	else:
367	try:
368	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
369	except:
370	# On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
371	if getHostOs() == 'win':
372	if uPythonVer < (3 << 16):
373	offComma = sMode.find(',');
374	if offComma < 0:
375	return open(sFile, sMode + 'N');
376	return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
377
378	# Just in case.
379	return open(sFile, sMode);
380
381	oFile = open(sFile, sMode);
382	#try:
383	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
384	#except:
385	# pass;
386	return oFile;
387
388	def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
389	"""
390	Wrapper around open() that tries it's best to make sure the file isn't
391	inherited by child processes.
392
393	This is a best effort thing at the moment as it doesn't synchronizes with
394	child process spawning in any way. Thus it can be subject to races in
395	multithreaded programs.
396	"""
397
398	if getHostOs() == 'win':
399	# Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
400	# pylint: disable=no-member,c-extension-no-member
401	fAccess = 0;
402	fDisposition = win32file.OPEN_EXISTING;
403	if 'r' in sMode or '+' in sMode:
404	fAccess \|= win32file.GENERIC_READ;
405	if 'a' in sMode:
406	fAccess \|= win32file.GENERIC_WRITE;
407	fDisposition = win32file.OPEN_ALWAYS;
408	elif 'w' in sMode:
409	fAccess = win32file.GENERIC_WRITE;
410	if '+' in sMode:
411	fDisposition = win32file.OPEN_ALWAYS;
412	fAccess \|= win32file.GENERIC_READ;
413	else:
414	fDisposition = win32file.CREATE_ALWAYS;
415	if not fAccess:
416	fAccess \|= win32file.GENERIC_READ;
417	fSharing = (win32file.FILE_SHARE_READ \| win32file.FILE_SHARE_WRITE
418	\| win32file.FILE_SHARE_DELETE);
419	hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
420	if 'a' in sMode:
421	win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
422
423	# Turn the NT handle into a CRT file descriptor.
424	hDetachedFile = hFile.Detach();
425	if fAccess == win32file.GENERIC_READ:
426	fOpen = os.O_RDONLY;
427	elif fAccess == win32file.GENERIC_WRITE:
428	fOpen = os.O_WRONLY;
429	else:
430	fOpen = os.O_RDWR;
431	# pulint: enable=no-member,c-extension-no-member
432	if 'a' in sMode:
433	fOpen \|= os.O_APPEND;
434	if 'b' in sMode or 't' in sMode:
435	fOpen \|= os.O_TEXT; # pylint: disable=no-member
436	fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
437
438	# Tell python to use this handle.
439	oFile = os.fdopen(fdFile, sMode);
440	else:
441	oFile = open(sFile, sMode);
442
443	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
444	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
445	if uPythonVer < ((3 << 16) \| 4):
446	try:
447	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
448	except:
449	pass;
450	else:
451	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
452	return oFile;
453
454	def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
455	"""
456	No exceptions os.readlink wrapper.
457	"""
458	try:
459	sRet = os.readlink(sPath); # pylint: disable=no-member
460	except:
461	return sXcptRet;
462	if hasattr(sRet, 'decode'):
463	sRet = sRet.decode(sEncoding, 'ignore');
464	return sRet;
465
466	def readFile(sFile, sMode = 'rb'):
467	"""
468	Reads the entire file.
469	"""
470	oFile = open(sFile, sMode);
471	sRet = oFile.read();
472	oFile.close();
473	return sRet;
474
475	def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
476	"""
477	No exceptions common.readFile wrapper.
478	"""
479	try:
480	sRet = readFile(sFile, sMode);
481	except:
482	sRet = sXcptRet;
483	if sEncoding is not None and hasattr(sRet, 'decode'):
484	sRet = sRet.decode(sEncoding, 'ignore');
485	return sRet;
486
487	def noxcptRmDir(sDir, oXcptRet = False):
488	"""
489	No exceptions os.rmdir wrapper.
490	"""
491	oRet = True;
492	try:
493	os.rmdir(sDir);
494	except:
495	oRet = oXcptRet;
496	return oRet;
497
498	def noxcptDeleteFile(sFile, oXcptRet = False):
499	"""
500	No exceptions os.remove wrapper.
501	"""
502	oRet = True;
503	try:
504	os.remove(sFile);
505	except:
506	oRet = oXcptRet;
507	return oRet;
508
509
510	def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
511	# type: (string, (string, stat) -> bool) -> bool
512	"""
513	Recursively walks a directory tree, calling fnCallback for each.
514
515	fnCallback takes a full path and stat object (can be None). It
516	returns a boolean value, False stops walking and returns immediately.
517
518	Returns True or False depending on fnCallback.
519	Returns None fIgnoreExceptions is True and an exception was raised by listdir.
520	"""
521	def __worker(sCurDir):
522	""" Worker for """
523	try:
524	asNames = os.listdir(sCurDir);
525	except:
526	if not fIgnoreExceptions:
527	raise;
528	return None;
529	rc = True;
530	for sName in asNames:
531	if sName not in [ '.', '..' ]:
532	sFullName = os.path.join(sCurDir, sName);
533	try: oStat = os.lstat(sFullName);
534	except: oStat = None;
535	if fnCallback(sFullName, oStat) is False:
536	return False;
537	if oStat is not None and stat.S_ISDIR(oStat.st_mode):
538	rc = __worker(sFullName);
539	if rc is False:
540	break;
541	return rc;
542
543	# Ensure unicode path here so listdir also returns unicode on windows.
544	## @todo figure out unicode stuff on non-windows.
545	if sys.platform == 'win32':
546	sDir = unicode(sDir);
547	return __worker(sDir);
548
549
550
551	def formatFileMode(uMode):
552	# type: (int) -> string
553	"""
554	Format a st_mode value 'ls -la' fasion.
555	Returns string.
556	"""
557	if stat.S_ISDIR(uMode): sMode = 'd';
558	elif stat.S_ISREG(uMode): sMode = '-';
559	elif stat.S_ISLNK(uMode): sMode = 'l';
560	elif stat.S_ISFIFO(uMode): sMode = 'p';
561	elif stat.S_ISCHR(uMode): sMode = 'c';
562	elif stat.S_ISBLK(uMode): sMode = 'b';
563	elif stat.S_ISSOCK(uMode): sMode = 's';
564	else: sMode = '?';
565	## @todo sticky bits.
566	sMode += 'r' if uMode & stat.S_IRUSR else '-';
567	sMode += 'w' if uMode & stat.S_IWUSR else '-';
568	sMode += 'x' if uMode & stat.S_IXUSR else '-';
569	sMode += 'r' if uMode & stat.S_IRGRP else '-';
570	sMode += 'w' if uMode & stat.S_IWGRP else '-';
571	sMode += 'x' if uMode & stat.S_IXGRP else '-';
572	sMode += 'r' if uMode & stat.S_IROTH else '-';
573	sMode += 'w' if uMode & stat.S_IWOTH else '-';
574	sMode += 'x' if uMode & stat.S_IXOTH else '-';
575	sMode += ' ';
576	return sMode;
577
578
579	def formatFileStat(oStat):
580	# type: (stat) -> string
581	"""
582	Format a stat result 'ls -la' fasion (numeric IDs).
583	Returns string.
584	"""
585	return '%s %3s %4s %4s %10s %s' \
586	% (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
587	time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
588
589	## Good buffer for file operations.
590	g_cbGoodBufferSize = 256*1024;
591
592	## The original shutil.copyfileobj.
593	g_fnOriginalShCopyFileObj = None;
594
595	def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
596	""" shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
597	return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
598
599	def __installShUtilHacks(shutil):
600	""" Installs the shutil buffer size hacks. """
601	global g_fnOriginalShCopyFileObj;
602	if g_fnOriginalShCopyFileObj is None:
603	g_fnOriginalShCopyFileObj = shutil.copyfileobj;
604	shutil.copyfileobj = __myshutilcopyfileobj;
605	return True;
606
607
608	def copyFileSimple(sFileSrc, sFileDst):
609	"""
610	Wrapper around shutil.copyfile that simply copies the data of a regular file.
611	Raises exception on failure.
612	Return True for show.
613	"""
614	import shutil;
615	__installShUtilHacks(shutil);
616	return shutil.copyfile(sFileSrc, sFileDst);
617
618
619	def getDiskUsage(sPath):
620	"""
621	Get free space of a partition that corresponds to specified sPath in MB.
622
623	Returns partition free space value in MB.
624	"""
625	if platform.system() == 'Windows':
626	oCTypeFreeSpace = ctypes.c_ulonglong(0);
627	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
628	ctypes.pointer(oCTypeFreeSpace));
629	cbFreeSpace = oCTypeFreeSpace.value;
630	else:
631	oStats = os.statvfs(sPath); # pylint: disable=no-member
632	cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
633
634	# Convert to MB
635	cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
636
637	return cMbFreeSpace;
638
639
640
641	#
642	# SubProcess.
643	#
644
645	def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
646	"""
647	If the "executable" is a python script, insert the python interpreter at
648	the head of the argument list so that it will work on systems which doesn't
649	support hash-bang scripts.
650	"""
651
652	asArgs = dKeywordArgs.get('args');
653	if asArgs is None:
654	asArgs = aPositionalArgs[0];
655
656	if asArgs[0].endswith('.py'):
657	if sys.executable:
658	asArgs.insert(0, sys.executable);
659	else:
660	asArgs.insert(0, 'python');
661
662	# paranoia...
663	if dKeywordArgs.get('args') is not None:
664	dKeywordArgs['args'] = asArgs;
665	else:
666	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
667	return None;
668
669	def processPopenSafe(aPositionalArgs, *dKeywordArgs):
670	"""
671	Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
672	"""
673	if getHostOs() == 'win':
674	if dKeywordArgs.get('creationflags', 0) == 0:
675	dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
676	return subprocess.Popen(aPositionalArgs, *dKeywordArgs);
677
678	def processCall(aPositionalArgs, *dKeywordArgs):
679	"""
680	Wrapper around subprocess.call to deal with its absence in older
681	python versions.
682	Returns process exit code (see subprocess.poll).
683	"""
684	assert dKeywordArgs.get('stdout') is None;
685	assert dKeywordArgs.get('stderr') is None;
686	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
687	oProcess = processPopenSafe(aPositionalArgs, *dKeywordArgs);
688	return oProcess.wait();
689
690	def processOutputChecked(aPositionalArgs, *dKeywordArgs):
691	"""
692	Wrapper around subprocess.check_output to deal with its absense in older
693	python versions.
694
695	Extra keywords for specifying now output is to be decoded:
696	sEncoding='utf-8
697	fIgnoreEncoding=True/False
698	"""
699	sEncoding = dKeywordArgs.get('sEncoding');
700	if sEncoding is not None: del dKeywordArgs['sEncoding'];
701	else: sEncoding = 'utf-8';
702
703	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
704	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
705	else: fIgnoreEncoding = True;
706
707	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
708	oProcess = processPopenSafe(stdout=subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
709
710	sOutput, _ = oProcess.communicate();
711	iExitCode = oProcess.poll();
712
713	if iExitCode != 0:
714	asArgs = dKeywordArgs.get('args');
715	if asArgs is None:
716	asArgs = aPositionalArgs[0];
717	print(sOutput);
718	raise subprocess.CalledProcessError(iExitCode, asArgs);
719
720	if hasattr(sOutput, 'decode'):
721	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
722	return sOutput;
723
724	def processOutputUnchecked(aPositionalArgs, *dKeywordArgs):
725	"""
726	Similar to processOutputChecked, but returns status code and both stdout
727	and stderr results.
728
729	Extra keywords for specifying now output is to be decoded:
730	sEncoding='utf-8
731	fIgnoreEncoding=True/False
732	"""
733	sEncoding = dKeywordArgs.get('sEncoding');
734	if sEncoding is not None: del dKeywordArgs['sEncoding'];
735	else: sEncoding = 'utf-8';
736
737	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
738	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
739	else: fIgnoreEncoding = True;
740
741	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
742	oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
743
744	sOutput, sError = oProcess.communicate();
745	iExitCode = oProcess.poll();
746
747	if hasattr(sOutput, 'decode'):
748	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
749	if hasattr(sError, 'decode'):
750	sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
751	return (iExitCode, sOutput, sError);
752
753	g_fOldSudo = None;
754	def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
755	"""
756	Adds 'sudo' (or similar) to the args parameter, whereever it is.
757	"""
758
759	# Are we root?
760	fIsRoot = True;
761	try:
762	fIsRoot = os.getuid() == 0; # pylint: disable=no-member
763	except:
764	pass;
765
766	# If not, prepend sudo (non-interactive, simulate initial login).
767	if fIsRoot is not True:
768	asArgs = dKeywordArgs.get('args');
769	if asArgs is None:
770	asArgs = aPositionalArgs[0];
771
772	# Detect old sudo.
773	global g_fOldSudo;
774	if g_fOldSudo is None:
775	try:
776	sVersion = str(processOutputChecked(['sudo', '-V']));
777	except:
778	sVersion = '1.7.0';
779	sVersion = sVersion.strip().split('\n')[0];
780	sVersion = sVersion.replace('Sudo version', '').strip();
781	g_fOldSudo = len(sVersion) >= 4 \
782	and sVersion[0] == '1' \
783	and sVersion[1] == '.' \
784	and sVersion[2] <= '6' \
785	and sVersion[3] == '.';
786
787	asArgs.insert(0, 'sudo');
788	if not g_fOldSudo:
789	asArgs.insert(1, '-n');
790	if fInitialEnv and not g_fOldSudo:
791	asArgs.insert(1, '-i');
792
793	# paranoia...
794	if dKeywordArgs.get('args') is not None:
795	dKeywordArgs['args'] = asArgs;
796	else:
797	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
798	return None;
799
800
801	def sudoProcessCall(aPositionalArgs, *dKeywordArgs):
802	"""
803	sudo (or similar) + subprocess.call
804	"""
805	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
806	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
807	return processCall(aPositionalArgs, *dKeywordArgs);
808
809	def sudoProcessOutputChecked(aPositionalArgs, *dKeywordArgs):
810	"""
811	sudo (or similar) + subprocess.check_output.
812	"""
813	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
814	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
815	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
816
817	def sudoProcessOutputCheckedNoI(aPositionalArgs, *dKeywordArgs):
818	"""
819	sudo (or similar) + subprocess.check_output, except '-i' isn't used.
820	"""
821	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
822	_sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
823	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
824
825	def sudoProcessPopen(aPositionalArgs, *dKeywordArgs):
826	"""
827	sudo (or similar) + processPopenSafe.
828	"""
829	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
830	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
831	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
832
833
834	#
835	# Generic process stuff.
836	#
837
838	def processInterrupt(uPid):
839	"""
840	Sends a SIGINT or equivalent to interrupt the specified process.
841	Returns True on success, False on failure.
842
843	On Windows hosts this may not work unless the process happens to be a
844	process group leader.
845	"""
846	if sys.platform == 'win32':
847	try:
848	win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
849	uPid);
850	fRc = True;
851	except:
852	fRc = False;
853	else:
854	try:
855	os.kill(uPid, signal.SIGINT);
856	fRc = True;
857	except:
858	fRc = False;
859	return fRc;
860
861	def sendUserSignal1(uPid):
862	"""
863	Sends a SIGUSR1 or equivalent to nudge the process into shutting down
864	(VBoxSVC) or something.
865	Returns True on success, False on failure or if not supported (win).
866
867	On Windows hosts this may not work unless the process happens to be a
868	process group leader.
869	"""
870	if sys.platform == 'win32':
871	fRc = False;
872	else:
873	try:
874	os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
875	fRc = True;
876	except:
877	fRc = False;
878	return fRc;
879
880	def processTerminate(uPid):
881	"""
882	Terminates the process in a nice manner (SIGTERM or equivalent).
883	Returns True on success, False on failure.
884	"""
885	fRc = False;
886	if sys.platform == 'win32':
887	try:
888	hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
889	False, uPid);
890	except:
891	pass;
892	else:
893	try:
894	win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
895	0x40010004); # DBG_TERMINATE_PROCESS
896	fRc = True;
897	except:
898	pass;
899	hProcess.Close(); #win32api.CloseHandle(hProcess)
900	else:
901	try:
902	os.kill(uPid, signal.SIGTERM);
903	fRc = True;
904	except:
905	pass;
906	return fRc;
907
908	def processKill(uPid):
909	"""
910	Terminates the process with extreme prejudice (SIGKILL).
911	Returns True on success, False on failure.
912	"""
913	if sys.platform == 'win32':
914	fRc = processTerminate(uPid);
915	else:
916	try:
917	os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
918	fRc = True;
919	except:
920	fRc = False;
921	return fRc;
922
923	def processKillWithNameCheck(uPid, sName):
924	"""
925	Like processKill(), but checks if the process name matches before killing
926	it. This is intended for killing using potentially stale pid values.
927
928	Returns True on success, False on failure.
929	"""
930
931	if processCheckPidAndName(uPid, sName) is not True:
932	return False;
933	return processKill(uPid);
934
935
936	def processExists(uPid):
937	"""
938	Checks if the specified process exits.
939	This will only work if we can signal/open the process.
940
941	Returns True if it positively exists, False otherwise.
942	"""
943	if sys.platform == 'win32':
944	fRc = False;
945	# We try open the process for waiting since this is generally only forbidden in a very few cases.
946	try:
947	hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
948	False, uPid);
949	except pywintypes.error as oXcpt: # pylint: disable=no-member
950	if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
951	fRc = True;
952	except Exception as oXcpt:
953	pass;
954	else:
955	hProcess.Close();
956	fRc = True;
957	else:
958	try:
959	os.kill(uPid, 0);
960	fRc = True;
961	except: ## @todo check error code.
962	fRc = False;
963	return fRc;
964
965	def processCheckPidAndName(uPid, sName):
966	"""
967	Checks if a process PID and NAME matches.
968	"""
969	fRc = processExists(uPid);
970	if fRc is not True:
971	return False;
972
973	if sys.platform == 'win32':
974	try:
975	from win32com.client import GetObject; # pylint: disable=import-error
976	oWmi = GetObject('winmgmts:');
977	aoProcesses = oWmi.InstancesOf('Win32_Process');
978	for oProcess in aoProcesses:
979	if long(oProcess.Properties_("ProcessId").Value) == uPid:
980	sCurName = oProcess.Properties_("Name").Value;
981	#reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
982	sName = sName.lower();
983	sCurName = sCurName.lower();
984	if os.path.basename(sName) == sName:
985	sCurName = os.path.basename(sCurName);
986
987	if sCurName == sName \
988	or sCurName + '.exe' == sName \
989	or sCurName == sName + '.exe':
990	fRc = True;
991	break;
992	except:
993	#reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
994	pass;
995	else:
996	if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
997	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
998	elif sys.platform in ('sunos5',):
999	asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1000	elif sys.platform in ('darwin',):
1001	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
1002	else:
1003	asPsCmd = None;
1004
1005	if asPsCmd is not None:
1006	try:
1007	oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
1008	sCurName = oPs.communicate()[0];
1009	iExitCode = oPs.wait();
1010	except:
1011	#reporter.logXcpt();
1012	return False;
1013
1014	# ps fails with non-zero exit code if the pid wasn't found.
1015	if iExitCode != 0:
1016	return False;
1017	if sCurName is None:
1018	return False;
1019	sCurName = sCurName.strip();
1020	if not sCurName:
1021	return False;
1022
1023	if os.path.basename(sName) == sName:
1024	sCurName = os.path.basename(sCurName);
1025	elif os.path.basename(sCurName) == sCurName:
1026	sName = os.path.basename(sName);
1027
1028	if sCurName != sName:
1029	return False;
1030
1031	fRc = True;
1032	return fRc;
1033
1034	def processGetInfo(uPid, fSudo = False):
1035	"""
1036	Tries to acquire state information of the given process.
1037
1038	Returns a string with the information on success or None on failure or
1039	if the host is not supported.
1040
1041	Note that the format of the information is host system dependent and will
1042	likely differ much between different hosts.
1043	"""
1044	fRc = processExists(uPid);
1045	if fRc is not True:
1046	return None;
1047
1048	sHostOs = getHostOs();
1049	if sHostOs in [ 'linux',]:
1050	sGdb = '/usr/bin/gdb';
1051	if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1052	if not os.path.isfile(sGdb): sGdb = 'gdb';
1053	aasCmd = [
1054	[ sGdb, '-batch',
1055	'-ex', 'set pagination off',
1056	'-ex', 'thread apply all bt',
1057	'-ex', 'info proc mapping',
1058	'-ex', 'info sharedlibrary',
1059	'-p', '%u' % (uPid,), ],
1060	];
1061	elif sHostOs == 'darwin':
1062	# LLDB doesn't work in batch mode when attaching to a process, at least
1063	# with macOS Sierra (10.12). GDB might not be installed. Use the sample
1064	# tool instead with a 1 second duration and 1000ms sampling interval to
1065	# get one stack trace. For the process mappings use vmmap.
1066	aasCmd = [
1067	[ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1068	[ '/usr/bin/vmmap', '%u' % (uPid,), ],
1069	];
1070	elif sHostOs == 'solaris':
1071	aasCmd = [
1072	[ '/usr/bin/pstack', '%u' % (uPid,), ],
1073	[ '/usr/bin/pmap', '%u' % (uPid,), ],
1074	];
1075	else:
1076	aasCmd = [];
1077
1078	sInfo = '';
1079	for asCmd in aasCmd:
1080	try:
1081	if fSudo:
1082	sThisInfo = sudoProcessOutputChecked(asCmd);
1083	else:
1084	sThisInfo = processOutputChecked(asCmd);
1085	if sThisInfo is not None:
1086	sInfo += sThisInfo;
1087	except:
1088	pass;
1089	if not sInfo:
1090	sInfo = None;
1091
1092	return sInfo;
1093
1094
1095	class ProcessInfo(object):
1096	"""Process info."""
1097	def __init__(self, iPid):
1098	self.iPid = iPid;
1099	self.iParentPid = None;
1100	self.sImage = None;
1101	self.sName = None;
1102	self.asArgs = None;
1103	self.sCwd = None;
1104	self.iGid = None;
1105	self.iUid = None;
1106	self.iProcGroup = None;
1107	self.iSessionId = None;
1108
1109	def loadAll(self):
1110	"""Load all the info."""
1111	sOs = getHostOs();
1112	if sOs == 'linux':
1113	sProc = '/proc/%s/' % (self.iPid,);
1114	if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1115	if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1116	if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1117	#elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1118	# sProc = '/proc/%s/' % (self.iPid,);
1119	# if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1120	# if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1121	else:
1122	pass;
1123	if self.sName is None and self.sImage is not None:
1124	self.sName = self.sImage;
1125
1126	def windowsGrabProcessInfo(self, oProcess):
1127	"""Windows specific loadAll."""
1128	try: self.sName = oProcess.Properties_("Name").Value;
1129	except: pass;
1130	try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1131	except: pass;
1132	try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1133	except: pass;
1134	try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1135	except: pass;
1136	try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1137	except: pass;
1138	if self.sName is None and self.sImage is not None:
1139	self.sName = self.sImage;
1140
1141	def getBaseImageName(self):
1142	"""
1143	Gets the base image name if available, use the process name if not available.
1144	Returns image/process base name or None.
1145	"""
1146	sRet = self.sImage if self.sName is None else self.sName;
1147	if sRet is None:
1148	self.loadAll();
1149	sRet = self.sImage if self.sName is None else self.sName;
1150	if sRet is None:
1151	if not self.asArgs:
1152	return None;
1153	sRet = self.asArgs[0];
1154	if not sRet:
1155	return None;
1156	return os.path.basename(sRet);
1157
1158	def getBaseImageNameNoExeSuff(self):
1159	"""
1160	Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1161	"""
1162	sRet = self.getBaseImageName();
1163	if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1164	if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1165	sRet = sRet[:-4];
1166	return sRet;
1167
1168
1169	def processListAll():
1170	"""
1171	Return a list of ProcessInfo objects for all the processes in the system
1172	that the current user can see.
1173	"""
1174	asProcesses = [];
1175
1176	sOs = getHostOs();
1177	if sOs == 'win':
1178	from win32com.client import GetObject; # pylint: disable=import-error
1179	oWmi = GetObject('winmgmts:');
1180	aoProcesses = oWmi.InstancesOf('Win32_Process');
1181	for oProcess in aoProcesses:
1182	try:
1183	iPid = int(oProcess.Properties_("ProcessId").Value);
1184	except:
1185	continue;
1186	oMyInfo = ProcessInfo(iPid);
1187	oMyInfo.windowsGrabProcessInfo(oProcess);
1188	asProcesses.append(oMyInfo);
1189	return asProcesses;
1190
1191	if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1192	try:
1193	asDirs = os.listdir('/proc');
1194	except:
1195	asDirs = [];
1196	for sDir in asDirs:
1197	if sDir.isdigit():
1198	asProcesses.append(ProcessInfo(int(sDir),));
1199	return asProcesses;
1200
1201	#
1202	# The other OSes parses the output from the 'ps' utility.
1203	#
1204	asPsCmd = [
1205	'/bin/ps', # 0
1206	'-A', # 1
1207	'-o', 'pid=', # 2,3
1208	'-o', 'ppid=', # 4,5
1209	'-o', 'pgid=', # 6,7
1210	'-o', 'sid=', # 8,9
1211	'-o', 'uid=', # 10,11
1212	'-o', 'gid=', # 12,13
1213	'-o', 'comm=' # 14,15
1214	];
1215
1216	if sOs == 'darwin':
1217	assert asPsCmd[9] == 'sid=';
1218	asPsCmd[9] = 'sess=';
1219	elif sOs == 'solaris':
1220	asPsCmd[0] = '/usr/bin/ps';
1221
1222	try:
1223	sRaw = processOutputChecked(asPsCmd);
1224	except:
1225	return asProcesses;
1226
1227	for sLine in sRaw.split('\n'):
1228	sLine = sLine.lstrip();
1229	if len(sLine) < 7 or not sLine[0].isdigit():
1230	continue;
1231
1232	iField = 0;
1233	off = 0;
1234	aoFields = [None, None, None, None, None, None, None];
1235	while iField < 7:
1236	# Eat whitespace.
1237	while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1238	off += 1;
1239
1240	# Final field / EOL.
1241	if iField == 6:
1242	aoFields[6] = sLine[off:];
1243	break;
1244	if off >= len(sLine):
1245	break;
1246
1247	# Generic field parsing.
1248	offStart = off;
1249	off += 1;
1250	while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1251	off += 1;
1252	try:
1253	if iField != 3:
1254	aoFields[iField] = int(sLine[offStart:off]);
1255	else:
1256	aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1257	except:
1258	pass;
1259	iField += 1;
1260
1261	if aoFields[0] is not None:
1262	oMyInfo = ProcessInfo(aoFields[0]);
1263	oMyInfo.iParentPid = aoFields[1];
1264	oMyInfo.iProcGroup = aoFields[2];
1265	oMyInfo.iSessionId = aoFields[3];
1266	oMyInfo.iUid = aoFields[4];
1267	oMyInfo.iGid = aoFields[5];
1268	oMyInfo.sName = aoFields[6];
1269	asProcesses.append(oMyInfo);
1270
1271	return asProcesses;
1272
1273
1274	def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1275	"""
1276	Looks for information regarding the demise of the given process.
1277	"""
1278	sOs = getHostOs();
1279	if sOs == 'darwin':
1280	#
1281	# On darwin we look for crash and diagnostic reports.
1282	#
1283	asLogDirs = [
1284	u'/Library/Logs/DiagnosticReports/',
1285	u'/Library/Logs/CrashReporter/',
1286	u'~/Library/Logs/DiagnosticReports/',
1287	u'~/Library/Logs/CrashReporter/',
1288	];
1289	for sDir in asLogDirs:
1290	sDir = os.path.expanduser(sDir);
1291	if not os.path.isdir(sDir):
1292	continue;
1293	try:
1294	asDirEntries = os.listdir(sDir);
1295	except:
1296	continue;
1297	for sEntry in asDirEntries:
1298	# Only interested in .crash files.
1299	_, sSuff = os.path.splitext(sEntry);
1300	if sSuff != '.crash':
1301	continue;
1302
1303	# The pid can be found at the end of the first line.
1304	sFull = os.path.join(sDir, sEntry);
1305	try:
1306	oFile = open(sFull, 'r');
1307	sFirstLine = oFile.readline();
1308	oFile.close();
1309	except:
1310	continue;
1311	if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1312	continue;
1313	offPid = len(sFirstLine) - 3;
1314	while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1315	offPid -= 1;
1316	try: uReportPid = int(sFirstLine[offPid:-2]);
1317	except: continue;
1318
1319	# Does the pid we found match?
1320	if uReportPid == uPid:
1321	fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1322	fnCrashFile(sFull, False);
1323	elif sOs == 'win':
1324	#
1325	# Getting WER reports would be great, however we have trouble match the
1326	# PID to those as they seems not to mention it in the brief reports.
1327	# Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1328	# location - see the windows readme for the testbox script) and what
1329	# the MSDN article lists for now.
1330	#
1331	# It's been observed on Windows server 2012 that the dump files takes
1332	# the form: <processimage>.<decimal-pid>.dmp
1333	#
1334	asDmpDirs = [
1335	u'%SystemDrive%/CrashDumps/', # Testboxes.
1336	u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1337	u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1338	u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1339	u'%WINDIR%/ServiceProfiles/',
1340	u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1341	];
1342	sMatchSuffix = '.%u.dmp' % (uPid,);
1343
1344	for sDir in asDmpDirs:
1345	sDir = os.path.expandvars(sDir);
1346	if not os.path.isdir(sDir):
1347	continue;
1348	try:
1349	asDirEntries = os.listdir(sDir);
1350	except:
1351	continue;
1352	for sEntry in asDirEntries:
1353	if sEntry.endswith(sMatchSuffix):
1354	sFull = os.path.join(sDir, sEntry);
1355	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1356	fnCrashFile(sFull, True);
1357
1358	else:
1359	pass; ## TODO
1360	return None;
1361
1362
1363	#
1364	# Time.
1365	#
1366
1367	#
1368	# The following test case shows how time.time() only have ~ms resolution
1369	# on Windows (tested W10) and why it therefore makes sense to try use
1370	# performance counters.
1371	#
1372	# Note! We cannot use time.clock() as the timestamp must be portable across
1373	# processes. See timeout testcase problem on win hosts (no logs).
1374	# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1375	#
1376	#import sys;
1377	#import time;
1378	#from common import utils;
1379	#
1380	#atSeries = [];
1381	#for i in xrange(1,160):
1382	# if i == 159: time.sleep(10);
1383	# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1384	#
1385	#tPrev = atSeries[0]
1386	#for tCur in atSeries:
1387	# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1388	# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1389	# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1390	# print '';
1391	# tPrev = tCur
1392	#
1393	#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1394	#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1395	#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1396
1397	g_fWinUseWinPerfCounter = sys.platform == 'win32';
1398	g_fpWinPerfCounterFreq = None;
1399	g_oFuncwinQueryPerformanceCounter = None;
1400
1401	def _winInitPerfCounter():
1402	""" Initializes the use of performance counters. """
1403	global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1404
1405	uFrequency = ctypes.c_ulonglong(0);
1406	if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1407	if uFrequency.value >= 1000:
1408	#print 'uFrequency = %s' % (uFrequency,);
1409	#print 'type(uFrequency) = %s' % (type(uFrequency),);
1410	g_fpWinPerfCounterFreq = float(uFrequency.value);
1411
1412	# Check that querying the counter works too.
1413	global g_oFuncwinQueryPerformanceCounter
1414	g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1415	uCurValue = ctypes.c_ulonglong(0);
1416	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1417	if uCurValue.value > 0:
1418	return True;
1419	g_fWinUseWinPerfCounter = False;
1420	return False;
1421
1422	def _winFloatTime():
1423	""" Gets floating point time on windows. """
1424	if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1425	uCurValue = ctypes.c_ulonglong(0);
1426	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1427	return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1428	return time.time();
1429
1430	def timestampNano():
1431	"""
1432	Gets a nanosecond timestamp.
1433	"""
1434	if g_fWinUseWinPerfCounter is True:
1435	return long(_winFloatTime() * 1000000000);
1436	return long(time.time() * 1000000000);
1437
1438	def timestampMilli():
1439	"""
1440	Gets a millisecond timestamp.
1441	"""
1442	if g_fWinUseWinPerfCounter is True:
1443	return long(_winFloatTime() * 1000);
1444	return long(time.time() * 1000);
1445
1446	def timestampSecond():
1447	"""
1448	Gets a second timestamp.
1449	"""
1450	if g_fWinUseWinPerfCounter is True:
1451	return long(_winFloatTime());
1452	return long(time.time());
1453
1454	def secondsSinceUnixEpoch():
1455	"""
1456	Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1457	"""
1458	## ASSUMES This returns unix epoch time on all systems we care about...
1459	return time.time();
1460
1461	def getTimePrefix():
1462	"""
1463	Returns a timestamp prefix, typically used for logging. UTC.
1464	"""
1465	try:
1466	oNow = datetime.datetime.utcnow();
1467	sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1468	except:
1469	sTs = 'getTimePrefix-exception';
1470	return sTs;
1471
1472	def getTimePrefixAndIsoTimestamp():
1473	"""
1474	Returns current UTC as log prefix and iso timestamp.
1475	"""
1476	try:
1477	oNow = datetime.datetime.utcnow();
1478	sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1479	sTsIso = formatIsoTimestamp(oNow);
1480	except:
1481	sTsPrf = sTsIso = 'getTimePrefix-exception';
1482	return (sTsPrf, sTsIso);
1483
1484	class UtcTzInfo(datetime.tzinfo):
1485	"""UTC TZ Info Class"""
1486	def utcoffset(self, _):
1487	return datetime.timedelta(0);
1488	def tzname(self, _):
1489	return "UTC";
1490	def dst(self, _):
1491	return datetime.timedelta(0);
1492
1493	class GenTzInfo(datetime.tzinfo):
1494	"""Generic TZ Info Class"""
1495	def __init__(self, offInMin):
1496	datetime.tzinfo.__init__(self);
1497	self.offInMin = offInMin;
1498	def utcoffset(self, _):
1499	return datetime.timedelta(minutes = self.offInMin);
1500	def tzname(self, _):
1501	if self.offInMin >= 0:
1502	return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1503	return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1504	def dst(self, _):
1505	return datetime.timedelta(0);
1506
1507	def formatIsoTimestamp(oNow):
1508	"""Formats the datetime object as an ISO timestamp."""
1509	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1510	sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1511	return sTs;
1512
1513	def getIsoTimestamp():
1514	"""Returns the current UTC timestamp as a string."""
1515	return formatIsoTimestamp(datetime.datetime.utcnow());
1516
1517	def formatShortIsoTimestamp(oNow):
1518	"""Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1519	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1520	return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1521
1522	def getShortIsoTimestamp():
1523	"""Returns the current UTC timestamp as a string, but w/o microseconds."""
1524	return formatShortIsoTimestamp(datetime.datetime.utcnow());
1525
1526	def convertDateTimeToZulu(oDateTime):
1527	""" Converts oDateTime to zulu time if it has timezone info. """
1528	if oDateTime.tzinfo is not None:
1529	oDateTime = oDateTime.astimezone(UtcTzInfo());
1530	else:
1531	oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1532	return oDateTime;
1533
1534	def parseIsoTimestamp(sTs):
1535	"""
1536	Parses a typical ISO timestamp, returing a datetime object, reasonably
1537	forgiving, but will throw weird indexing/conversion errors if the input
1538	is malformed.
1539	"""
1540	# YYYY-MM-DD
1541	iYear = int(sTs[0:4]);
1542	assert(sTs[4] == '-');
1543	iMonth = int(sTs[5:7]);
1544	assert(sTs[7] == '-');
1545	iDay = int(sTs[8:10]);
1546
1547	# Skip separator
1548	sTime = sTs[10:];
1549	while sTime[0] in 'Tt \t\n\r':
1550	sTime = sTime[1:];
1551
1552	# HH:MM:SS
1553	iHour = int(sTime[0:2]);
1554	assert(sTime[2] == ':');
1555	iMin = int(sTime[3:5]);
1556	assert(sTime[5] == ':');
1557	iSec = int(sTime[6:8]);
1558
1559	# Fraction?
1560	offTime = 8;
1561	iMicroseconds = 0;
1562	if offTime < len(sTime) and sTime[offTime] in '.,':
1563	offTime += 1;
1564	cchFraction = 0;
1565	while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1566	cchFraction += 1;
1567	if cchFraction > 0:
1568	iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1569	offTime += cchFraction;
1570	while cchFraction < 6:
1571	iMicroseconds *= 10;
1572	cchFraction += 1;
1573	while cchFraction > 6:
1574	iMicroseconds = iMicroseconds // 10;
1575	cchFraction -= 1;
1576
1577	# Naive?
1578	if offTime >= len(sTime):
1579	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1580
1581	# Zulu?
1582	if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1583	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1584
1585	# Some kind of offset afterwards, and strptime is useless. sigh.
1586	if sTime[offTime] in '+-':
1587	chSign = sTime[offTime];
1588	offTime += 1;
1589	cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1590	offTime += 2;
1591	if offTime < len(sTime) and sTime[offTime] in ':':
1592	offTime += 1;
1593	if offTime + 2 <= len(sTime):
1594	cMinTz += int(sTime[offTime : (offTime + 2)]);
1595	offTime += 2;
1596	assert offTime == len(sTime);
1597	if chSign == '-':
1598	cMinTz = -cMinTz;
1599	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1600	assert False, sTs;
1601	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1602
1603	def normalizeIsoTimestampToZulu(sTs):
1604	"""
1605	Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1606	+ convertDateTimeToZulu + formatIsoTimestamp).
1607	Returns ISO tiemstamp string.
1608	"""
1609	return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1610
1611	def getLocalHourOfWeek():
1612	""" Local hour of week (0 based). """
1613	oNow = datetime.datetime.now();
1614	return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1615
1616
1617	def formatIntervalSeconds(cSeconds):
1618	""" Format a seconds interval into a nice 01h 00m 22s string """
1619	# Two simple special cases.
1620	if cSeconds < 60:
1621	return '%ss' % (cSeconds,);
1622	if cSeconds < 3600:
1623	cMins = cSeconds // 60;
1624	cSecs = cSeconds % 60;
1625	if cSecs == 0:
1626	return '%sm' % (cMins,);
1627	return '%sm %ss' % (cMins, cSecs,);
1628
1629	# Generic and a bit slower.
1630	cDays = cSeconds // 86400;
1631	cSeconds %= 86400;
1632	cHours = cSeconds // 3600;
1633	cSeconds %= 3600;
1634	cMins = cSeconds // 60;
1635	cSecs = cSeconds % 60;
1636	sRet = '';
1637	if cDays > 0:
1638	sRet = '%sd ' % (cDays,);
1639	if cHours > 0:
1640	sRet += '%sh ' % (cHours,);
1641	if cMins > 0:
1642	sRet += '%sm ' % (cMins,);
1643	if cSecs > 0:
1644	sRet += '%ss ' % (cSecs,);
1645	assert sRet; assert sRet[-1] == ' ';
1646	return sRet[:-1];
1647
1648	def formatIntervalSeconds2(oSeconds):
1649	"""
1650	Flexible input version of formatIntervalSeconds for use in WUI forms where
1651	data is usually already string form.
1652	"""
1653	if isinstance(oSeconds, (int, long)):
1654	return formatIntervalSeconds(oSeconds);
1655	if not isString(oSeconds):
1656	try:
1657	lSeconds = long(oSeconds);
1658	except:
1659	pass;
1660	else:
1661	if lSeconds >= 0:
1662	return formatIntervalSeconds2(lSeconds);
1663	return oSeconds;
1664
1665	def parseIntervalSeconds(sString):
1666	"""
1667	Reverse of formatIntervalSeconds.
1668
1669	Returns (cSeconds, sError), where sError is None on success.
1670	"""
1671
1672	# We might given non-strings, just return them without any fuss.
1673	if not isString(sString):
1674	if isinstance(sString, (int, long)) or sString is None:
1675	return (sString, None);
1676	## @todo time/date objects?
1677	return (int(sString), None);
1678
1679	# Strip it and make sure it's not empty.
1680	sString = sString.strip();
1681	if not sString:
1682	return (0, 'Empty interval string.');
1683
1684	#
1685	# Split up the input into a list of 'valueN, unitN, ...'.
1686	#
1687	# Don't want to spend too much time trying to make re.split do exactly what
1688	# I need here, so please forgive the extra pass I'm making here.
1689	#
1690	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1691	asParts = [];
1692	for sPart in asRawParts:
1693	sPart = sPart.strip();
1694	if sPart:
1695	asParts.append(sPart);
1696	if not asParts:
1697	return (0, 'Empty interval string or something?');
1698
1699	#
1700	# Process them one or two at the time.
1701	#
1702	cSeconds = 0;
1703	asErrors = [];
1704	i = 0;
1705	while i < len(asParts):
1706	sNumber = asParts[i];
1707	i += 1;
1708	if sNumber.isdigit():
1709	iNumber = int(sNumber);
1710
1711	sUnit = 's';
1712	if i < len(asParts) and not asParts[i].isdigit():
1713	sUnit = asParts[i];
1714	i += 1;
1715
1716	sUnitLower = sUnit.lower();
1717	if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1718	pass;
1719	elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1720	iNumber *= 60;
1721	elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1722	iNumber *= 3600;
1723	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1724	iNumber *= 86400;
1725	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1726	iNumber = 7 86400;
1727	else:
1728	asErrors.append('Unknown unit "%s".' % (sUnit,));
1729	cSeconds += iNumber;
1730	else:
1731	asErrors.append('Bad number "%s".' % (sNumber,));
1732	return (cSeconds, None if not asErrors else ' '.join(asErrors));
1733
1734	def formatIntervalHours(cHours):
1735	""" Format a hours interval into a nice 1w 2d 1h string. """
1736	# Simple special cases.
1737	if cHours < 24:
1738	return '%sh' % (cHours,);
1739
1740	# Generic and a bit slower.
1741	cWeeks = cHours / (7 * 24);
1742	cHours %= 7 * 24;
1743	cDays = cHours / 24;
1744	cHours %= 24;
1745	sRet = '';
1746	if cWeeks > 0:
1747	sRet = '%sw ' % (cWeeks,);
1748	if cDays > 0:
1749	sRet = '%sd ' % (cDays,);
1750	if cHours > 0:
1751	sRet += '%sh ' % (cHours,);
1752	assert sRet; assert sRet[-1] == ' ';
1753	return sRet[:-1];
1754
1755	def parseIntervalHours(sString):
1756	"""
1757	Reverse of formatIntervalHours.
1758
1759	Returns (cHours, sError), where sError is None on success.
1760	"""
1761
1762	# We might given non-strings, just return them without any fuss.
1763	if not isString(sString):
1764	if isinstance(sString, (int, long)) or sString is None:
1765	return (sString, None);
1766	## @todo time/date objects?
1767	return (int(sString), None);
1768
1769	# Strip it and make sure it's not empty.
1770	sString = sString.strip();
1771	if not sString:
1772	return (0, 'Empty interval string.');
1773
1774	#
1775	# Split up the input into a list of 'valueN, unitN, ...'.
1776	#
1777	# Don't want to spend too much time trying to make re.split do exactly what
1778	# I need here, so please forgive the extra pass I'm making here.
1779	#
1780	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1781	asParts = [];
1782	for sPart in asRawParts:
1783	sPart = sPart.strip();
1784	if sPart:
1785	asParts.append(sPart);
1786	if not asParts:
1787	return (0, 'Empty interval string or something?');
1788
1789	#
1790	# Process them one or two at the time.
1791	#
1792	cHours = 0;
1793	asErrors = [];
1794	i = 0;
1795	while i < len(asParts):
1796	sNumber = asParts[i];
1797	i += 1;
1798	if sNumber.isdigit():
1799	iNumber = int(sNumber);
1800
1801	sUnit = 'h';
1802	if i < len(asParts) and not asParts[i].isdigit():
1803	sUnit = asParts[i];
1804	i += 1;
1805
1806	sUnitLower = sUnit.lower();
1807	if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1808	pass;
1809	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1810	iNumber *= 24;
1811	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1812	iNumber = 7 24;
1813	else:
1814	asErrors.append('Unknown unit "%s".' % (sUnit,));
1815	cHours += iNumber;
1816	else:
1817	asErrors.append('Bad number "%s".' % (sNumber,));
1818	return (cHours, None if not asErrors else ' '.join(asErrors));
1819
1820
1821	#
1822	# Introspection.
1823	#
1824
1825	def getCallerName(oFrame=None, iFrame=2):
1826	"""
1827	Returns the name of the caller's caller.
1828	"""
1829	if oFrame is None:
1830	try:
1831	raise Exception();
1832	except:
1833	oFrame = sys.exc_info()[2].tb_frame.f_back;
1834	while iFrame > 1:
1835	if oFrame is not None:
1836	oFrame = oFrame.f_back;
1837	iFrame = iFrame - 1;
1838	if oFrame is not None:
1839	sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1840	return sName;
1841	return "unknown";
1842
1843
1844	def getXcptInfo(cFrames = 1):
1845	"""
1846	Gets text detailing the exception. (Good for logging.)
1847	Returns list of info strings.
1848	"""
1849
1850	#
1851	# Try get exception info.
1852	#
1853	try:
1854	oType, oValue, oTraceback = sys.exc_info();
1855	except:
1856	oType = oValue = oTraceback = None;
1857	if oType is not None:
1858
1859	#
1860	# Try format the info
1861	#
1862	asRet = [];
1863	try:
1864	try:
1865	asRet = asRet + traceback.format_exception_only(oType, oValue);
1866	asTraceBack = traceback.format_tb(oTraceback);
1867	if cFrames is not None and cFrames <= 1:
1868	asRet.append(asTraceBack[-1]);
1869	else:
1870	asRet.append('Traceback:')
1871	for iFrame in range(min(cFrames, len(asTraceBack))):
1872	asRet.append(asTraceBack[-iFrame - 1]);
1873	asRet.append('Stack:')
1874	asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1875	except:
1876	asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1877
1878	if not asRet:
1879	asRet.append('No exception info...');
1880	except:
1881	asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1882	else:
1883	asRet = ['Couldn\'t find exception traceback.'];
1884	return asRet;
1885
1886
1887	def getObjectTypeName(oObject):
1888	"""
1889	Get the type name of the given object.
1890	"""
1891	if oObject is None:
1892	return 'None';
1893
1894	# Get the type object.
1895	try:
1896	oType = type(oObject);
1897	except:
1898	return 'type-throws-exception';
1899
1900	# Python 2.x only: Handle old-style object wrappers.
1901	if sys.version_info[0] < 3:
1902	try:
1903	from types import InstanceType; # pylint: disable=no-name-in-module
1904	if oType == InstanceType:
1905	oType = oObject.__class__;
1906	except:
1907	pass;
1908
1909	# Get the name.
1910	try:
1911	return oType.__name__;
1912	except:
1913	return '__type__-throws-exception';
1914
1915
1916	def chmodPlusX(sFile):
1917	"""
1918	Makes the specified file or directory executable.
1919	Returns success indicator, no exceptions.
1920
1921	Note! Symbolic links are followed and the target will be changed.
1922	"""
1923	try:
1924	oStat = os.stat(sFile);
1925	except:
1926	return False;
1927	try:
1928	os.chmod(sFile, oStat.st_mode \| stat.S_IXUSR \| stat.S_IXGRP \| stat.S_IXOTH);
1929	except:
1930	return False;
1931	return True;
1932
1933
1934	#
1935	# TestSuite stuff.
1936	#
1937
1938	def isRunningFromCheckout(cScriptDepth = 1):
1939	"""
1940	Checks if we're running from the SVN checkout or not.
1941	"""
1942
1943	try:
1944	sFile = __file__;
1945	cScriptDepth = 1;
1946	except:
1947	sFile = sys.argv[0];
1948
1949	sDir = os.path.abspath(sFile);
1950	while cScriptDepth >= 0:
1951	sDir = os.path.dirname(sDir);
1952	if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1953	or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1954	return True;
1955	cScriptDepth -= 1;
1956
1957	return False;
1958
1959
1960	#
1961	# Bourne shell argument fun.
1962	#
1963
1964
1965	def argsSplit(sCmdLine):
1966	"""
1967	Given a bourne shell command line invocation, split it up into arguments
1968	assuming IFS is space.
1969	Returns None on syntax error.
1970	"""
1971	## @todo bourne shell argument parsing!
1972	return sCmdLine.split(' ');
1973
1974	def argsGetFirst(sCmdLine):
1975	"""
1976	Given a bourne shell command line invocation, get return the first argument
1977	assuming IFS is space.
1978	Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1979	"""
1980	asArgs = argsSplit(sCmdLine);
1981	if not asArgs:
1982	return None;
1983
1984	return asArgs[0];
1985
1986	#
1987	# String helpers.
1988	#
1989
1990	def stricmp(sFirst, sSecond):
1991	"""
1992	Compares to strings in an case insensitive fashion.
1993
1994	Python doesn't seem to have any way of doing the correctly, so this is just
1995	an approximation using lower.
1996	"""
1997	if sFirst == sSecond:
1998	return 0;
1999	sLower1 = sFirst.lower();
2000	sLower2 = sSecond.lower();
2001	if sLower1 == sLower2:
2002	return 0;
2003	if sLower1 < sLower2:
2004	return -1;
2005	return 1;
2006
2007
2008	def versionCompare(sVer1, sVer2):
2009	"""
2010	Compares to version strings in a fashion similar to RTStrVersionCompare.
2011	"""
2012
2013	## @todo implement me!!
2014
2015	if sVer1 == sVer2:
2016	return 0;
2017	if sVer1 < sVer2:
2018	return -1;
2019	return 1;
2020
2021
2022	def formatNumber(lNum, sThousandSep = ' '):
2023	"""
2024	Formats a decimal number with pretty separators.
2025	"""
2026	sNum = str(lNum);
2027	sRet = sNum[-3:];
2028	off = len(sNum) - 3;
2029	while off > 0:
2030	off -= 3;
2031	sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2032	return sRet;
2033
2034
2035	def formatNumberNbsp(lNum):
2036	"""
2037	Formats a decimal number with pretty separators.
2038	"""
2039	sRet = formatNumber(lNum);
2040	return unicode(sRet).replace(' ', u'\u00a0');
2041
2042
2043	def isString(oString):
2044	"""
2045	Checks if the object is a string object, hiding difference between python 2 and 3.
2046
2047	Returns True if it's a string of some kind.
2048	Returns False if not.
2049	"""
2050	if sys.version_info[0] >= 3:
2051	return isinstance(oString, str);
2052	return isinstance(oString, basestring); # pylint: disable=undefined-variable
2053
2054
2055	def hasNonAsciiCharacters(sText):
2056	"""
2057	Returns True is specified string has non-ASCII characters, False if ASCII only.
2058	"""
2059	if isString(sText):
2060	for ch in sText:
2061	if ord(ch) >= 128:
2062	return True;
2063	else:
2064	# Probably byte array or some such thing.
2065	for ch in sText:
2066	if ch >= 128 or ch < 0:
2067	return True;
2068	return False;
2069
2070
2071	#
2072	# Unpacking.
2073	#
2074
2075	def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2076	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2077	"""
2078	Worker for unpackFile that deals with ZIP files, same function signature.
2079	"""
2080	import zipfile
2081	if fnError is None:
2082	fnError = fnLog;
2083
2084	fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2085
2086	# Open it.
2087	try: oZipFile = zipfile.ZipFile(sArchive, 'r')
2088	except Exception as oXcpt:
2089	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2090	return None;
2091
2092	# Extract all members.
2093	asMembers = [];
2094	try:
2095	for sMember in oZipFile.namelist():
2096	if fnFilter is None or fnFilter(sMember) is not False:
2097	if sMember.endswith('/'):
2098	os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2099	else:
2100	oZipFile.extract(sMember, sDstDir);
2101	asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2102	except Exception as oXcpt:
2103	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2104	asMembers = None;
2105
2106	# close it.
2107	try: oZipFile.close();
2108	except Exception as oXcpt:
2109	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2110	asMembers = None;
2111
2112	return asMembers;
2113
2114
2115	## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2116	g_fTarCopyFileObjOverriddend = False;
2117
2118	def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2119	""" tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2120	_ = bufsize;
2121	if length is None:
2122	__myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2123	elif length > 0:
2124	cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2125	for _ in xrange(cFull):
2126	abBuffer = src.read(g_cbGoodBufferSize);
2127	dst.write(abBuffer);
2128	if len(abBuffer) != g_cbGoodBufferSize:
2129	raise exception('unexpected end of source file');
2130	if cbRemainder > 0:
2131	abBuffer = src.read(cbRemainder);
2132	dst.write(abBuffer);
2133	if len(abBuffer) != cbRemainder:
2134	raise exception('unexpected end of source file');
2135
2136
2137	def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2138	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2139	"""
2140	Worker for unpackFile that deals with tarballs, same function signature.
2141	"""
2142	import shutil;
2143	import tarfile;
2144	if fnError is None:
2145	fnError = fnLog;
2146
2147	fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2148
2149	#
2150	# Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2151	# 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2152	# 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2153	#
2154	if True is True: # pylint: disable=comparison-with-itself
2155	__installShUtilHacks(shutil);
2156	global g_fTarCopyFileObjOverriddend;
2157	if g_fTarCopyFileObjOverriddend is False:
2158	g_fTarCopyFileObjOverriddend = True;
2159	#if sys.hexversion < 0x03060000:
2160	tarfile.copyfileobj = __mytarfilecopyfileobj;
2161
2162	#
2163	# Open it.
2164	#
2165	# Note! We not using 'r:*' because we cannot allow seeking compressed files!
2166	# That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2167	#
2168	try:
2169	if sys.hexversion >= 0x03060000:
2170	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2171	else:
2172	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize);
2173	except Exception as oXcpt:
2174	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2175	return None;
2176
2177	# Extract all members.
2178	asMembers = [];
2179	try:
2180	for oTarInfo in oTarFile:
2181	try:
2182	if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2183	if oTarInfo.islnk():
2184	# Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2185	# in the compressed tar stream. So, fall back on shutil.copy2 instead.
2186	sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2187	sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2188	sParentDir = os.path.dirname(sLinkFile);
2189	try: os.unlink(sLinkFile);
2190	except: pass;
2191	if sParentDir and not os.path.exists(sParentDir):
2192	os.makedirs(sParentDir);
2193	try: os.link(sLinkTarget, sLinkFile);
2194	except: shutil.copy2(sLinkTarget, sLinkFile);
2195	else:
2196	if oTarInfo.isdir():
2197	# Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2198	oTarInfo.mode \|= 0x1c0; # (octal: 0700)
2199	oTarFile.extract(oTarInfo, sDstDir);
2200	asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2201	except Exception as oXcpt:
2202	fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2203	for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2204	fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2205	for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2206	fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2207	asMembers = None;
2208	break;
2209	except Exception as oXcpt:
2210	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2211	asMembers = None;
2212
2213	#
2214	# Finally, close it.
2215	#
2216	try: oTarFile.close();
2217	except Exception as oXcpt:
2218	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2219	asMembers = None;
2220
2221	return asMembers;
2222
2223
2224	def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2225	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2226	"""
2227	Unpacks the given file if it has a know archive extension, otherwise do
2228	nothing.
2229
2230	fnLog & fnError both take a string parameter.
2231
2232	fnFilter takes a member name (string) and returns True if it's included
2233	and False if excluded.
2234
2235	Returns list of the extracted files (full path) on success.
2236	Returns empty list if not a supported archive format.
2237	Returns None on failure. Raises no exceptions.
2238	"""
2239	sBaseNameLower = os.path.basename(sArchive).lower();
2240
2241	#
2242	# Zip file?
2243	#
2244	if sBaseNameLower.endswith('.zip'):
2245	return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2246
2247	#
2248	# Tarball?
2249	#
2250	if sBaseNameLower.endswith('.tar') \
2251	or sBaseNameLower.endswith('.tar.gz') \
2252	or sBaseNameLower.endswith('.tgz') \
2253	or sBaseNameLower.endswith('.tar.bz2'):
2254	return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2255
2256	#
2257	# Cannot classify it from the name, so just return that to the caller.
2258	#
2259	fnLog('Not unpacking "%s".' % (sArchive,));
2260	return [];
2261
2262
2263	#
2264	# Misc.
2265	#
2266	def areBytesEqual(oLeft, oRight):
2267	"""
2268	Compares two byte arrays, strings or whatnot.
2269
2270	returns true / false accordingly.
2271	"""
2272
2273	# If both are None, consider them equal (bogus?):
2274	if oLeft is None and oRight is None:
2275	return True;
2276
2277	# If just one is None, they can't match:
2278	if oLeft is None or oRight is None:
2279	return False;
2280
2281	# If both have the same type, use the compare operator of the class:
2282	if type(oLeft) is type(oRight):
2283	#print('same type: %s' % (oLeft == oRight,));
2284	return oLeft == oRight;
2285
2286	# On the offchance that they're both strings, but of different types.
2287	if isString(oLeft) and isString(oRight):
2288	#print('string compare: %s' % (oLeft == oRight,));
2289	return oLeft == oRight;
2290
2291	#
2292	# See if byte/buffer stuff that can be compared directory. If not convert
2293	# strings to bytes.
2294	#
2295	# Note! For 2.x, we must convert both sides to the buffer type or the
2296	# comparison may fail despite it working okay in test cases.
2297	#
2298	if sys.version_info[0] >= 3:
2299	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2300	return oLeft == oRight;
2301
2302	if isString(oLeft):
2303	try: oLeft = bytes(oLeft, 'utf-8');
2304	except: pass;
2305	if isString(oRight):
2306	try: oRight = bytes(oRight, 'utf-8');
2307	except: pass;
2308	else:
2309	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2310	if isinstance(oLeft, bytearray):
2311	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2312	else:
2313	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2314	#print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2315	return oLeft == oRight;
2316
2317	if isString(oLeft):
2318	try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2319	except: pass;
2320	if isString(oRight):
2321	try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2322	except: pass;
2323
2324	# Check if we now have the same type for both:
2325	if type(oLeft) is type(oRight):
2326	#print('same type now: %s' % (oLeft == oRight,));
2327	return oLeft == oRight;
2328
2329	# Check if we now have buffer/memoryview vs bytes/bytesarray again.
2330	if sys.version_info[0] >= 3:
2331	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2332	return oLeft == oRight;
2333	else:
2334	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2335	if isinstance(oLeft, bytearray):
2336	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2337	else:
2338	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2339	#print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2340	return oLeft == oRight;
2341
2342	# Do item by item comparison:
2343	if len(oLeft) != len(oRight):
2344	#print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2345	return False;
2346	i = len(oLeft);
2347	while i > 0:
2348	i = i - 1;
2349
2350	iElmLeft = oLeft[i];
2351	if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2352	iElmLeft = ord(iElmLeft);
2353
2354	iElmRight = oRight[i];
2355	if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2356	iElmRight = ord(iElmRight);
2357
2358	if iElmLeft != iElmRight:
2359	#print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2360	return False;
2361	return True;
2362
2363
2364	#
2365	# Unit testing.
2366	#
2367
2368	# pylint: disable=missing-docstring
2369	# pylint: disable=undefined-variable
2370	class BuildCategoryDataTestCase(unittest.TestCase):
2371	def testIntervalSeconds(self):
2372	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2373	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2374	self.assertEqual(parseIntervalSeconds('123'), (123, None));
2375	self.assertEqual(parseIntervalSeconds(123), (123, None));
2376	self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2377	self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2378	self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2379	self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2380	self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2381	self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2382	self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2383
2384	def testZuluNormalization(self):
2385	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2386	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2387	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2388	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2389	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2390
2391	def testHasNonAsciiChars(self):
2392	self.assertEqual(hasNonAsciiCharacters(''), False);
2393	self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB&$%&)@#(!)@(#!(#$&#$&%*Y@#$IQWN---00;'), False);
2394	self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2395	self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2396	self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2397	self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2398	self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2399	self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2400	self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2401	self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2402
2403	def testAreBytesEqual(self):
2404	self.assertEqual(areBytesEqual(None, None), True);
2405	self.assertEqual(areBytesEqual(None, ''), False);
2406	self.assertEqual(areBytesEqual('', ''), True);
2407	self.assertEqual(areBytesEqual('1', '1'), True);
2408	self.assertEqual(areBytesEqual('12345', '1234'), False);
2409	self.assertEqual(areBytesEqual('1234', '1234'), True);
2410	self.assertEqual(areBytesEqual('1234', b'1234'), True);
2411	self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2412	self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2413	self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2414	self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2415	self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2416	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2417	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2418	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2419	if sys.version_info[0] >= 3:
2420	pass;
2421	else:
2422	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2423	bytearray([0x31,0x32,0x33,0x34])), True);
2424	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2425	bytearray([0x99,0x32,0x32,0x34])), False);
2426	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2427	buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2428	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2429	buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2430	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2431	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2432	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2433
2434	if __name__ == '__main__':
2435	unittest.main();
2436	# not reached.
2437

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 86208

Download in other formats: