utils.py@ 83585

Last change on this file since 83585 was 83585, checked in by vboxsync, 5 years ago
common/utils.py: Added getShortIsoTimestamp and formatShortIsoTimestamp.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 83.2 KB

Line
1	# -- coding: utf-8 --
2	# $Id: utils.py 83585 2020-04-06 10:58:19Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Common Utility Functions.
7	"""
8
9	from __future__ import print_function;
10
11	__copyright__ = \
12	"""
13	Copyright (C) 2012-2020 Oracle Corporation
14
15	This file is part of VirtualBox Open Source Edition (OSE), as
16	available from http://www.virtualbox.org. This file is free software;
17	you can redistribute it and/or modify it under the terms of the GNU
18	General Public License (GPL) as published by the Free Software
19	Foundation, in version 2 as it comes in the "COPYING" file of the
20	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23	The contents of this file may alternatively be used under the terms
24	of the Common Development and Distribution License Version 1.0
25	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26	VirtualBox OSE distribution, in which case the provisions of the
27	CDDL are applicable instead of those of the GPL.
28
29	You may elect to license modified versions of this file under the
30	terms and conditions of either the GPL or the CDDL or both.
31	"""
32	__version__ = "$Revision: 83585 $"
33
34
35	# Standard Python imports.
36	import datetime;
37	import os;
38	import platform;
39	import re;
40	import stat;
41	import subprocess;
42	import sys;
43	import time;
44	import traceback;
45	import unittest;
46
47	if sys.platform == 'win32':
48	import ctypes;
49	import msvcrt; # pylint: disable=import-error
50	import win32api; # pylint: disable=import-error
51	import win32con; # pylint: disable=import-error
52	import win32console; # pylint: disable=import-error
53	import win32file; # pylint: disable=import-error
54	import win32process; # pylint: disable=import-error
55	import winerror; # pylint: disable=import-error
56	import pywintypes; # pylint: disable=import-error
57	else:
58	import signal;
59
60	# Python 3 hacks:
61	if sys.version_info[0] >= 3:
62	unicode = str; # pylint: disable=redefined-builtin,invalid-name
63	xrange = range; # pylint: disable=redefined-builtin,invalid-name
64	long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67	#
68	# Strings.
69	#
70
71	def toUnicode(sString, encoding = None, errors = 'strict'):
72	"""
73	A little like the python 2 unicode() function.
74	"""
75	if sys.version_info[0] >= 3:
76	if isinstance(sString, bytes):
77	return str(sString, encoding if encoding else 'utf-8', errors);
78	else:
79	if not isinstance(sString, unicode):
80	return unicode(sString, encoding if encoding else 'utf-8', errors);
81	return sString;
82
83
84
85	#
86	# Output.
87	#
88
89	def printOut(sString):
90	"""
91	Outputs a string to standard output, dealing with python 2.x encoding stupidity.
92	"""
93	sStreamEncoding = sys.stdout.encoding;
94	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
95	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
96	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
97	print(sString);
98	else:
99	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
100
101	def printErr(sString):
102	"""
103	Outputs a string to standard error, dealing with python 2.x encoding stupidity.
104	"""
105	sStreamEncoding = sys.stderr.encoding;
106	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
107	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
108	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
109	print(sString, file = sys.stderr);
110	else:
111	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
112
113
114	#
115	# Host OS and CPU.
116	#
117
118	def getHostOs():
119	"""
120	Gets the host OS name (short).
121
122	See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
123	"""
124	sPlatform = platform.system();
125	if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
126	sPlatform = sPlatform.lower();
127	elif sPlatform == 'Windows':
128	sPlatform = 'win';
129	elif sPlatform == 'SunOS':
130	sPlatform = 'solaris';
131	else:
132	raise Exception('Unsupported platform "%s"' % (sPlatform,));
133	return sPlatform;
134
135	g_sHostArch = None;
136
137	def getHostArch():
138	"""
139	Gets the host CPU architecture.
140
141	See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
142	"""
143	global g_sHostArch;
144	if g_sHostArch is None:
145	sArch = platform.machine();
146	if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
147	sArch = 'x86';
148	elif sArch in ('AMD64', 'amd64', 'x86_64'):
149	sArch = 'amd64';
150	elif sArch == 'i86pc': # SunOS
151	if platform.architecture()[0] == '64bit':
152	sArch = 'amd64';
153	else:
154	try:
155	sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
156	except:
157	pass;
158	sArch = sArch.strip();
159	if sArch != 'amd64':
160	sArch = 'x86';
161	else:
162	raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
163	g_sHostArch = sArch;
164	return g_sHostArch;
165
166
167	def getHostOsDotArch():
168	"""
169	Gets the 'os.arch' for the host.
170	"""
171	return '%s.%s' % (getHostOs(), getHostArch());
172
173
174	def isValidOs(sOs):
175	"""
176	Validates the OS name.
177	"""
178	if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
179	'os2', 'solaris', 'win', 'os-agnostic'):
180	return True;
181	return False;
182
183
184	def isValidArch(sArch):
185	"""
186	Validates the CPU architecture name.
187	"""
188	if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
189	'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
190	return True;
191	return False;
192
193	def isValidOsDotArch(sOsDotArch):
194	"""
195	Validates the 'os.arch' string.
196	"""
197
198	asParts = sOsDotArch.split('.');
199	if asParts.length() != 2:
200	return False;
201	return isValidOs(asParts[0]) \
202	and isValidArch(asParts[1]);
203
204	def getHostOsVersion():
205	"""
206	Returns the host OS version. This is platform.release with additional
207	distro indicator on linux.
208	"""
209	sVersion = platform.release();
210	sOs = getHostOs();
211	if sOs == 'linux':
212	sDist = '';
213	try:
214	# try /etc/lsb-release first to distinguish between Debian and Ubuntu
215	oFile = open('/etc/lsb-release');
216	for sLine in oFile:
217	oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s=)\s"(.)"', sLine);
218	if oMatch is not None:
219	sDist = oMatch.group(1).strip();
220	except:
221	pass;
222	if sDist:
223	sVersion += ' / ' + sDist;
224	else:
225	asFiles = \
226	[
227	[ '/etc/debian_version', 'Debian v'],
228	[ '/etc/gentoo-release', '' ],
229	[ '/etc/oracle-release', '' ],
230	[ '/etc/redhat-release', '' ],
231	[ '/etc/SuSE-release', '' ],
232	];
233	for sFile, sPrefix in asFiles:
234	if os.path.isfile(sFile):
235	try:
236	oFile = open(sFile);
237	sLine = oFile.readline();
238	oFile.close();
239	except:
240	continue;
241	sLine = sLine.strip()
242	if sLine:
243	sVersion += ' / ' + sPrefix + sLine;
244	break;
245
246	elif sOs == 'solaris':
247	sVersion = platform.version();
248	if os.path.isfile('/etc/release'):
249	try:
250	oFile = open('/etc/release');
251	sLast = oFile.readlines()[-1];
252	oFile.close();
253	sLast = sLast.strip();
254	if sLast:
255	sVersion += ' (' + sLast + ')';
256	except:
257	pass;
258
259	elif sOs == 'darwin':
260	sOsxVersion = platform.mac_ver()[0];
261	codenames = {"4": "Tiger",
262	"5": "Leopard",
263	"6": "Snow Leopard",
264	"7": "Lion",
265	"8": "Mountain Lion",
266	"9": "Mavericks",
267	"10": "Yosemite",
268	"11": "El Capitan",
269	"12": "Sierra",
270	"13": "High Sierra",
271	"14": "Unknown 14", }
272	sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
273
274	elif sOs == 'win':
275	class OSVersionInfoEx(ctypes.Structure):
276	""" OSVERSIONEX """
277	kaFields = [
278	('dwOSVersionInfoSize', ctypes.c_ulong),
279	('dwMajorVersion', ctypes.c_ulong),
280	('dwMinorVersion', ctypes.c_ulong),
281	('dwBuildNumber', ctypes.c_ulong),
282	('dwPlatformId', ctypes.c_ulong),
283	('szCSDVersion', ctypes.c_wchar*128),
284	('wServicePackMajor', ctypes.c_ushort),
285	('wServicePackMinor', ctypes.c_ushort),
286	('wSuiteMask', ctypes.c_ushort),
287	('wProductType', ctypes.c_byte),
288	('wReserved', ctypes.c_byte)]
289	_fields_ = kaFields # pylint: disable=invalid-name
290
291	def __init__(self):
292	super(OSVersionInfoEx, self).__init__()
293	self.dwOSVersionInfoSize = ctypes.sizeof(self)
294
295	oOsVersion = OSVersionInfoEx()
296	rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
297	if rc == 0:
298	# Python platform.release() is not reliable for newer server releases
299	if oOsVersion.wProductType != 1:
300	if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
301	sVersion = '2016Server';
302	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
303	sVersion = '2012ServerR2';
304	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
305	sVersion = '2012Server';
306	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
307	sVersion = '2008ServerR2';
308	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
309	sVersion = '2008Server';
310	elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
311	sVersion = '2003Server';
312	sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
313	if oOsVersion.wServicePackMajor:
314	sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
315	if oOsVersion.wServicePackMinor:
316	sVersion += '.' + str(oOsVersion.wServicePackMinor)
317
318	return sVersion;
319
320	def getPresentCpuCount():
321	"""
322	Gets the number of CPUs present in the system.
323
324	This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
325	that we return the active count rather than the maximum count. If we don't,
326	we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
327	never will have.
328
329	@todo This is probably not exactly what we get on non-windows...
330	"""
331
332	if getHostOs() == 'win':
333	fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
334	if fnGetActiveProcessorCount:
335	cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
336	if cCpus > 0:
337	return cCpus;
338
339	import multiprocessing
340	return multiprocessing.cpu_count();
341
342
343	#
344	# File system.
345	#
346
347	def openNoInherit(sFile, sMode = 'r'):
348	"""
349	Wrapper around open() that tries it's best to make sure the file isn't
350	inherited by child processes.
351
352	This is a best effort thing at the moment as it doesn't synchronizes with
353	child process spawning in any way. Thus it can be subject to races in
354	multithreaded programs.
355	"""
356
357	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
358	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
359	if uPythonVer >= ((3 << 16) \| 4):
360	oFile = open(sFile, sMode);
361	else:
362	try:
363	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
364	except:
365	# On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
366	if getHostOs() == 'win':
367	if uPythonVer < (3 << 16):
368	offComma = sMode.find(',');
369	if offComma < 0:
370	return open(sFile, sMode + 'N');
371	return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
372
373	# Just in case.
374	return open(sFile, sMode);
375
376	oFile = open(sFile, sMode);
377	#try:
378	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
379	#except:
380	# pass;
381	return oFile;
382
383	def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
384	"""
385	Wrapper around open() that tries it's best to make sure the file isn't
386	inherited by child processes.
387
388	This is a best effort thing at the moment as it doesn't synchronizes with
389	child process spawning in any way. Thus it can be subject to races in
390	multithreaded programs.
391	"""
392
393	if getHostOs() == 'win':
394	# Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
395	# pylint: disable=no-member,c-extension-no-member
396	fAccess = 0;
397	fDisposition = win32file.OPEN_EXISTING;
398	if 'r' in sMode or '+' in sMode:
399	fAccess \|= win32file.GENERIC_READ;
400	if 'a' in sMode:
401	fAccess \|= win32file.GENERIC_WRITE;
402	fDisposition = win32file.OPEN_ALWAYS;
403	elif 'w' in sMode:
404	fAccess = win32file.GENERIC_WRITE;
405	if '+' in sMode:
406	fDisposition = win32file.OPEN_ALWAYS;
407	fAccess \|= win32file.GENERIC_READ;
408	else:
409	fDisposition = win32file.CREATE_ALWAYS;
410	if not fAccess:
411	fAccess \|= win32file.GENERIC_READ;
412	fSharing = (win32file.FILE_SHARE_READ \| win32file.FILE_SHARE_WRITE
413	\| win32file.FILE_SHARE_DELETE);
414	hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
415	if 'a' in sMode:
416	win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
417
418	# Turn the NT handle into a CRT file descriptor.
419	hDetachedFile = hFile.Detach();
420	if fAccess == win32file.GENERIC_READ:
421	fOpen = os.O_RDONLY;
422	elif fAccess == win32file.GENERIC_WRITE:
423	fOpen = os.O_WRONLY;
424	else:
425	fOpen = os.O_RDWR;
426	# pulint: enable=no-member,c-extension-no-member
427	if 'a' in sMode:
428	fOpen \|= os.O_APPEND;
429	if 'b' in sMode or 't' in sMode:
430	fOpen \|= os.O_TEXT; # pylint: disable=no-member
431	fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
432
433	# Tell python to use this handle.
434	oFile = os.fdopen(fdFile, sMode);
435	else:
436	oFile = open(sFile, sMode);
437
438	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
439	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
440	if uPythonVer < ((3 << 16) \| 4):
441	try:
442	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
443	except:
444	pass;
445	else:
446	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
447	return oFile;
448
449	def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
450	"""
451	No exceptions os.readlink wrapper.
452	"""
453	try:
454	sRet = os.readlink(sPath); # pylint: disable=no-member
455	except:
456	return sXcptRet;
457	if hasattr(sRet, 'decode'):
458	sRet = sRet.decode(sEncoding, 'ignore');
459	return sRet;
460
461	def readFile(sFile, sMode = 'rb'):
462	"""
463	Reads the entire file.
464	"""
465	oFile = open(sFile, sMode);
466	sRet = oFile.read();
467	oFile.close();
468	return sRet;
469
470	def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
471	"""
472	No exceptions common.readFile wrapper.
473	"""
474	try:
475	sRet = readFile(sFile, sMode);
476	except:
477	sRet = sXcptRet;
478	if sEncoding is not None and hasattr(sRet, 'decode'):
479	sRet = sRet.decode(sEncoding, 'ignore');
480	return sRet;
481
482	def noxcptRmDir(sDir, oXcptRet = False):
483	"""
484	No exceptions os.rmdir wrapper.
485	"""
486	oRet = True;
487	try:
488	os.rmdir(sDir);
489	except:
490	oRet = oXcptRet;
491	return oRet;
492
493	def noxcptDeleteFile(sFile, oXcptRet = False):
494	"""
495	No exceptions os.remove wrapper.
496	"""
497	oRet = True;
498	try:
499	os.remove(sFile);
500	except:
501	oRet = oXcptRet;
502	return oRet;
503
504
505	def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
506	# type: (string, (string, stat) -> bool) -> bool
507	"""
508	Recursively walks a directory tree, calling fnCallback for each.
509
510	fnCallback takes a full path and stat object (can be None). It
511	returns a boolean value, False stops walking and returns immediately.
512
513	Returns True or False depending on fnCallback.
514	Returns None fIgnoreExceptions is True and an exception was raised by listdir.
515	"""
516	def __worker(sCurDir):
517	""" Worker for """
518	try:
519	asNames = os.listdir(sCurDir);
520	except:
521	if not fIgnoreExceptions:
522	raise;
523	return None;
524	rc = True;
525	for sName in asNames:
526	if sName not in [ '.', '..' ]:
527	sFullName = os.path.join(sCurDir, sName);
528	try: oStat = os.lstat(sFullName);
529	except: oStat = None;
530	if fnCallback(sFullName, oStat) is False:
531	return False;
532	if oStat is not None and stat.S_ISDIR(oStat.st_mode):
533	rc = __worker(sFullName);
534	if rc is False:
535	break;
536	return rc;
537
538	# Ensure unicode path here so listdir also returns unicode on windows.
539	## @todo figure out unicode stuff on non-windows.
540	if sys.platform == 'win32':
541	sDir = unicode(sDir);
542	return __worker(sDir);
543
544
545
546	def formatFileMode(uMode):
547	# type: (int) -> string
548	"""
549	Format a st_mode value 'ls -la' fasion.
550	Returns string.
551	"""
552	if stat.S_ISDIR(uMode): sMode = 'd';
553	elif stat.S_ISREG(uMode): sMode = '-';
554	elif stat.S_ISLNK(uMode): sMode = 'l';
555	elif stat.S_ISFIFO(uMode): sMode = 'p';
556	elif stat.S_ISCHR(uMode): sMode = 'c';
557	elif stat.S_ISBLK(uMode): sMode = 'b';
558	elif stat.S_ISSOCK(uMode): sMode = 's';
559	else: sMode = '?';
560	## @todo sticky bits.
561	sMode += 'r' if uMode & stat.S_IRUSR else '-';
562	sMode += 'w' if uMode & stat.S_IWUSR else '-';
563	sMode += 'x' if uMode & stat.S_IXUSR else '-';
564	sMode += 'r' if uMode & stat.S_IRGRP else '-';
565	sMode += 'w' if uMode & stat.S_IWGRP else '-';
566	sMode += 'x' if uMode & stat.S_IXGRP else '-';
567	sMode += 'r' if uMode & stat.S_IROTH else '-';
568	sMode += 'w' if uMode & stat.S_IWOTH else '-';
569	sMode += 'x' if uMode & stat.S_IXOTH else '-';
570	sMode += ' ';
571	return sMode;
572
573
574	def formatFileStat(oStat):
575	# type: (stat) -> string
576	"""
577	Format a stat result 'ls -la' fasion (numeric IDs).
578	Returns string.
579	"""
580	return '%s %3s %4s %4s %10s %s' \
581	% (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
582	time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
583
584	## Good buffer for file operations.
585	g_cbGoodBufferSize = 256*1024;
586
587	## The original shutil.copyfileobj.
588	g_fnOriginalShCopyFileObj = None;
589
590	def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
591	""" shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
592	return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
593
594	def __installShUtilHacks(shutil):
595	""" Installs the shutil buffer size hacks. """
596	global g_fnOriginalShCopyFileObj;
597	if g_fnOriginalShCopyFileObj is None:
598	g_fnOriginalShCopyFileObj = shutil.copyfileobj;
599	shutil.copyfileobj = __myshutilcopyfileobj;
600	return True;
601
602
603	def copyFileSimple(sFileSrc, sFileDst):
604	"""
605	Wrapper around shutil.copyfile that simply copies the data of a regular file.
606	Raises exception on failure.
607	Return True for show.
608	"""
609	import shutil;
610	__installShUtilHacks(shutil);
611	return shutil.copyfile(sFileSrc, sFileDst);
612
613
614	def getDiskUsage(sPath):
615	"""
616	Get free space of a partition that corresponds to specified sPath in MB.
617
618	Returns partition free space value in MB.
619	"""
620	if platform.system() == 'Windows':
621	oCTypeFreeSpace = ctypes.c_ulonglong(0);
622	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
623	ctypes.pointer(oCTypeFreeSpace));
624	cbFreeSpace = oCTypeFreeSpace.value;
625	else:
626	oStats = os.statvfs(sPath); # pylint: disable=no-member
627	cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
628
629	# Convert to MB
630	cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
631
632	return cMbFreeSpace;
633
634
635
636	#
637	# SubProcess.
638	#
639
640	def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
641	"""
642	If the "executable" is a python script, insert the python interpreter at
643	the head of the argument list so that it will work on systems which doesn't
644	support hash-bang scripts.
645	"""
646
647	asArgs = dKeywordArgs.get('args');
648	if asArgs is None:
649	asArgs = aPositionalArgs[0];
650
651	if asArgs[0].endswith('.py'):
652	if sys.executable:
653	asArgs.insert(0, sys.executable);
654	else:
655	asArgs.insert(0, 'python');
656
657	# paranoia...
658	if dKeywordArgs.get('args') is not None:
659	dKeywordArgs['args'] = asArgs;
660	else:
661	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
662	return None;
663
664	def processPopenSafe(aPositionalArgs, *dKeywordArgs):
665	"""
666	Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
667	"""
668	if getHostOs() == 'win':
669	if dKeywordArgs.get('creationflags', 0) == 0:
670	dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
671	return subprocess.Popen(aPositionalArgs, *dKeywordArgs);
672
673	def processCall(aPositionalArgs, *dKeywordArgs):
674	"""
675	Wrapper around subprocess.call to deal with its absence in older
676	python versions.
677	Returns process exit code (see subprocess.poll).
678	"""
679	assert dKeywordArgs.get('stdout') is None;
680	assert dKeywordArgs.get('stderr') is None;
681	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
682	oProcess = processPopenSafe(aPositionalArgs, *dKeywordArgs);
683	return oProcess.wait();
684
685	def processOutputChecked(aPositionalArgs, *dKeywordArgs):
686	"""
687	Wrapper around subprocess.check_output to deal with its absense in older
688	python versions.
689
690	Extra keywords for specifying now output is to be decoded:
691	sEncoding='utf-8
692	fIgnoreEncoding=True/False
693	"""
694	sEncoding = dKeywordArgs.get('sEncoding');
695	if sEncoding is not None: del dKeywordArgs['sEncoding'];
696	else: sEncoding = 'utf-8';
697
698	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
699	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
700	else: fIgnoreEncoding = True;
701
702	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
703	oProcess = processPopenSafe(stdout=subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
704
705	sOutput, _ = oProcess.communicate();
706	iExitCode = oProcess.poll();
707
708	if iExitCode != 0:
709	asArgs = dKeywordArgs.get('args');
710	if asArgs is None:
711	asArgs = aPositionalArgs[0];
712	print(sOutput);
713	raise subprocess.CalledProcessError(iExitCode, asArgs);
714
715	if hasattr(sOutput, 'decode'):
716	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
717	return sOutput;
718
719	def processOutputUnchecked(aPositionalArgs, *dKeywordArgs):
720	"""
721	Similar to processOutputChecked, but returns status code and both stdout
722	and stderr results.
723
724	Extra keywords for specifying now output is to be decoded:
725	sEncoding='utf-8
726	fIgnoreEncoding=True/False
727	"""
728	sEncoding = dKeywordArgs.get('sEncoding');
729	if sEncoding is not None: del dKeywordArgs['sEncoding'];
730	else: sEncoding = 'utf-8';
731
732	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
733	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
734	else: fIgnoreEncoding = True;
735
736	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
737	oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
738
739	sOutput, sError = oProcess.communicate();
740	iExitCode = oProcess.poll();
741
742	if hasattr(sOutput, 'decode'):
743	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
744	if hasattr(sError, 'decode'):
745	sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
746	return (iExitCode, sOutput, sError);
747
748	g_fOldSudo = None;
749	def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
750	"""
751	Adds 'sudo' (or similar) to the args parameter, whereever it is.
752	"""
753
754	# Are we root?
755	fIsRoot = True;
756	try:
757	fIsRoot = os.getuid() == 0; # pylint: disable=no-member
758	except:
759	pass;
760
761	# If not, prepend sudo (non-interactive, simulate initial login).
762	if fIsRoot is not True:
763	asArgs = dKeywordArgs.get('args');
764	if asArgs is None:
765	asArgs = aPositionalArgs[0];
766
767	# Detect old sudo.
768	global g_fOldSudo;
769	if g_fOldSudo is None:
770	try:
771	sVersion = str(processOutputChecked(['sudo', '-V']));
772	except:
773	sVersion = '1.7.0';
774	sVersion = sVersion.strip().split('\n')[0];
775	sVersion = sVersion.replace('Sudo version', '').strip();
776	g_fOldSudo = len(sVersion) >= 4 \
777	and sVersion[0] == '1' \
778	and sVersion[1] == '.' \
779	and sVersion[2] <= '6' \
780	and sVersion[3] == '.';
781
782	asArgs.insert(0, 'sudo');
783	if not g_fOldSudo:
784	asArgs.insert(1, '-n');
785	if fInitialEnv and not g_fOldSudo:
786	asArgs.insert(1, '-i');
787
788	# paranoia...
789	if dKeywordArgs.get('args') is not None:
790	dKeywordArgs['args'] = asArgs;
791	else:
792	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
793	return None;
794
795
796	def sudoProcessCall(aPositionalArgs, *dKeywordArgs):
797	"""
798	sudo (or similar) + subprocess.call
799	"""
800	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
801	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
802	return processCall(aPositionalArgs, *dKeywordArgs);
803
804	def sudoProcessOutputChecked(aPositionalArgs, *dKeywordArgs):
805	"""
806	sudo (or similar) + subprocess.check_output.
807	"""
808	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
809	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
810	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
811
812	def sudoProcessOutputCheckedNoI(aPositionalArgs, *dKeywordArgs):
813	"""
814	sudo (or similar) + subprocess.check_output, except '-i' isn't used.
815	"""
816	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
817	_sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
818	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
819
820	def sudoProcessPopen(aPositionalArgs, *dKeywordArgs):
821	"""
822	sudo (or similar) + processPopenSafe.
823	"""
824	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
825	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
826	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
827
828
829	#
830	# Generic process stuff.
831	#
832
833	def processInterrupt(uPid):
834	"""
835	Sends a SIGINT or equivalent to interrupt the specified process.
836	Returns True on success, False on failure.
837
838	On Windows hosts this may not work unless the process happens to be a
839	process group leader.
840	"""
841	if sys.platform == 'win32':
842	try:
843	win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
844	uPid);
845	fRc = True;
846	except:
847	fRc = False;
848	else:
849	try:
850	os.kill(uPid, signal.SIGINT);
851	fRc = True;
852	except:
853	fRc = False;
854	return fRc;
855
856	def sendUserSignal1(uPid):
857	"""
858	Sends a SIGUSR1 or equivalent to nudge the process into shutting down
859	(VBoxSVC) or something.
860	Returns True on success, False on failure or if not supported (win).
861
862	On Windows hosts this may not work unless the process happens to be a
863	process group leader.
864	"""
865	if sys.platform == 'win32':
866	fRc = False;
867	else:
868	try:
869	os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
870	fRc = True;
871	except:
872	fRc = False;
873	return fRc;
874
875	def processTerminate(uPid):
876	"""
877	Terminates the process in a nice manner (SIGTERM or equivalent).
878	Returns True on success, False on failure.
879	"""
880	fRc = False;
881	if sys.platform == 'win32':
882	try:
883	hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
884	False, uPid);
885	except:
886	pass;
887	else:
888	try:
889	win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
890	0x40010004); # DBG_TERMINATE_PROCESS
891	fRc = True;
892	except:
893	pass;
894	hProcess.Close(); #win32api.CloseHandle(hProcess)
895	else:
896	try:
897	os.kill(uPid, signal.SIGTERM);
898	fRc = True;
899	except:
900	pass;
901	return fRc;
902
903	def processKill(uPid):
904	"""
905	Terminates the process with extreme prejudice (SIGKILL).
906	Returns True on success, False on failure.
907	"""
908	if sys.platform == 'win32':
909	fRc = processTerminate(uPid);
910	else:
911	try:
912	os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
913	fRc = True;
914	except:
915	fRc = False;
916	return fRc;
917
918	def processKillWithNameCheck(uPid, sName):
919	"""
920	Like processKill(), but checks if the process name matches before killing
921	it. This is intended for killing using potentially stale pid values.
922
923	Returns True on success, False on failure.
924	"""
925
926	if processCheckPidAndName(uPid, sName) is not True:
927	return False;
928	return processKill(uPid);
929
930
931	def processExists(uPid):
932	"""
933	Checks if the specified process exits.
934	This will only work if we can signal/open the process.
935
936	Returns True if it positively exists, False otherwise.
937	"""
938	if sys.platform == 'win32':
939	fRc = False;
940	# We try open the process for waiting since this is generally only forbidden in a very few cases.
941	try:
942	hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
943	False, uPid);
944	except pywintypes.error as oXcpt: # pylint: disable=no-member
945	if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
946	fRc = True;
947	except Exception as oXcpt:
948	pass;
949	else:
950	hProcess.Close();
951	fRc = True;
952	else:
953	try:
954	os.kill(uPid, 0);
955	fRc = True;
956	except: ## @todo check error code.
957	fRc = False;
958	return fRc;
959
960	def processCheckPidAndName(uPid, sName):
961	"""
962	Checks if a process PID and NAME matches.
963	"""
964	fRc = processExists(uPid);
965	if fRc is not True:
966	return False;
967
968	if sys.platform == 'win32':
969	try:
970	from win32com.client import GetObject; # pylint: disable=import-error
971	oWmi = GetObject('winmgmts:');
972	aoProcesses = oWmi.InstancesOf('Win32_Process');
973	for oProcess in aoProcesses:
974	if long(oProcess.Properties_("ProcessId").Value) == uPid:
975	sCurName = oProcess.Properties_("Name").Value;
976	#reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
977	sName = sName.lower();
978	sCurName = sCurName.lower();
979	if os.path.basename(sName) == sName:
980	sCurName = os.path.basename(sCurName);
981
982	if sCurName == sName \
983	or sCurName + '.exe' == sName \
984	or sCurName == sName + '.exe':
985	fRc = True;
986	break;
987	except:
988	#reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
989	pass;
990	else:
991	if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
992	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
993	elif sys.platform in ('sunos5',):
994	asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
995	elif sys.platform in ('darwin',):
996	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
997	else:
998	asPsCmd = None;
999
1000	if asPsCmd is not None:
1001	try:
1002	oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
1003	sCurName = oPs.communicate()[0];
1004	iExitCode = oPs.wait();
1005	except:
1006	#reporter.logXcpt();
1007	return False;
1008
1009	# ps fails with non-zero exit code if the pid wasn't found.
1010	if iExitCode != 0:
1011	return False;
1012	if sCurName is None:
1013	return False;
1014	sCurName = sCurName.strip();
1015	if not sCurName:
1016	return False;
1017
1018	if os.path.basename(sName) == sName:
1019	sCurName = os.path.basename(sCurName);
1020	elif os.path.basename(sCurName) == sCurName:
1021	sName = os.path.basename(sName);
1022
1023	if sCurName != sName:
1024	return False;
1025
1026	fRc = True;
1027	return fRc;
1028
1029	def processGetInfo(uPid, fSudo = False):
1030	"""
1031	Tries to acquire state information of the given process.
1032
1033	Returns a string with the information on success or None on failure or
1034	if the host is not supported.
1035
1036	Note that the format of the information is host system dependent and will
1037	likely differ much between different hosts.
1038	"""
1039	fRc = processExists(uPid);
1040	if fRc is not True:
1041	return None;
1042
1043	sHostOs = getHostOs();
1044	if sHostOs in [ 'linux',]:
1045	sGdb = '/usr/bin/gdb';
1046	if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1047	if not os.path.isfile(sGdb): sGdb = 'gdb';
1048	aasCmd = [
1049	[ sGdb, '-batch',
1050	'-ex', 'set pagination off',
1051	'-ex', 'thread apply all bt',
1052	'-ex', 'info proc mapping',
1053	'-ex', 'info sharedlibrary',
1054	'-p', '%u' % (uPid,), ],
1055	];
1056	elif sHostOs == 'darwin':
1057	# LLDB doesn't work in batch mode when attaching to a process, at least
1058	# with macOS Sierra (10.12). GDB might not be installed. Use the sample
1059	# tool instead with a 1 second duration and 1000ms sampling interval to
1060	# get one stack trace. For the process mappings use vmmap.
1061	aasCmd = [
1062	[ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1063	[ '/usr/bin/vmmap', '%u' % (uPid,), ],
1064	];
1065	elif sHostOs == 'solaris':
1066	aasCmd = [
1067	[ '/usr/bin/pstack', '%u' % (uPid,), ],
1068	[ '/usr/bin/pmap', '%u' % (uPid,), ],
1069	];
1070	else:
1071	aasCmd = [];
1072
1073	sInfo = '';
1074	for asCmd in aasCmd:
1075	try:
1076	if fSudo:
1077	sThisInfo = sudoProcessOutputChecked(asCmd);
1078	else:
1079	sThisInfo = processOutputChecked(asCmd);
1080	if sThisInfo is not None:
1081	sInfo += sThisInfo;
1082	except:
1083	pass;
1084	if not sInfo:
1085	sInfo = None;
1086
1087	return sInfo;
1088
1089
1090	class ProcessInfo(object):
1091	"""Process info."""
1092	def __init__(self, iPid):
1093	self.iPid = iPid;
1094	self.iParentPid = None;
1095	self.sImage = None;
1096	self.sName = None;
1097	self.asArgs = None;
1098	self.sCwd = None;
1099	self.iGid = None;
1100	self.iUid = None;
1101	self.iProcGroup = None;
1102	self.iSessionId = None;
1103
1104	def loadAll(self):
1105	"""Load all the info."""
1106	sOs = getHostOs();
1107	if sOs == 'linux':
1108	sProc = '/proc/%s/' % (self.iPid,);
1109	if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1110	if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1111	if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1112	#elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1113	# sProc = '/proc/%s/' % (self.iPid,);
1114	# if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1115	# if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1116	else:
1117	pass;
1118	if self.sName is None and self.sImage is not None:
1119	self.sName = self.sImage;
1120
1121	def windowsGrabProcessInfo(self, oProcess):
1122	"""Windows specific loadAll."""
1123	try: self.sName = oProcess.Properties_("Name").Value;
1124	except: pass;
1125	try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1126	except: pass;
1127	try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1128	except: pass;
1129	try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1130	except: pass;
1131	try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1132	except: pass;
1133	if self.sName is None and self.sImage is not None:
1134	self.sName = self.sImage;
1135
1136	def getBaseImageName(self):
1137	"""
1138	Gets the base image name if available, use the process name if not available.
1139	Returns image/process base name or None.
1140	"""
1141	sRet = self.sImage if self.sName is None else self.sName;
1142	if sRet is None:
1143	self.loadAll();
1144	sRet = self.sImage if self.sName is None else self.sName;
1145	if sRet is None:
1146	if not self.asArgs:
1147	return None;
1148	sRet = self.asArgs[0];
1149	if not sRet:
1150	return None;
1151	return os.path.basename(sRet);
1152
1153	def getBaseImageNameNoExeSuff(self):
1154	"""
1155	Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1156	"""
1157	sRet = self.getBaseImageName();
1158	if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1159	if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1160	sRet = sRet[:-4];
1161	return sRet;
1162
1163
1164	def processListAll():
1165	"""
1166	Return a list of ProcessInfo objects for all the processes in the system
1167	that the current user can see.
1168	"""
1169	asProcesses = [];
1170
1171	sOs = getHostOs();
1172	if sOs == 'win':
1173	from win32com.client import GetObject; # pylint: disable=import-error
1174	oWmi = GetObject('winmgmts:');
1175	aoProcesses = oWmi.InstancesOf('Win32_Process');
1176	for oProcess in aoProcesses:
1177	try:
1178	iPid = int(oProcess.Properties_("ProcessId").Value);
1179	except:
1180	continue;
1181	oMyInfo = ProcessInfo(iPid);
1182	oMyInfo.windowsGrabProcessInfo(oProcess);
1183	asProcesses.append(oMyInfo);
1184	return asProcesses;
1185
1186	if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1187	try:
1188	asDirs = os.listdir('/proc');
1189	except:
1190	asDirs = [];
1191	for sDir in asDirs:
1192	if sDir.isdigit():
1193	asProcesses.append(ProcessInfo(int(sDir),));
1194	return asProcesses;
1195
1196	#
1197	# The other OSes parses the output from the 'ps' utility.
1198	#
1199	asPsCmd = [
1200	'/bin/ps', # 0
1201	'-A', # 1
1202	'-o', 'pid=', # 2,3
1203	'-o', 'ppid=', # 4,5
1204	'-o', 'pgid=', # 6,7
1205	'-o', 'sid=', # 8,9
1206	'-o', 'uid=', # 10,11
1207	'-o', 'gid=', # 12,13
1208	'-o', 'comm=' # 14,15
1209	];
1210
1211	if sOs == 'darwin':
1212	assert asPsCmd[9] == 'sid=';
1213	asPsCmd[9] = 'sess=';
1214	elif sOs == 'solaris':
1215	asPsCmd[0] = '/usr/bin/ps';
1216
1217	try:
1218	sRaw = processOutputChecked(asPsCmd);
1219	except:
1220	return asProcesses;
1221
1222	for sLine in sRaw.split('\n'):
1223	sLine = sLine.lstrip();
1224	if len(sLine) < 7 or not sLine[0].isdigit():
1225	continue;
1226
1227	iField = 0;
1228	off = 0;
1229	aoFields = [None, None, None, None, None, None, None];
1230	while iField < 7:
1231	# Eat whitespace.
1232	while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1233	off += 1;
1234
1235	# Final field / EOL.
1236	if iField == 6:
1237	aoFields[6] = sLine[off:];
1238	break;
1239	if off >= len(sLine):
1240	break;
1241
1242	# Generic field parsing.
1243	offStart = off;
1244	off += 1;
1245	while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1246	off += 1;
1247	try:
1248	if iField != 3:
1249	aoFields[iField] = int(sLine[offStart:off]);
1250	else:
1251	aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1252	except:
1253	pass;
1254	iField += 1;
1255
1256	if aoFields[0] is not None:
1257	oMyInfo = ProcessInfo(aoFields[0]);
1258	oMyInfo.iParentPid = aoFields[1];
1259	oMyInfo.iProcGroup = aoFields[2];
1260	oMyInfo.iSessionId = aoFields[3];
1261	oMyInfo.iUid = aoFields[4];
1262	oMyInfo.iGid = aoFields[5];
1263	oMyInfo.sName = aoFields[6];
1264	asProcesses.append(oMyInfo);
1265
1266	return asProcesses;
1267
1268
1269	def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1270	"""
1271	Looks for information regarding the demise of the given process.
1272	"""
1273	sOs = getHostOs();
1274	if sOs == 'darwin':
1275	#
1276	# On darwin we look for crash and diagnostic reports.
1277	#
1278	asLogDirs = [
1279	u'/Library/Logs/DiagnosticReports/',
1280	u'/Library/Logs/CrashReporter/',
1281	u'~/Library/Logs/DiagnosticReports/',
1282	u'~/Library/Logs/CrashReporter/',
1283	];
1284	for sDir in asLogDirs:
1285	sDir = os.path.expanduser(sDir);
1286	if not os.path.isdir(sDir):
1287	continue;
1288	try:
1289	asDirEntries = os.listdir(sDir);
1290	except:
1291	continue;
1292	for sEntry in asDirEntries:
1293	# Only interested in .crash files.
1294	_, sSuff = os.path.splitext(sEntry);
1295	if sSuff != '.crash':
1296	continue;
1297
1298	# The pid can be found at the end of the first line.
1299	sFull = os.path.join(sDir, sEntry);
1300	try:
1301	oFile = open(sFull, 'r');
1302	sFirstLine = oFile.readline();
1303	oFile.close();
1304	except:
1305	continue;
1306	if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1307	continue;
1308	offPid = len(sFirstLine) - 3;
1309	while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1310	offPid -= 1;
1311	try: uReportPid = int(sFirstLine[offPid:-2]);
1312	except: continue;
1313
1314	# Does the pid we found match?
1315	if uReportPid == uPid:
1316	fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1317	fnCrashFile(sFull, False);
1318	elif sOs == 'win':
1319	#
1320	# Getting WER reports would be great, however we have trouble match the
1321	# PID to those as they seems not to mention it in the brief reports.
1322	# Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1323	# location - see the windows readme for the testbox script) and what
1324	# the MSDN article lists for now.
1325	#
1326	# It's been observed on Windows server 2012 that the dump files takes
1327	# the form: <processimage>.<decimal-pid>.dmp
1328	#
1329	asDmpDirs = [
1330	u'%SystemDrive%/CrashDumps/', # Testboxes.
1331	u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1332	u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1333	u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1334	u'%WINDIR%/ServiceProfiles/',
1335	u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1336	];
1337	sMatchSuffix = '.%u.dmp' % (uPid,);
1338
1339	for sDir in asDmpDirs:
1340	sDir = os.path.expandvars(sDir);
1341	if not os.path.isdir(sDir):
1342	continue;
1343	try:
1344	asDirEntries = os.listdir(sDir);
1345	except:
1346	continue;
1347	for sEntry in asDirEntries:
1348	if sEntry.endswith(sMatchSuffix):
1349	sFull = os.path.join(sDir, sEntry);
1350	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1351	fnCrashFile(sFull, True);
1352
1353	else:
1354	pass; ## TODO
1355	return None;
1356
1357
1358	#
1359	# Time.
1360	#
1361
1362	#
1363	# The following test case shows how time.time() only have ~ms resolution
1364	# on Windows (tested W10) and why it therefore makes sense to try use
1365	# performance counters.
1366	#
1367	# Note! We cannot use time.clock() as the timestamp must be portable across
1368	# processes. See timeout testcase problem on win hosts (no logs).
1369	# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1370	#
1371	#import sys;
1372	#import time;
1373	#from common import utils;
1374	#
1375	#atSeries = [];
1376	#for i in xrange(1,160):
1377	# if i == 159: time.sleep(10);
1378	# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1379	#
1380	#tPrev = atSeries[0]
1381	#for tCur in atSeries:
1382	# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1383	# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1384	# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1385	# print '';
1386	# tPrev = tCur
1387	#
1388	#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1389	#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1390	#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1391
1392	g_fWinUseWinPerfCounter = sys.platform == 'win32';
1393	g_fpWinPerfCounterFreq = None;
1394	g_oFuncwinQueryPerformanceCounter = None;
1395
1396	def _winInitPerfCounter():
1397	""" Initializes the use of performance counters. """
1398	global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1399
1400	uFrequency = ctypes.c_ulonglong(0);
1401	if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1402	if uFrequency.value >= 1000:
1403	#print 'uFrequency = %s' % (uFrequency,);
1404	#print 'type(uFrequency) = %s' % (type(uFrequency),);
1405	g_fpWinPerfCounterFreq = float(uFrequency.value);
1406
1407	# Check that querying the counter works too.
1408	global g_oFuncwinQueryPerformanceCounter
1409	g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1410	uCurValue = ctypes.c_ulonglong(0);
1411	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1412	if uCurValue.value > 0:
1413	return True;
1414	g_fWinUseWinPerfCounter = False;
1415	return False;
1416
1417	def _winFloatTime():
1418	""" Gets floating point time on windows. """
1419	if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1420	uCurValue = ctypes.c_ulonglong(0);
1421	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1422	return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1423	return time.time();
1424
1425	def timestampNano():
1426	"""
1427	Gets a nanosecond timestamp.
1428	"""
1429	if g_fWinUseWinPerfCounter is True:
1430	return long(_winFloatTime() * 1000000000);
1431	return long(time.time() * 1000000000);
1432
1433	def timestampMilli():
1434	"""
1435	Gets a millisecond timestamp.
1436	"""
1437	if g_fWinUseWinPerfCounter is True:
1438	return long(_winFloatTime() * 1000);
1439	return long(time.time() * 1000);
1440
1441	def timestampSecond():
1442	"""
1443	Gets a second timestamp.
1444	"""
1445	if g_fWinUseWinPerfCounter is True:
1446	return long(_winFloatTime());
1447	return long(time.time());
1448
1449	def secondsSinceUnixEpoch():
1450	"""
1451	Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1452	"""
1453	## ASSUMES This returns unix epoch time on all systems we care about...
1454	return time.time();
1455
1456	def getTimePrefix():
1457	"""
1458	Returns a timestamp prefix, typically used for logging. UTC.
1459	"""
1460	try:
1461	oNow = datetime.datetime.utcnow();
1462	sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1463	except:
1464	sTs = 'getTimePrefix-exception';
1465	return sTs;
1466
1467	def getTimePrefixAndIsoTimestamp():
1468	"""
1469	Returns current UTC as log prefix and iso timestamp.
1470	"""
1471	try:
1472	oNow = datetime.datetime.utcnow();
1473	sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1474	sTsIso = formatIsoTimestamp(oNow);
1475	except:
1476	sTsPrf = sTsIso = 'getTimePrefix-exception';
1477	return (sTsPrf, sTsIso);
1478
1479	class UtcTzInfo(datetime.tzinfo):
1480	"""UTC TZ Info Class"""
1481	def utcoffset(self, _):
1482	return datetime.timedelta(0);
1483	def tzname(self, _):
1484	return "UTC";
1485	def dst(self, _):
1486	return datetime.timedelta(0);
1487
1488	class GenTzInfo(datetime.tzinfo):
1489	"""Generic TZ Info Class"""
1490	def __init__(self, offInMin):
1491	datetime.tzinfo.__init__(self);
1492	self.offInMin = offInMin;
1493	def utcoffset(self, _):
1494	return datetime.timedelta(minutes = self.offInMin);
1495	def tzname(self, _):
1496	if self.offInMin >= 0:
1497	return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1498	return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1499	def dst(self, _):
1500	return datetime.timedelta(0);
1501
1502	def formatIsoTimestamp(oNow):
1503	"""Formats the datetime object as an ISO timestamp."""
1504	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1505	sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1506	return sTs;
1507
1508	def getIsoTimestamp():
1509	"""Returns the current UTC timestamp as a string."""
1510	return formatIsoTimestamp(datetime.datetime.utcnow());
1511
1512	def formatShortIsoTimestamp(oNow):
1513	"""Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1514	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1515	return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1516
1517	def getShortIsoTimestamp():
1518	"""Returns the current UTC timestamp as a string, but w/o microseconds."""
1519	return formatShortIsoTimestamp(datetime.datetime.utcnow());
1520
1521	def convertDateTimeToZulu(oDateTime):
1522	""" Converts oDateTime to zulu time if it has timezone info. """
1523	if oDateTime.tzinfo is not None:
1524	oDateTime = oDateTime.astimezone(UtcTzInfo());
1525	else:
1526	oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1527	return oDateTime;
1528
1529	def parseIsoTimestamp(sTs):
1530	"""
1531	Parses a typical ISO timestamp, returing a datetime object, reasonably
1532	forgiving, but will throw weird indexing/conversion errors if the input
1533	is malformed.
1534	"""
1535	# YYYY-MM-DD
1536	iYear = int(sTs[0:4]);
1537	assert(sTs[4] == '-');
1538	iMonth = int(sTs[5:7]);
1539	assert(sTs[7] == '-');
1540	iDay = int(sTs[8:10]);
1541
1542	# Skip separator
1543	sTime = sTs[10:];
1544	while sTime[0] in 'Tt \t\n\r':
1545	sTime = sTime[1:];
1546
1547	# HH:MM:SS
1548	iHour = int(sTime[0:2]);
1549	assert(sTime[2] == ':');
1550	iMin = int(sTime[3:5]);
1551	assert(sTime[5] == ':');
1552	iSec = int(sTime[6:8]);
1553
1554	# Fraction?
1555	offTime = 8;
1556	iMicroseconds = 0;
1557	if offTime < len(sTime) and sTime[offTime] in '.,':
1558	offTime += 1;
1559	cchFraction = 0;
1560	while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1561	cchFraction += 1;
1562	if cchFraction > 0:
1563	iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1564	offTime += cchFraction;
1565	while cchFraction < 6:
1566	iMicroseconds *= 10;
1567	cchFraction += 1;
1568	while cchFraction > 6:
1569	iMicroseconds = iMicroseconds // 10;
1570	cchFraction -= 1;
1571
1572	# Naive?
1573	if offTime >= len(sTime):
1574	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1575
1576	# Zulu?
1577	if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1578	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1579
1580	# Some kind of offset afterwards, and strptime is useless. sigh.
1581	if sTime[offTime] in '+-':
1582	chSign = sTime[offTime];
1583	offTime += 1;
1584	cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1585	offTime += 2;
1586	if offTime < len(sTime) and sTime[offTime] in ':':
1587	offTime += 1;
1588	if offTime + 2 <= len(sTime):
1589	cMinTz += int(sTime[offTime : (offTime + 2)]);
1590	offTime += 2;
1591	assert offTime == len(sTime);
1592	if chSign == '-':
1593	cMinTz = -cMinTz;
1594	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1595	assert False, sTs;
1596	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1597
1598	def normalizeIsoTimestampToZulu(sTs):
1599	"""
1600	Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1601	+ convertDateTimeToZulu + formatIsoTimestamp).
1602	Returns ISO tiemstamp string.
1603	"""
1604	return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1605
1606	def getLocalHourOfWeek():
1607	""" Local hour of week (0 based). """
1608	oNow = datetime.datetime.now();
1609	return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1610
1611
1612	def formatIntervalSeconds(cSeconds):
1613	""" Format a seconds interval into a nice 01h 00m 22s string """
1614	# Two simple special cases.
1615	if cSeconds < 60:
1616	return '%ss' % (cSeconds,);
1617	if cSeconds < 3600:
1618	cMins = cSeconds // 60;
1619	cSecs = cSeconds % 60;
1620	if cSecs == 0:
1621	return '%sm' % (cMins,);
1622	return '%sm %ss' % (cMins, cSecs,);
1623
1624	# Generic and a bit slower.
1625	cDays = cSeconds // 86400;
1626	cSeconds %= 86400;
1627	cHours = cSeconds // 3600;
1628	cSeconds %= 3600;
1629	cMins = cSeconds // 60;
1630	cSecs = cSeconds % 60;
1631	sRet = '';
1632	if cDays > 0:
1633	sRet = '%sd ' % (cDays,);
1634	if cHours > 0:
1635	sRet += '%sh ' % (cHours,);
1636	if cMins > 0:
1637	sRet += '%sm ' % (cMins,);
1638	if cSecs > 0:
1639	sRet += '%ss ' % (cSecs,);
1640	assert sRet; assert sRet[-1] == ' ';
1641	return sRet[:-1];
1642
1643	def formatIntervalSeconds2(oSeconds):
1644	"""
1645	Flexible input version of formatIntervalSeconds for use in WUI forms where
1646	data is usually already string form.
1647	"""
1648	if isinstance(oSeconds, (int, long)):
1649	return formatIntervalSeconds(oSeconds);
1650	if not isString(oSeconds):
1651	try:
1652	lSeconds = long(oSeconds);
1653	except:
1654	pass;
1655	else:
1656	if lSeconds >= 0:
1657	return formatIntervalSeconds2(lSeconds);
1658	return oSeconds;
1659
1660	def parseIntervalSeconds(sString):
1661	"""
1662	Reverse of formatIntervalSeconds.
1663
1664	Returns (cSeconds, sError), where sError is None on success.
1665	"""
1666
1667	# We might given non-strings, just return them without any fuss.
1668	if not isString(sString):
1669	if isinstance(sString, (int, long)) or sString is None:
1670	return (sString, None);
1671	## @todo time/date objects?
1672	return (int(sString), None);
1673
1674	# Strip it and make sure it's not empty.
1675	sString = sString.strip();
1676	if not sString:
1677	return (0, 'Empty interval string.');
1678
1679	#
1680	# Split up the input into a list of 'valueN, unitN, ...'.
1681	#
1682	# Don't want to spend too much time trying to make re.split do exactly what
1683	# I need here, so please forgive the extra pass I'm making here.
1684	#
1685	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1686	asParts = [];
1687	for sPart in asRawParts:
1688	sPart = sPart.strip();
1689	if sPart:
1690	asParts.append(sPart);
1691	if not asParts:
1692	return (0, 'Empty interval string or something?');
1693
1694	#
1695	# Process them one or two at the time.
1696	#
1697	cSeconds = 0;
1698	asErrors = [];
1699	i = 0;
1700	while i < len(asParts):
1701	sNumber = asParts[i];
1702	i += 1;
1703	if sNumber.isdigit():
1704	iNumber = int(sNumber);
1705
1706	sUnit = 's';
1707	if i < len(asParts) and not asParts[i].isdigit():
1708	sUnit = asParts[i];
1709	i += 1;
1710
1711	sUnitLower = sUnit.lower();
1712	if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1713	pass;
1714	elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1715	iNumber *= 60;
1716	elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1717	iNumber *= 3600;
1718	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1719	iNumber *= 86400;
1720	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1721	iNumber = 7 86400;
1722	else:
1723	asErrors.append('Unknown unit "%s".' % (sUnit,));
1724	cSeconds += iNumber;
1725	else:
1726	asErrors.append('Bad number "%s".' % (sNumber,));
1727	return (cSeconds, None if not asErrors else ' '.join(asErrors));
1728
1729	def formatIntervalHours(cHours):
1730	""" Format a hours interval into a nice 1w 2d 1h string. """
1731	# Simple special cases.
1732	if cHours < 24:
1733	return '%sh' % (cHours,);
1734
1735	# Generic and a bit slower.
1736	cWeeks = cHours / (7 * 24);
1737	cHours %= 7 * 24;
1738	cDays = cHours / 24;
1739	cHours %= 24;
1740	sRet = '';
1741	if cWeeks > 0:
1742	sRet = '%sw ' % (cWeeks,);
1743	if cDays > 0:
1744	sRet = '%sd ' % (cDays,);
1745	if cHours > 0:
1746	sRet += '%sh ' % (cHours,);
1747	assert sRet; assert sRet[-1] == ' ';
1748	return sRet[:-1];
1749
1750	def parseIntervalHours(sString):
1751	"""
1752	Reverse of formatIntervalHours.
1753
1754	Returns (cHours, sError), where sError is None on success.
1755	"""
1756
1757	# We might given non-strings, just return them without any fuss.
1758	if not isString(sString):
1759	if isinstance(sString, (int, long)) or sString is None:
1760	return (sString, None);
1761	## @todo time/date objects?
1762	return (int(sString), None);
1763
1764	# Strip it and make sure it's not empty.
1765	sString = sString.strip();
1766	if not sString:
1767	return (0, 'Empty interval string.');
1768
1769	#
1770	# Split up the input into a list of 'valueN, unitN, ...'.
1771	#
1772	# Don't want to spend too much time trying to make re.split do exactly what
1773	# I need here, so please forgive the extra pass I'm making here.
1774	#
1775	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1776	asParts = [];
1777	for sPart in asRawParts:
1778	sPart = sPart.strip();
1779	if sPart:
1780	asParts.append(sPart);
1781	if not asParts:
1782	return (0, 'Empty interval string or something?');
1783
1784	#
1785	# Process them one or two at the time.
1786	#
1787	cHours = 0;
1788	asErrors = [];
1789	i = 0;
1790	while i < len(asParts):
1791	sNumber = asParts[i];
1792	i += 1;
1793	if sNumber.isdigit():
1794	iNumber = int(sNumber);
1795
1796	sUnit = 'h';
1797	if i < len(asParts) and not asParts[i].isdigit():
1798	sUnit = asParts[i];
1799	i += 1;
1800
1801	sUnitLower = sUnit.lower();
1802	if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1803	pass;
1804	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1805	iNumber *= 24;
1806	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1807	iNumber = 7 24;
1808	else:
1809	asErrors.append('Unknown unit "%s".' % (sUnit,));
1810	cHours += iNumber;
1811	else:
1812	asErrors.append('Bad number "%s".' % (sNumber,));
1813	return (cHours, None if not asErrors else ' '.join(asErrors));
1814
1815
1816	#
1817	# Introspection.
1818	#
1819
1820	def getCallerName(oFrame=None, iFrame=2):
1821	"""
1822	Returns the name of the caller's caller.
1823	"""
1824	if oFrame is None:
1825	try:
1826	raise Exception();
1827	except:
1828	oFrame = sys.exc_info()[2].tb_frame.f_back;
1829	while iFrame > 1:
1830	if oFrame is not None:
1831	oFrame = oFrame.f_back;
1832	iFrame = iFrame - 1;
1833	if oFrame is not None:
1834	sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1835	return sName;
1836	return "unknown";
1837
1838
1839	def getXcptInfo(cFrames = 1):
1840	"""
1841	Gets text detailing the exception. (Good for logging.)
1842	Returns list of info strings.
1843	"""
1844
1845	#
1846	# Try get exception info.
1847	#
1848	try:
1849	oType, oValue, oTraceback = sys.exc_info();
1850	except:
1851	oType = oValue = oTraceback = None;
1852	if oType is not None:
1853
1854	#
1855	# Try format the info
1856	#
1857	asRet = [];
1858	try:
1859	try:
1860	asRet = asRet + traceback.format_exception_only(oType, oValue);
1861	asTraceBack = traceback.format_tb(oTraceback);
1862	if cFrames is not None and cFrames <= 1:
1863	asRet.append(asTraceBack[-1]);
1864	else:
1865	asRet.append('Traceback:')
1866	for iFrame in range(min(cFrames, len(asTraceBack))):
1867	asRet.append(asTraceBack[-iFrame - 1]);
1868	asRet.append('Stack:')
1869	asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1870	except:
1871	asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1872
1873	if not asRet:
1874	asRet.append('No exception info...');
1875	except:
1876	asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1877	else:
1878	asRet = ['Couldn\'t find exception traceback.'];
1879	return asRet;
1880
1881
1882	def getObjectTypeName(oObject):
1883	"""
1884	Get the type name of the given object.
1885	"""
1886	if oObject is None:
1887	return 'None';
1888
1889	# Get the type object.
1890	try:
1891	oType = type(oObject);
1892	except:
1893	return 'type-throws-exception';
1894
1895	# Python 2.x only: Handle old-style object wrappers.
1896	if sys.version_info[0] < 3:
1897	try:
1898	from types import InstanceType; # pylint: disable=no-name-in-module
1899	if oType == InstanceType:
1900	oType = oObject.__class__;
1901	except:
1902	pass;
1903
1904	# Get the name.
1905	try:
1906	return oType.__name__;
1907	except:
1908	return '__type__-throws-exception';
1909
1910
1911	def chmodPlusX(sFile):
1912	"""
1913	Makes the specified file or directory executable.
1914	Returns success indicator, no exceptions.
1915
1916	Note! Symbolic links are followed and the target will be changed.
1917	"""
1918	try:
1919	oStat = os.stat(sFile);
1920	except:
1921	return False;
1922	try:
1923	os.chmod(sFile, oStat.st_mode \| stat.S_IXUSR \| stat.S_IXGRP \| stat.S_IXOTH);
1924	except:
1925	return False;
1926	return True;
1927
1928
1929	#
1930	# TestSuite stuff.
1931	#
1932
1933	def isRunningFromCheckout(cScriptDepth = 1):
1934	"""
1935	Checks if we're running from the SVN checkout or not.
1936	"""
1937
1938	try:
1939	sFile = __file__;
1940	cScriptDepth = 1;
1941	except:
1942	sFile = sys.argv[0];
1943
1944	sDir = os.path.abspath(sFile);
1945	while cScriptDepth >= 0:
1946	sDir = os.path.dirname(sDir);
1947	if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1948	or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1949	return True;
1950	cScriptDepth -= 1;
1951
1952	return False;
1953
1954
1955	#
1956	# Bourne shell argument fun.
1957	#
1958
1959
1960	def argsSplit(sCmdLine):
1961	"""
1962	Given a bourne shell command line invocation, split it up into arguments
1963	assuming IFS is space.
1964	Returns None on syntax error.
1965	"""
1966	## @todo bourne shell argument parsing!
1967	return sCmdLine.split(' ');
1968
1969	def argsGetFirst(sCmdLine):
1970	"""
1971	Given a bourne shell command line invocation, get return the first argument
1972	assuming IFS is space.
1973	Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1974	"""
1975	asArgs = argsSplit(sCmdLine);
1976	if not asArgs:
1977	return None;
1978
1979	return asArgs[0];
1980
1981	#
1982	# String helpers.
1983	#
1984
1985	def stricmp(sFirst, sSecond):
1986	"""
1987	Compares to strings in an case insensitive fashion.
1988
1989	Python doesn't seem to have any way of doing the correctly, so this is just
1990	an approximation using lower.
1991	"""
1992	if sFirst == sSecond:
1993	return 0;
1994	sLower1 = sFirst.lower();
1995	sLower2 = sSecond.lower();
1996	if sLower1 == sLower2:
1997	return 0;
1998	if sLower1 < sLower2:
1999	return -1;
2000	return 1;
2001
2002
2003	def versionCompare(sVer1, sVer2):
2004	"""
2005	Compares to version strings in a fashion similar to RTStrVersionCompare.
2006	"""
2007
2008	## @todo implement me!!
2009
2010	if sVer1 == sVer2:
2011	return 0;
2012	if sVer1 < sVer2:
2013	return -1;
2014	return 1;
2015
2016
2017	def formatNumber(lNum, sThousandSep = ' '):
2018	"""
2019	Formats a decimal number with pretty separators.
2020	"""
2021	sNum = str(lNum);
2022	sRet = sNum[-3:];
2023	off = len(sNum) - 3;
2024	while off > 0:
2025	off -= 3;
2026	sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2027	return sRet;
2028
2029
2030	def formatNumberNbsp(lNum):
2031	"""
2032	Formats a decimal number with pretty separators.
2033	"""
2034	sRet = formatNumber(lNum);
2035	return unicode(sRet).replace(' ', u'\u00a0');
2036
2037
2038	def isString(oString):
2039	"""
2040	Checks if the object is a string object, hiding difference between python 2 and 3.
2041
2042	Returns True if it's a string of some kind.
2043	Returns False if not.
2044	"""
2045	if sys.version_info[0] >= 3:
2046	return isinstance(oString, str);
2047	return isinstance(oString, basestring); # pylint: disable=undefined-variable
2048
2049
2050	def hasNonAsciiCharacters(sText):
2051	"""
2052	Returns True is specified string has non-ASCII characters, False if ASCII only.
2053	"""
2054	if isString(sText):
2055	for ch in sText:
2056	if ord(ch) >= 128:
2057	return True;
2058	else:
2059	# Probably byte array or some such thing.
2060	for ch in sText:
2061	if ch >= 128 or ch < 0:
2062	return True;
2063	return False;
2064
2065
2066	#
2067	# Unpacking.
2068	#
2069
2070	def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2071	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2072	"""
2073	Worker for unpackFile that deals with ZIP files, same function signature.
2074	"""
2075	import zipfile
2076	if fnError is None:
2077	fnError = fnLog;
2078
2079	fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2080
2081	# Open it.
2082	try: oZipFile = zipfile.ZipFile(sArchive, 'r')
2083	except Exception as oXcpt:
2084	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2085	return None;
2086
2087	# Extract all members.
2088	asMembers = [];
2089	try:
2090	for sMember in oZipFile.namelist():
2091	if fnFilter is None or fnFilter(sMember) is not False:
2092	if sMember.endswith('/'):
2093	os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2094	else:
2095	oZipFile.extract(sMember, sDstDir);
2096	asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2097	except Exception as oXcpt:
2098	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2099	asMembers = None;
2100
2101	# close it.
2102	try: oZipFile.close();
2103	except Exception as oXcpt:
2104	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2105	asMembers = None;
2106
2107	return asMembers;
2108
2109
2110	## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2111	g_fTarCopyFileObjOverriddend = False;
2112
2113	def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2114	""" tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2115	_ = bufsize;
2116	if length is None:
2117	__myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2118	elif length > 0:
2119	cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2120	for _ in xrange(cFull):
2121	abBuffer = src.read(g_cbGoodBufferSize);
2122	dst.write(abBuffer);
2123	if len(abBuffer) != g_cbGoodBufferSize:
2124	raise exception('unexpected end of source file');
2125	if cbRemainder > 0:
2126	abBuffer = src.read(cbRemainder);
2127	dst.write(abBuffer);
2128	if len(abBuffer) != cbRemainder:
2129	raise exception('unexpected end of source file');
2130
2131
2132	def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2133	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2134	"""
2135	Worker for unpackFile that deals with tarballs, same function signature.
2136	"""
2137	import shutil;
2138	import tarfile;
2139	if fnError is None:
2140	fnError = fnLog;
2141
2142	fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2143
2144	#
2145	# Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2146	# 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2147	# 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2148	#
2149	if True is True: # pylint: disable=comparison-with-itself
2150	__installShUtilHacks(shutil);
2151	global g_fTarCopyFileObjOverriddend;
2152	if g_fTarCopyFileObjOverriddend is False:
2153	g_fTarCopyFileObjOverriddend = True;
2154	#if sys.hexversion < 0x03060000:
2155	tarfile.copyfileobj = __mytarfilecopyfileobj;
2156
2157	#
2158	# Open it.
2159	#
2160	# Note! We not using 'r:*' because we cannot allow seeking compressed files!
2161	# That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2162	#
2163	try:
2164	if sys.hexversion >= 0x03060000:
2165	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2166	else:
2167	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize);
2168	except Exception as oXcpt:
2169	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2170	return None;
2171
2172	# Extract all members.
2173	asMembers = [];
2174	try:
2175	for oTarInfo in oTarFile:
2176	try:
2177	if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2178	if oTarInfo.islnk():
2179	# Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2180	# in the compressed tar stream. So, fall back on shutil.copy2 instead.
2181	sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2182	sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2183	sParentDir = os.path.dirname(sLinkFile);
2184	try: os.unlink(sLinkFile);
2185	except: pass;
2186	if sParentDir and not os.path.exists(sParentDir):
2187	os.makedirs(sParentDir);
2188	try: os.link(sLinkTarget, sLinkFile);
2189	except: shutil.copy2(sLinkTarget, sLinkFile);
2190	else:
2191	if oTarInfo.isdir():
2192	# Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2193	oTarInfo.mode \|= 0x1c0; # (octal: 0700)
2194	oTarFile.extract(oTarInfo, sDstDir);
2195	asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2196	except Exception as oXcpt:
2197	fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2198	for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2199	fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2200	for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2201	fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2202	asMembers = None;
2203	break;
2204	except Exception as oXcpt:
2205	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2206	asMembers = None;
2207
2208	#
2209	# Finally, close it.
2210	#
2211	try: oTarFile.close();
2212	except Exception as oXcpt:
2213	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2214	asMembers = None;
2215
2216	return asMembers;
2217
2218
2219	def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2220	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2221	"""
2222	Unpacks the given file if it has a know archive extension, otherwise do
2223	nothing.
2224
2225	fnLog & fnError both take a string parameter.
2226
2227	fnFilter takes a member name (string) and returns True if it's included
2228	and False if excluded.
2229
2230	Returns list of the extracted files (full path) on success.
2231	Returns empty list if not a supported archive format.
2232	Returns None on failure. Raises no exceptions.
2233	"""
2234	sBaseNameLower = os.path.basename(sArchive).lower();
2235
2236	#
2237	# Zip file?
2238	#
2239	if sBaseNameLower.endswith('.zip'):
2240	return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2241
2242	#
2243	# Tarball?
2244	#
2245	if sBaseNameLower.endswith('.tar') \
2246	or sBaseNameLower.endswith('.tar.gz') \
2247	or sBaseNameLower.endswith('.tgz') \
2248	or sBaseNameLower.endswith('.tar.bz2'):
2249	return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2250
2251	#
2252	# Cannot classify it from the name, so just return that to the caller.
2253	#
2254	fnLog('Not unpacking "%s".' % (sArchive,));
2255	return [];
2256
2257
2258	#
2259	# Misc.
2260	#
2261	def areBytesEqual(oLeft, oRight):
2262	"""
2263	Compares two byte arrays, strings or whatnot.
2264
2265	returns true / false accordingly.
2266	"""
2267
2268	# If both are None, consider them equal (bogus?):
2269	if oLeft is None and oRight is None:
2270	return True;
2271
2272	# If just one is None, they can't match:
2273	if oLeft is None or oRight is None:
2274	return False;
2275
2276	# If both have the same type, use the compare operator of the class:
2277	if type(oLeft) is type(oRight):
2278	#print('same type: %s' % (oLeft == oRight,));
2279	return oLeft == oRight;
2280
2281	# On the offchance that they're both strings, but of different types.
2282	if isString(oLeft) and isString(oRight):
2283	#print('string compare: %s' % (oLeft == oRight,));
2284	return oLeft == oRight;
2285
2286	#
2287	# See if byte/buffer stuff that can be compared directory. If not convert
2288	# strings to bytes.
2289	#
2290	# Note! For 2.x, we must convert both sides to the buffer type or the
2291	# comparison may fail despite it working okay in test cases.
2292	#
2293	if sys.version_info[0] >= 3:
2294	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2295	return oLeft == oRight;
2296
2297	if isString(oLeft):
2298	try: oLeft = bytes(oLeft, 'utf-8');
2299	except: pass;
2300	if isString(oRight):
2301	try: oRight = bytes(oRight, 'utf-8');
2302	except: pass;
2303	else:
2304	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2305	if isinstance(oLeft, bytearray):
2306	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2307	else:
2308	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2309	#print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2310	return oLeft == oRight;
2311
2312	if isString(oLeft):
2313	try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2314	except: pass;
2315	if isString(oRight):
2316	try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2317	except: pass;
2318
2319	# Check if we now have the same type for both:
2320	if type(oLeft) is type(oRight):
2321	#print('same type now: %s' % (oLeft == oRight,));
2322	return oLeft == oRight;
2323
2324	# Check if we now have buffer/memoryview vs bytes/bytesarray again.
2325	if sys.version_info[0] >= 3:
2326	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2327	return oLeft == oRight;
2328	else:
2329	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2330	if isinstance(oLeft, bytearray):
2331	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2332	else:
2333	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2334	#print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2335	return oLeft == oRight;
2336
2337	# Do item by item comparison:
2338	if len(oLeft) != len(oRight):
2339	#print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2340	return False;
2341	i = len(oLeft);
2342	while i > 0:
2343	i = i - 1;
2344
2345	iElmLeft = oLeft[i];
2346	if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2347	iElmLeft = ord(iElmLeft);
2348
2349	iElmRight = oRight[i];
2350	if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2351	iElmRight = ord(iElmRight);
2352
2353	if iElmLeft != iElmRight:
2354	#print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2355	return False;
2356	return True;
2357
2358
2359	#
2360	# Unit testing.
2361	#
2362
2363	# pylint: disable=missing-docstring
2364	# pylint: disable=undefined-variable
2365	class BuildCategoryDataTestCase(unittest.TestCase):
2366	def testIntervalSeconds(self):
2367	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2368	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2369	self.assertEqual(parseIntervalSeconds('123'), (123, None));
2370	self.assertEqual(parseIntervalSeconds(123), (123, None));
2371	self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2372	self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2373	self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2374	self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2375	self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2376	self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2377	self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2378
2379	def testZuluNormalization(self):
2380	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2381	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2382	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2383	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2384	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2385
2386	def testHasNonAsciiChars(self):
2387	self.assertEqual(hasNonAsciiCharacters(''), False);
2388	self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB&$%&)@#(!)@(#!(#$&#$&%*Y@#$IQWN---00;'), False);
2389	self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2390	self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2391	self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2392	self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2393	self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2394	self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2395	self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2396	self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2397
2398	def testAreBytesEqual(self):
2399	self.assertEqual(areBytesEqual(None, None), True);
2400	self.assertEqual(areBytesEqual(None, ''), False);
2401	self.assertEqual(areBytesEqual('', ''), True);
2402	self.assertEqual(areBytesEqual('1', '1'), True);
2403	self.assertEqual(areBytesEqual('12345', '1234'), False);
2404	self.assertEqual(areBytesEqual('1234', '1234'), True);
2405	self.assertEqual(areBytesEqual('1234', b'1234'), True);
2406	self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2407	self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2408	self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2409	self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2410	self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2411	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2412	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2413	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2414	if sys.version_info[0] >= 3:
2415	pass;
2416	else:
2417	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2418	bytearray([0x31,0x32,0x33,0x34])), True);
2419	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2420	bytearray([0x99,0x32,0x32,0x34])), False);
2421	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2422	buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2423	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2424	buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2425	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2426	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2427	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2428
2429	if __name__ == '__main__':
2430	unittest.main();
2431	# not reached.
2432

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 83585

Download in other formats: