utils.py@ 83357

Last change on this file since 83357 was 83357, checked in by vboxsync, 5 years ago
common/utils.py: Added a few functions of parsing and managing iso timestamps.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 81.6 KB

Line
1	# -- coding: utf-8 --
2	# $Id: utils.py 83357 2020-03-21 13:01:30Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Common Utility Functions.
7	"""
8
9	from __future__ import print_function;
10
11	__copyright__ = \
12	"""
13	Copyright (C) 2012-2020 Oracle Corporation
14
15	This file is part of VirtualBox Open Source Edition (OSE), as
16	available from http://www.virtualbox.org. This file is free software;
17	you can redistribute it and/or modify it under the terms of the GNU
18	General Public License (GPL) as published by the Free Software
19	Foundation, in version 2 as it comes in the "COPYING" file of the
20	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23	The contents of this file may alternatively be used under the terms
24	of the Common Development and Distribution License Version 1.0
25	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26	VirtualBox OSE distribution, in which case the provisions of the
27	CDDL are applicable instead of those of the GPL.
28
29	You may elect to license modified versions of this file under the
30	terms and conditions of either the GPL or the CDDL or both.
31	"""
32	__version__ = "$Revision: 83357 $"
33
34
35	# Standard Python imports.
36	import datetime;
37	import os;
38	import platform;
39	import re;
40	import stat;
41	import subprocess;
42	import sys;
43	import time;
44	import traceback;
45	import unittest;
46
47	if sys.platform == 'win32':
48	import ctypes;
49	import msvcrt; # pylint: disable=import-error
50	import win32api; # pylint: disable=import-error
51	import win32con; # pylint: disable=import-error
52	import win32console; # pylint: disable=import-error
53	import win32file; # pylint: disable=import-error
54	import win32process; # pylint: disable=import-error
55	import winerror; # pylint: disable=import-error
56	import pywintypes; # pylint: disable=import-error
57	else:
58	import signal;
59
60	# Python 3 hacks:
61	if sys.version_info[0] >= 3:
62	unicode = str; # pylint: disable=redefined-builtin,invalid-name
63	xrange = range; # pylint: disable=redefined-builtin,invalid-name
64	long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67	#
68	# Strings.
69	#
70
71	def toUnicode(sString, encoding = None, errors = 'strict'):
72	"""
73	A little like the python 2 unicode() function.
74	"""
75	if sys.version_info[0] >= 3:
76	if isinstance(sString, bytes):
77	return str(sString, encoding if encoding else 'utf-8', errors);
78	else:
79	if not isinstance(sString, unicode):
80	return unicode(sString, encoding if encoding else 'utf-8', errors);
81	return sString;
82
83
84
85	#
86	# Output.
87	#
88
89	def printOut(sString):
90	"""
91	Outputs a string to standard output, dealing with python 2.x encoding stupidity.
92	"""
93	sStreamEncoding = sys.stdout.encoding;
94	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
95	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
96	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
97	print(sString);
98	else:
99	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
100
101	def printErr(sString):
102	"""
103	Outputs a string to standard error, dealing with python 2.x encoding stupidity.
104	"""
105	sStreamEncoding = sys.stderr.encoding;
106	if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
107	sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
108	if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
109	print(sString, file = sys.stderr);
110	else:
111	print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
112
113
114	#
115	# Host OS and CPU.
116	#
117
118	def getHostOs():
119	"""
120	Gets the host OS name (short).
121
122	See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
123	"""
124	sPlatform = platform.system();
125	if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
126	sPlatform = sPlatform.lower();
127	elif sPlatform == 'Windows':
128	sPlatform = 'win';
129	elif sPlatform == 'SunOS':
130	sPlatform = 'solaris';
131	else:
132	raise Exception('Unsupported platform "%s"' % (sPlatform,));
133	return sPlatform;
134
135	g_sHostArch = None;
136
137	def getHostArch():
138	"""
139	Gets the host CPU architecture.
140
141	See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
142	"""
143	global g_sHostArch;
144	if g_sHostArch is None:
145	sArch = platform.machine();
146	if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
147	sArch = 'x86';
148	elif sArch in ('AMD64', 'amd64', 'x86_64'):
149	sArch = 'amd64';
150	elif sArch == 'i86pc': # SunOS
151	if platform.architecture()[0] == '64bit':
152	sArch = 'amd64';
153	else:
154	try:
155	sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
156	except:
157	pass;
158	sArch = sArch.strip();
159	if sArch != 'amd64':
160	sArch = 'x86';
161	else:
162	raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
163	g_sHostArch = sArch;
164	return g_sHostArch;
165
166
167	def getHostOsDotArch():
168	"""
169	Gets the 'os.arch' for the host.
170	"""
171	return '%s.%s' % (getHostOs(), getHostArch());
172
173
174	def isValidOs(sOs):
175	"""
176	Validates the OS name.
177	"""
178	if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
179	'os2', 'solaris', 'win', 'os-agnostic'):
180	return True;
181	return False;
182
183
184	def isValidArch(sArch):
185	"""
186	Validates the CPU architecture name.
187	"""
188	if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
189	'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
190	return True;
191	return False;
192
193	def isValidOsDotArch(sOsDotArch):
194	"""
195	Validates the 'os.arch' string.
196	"""
197
198	asParts = sOsDotArch.split('.');
199	if asParts.length() != 2:
200	return False;
201	return isValidOs(asParts[0]) \
202	and isValidArch(asParts[1]);
203
204	def getHostOsVersion():
205	"""
206	Returns the host OS version. This is platform.release with additional
207	distro indicator on linux.
208	"""
209	sVersion = platform.release();
210	sOs = getHostOs();
211	if sOs == 'linux':
212	sDist = '';
213	try:
214	# try /etc/lsb-release first to distinguish between Debian and Ubuntu
215	oFile = open('/etc/lsb-release');
216	for sLine in oFile:
217	oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s=)\s"(.)"', sLine);
218	if oMatch is not None:
219	sDist = oMatch.group(1).strip();
220	except:
221	pass;
222	if sDist:
223	sVersion += ' / ' + sDist;
224	else:
225	asFiles = \
226	[
227	[ '/etc/debian_version', 'Debian v'],
228	[ '/etc/gentoo-release', '' ],
229	[ '/etc/oracle-release', '' ],
230	[ '/etc/redhat-release', '' ],
231	[ '/etc/SuSE-release', '' ],
232	];
233	for sFile, sPrefix in asFiles:
234	if os.path.isfile(sFile):
235	try:
236	oFile = open(sFile);
237	sLine = oFile.readline();
238	oFile.close();
239	except:
240	continue;
241	sLine = sLine.strip()
242	if sLine:
243	sVersion += ' / ' + sPrefix + sLine;
244	break;
245
246	elif sOs == 'solaris':
247	sVersion = platform.version();
248	if os.path.isfile('/etc/release'):
249	try:
250	oFile = open('/etc/release');
251	sLast = oFile.readlines()[-1];
252	oFile.close();
253	sLast = sLast.strip();
254	if sLast:
255	sVersion += ' (' + sLast + ')';
256	except:
257	pass;
258
259	elif sOs == 'darwin':
260	sOsxVersion = platform.mac_ver()[0];
261	codenames = {"4": "Tiger",
262	"5": "Leopard",
263	"6": "Snow Leopard",
264	"7": "Lion",
265	"8": "Mountain Lion",
266	"9": "Mavericks",
267	"10": "Yosemite",
268	"11": "El Capitan",
269	"12": "Sierra",
270	"13": "High Sierra",
271	"14": "Unknown 14", }
272	sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
273
274	elif sOs == 'win':
275	class OSVersionInfoEx(ctypes.Structure):
276	""" OSVERSIONEX """
277	kaFields = [
278	('dwOSVersionInfoSize', ctypes.c_ulong),
279	('dwMajorVersion', ctypes.c_ulong),
280	('dwMinorVersion', ctypes.c_ulong),
281	('dwBuildNumber', ctypes.c_ulong),
282	('dwPlatformId', ctypes.c_ulong),
283	('szCSDVersion', ctypes.c_wchar*128),
284	('wServicePackMajor', ctypes.c_ushort),
285	('wServicePackMinor', ctypes.c_ushort),
286	('wSuiteMask', ctypes.c_ushort),
287	('wProductType', ctypes.c_byte),
288	('wReserved', ctypes.c_byte)]
289	_fields_ = kaFields # pylint: disable=invalid-name
290
291	def __init__(self):
292	super(OSVersionInfoEx, self).__init__()
293	self.dwOSVersionInfoSize = ctypes.sizeof(self)
294
295	oOsVersion = OSVersionInfoEx()
296	rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
297	if rc == 0:
298	# Python platform.release() is not reliable for newer server releases
299	if oOsVersion.wProductType != 1:
300	if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
301	sVersion = '2016Server';
302	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
303	sVersion = '2012ServerR2';
304	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
305	sVersion = '2012Server';
306	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
307	sVersion = '2008ServerR2';
308	elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
309	sVersion = '2008Server';
310	elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
311	sVersion = '2003Server';
312	sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
313	if oOsVersion.wServicePackMajor:
314	sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
315	if oOsVersion.wServicePackMinor:
316	sVersion += '.' + str(oOsVersion.wServicePackMinor)
317
318	return sVersion;
319
320	def getPresentCpuCount():
321	"""
322	Gets the number of CPUs present in the system.
323
324	This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
325	that we return the active count rather than the maximum count. If we don't,
326	we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
327	never will have.
328
329	@todo This is probably not exactly what we get on non-windows...
330	"""
331
332	if getHostOs() == 'win':
333	fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
334	if fnGetActiveProcessorCount:
335	cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
336	if cCpus > 0:
337	return cCpus;
338
339	import multiprocessing
340	return multiprocessing.cpu_count();
341
342
343	#
344	# File system.
345	#
346
347	def openNoInherit(sFile, sMode = 'r'):
348	"""
349	Wrapper around open() that tries it's best to make sure the file isn't
350	inherited by child processes.
351
352	This is a best effort thing at the moment as it doesn't synchronizes with
353	child process spawning in any way. Thus it can be subject to races in
354	multithreaded programs.
355	"""
356
357	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
358	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
359	if uPythonVer >= ((3 << 16) \| 4):
360	oFile = open(sFile, sMode);
361	else:
362	try:
363	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
364	except:
365	# On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
366	if getHostOs() == 'win':
367	if uPythonVer < (3 << 16):
368	offComma = sMode.find(',');
369	if offComma < 0:
370	return open(sFile, sMode + 'N');
371	return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
372
373	# Just in case.
374	return open(sFile, sMode);
375
376	oFile = open(sFile, sMode);
377	#try:
378	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
379	#except:
380	# pass;
381	return oFile;
382
383	def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
384	"""
385	Wrapper around open() that tries it's best to make sure the file isn't
386	inherited by child processes.
387
388	This is a best effort thing at the moment as it doesn't synchronizes with
389	child process spawning in any way. Thus it can be subject to races in
390	multithreaded programs.
391	"""
392
393	if getHostOs() == 'win':
394	# Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
395	# pylint: disable=no-member,c-extension-no-member
396	fAccess = 0;
397	fDisposition = win32file.OPEN_EXISTING;
398	if 'r' in sMode or '+' in sMode:
399	fAccess \|= win32file.GENERIC_READ;
400	if 'a' in sMode:
401	fAccess \|= win32file.GENERIC_WRITE;
402	fDisposition = win32file.OPEN_ALWAYS;
403	elif 'w' in sMode:
404	fAccess = win32file.GENERIC_WRITE;
405	if '+' in sMode:
406	fDisposition = win32file.OPEN_ALWAYS;
407	fAccess \|= win32file.GENERIC_READ;
408	else:
409	fDisposition = win32file.CREATE_ALWAYS;
410	if not fAccess:
411	fAccess \|= win32file.GENERIC_READ;
412	fSharing = (win32file.FILE_SHARE_READ \| win32file.FILE_SHARE_WRITE
413	\| win32file.FILE_SHARE_DELETE);
414	hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
415	if 'a' in sMode:
416	win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
417
418	# Turn the NT handle into a CRT file descriptor.
419	hDetachedFile = hFile.Detach();
420	if fAccess == win32file.GENERIC_READ:
421	fOpen = os.O_RDONLY;
422	elif fAccess == win32file.GENERIC_WRITE:
423	fOpen = os.O_WRONLY;
424	else:
425	fOpen = os.O_RDWR;
426	# pulint: enable=no-member,c-extension-no-member
427	if 'a' in sMode:
428	fOpen \|= os.O_APPEND;
429	if 'b' in sMode or 't' in sMode:
430	fOpen \|= os.O_TEXT; # pylint: disable=no-member
431	fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
432
433	# Tell python to use this handle.
434	oFile = os.fdopen(fdFile, sMode);
435	else:
436	oFile = open(sFile, sMode);
437
438	# Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
439	uPythonVer = (sys.version_info[0] << 16) \| (sys.version_info[1] & 0xffff);
440	if uPythonVer < ((3 << 16) \| 4):
441	try:
442	from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
443	except:
444	pass;
445	else:
446	fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) \| FD_CLOEXEC);
447	return oFile;
448
449	def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
450	"""
451	No exceptions os.readlink wrapper.
452	"""
453	try:
454	sRet = os.readlink(sPath); # pylint: disable=no-member
455	except:
456	return sXcptRet;
457	if hasattr(sRet, 'decode'):
458	sRet = sRet.decode(sEncoding, 'ignore');
459	return sRet;
460
461	def readFile(sFile, sMode = 'rb'):
462	"""
463	Reads the entire file.
464	"""
465	oFile = open(sFile, sMode);
466	sRet = oFile.read();
467	oFile.close();
468	return sRet;
469
470	def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
471	"""
472	No exceptions common.readFile wrapper.
473	"""
474	try:
475	sRet = readFile(sFile, sMode);
476	except:
477	sRet = sXcptRet;
478	if sEncoding is not None and hasattr(sRet, 'decode'):
479	sRet = sRet.decode(sEncoding, 'ignore');
480	return sRet;
481
482	def noxcptRmDir(sDir, oXcptRet = False):
483	"""
484	No exceptions os.rmdir wrapper.
485	"""
486	oRet = True;
487	try:
488	os.rmdir(sDir);
489	except:
490	oRet = oXcptRet;
491	return oRet;
492
493	def noxcptDeleteFile(sFile, oXcptRet = False):
494	"""
495	No exceptions os.remove wrapper.
496	"""
497	oRet = True;
498	try:
499	os.remove(sFile);
500	except:
501	oRet = oXcptRet;
502	return oRet;
503
504
505	def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
506	# type: (string, (string, stat) -> bool) -> bool
507	"""
508	Recursively walks a directory tree, calling fnCallback for each.
509
510	fnCallback takes a full path and stat object (can be None). It
511	returns a boolean value, False stops walking and returns immediately.
512
513	Returns True or False depending on fnCallback.
514	Returns None fIgnoreExceptions is True and an exception was raised by listdir.
515	"""
516	def __worker(sCurDir):
517	""" Worker for """
518	try:
519	asNames = os.listdir(sCurDir);
520	except:
521	if not fIgnoreExceptions:
522	raise;
523	return None;
524	rc = True;
525	for sName in asNames:
526	if sName not in [ '.', '..' ]:
527	sFullName = os.path.join(sCurDir, sName);
528	try: oStat = os.lstat(sFullName);
529	except: oStat = None;
530	if fnCallback(sFullName, oStat) is False:
531	return False;
532	if oStat is not None and stat.S_ISDIR(oStat.st_mode):
533	rc = __worker(sFullName);
534	if rc is False:
535	break;
536	return rc;
537
538	# Ensure unicode path here so listdir also returns unicode on windows.
539	## @todo figure out unicode stuff on non-windows.
540	if sys.platform == 'win32':
541	sDir = unicode(sDir);
542	return __worker(sDir);
543
544
545
546	def formatFileMode(uMode):
547	# type: (int) -> string
548	"""
549	Format a st_mode value 'ls -la' fasion.
550	Returns string.
551	"""
552	if stat.S_ISDIR(uMode): sMode = 'd';
553	elif stat.S_ISREG(uMode): sMode = '-';
554	elif stat.S_ISLNK(uMode): sMode = 'l';
555	elif stat.S_ISFIFO(uMode): sMode = 'p';
556	elif stat.S_ISCHR(uMode): sMode = 'c';
557	elif stat.S_ISBLK(uMode): sMode = 'b';
558	elif stat.S_ISSOCK(uMode): sMode = 's';
559	else: sMode = '?';
560	## @todo sticky bits.
561	sMode += 'r' if uMode & stat.S_IRUSR else '-';
562	sMode += 'w' if uMode & stat.S_IWUSR else '-';
563	sMode += 'x' if uMode & stat.S_IXUSR else '-';
564	sMode += 'r' if uMode & stat.S_IRGRP else '-';
565	sMode += 'w' if uMode & stat.S_IWGRP else '-';
566	sMode += 'x' if uMode & stat.S_IXGRP else '-';
567	sMode += 'r' if uMode & stat.S_IROTH else '-';
568	sMode += 'w' if uMode & stat.S_IWOTH else '-';
569	sMode += 'x' if uMode & stat.S_IXOTH else '-';
570	sMode += ' ';
571	return sMode;
572
573
574	def formatFileStat(oStat):
575	# type: (stat) -> string
576	"""
577	Format a stat result 'ls -la' fasion (numeric IDs).
578	Returns string.
579	"""
580	return '%s %3s %4s %4s %10s %s' \
581	% (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
582	time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
583
584	## Good buffer for file operations.
585	g_cbGoodBufferSize = 256*1024;
586
587	## The original shutil.copyfileobj.
588	g_fnOriginalShCopyFileObj = None;
589
590	def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
591	""" shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
592	return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
593
594	def __installShUtilHacks(shutil):
595	""" Installs the shutil buffer size hacks. """
596	global g_fnOriginalShCopyFileObj;
597	if g_fnOriginalShCopyFileObj is None:
598	g_fnOriginalShCopyFileObj = shutil.copyfileobj;
599	shutil.copyfileobj = __myshutilcopyfileobj;
600	return True;
601
602
603	def copyFileSimple(sFileSrc, sFileDst):
604	"""
605	Wrapper around shutil.copyfile that simply copies the data of a regular file.
606	Raises exception on failure.
607	Return True for show.
608	"""
609	import shutil;
610	__installShUtilHacks(shutil);
611	return shutil.copyfile(sFileSrc, sFileDst);
612
613
614	def getDiskUsage(sPath):
615	"""
616	Get free space of a partition that corresponds to specified sPath in MB.
617
618	Returns partition free space value in MB.
619	"""
620	if platform.system() == 'Windows':
621	oCTypeFreeSpace = ctypes.c_ulonglong(0);
622	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
623	ctypes.pointer(oCTypeFreeSpace));
624	cbFreeSpace = oCTypeFreeSpace.value;
625	else:
626	oStats = os.statvfs(sPath); # pylint: disable=no-member
627	cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
628
629	# Convert to MB
630	cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
631
632	return cMbFreeSpace;
633
634
635
636	#
637	# SubProcess.
638	#
639
640	def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
641	"""
642	If the "executable" is a python script, insert the python interpreter at
643	the head of the argument list so that it will work on systems which doesn't
644	support hash-bang scripts.
645	"""
646
647	asArgs = dKeywordArgs.get('args');
648	if asArgs is None:
649	asArgs = aPositionalArgs[0];
650
651	if asArgs[0].endswith('.py'):
652	if sys.executable:
653	asArgs.insert(0, sys.executable);
654	else:
655	asArgs.insert(0, 'python');
656
657	# paranoia...
658	if dKeywordArgs.get('args') is not None:
659	dKeywordArgs['args'] = asArgs;
660	else:
661	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
662	return None;
663
664	def processPopenSafe(aPositionalArgs, *dKeywordArgs):
665	"""
666	Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
667	"""
668	if getHostOs() == 'win':
669	if dKeywordArgs.get('creationflags', 0) == 0:
670	dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
671	return subprocess.Popen(aPositionalArgs, *dKeywordArgs);
672
673	def processCall(aPositionalArgs, *dKeywordArgs):
674	"""
675	Wrapper around subprocess.call to deal with its absence in older
676	python versions.
677	Returns process exit code (see subprocess.poll).
678	"""
679	assert dKeywordArgs.get('stdout') is None;
680	assert dKeywordArgs.get('stderr') is None;
681	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
682	oProcess = processPopenSafe(aPositionalArgs, *dKeywordArgs);
683	return oProcess.wait();
684
685	def processOutputChecked(aPositionalArgs, *dKeywordArgs):
686	"""
687	Wrapper around subprocess.check_output to deal with its absense in older
688	python versions.
689
690	Extra keywords for specifying now output is to be decoded:
691	sEncoding='utf-8
692	fIgnoreEncoding=True/False
693	"""
694	sEncoding = dKeywordArgs.get('sEncoding');
695	if sEncoding is not None: del dKeywordArgs['sEncoding'];
696	else: sEncoding = 'utf-8';
697
698	fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
699	if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
700	else: fIgnoreEncoding = True;
701
702	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
703	oProcess = processPopenSafe(stdout=subprocess.PIPE, aPositionalArgs, *dKeywordArgs);
704
705	sOutput, _ = oProcess.communicate();
706	iExitCode = oProcess.poll();
707
708	if iExitCode != 0:
709	asArgs = dKeywordArgs.get('args');
710	if asArgs is None:
711	asArgs = aPositionalArgs[0];
712	print(sOutput);
713	raise subprocess.CalledProcessError(iExitCode, asArgs);
714
715	if hasattr(sOutput, 'decode'):
716	sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
717	return sOutput;
718
719	g_fOldSudo = None;
720	def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
721	"""
722	Adds 'sudo' (or similar) to the args parameter, whereever it is.
723	"""
724
725	# Are we root?
726	fIsRoot = True;
727	try:
728	fIsRoot = os.getuid() == 0; # pylint: disable=no-member
729	except:
730	pass;
731
732	# If not, prepend sudo (non-interactive, simulate initial login).
733	if fIsRoot is not True:
734	asArgs = dKeywordArgs.get('args');
735	if asArgs is None:
736	asArgs = aPositionalArgs[0];
737
738	# Detect old sudo.
739	global g_fOldSudo;
740	if g_fOldSudo is None:
741	try:
742	sVersion = str(processOutputChecked(['sudo', '-V']));
743	except:
744	sVersion = '1.7.0';
745	sVersion = sVersion.strip().split('\n')[0];
746	sVersion = sVersion.replace('Sudo version', '').strip();
747	g_fOldSudo = len(sVersion) >= 4 \
748	and sVersion[0] == '1' \
749	and sVersion[1] == '.' \
750	and sVersion[2] <= '6' \
751	and sVersion[3] == '.';
752
753	asArgs.insert(0, 'sudo');
754	if not g_fOldSudo:
755	asArgs.insert(1, '-n');
756	if fInitialEnv and not g_fOldSudo:
757	asArgs.insert(1, '-i');
758
759	# paranoia...
760	if dKeywordArgs.get('args') is not None:
761	dKeywordArgs['args'] = asArgs;
762	else:
763	aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
764	return None;
765
766
767	def sudoProcessCall(aPositionalArgs, *dKeywordArgs):
768	"""
769	sudo (or similar) + subprocess.call
770	"""
771	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
772	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
773	return processCall(aPositionalArgs, *dKeywordArgs);
774
775	def sudoProcessOutputChecked(aPositionalArgs, *dKeywordArgs):
776	"""
777	sudo (or similar) + subprocess.check_output.
778	"""
779	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
780	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
781	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
782
783	def sudoProcessOutputCheckedNoI(aPositionalArgs, *dKeywordArgs):
784	"""
785	sudo (or similar) + subprocess.check_output, except '-i' isn't used.
786	"""
787	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
788	_sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
789	return processOutputChecked(aPositionalArgs, *dKeywordArgs);
790
791	def sudoProcessPopen(aPositionalArgs, *dKeywordArgs):
792	"""
793	sudo (or similar) + processPopenSafe.
794	"""
795	_processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
796	_sudoFixArguments(aPositionalArgs, dKeywordArgs);
797	return processPopenSafe(aPositionalArgs, *dKeywordArgs);
798
799
800	#
801	# Generic process stuff.
802	#
803
804	def processInterrupt(uPid):
805	"""
806	Sends a SIGINT or equivalent to interrupt the specified process.
807	Returns True on success, False on failure.
808
809	On Windows hosts this may not work unless the process happens to be a
810	process group leader.
811	"""
812	if sys.platform == 'win32':
813	try:
814	win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
815	uPid);
816	fRc = True;
817	except:
818	fRc = False;
819	else:
820	try:
821	os.kill(uPid, signal.SIGINT);
822	fRc = True;
823	except:
824	fRc = False;
825	return fRc;
826
827	def sendUserSignal1(uPid):
828	"""
829	Sends a SIGUSR1 or equivalent to nudge the process into shutting down
830	(VBoxSVC) or something.
831	Returns True on success, False on failure or if not supported (win).
832
833	On Windows hosts this may not work unless the process happens to be a
834	process group leader.
835	"""
836	if sys.platform == 'win32':
837	fRc = False;
838	else:
839	try:
840	os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
841	fRc = True;
842	except:
843	fRc = False;
844	return fRc;
845
846	def processTerminate(uPid):
847	"""
848	Terminates the process in a nice manner (SIGTERM or equivalent).
849	Returns True on success, False on failure.
850	"""
851	fRc = False;
852	if sys.platform == 'win32':
853	try:
854	hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
855	False, uPid);
856	except:
857	pass;
858	else:
859	try:
860	win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
861	0x40010004); # DBG_TERMINATE_PROCESS
862	fRc = True;
863	except:
864	pass;
865	hProcess.Close(); #win32api.CloseHandle(hProcess)
866	else:
867	try:
868	os.kill(uPid, signal.SIGTERM);
869	fRc = True;
870	except:
871	pass;
872	return fRc;
873
874	def processKill(uPid):
875	"""
876	Terminates the process with extreme prejudice (SIGKILL).
877	Returns True on success, False on failure.
878	"""
879	if sys.platform == 'win32':
880	fRc = processTerminate(uPid);
881	else:
882	try:
883	os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
884	fRc = True;
885	except:
886	fRc = False;
887	return fRc;
888
889	def processKillWithNameCheck(uPid, sName):
890	"""
891	Like processKill(), but checks if the process name matches before killing
892	it. This is intended for killing using potentially stale pid values.
893
894	Returns True on success, False on failure.
895	"""
896
897	if processCheckPidAndName(uPid, sName) is not True:
898	return False;
899	return processKill(uPid);
900
901
902	def processExists(uPid):
903	"""
904	Checks if the specified process exits.
905	This will only work if we can signal/open the process.
906
907	Returns True if it positively exists, False otherwise.
908	"""
909	if sys.platform == 'win32':
910	fRc = False;
911	# We try open the process for waiting since this is generally only forbidden in a very few cases.
912	try:
913	hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
914	False, uPid);
915	except pywintypes.error as oXcpt: # pylint: disable=no-member
916	if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
917	fRc = True;
918	except Exception as oXcpt:
919	pass;
920	else:
921	hProcess.Close();
922	fRc = True;
923	else:
924	try:
925	os.kill(uPid, 0);
926	fRc = True;
927	except: ## @todo check error code.
928	fRc = False;
929	return fRc;
930
931	def processCheckPidAndName(uPid, sName):
932	"""
933	Checks if a process PID and NAME matches.
934	"""
935	fRc = processExists(uPid);
936	if fRc is not True:
937	return False;
938
939	if sys.platform == 'win32':
940	try:
941	from win32com.client import GetObject; # pylint: disable=import-error
942	oWmi = GetObject('winmgmts:');
943	aoProcesses = oWmi.InstancesOf('Win32_Process');
944	for oProcess in aoProcesses:
945	if long(oProcess.Properties_("ProcessId").Value) == uPid:
946	sCurName = oProcess.Properties_("Name").Value;
947	#reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
948	sName = sName.lower();
949	sCurName = sCurName.lower();
950	if os.path.basename(sName) == sName:
951	sCurName = os.path.basename(sCurName);
952
953	if sCurName == sName \
954	or sCurName + '.exe' == sName \
955	or sCurName == sName + '.exe':
956	fRc = True;
957	break;
958	except:
959	#reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
960	pass;
961	else:
962	if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
963	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
964	elif sys.platform in ('sunos5',):
965	asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
966	elif sys.platform in ('darwin',):
967	asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
968	else:
969	asPsCmd = None;
970
971	if asPsCmd is not None:
972	try:
973	oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
974	sCurName = oPs.communicate()[0];
975	iExitCode = oPs.wait();
976	except:
977	#reporter.logXcpt();
978	return False;
979
980	# ps fails with non-zero exit code if the pid wasn't found.
981	if iExitCode != 0:
982	return False;
983	if sCurName is None:
984	return False;
985	sCurName = sCurName.strip();
986	if not sCurName:
987	return False;
988
989	if os.path.basename(sName) == sName:
990	sCurName = os.path.basename(sCurName);
991	elif os.path.basename(sCurName) == sCurName:
992	sName = os.path.basename(sName);
993
994	if sCurName != sName:
995	return False;
996
997	fRc = True;
998	return fRc;
999
1000	def processGetInfo(uPid, fSudo = False):
1001	"""
1002	Tries to acquire state information of the given process.
1003
1004	Returns a string with the information on success or None on failure or
1005	if the host is not supported.
1006
1007	Note that the format of the information is host system dependent and will
1008	likely differ much between different hosts.
1009	"""
1010	fRc = processExists(uPid);
1011	if fRc is not True:
1012	return None;
1013
1014	sHostOs = getHostOs();
1015	if sHostOs in [ 'linux',]:
1016	sGdb = '/usr/bin/gdb';
1017	if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1018	if not os.path.isfile(sGdb): sGdb = 'gdb';
1019	aasCmd = [
1020	[ sGdb, '-batch',
1021	'-ex', 'set pagination off',
1022	'-ex', 'thread apply all bt',
1023	'-ex', 'info proc mapping',
1024	'-ex', 'info sharedlibrary',
1025	'-p', '%u' % (uPid,), ],
1026	];
1027	elif sHostOs == 'darwin':
1028	# LLDB doesn't work in batch mode when attaching to a process, at least
1029	# with macOS Sierra (10.12). GDB might not be installed. Use the sample
1030	# tool instead with a 1 second duration and 1000ms sampling interval to
1031	# get one stack trace. For the process mappings use vmmap.
1032	aasCmd = [
1033	[ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1034	[ '/usr/bin/vmmap', '%u' % (uPid,), ],
1035	];
1036	elif sHostOs == 'solaris':
1037	aasCmd = [
1038	[ '/usr/bin/pstack', '%u' % (uPid,), ],
1039	[ '/usr/bin/pmap', '%u' % (uPid,), ],
1040	];
1041	else:
1042	aasCmd = [];
1043
1044	sInfo = '';
1045	for asCmd in aasCmd:
1046	try:
1047	if fSudo:
1048	sThisInfo = sudoProcessOutputChecked(asCmd);
1049	else:
1050	sThisInfo = processOutputChecked(asCmd);
1051	if sThisInfo is not None:
1052	sInfo += sThisInfo;
1053	except:
1054	pass;
1055	if not sInfo:
1056	sInfo = None;
1057
1058	return sInfo;
1059
1060
1061	class ProcessInfo(object):
1062	"""Process info."""
1063	def __init__(self, iPid):
1064	self.iPid = iPid;
1065	self.iParentPid = None;
1066	self.sImage = None;
1067	self.sName = None;
1068	self.asArgs = None;
1069	self.sCwd = None;
1070	self.iGid = None;
1071	self.iUid = None;
1072	self.iProcGroup = None;
1073	self.iSessionId = None;
1074
1075	def loadAll(self):
1076	"""Load all the info."""
1077	sOs = getHostOs();
1078	if sOs == 'linux':
1079	sProc = '/proc/%s/' % (self.iPid,);
1080	if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1081	if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1082	if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1083	#elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1084	# sProc = '/proc/%s/' % (self.iPid,);
1085	# if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1086	# if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1087	else:
1088	pass;
1089	if self.sName is None and self.sImage is not None:
1090	self.sName = self.sImage;
1091
1092	def windowsGrabProcessInfo(self, oProcess):
1093	"""Windows specific loadAll."""
1094	try: self.sName = oProcess.Properties_("Name").Value;
1095	except: pass;
1096	try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1097	except: pass;
1098	try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1099	except: pass;
1100	try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1101	except: pass;
1102	try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1103	except: pass;
1104	if self.sName is None and self.sImage is not None:
1105	self.sName = self.sImage;
1106
1107	def getBaseImageName(self):
1108	"""
1109	Gets the base image name if available, use the process name if not available.
1110	Returns image/process base name or None.
1111	"""
1112	sRet = self.sImage if self.sName is None else self.sName;
1113	if sRet is None:
1114	self.loadAll();
1115	sRet = self.sImage if self.sName is None else self.sName;
1116	if sRet is None:
1117	if not self.asArgs:
1118	return None;
1119	sRet = self.asArgs[0];
1120	if not sRet:
1121	return None;
1122	return os.path.basename(sRet);
1123
1124	def getBaseImageNameNoExeSuff(self):
1125	"""
1126	Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1127	"""
1128	sRet = self.getBaseImageName();
1129	if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1130	if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1131	sRet = sRet[:-4];
1132	return sRet;
1133
1134
1135	def processListAll():
1136	"""
1137	Return a list of ProcessInfo objects for all the processes in the system
1138	that the current user can see.
1139	"""
1140	asProcesses = [];
1141
1142	sOs = getHostOs();
1143	if sOs == 'win':
1144	from win32com.client import GetObject; # pylint: disable=import-error
1145	oWmi = GetObject('winmgmts:');
1146	aoProcesses = oWmi.InstancesOf('Win32_Process');
1147	for oProcess in aoProcesses:
1148	try:
1149	iPid = int(oProcess.Properties_("ProcessId").Value);
1150	except:
1151	continue;
1152	oMyInfo = ProcessInfo(iPid);
1153	oMyInfo.windowsGrabProcessInfo(oProcess);
1154	asProcesses.append(oMyInfo);
1155	return asProcesses;
1156
1157	if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1158	try:
1159	asDirs = os.listdir('/proc');
1160	except:
1161	asDirs = [];
1162	for sDir in asDirs:
1163	if sDir.isdigit():
1164	asProcesses.append(ProcessInfo(int(sDir),));
1165	return asProcesses;
1166
1167	#
1168	# The other OSes parses the output from the 'ps' utility.
1169	#
1170	asPsCmd = [
1171	'/bin/ps', # 0
1172	'-A', # 1
1173	'-o', 'pid=', # 2,3
1174	'-o', 'ppid=', # 4,5
1175	'-o', 'pgid=', # 6,7
1176	'-o', 'sid=', # 8,9
1177	'-o', 'uid=', # 10,11
1178	'-o', 'gid=', # 12,13
1179	'-o', 'comm=' # 14,15
1180	];
1181
1182	if sOs == 'darwin':
1183	assert asPsCmd[9] == 'sid=';
1184	asPsCmd[9] = 'sess=';
1185	elif sOs == 'solaris':
1186	asPsCmd[0] = '/usr/bin/ps';
1187
1188	try:
1189	sRaw = processOutputChecked(asPsCmd);
1190	except:
1191	return asProcesses;
1192
1193	for sLine in sRaw.split('\n'):
1194	sLine = sLine.lstrip();
1195	if len(sLine) < 7 or not sLine[0].isdigit():
1196	continue;
1197
1198	iField = 0;
1199	off = 0;
1200	aoFields = [None, None, None, None, None, None, None];
1201	while iField < 7:
1202	# Eat whitespace.
1203	while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1204	off += 1;
1205
1206	# Final field / EOL.
1207	if iField == 6:
1208	aoFields[6] = sLine[off:];
1209	break;
1210	if off >= len(sLine):
1211	break;
1212
1213	# Generic field parsing.
1214	offStart = off;
1215	off += 1;
1216	while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1217	off += 1;
1218	try:
1219	if iField != 3:
1220	aoFields[iField] = int(sLine[offStart:off]);
1221	else:
1222	aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1223	except:
1224	pass;
1225	iField += 1;
1226
1227	if aoFields[0] is not None:
1228	oMyInfo = ProcessInfo(aoFields[0]);
1229	oMyInfo.iParentPid = aoFields[1];
1230	oMyInfo.iProcGroup = aoFields[2];
1231	oMyInfo.iSessionId = aoFields[3];
1232	oMyInfo.iUid = aoFields[4];
1233	oMyInfo.iGid = aoFields[5];
1234	oMyInfo.sName = aoFields[6];
1235	asProcesses.append(oMyInfo);
1236
1237	return asProcesses;
1238
1239
1240	def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1241	"""
1242	Looks for information regarding the demise of the given process.
1243	"""
1244	sOs = getHostOs();
1245	if sOs == 'darwin':
1246	#
1247	# On darwin we look for crash and diagnostic reports.
1248	#
1249	asLogDirs = [
1250	u'/Library/Logs/DiagnosticReports/',
1251	u'/Library/Logs/CrashReporter/',
1252	u'~/Library/Logs/DiagnosticReports/',
1253	u'~/Library/Logs/CrashReporter/',
1254	];
1255	for sDir in asLogDirs:
1256	sDir = os.path.expanduser(sDir);
1257	if not os.path.isdir(sDir):
1258	continue;
1259	try:
1260	asDirEntries = os.listdir(sDir);
1261	except:
1262	continue;
1263	for sEntry in asDirEntries:
1264	# Only interested in .crash files.
1265	_, sSuff = os.path.splitext(sEntry);
1266	if sSuff != '.crash':
1267	continue;
1268
1269	# The pid can be found at the end of the first line.
1270	sFull = os.path.join(sDir, sEntry);
1271	try:
1272	oFile = open(sFull, 'r');
1273	sFirstLine = oFile.readline();
1274	oFile.close();
1275	except:
1276	continue;
1277	if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1278	continue;
1279	offPid = len(sFirstLine) - 3;
1280	while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1281	offPid -= 1;
1282	try: uReportPid = int(sFirstLine[offPid:-2]);
1283	except: continue;
1284
1285	# Does the pid we found match?
1286	if uReportPid == uPid:
1287	fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1288	fnCrashFile(sFull, False);
1289	elif sOs == 'win':
1290	#
1291	# Getting WER reports would be great, however we have trouble match the
1292	# PID to those as they seems not to mention it in the brief reports.
1293	# Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1294	# location - see the windows readme for the testbox script) and what
1295	# the MSDN article lists for now.
1296	#
1297	# It's been observed on Windows server 2012 that the dump files takes
1298	# the form: <processimage>.<decimal-pid>.dmp
1299	#
1300	asDmpDirs = [
1301	u'%SystemDrive%/CrashDumps/', # Testboxes.
1302	u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1303	u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1304	u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1305	u'%WINDIR%/ServiceProfiles/',
1306	u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1307	];
1308	sMatchSuffix = '.%u.dmp' % (uPid,);
1309
1310	for sDir in asDmpDirs:
1311	sDir = os.path.expandvars(sDir);
1312	if not os.path.isdir(sDir):
1313	continue;
1314	try:
1315	asDirEntries = os.listdir(sDir);
1316	except:
1317	continue;
1318	for sEntry in asDirEntries:
1319	if sEntry.endswith(sMatchSuffix):
1320	sFull = os.path.join(sDir, sEntry);
1321	fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1322	fnCrashFile(sFull, True);
1323
1324	else:
1325	pass; ## TODO
1326	return None;
1327
1328
1329	#
1330	# Time.
1331	#
1332
1333	#
1334	# The following test case shows how time.time() only have ~ms resolution
1335	# on Windows (tested W10) and why it therefore makes sense to try use
1336	# performance counters.
1337	#
1338	# Note! We cannot use time.clock() as the timestamp must be portable across
1339	# processes. See timeout testcase problem on win hosts (no logs).
1340	# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1341	#
1342	#import sys;
1343	#import time;
1344	#from common import utils;
1345	#
1346	#atSeries = [];
1347	#for i in xrange(1,160):
1348	# if i == 159: time.sleep(10);
1349	# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1350	#
1351	#tPrev = atSeries[0]
1352	#for tCur in atSeries:
1353	# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1354	# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1355	# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1356	# print '';
1357	# tPrev = tCur
1358	#
1359	#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1360	#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1361	#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1362
1363	g_fWinUseWinPerfCounter = sys.platform == 'win32';
1364	g_fpWinPerfCounterFreq = None;
1365	g_oFuncwinQueryPerformanceCounter = None;
1366
1367	def _winInitPerfCounter():
1368	""" Initializes the use of performance counters. """
1369	global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1370
1371	uFrequency = ctypes.c_ulonglong(0);
1372	if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1373	if uFrequency.value >= 1000:
1374	#print 'uFrequency = %s' % (uFrequency,);
1375	#print 'type(uFrequency) = %s' % (type(uFrequency),);
1376	g_fpWinPerfCounterFreq = float(uFrequency.value);
1377
1378	# Check that querying the counter works too.
1379	global g_oFuncwinQueryPerformanceCounter
1380	g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1381	uCurValue = ctypes.c_ulonglong(0);
1382	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1383	if uCurValue.value > 0:
1384	return True;
1385	g_fWinUseWinPerfCounter = False;
1386	return False;
1387
1388	def _winFloatTime():
1389	""" Gets floating point time on windows. """
1390	if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1391	uCurValue = ctypes.c_ulonglong(0);
1392	if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1393	return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1394	return time.time();
1395
1396	def timestampNano():
1397	"""
1398	Gets a nanosecond timestamp.
1399	"""
1400	if g_fWinUseWinPerfCounter is True:
1401	return long(_winFloatTime() * 1000000000);
1402	return long(time.time() * 1000000000);
1403
1404	def timestampMilli():
1405	"""
1406	Gets a millisecond timestamp.
1407	"""
1408	if g_fWinUseWinPerfCounter is True:
1409	return long(_winFloatTime() * 1000);
1410	return long(time.time() * 1000);
1411
1412	def timestampSecond():
1413	"""
1414	Gets a second timestamp.
1415	"""
1416	if g_fWinUseWinPerfCounter is True:
1417	return long(_winFloatTime());
1418	return long(time.time());
1419
1420	def secondsSinceUnixEpoch():
1421	"""
1422	Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1423	"""
1424	## ASSUMES This returns unix epoch time on all systems we care about...
1425	return time.time();
1426
1427	def getTimePrefix():
1428	"""
1429	Returns a timestamp prefix, typically used for logging. UTC.
1430	"""
1431	try:
1432	oNow = datetime.datetime.utcnow();
1433	sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1434	except:
1435	sTs = 'getTimePrefix-exception';
1436	return sTs;
1437
1438	def getTimePrefixAndIsoTimestamp():
1439	"""
1440	Returns current UTC as log prefix and iso timestamp.
1441	"""
1442	try:
1443	oNow = datetime.datetime.utcnow();
1444	sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1445	sTsIso = formatIsoTimestamp(oNow);
1446	except:
1447	sTsPrf = sTsIso = 'getTimePrefix-exception';
1448	return (sTsPrf, sTsIso);
1449
1450	class UtcTzInfo(datetime.tzinfo):
1451	"""UTC TZ Info Class"""
1452	def utcoffset(self, _):
1453	return datetime.timedelta(0);
1454	def tzname(self, _):
1455	return "UTC";
1456	def dst(self, _):
1457	return datetime.timedelta(0);
1458
1459	class GenTzInfo(datetime.tzinfo):
1460	"""Generic TZ Info Class"""
1461	def __init__(self, offInMin):
1462	datetime.tzinfo.__init__(self);
1463	self.offInMin = offInMin;
1464	def utcoffset(self, _):
1465	return datetime.timedelta(minutes = self.offInMin);
1466	def tzname(self, _):
1467	if self.offInMin >= 0:
1468	return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1469	return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1470	def dst(self, _):
1471	return datetime.timedelta(0);
1472
1473	def formatIsoTimestamp(oNow):
1474	"""Formats the datetime object as an ISO timestamp."""
1475	assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1476	sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1477	return sTs;
1478
1479	def getIsoTimestamp():
1480	"""Returns the current UTC timestamp as a string."""
1481	return formatIsoTimestamp(datetime.datetime.utcnow());
1482
1483	def convertDateTimeToZulu(oDateTime):
1484	""" Converts oDateTime to zulu time if it has timezone info. """
1485	if oDateTime.tzinfo is not None:
1486	oDateTime = oDateTime.astimezone(UtcTzInfo());
1487	else:
1488	oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1489	return oDateTime;
1490
1491	def parseIsoTimestamp(sTs):
1492	"""
1493	Parses a typical ISO timestamp, returing a datetime object, reasonably
1494	forgiving, but will throw weird indexing/conversion errors if the input
1495	is malformed.
1496	"""
1497	# YYYY-MM-DD
1498	iYear = int(sTs[0:4]);
1499	assert(sTs[4] == '-');
1500	iMonth = int(sTs[5:7]);
1501	assert(sTs[7] == '-');
1502	iDay = int(sTs[8:10]);
1503
1504	# Skip separator
1505	sTime = sTs[10:];
1506	while sTime[0] in 'Tt \t\n\r':
1507	sTime = sTime[1:];
1508
1509	# HH:MM:SS
1510	iHour = int(sTime[0:2]);
1511	assert(sTime[2] == ':');
1512	iMin = int(sTime[3:5]);
1513	assert(sTime[5] == ':');
1514	iSec = int(sTime[6:8]);
1515
1516	# Fraction?
1517	offTime = 8;
1518	iMicroseconds = 0;
1519	if offTime < len(sTime) and sTime[offTime] in '.,':
1520	offTime += 1;
1521	cchFraction = 0;
1522	while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1523	cchFraction += 1;
1524	if cchFraction > 0:
1525	iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1526	offTime += cchFraction;
1527	while cchFraction < 6:
1528	iMicroseconds *= 10;
1529	cchFraction += 1;
1530	while cchFraction > 6:
1531	iMicroseconds = iMicroseconds // 10;
1532	cchFraction -= 1;
1533
1534	# Naive?
1535	if offTime >= len(sTime):
1536	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1537
1538	# Zulu?
1539	if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1540	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1541
1542	# Some kind of offset afterwards, and strptime is useless. sigh.
1543	if sTime[offTime] in '+-':
1544	chSign = sTime[offTime];
1545	offTime += 1;
1546	cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1547	offTime += 2;
1548	if offTime < len(sTime) and sTime[offTime] in ':':
1549	offTime += 1;
1550	if offTime + 2 <= len(sTime):
1551	cMinTz += int(sTime[offTime : (offTime + 2)]);
1552	offTime += 2;
1553	assert offTime == len(sTime);
1554	if chSign == '-':
1555	cMinTz = -cMinTz;
1556	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1557	assert False, sTs;
1558	return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1559
1560	def normalizeIsoTimestampToZulu(sTs):
1561	"""
1562	Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1563	+ convertDateTimeToZulu + formatIsoTimestamp).
1564	Returns ISO tiemstamp string.
1565	"""
1566	return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1567
1568	def getLocalHourOfWeek():
1569	""" Local hour of week (0 based). """
1570	oNow = datetime.datetime.now();
1571	return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1572
1573
1574	def formatIntervalSeconds(cSeconds):
1575	""" Format a seconds interval into a nice 01h 00m 22s string """
1576	# Two simple special cases.
1577	if cSeconds < 60:
1578	return '%ss' % (cSeconds,);
1579	if cSeconds < 3600:
1580	cMins = cSeconds // 60;
1581	cSecs = cSeconds % 60;
1582	if cSecs == 0:
1583	return '%sm' % (cMins,);
1584	return '%sm %ss' % (cMins, cSecs,);
1585
1586	# Generic and a bit slower.
1587	cDays = cSeconds // 86400;
1588	cSeconds %= 86400;
1589	cHours = cSeconds // 3600;
1590	cSeconds %= 3600;
1591	cMins = cSeconds // 60;
1592	cSecs = cSeconds % 60;
1593	sRet = '';
1594	if cDays > 0:
1595	sRet = '%sd ' % (cDays,);
1596	if cHours > 0:
1597	sRet += '%sh ' % (cHours,);
1598	if cMins > 0:
1599	sRet += '%sm ' % (cMins,);
1600	if cSecs > 0:
1601	sRet += '%ss ' % (cSecs,);
1602	assert sRet; assert sRet[-1] == ' ';
1603	return sRet[:-1];
1604
1605	def formatIntervalSeconds2(oSeconds):
1606	"""
1607	Flexible input version of formatIntervalSeconds for use in WUI forms where
1608	data is usually already string form.
1609	"""
1610	if isinstance(oSeconds, (int, long)):
1611	return formatIntervalSeconds(oSeconds);
1612	if not isString(oSeconds):
1613	try:
1614	lSeconds = long(oSeconds);
1615	except:
1616	pass;
1617	else:
1618	if lSeconds >= 0:
1619	return formatIntervalSeconds2(lSeconds);
1620	return oSeconds;
1621
1622	def parseIntervalSeconds(sString):
1623	"""
1624	Reverse of formatIntervalSeconds.
1625
1626	Returns (cSeconds, sError), where sError is None on success.
1627	"""
1628
1629	# We might given non-strings, just return them without any fuss.
1630	if not isString(sString):
1631	if isinstance(sString, (int, long)) or sString is None:
1632	return (sString, None);
1633	## @todo time/date objects?
1634	return (int(sString), None);
1635
1636	# Strip it and make sure it's not empty.
1637	sString = sString.strip();
1638	if not sString:
1639	return (0, 'Empty interval string.');
1640
1641	#
1642	# Split up the input into a list of 'valueN, unitN, ...'.
1643	#
1644	# Don't want to spend too much time trying to make re.split do exactly what
1645	# I need here, so please forgive the extra pass I'm making here.
1646	#
1647	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1648	asParts = [];
1649	for sPart in asRawParts:
1650	sPart = sPart.strip();
1651	if sPart:
1652	asParts.append(sPart);
1653	if not asParts:
1654	return (0, 'Empty interval string or something?');
1655
1656	#
1657	# Process them one or two at the time.
1658	#
1659	cSeconds = 0;
1660	asErrors = [];
1661	i = 0;
1662	while i < len(asParts):
1663	sNumber = asParts[i];
1664	i += 1;
1665	if sNumber.isdigit():
1666	iNumber = int(sNumber);
1667
1668	sUnit = 's';
1669	if i < len(asParts) and not asParts[i].isdigit():
1670	sUnit = asParts[i];
1671	i += 1;
1672
1673	sUnitLower = sUnit.lower();
1674	if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1675	pass;
1676	elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1677	iNumber *= 60;
1678	elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1679	iNumber *= 3600;
1680	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1681	iNumber *= 86400;
1682	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1683	iNumber = 7 86400;
1684	else:
1685	asErrors.append('Unknown unit "%s".' % (sUnit,));
1686	cSeconds += iNumber;
1687	else:
1688	asErrors.append('Bad number "%s".' % (sNumber,));
1689	return (cSeconds, None if not asErrors else ' '.join(asErrors));
1690
1691	def formatIntervalHours(cHours):
1692	""" Format a hours interval into a nice 1w 2d 1h string. """
1693	# Simple special cases.
1694	if cHours < 24:
1695	return '%sh' % (cHours,);
1696
1697	# Generic and a bit slower.
1698	cWeeks = cHours / (7 * 24);
1699	cHours %= 7 * 24;
1700	cDays = cHours / 24;
1701	cHours %= 24;
1702	sRet = '';
1703	if cWeeks > 0:
1704	sRet = '%sw ' % (cWeeks,);
1705	if cDays > 0:
1706	sRet = '%sd ' % (cDays,);
1707	if cHours > 0:
1708	sRet += '%sh ' % (cHours,);
1709	assert sRet; assert sRet[-1] == ' ';
1710	return sRet[:-1];
1711
1712	def parseIntervalHours(sString):
1713	"""
1714	Reverse of formatIntervalHours.
1715
1716	Returns (cHours, sError), where sError is None on success.
1717	"""
1718
1719	# We might given non-strings, just return them without any fuss.
1720	if not isString(sString):
1721	if isinstance(sString, (int, long)) or sString is None:
1722	return (sString, None);
1723	## @todo time/date objects?
1724	return (int(sString), None);
1725
1726	# Strip it and make sure it's not empty.
1727	sString = sString.strip();
1728	if not sString:
1729	return (0, 'Empty interval string.');
1730
1731	#
1732	# Split up the input into a list of 'valueN, unitN, ...'.
1733	#
1734	# Don't want to spend too much time trying to make re.split do exactly what
1735	# I need here, so please forgive the extra pass I'm making here.
1736	#
1737	asRawParts = re.split(r'\s([0-9]+)\s([^0-9,;])[\s,;]', sString);
1738	asParts = [];
1739	for sPart in asRawParts:
1740	sPart = sPart.strip();
1741	if sPart:
1742	asParts.append(sPart);
1743	if not asParts:
1744	return (0, 'Empty interval string or something?');
1745
1746	#
1747	# Process them one or two at the time.
1748	#
1749	cHours = 0;
1750	asErrors = [];
1751	i = 0;
1752	while i < len(asParts):
1753	sNumber = asParts[i];
1754	i += 1;
1755	if sNumber.isdigit():
1756	iNumber = int(sNumber);
1757
1758	sUnit = 'h';
1759	if i < len(asParts) and not asParts[i].isdigit():
1760	sUnit = asParts[i];
1761	i += 1;
1762
1763	sUnitLower = sUnit.lower();
1764	if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1765	pass;
1766	elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1767	iNumber *= 24;
1768	elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1769	iNumber = 7 24;
1770	else:
1771	asErrors.append('Unknown unit "%s".' % (sUnit,));
1772	cHours += iNumber;
1773	else:
1774	asErrors.append('Bad number "%s".' % (sNumber,));
1775	return (cHours, None if not asErrors else ' '.join(asErrors));
1776
1777
1778	#
1779	# Introspection.
1780	#
1781
1782	def getCallerName(oFrame=None, iFrame=2):
1783	"""
1784	Returns the name of the caller's caller.
1785	"""
1786	if oFrame is None:
1787	try:
1788	raise Exception();
1789	except:
1790	oFrame = sys.exc_info()[2].tb_frame.f_back;
1791	while iFrame > 1:
1792	if oFrame is not None:
1793	oFrame = oFrame.f_back;
1794	iFrame = iFrame - 1;
1795	if oFrame is not None:
1796	sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1797	return sName;
1798	return "unknown";
1799
1800
1801	def getXcptInfo(cFrames = 1):
1802	"""
1803	Gets text detailing the exception. (Good for logging.)
1804	Returns list of info strings.
1805	"""
1806
1807	#
1808	# Try get exception info.
1809	#
1810	try:
1811	oType, oValue, oTraceback = sys.exc_info();
1812	except:
1813	oType = oValue = oTraceback = None;
1814	if oType is not None:
1815
1816	#
1817	# Try format the info
1818	#
1819	asRet = [];
1820	try:
1821	try:
1822	asRet = asRet + traceback.format_exception_only(oType, oValue);
1823	asTraceBack = traceback.format_tb(oTraceback);
1824	if cFrames is not None and cFrames <= 1:
1825	asRet.append(asTraceBack[-1]);
1826	else:
1827	asRet.append('Traceback:')
1828	for iFrame in range(min(cFrames, len(asTraceBack))):
1829	asRet.append(asTraceBack[-iFrame - 1]);
1830	asRet.append('Stack:')
1831	asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1832	except:
1833	asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1834
1835	if not asRet:
1836	asRet.append('No exception info...');
1837	except:
1838	asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1839	else:
1840	asRet = ['Couldn\'t find exception traceback.'];
1841	return asRet;
1842
1843
1844	def getObjectTypeName(oObject):
1845	"""
1846	Get the type name of the given object.
1847	"""
1848	if oObject is None:
1849	return 'None';
1850
1851	# Get the type object.
1852	try:
1853	oType = type(oObject);
1854	except:
1855	return 'type-throws-exception';
1856
1857	# Python 2.x only: Handle old-style object wrappers.
1858	if sys.version_info[0] < 3:
1859	try:
1860	from types import InstanceType; # pylint: disable=no-name-in-module
1861	if oType == InstanceType:
1862	oType = oObject.__class__;
1863	except:
1864	pass;
1865
1866	# Get the name.
1867	try:
1868	return oType.__name__;
1869	except:
1870	return '__type__-throws-exception';
1871
1872
1873	def chmodPlusX(sFile):
1874	"""
1875	Makes the specified file or directory executable.
1876	Returns success indicator, no exceptions.
1877
1878	Note! Symbolic links are followed and the target will be changed.
1879	"""
1880	try:
1881	oStat = os.stat(sFile);
1882	except:
1883	return False;
1884	try:
1885	os.chmod(sFile, oStat.st_mode \| stat.S_IXUSR \| stat.S_IXGRP \| stat.S_IXOTH);
1886	except:
1887	return False;
1888	return True;
1889
1890
1891	#
1892	# TestSuite stuff.
1893	#
1894
1895	def isRunningFromCheckout(cScriptDepth = 1):
1896	"""
1897	Checks if we're running from the SVN checkout or not.
1898	"""
1899
1900	try:
1901	sFile = __file__;
1902	cScriptDepth = 1;
1903	except:
1904	sFile = sys.argv[0];
1905
1906	sDir = os.path.abspath(sFile);
1907	while cScriptDepth >= 0:
1908	sDir = os.path.dirname(sDir);
1909	if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1910	or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1911	return True;
1912	cScriptDepth -= 1;
1913
1914	return False;
1915
1916
1917	#
1918	# Bourne shell argument fun.
1919	#
1920
1921
1922	def argsSplit(sCmdLine):
1923	"""
1924	Given a bourne shell command line invocation, split it up into arguments
1925	assuming IFS is space.
1926	Returns None on syntax error.
1927	"""
1928	## @todo bourne shell argument parsing!
1929	return sCmdLine.split(' ');
1930
1931	def argsGetFirst(sCmdLine):
1932	"""
1933	Given a bourne shell command line invocation, get return the first argument
1934	assuming IFS is space.
1935	Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1936	"""
1937	asArgs = argsSplit(sCmdLine);
1938	if not asArgs:
1939	return None;
1940
1941	return asArgs[0];
1942
1943	#
1944	# String helpers.
1945	#
1946
1947	def stricmp(sFirst, sSecond):
1948	"""
1949	Compares to strings in an case insensitive fashion.
1950
1951	Python doesn't seem to have any way of doing the correctly, so this is just
1952	an approximation using lower.
1953	"""
1954	if sFirst == sSecond:
1955	return 0;
1956	sLower1 = sFirst.lower();
1957	sLower2 = sSecond.lower();
1958	if sLower1 == sLower2:
1959	return 0;
1960	if sLower1 < sLower2:
1961	return -1;
1962	return 1;
1963
1964
1965	def versionCompare(sVer1, sVer2):
1966	"""
1967	Compares to version strings in a fashion similar to RTStrVersionCompare.
1968	"""
1969
1970	## @todo implement me!!
1971
1972	if sVer1 == sVer2:
1973	return 0;
1974	if sVer1 < sVer2:
1975	return -1;
1976	return 1;
1977
1978
1979	def formatNumber(lNum, sThousandSep = ' '):
1980	"""
1981	Formats a decimal number with pretty separators.
1982	"""
1983	sNum = str(lNum);
1984	sRet = sNum[-3:];
1985	off = len(sNum) - 3;
1986	while off > 0:
1987	off -= 3;
1988	sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
1989	return sRet;
1990
1991
1992	def formatNumberNbsp(lNum):
1993	"""
1994	Formats a decimal number with pretty separators.
1995	"""
1996	sRet = formatNumber(lNum);
1997	return unicode(sRet).replace(' ', u'\u00a0');
1998
1999
2000	def isString(oString):
2001	"""
2002	Checks if the object is a string object, hiding difference between python 2 and 3.
2003
2004	Returns True if it's a string of some kind.
2005	Returns False if not.
2006	"""
2007	if sys.version_info[0] >= 3:
2008	return isinstance(oString, str);
2009	return isinstance(oString, basestring); # pylint: disable=undefined-variable
2010
2011
2012	def hasNonAsciiCharacters(sText):
2013	"""
2014	Returns True is specified string has non-ASCII characters, False if ASCII only.
2015	"""
2016	if isString(sText):
2017	for ch in sText:
2018	if ord(ch) >= 128:
2019	return True;
2020	else:
2021	# Probably byte array or some such thing.
2022	for ch in sText:
2023	if ch >= 128 or ch < 0:
2024	return True;
2025	return False;
2026
2027
2028	#
2029	# Unpacking.
2030	#
2031
2032	def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2033	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2034	"""
2035	Worker for unpackFile that deals with ZIP files, same function signature.
2036	"""
2037	import zipfile
2038	if fnError is None:
2039	fnError = fnLog;
2040
2041	fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2042
2043	# Open it.
2044	try: oZipFile = zipfile.ZipFile(sArchive, 'r')
2045	except Exception as oXcpt:
2046	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2047	return None;
2048
2049	# Extract all members.
2050	asMembers = [];
2051	try:
2052	for sMember in oZipFile.namelist():
2053	if fnFilter is None or fnFilter(sMember) is not False:
2054	if sMember.endswith('/'):
2055	os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2056	else:
2057	oZipFile.extract(sMember, sDstDir);
2058	asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2059	except Exception as oXcpt:
2060	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2061	asMembers = None;
2062
2063	# close it.
2064	try: oZipFile.close();
2065	except Exception as oXcpt:
2066	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2067	asMembers = None;
2068
2069	return asMembers;
2070
2071
2072	## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2073	g_fTarCopyFileObjOverriddend = False;
2074
2075	def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2076	""" tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2077	_ = bufsize;
2078	if length is None:
2079	__myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2080	elif length > 0:
2081	cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2082	for _ in xrange(cFull):
2083	abBuffer = src.read(g_cbGoodBufferSize);
2084	dst.write(abBuffer);
2085	if len(abBuffer) != g_cbGoodBufferSize:
2086	raise exception('unexpected end of source file');
2087	if cbRemainder > 0:
2088	abBuffer = src.read(cbRemainder);
2089	dst.write(abBuffer);
2090	if len(abBuffer) != cbRemainder:
2091	raise exception('unexpected end of source file');
2092
2093
2094	def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2095	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2096	"""
2097	Worker for unpackFile that deals with tarballs, same function signature.
2098	"""
2099	import shutil;
2100	import tarfile;
2101	if fnError is None:
2102	fnError = fnLog;
2103
2104	fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2105
2106	#
2107	# Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2108	# 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2109	# 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2110	#
2111	if True is True: # pylint: disable=comparison-with-itself
2112	__installShUtilHacks(shutil);
2113	global g_fTarCopyFileObjOverriddend;
2114	if g_fTarCopyFileObjOverriddend is False:
2115	g_fTarCopyFileObjOverriddend = True;
2116	#if sys.hexversion < 0x03060000:
2117	tarfile.copyfileobj = __mytarfilecopyfileobj;
2118
2119	#
2120	# Open it.
2121	#
2122	# Note! We not using 'r:*' because we cannot allow seeking compressed files!
2123	# That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2124	#
2125	try:
2126	if sys.hexversion >= 0x03060000:
2127	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2128	else:
2129	oTarFile = tarfile.open(sArchive, 'r\|*', bufsize = g_cbGoodBufferSize);
2130	except Exception as oXcpt:
2131	fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2132	return None;
2133
2134	# Extract all members.
2135	asMembers = [];
2136	try:
2137	for oTarInfo in oTarFile:
2138	try:
2139	if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2140	if oTarInfo.islnk():
2141	# Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2142	# in the compressed tar stream. So, fall back on shutil.copy2 instead.
2143	sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2144	sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2145	sParentDir = os.path.dirname(sLinkFile);
2146	try: os.unlink(sLinkFile);
2147	except: pass;
2148	if sParentDir and not os.path.exists(sParentDir):
2149	os.makedirs(sParentDir);
2150	try: os.link(sLinkTarget, sLinkFile);
2151	except: shutil.copy2(sLinkTarget, sLinkFile);
2152	else:
2153	if oTarInfo.isdir():
2154	# Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2155	oTarInfo.mode \|= 0x1c0; # (octal: 0700)
2156	oTarFile.extract(oTarInfo, sDstDir);
2157	asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2158	except Exception as oXcpt:
2159	fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2160	for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2161	fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2162	for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2163	fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2164	asMembers = None;
2165	break;
2166	except Exception as oXcpt:
2167	fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2168	asMembers = None;
2169
2170	#
2171	# Finally, close it.
2172	#
2173	try: oTarFile.close();
2174	except Exception as oXcpt:
2175	fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2176	asMembers = None;
2177
2178	return asMembers;
2179
2180
2181	def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2182	# type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2183	"""
2184	Unpacks the given file if it has a know archive extension, otherwise do
2185	nothing.
2186
2187	fnLog & fnError both take a string parameter.
2188
2189	fnFilter takes a member name (string) and returns True if it's included
2190	and False if excluded.
2191
2192	Returns list of the extracted files (full path) on success.
2193	Returns empty list if not a supported archive format.
2194	Returns None on failure. Raises no exceptions.
2195	"""
2196	sBaseNameLower = os.path.basename(sArchive).lower();
2197
2198	#
2199	# Zip file?
2200	#
2201	if sBaseNameLower.endswith('.zip'):
2202	return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2203
2204	#
2205	# Tarball?
2206	#
2207	if sBaseNameLower.endswith('.tar') \
2208	or sBaseNameLower.endswith('.tar.gz') \
2209	or sBaseNameLower.endswith('.tgz') \
2210	or sBaseNameLower.endswith('.tar.bz2'):
2211	return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2212
2213	#
2214	# Cannot classify it from the name, so just return that to the caller.
2215	#
2216	fnLog('Not unpacking "%s".' % (sArchive,));
2217	return [];
2218
2219
2220	#
2221	# Misc.
2222	#
2223	def areBytesEqual(oLeft, oRight):
2224	"""
2225	Compares two byte arrays, strings or whatnot.
2226
2227	returns true / false accordingly.
2228	"""
2229
2230	# If both are None, consider them equal (bogus?):
2231	if oLeft is None and oRight is None:
2232	return True;
2233
2234	# If just one is None, they can't match:
2235	if oLeft is None or oRight is None:
2236	return False;
2237
2238	# If both have the same type, use the compare operator of the class:
2239	if type(oLeft) is type(oRight):
2240	#print('same type: %s' % (oLeft == oRight,));
2241	return oLeft == oRight;
2242
2243	# On the offchance that they're both strings, but of different types.
2244	if isString(oLeft) and isString(oRight):
2245	#print('string compare: %s' % (oLeft == oRight,));
2246	return oLeft == oRight;
2247
2248	#
2249	# See if byte/buffer stuff that can be compared directory. If not convert
2250	# strings to bytes.
2251	#
2252	# Note! For 2.x, we must convert both sides to the buffer type or the
2253	# comparison may fail despite it working okay in test cases.
2254	#
2255	if sys.version_info[0] >= 3:
2256	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2257	return oLeft == oRight;
2258
2259	if isString(oLeft):
2260	try: oLeft = bytes(oLeft, 'utf-8');
2261	except: pass;
2262	if isString(oRight):
2263	try: oRight = bytes(oRight, 'utf-8');
2264	except: pass;
2265	else:
2266	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2267	if isinstance(oLeft, bytearray):
2268	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2269	else:
2270	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2271	#print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2272	return oLeft == oRight;
2273
2274	if isString(oLeft):
2275	try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2276	except: pass;
2277	if isString(oRight):
2278	try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2279	except: pass;
2280
2281	# Check if we now have the same type for both:
2282	if type(oLeft) is type(oRight):
2283	#print('same type now: %s' % (oLeft == oRight,));
2284	return oLeft == oRight;
2285
2286	# Check if we now have buffer/memoryview vs bytes/bytesarray again.
2287	if sys.version_info[0] >= 3:
2288	if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2289	return oLeft == oRight;
2290	else:
2291	if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2292	if isinstance(oLeft, bytearray):
2293	oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2294	else:
2295	oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2296	#print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2297	return oLeft == oRight;
2298
2299	# Do item by item comparison:
2300	if len(oLeft) != len(oRight):
2301	#print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2302	return False;
2303	i = len(oLeft);
2304	while i > 0:
2305	i = i - 1;
2306
2307	iElmLeft = oLeft[i];
2308	if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2309	iElmLeft = ord(iElmLeft);
2310
2311	iElmRight = oRight[i];
2312	if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2313	iElmRight = ord(iElmRight);
2314
2315	if iElmLeft != iElmRight:
2316	#print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2317	return False;
2318	return True;
2319
2320
2321	#
2322	# Unit testing.
2323	#
2324
2325	# pylint: disable=missing-docstring
2326	# pylint: disable=undefined-variable
2327	class BuildCategoryDataTestCase(unittest.TestCase):
2328	def testIntervalSeconds(self):
2329	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2330	self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2331	self.assertEqual(parseIntervalSeconds('123'), (123, None));
2332	self.assertEqual(parseIntervalSeconds(123), (123, None));
2333	self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2334	self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2335	self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2336	self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2337	self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2338	self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2339	self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2340
2341	def testZuluNormalization(self):
2342	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2343	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2344	self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2345	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2346	self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2347
2348	def testHasNonAsciiChars(self):
2349	self.assertEqual(hasNonAsciiCharacters(''), False);
2350	self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB&$%&)@#(!)@(#!(#$&#$&%*Y@#$IQWN---00;'), False);
2351	self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2352	self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2353	self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2354	self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2355	self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2356	self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2357	self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2358	self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2359
2360	def testAreBytesEqual(self):
2361	self.assertEqual(areBytesEqual(None, None), True);
2362	self.assertEqual(areBytesEqual(None, ''), False);
2363	self.assertEqual(areBytesEqual('', ''), True);
2364	self.assertEqual(areBytesEqual('1', '1'), True);
2365	self.assertEqual(areBytesEqual('12345', '1234'), False);
2366	self.assertEqual(areBytesEqual('1234', '1234'), True);
2367	self.assertEqual(areBytesEqual('1234', b'1234'), True);
2368	self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2369	self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2370	self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2371	self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2372	self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2373	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2374	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2375	self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2376	if sys.version_info[0] >= 3:
2377	pass;
2378	else:
2379	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2380	bytearray([0x31,0x32,0x33,0x34])), True);
2381	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2382	bytearray([0x99,0x32,0x32,0x34])), False);
2383	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2384	buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2385	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2386	buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2387	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2388	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2389	self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2390
2391	if __name__ == '__main__':
2392	unittest.main();
2393	# not reached.
2394

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 83357

Download in other formats: