VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 86918

Last change on this file since 86918 was 86918, checked in by vboxsync, 4 years ago

ValKit/utils.py: Made the timestamp parser treat the seconds part as optional, just like the javascript version of the code does. bugref:9787
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 84.3 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: utils.py 86918 2020-11-19 11:08:06Z vboxsync $
3# pylint: disable=too-many-lines
4
5"""
6Common Utility Functions.
7"""
8
9from __future__ import print_function;
10
11__copyright__ = \
12"""
13Copyright (C) 2012-2020 Oracle Corporation
14
15This file is part of VirtualBox Open Source Edition (OSE), as
16available from http://www.virtualbox.org. This file is free software;
17you can redistribute it and/or modify it under the terms of the GNU
18General Public License (GPL) as published by the Free Software
19Foundation, in version 2 as it comes in the "COPYING" file of the
20VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23The contents of this file may alternatively be used under the terms
24of the Common Development and Distribution License Version 1.0
25(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26VirtualBox OSE distribution, in which case the provisions of the
27CDDL are applicable instead of those of the GPL.
28
29You may elect to license modified versions of this file under the
30terms and conditions of either the GPL or the CDDL or both.
31"""
32__version__ = "$Revision: 86918 $"
33
34
35# Standard Python imports.
36import datetime;
37import os;
38import platform;
39import re;
40import stat;
41import subprocess;
42import sys;
43import time;
44import traceback;
45import unittest;
46
47if sys.platform == 'win32':
48 import ctypes;
49 import msvcrt; # pylint: disable=import-error
50 import win32api; # pylint: disable=import-error
51 import win32con; # pylint: disable=import-error
52 import win32console; # pylint: disable=import-error
53 import win32file; # pylint: disable=import-error
54 import win32process; # pylint: disable=import-error
55 import winerror; # pylint: disable=import-error
56 import pywintypes; # pylint: disable=import-error
57else:
58 import signal;
59
60# Python 3 hacks:
61if sys.version_info[0] >= 3:
62 unicode = str; # pylint: disable=redefined-builtin,invalid-name
63 xrange = range; # pylint: disable=redefined-builtin,invalid-name
64 long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67#
68# Strings.
69#
70
71def toUnicode(sString, encoding = None, errors = 'strict'):
72 """
73 A little like the python 2 unicode() function.
74 """
75 if sys.version_info[0] >= 3:
76 if isinstance(sString, bytes):
77 return str(sString, encoding if encoding else 'utf-8', errors);
78 else:
79 if not isinstance(sString, unicode):
80 return unicode(sString, encoding if encoding else 'utf-8', errors);
81 return sString;
82
83
84
85#
86# Output.
87#
88
89def printOut(sString):
90 """
91 Outputs a string to standard output, dealing with python 2.x encoding stupidity.
92 """
93 sStreamEncoding = sys.stdout.encoding;
94 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
95 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
96 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
97 print(sString);
98 else:
99 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
100
101def printErr(sString):
102 """
103 Outputs a string to standard error, dealing with python 2.x encoding stupidity.
104 """
105 sStreamEncoding = sys.stderr.encoding;
106 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
107 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
108 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
109 print(sString, file = sys.stderr);
110 else:
111 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
112
113
114#
115# Host OS and CPU.
116#
117
118def getHostOs():
119 """
120 Gets the host OS name (short).
121
122 See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
123 """
124 sPlatform = platform.system();
125 if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
126 sPlatform = sPlatform.lower();
127 elif sPlatform == 'Windows':
128 sPlatform = 'win';
129 elif sPlatform == 'SunOS':
130 sPlatform = 'solaris';
131 else:
132 raise Exception('Unsupported platform "%s"' % (sPlatform,));
133 return sPlatform;
134
135g_sHostArch = None;
136
137def getHostArch():
138 """
139 Gets the host CPU architecture.
140
141 See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
142 """
143 global g_sHostArch;
144 if g_sHostArch is None:
145 sArch = platform.machine();
146 if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
147 sArch = 'x86';
148 elif sArch in ('AMD64', 'amd64', 'x86_64'):
149 sArch = 'amd64';
150 elif sArch == 'i86pc': # SunOS
151 if platform.architecture()[0] == '64bit':
152 sArch = 'amd64';
153 else:
154 try:
155 sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
156 except:
157 pass;
158 sArch = sArch.strip();
159 if sArch != 'amd64':
160 sArch = 'x86';
161 else:
162 raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
163 g_sHostArch = sArch;
164 return g_sHostArch;
165
166
167def getHostOsDotArch():
168 """
169 Gets the 'os.arch' for the host.
170 """
171 return '%s.%s' % (getHostOs(), getHostArch());
172
173
174def isValidOs(sOs):
175 """
176 Validates the OS name.
177 """
178 if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
179 'os2', 'solaris', 'win', 'os-agnostic'):
180 return True;
181 return False;
182
183
184def isValidArch(sArch):
185 """
186 Validates the CPU architecture name.
187 """
188 if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
189 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
190 return True;
191 return False;
192
193def isValidOsDotArch(sOsDotArch):
194 """
195 Validates the 'os.arch' string.
196 """
197
198 asParts = sOsDotArch.split('.');
199 if asParts.length() != 2:
200 return False;
201 return isValidOs(asParts[0]) \
202 and isValidArch(asParts[1]);
203
204def getHostOsVersion():
205 """
206 Returns the host OS version. This is platform.release with additional
207 distro indicator on linux.
208 """
209 sVersion = platform.release();
210 sOs = getHostOs();
211 if sOs == 'linux':
212 sDist = '';
213 try:
214 # try /etc/lsb-release first to distinguish between Debian and Ubuntu
215 oFile = open('/etc/lsb-release');
216 for sLine in oFile:
217 oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
218 if oMatch is not None:
219 sDist = oMatch.group(1).strip();
220 except:
221 pass;
222 if sDist:
223 sVersion += ' / ' + sDist;
224 else:
225 asFiles = \
226 [
227 [ '/etc/debian_version', 'Debian v'],
228 [ '/etc/gentoo-release', '' ],
229 [ '/etc/oracle-release', '' ],
230 [ '/etc/redhat-release', '' ],
231 [ '/etc/SuSE-release', '' ],
232 ];
233 for sFile, sPrefix in asFiles:
234 if os.path.isfile(sFile):
235 try:
236 oFile = open(sFile);
237 sLine = oFile.readline();
238 oFile.close();
239 except:
240 continue;
241 sLine = sLine.strip()
242 if sLine:
243 sVersion += ' / ' + sPrefix + sLine;
244 break;
245
246 elif sOs == 'solaris':
247 sVersion = platform.version();
248 if os.path.isfile('/etc/release'):
249 try:
250 oFile = open('/etc/release');
251 sLast = oFile.readlines()[-1];
252 oFile.close();
253 sLast = sLast.strip();
254 if sLast:
255 sVersion += ' (' + sLast + ')';
256 except:
257 pass;
258
259 elif sOs == 'darwin':
260 sOsxVersion = platform.mac_ver()[0];
261 codenames = {"4": "Tiger",
262 "5": "Leopard",
263 "6": "Snow Leopard",
264 "7": "Lion",
265 "8": "Mountain Lion",
266 "9": "Mavericks",
267 "10": "Yosemite",
268 "11": "El Capitan",
269 "12": "Sierra",
270 "13": "High Sierra",
271 "14": "Mojave",
272 "15": "Catalina",
273 "16": "Unknown 16",
274 "17": "Unknown 17",
275 "18": "Unknown 18",
276 "19": "Unknown 19", }
277 sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
278
279 elif sOs == 'win':
280 class OSVersionInfoEx(ctypes.Structure):
281 """ OSVERSIONEX """
282 kaFields = [
283 ('dwOSVersionInfoSize', ctypes.c_ulong),
284 ('dwMajorVersion', ctypes.c_ulong),
285 ('dwMinorVersion', ctypes.c_ulong),
286 ('dwBuildNumber', ctypes.c_ulong),
287 ('dwPlatformId', ctypes.c_ulong),
288 ('szCSDVersion', ctypes.c_wchar*128),
289 ('wServicePackMajor', ctypes.c_ushort),
290 ('wServicePackMinor', ctypes.c_ushort),
291 ('wSuiteMask', ctypes.c_ushort),
292 ('wProductType', ctypes.c_byte),
293 ('wReserved', ctypes.c_byte)]
294 _fields_ = kaFields # pylint: disable=invalid-name
295
296 def __init__(self):
297 super(OSVersionInfoEx, self).__init__()
298 self.dwOSVersionInfoSize = ctypes.sizeof(self)
299
300 oOsVersion = OSVersionInfoEx()
301 rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
302 if rc == 0:
303 # Python platform.release() is not reliable for newer server releases
304 if oOsVersion.wProductType != 1:
305 if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
306 sVersion = '2016Server';
307 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
308 sVersion = '2012ServerR2';
309 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
310 sVersion = '2012Server';
311 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
312 sVersion = '2008ServerR2';
313 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
314 sVersion = '2008Server';
315 elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
316 sVersion = '2003Server';
317 sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
318 if oOsVersion.wServicePackMajor:
319 sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
320 if oOsVersion.wServicePackMinor:
321 sVersion += '.' + str(oOsVersion.wServicePackMinor)
322
323 return sVersion;
324
325def getPresentCpuCount():
326 """
327 Gets the number of CPUs present in the system.
328
329 This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
330 that we return the active count rather than the maximum count. If we don't,
331 we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
332 never will have.
333
334 @todo This is probably not exactly what we get on non-windows...
335 """
336
337 if getHostOs() == 'win':
338 fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
339 if fnGetActiveProcessorCount:
340 cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
341 if cCpus > 0:
342 return cCpus;
343
344 import multiprocessing
345 return multiprocessing.cpu_count();
346
347
348#
349# File system.
350#
351
352def openNoInherit(sFile, sMode = 'r'):
353 """
354 Wrapper around open() that tries it's best to make sure the file isn't
355 inherited by child processes.
356
357 This is a best effort thing at the moment as it doesn't synchronizes with
358 child process spawning in any way. Thus it can be subject to races in
359 multithreaded programs.
360 """
361
362 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
363 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
364 if uPythonVer >= ((3 << 16) | 4):
365 oFile = open(sFile, sMode);
366 else:
367 try:
368 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
369 except:
370 # On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
371 if getHostOs() == 'win':
372 if uPythonVer < (3 << 16):
373 offComma = sMode.find(',');
374 if offComma < 0:
375 return open(sFile, sMode + 'N');
376 return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
377
378 # Just in case.
379 return open(sFile, sMode);
380
381 oFile = open(sFile, sMode);
382 #try:
383 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
384 #except:
385 # pass;
386 return oFile;
387
388def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
389 """
390 Wrapper around open() that tries it's best to make sure the file isn't
391 inherited by child processes.
392
393 This is a best effort thing at the moment as it doesn't synchronizes with
394 child process spawning in any way. Thus it can be subject to races in
395 multithreaded programs.
396 """
397
398 if getHostOs() == 'win':
399 # Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
400 # pylint: disable=no-member,c-extension-no-member
401 fAccess = 0;
402 fDisposition = win32file.OPEN_EXISTING;
403 if 'r' in sMode or '+' in sMode:
404 fAccess |= win32file.GENERIC_READ;
405 if 'a' in sMode:
406 fAccess |= win32file.GENERIC_WRITE;
407 fDisposition = win32file.OPEN_ALWAYS;
408 elif 'w' in sMode:
409 fAccess = win32file.GENERIC_WRITE;
410 if '+' in sMode:
411 fDisposition = win32file.OPEN_ALWAYS;
412 fAccess |= win32file.GENERIC_READ;
413 else:
414 fDisposition = win32file.CREATE_ALWAYS;
415 if not fAccess:
416 fAccess |= win32file.GENERIC_READ;
417 fSharing = (win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE
418 | win32file.FILE_SHARE_DELETE);
419 hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
420 if 'a' in sMode:
421 win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
422
423 # Turn the NT handle into a CRT file descriptor.
424 hDetachedFile = hFile.Detach();
425 if fAccess == win32file.GENERIC_READ:
426 fOpen = os.O_RDONLY;
427 elif fAccess == win32file.GENERIC_WRITE:
428 fOpen = os.O_WRONLY;
429 else:
430 fOpen = os.O_RDWR;
431 # pulint: enable=no-member,c-extension-no-member
432 if 'a' in sMode:
433 fOpen |= os.O_APPEND;
434 if 'b' in sMode or 't' in sMode:
435 fOpen |= os.O_TEXT; # pylint: disable=no-member
436 fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
437
438 # Tell python to use this handle.
439 oFile = os.fdopen(fdFile, sMode);
440 else:
441 oFile = open(sFile, sMode);
442
443 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
444 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
445 if uPythonVer < ((3 << 16) | 4):
446 try:
447 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
448 except:
449 pass;
450 else:
451 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
452 return oFile;
453
454def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
455 """
456 No exceptions os.readlink wrapper.
457 """
458 try:
459 sRet = os.readlink(sPath); # pylint: disable=no-member
460 except:
461 return sXcptRet;
462 if hasattr(sRet, 'decode'):
463 sRet = sRet.decode(sEncoding, 'ignore');
464 return sRet;
465
466def readFile(sFile, sMode = 'rb'):
467 """
468 Reads the entire file.
469 """
470 oFile = open(sFile, sMode);
471 sRet = oFile.read();
472 oFile.close();
473 return sRet;
474
475def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
476 """
477 No exceptions common.readFile wrapper.
478 """
479 try:
480 sRet = readFile(sFile, sMode);
481 except:
482 sRet = sXcptRet;
483 if sEncoding is not None and hasattr(sRet, 'decode'):
484 sRet = sRet.decode(sEncoding, 'ignore');
485 return sRet;
486
487def noxcptRmDir(sDir, oXcptRet = False):
488 """
489 No exceptions os.rmdir wrapper.
490 """
491 oRet = True;
492 try:
493 os.rmdir(sDir);
494 except:
495 oRet = oXcptRet;
496 return oRet;
497
498def noxcptDeleteFile(sFile, oXcptRet = False):
499 """
500 No exceptions os.remove wrapper.
501 """
502 oRet = True;
503 try:
504 os.remove(sFile);
505 except:
506 oRet = oXcptRet;
507 return oRet;
508
509
510def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
511 # type: (string, (string, stat) -> bool) -> bool
512 """
513 Recursively walks a directory tree, calling fnCallback for each.
514
515 fnCallback takes a full path and stat object (can be None). It
516 returns a boolean value, False stops walking and returns immediately.
517
518 Returns True or False depending on fnCallback.
519 Returns None fIgnoreExceptions is True and an exception was raised by listdir.
520 """
521 def __worker(sCurDir):
522 """ Worker for """
523 try:
524 asNames = os.listdir(sCurDir);
525 except:
526 if not fIgnoreExceptions:
527 raise;
528 return None;
529 rc = True;
530 for sName in asNames:
531 if sName not in [ '.', '..' ]:
532 sFullName = os.path.join(sCurDir, sName);
533 try: oStat = os.lstat(sFullName);
534 except: oStat = None;
535 if fnCallback(sFullName, oStat) is False:
536 return False;
537 if oStat is not None and stat.S_ISDIR(oStat.st_mode):
538 rc = __worker(sFullName);
539 if rc is False:
540 break;
541 return rc;
542
543 # Ensure unicode path here so listdir also returns unicode on windows.
544 ## @todo figure out unicode stuff on non-windows.
545 if sys.platform == 'win32':
546 sDir = unicode(sDir);
547 return __worker(sDir);
548
549
550
551def formatFileMode(uMode):
552 # type: (int) -> string
553 """
554 Format a st_mode value 'ls -la' fasion.
555 Returns string.
556 """
557 if stat.S_ISDIR(uMode): sMode = 'd';
558 elif stat.S_ISREG(uMode): sMode = '-';
559 elif stat.S_ISLNK(uMode): sMode = 'l';
560 elif stat.S_ISFIFO(uMode): sMode = 'p';
561 elif stat.S_ISCHR(uMode): sMode = 'c';
562 elif stat.S_ISBLK(uMode): sMode = 'b';
563 elif stat.S_ISSOCK(uMode): sMode = 's';
564 else: sMode = '?';
565 ## @todo sticky bits.
566 sMode += 'r' if uMode & stat.S_IRUSR else '-';
567 sMode += 'w' if uMode & stat.S_IWUSR else '-';
568 sMode += 'x' if uMode & stat.S_IXUSR else '-';
569 sMode += 'r' if uMode & stat.S_IRGRP else '-';
570 sMode += 'w' if uMode & stat.S_IWGRP else '-';
571 sMode += 'x' if uMode & stat.S_IXGRP else '-';
572 sMode += 'r' if uMode & stat.S_IROTH else '-';
573 sMode += 'w' if uMode & stat.S_IWOTH else '-';
574 sMode += 'x' if uMode & stat.S_IXOTH else '-';
575 sMode += ' ';
576 return sMode;
577
578
579def formatFileStat(oStat):
580 # type: (stat) -> string
581 """
582 Format a stat result 'ls -la' fasion (numeric IDs).
583 Returns string.
584 """
585 return '%s %3s %4s %4s %10s %s' \
586 % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
587 time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
588
589## Good buffer for file operations.
590g_cbGoodBufferSize = 256*1024;
591
592## The original shutil.copyfileobj.
593g_fnOriginalShCopyFileObj = None;
594
595def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
596 """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
597 return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
598
599def __installShUtilHacks(shutil):
600 """ Installs the shutil buffer size hacks. """
601 global g_fnOriginalShCopyFileObj;
602 if g_fnOriginalShCopyFileObj is None:
603 g_fnOriginalShCopyFileObj = shutil.copyfileobj;
604 shutil.copyfileobj = __myshutilcopyfileobj;
605 return True;
606
607
608def copyFileSimple(sFileSrc, sFileDst):
609 """
610 Wrapper around shutil.copyfile that simply copies the data of a regular file.
611 Raises exception on failure.
612 Return True for show.
613 """
614 import shutil;
615 __installShUtilHacks(shutil);
616 return shutil.copyfile(sFileSrc, sFileDst);
617
618
619def getDiskUsage(sPath):
620 """
621 Get free space of a partition that corresponds to specified sPath in MB.
622
623 Returns partition free space value in MB.
624 """
625 if platform.system() == 'Windows':
626 oCTypeFreeSpace = ctypes.c_ulonglong(0);
627 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
628 ctypes.pointer(oCTypeFreeSpace));
629 cbFreeSpace = oCTypeFreeSpace.value;
630 else:
631 oStats = os.statvfs(sPath); # pylint: disable=no-member
632 cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
633
634 # Convert to MB
635 cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
636
637 return cMbFreeSpace;
638
639
640
641#
642# SubProcess.
643#
644
645def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
646 """
647 If the "executable" is a python script, insert the python interpreter at
648 the head of the argument list so that it will work on systems which doesn't
649 support hash-bang scripts.
650 """
651
652 asArgs = dKeywordArgs.get('args');
653 if asArgs is None:
654 asArgs = aPositionalArgs[0];
655
656 if asArgs[0].endswith('.py'):
657 if sys.executable:
658 asArgs.insert(0, sys.executable);
659 else:
660 asArgs.insert(0, 'python');
661
662 # paranoia...
663 if dKeywordArgs.get('args') is not None:
664 dKeywordArgs['args'] = asArgs;
665 else:
666 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
667 return None;
668
669def processPopenSafe(*aPositionalArgs, **dKeywordArgs):
670 """
671 Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
672 """
673 if getHostOs() == 'win':
674 if dKeywordArgs.get('creationflags', 0) == 0:
675 dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
676 return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
677
678def processCall(*aPositionalArgs, **dKeywordArgs):
679 """
680 Wrapper around subprocess.call to deal with its absence in older
681 python versions.
682 Returns process exit code (see subprocess.poll).
683 """
684 assert dKeywordArgs.get('stdout') is None;
685 assert dKeywordArgs.get('stderr') is None;
686 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
687 oProcess = processPopenSafe(*aPositionalArgs, **dKeywordArgs);
688 return oProcess.wait();
689
690def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
691 """
692 Wrapper around subprocess.check_output to deal with its absense in older
693 python versions.
694
695 Extra keywords for specifying now output is to be decoded:
696 sEncoding='utf-8
697 fIgnoreEncoding=True/False
698 """
699 sEncoding = dKeywordArgs.get('sEncoding');
700 if sEncoding is not None: del dKeywordArgs['sEncoding'];
701 else: sEncoding = 'utf-8';
702
703 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
704 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
705 else: fIgnoreEncoding = True;
706
707 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
708 oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
709
710 sOutput, _ = oProcess.communicate();
711 iExitCode = oProcess.poll();
712
713 if iExitCode != 0:
714 asArgs = dKeywordArgs.get('args');
715 if asArgs is None:
716 asArgs = aPositionalArgs[0];
717 print(sOutput);
718 raise subprocess.CalledProcessError(iExitCode, asArgs);
719
720 if hasattr(sOutput, 'decode'):
721 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
722 return sOutput;
723
724def processOutputUnchecked(*aPositionalArgs, **dKeywordArgs):
725 """
726 Similar to processOutputChecked, but returns status code and both stdout
727 and stderr results.
728
729 Extra keywords for specifying now output is to be decoded:
730 sEncoding='utf-8
731 fIgnoreEncoding=True/False
732 """
733 sEncoding = dKeywordArgs.get('sEncoding');
734 if sEncoding is not None: del dKeywordArgs['sEncoding'];
735 else: sEncoding = 'utf-8';
736
737 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
738 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
739 else: fIgnoreEncoding = True;
740
741 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
742 oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
743
744 sOutput, sError = oProcess.communicate();
745 iExitCode = oProcess.poll();
746
747 if hasattr(sOutput, 'decode'):
748 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
749 if hasattr(sError, 'decode'):
750 sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
751 return (iExitCode, sOutput, sError);
752
753g_fOldSudo = None;
754def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
755 """
756 Adds 'sudo' (or similar) to the args parameter, whereever it is.
757 """
758
759 # Are we root?
760 fIsRoot = True;
761 try:
762 fIsRoot = os.getuid() == 0; # pylint: disable=no-member
763 except:
764 pass;
765
766 # If not, prepend sudo (non-interactive, simulate initial login).
767 if fIsRoot is not True:
768 asArgs = dKeywordArgs.get('args');
769 if asArgs is None:
770 asArgs = aPositionalArgs[0];
771
772 # Detect old sudo.
773 global g_fOldSudo;
774 if g_fOldSudo is None:
775 try:
776 sVersion = str(processOutputChecked(['sudo', '-V']));
777 except:
778 sVersion = '1.7.0';
779 sVersion = sVersion.strip().split('\n')[0];
780 sVersion = sVersion.replace('Sudo version', '').strip();
781 g_fOldSudo = len(sVersion) >= 4 \
782 and sVersion[0] == '1' \
783 and sVersion[1] == '.' \
784 and sVersion[2] <= '6' \
785 and sVersion[3] == '.';
786
787 asArgs.insert(0, 'sudo');
788 if not g_fOldSudo:
789 asArgs.insert(1, '-n');
790 if fInitialEnv and not g_fOldSudo:
791 asArgs.insert(1, '-i');
792
793 # paranoia...
794 if dKeywordArgs.get('args') is not None:
795 dKeywordArgs['args'] = asArgs;
796 else:
797 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
798 return None;
799
800
801def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
802 """
803 sudo (or similar) + subprocess.call
804 """
805 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
806 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
807 return processCall(*aPositionalArgs, **dKeywordArgs);
808
809def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
810 """
811 sudo (or similar) + subprocess.check_output.
812 """
813 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
814 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
815 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
816
817def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
818 """
819 sudo (or similar) + subprocess.check_output, except '-i' isn't used.
820 """
821 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
822 _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
823 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
824
825def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
826 """
827 sudo (or similar) + processPopenSafe.
828 """
829 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
830 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
831 return processPopenSafe(*aPositionalArgs, **dKeywordArgs);
832
833
834def whichProgram(sName, sPath = None):
835 """
836 Works similar to the 'which' utility on unix.
837
838 Returns path to the given program if found.
839 Returns None if not found.
840 """
841 sHost = getHostOs();
842 sSep = ';' if sHost in [ 'win', 'os2' ] else ':';
843
844 if sPath is None:
845 if sHost == 'win':
846 sPath = os.environ.get('Path', None);
847 else:
848 sPath = os.environ.get('PATH', None);
849 if sPath is None:
850 return None;
851
852 for sDir in sPath.split(sSep):
853 if sDir.strip() != '':
854 sTest = os.path.abspath(os.path.join(sDir, sName));
855 else:
856 sTest = os.path.abspath(sName);
857 if os.path.exists(sTest):
858 return sTest;
859
860 return None;
861
862#
863# Generic process stuff.
864#
865
866def processInterrupt(uPid):
867 """
868 Sends a SIGINT or equivalent to interrupt the specified process.
869 Returns True on success, False on failure.
870
871 On Windows hosts this may not work unless the process happens to be a
872 process group leader.
873 """
874 if sys.platform == 'win32':
875 try:
876 win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
877 uPid);
878 fRc = True;
879 except:
880 fRc = False;
881 else:
882 try:
883 os.kill(uPid, signal.SIGINT);
884 fRc = True;
885 except:
886 fRc = False;
887 return fRc;
888
889def sendUserSignal1(uPid):
890 """
891 Sends a SIGUSR1 or equivalent to nudge the process into shutting down
892 (VBoxSVC) or something.
893 Returns True on success, False on failure or if not supported (win).
894
895 On Windows hosts this may not work unless the process happens to be a
896 process group leader.
897 """
898 if sys.platform == 'win32':
899 fRc = False;
900 else:
901 try:
902 os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
903 fRc = True;
904 except:
905 fRc = False;
906 return fRc;
907
908def processTerminate(uPid):
909 """
910 Terminates the process in a nice manner (SIGTERM or equivalent).
911 Returns True on success, False on failure.
912 """
913 fRc = False;
914 if sys.platform == 'win32':
915 try:
916 hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
917 False, uPid);
918 except:
919 pass;
920 else:
921 try:
922 win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
923 0x40010004); # DBG_TERMINATE_PROCESS
924 fRc = True;
925 except:
926 pass;
927 hProcess.Close(); #win32api.CloseHandle(hProcess)
928 else:
929 try:
930 os.kill(uPid, signal.SIGTERM);
931 fRc = True;
932 except:
933 pass;
934 return fRc;
935
936def processKill(uPid):
937 """
938 Terminates the process with extreme prejudice (SIGKILL).
939 Returns True on success, False on failure.
940 """
941 if sys.platform == 'win32':
942 fRc = processTerminate(uPid);
943 else:
944 try:
945 os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
946 fRc = True;
947 except:
948 fRc = False;
949 return fRc;
950
951def processKillWithNameCheck(uPid, sName):
952 """
953 Like processKill(), but checks if the process name matches before killing
954 it. This is intended for killing using potentially stale pid values.
955
956 Returns True on success, False on failure.
957 """
958
959 if processCheckPidAndName(uPid, sName) is not True:
960 return False;
961 return processKill(uPid);
962
963
964def processExists(uPid):
965 """
966 Checks if the specified process exits.
967 This will only work if we can signal/open the process.
968
969 Returns True if it positively exists, False otherwise.
970 """
971 if sys.platform == 'win32':
972 fRc = False;
973 # We try open the process for waiting since this is generally only forbidden in a very few cases.
974 try:
975 hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
976 False, uPid);
977 except pywintypes.error as oXcpt: # pylint: disable=no-member
978 if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
979 fRc = True;
980 except Exception as oXcpt:
981 pass;
982 else:
983 hProcess.Close();
984 fRc = True;
985 else:
986 try:
987 os.kill(uPid, 0);
988 fRc = True;
989 except: ## @todo check error code.
990 fRc = False;
991 return fRc;
992
993def processCheckPidAndName(uPid, sName):
994 """
995 Checks if a process PID and NAME matches.
996 """
997 fRc = processExists(uPid);
998 if fRc is not True:
999 return False;
1000
1001 if sys.platform == 'win32':
1002 try:
1003 from win32com.client import GetObject; # pylint: disable=import-error
1004 oWmi = GetObject('winmgmts:');
1005 aoProcesses = oWmi.InstancesOf('Win32_Process');
1006 for oProcess in aoProcesses:
1007 if long(oProcess.Properties_("ProcessId").Value) == uPid:
1008 sCurName = oProcess.Properties_("Name").Value;
1009 #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
1010 sName = sName.lower();
1011 sCurName = sCurName.lower();
1012 if os.path.basename(sName) == sName:
1013 sCurName = os.path.basename(sCurName);
1014
1015 if sCurName == sName \
1016 or sCurName + '.exe' == sName \
1017 or sCurName == sName + '.exe':
1018 fRc = True;
1019 break;
1020 except:
1021 #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
1022 pass;
1023 else:
1024 if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
1025 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1026 elif sys.platform in ('sunos5',):
1027 asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1028 elif sys.platform in ('darwin',):
1029 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
1030 else:
1031 asPsCmd = None;
1032
1033 if asPsCmd is not None:
1034 try:
1035 oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
1036 sCurName = oPs.communicate()[0];
1037 iExitCode = oPs.wait();
1038 except:
1039 #reporter.logXcpt();
1040 return False;
1041
1042 # ps fails with non-zero exit code if the pid wasn't found.
1043 if iExitCode != 0:
1044 return False;
1045 if sCurName is None:
1046 return False;
1047 sCurName = sCurName.strip();
1048 if not sCurName:
1049 return False;
1050
1051 if os.path.basename(sName) == sName:
1052 sCurName = os.path.basename(sCurName);
1053 elif os.path.basename(sCurName) == sCurName:
1054 sName = os.path.basename(sName);
1055
1056 if sCurName != sName:
1057 return False;
1058
1059 fRc = True;
1060 return fRc;
1061
1062def processGetInfo(uPid, fSudo = False):
1063 """
1064 Tries to acquire state information of the given process.
1065
1066 Returns a string with the information on success or None on failure or
1067 if the host is not supported.
1068
1069 Note that the format of the information is host system dependent and will
1070 likely differ much between different hosts.
1071 """
1072 fRc = processExists(uPid);
1073 if fRc is not True:
1074 return None;
1075
1076 sHostOs = getHostOs();
1077 if sHostOs in [ 'linux',]:
1078 sGdb = '/usr/bin/gdb';
1079 if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1080 if not os.path.isfile(sGdb): sGdb = 'gdb';
1081 aasCmd = [
1082 [ sGdb, '-batch',
1083 '-ex', 'set pagination off',
1084 '-ex', 'thread apply all bt',
1085 '-ex', 'info proc mapping',
1086 '-ex', 'info sharedlibrary',
1087 '-p', '%u' % (uPid,), ],
1088 ];
1089 elif sHostOs == 'darwin':
1090 # LLDB doesn't work in batch mode when attaching to a process, at least
1091 # with macOS Sierra (10.12). GDB might not be installed. Use the sample
1092 # tool instead with a 1 second duration and 1000ms sampling interval to
1093 # get one stack trace. For the process mappings use vmmap.
1094 aasCmd = [
1095 [ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1096 [ '/usr/bin/vmmap', '%u' % (uPid,), ],
1097 ];
1098 elif sHostOs == 'solaris':
1099 aasCmd = [
1100 [ '/usr/bin/pstack', '%u' % (uPid,), ],
1101 [ '/usr/bin/pmap', '%u' % (uPid,), ],
1102 ];
1103 else:
1104 aasCmd = [];
1105
1106 sInfo = '';
1107 for asCmd in aasCmd:
1108 try:
1109 if fSudo:
1110 sThisInfo = sudoProcessOutputChecked(asCmd);
1111 else:
1112 sThisInfo = processOutputChecked(asCmd);
1113 if sThisInfo is not None:
1114 sInfo += sThisInfo;
1115 except:
1116 pass;
1117 if not sInfo:
1118 sInfo = None;
1119
1120 return sInfo;
1121
1122
1123class ProcessInfo(object):
1124 """Process info."""
1125 def __init__(self, iPid):
1126 self.iPid = iPid;
1127 self.iParentPid = None;
1128 self.sImage = None;
1129 self.sName = None;
1130 self.asArgs = None;
1131 self.sCwd = None;
1132 self.iGid = None;
1133 self.iUid = None;
1134 self.iProcGroup = None;
1135 self.iSessionId = None;
1136
1137 def loadAll(self):
1138 """Load all the info."""
1139 sOs = getHostOs();
1140 if sOs == 'linux':
1141 sProc = '/proc/%s/' % (self.iPid,);
1142 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1143 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1144 if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1145 #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1146 # sProc = '/proc/%s/' % (self.iPid,);
1147 # if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1148 # if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1149 else:
1150 pass;
1151 if self.sName is None and self.sImage is not None:
1152 self.sName = self.sImage;
1153
1154 def windowsGrabProcessInfo(self, oProcess):
1155 """Windows specific loadAll."""
1156 try: self.sName = oProcess.Properties_("Name").Value;
1157 except: pass;
1158 try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1159 except: pass;
1160 try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1161 except: pass;
1162 try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1163 except: pass;
1164 try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1165 except: pass;
1166 if self.sName is None and self.sImage is not None:
1167 self.sName = self.sImage;
1168
1169 def getBaseImageName(self):
1170 """
1171 Gets the base image name if available, use the process name if not available.
1172 Returns image/process base name or None.
1173 """
1174 sRet = self.sImage if self.sName is None else self.sName;
1175 if sRet is None:
1176 self.loadAll();
1177 sRet = self.sImage if self.sName is None else self.sName;
1178 if sRet is None:
1179 if not self.asArgs:
1180 return None;
1181 sRet = self.asArgs[0];
1182 if not sRet:
1183 return None;
1184 return os.path.basename(sRet);
1185
1186 def getBaseImageNameNoExeSuff(self):
1187 """
1188 Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1189 """
1190 sRet = self.getBaseImageName();
1191 if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1192 if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1193 sRet = sRet[:-4];
1194 return sRet;
1195
1196
1197def processListAll():
1198 """
1199 Return a list of ProcessInfo objects for all the processes in the system
1200 that the current user can see.
1201 """
1202 asProcesses = [];
1203
1204 sOs = getHostOs();
1205 if sOs == 'win':
1206 from win32com.client import GetObject; # pylint: disable=import-error
1207 oWmi = GetObject('winmgmts:');
1208 aoProcesses = oWmi.InstancesOf('Win32_Process');
1209 for oProcess in aoProcesses:
1210 try:
1211 iPid = int(oProcess.Properties_("ProcessId").Value);
1212 except:
1213 continue;
1214 oMyInfo = ProcessInfo(iPid);
1215 oMyInfo.windowsGrabProcessInfo(oProcess);
1216 asProcesses.append(oMyInfo);
1217 return asProcesses;
1218
1219 if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1220 try:
1221 asDirs = os.listdir('/proc');
1222 except:
1223 asDirs = [];
1224 for sDir in asDirs:
1225 if sDir.isdigit():
1226 asProcesses.append(ProcessInfo(int(sDir),));
1227 return asProcesses;
1228
1229 #
1230 # The other OSes parses the output from the 'ps' utility.
1231 #
1232 asPsCmd = [
1233 '/bin/ps', # 0
1234 '-A', # 1
1235 '-o', 'pid=', # 2,3
1236 '-o', 'ppid=', # 4,5
1237 '-o', 'pgid=', # 6,7
1238 '-o', 'sid=', # 8,9
1239 '-o', 'uid=', # 10,11
1240 '-o', 'gid=', # 12,13
1241 '-o', 'comm=' # 14,15
1242 ];
1243
1244 if sOs == 'darwin':
1245 assert asPsCmd[9] == 'sid=';
1246 asPsCmd[9] = 'sess=';
1247 elif sOs == 'solaris':
1248 asPsCmd[0] = '/usr/bin/ps';
1249
1250 try:
1251 sRaw = processOutputChecked(asPsCmd);
1252 except:
1253 return asProcesses;
1254
1255 for sLine in sRaw.split('\n'):
1256 sLine = sLine.lstrip();
1257 if len(sLine) < 7 or not sLine[0].isdigit():
1258 continue;
1259
1260 iField = 0;
1261 off = 0;
1262 aoFields = [None, None, None, None, None, None, None];
1263 while iField < 7:
1264 # Eat whitespace.
1265 while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1266 off += 1;
1267
1268 # Final field / EOL.
1269 if iField == 6:
1270 aoFields[6] = sLine[off:];
1271 break;
1272 if off >= len(sLine):
1273 break;
1274
1275 # Generic field parsing.
1276 offStart = off;
1277 off += 1;
1278 while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1279 off += 1;
1280 try:
1281 if iField != 3:
1282 aoFields[iField] = int(sLine[offStart:off]);
1283 else:
1284 aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1285 except:
1286 pass;
1287 iField += 1;
1288
1289 if aoFields[0] is not None:
1290 oMyInfo = ProcessInfo(aoFields[0]);
1291 oMyInfo.iParentPid = aoFields[1];
1292 oMyInfo.iProcGroup = aoFields[2];
1293 oMyInfo.iSessionId = aoFields[3];
1294 oMyInfo.iUid = aoFields[4];
1295 oMyInfo.iGid = aoFields[5];
1296 oMyInfo.sName = aoFields[6];
1297 asProcesses.append(oMyInfo);
1298
1299 return asProcesses;
1300
1301
1302def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1303 """
1304 Looks for information regarding the demise of the given process.
1305 """
1306 sOs = getHostOs();
1307 if sOs == 'darwin':
1308 #
1309 # On darwin we look for crash and diagnostic reports.
1310 #
1311 asLogDirs = [
1312 u'/Library/Logs/DiagnosticReports/',
1313 u'/Library/Logs/CrashReporter/',
1314 u'~/Library/Logs/DiagnosticReports/',
1315 u'~/Library/Logs/CrashReporter/',
1316 ];
1317 for sDir in asLogDirs:
1318 sDir = os.path.expanduser(sDir);
1319 if not os.path.isdir(sDir):
1320 continue;
1321 try:
1322 asDirEntries = os.listdir(sDir);
1323 except:
1324 continue;
1325 for sEntry in asDirEntries:
1326 # Only interested in .crash files.
1327 _, sSuff = os.path.splitext(sEntry);
1328 if sSuff != '.crash':
1329 continue;
1330
1331 # The pid can be found at the end of the first line.
1332 sFull = os.path.join(sDir, sEntry);
1333 try:
1334 oFile = open(sFull, 'r');
1335 sFirstLine = oFile.readline();
1336 oFile.close();
1337 except:
1338 continue;
1339 if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1340 continue;
1341 offPid = len(sFirstLine) - 3;
1342 while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1343 offPid -= 1;
1344 try: uReportPid = int(sFirstLine[offPid:-2]);
1345 except: continue;
1346
1347 # Does the pid we found match?
1348 if uReportPid == uPid:
1349 fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1350 fnCrashFile(sFull, False);
1351 elif sOs == 'win':
1352 #
1353 # Getting WER reports would be great, however we have trouble match the
1354 # PID to those as they seems not to mention it in the brief reports.
1355 # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1356 # location - see the windows readme for the testbox script) and what
1357 # the MSDN article lists for now.
1358 #
1359 # It's been observed on Windows server 2012 that the dump files takes
1360 # the form: <processimage>.<decimal-pid>.dmp
1361 #
1362 asDmpDirs = [
1363 u'%SystemDrive%/CrashDumps/', # Testboxes.
1364 u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1365 u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1366 u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1367 u'%WINDIR%/ServiceProfiles/',
1368 u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1369 ];
1370 sMatchSuffix = '.%u.dmp' % (uPid,);
1371
1372 for sDir in asDmpDirs:
1373 sDir = os.path.expandvars(sDir);
1374 if not os.path.isdir(sDir):
1375 continue;
1376 try:
1377 asDirEntries = os.listdir(sDir);
1378 except:
1379 continue;
1380 for sEntry in asDirEntries:
1381 if sEntry.endswith(sMatchSuffix):
1382 sFull = os.path.join(sDir, sEntry);
1383 fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1384 fnCrashFile(sFull, True);
1385
1386 else:
1387 pass; ## TODO
1388 return None;
1389
1390
1391#
1392# Time.
1393#
1394
1395#
1396# The following test case shows how time.time() only have ~ms resolution
1397# on Windows (tested W10) and why it therefore makes sense to try use
1398# performance counters.
1399#
1400# Note! We cannot use time.clock() as the timestamp must be portable across
1401# processes. See timeout testcase problem on win hosts (no logs).
1402# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1403#
1404#import sys;
1405#import time;
1406#from common import utils;
1407#
1408#atSeries = [];
1409#for i in xrange(1,160):
1410# if i == 159: time.sleep(10);
1411# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1412#
1413#tPrev = atSeries[0]
1414#for tCur in atSeries:
1415# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1416# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1417# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1418# print '';
1419# tPrev = tCur
1420#
1421#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1422#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1423#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1424
1425g_fWinUseWinPerfCounter = sys.platform == 'win32';
1426g_fpWinPerfCounterFreq = None;
1427g_oFuncwinQueryPerformanceCounter = None;
1428
1429def _winInitPerfCounter():
1430 """ Initializes the use of performance counters. """
1431 global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1432
1433 uFrequency = ctypes.c_ulonglong(0);
1434 if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1435 if uFrequency.value >= 1000:
1436 #print 'uFrequency = %s' % (uFrequency,);
1437 #print 'type(uFrequency) = %s' % (type(uFrequency),);
1438 g_fpWinPerfCounterFreq = float(uFrequency.value);
1439
1440 # Check that querying the counter works too.
1441 global g_oFuncwinQueryPerformanceCounter
1442 g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1443 uCurValue = ctypes.c_ulonglong(0);
1444 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1445 if uCurValue.value > 0:
1446 return True;
1447 g_fWinUseWinPerfCounter = False;
1448 return False;
1449
1450def _winFloatTime():
1451 """ Gets floating point time on windows. """
1452 if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1453 uCurValue = ctypes.c_ulonglong(0);
1454 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1455 return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1456 return time.time();
1457
1458def timestampNano():
1459 """
1460 Gets a nanosecond timestamp.
1461 """
1462 if g_fWinUseWinPerfCounter is True:
1463 return long(_winFloatTime() * 1000000000);
1464 return long(time.time() * 1000000000);
1465
1466def timestampMilli():
1467 """
1468 Gets a millisecond timestamp.
1469 """
1470 if g_fWinUseWinPerfCounter is True:
1471 return long(_winFloatTime() * 1000);
1472 return long(time.time() * 1000);
1473
1474def timestampSecond():
1475 """
1476 Gets a second timestamp.
1477 """
1478 if g_fWinUseWinPerfCounter is True:
1479 return long(_winFloatTime());
1480 return long(time.time());
1481
1482def secondsSinceUnixEpoch():
1483 """
1484 Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1485 """
1486 ## ASSUMES This returns unix epoch time on all systems we care about...
1487 return time.time();
1488
1489def getTimePrefix():
1490 """
1491 Returns a timestamp prefix, typically used for logging. UTC.
1492 """
1493 try:
1494 oNow = datetime.datetime.utcnow();
1495 sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1496 except:
1497 sTs = 'getTimePrefix-exception';
1498 return sTs;
1499
1500def getTimePrefixAndIsoTimestamp():
1501 """
1502 Returns current UTC as log prefix and iso timestamp.
1503 """
1504 try:
1505 oNow = datetime.datetime.utcnow();
1506 sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1507 sTsIso = formatIsoTimestamp(oNow);
1508 except:
1509 sTsPrf = sTsIso = 'getTimePrefix-exception';
1510 return (sTsPrf, sTsIso);
1511
1512class UtcTzInfo(datetime.tzinfo):
1513 """UTC TZ Info Class"""
1514 def utcoffset(self, _):
1515 return datetime.timedelta(0);
1516 def tzname(self, _):
1517 return "UTC";
1518 def dst(self, _):
1519 return datetime.timedelta(0);
1520
1521class GenTzInfo(datetime.tzinfo):
1522 """Generic TZ Info Class"""
1523 def __init__(self, offInMin):
1524 datetime.tzinfo.__init__(self);
1525 self.offInMin = offInMin;
1526 def utcoffset(self, _):
1527 return datetime.timedelta(minutes = self.offInMin);
1528 def tzname(self, _):
1529 if self.offInMin >= 0:
1530 return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1531 return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1532 def dst(self, _):
1533 return datetime.timedelta(0);
1534
1535def formatIsoTimestamp(oNow):
1536 """Formats the datetime object as an ISO timestamp."""
1537 assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1538 sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1539 return sTs;
1540
1541def getIsoTimestamp():
1542 """Returns the current UTC timestamp as a string."""
1543 return formatIsoTimestamp(datetime.datetime.utcnow());
1544
1545def formatShortIsoTimestamp(oNow):
1546 """Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1547 assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1548 return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1549
1550def getShortIsoTimestamp():
1551 """Returns the current UTC timestamp as a string, but w/o microseconds."""
1552 return formatShortIsoTimestamp(datetime.datetime.utcnow());
1553
1554def convertDateTimeToZulu(oDateTime):
1555 """ Converts oDateTime to zulu time if it has timezone info. """
1556 if oDateTime.tzinfo is not None:
1557 oDateTime = oDateTime.astimezone(UtcTzInfo());
1558 else:
1559 oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1560 return oDateTime;
1561
1562def parseIsoTimestamp(sTs):
1563 """
1564 Parses a typical ISO timestamp, returing a datetime object, reasonably
1565 forgiving, but will throw weird indexing/conversion errors if the input
1566 is malformed.
1567 """
1568 # YYYY-MM-DD
1569 iYear = int(sTs[0:4]);
1570 assert(sTs[4] == '-');
1571 iMonth = int(sTs[5:7]);
1572 assert(sTs[7] == '-');
1573 iDay = int(sTs[8:10]);
1574
1575 # Skip separator
1576 sTime = sTs[10:];
1577 while sTime[0] in 'Tt \t\n\r':
1578 sTime = sTime[1:];
1579
1580 # HH:MM[:SS]
1581 iHour = int(sTime[0:2]);
1582 assert(sTime[2] == ':');
1583 iMin = int(sTime[3:5]);
1584 if (sTime[5] == ':')
1585 iSec = int(sTime[6:8]);
1586
1587 # Fraction?
1588 offTime = 8;
1589 iMicroseconds = 0;
1590 if offTime < len(sTime) and sTime[offTime] in '.,':
1591 offTime += 1;
1592 cchFraction = 0;
1593 while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1594 cchFraction += 1;
1595 if cchFraction > 0:
1596 iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1597 offTime += cchFraction;
1598 while cchFraction < 6:
1599 iMicroseconds *= 10;
1600 cchFraction += 1;
1601 while cchFraction > 6:
1602 iMicroseconds = iMicroseconds // 10;
1603 cchFraction -= 1;
1604
1605 else:
1606 iSec = 0;
1607 iMicroseconds = 0;
1608 offTime = 5;
1609
1610 # Naive?
1611 if offTime >= len(sTime):
1612 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1613
1614 # Zulu?
1615 if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1616 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1617
1618 # Some kind of offset afterwards, and strptime is useless. sigh.
1619 if sTime[offTime] in '+-':
1620 chSign = sTime[offTime];
1621 offTime += 1;
1622 cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1623 offTime += 2;
1624 if offTime < len(sTime) and sTime[offTime] in ':':
1625 offTime += 1;
1626 if offTime + 2 <= len(sTime):
1627 cMinTz += int(sTime[offTime : (offTime + 2)]);
1628 offTime += 2;
1629 assert offTime == len(sTime);
1630 if chSign == '-':
1631 cMinTz = -cMinTz;
1632 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1633 assert False, sTs;
1634 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1635
1636def normalizeIsoTimestampToZulu(sTs):
1637 """
1638 Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1639 + convertDateTimeToZulu + formatIsoTimestamp).
1640 Returns ISO tiemstamp string.
1641 """
1642 return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1643
1644def getLocalHourOfWeek():
1645 """ Local hour of week (0 based). """
1646 oNow = datetime.datetime.now();
1647 return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1648
1649
1650def formatIntervalSeconds(cSeconds):
1651 """ Format a seconds interval into a nice 01h 00m 22s string """
1652 # Two simple special cases.
1653 if cSeconds < 60:
1654 return '%ss' % (cSeconds,);
1655 if cSeconds < 3600:
1656 cMins = cSeconds // 60;
1657 cSecs = cSeconds % 60;
1658 if cSecs == 0:
1659 return '%sm' % (cMins,);
1660 return '%sm %ss' % (cMins, cSecs,);
1661
1662 # Generic and a bit slower.
1663 cDays = cSeconds // 86400;
1664 cSeconds %= 86400;
1665 cHours = cSeconds // 3600;
1666 cSeconds %= 3600;
1667 cMins = cSeconds // 60;
1668 cSecs = cSeconds % 60;
1669 sRet = '';
1670 if cDays > 0:
1671 sRet = '%sd ' % (cDays,);
1672 if cHours > 0:
1673 sRet += '%sh ' % (cHours,);
1674 if cMins > 0:
1675 sRet += '%sm ' % (cMins,);
1676 if cSecs > 0:
1677 sRet += '%ss ' % (cSecs,);
1678 assert sRet; assert sRet[-1] == ' ';
1679 return sRet[:-1];
1680
1681def formatIntervalSeconds2(oSeconds):
1682 """
1683 Flexible input version of formatIntervalSeconds for use in WUI forms where
1684 data is usually already string form.
1685 """
1686 if isinstance(oSeconds, (int, long)):
1687 return formatIntervalSeconds(oSeconds);
1688 if not isString(oSeconds):
1689 try:
1690 lSeconds = long(oSeconds);
1691 except:
1692 pass;
1693 else:
1694 if lSeconds >= 0:
1695 return formatIntervalSeconds2(lSeconds);
1696 return oSeconds;
1697
1698def parseIntervalSeconds(sString):
1699 """
1700 Reverse of formatIntervalSeconds.
1701
1702 Returns (cSeconds, sError), where sError is None on success.
1703 """
1704
1705 # We might given non-strings, just return them without any fuss.
1706 if not isString(sString):
1707 if isinstance(sString, (int, long)) or sString is None:
1708 return (sString, None);
1709 ## @todo time/date objects?
1710 return (int(sString), None);
1711
1712 # Strip it and make sure it's not empty.
1713 sString = sString.strip();
1714 if not sString:
1715 return (0, 'Empty interval string.');
1716
1717 #
1718 # Split up the input into a list of 'valueN, unitN, ...'.
1719 #
1720 # Don't want to spend too much time trying to make re.split do exactly what
1721 # I need here, so please forgive the extra pass I'm making here.
1722 #
1723 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1724 asParts = [];
1725 for sPart in asRawParts:
1726 sPart = sPart.strip();
1727 if sPart:
1728 asParts.append(sPart);
1729 if not asParts:
1730 return (0, 'Empty interval string or something?');
1731
1732 #
1733 # Process them one or two at the time.
1734 #
1735 cSeconds = 0;
1736 asErrors = [];
1737 i = 0;
1738 while i < len(asParts):
1739 sNumber = asParts[i];
1740 i += 1;
1741 if sNumber.isdigit():
1742 iNumber = int(sNumber);
1743
1744 sUnit = 's';
1745 if i < len(asParts) and not asParts[i].isdigit():
1746 sUnit = asParts[i];
1747 i += 1;
1748
1749 sUnitLower = sUnit.lower();
1750 if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1751 pass;
1752 elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1753 iNumber *= 60;
1754 elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1755 iNumber *= 3600;
1756 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1757 iNumber *= 86400;
1758 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1759 iNumber *= 7 * 86400;
1760 else:
1761 asErrors.append('Unknown unit "%s".' % (sUnit,));
1762 cSeconds += iNumber;
1763 else:
1764 asErrors.append('Bad number "%s".' % (sNumber,));
1765 return (cSeconds, None if not asErrors else ' '.join(asErrors));
1766
1767def formatIntervalHours(cHours):
1768 """ Format a hours interval into a nice 1w 2d 1h string. """
1769 # Simple special cases.
1770 if cHours < 24:
1771 return '%sh' % (cHours,);
1772
1773 # Generic and a bit slower.
1774 cWeeks = cHours / (7 * 24);
1775 cHours %= 7 * 24;
1776 cDays = cHours / 24;
1777 cHours %= 24;
1778 sRet = '';
1779 if cWeeks > 0:
1780 sRet = '%sw ' % (cWeeks,);
1781 if cDays > 0:
1782 sRet = '%sd ' % (cDays,);
1783 if cHours > 0:
1784 sRet += '%sh ' % (cHours,);
1785 assert sRet; assert sRet[-1] == ' ';
1786 return sRet[:-1];
1787
1788def parseIntervalHours(sString):
1789 """
1790 Reverse of formatIntervalHours.
1791
1792 Returns (cHours, sError), where sError is None on success.
1793 """
1794
1795 # We might given non-strings, just return them without any fuss.
1796 if not isString(sString):
1797 if isinstance(sString, (int, long)) or sString is None:
1798 return (sString, None);
1799 ## @todo time/date objects?
1800 return (int(sString), None);
1801
1802 # Strip it and make sure it's not empty.
1803 sString = sString.strip();
1804 if not sString:
1805 return (0, 'Empty interval string.');
1806
1807 #
1808 # Split up the input into a list of 'valueN, unitN, ...'.
1809 #
1810 # Don't want to spend too much time trying to make re.split do exactly what
1811 # I need here, so please forgive the extra pass I'm making here.
1812 #
1813 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1814 asParts = [];
1815 for sPart in asRawParts:
1816 sPart = sPart.strip();
1817 if sPart:
1818 asParts.append(sPart);
1819 if not asParts:
1820 return (0, 'Empty interval string or something?');
1821
1822 #
1823 # Process them one or two at the time.
1824 #
1825 cHours = 0;
1826 asErrors = [];
1827 i = 0;
1828 while i < len(asParts):
1829 sNumber = asParts[i];
1830 i += 1;
1831 if sNumber.isdigit():
1832 iNumber = int(sNumber);
1833
1834 sUnit = 'h';
1835 if i < len(asParts) and not asParts[i].isdigit():
1836 sUnit = asParts[i];
1837 i += 1;
1838
1839 sUnitLower = sUnit.lower();
1840 if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1841 pass;
1842 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1843 iNumber *= 24;
1844 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1845 iNumber *= 7 * 24;
1846 else:
1847 asErrors.append('Unknown unit "%s".' % (sUnit,));
1848 cHours += iNumber;
1849 else:
1850 asErrors.append('Bad number "%s".' % (sNumber,));
1851 return (cHours, None if not asErrors else ' '.join(asErrors));
1852
1853
1854#
1855# Introspection.
1856#
1857
1858def getCallerName(oFrame=None, iFrame=2):
1859 """
1860 Returns the name of the caller's caller.
1861 """
1862 if oFrame is None:
1863 try:
1864 raise Exception();
1865 except:
1866 oFrame = sys.exc_info()[2].tb_frame.f_back;
1867 while iFrame > 1:
1868 if oFrame is not None:
1869 oFrame = oFrame.f_back;
1870 iFrame = iFrame - 1;
1871 if oFrame is not None:
1872 sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1873 return sName;
1874 return "unknown";
1875
1876
1877def getXcptInfo(cFrames = 1):
1878 """
1879 Gets text detailing the exception. (Good for logging.)
1880 Returns list of info strings.
1881 """
1882
1883 #
1884 # Try get exception info.
1885 #
1886 try:
1887 oType, oValue, oTraceback = sys.exc_info();
1888 except:
1889 oType = oValue = oTraceback = None;
1890 if oType is not None:
1891
1892 #
1893 # Try format the info
1894 #
1895 asRet = [];
1896 try:
1897 try:
1898 asRet = asRet + traceback.format_exception_only(oType, oValue);
1899 asTraceBack = traceback.format_tb(oTraceback);
1900 if cFrames is not None and cFrames <= 1:
1901 asRet.append(asTraceBack[-1]);
1902 else:
1903 asRet.append('Traceback:')
1904 for iFrame in range(min(cFrames, len(asTraceBack))):
1905 asRet.append(asTraceBack[-iFrame - 1]);
1906 asRet.append('Stack:')
1907 asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1908 except:
1909 asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1910
1911 if not asRet:
1912 asRet.append('No exception info...');
1913 except:
1914 asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1915 else:
1916 asRet = ['Couldn\'t find exception traceback.'];
1917 return asRet;
1918
1919
1920def getObjectTypeName(oObject):
1921 """
1922 Get the type name of the given object.
1923 """
1924 if oObject is None:
1925 return 'None';
1926
1927 # Get the type object.
1928 try:
1929 oType = type(oObject);
1930 except:
1931 return 'type-throws-exception';
1932
1933 # Python 2.x only: Handle old-style object wrappers.
1934 if sys.version_info[0] < 3:
1935 try:
1936 from types import InstanceType; # pylint: disable=no-name-in-module
1937 if oType == InstanceType:
1938 oType = oObject.__class__;
1939 except:
1940 pass;
1941
1942 # Get the name.
1943 try:
1944 return oType.__name__;
1945 except:
1946 return '__type__-throws-exception';
1947
1948
1949def chmodPlusX(sFile):
1950 """
1951 Makes the specified file or directory executable.
1952 Returns success indicator, no exceptions.
1953
1954 Note! Symbolic links are followed and the target will be changed.
1955 """
1956 try:
1957 oStat = os.stat(sFile);
1958 except:
1959 return False;
1960 try:
1961 os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
1962 except:
1963 return False;
1964 return True;
1965
1966
1967#
1968# TestSuite stuff.
1969#
1970
1971def isRunningFromCheckout(cScriptDepth = 1):
1972 """
1973 Checks if we're running from the SVN checkout or not.
1974 """
1975
1976 try:
1977 sFile = __file__;
1978 cScriptDepth = 1;
1979 except:
1980 sFile = sys.argv[0];
1981
1982 sDir = os.path.abspath(sFile);
1983 while cScriptDepth >= 0:
1984 sDir = os.path.dirname(sDir);
1985 if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1986 or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1987 return True;
1988 cScriptDepth -= 1;
1989
1990 return False;
1991
1992
1993#
1994# Bourne shell argument fun.
1995#
1996
1997
1998def argsSplit(sCmdLine):
1999 """
2000 Given a bourne shell command line invocation, split it up into arguments
2001 assuming IFS is space.
2002 Returns None on syntax error.
2003 """
2004 ## @todo bourne shell argument parsing!
2005 return sCmdLine.split(' ');
2006
2007def argsGetFirst(sCmdLine):
2008 """
2009 Given a bourne shell command line invocation, get return the first argument
2010 assuming IFS is space.
2011 Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
2012 """
2013 asArgs = argsSplit(sCmdLine);
2014 if not asArgs:
2015 return None;
2016
2017 return asArgs[0];
2018
2019#
2020# String helpers.
2021#
2022
2023def stricmp(sFirst, sSecond):
2024 """
2025 Compares to strings in an case insensitive fashion.
2026
2027 Python doesn't seem to have any way of doing the correctly, so this is just
2028 an approximation using lower.
2029 """
2030 if sFirst == sSecond:
2031 return 0;
2032 sLower1 = sFirst.lower();
2033 sLower2 = sSecond.lower();
2034 if sLower1 == sLower2:
2035 return 0;
2036 if sLower1 < sLower2:
2037 return -1;
2038 return 1;
2039
2040
2041def versionCompare(sVer1, sVer2):
2042 """
2043 Compares to version strings in a fashion similar to RTStrVersionCompare.
2044 """
2045
2046 ## @todo implement me!!
2047
2048 if sVer1 == sVer2:
2049 return 0;
2050 if sVer1 < sVer2:
2051 return -1;
2052 return 1;
2053
2054
2055def formatNumber(lNum, sThousandSep = ' '):
2056 """
2057 Formats a decimal number with pretty separators.
2058 """
2059 sNum = str(lNum);
2060 sRet = sNum[-3:];
2061 off = len(sNum) - 3;
2062 while off > 0:
2063 off -= 3;
2064 sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2065 return sRet;
2066
2067
2068def formatNumberNbsp(lNum):
2069 """
2070 Formats a decimal number with pretty separators.
2071 """
2072 sRet = formatNumber(lNum);
2073 return unicode(sRet).replace(' ', u'\u00a0');
2074
2075
2076def isString(oString):
2077 """
2078 Checks if the object is a string object, hiding difference between python 2 and 3.
2079
2080 Returns True if it's a string of some kind.
2081 Returns False if not.
2082 """
2083 if sys.version_info[0] >= 3:
2084 return isinstance(oString, str);
2085 return isinstance(oString, basestring); # pylint: disable=undefined-variable
2086
2087
2088def hasNonAsciiCharacters(sText):
2089 """
2090 Returns True is specified string has non-ASCII characters, False if ASCII only.
2091 """
2092 if isString(sText):
2093 for ch in sText:
2094 if ord(ch) >= 128:
2095 return True;
2096 else:
2097 # Probably byte array or some such thing.
2098 for ch in sText:
2099 if ch >= 128 or ch < 0:
2100 return True;
2101 return False;
2102
2103
2104#
2105# Unpacking.
2106#
2107
2108def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2109 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2110 """
2111 Worker for unpackFile that deals with ZIP files, same function signature.
2112 """
2113 import zipfile
2114 if fnError is None:
2115 fnError = fnLog;
2116
2117 fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2118
2119 # Open it.
2120 try: oZipFile = zipfile.ZipFile(sArchive, 'r')
2121 except Exception as oXcpt:
2122 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2123 return None;
2124
2125 # Extract all members.
2126 asMembers = [];
2127 try:
2128 for sMember in oZipFile.namelist():
2129 if fnFilter is None or fnFilter(sMember) is not False:
2130 if sMember.endswith('/'):
2131 os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2132 else:
2133 oZipFile.extract(sMember, sDstDir);
2134 asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2135 except Exception as oXcpt:
2136 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2137 asMembers = None;
2138
2139 # close it.
2140 try: oZipFile.close();
2141 except Exception as oXcpt:
2142 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2143 asMembers = None;
2144
2145 return asMembers;
2146
2147
2148## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2149g_fTarCopyFileObjOverriddend = False;
2150
2151def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2152 """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2153 _ = bufsize;
2154 if length is None:
2155 __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2156 elif length > 0:
2157 cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2158 for _ in xrange(cFull):
2159 abBuffer = src.read(g_cbGoodBufferSize);
2160 dst.write(abBuffer);
2161 if len(abBuffer) != g_cbGoodBufferSize:
2162 raise exception('unexpected end of source file');
2163 if cbRemainder > 0:
2164 abBuffer = src.read(cbRemainder);
2165 dst.write(abBuffer);
2166 if len(abBuffer) != cbRemainder:
2167 raise exception('unexpected end of source file');
2168
2169
2170def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2171 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2172 """
2173 Worker for unpackFile that deals with tarballs, same function signature.
2174 """
2175 import shutil;
2176 import tarfile;
2177 if fnError is None:
2178 fnError = fnLog;
2179
2180 fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2181
2182 #
2183 # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2184 # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2185 # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2186 #
2187 if True is True: # pylint: disable=comparison-with-itself
2188 __installShUtilHacks(shutil);
2189 global g_fTarCopyFileObjOverriddend;
2190 if g_fTarCopyFileObjOverriddend is False:
2191 g_fTarCopyFileObjOverriddend = True;
2192 #if sys.hexversion < 0x03060000:
2193 tarfile.copyfileobj = __mytarfilecopyfileobj;
2194
2195 #
2196 # Open it.
2197 #
2198 # Note! We not using 'r:*' because we cannot allow seeking compressed files!
2199 # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2200 #
2201 try:
2202 if sys.hexversion >= 0x03060000:
2203 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2204 else:
2205 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
2206 except Exception as oXcpt:
2207 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2208 return None;
2209
2210 # Extract all members.
2211 asMembers = [];
2212 try:
2213 for oTarInfo in oTarFile:
2214 try:
2215 if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2216 if oTarInfo.islnk():
2217 # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2218 # in the compressed tar stream. So, fall back on shutil.copy2 instead.
2219 sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2220 sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2221 sParentDir = os.path.dirname(sLinkFile);
2222 try: os.unlink(sLinkFile);
2223 except: pass;
2224 if sParentDir and not os.path.exists(sParentDir):
2225 os.makedirs(sParentDir);
2226 try: os.link(sLinkTarget, sLinkFile);
2227 except: shutil.copy2(sLinkTarget, sLinkFile);
2228 else:
2229 if oTarInfo.isdir():
2230 # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2231 oTarInfo.mode |= 0x1c0; # (octal: 0700)
2232 oTarFile.extract(oTarInfo, sDstDir);
2233 asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2234 except Exception as oXcpt:
2235 fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2236 for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2237 fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2238 for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2239 fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2240 asMembers = None;
2241 break;
2242 except Exception as oXcpt:
2243 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2244 asMembers = None;
2245
2246 #
2247 # Finally, close it.
2248 #
2249 try: oTarFile.close();
2250 except Exception as oXcpt:
2251 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2252 asMembers = None;
2253
2254 return asMembers;
2255
2256
2257def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2258 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2259 """
2260 Unpacks the given file if it has a know archive extension, otherwise do
2261 nothing.
2262
2263 fnLog & fnError both take a string parameter.
2264
2265 fnFilter takes a member name (string) and returns True if it's included
2266 and False if excluded.
2267
2268 Returns list of the extracted files (full path) on success.
2269 Returns empty list if not a supported archive format.
2270 Returns None on failure. Raises no exceptions.
2271 """
2272 sBaseNameLower = os.path.basename(sArchive).lower();
2273
2274 #
2275 # Zip file?
2276 #
2277 if sBaseNameLower.endswith('.zip'):
2278 return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2279
2280 #
2281 # Tarball?
2282 #
2283 if sBaseNameLower.endswith('.tar') \
2284 or sBaseNameLower.endswith('.tar.gz') \
2285 or sBaseNameLower.endswith('.tgz') \
2286 or sBaseNameLower.endswith('.tar.bz2'):
2287 return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2288
2289 #
2290 # Cannot classify it from the name, so just return that to the caller.
2291 #
2292 fnLog('Not unpacking "%s".' % (sArchive,));
2293 return [];
2294
2295
2296#
2297# Misc.
2298#
2299def areBytesEqual(oLeft, oRight):
2300 """
2301 Compares two byte arrays, strings or whatnot.
2302
2303 returns true / false accordingly.
2304 """
2305
2306 # If both are None, consider them equal (bogus?):
2307 if oLeft is None and oRight is None:
2308 return True;
2309
2310 # If just one is None, they can't match:
2311 if oLeft is None or oRight is None:
2312 return False;
2313
2314 # If both have the same type, use the compare operator of the class:
2315 if type(oLeft) is type(oRight):
2316 #print('same type: %s' % (oLeft == oRight,));
2317 return oLeft == oRight;
2318
2319 # On the offchance that they're both strings, but of different types.
2320 if isString(oLeft) and isString(oRight):
2321 #print('string compare: %s' % (oLeft == oRight,));
2322 return oLeft == oRight;
2323
2324 #
2325 # See if byte/buffer stuff that can be compared directory. If not convert
2326 # strings to bytes.
2327 #
2328 # Note! For 2.x, we must convert both sides to the buffer type or the
2329 # comparison may fail despite it working okay in test cases.
2330 #
2331 if sys.version_info[0] >= 3:
2332 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2333 return oLeft == oRight;
2334
2335 if isString(oLeft):
2336 try: oLeft = bytes(oLeft, 'utf-8');
2337 except: pass;
2338 if isString(oRight):
2339 try: oRight = bytes(oRight, 'utf-8');
2340 except: pass;
2341 else:
2342 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2343 if isinstance(oLeft, bytearray):
2344 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2345 else:
2346 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2347 #print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2348 return oLeft == oRight;
2349
2350 if isString(oLeft):
2351 try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2352 except: pass;
2353 if isString(oRight):
2354 try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2355 except: pass;
2356
2357 # Check if we now have the same type for both:
2358 if type(oLeft) is type(oRight):
2359 #print('same type now: %s' % (oLeft == oRight,));
2360 return oLeft == oRight;
2361
2362 # Check if we now have buffer/memoryview vs bytes/bytesarray again.
2363 if sys.version_info[0] >= 3:
2364 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2365 return oLeft == oRight;
2366 else:
2367 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2368 if isinstance(oLeft, bytearray):
2369 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2370 else:
2371 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2372 #print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2373 return oLeft == oRight;
2374
2375 # Do item by item comparison:
2376 if len(oLeft) != len(oRight):
2377 #print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2378 return False;
2379 i = len(oLeft);
2380 while i > 0:
2381 i = i - 1;
2382
2383 iElmLeft = oLeft[i];
2384 if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2385 iElmLeft = ord(iElmLeft);
2386
2387 iElmRight = oRight[i];
2388 if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2389 iElmRight = ord(iElmRight);
2390
2391 if iElmLeft != iElmRight:
2392 #print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2393 return False;
2394 return True;
2395
2396
2397#
2398# Unit testing.
2399#
2400
2401# pylint: disable=missing-docstring
2402# pylint: disable=undefined-variable
2403class BuildCategoryDataTestCase(unittest.TestCase):
2404 def testIntervalSeconds(self):
2405 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2406 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2407 self.assertEqual(parseIntervalSeconds('123'), (123, None));
2408 self.assertEqual(parseIntervalSeconds(123), (123, None));
2409 self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2410 self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2411 self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2412 self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2413 self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2414 self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2415 self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2416
2417 def testZuluNormalization(self):
2418 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2419 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2420 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2421 self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2422 self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2423
2424 def testHasNonAsciiChars(self):
2425 self.assertEqual(hasNonAsciiCharacters(''), False);
2426 self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
2427 self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2428 self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2429 self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2430 self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2431 self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2432 self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2433 self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2434 self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2435
2436 def testAreBytesEqual(self):
2437 self.assertEqual(areBytesEqual(None, None), True);
2438 self.assertEqual(areBytesEqual(None, ''), False);
2439 self.assertEqual(areBytesEqual('', ''), True);
2440 self.assertEqual(areBytesEqual('1', '1'), True);
2441 self.assertEqual(areBytesEqual('12345', '1234'), False);
2442 self.assertEqual(areBytesEqual('1234', '1234'), True);
2443 self.assertEqual(areBytesEqual('1234', b'1234'), True);
2444 self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2445 self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2446 self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2447 self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2448 self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2449 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2450 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2451 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2452 if sys.version_info[0] >= 3:
2453 pass;
2454 else:
2455 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2456 bytearray([0x31,0x32,0x33,0x34])), True);
2457 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2458 bytearray([0x99,0x32,0x32,0x34])), False);
2459 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2460 buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2461 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2462 buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2463 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2464 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2465 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2466
2467if __name__ == '__main__':
2468 unittest.main();
2469 # not reached.
2470
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette