VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 92889

Last change on this file since 92889 was 92889, checked in by vboxsync, 3 years ago

ValKit/utils.py: Better processExits for all unix platforms.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 84.6 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: utils.py 92889 2021-12-13 21:33:56Z vboxsync $
3# pylint: disable=too-many-lines
4
5"""
6Common Utility Functions.
7"""
8
9from __future__ import print_function;
10
11__copyright__ = \
12"""
13Copyright (C) 2012-2020 Oracle Corporation
14
15This file is part of VirtualBox Open Source Edition (OSE), as
16available from http://www.virtualbox.org. This file is free software;
17you can redistribute it and/or modify it under the terms of the GNU
18General Public License (GPL) as published by the Free Software
19Foundation, in version 2 as it comes in the "COPYING" file of the
20VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23The contents of this file may alternatively be used under the terms
24of the Common Development and Distribution License Version 1.0
25(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26VirtualBox OSE distribution, in which case the provisions of the
27CDDL are applicable instead of those of the GPL.
28
29You may elect to license modified versions of this file under the
30terms and conditions of either the GPL or the CDDL or both.
31"""
32__version__ = "$Revision: 92889 $"
33
34
35# Standard Python imports.
36import datetime;
37import errno;
38import os;
39import platform;
40import re;
41import stat;
42import subprocess;
43import sys;
44import time;
45import traceback;
46import unittest;
47
48if sys.platform == 'win32':
49 import ctypes;
50 import msvcrt; # pylint: disable=import-error
51 import win32api; # pylint: disable=import-error
52 import win32con; # pylint: disable=import-error
53 import win32console; # pylint: disable=import-error
54 import win32file; # pylint: disable=import-error
55 import win32process; # pylint: disable=import-error
56 import winerror; # pylint: disable=import-error
57 import pywintypes; # pylint: disable=import-error
58else:
59 import signal;
60
61# Python 3 hacks:
62if sys.version_info[0] >= 3:
63 unicode = str; # pylint: disable=redefined-builtin,invalid-name
64 xrange = range; # pylint: disable=redefined-builtin,invalid-name
65 long = int; # pylint: disable=redefined-builtin,invalid-name
66
67
68#
69# Strings.
70#
71
72def toUnicode(sString, encoding = None, errors = 'strict'):
73 """
74 A little like the python 2 unicode() function.
75 """
76 if sys.version_info[0] >= 3:
77 if isinstance(sString, bytes):
78 return str(sString, encoding if encoding else 'utf-8', errors);
79 else:
80 if not isinstance(sString, unicode):
81 return unicode(sString, encoding if encoding else 'utf-8', errors);
82 return sString;
83
84
85
86#
87# Output.
88#
89
90def printOut(sString):
91 """
92 Outputs a string to standard output, dealing with python 2.x encoding stupidity.
93 """
94 sStreamEncoding = sys.stdout.encoding;
95 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
96 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
97 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
98 print(sString);
99 else:
100 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
101
102def printErr(sString):
103 """
104 Outputs a string to standard error, dealing with python 2.x encoding stupidity.
105 """
106 sStreamEncoding = sys.stderr.encoding;
107 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
108 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
109 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
110 print(sString, file = sys.stderr);
111 else:
112 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
113
114
115#
116# Host OS and CPU.
117#
118
119def getHostOs():
120 """
121 Gets the host OS name (short).
122
123 See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
124 """
125 sPlatform = platform.system();
126 if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
127 sPlatform = sPlatform.lower();
128 elif sPlatform == 'Windows':
129 sPlatform = 'win';
130 elif sPlatform == 'SunOS':
131 sPlatform = 'solaris';
132 else:
133 raise Exception('Unsupported platform "%s"' % (sPlatform,));
134 return sPlatform;
135
136g_sHostArch = None;
137
138def getHostArch():
139 """
140 Gets the host CPU architecture.
141
142 See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
143 """
144 global g_sHostArch;
145 if g_sHostArch is None:
146 sArch = platform.machine();
147 if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
148 sArch = 'x86';
149 elif sArch in ('AMD64', 'amd64', 'x86_64'):
150 sArch = 'amd64';
151 elif sArch == 'i86pc': # SunOS
152 if platform.architecture()[0] == '64bit':
153 sArch = 'amd64';
154 else:
155 try:
156 sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
157 except:
158 pass;
159 sArch = sArch.strip();
160 if sArch != 'amd64':
161 sArch = 'x86';
162 else:
163 raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
164 g_sHostArch = sArch;
165 return g_sHostArch;
166
167
168def getHostOsDotArch():
169 """
170 Gets the 'os.arch' for the host.
171 """
172 return '%s.%s' % (getHostOs(), getHostArch());
173
174
175def isValidOs(sOs):
176 """
177 Validates the OS name.
178 """
179 if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
180 'os2', 'solaris', 'win', 'os-agnostic'):
181 return True;
182 return False;
183
184
185def isValidArch(sArch):
186 """
187 Validates the CPU architecture name.
188 """
189 if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
190 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
191 return True;
192 return False;
193
194def isValidOsDotArch(sOsDotArch):
195 """
196 Validates the 'os.arch' string.
197 """
198
199 asParts = sOsDotArch.split('.');
200 if asParts.length() != 2:
201 return False;
202 return isValidOs(asParts[0]) \
203 and isValidArch(asParts[1]);
204
205def getHostOsVersion():
206 """
207 Returns the host OS version. This is platform.release with additional
208 distro indicator on linux.
209 """
210 sVersion = platform.release();
211 sOs = getHostOs();
212 if sOs == 'linux':
213 sDist = '';
214 try:
215 # try /etc/lsb-release first to distinguish between Debian and Ubuntu
216 oFile = open('/etc/lsb-release');
217 for sLine in oFile:
218 oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
219 if oMatch is not None:
220 sDist = oMatch.group(1).strip();
221 except:
222 pass;
223 if sDist:
224 sVersion += ' / ' + sDist;
225 else:
226 asFiles = \
227 [
228 [ '/etc/debian_version', 'Debian v'],
229 [ '/etc/gentoo-release', '' ],
230 [ '/etc/oracle-release', '' ],
231 [ '/etc/redhat-release', '' ],
232 [ '/etc/SuSE-release', '' ],
233 ];
234 for sFile, sPrefix in asFiles:
235 if os.path.isfile(sFile):
236 try:
237 oFile = open(sFile);
238 sLine = oFile.readline();
239 oFile.close();
240 except:
241 continue;
242 sLine = sLine.strip()
243 if sLine:
244 sVersion += ' / ' + sPrefix + sLine;
245 break;
246
247 elif sOs == 'solaris':
248 sVersion = platform.version();
249 if os.path.isfile('/etc/release'):
250 try:
251 oFile = open('/etc/release');
252 sLast = oFile.readlines()[-1];
253 oFile.close();
254 sLast = sLast.strip();
255 if sLast:
256 sVersion += ' (' + sLast + ')';
257 except:
258 pass;
259
260 elif sOs == 'darwin':
261 sOsxVersion = platform.mac_ver()[0];
262 codenames = {"4": "Tiger",
263 "5": "Leopard",
264 "6": "Snow Leopard",
265 "7": "Lion",
266 "8": "Mountain Lion",
267 "9": "Mavericks",
268 "10": "Yosemite",
269 "11": "El Capitan",
270 "12": "Sierra",
271 "13": "High Sierra",
272 "14": "Mojave",
273 "15": "Catalina",
274 "16": "Unknown 16",
275 "17": "Unknown 17",
276 "18": "Unknown 18",
277 "19": "Unknown 19", }
278 sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
279
280 elif sOs == 'win':
281 class OSVersionInfoEx(ctypes.Structure):
282 """ OSVERSIONEX """
283 kaFields = [
284 ('dwOSVersionInfoSize', ctypes.c_ulong),
285 ('dwMajorVersion', ctypes.c_ulong),
286 ('dwMinorVersion', ctypes.c_ulong),
287 ('dwBuildNumber', ctypes.c_ulong),
288 ('dwPlatformId', ctypes.c_ulong),
289 ('szCSDVersion', ctypes.c_wchar*128),
290 ('wServicePackMajor', ctypes.c_ushort),
291 ('wServicePackMinor', ctypes.c_ushort),
292 ('wSuiteMask', ctypes.c_ushort),
293 ('wProductType', ctypes.c_byte),
294 ('wReserved', ctypes.c_byte)]
295 _fields_ = kaFields # pylint: disable=invalid-name
296
297 def __init__(self):
298 super(OSVersionInfoEx, self).__init__()
299 self.dwOSVersionInfoSize = ctypes.sizeof(self)
300
301 oOsVersion = OSVersionInfoEx()
302 rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
303 if rc == 0:
304 # Python platform.release() is not reliable for newer server releases
305 if oOsVersion.wProductType != 1:
306 if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
307 sVersion = '2016Server';
308 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
309 sVersion = '2012ServerR2';
310 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
311 sVersion = '2012Server';
312 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
313 sVersion = '2008ServerR2';
314 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
315 sVersion = '2008Server';
316 elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
317 sVersion = '2003Server';
318 sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
319 if oOsVersion.wServicePackMajor:
320 sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
321 if oOsVersion.wServicePackMinor:
322 sVersion += '.' + str(oOsVersion.wServicePackMinor)
323
324 return sVersion;
325
326def getPresentCpuCount():
327 """
328 Gets the number of CPUs present in the system.
329
330 This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
331 that we return the active count rather than the maximum count. If we don't,
332 we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
333 never will have.
334
335 @todo This is probably not exactly what we get on non-windows...
336 """
337
338 if getHostOs() == 'win':
339 fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
340 if fnGetActiveProcessorCount:
341 cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
342 if cCpus > 0:
343 return cCpus;
344
345 import multiprocessing
346 return multiprocessing.cpu_count();
347
348
349#
350# File system.
351#
352
353def openNoInherit(sFile, sMode = 'r'):
354 """
355 Wrapper around open() that tries it's best to make sure the file isn't
356 inherited by child processes.
357
358 This is a best effort thing at the moment as it doesn't synchronizes with
359 child process spawning in any way. Thus it can be subject to races in
360 multithreaded programs.
361 """
362
363 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
364 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
365 if uPythonVer >= ((3 << 16) | 4):
366 oFile = open(sFile, sMode);
367 else:
368 try:
369 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
370 except:
371 # On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
372 if getHostOs() == 'win':
373 if uPythonVer < (3 << 16):
374 offComma = sMode.find(',');
375 if offComma < 0:
376 return open(sFile, sMode + 'N');
377 return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
378
379 # Just in case.
380 return open(sFile, sMode);
381
382 oFile = open(sFile, sMode);
383 #try:
384 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
385 #except:
386 # pass;
387 return oFile;
388
389def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
390 """
391 Wrapper around open() that tries it's best to make sure the file isn't
392 inherited by child processes.
393
394 This is a best effort thing at the moment as it doesn't synchronizes with
395 child process spawning in any way. Thus it can be subject to races in
396 multithreaded programs.
397 """
398
399 if getHostOs() == 'win':
400 # Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
401 # pylint: disable=no-member,c-extension-no-member
402 fAccess = 0;
403 fDisposition = win32file.OPEN_EXISTING;
404 if 'r' in sMode or '+' in sMode:
405 fAccess |= win32file.GENERIC_READ;
406 if 'a' in sMode:
407 fAccess |= win32file.GENERIC_WRITE;
408 fDisposition = win32file.OPEN_ALWAYS;
409 elif 'w' in sMode:
410 fAccess = win32file.GENERIC_WRITE;
411 if '+' in sMode:
412 fDisposition = win32file.OPEN_ALWAYS;
413 fAccess |= win32file.GENERIC_READ;
414 else:
415 fDisposition = win32file.CREATE_ALWAYS;
416 if not fAccess:
417 fAccess |= win32file.GENERIC_READ;
418 fSharing = (win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE
419 | win32file.FILE_SHARE_DELETE);
420 hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
421 if 'a' in sMode:
422 win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
423
424 # Turn the NT handle into a CRT file descriptor.
425 hDetachedFile = hFile.Detach();
426 if fAccess == win32file.GENERIC_READ:
427 fOpen = os.O_RDONLY;
428 elif fAccess == win32file.GENERIC_WRITE:
429 fOpen = os.O_WRONLY;
430 else:
431 fOpen = os.O_RDWR;
432 # pulint: enable=no-member,c-extension-no-member
433 if 'a' in sMode:
434 fOpen |= os.O_APPEND;
435 if 'b' in sMode or 't' in sMode:
436 fOpen |= os.O_TEXT; # pylint: disable=no-member
437 fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
438
439 # Tell python to use this handle.
440 oFile = os.fdopen(fdFile, sMode);
441 else:
442 oFile = open(sFile, sMode);
443
444 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
445 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
446 if uPythonVer < ((3 << 16) | 4):
447 try:
448 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
449 except:
450 pass;
451 else:
452 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
453 return oFile;
454
455def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
456 """
457 No exceptions os.readlink wrapper.
458 """
459 try:
460 sRet = os.readlink(sPath); # pylint: disable=no-member
461 except:
462 return sXcptRet;
463 if hasattr(sRet, 'decode'):
464 sRet = sRet.decode(sEncoding, 'ignore');
465 return sRet;
466
467def readFile(sFile, sMode = 'rb'):
468 """
469 Reads the entire file.
470 """
471 oFile = open(sFile, sMode);
472 sRet = oFile.read();
473 oFile.close();
474 return sRet;
475
476def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
477 """
478 No exceptions common.readFile wrapper.
479 """
480 try:
481 sRet = readFile(sFile, sMode);
482 except:
483 sRet = sXcptRet;
484 if sEncoding is not None and hasattr(sRet, 'decode'):
485 sRet = sRet.decode(sEncoding, 'ignore');
486 return sRet;
487
488def noxcptRmDir(sDir, oXcptRet = False):
489 """
490 No exceptions os.rmdir wrapper.
491 """
492 oRet = True;
493 try:
494 os.rmdir(sDir);
495 except:
496 oRet = oXcptRet;
497 return oRet;
498
499def noxcptDeleteFile(sFile, oXcptRet = False):
500 """
501 No exceptions os.remove wrapper.
502 """
503 oRet = True;
504 try:
505 os.remove(sFile);
506 except:
507 oRet = oXcptRet;
508 return oRet;
509
510
511def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
512 # type: (string, (string, stat) -> bool) -> bool
513 """
514 Recursively walks a directory tree, calling fnCallback for each.
515
516 fnCallback takes a full path and stat object (can be None). It
517 returns a boolean value, False stops walking and returns immediately.
518
519 Returns True or False depending on fnCallback.
520 Returns None fIgnoreExceptions is True and an exception was raised by listdir.
521 """
522 def __worker(sCurDir):
523 """ Worker for """
524 try:
525 asNames = os.listdir(sCurDir);
526 except:
527 if not fIgnoreExceptions:
528 raise;
529 return None;
530 rc = True;
531 for sName in asNames:
532 if sName not in [ '.', '..' ]:
533 sFullName = os.path.join(sCurDir, sName);
534 try: oStat = os.lstat(sFullName);
535 except: oStat = None;
536 if fnCallback(sFullName, oStat) is False:
537 return False;
538 if oStat is not None and stat.S_ISDIR(oStat.st_mode):
539 rc = __worker(sFullName);
540 if rc is False:
541 break;
542 return rc;
543
544 # Ensure unicode path here so listdir also returns unicode on windows.
545 ## @todo figure out unicode stuff on non-windows.
546 if sys.platform == 'win32':
547 sDir = unicode(sDir);
548 return __worker(sDir);
549
550
551
552def formatFileMode(uMode):
553 # type: (int) -> string
554 """
555 Format a st_mode value 'ls -la' fasion.
556 Returns string.
557 """
558 if stat.S_ISDIR(uMode): sMode = 'd';
559 elif stat.S_ISREG(uMode): sMode = '-';
560 elif stat.S_ISLNK(uMode): sMode = 'l';
561 elif stat.S_ISFIFO(uMode): sMode = 'p';
562 elif stat.S_ISCHR(uMode): sMode = 'c';
563 elif stat.S_ISBLK(uMode): sMode = 'b';
564 elif stat.S_ISSOCK(uMode): sMode = 's';
565 else: sMode = '?';
566 ## @todo sticky bits.
567 sMode += 'r' if uMode & stat.S_IRUSR else '-';
568 sMode += 'w' if uMode & stat.S_IWUSR else '-';
569 sMode += 'x' if uMode & stat.S_IXUSR else '-';
570 sMode += 'r' if uMode & stat.S_IRGRP else '-';
571 sMode += 'w' if uMode & stat.S_IWGRP else '-';
572 sMode += 'x' if uMode & stat.S_IXGRP else '-';
573 sMode += 'r' if uMode & stat.S_IROTH else '-';
574 sMode += 'w' if uMode & stat.S_IWOTH else '-';
575 sMode += 'x' if uMode & stat.S_IXOTH else '-';
576 sMode += ' ';
577 return sMode;
578
579
580def formatFileStat(oStat):
581 # type: (stat) -> string
582 """
583 Format a stat result 'ls -la' fasion (numeric IDs).
584 Returns string.
585 """
586 return '%s %3s %4s %4s %10s %s' \
587 % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
588 time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
589
590## Good buffer for file operations.
591g_cbGoodBufferSize = 256*1024;
592
593## The original shutil.copyfileobj.
594g_fnOriginalShCopyFileObj = None;
595
596def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
597 """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
598 return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
599
600def __installShUtilHacks(shutil):
601 """ Installs the shutil buffer size hacks. """
602 global g_fnOriginalShCopyFileObj;
603 if g_fnOriginalShCopyFileObj is None:
604 g_fnOriginalShCopyFileObj = shutil.copyfileobj;
605 shutil.copyfileobj = __myshutilcopyfileobj;
606 return True;
607
608
609def copyFileSimple(sFileSrc, sFileDst):
610 """
611 Wrapper around shutil.copyfile that simply copies the data of a regular file.
612 Raises exception on failure.
613 Return True for show.
614 """
615 import shutil;
616 __installShUtilHacks(shutil);
617 return shutil.copyfile(sFileSrc, sFileDst);
618
619
620def getDiskUsage(sPath):
621 """
622 Get free space of a partition that corresponds to specified sPath in MB.
623
624 Returns partition free space value in MB.
625 """
626 if platform.system() == 'Windows':
627 oCTypeFreeSpace = ctypes.c_ulonglong(0);
628 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
629 ctypes.pointer(oCTypeFreeSpace));
630 cbFreeSpace = oCTypeFreeSpace.value;
631 else:
632 oStats = os.statvfs(sPath); # pylint: disable=no-member
633 cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
634
635 # Convert to MB
636 cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
637
638 return cMbFreeSpace;
639
640
641
642#
643# SubProcess.
644#
645
646def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
647 """
648 If the "executable" is a python script, insert the python interpreter at
649 the head of the argument list so that it will work on systems which doesn't
650 support hash-bang scripts.
651 """
652
653 asArgs = dKeywordArgs.get('args');
654 if asArgs is None:
655 asArgs = aPositionalArgs[0];
656
657 if asArgs[0].endswith('.py'):
658 if sys.executable:
659 asArgs.insert(0, sys.executable);
660 else:
661 asArgs.insert(0, 'python');
662
663 # paranoia...
664 if dKeywordArgs.get('args') is not None:
665 dKeywordArgs['args'] = asArgs;
666 else:
667 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
668 return None;
669
670def processPopenSafe(*aPositionalArgs, **dKeywordArgs):
671 """
672 Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
673 """
674 if getHostOs() == 'win':
675 if dKeywordArgs.get('creationflags', 0) == 0:
676 dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
677 return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
678
679def processCall(*aPositionalArgs, **dKeywordArgs):
680 """
681 Wrapper around subprocess.call to deal with its absence in older
682 python versions.
683 Returns process exit code (see subprocess.poll).
684 """
685 assert dKeywordArgs.get('stdout') is None;
686 assert dKeywordArgs.get('stderr') is None;
687 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
688 oProcess = processPopenSafe(*aPositionalArgs, **dKeywordArgs);
689 return oProcess.wait();
690
691def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
692 """
693 Wrapper around subprocess.check_output to deal with its absense in older
694 python versions.
695
696 Extra keywords for specifying now output is to be decoded:
697 sEncoding='utf-8
698 fIgnoreEncoding=True/False
699 """
700 sEncoding = dKeywordArgs.get('sEncoding');
701 if sEncoding is not None: del dKeywordArgs['sEncoding'];
702 else: sEncoding = 'utf-8';
703
704 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
705 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
706 else: fIgnoreEncoding = True;
707
708 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
709 oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
710
711 sOutput, _ = oProcess.communicate();
712 iExitCode = oProcess.poll();
713
714 if iExitCode != 0:
715 asArgs = dKeywordArgs.get('args');
716 if asArgs is None:
717 asArgs = aPositionalArgs[0];
718 print(sOutput);
719 raise subprocess.CalledProcessError(iExitCode, asArgs);
720
721 if hasattr(sOutput, 'decode'):
722 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
723 return sOutput;
724
725def processOutputUnchecked(*aPositionalArgs, **dKeywordArgs):
726 """
727 Similar to processOutputChecked, but returns status code and both stdout
728 and stderr results.
729
730 Extra keywords for specifying now output is to be decoded:
731 sEncoding='utf-8
732 fIgnoreEncoding=True/False
733 """
734 sEncoding = dKeywordArgs.get('sEncoding');
735 if sEncoding is not None: del dKeywordArgs['sEncoding'];
736 else: sEncoding = 'utf-8';
737
738 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
739 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
740 else: fIgnoreEncoding = True;
741
742 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
743 oProcess = processPopenSafe(stdout = subprocess.PIPE, stderr = subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
744
745 sOutput, sError = oProcess.communicate();
746 iExitCode = oProcess.poll();
747
748 if hasattr(sOutput, 'decode'):
749 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
750 if hasattr(sError, 'decode'):
751 sError = sError.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
752 return (iExitCode, sOutput, sError);
753
754g_fOldSudo = None;
755def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
756 """
757 Adds 'sudo' (or similar) to the args parameter, whereever it is.
758 """
759
760 # Are we root?
761 fIsRoot = True;
762 try:
763 fIsRoot = os.getuid() == 0; # pylint: disable=no-member
764 except:
765 pass;
766
767 # If not, prepend sudo (non-interactive, simulate initial login).
768 if fIsRoot is not True:
769 asArgs = dKeywordArgs.get('args');
770 if asArgs is None:
771 asArgs = aPositionalArgs[0];
772
773 # Detect old sudo.
774 global g_fOldSudo;
775 if g_fOldSudo is None:
776 try:
777 sVersion = str(processOutputChecked(['sudo', '-V']));
778 except:
779 sVersion = '1.7.0';
780 sVersion = sVersion.strip().split('\n')[0];
781 sVersion = sVersion.replace('Sudo version', '').strip();
782 g_fOldSudo = len(sVersion) >= 4 \
783 and sVersion[0] == '1' \
784 and sVersion[1] == '.' \
785 and sVersion[2] <= '6' \
786 and sVersion[3] == '.';
787
788 asArgs.insert(0, 'sudo');
789 if not g_fOldSudo:
790 asArgs.insert(1, '-n');
791 if fInitialEnv and not g_fOldSudo:
792 asArgs.insert(1, '-i');
793
794 # paranoia...
795 if dKeywordArgs.get('args') is not None:
796 dKeywordArgs['args'] = asArgs;
797 else:
798 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
799 return None;
800
801
802def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
803 """
804 sudo (or similar) + subprocess.call
805 """
806 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
807 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
808 return processCall(*aPositionalArgs, **dKeywordArgs);
809
810def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
811 """
812 sudo (or similar) + subprocess.check_output.
813 """
814 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
815 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
816 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
817
818def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
819 """
820 sudo (or similar) + subprocess.check_output, except '-i' isn't used.
821 """
822 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
823 _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
824 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
825
826def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
827 """
828 sudo (or similar) + processPopenSafe.
829 """
830 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
831 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
832 return processPopenSafe(*aPositionalArgs, **dKeywordArgs);
833
834
835def whichProgram(sName, sPath = None):
836 """
837 Works similar to the 'which' utility on unix.
838
839 Returns path to the given program if found.
840 Returns None if not found.
841 """
842 sHost = getHostOs();
843 sSep = ';' if sHost in [ 'win', 'os2' ] else ':';
844
845 if sPath is None:
846 if sHost == 'win':
847 sPath = os.environ.get('Path', None);
848 else:
849 sPath = os.environ.get('PATH', None);
850 if sPath is None:
851 return None;
852
853 for sDir in sPath.split(sSep):
854 if sDir.strip() != '':
855 sTest = os.path.abspath(os.path.join(sDir, sName));
856 else:
857 sTest = os.path.abspath(sName);
858 if os.path.exists(sTest):
859 return sTest;
860
861 return None;
862
863#
864# Generic process stuff.
865#
866
867def processInterrupt(uPid):
868 """
869 Sends a SIGINT or equivalent to interrupt the specified process.
870 Returns True on success, False on failure.
871
872 On Windows hosts this may not work unless the process happens to be a
873 process group leader.
874 """
875 if sys.platform == 'win32':
876 try:
877 win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
878 uPid);
879 fRc = True;
880 except:
881 fRc = False;
882 else:
883 try:
884 os.kill(uPid, signal.SIGINT);
885 fRc = True;
886 except:
887 fRc = False;
888 return fRc;
889
890def sendUserSignal1(uPid):
891 """
892 Sends a SIGUSR1 or equivalent to nudge the process into shutting down
893 (VBoxSVC) or something.
894 Returns True on success, False on failure or if not supported (win).
895
896 On Windows hosts this may not work unless the process happens to be a
897 process group leader.
898 """
899 if sys.platform == 'win32':
900 fRc = False;
901 else:
902 try:
903 os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
904 fRc = True;
905 except:
906 fRc = False;
907 return fRc;
908
909def processTerminate(uPid):
910 """
911 Terminates the process in a nice manner (SIGTERM or equivalent).
912 Returns True on success, False on failure.
913 """
914 fRc = False;
915 if sys.platform == 'win32':
916 try:
917 hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
918 False, uPid);
919 except:
920 pass;
921 else:
922 try:
923 win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
924 0x40010004); # DBG_TERMINATE_PROCESS
925 fRc = True;
926 except:
927 pass;
928 hProcess.Close(); #win32api.CloseHandle(hProcess)
929 else:
930 try:
931 os.kill(uPid, signal.SIGTERM);
932 fRc = True;
933 except:
934 pass;
935 return fRc;
936
937def processKill(uPid):
938 """
939 Terminates the process with extreme prejudice (SIGKILL).
940 Returns True on success, False on failure.
941 """
942 if sys.platform == 'win32':
943 fRc = processTerminate(uPid);
944 else:
945 try:
946 os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
947 fRc = True;
948 except:
949 fRc = False;
950 return fRc;
951
952def processKillWithNameCheck(uPid, sName):
953 """
954 Like processKill(), but checks if the process name matches before killing
955 it. This is intended for killing using potentially stale pid values.
956
957 Returns True on success, False on failure.
958 """
959
960 if processCheckPidAndName(uPid, sName) is not True:
961 return False;
962 return processKill(uPid);
963
964
965def processExists(uPid):
966 """
967 Checks if the specified process exits.
968 This will only work if we can signal/open the process.
969
970 Returns True if it positively exists, False otherwise.
971 """
972 sHostOs = getHostOs();
973 if sHostOs == 'win':
974 fRc = False;
975 # We try open the process for waiting since this is generally only forbidden in a very few cases.
976 try:
977 hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
978 False, uPid);
979 except pywintypes.error as oXcpt: # pylint: disable=no-member
980 if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
981 fRc = True;
982 except Exception as oXcpt:
983 pass;
984 else:
985 hProcess.Close();
986 fRc = True;
987 else:
988 fRc = False;
989 try:
990 os.kill(uPid, 0);
991 fRc = True;
992 except OSError as oXcpt:
993 if oXcpt.errno == errno.EPERM:
994 fRc = True;
995 except:
996 pass;
997 return fRc;
998
999def processCheckPidAndName(uPid, sName):
1000 """
1001 Checks if a process PID and NAME matches.
1002 """
1003 fRc = processExists(uPid);
1004 if fRc is not True:
1005 return False;
1006
1007 if sys.platform == 'win32':
1008 try:
1009 from win32com.client import GetObject; # pylint: disable=import-error
1010 oWmi = GetObject('winmgmts:');
1011 aoProcesses = oWmi.InstancesOf('Win32_Process');
1012 for oProcess in aoProcesses:
1013 if long(oProcess.Properties_("ProcessId").Value) == uPid:
1014 sCurName = oProcess.Properties_("Name").Value;
1015 #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
1016 sName = sName.lower();
1017 sCurName = sCurName.lower();
1018 if os.path.basename(sName) == sName:
1019 sCurName = os.path.basename(sCurName);
1020
1021 if sCurName == sName \
1022 or sCurName + '.exe' == sName \
1023 or sCurName == sName + '.exe':
1024 fRc = True;
1025 break;
1026 except:
1027 #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
1028 pass;
1029 else:
1030 if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
1031 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1032 elif sys.platform in ('sunos5',):
1033 asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
1034 elif sys.platform in ('darwin',):
1035 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
1036 else:
1037 asPsCmd = None;
1038
1039 if asPsCmd is not None:
1040 try:
1041 oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
1042 sCurName = oPs.communicate()[0];
1043 iExitCode = oPs.wait();
1044 except:
1045 #reporter.logXcpt();
1046 return False;
1047
1048 # ps fails with non-zero exit code if the pid wasn't found.
1049 if iExitCode != 0:
1050 return False;
1051 if sCurName is None:
1052 return False;
1053 sCurName = sCurName.strip();
1054 if not sCurName:
1055 return False;
1056
1057 if os.path.basename(sName) == sName:
1058 sCurName = os.path.basename(sCurName);
1059 elif os.path.basename(sCurName) == sCurName:
1060 sName = os.path.basename(sName);
1061
1062 if sCurName != sName:
1063 return False;
1064
1065 fRc = True;
1066 return fRc;
1067
1068def processGetInfo(uPid, fSudo = False):
1069 """
1070 Tries to acquire state information of the given process.
1071
1072 Returns a string with the information on success or None on failure or
1073 if the host is not supported.
1074
1075 Note that the format of the information is host system dependent and will
1076 likely differ much between different hosts.
1077 """
1078 fRc = processExists(uPid);
1079 if fRc is not True:
1080 return None;
1081
1082 sHostOs = getHostOs();
1083 if sHostOs in [ 'linux',]:
1084 sGdb = '/usr/bin/gdb';
1085 if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1086 if not os.path.isfile(sGdb): sGdb = 'gdb';
1087 aasCmd = [
1088 [ sGdb, '-batch',
1089 '-ex', 'set pagination off',
1090 '-ex', 'thread apply all bt',
1091 '-ex', 'info proc mapping',
1092 '-ex', 'info sharedlibrary',
1093 '-p', '%u' % (uPid,), ],
1094 ];
1095 elif sHostOs == 'darwin':
1096 # LLDB doesn't work in batch mode when attaching to a process, at least
1097 # with macOS Sierra (10.12). GDB might not be installed. Use the sample
1098 # tool instead with a 1 second duration and 1000ms sampling interval to
1099 # get one stack trace. For the process mappings use vmmap.
1100 aasCmd = [
1101 [ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1102 [ '/usr/bin/vmmap', '%u' % (uPid,), ],
1103 ];
1104 elif sHostOs == 'solaris':
1105 aasCmd = [
1106 [ '/usr/bin/pstack', '%u' % (uPid,), ],
1107 [ '/usr/bin/pmap', '%u' % (uPid,), ],
1108 ];
1109 else:
1110 aasCmd = [];
1111
1112 sInfo = '';
1113 for asCmd in aasCmd:
1114 try:
1115 if fSudo:
1116 sThisInfo = sudoProcessOutputChecked(asCmd);
1117 else:
1118 sThisInfo = processOutputChecked(asCmd);
1119 if sThisInfo is not None:
1120 sInfo += sThisInfo;
1121 except:
1122 pass;
1123 if not sInfo:
1124 sInfo = None;
1125
1126 return sInfo;
1127
1128
1129class ProcessInfo(object):
1130 """Process info."""
1131 def __init__(self, iPid):
1132 self.iPid = iPid;
1133 self.iParentPid = None;
1134 self.sImage = None;
1135 self.sName = None;
1136 self.asArgs = None;
1137 self.sCwd = None;
1138 self.iGid = None;
1139 self.iUid = None;
1140 self.iProcGroup = None;
1141 self.iSessionId = None;
1142
1143 def loadAll(self):
1144 """Load all the info."""
1145 sOs = getHostOs();
1146 if sOs == 'linux':
1147 sProc = '/proc/%s/' % (self.iPid,);
1148 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1149 if self.sImage is None:
1150 self.sImage = noxcptReadFile(sProc + 'comm', None);
1151 if self.sImage: self.sImage = self.sImage.strip();
1152 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1153 if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1154 #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1155 # sProc = '/proc/%s/' % (self.iPid,);
1156 # if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1157 # if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1158 else:
1159 pass;
1160 if self.sName is None and self.sImage is not None:
1161 self.sName = self.sImage;
1162
1163 def windowsGrabProcessInfo(self, oProcess):
1164 """Windows specific loadAll."""
1165 try: self.sName = oProcess.Properties_("Name").Value;
1166 except: pass;
1167 try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1168 except: pass;
1169 try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1170 except: pass;
1171 try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1172 except: pass;
1173 try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1174 except: pass;
1175 if self.sName is None and self.sImage is not None:
1176 self.sName = self.sImage;
1177
1178 def getBaseImageName(self):
1179 """
1180 Gets the base image name if available, use the process name if not available.
1181 Returns image/process base name or None.
1182 """
1183 sRet = self.sImage if self.sName is None else self.sName;
1184 if sRet is None:
1185 self.loadAll();
1186 sRet = self.sImage if self.sName is None else self.sName;
1187 if sRet is None:
1188 if not self.asArgs:
1189 return None;
1190 sRet = self.asArgs[0];
1191 if not sRet:
1192 return None;
1193 return os.path.basename(sRet);
1194
1195 def getBaseImageNameNoExeSuff(self):
1196 """
1197 Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1198 """
1199 sRet = self.getBaseImageName();
1200 if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1201 if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1202 sRet = sRet[:-4];
1203 return sRet;
1204
1205
1206def processListAll():
1207 """
1208 Return a list of ProcessInfo objects for all the processes in the system
1209 that the current user can see.
1210 """
1211 asProcesses = [];
1212
1213 sOs = getHostOs();
1214 if sOs == 'win':
1215 from win32com.client import GetObject; # pylint: disable=import-error
1216 oWmi = GetObject('winmgmts:');
1217 aoProcesses = oWmi.InstancesOf('Win32_Process');
1218 for oProcess in aoProcesses:
1219 try:
1220 iPid = int(oProcess.Properties_("ProcessId").Value);
1221 except:
1222 continue;
1223 oMyInfo = ProcessInfo(iPid);
1224 oMyInfo.windowsGrabProcessInfo(oProcess);
1225 asProcesses.append(oMyInfo);
1226 return asProcesses;
1227
1228 if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1229 try:
1230 asDirs = os.listdir('/proc');
1231 except:
1232 asDirs = [];
1233 for sDir in asDirs:
1234 if sDir.isdigit():
1235 asProcesses.append(ProcessInfo(int(sDir),));
1236 return asProcesses;
1237
1238 #
1239 # The other OSes parses the output from the 'ps' utility.
1240 #
1241 asPsCmd = [
1242 '/bin/ps', # 0
1243 '-A', # 1
1244 '-o', 'pid=', # 2,3
1245 '-o', 'ppid=', # 4,5
1246 '-o', 'pgid=', # 6,7
1247 '-o', 'sid=', # 8,9
1248 '-o', 'uid=', # 10,11
1249 '-o', 'gid=', # 12,13
1250 '-o', 'comm=' # 14,15
1251 ];
1252
1253 if sOs == 'darwin':
1254 assert asPsCmd[9] == 'sid=';
1255 asPsCmd[9] = 'sess=';
1256 elif sOs == 'solaris':
1257 asPsCmd[0] = '/usr/bin/ps';
1258
1259 try:
1260 sRaw = processOutputChecked(asPsCmd);
1261 except:
1262 return asProcesses;
1263
1264 for sLine in sRaw.split('\n'):
1265 sLine = sLine.lstrip();
1266 if len(sLine) < 7 or not sLine[0].isdigit():
1267 continue;
1268
1269 iField = 0;
1270 off = 0;
1271 aoFields = [None, None, None, None, None, None, None];
1272 while iField < 7:
1273 # Eat whitespace.
1274 while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1275 off += 1;
1276
1277 # Final field / EOL.
1278 if iField == 6:
1279 aoFields[6] = sLine[off:];
1280 break;
1281 if off >= len(sLine):
1282 break;
1283
1284 # Generic field parsing.
1285 offStart = off;
1286 off += 1;
1287 while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1288 off += 1;
1289 try:
1290 if iField != 3:
1291 aoFields[iField] = int(sLine[offStart:off]);
1292 else:
1293 aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1294 except:
1295 pass;
1296 iField += 1;
1297
1298 if aoFields[0] is not None:
1299 oMyInfo = ProcessInfo(aoFields[0]);
1300 oMyInfo.iParentPid = aoFields[1];
1301 oMyInfo.iProcGroup = aoFields[2];
1302 oMyInfo.iSessionId = aoFields[3];
1303 oMyInfo.iUid = aoFields[4];
1304 oMyInfo.iGid = aoFields[5];
1305 oMyInfo.sName = aoFields[6];
1306 asProcesses.append(oMyInfo);
1307
1308 return asProcesses;
1309
1310
1311def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1312 """
1313 Looks for information regarding the demise of the given process.
1314 """
1315 sOs = getHostOs();
1316 if sOs == 'darwin':
1317 #
1318 # On darwin we look for crash and diagnostic reports.
1319 #
1320 asLogDirs = [
1321 u'/Library/Logs/DiagnosticReports/',
1322 u'/Library/Logs/CrashReporter/',
1323 u'~/Library/Logs/DiagnosticReports/',
1324 u'~/Library/Logs/CrashReporter/',
1325 ];
1326 for sDir in asLogDirs:
1327 sDir = os.path.expanduser(sDir);
1328 if not os.path.isdir(sDir):
1329 continue;
1330 try:
1331 asDirEntries = os.listdir(sDir);
1332 except:
1333 continue;
1334 for sEntry in asDirEntries:
1335 # Only interested in .crash files.
1336 _, sSuff = os.path.splitext(sEntry);
1337 if sSuff != '.crash':
1338 continue;
1339
1340 # The pid can be found at the end of the first line.
1341 sFull = os.path.join(sDir, sEntry);
1342 try:
1343 oFile = open(sFull, 'r');
1344 sFirstLine = oFile.readline();
1345 oFile.close();
1346 except:
1347 continue;
1348 if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1349 continue;
1350 offPid = len(sFirstLine) - 3;
1351 while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1352 offPid -= 1;
1353 try: uReportPid = int(sFirstLine[offPid:-2]);
1354 except: continue;
1355
1356 # Does the pid we found match?
1357 if uReportPid == uPid:
1358 fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1359 fnCrashFile(sFull, False);
1360 elif sOs == 'win':
1361 #
1362 # Getting WER reports would be great, however we have trouble match the
1363 # PID to those as they seems not to mention it in the brief reports.
1364 # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1365 # location - see the windows readme for the testbox script) and what
1366 # the MSDN article lists for now.
1367 #
1368 # It's been observed on Windows server 2012 that the dump files takes
1369 # the form: <processimage>.<decimal-pid>.dmp
1370 #
1371 asDmpDirs = [
1372 u'%SystemDrive%/CrashDumps/', # Testboxes.
1373 u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1374 u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1375 u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1376 u'%WINDIR%/ServiceProfiles/',
1377 u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1378 ];
1379 sMatchSuffix = '.%u.dmp' % (uPid,);
1380
1381 for sDir in asDmpDirs:
1382 sDir = os.path.expandvars(sDir);
1383 if not os.path.isdir(sDir):
1384 continue;
1385 try:
1386 asDirEntries = os.listdir(sDir);
1387 except:
1388 continue;
1389 for sEntry in asDirEntries:
1390 if sEntry.endswith(sMatchSuffix):
1391 sFull = os.path.join(sDir, sEntry);
1392 fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1393 fnCrashFile(sFull, True);
1394
1395 else:
1396 pass; ## TODO
1397 return None;
1398
1399
1400#
1401# Time.
1402#
1403
1404#
1405# The following test case shows how time.time() only have ~ms resolution
1406# on Windows (tested W10) and why it therefore makes sense to try use
1407# performance counters.
1408#
1409# Note! We cannot use time.clock() as the timestamp must be portable across
1410# processes. See timeout testcase problem on win hosts (no logs).
1411# Also, time.clock() was axed in python 3.8 (https://bugs.python.org/issue31803).
1412#
1413#import sys;
1414#import time;
1415#from common import utils;
1416#
1417#atSeries = [];
1418#for i in xrange(1,160):
1419# if i == 159: time.sleep(10);
1420# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1421#
1422#tPrev = atSeries[0]
1423#for tCur in atSeries:
1424# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1425# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1426# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1427# print '';
1428# tPrev = tCur
1429#
1430#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1431#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1432#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1433
1434g_fWinUseWinPerfCounter = sys.platform == 'win32';
1435g_fpWinPerfCounterFreq = None;
1436g_oFuncwinQueryPerformanceCounter = None;
1437
1438def _winInitPerfCounter():
1439 """ Initializes the use of performance counters. """
1440 global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1441
1442 uFrequency = ctypes.c_ulonglong(0);
1443 if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1444 if uFrequency.value >= 1000:
1445 #print 'uFrequency = %s' % (uFrequency,);
1446 #print 'type(uFrequency) = %s' % (type(uFrequency),);
1447 g_fpWinPerfCounterFreq = float(uFrequency.value);
1448
1449 # Check that querying the counter works too.
1450 global g_oFuncwinQueryPerformanceCounter
1451 g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1452 uCurValue = ctypes.c_ulonglong(0);
1453 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1454 if uCurValue.value > 0:
1455 return True;
1456 g_fWinUseWinPerfCounter = False;
1457 return False;
1458
1459def _winFloatTime():
1460 """ Gets floating point time on windows. """
1461 if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1462 uCurValue = ctypes.c_ulonglong(0);
1463 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1464 return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1465 return time.time();
1466
1467def timestampNano():
1468 """
1469 Gets a nanosecond timestamp.
1470 """
1471 if g_fWinUseWinPerfCounter is True:
1472 return long(_winFloatTime() * 1000000000);
1473 return long(time.time() * 1000000000);
1474
1475def timestampMilli():
1476 """
1477 Gets a millisecond timestamp.
1478 """
1479 if g_fWinUseWinPerfCounter is True:
1480 return long(_winFloatTime() * 1000);
1481 return long(time.time() * 1000);
1482
1483def timestampSecond():
1484 """
1485 Gets a second timestamp.
1486 """
1487 if g_fWinUseWinPerfCounter is True:
1488 return long(_winFloatTime());
1489 return long(time.time());
1490
1491def secondsSinceUnixEpoch():
1492 """
1493 Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1494 """
1495 ## ASSUMES This returns unix epoch time on all systems we care about...
1496 return time.time();
1497
1498def getTimePrefix():
1499 """
1500 Returns a timestamp prefix, typically used for logging. UTC.
1501 """
1502 try:
1503 oNow = datetime.datetime.utcnow();
1504 sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1505 except:
1506 sTs = 'getTimePrefix-exception';
1507 return sTs;
1508
1509def getTimePrefixAndIsoTimestamp():
1510 """
1511 Returns current UTC as log prefix and iso timestamp.
1512 """
1513 try:
1514 oNow = datetime.datetime.utcnow();
1515 sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1516 sTsIso = formatIsoTimestamp(oNow);
1517 except:
1518 sTsPrf = sTsIso = 'getTimePrefix-exception';
1519 return (sTsPrf, sTsIso);
1520
1521class UtcTzInfo(datetime.tzinfo):
1522 """UTC TZ Info Class"""
1523 def utcoffset(self, _):
1524 return datetime.timedelta(0);
1525 def tzname(self, _):
1526 return "UTC";
1527 def dst(self, _):
1528 return datetime.timedelta(0);
1529
1530class GenTzInfo(datetime.tzinfo):
1531 """Generic TZ Info Class"""
1532 def __init__(self, offInMin):
1533 datetime.tzinfo.__init__(self);
1534 self.offInMin = offInMin;
1535 def utcoffset(self, _):
1536 return datetime.timedelta(minutes = self.offInMin);
1537 def tzname(self, _):
1538 if self.offInMin >= 0:
1539 return "+%02d%02d" % (self.offInMin // 60, self.offInMin % 60);
1540 return "-%02d%02d" % (-self.offInMin // 60, -self.offInMin % 60);
1541 def dst(self, _):
1542 return datetime.timedelta(0);
1543
1544def formatIsoTimestamp(oNow):
1545 """Formats the datetime object as an ISO timestamp."""
1546 assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1547 sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1548 return sTs;
1549
1550def getIsoTimestamp():
1551 """Returns the current UTC timestamp as a string."""
1552 return formatIsoTimestamp(datetime.datetime.utcnow());
1553
1554def formatShortIsoTimestamp(oNow):
1555 """Formats the datetime object as an ISO timestamp, but w/o microseconds."""
1556 assert oNow.tzinfo is None or isinstance(oNow.tzinfo, UtcTzInfo);
1557 return oNow.strftime('%Y-%m-%dT%H:%M:%SZ');
1558
1559def getShortIsoTimestamp():
1560 """Returns the current UTC timestamp as a string, but w/o microseconds."""
1561 return formatShortIsoTimestamp(datetime.datetime.utcnow());
1562
1563def convertDateTimeToZulu(oDateTime):
1564 """ Converts oDateTime to zulu time if it has timezone info. """
1565 if oDateTime.tzinfo is not None:
1566 oDateTime = oDateTime.astimezone(UtcTzInfo());
1567 else:
1568 oDateTime = oDateTime.replace(tzinfo = UtcTzInfo());
1569 return oDateTime;
1570
1571def parseIsoTimestamp(sTs):
1572 """
1573 Parses a typical ISO timestamp, returing a datetime object, reasonably
1574 forgiving, but will throw weird indexing/conversion errors if the input
1575 is malformed.
1576 """
1577 # YYYY-MM-DD
1578 iYear = int(sTs[0:4]);
1579 assert(sTs[4] == '-');
1580 iMonth = int(sTs[5:7]);
1581 assert(sTs[7] == '-');
1582 iDay = int(sTs[8:10]);
1583
1584 # Skip separator
1585 sTime = sTs[10:];
1586 while sTime[0] in 'Tt \t\n\r':
1587 sTime = sTime[1:];
1588
1589 # HH:MM[:SS]
1590 iHour = int(sTime[0:2]);
1591 assert(sTime[2] == ':');
1592 iMin = int(sTime[3:5]);
1593 if sTime[5] == ':':
1594 iSec = int(sTime[6:8]);
1595
1596 # Fraction?
1597 offTime = 8;
1598 iMicroseconds = 0;
1599 if offTime < len(sTime) and sTime[offTime] in '.,':
1600 offTime += 1;
1601 cchFraction = 0;
1602 while offTime + cchFraction < len(sTime) and sTime[offTime + cchFraction] in '0123456789':
1603 cchFraction += 1;
1604 if cchFraction > 0:
1605 iMicroseconds = int(sTime[offTime : (offTime + cchFraction)]);
1606 offTime += cchFraction;
1607 while cchFraction < 6:
1608 iMicroseconds *= 10;
1609 cchFraction += 1;
1610 while cchFraction > 6:
1611 iMicroseconds = iMicroseconds // 10;
1612 cchFraction -= 1;
1613
1614 else:
1615 iSec = 0;
1616 iMicroseconds = 0;
1617 offTime = 5;
1618
1619 # Naive?
1620 if offTime >= len(sTime):
1621 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1622
1623 # Zulu?
1624 if offTime >= len(sTime) or sTime[offTime] in 'Zz':
1625 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = UtcTzInfo());
1626
1627 # Some kind of offset afterwards, and strptime is useless. sigh.
1628 if sTime[offTime] in '+-':
1629 chSign = sTime[offTime];
1630 offTime += 1;
1631 cMinTz = int(sTime[offTime : (offTime + 2)]) * 60;
1632 offTime += 2;
1633 if offTime < len(sTime) and sTime[offTime] in ':':
1634 offTime += 1;
1635 if offTime + 2 <= len(sTime):
1636 cMinTz += int(sTime[offTime : (offTime + 2)]);
1637 offTime += 2;
1638 assert offTime == len(sTime);
1639 if chSign == '-':
1640 cMinTz = -cMinTz;
1641 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds, tzinfo = GenTzInfo(cMinTz));
1642 assert False, sTs;
1643 return datetime.datetime(iYear, iMonth, iDay, iHour, iMin, iSec, iMicroseconds);
1644
1645def normalizeIsoTimestampToZulu(sTs):
1646 """
1647 Takes a iso timestamp string and normalizes it (basically parseIsoTimestamp
1648 + convertDateTimeToZulu + formatIsoTimestamp).
1649 Returns ISO tiemstamp string.
1650 """
1651 return formatIsoTimestamp(convertDateTimeToZulu(parseIsoTimestamp(sTs)));
1652
1653def getLocalHourOfWeek():
1654 """ Local hour of week (0 based). """
1655 oNow = datetime.datetime.now();
1656 return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1657
1658
1659def formatIntervalSeconds(cSeconds):
1660 """ Format a seconds interval into a nice 01h 00m 22s string """
1661 # Two simple special cases.
1662 if cSeconds < 60:
1663 return '%ss' % (cSeconds,);
1664 if cSeconds < 3600:
1665 cMins = cSeconds // 60;
1666 cSecs = cSeconds % 60;
1667 if cSecs == 0:
1668 return '%sm' % (cMins,);
1669 return '%sm %ss' % (cMins, cSecs,);
1670
1671 # Generic and a bit slower.
1672 cDays = cSeconds // 86400;
1673 cSeconds %= 86400;
1674 cHours = cSeconds // 3600;
1675 cSeconds %= 3600;
1676 cMins = cSeconds // 60;
1677 cSecs = cSeconds % 60;
1678 sRet = '';
1679 if cDays > 0:
1680 sRet = '%sd ' % (cDays,);
1681 if cHours > 0:
1682 sRet += '%sh ' % (cHours,);
1683 if cMins > 0:
1684 sRet += '%sm ' % (cMins,);
1685 if cSecs > 0:
1686 sRet += '%ss ' % (cSecs,);
1687 assert sRet; assert sRet[-1] == ' ';
1688 return sRet[:-1];
1689
1690def formatIntervalSeconds2(oSeconds):
1691 """
1692 Flexible input version of formatIntervalSeconds for use in WUI forms where
1693 data is usually already string form.
1694 """
1695 if isinstance(oSeconds, (int, long)):
1696 return formatIntervalSeconds(oSeconds);
1697 if not isString(oSeconds):
1698 try:
1699 lSeconds = long(oSeconds);
1700 except:
1701 pass;
1702 else:
1703 if lSeconds >= 0:
1704 return formatIntervalSeconds2(lSeconds);
1705 return oSeconds;
1706
1707def parseIntervalSeconds(sString):
1708 """
1709 Reverse of formatIntervalSeconds.
1710
1711 Returns (cSeconds, sError), where sError is None on success.
1712 """
1713
1714 # We might given non-strings, just return them without any fuss.
1715 if not isString(sString):
1716 if isinstance(sString, (int, long)) or sString is None:
1717 return (sString, None);
1718 ## @todo time/date objects?
1719 return (int(sString), None);
1720
1721 # Strip it and make sure it's not empty.
1722 sString = sString.strip();
1723 if not sString:
1724 return (0, 'Empty interval string.');
1725
1726 #
1727 # Split up the input into a list of 'valueN, unitN, ...'.
1728 #
1729 # Don't want to spend too much time trying to make re.split do exactly what
1730 # I need here, so please forgive the extra pass I'm making here.
1731 #
1732 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1733 asParts = [];
1734 for sPart in asRawParts:
1735 sPart = sPart.strip();
1736 if sPart:
1737 asParts.append(sPart);
1738 if not asParts:
1739 return (0, 'Empty interval string or something?');
1740
1741 #
1742 # Process them one or two at the time.
1743 #
1744 cSeconds = 0;
1745 asErrors = [];
1746 i = 0;
1747 while i < len(asParts):
1748 sNumber = asParts[i];
1749 i += 1;
1750 if sNumber.isdigit():
1751 iNumber = int(sNumber);
1752
1753 sUnit = 's';
1754 if i < len(asParts) and not asParts[i].isdigit():
1755 sUnit = asParts[i];
1756 i += 1;
1757
1758 sUnitLower = sUnit.lower();
1759 if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1760 pass;
1761 elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1762 iNumber *= 60;
1763 elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1764 iNumber *= 3600;
1765 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1766 iNumber *= 86400;
1767 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1768 iNumber *= 7 * 86400;
1769 else:
1770 asErrors.append('Unknown unit "%s".' % (sUnit,));
1771 cSeconds += iNumber;
1772 else:
1773 asErrors.append('Bad number "%s".' % (sNumber,));
1774 return (cSeconds, None if not asErrors else ' '.join(asErrors));
1775
1776def formatIntervalHours(cHours):
1777 """ Format a hours interval into a nice 1w 2d 1h string. """
1778 # Simple special cases.
1779 if cHours < 24:
1780 return '%sh' % (cHours,);
1781
1782 # Generic and a bit slower.
1783 cWeeks = cHours / (7 * 24);
1784 cHours %= 7 * 24;
1785 cDays = cHours / 24;
1786 cHours %= 24;
1787 sRet = '';
1788 if cWeeks > 0:
1789 sRet = '%sw ' % (cWeeks,);
1790 if cDays > 0:
1791 sRet = '%sd ' % (cDays,);
1792 if cHours > 0:
1793 sRet += '%sh ' % (cHours,);
1794 assert sRet; assert sRet[-1] == ' ';
1795 return sRet[:-1];
1796
1797def parseIntervalHours(sString):
1798 """
1799 Reverse of formatIntervalHours.
1800
1801 Returns (cHours, sError), where sError is None on success.
1802 """
1803
1804 # We might given non-strings, just return them without any fuss.
1805 if not isString(sString):
1806 if isinstance(sString, (int, long)) or sString is None:
1807 return (sString, None);
1808 ## @todo time/date objects?
1809 return (int(sString), None);
1810
1811 # Strip it and make sure it's not empty.
1812 sString = sString.strip();
1813 if not sString:
1814 return (0, 'Empty interval string.');
1815
1816 #
1817 # Split up the input into a list of 'valueN, unitN, ...'.
1818 #
1819 # Don't want to spend too much time trying to make re.split do exactly what
1820 # I need here, so please forgive the extra pass I'm making here.
1821 #
1822 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1823 asParts = [];
1824 for sPart in asRawParts:
1825 sPart = sPart.strip();
1826 if sPart:
1827 asParts.append(sPart);
1828 if not asParts:
1829 return (0, 'Empty interval string or something?');
1830
1831 #
1832 # Process them one or two at the time.
1833 #
1834 cHours = 0;
1835 asErrors = [];
1836 i = 0;
1837 while i < len(asParts):
1838 sNumber = asParts[i];
1839 i += 1;
1840 if sNumber.isdigit():
1841 iNumber = int(sNumber);
1842
1843 sUnit = 'h';
1844 if i < len(asParts) and not asParts[i].isdigit():
1845 sUnit = asParts[i];
1846 i += 1;
1847
1848 sUnitLower = sUnit.lower();
1849 if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1850 pass;
1851 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1852 iNumber *= 24;
1853 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1854 iNumber *= 7 * 24;
1855 else:
1856 asErrors.append('Unknown unit "%s".' % (sUnit,));
1857 cHours += iNumber;
1858 else:
1859 asErrors.append('Bad number "%s".' % (sNumber,));
1860 return (cHours, None if not asErrors else ' '.join(asErrors));
1861
1862
1863#
1864# Introspection.
1865#
1866
1867def getCallerName(oFrame=None, iFrame=2):
1868 """
1869 Returns the name of the caller's caller.
1870 """
1871 if oFrame is None:
1872 try:
1873 raise Exception();
1874 except:
1875 oFrame = sys.exc_info()[2].tb_frame.f_back;
1876 while iFrame > 1:
1877 if oFrame is not None:
1878 oFrame = oFrame.f_back;
1879 iFrame = iFrame - 1;
1880 if oFrame is not None:
1881 sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1882 return sName;
1883 return "unknown";
1884
1885
1886def getXcptInfo(cFrames = 1):
1887 """
1888 Gets text detailing the exception. (Good for logging.)
1889 Returns list of info strings.
1890 """
1891
1892 #
1893 # Try get exception info.
1894 #
1895 try:
1896 oType, oValue, oTraceback = sys.exc_info();
1897 except:
1898 oType = oValue = oTraceback = None;
1899 if oType is not None:
1900
1901 #
1902 # Try format the info
1903 #
1904 asRet = [];
1905 try:
1906 try:
1907 asRet = asRet + traceback.format_exception_only(oType, oValue);
1908 asTraceBack = traceback.format_tb(oTraceback);
1909 if cFrames is not None and cFrames <= 1:
1910 asRet.append(asTraceBack[-1]);
1911 else:
1912 asRet.append('Traceback:')
1913 for iFrame in range(min(cFrames, len(asTraceBack))):
1914 asRet.append(asTraceBack[-iFrame - 1]);
1915 asRet.append('Stack:')
1916 asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1917 except:
1918 asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1919
1920 if not asRet:
1921 asRet.append('No exception info...');
1922 except:
1923 asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1924 else:
1925 asRet = ['Couldn\'t find exception traceback.'];
1926 return asRet;
1927
1928
1929def getObjectTypeName(oObject):
1930 """
1931 Get the type name of the given object.
1932 """
1933 if oObject is None:
1934 return 'None';
1935
1936 # Get the type object.
1937 try:
1938 oType = type(oObject);
1939 except:
1940 return 'type-throws-exception';
1941
1942 # Python 2.x only: Handle old-style object wrappers.
1943 if sys.version_info[0] < 3:
1944 try:
1945 from types import InstanceType; # pylint: disable=no-name-in-module
1946 if oType == InstanceType:
1947 oType = oObject.__class__;
1948 except:
1949 pass;
1950
1951 # Get the name.
1952 try:
1953 return oType.__name__;
1954 except:
1955 return '__type__-throws-exception';
1956
1957
1958def chmodPlusX(sFile):
1959 """
1960 Makes the specified file or directory executable.
1961 Returns success indicator, no exceptions.
1962
1963 Note! Symbolic links are followed and the target will be changed.
1964 """
1965 try:
1966 oStat = os.stat(sFile);
1967 except:
1968 return False;
1969 try:
1970 os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
1971 except:
1972 return False;
1973 return True;
1974
1975
1976#
1977# TestSuite stuff.
1978#
1979
1980def isRunningFromCheckout(cScriptDepth = 1):
1981 """
1982 Checks if we're running from the SVN checkout or not.
1983 """
1984
1985 try:
1986 sFile = __file__;
1987 cScriptDepth = 1;
1988 except:
1989 sFile = sys.argv[0];
1990
1991 sDir = os.path.abspath(sFile);
1992 while cScriptDepth >= 0:
1993 sDir = os.path.dirname(sDir);
1994 if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1995 or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1996 return True;
1997 cScriptDepth -= 1;
1998
1999 return False;
2000
2001
2002#
2003# Bourne shell argument fun.
2004#
2005
2006
2007def argsSplit(sCmdLine):
2008 """
2009 Given a bourne shell command line invocation, split it up into arguments
2010 assuming IFS is space.
2011 Returns None on syntax error.
2012 """
2013 ## @todo bourne shell argument parsing!
2014 return sCmdLine.split(' ');
2015
2016def argsGetFirst(sCmdLine):
2017 """
2018 Given a bourne shell command line invocation, get return the first argument
2019 assuming IFS is space.
2020 Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
2021 """
2022 asArgs = argsSplit(sCmdLine);
2023 if not asArgs:
2024 return None;
2025
2026 return asArgs[0];
2027
2028#
2029# String helpers.
2030#
2031
2032def stricmp(sFirst, sSecond):
2033 """
2034 Compares to strings in an case insensitive fashion.
2035
2036 Python doesn't seem to have any way of doing the correctly, so this is just
2037 an approximation using lower.
2038 """
2039 if sFirst == sSecond:
2040 return 0;
2041 sLower1 = sFirst.lower();
2042 sLower2 = sSecond.lower();
2043 if sLower1 == sLower2:
2044 return 0;
2045 if sLower1 < sLower2:
2046 return -1;
2047 return 1;
2048
2049
2050def versionCompare(sVer1, sVer2):
2051 """
2052 Compares to version strings in a fashion similar to RTStrVersionCompare.
2053 """
2054
2055 ## @todo implement me!!
2056
2057 if sVer1 == sVer2:
2058 return 0;
2059 if sVer1 < sVer2:
2060 return -1;
2061 return 1;
2062
2063
2064def formatNumber(lNum, sThousandSep = ' '):
2065 """
2066 Formats a decimal number with pretty separators.
2067 """
2068 sNum = str(lNum);
2069 sRet = sNum[-3:];
2070 off = len(sNum) - 3;
2071 while off > 0:
2072 off -= 3;
2073 sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
2074 return sRet;
2075
2076
2077def formatNumberNbsp(lNum):
2078 """
2079 Formats a decimal number with pretty separators.
2080 """
2081 sRet = formatNumber(lNum);
2082 return unicode(sRet).replace(' ', u'\u00a0');
2083
2084
2085def isString(oString):
2086 """
2087 Checks if the object is a string object, hiding difference between python 2 and 3.
2088
2089 Returns True if it's a string of some kind.
2090 Returns False if not.
2091 """
2092 if sys.version_info[0] >= 3:
2093 return isinstance(oString, str);
2094 return isinstance(oString, basestring); # pylint: disable=undefined-variable
2095
2096
2097def hasNonAsciiCharacters(sText):
2098 """
2099 Returns True is specified string has non-ASCII characters, False if ASCII only.
2100 """
2101 if isString(sText):
2102 for ch in sText:
2103 if ord(ch) >= 128:
2104 return True;
2105 else:
2106 # Probably byte array or some such thing.
2107 for ch in sText:
2108 if ch >= 128 or ch < 0:
2109 return True;
2110 return False;
2111
2112
2113#
2114# Unpacking.
2115#
2116
2117def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2118 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2119 """
2120 Worker for unpackFile that deals with ZIP files, same function signature.
2121 """
2122 import zipfile
2123 if fnError is None:
2124 fnError = fnLog;
2125
2126 fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
2127
2128 # Open it.
2129 try: oZipFile = zipfile.ZipFile(sArchive, 'r')
2130 except Exception as oXcpt:
2131 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2132 return None;
2133
2134 # Extract all members.
2135 asMembers = [];
2136 try:
2137 for sMember in oZipFile.namelist():
2138 if fnFilter is None or fnFilter(sMember) is not False:
2139 if sMember.endswith('/'):
2140 os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
2141 else:
2142 oZipFile.extract(sMember, sDstDir);
2143 asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
2144 except Exception as oXcpt:
2145 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2146 asMembers = None;
2147
2148 # close it.
2149 try: oZipFile.close();
2150 except Exception as oXcpt:
2151 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2152 asMembers = None;
2153
2154 return asMembers;
2155
2156
2157## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
2158g_fTarCopyFileObjOverriddend = False;
2159
2160def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
2161 """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
2162 _ = bufsize;
2163 if length is None:
2164 __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
2165 elif length > 0:
2166 cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
2167 for _ in xrange(cFull):
2168 abBuffer = src.read(g_cbGoodBufferSize);
2169 dst.write(abBuffer);
2170 if len(abBuffer) != g_cbGoodBufferSize:
2171 raise exception('unexpected end of source file');
2172 if cbRemainder > 0:
2173 abBuffer = src.read(cbRemainder);
2174 dst.write(abBuffer);
2175 if len(abBuffer) != cbRemainder:
2176 raise exception('unexpected end of source file');
2177
2178
2179def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2180 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2181 """
2182 Worker for unpackFile that deals with tarballs, same function signature.
2183 """
2184 import shutil;
2185 import tarfile;
2186 if fnError is None:
2187 fnError = fnLog;
2188
2189 fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
2190
2191 #
2192 # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2193 # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2194 # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2195 #
2196 if True is True: # pylint: disable=comparison-with-itself
2197 __installShUtilHacks(shutil);
2198 global g_fTarCopyFileObjOverriddend;
2199 if g_fTarCopyFileObjOverriddend is False:
2200 g_fTarCopyFileObjOverriddend = True;
2201 #if sys.hexversion < 0x03060000:
2202 tarfile.copyfileobj = __mytarfilecopyfileobj;
2203
2204 #
2205 # Open it.
2206 #
2207 # Note! We not using 'r:*' because we cannot allow seeking compressed files!
2208 # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2209 #
2210 try:
2211 if sys.hexversion >= 0x03060000:
2212 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2213 else:
2214 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
2215 except Exception as oXcpt:
2216 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2217 return None;
2218
2219 # Extract all members.
2220 asMembers = [];
2221 try:
2222 for oTarInfo in oTarFile:
2223 try:
2224 if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2225 if oTarInfo.islnk():
2226 # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2227 # in the compressed tar stream. So, fall back on shutil.copy2 instead.
2228 sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2229 sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2230 sParentDir = os.path.dirname(sLinkFile);
2231 try: os.unlink(sLinkFile);
2232 except: pass;
2233 if sParentDir and not os.path.exists(sParentDir):
2234 os.makedirs(sParentDir);
2235 try: os.link(sLinkTarget, sLinkFile);
2236 except: shutil.copy2(sLinkTarget, sLinkFile);
2237 else:
2238 if oTarInfo.isdir():
2239 # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2240 oTarInfo.mode |= 0x1c0; # (octal: 0700)
2241 oTarFile.extract(oTarInfo, sDstDir);
2242 asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2243 except Exception as oXcpt:
2244 fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2245 for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2246 fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2247 for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2248 fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2249 asMembers = None;
2250 break;
2251 except Exception as oXcpt:
2252 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2253 asMembers = None;
2254
2255 #
2256 # Finally, close it.
2257 #
2258 try: oTarFile.close();
2259 except Exception as oXcpt:
2260 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2261 asMembers = None;
2262
2263 return asMembers;
2264
2265
2266def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2267 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2268 """
2269 Unpacks the given file if it has a know archive extension, otherwise do
2270 nothing.
2271
2272 fnLog & fnError both take a string parameter.
2273
2274 fnFilter takes a member name (string) and returns True if it's included
2275 and False if excluded.
2276
2277 Returns list of the extracted files (full path) on success.
2278 Returns empty list if not a supported archive format.
2279 Returns None on failure. Raises no exceptions.
2280 """
2281 sBaseNameLower = os.path.basename(sArchive).lower();
2282
2283 #
2284 # Zip file?
2285 #
2286 if sBaseNameLower.endswith('.zip'):
2287 return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2288
2289 #
2290 # Tarball?
2291 #
2292 if sBaseNameLower.endswith('.tar') \
2293 or sBaseNameLower.endswith('.tar.gz') \
2294 or sBaseNameLower.endswith('.tgz') \
2295 or sBaseNameLower.endswith('.tar.bz2'):
2296 return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2297
2298 #
2299 # Cannot classify it from the name, so just return that to the caller.
2300 #
2301 fnLog('Not unpacking "%s".' % (sArchive,));
2302 return [];
2303
2304
2305#
2306# Misc.
2307#
2308def areBytesEqual(oLeft, oRight):
2309 """
2310 Compares two byte arrays, strings or whatnot.
2311
2312 returns true / false accordingly.
2313 """
2314
2315 # If both are None, consider them equal (bogus?):
2316 if oLeft is None and oRight is None:
2317 return True;
2318
2319 # If just one is None, they can't match:
2320 if oLeft is None or oRight is None:
2321 return False;
2322
2323 # If both have the same type, use the compare operator of the class:
2324 if type(oLeft) is type(oRight):
2325 #print('same type: %s' % (oLeft == oRight,));
2326 return oLeft == oRight;
2327
2328 # On the offchance that they're both strings, but of different types.
2329 if isString(oLeft) and isString(oRight):
2330 #print('string compare: %s' % (oLeft == oRight,));
2331 return oLeft == oRight;
2332
2333 #
2334 # See if byte/buffer stuff that can be compared directory. If not convert
2335 # strings to bytes.
2336 #
2337 # Note! For 2.x, we must convert both sides to the buffer type or the
2338 # comparison may fail despite it working okay in test cases.
2339 #
2340 if sys.version_info[0] >= 3:
2341 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2342 return oLeft == oRight;
2343
2344 if isString(oLeft):
2345 try: oLeft = bytes(oLeft, 'utf-8');
2346 except: pass;
2347 if isString(oRight):
2348 try: oRight = bytes(oRight, 'utf-8');
2349 except: pass;
2350 else:
2351 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2352 if isinstance(oLeft, bytearray):
2353 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2354 else:
2355 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2356 #print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2357 return oLeft == oRight;
2358
2359 if isString(oLeft):
2360 try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2361 except: pass;
2362 if isString(oRight):
2363 try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2364 except: pass;
2365
2366 # Check if we now have the same type for both:
2367 if type(oLeft) is type(oRight):
2368 #print('same type now: %s' % (oLeft == oRight,));
2369 return oLeft == oRight;
2370
2371 # Check if we now have buffer/memoryview vs bytes/bytesarray again.
2372 if sys.version_info[0] >= 3:
2373 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2374 return oLeft == oRight;
2375 else:
2376 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2377 if isinstance(oLeft, bytearray):
2378 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2379 else:
2380 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2381 #print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2382 return oLeft == oRight;
2383
2384 # Do item by item comparison:
2385 if len(oLeft) != len(oRight):
2386 #print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2387 return False;
2388 i = len(oLeft);
2389 while i > 0:
2390 i = i - 1;
2391
2392 iElmLeft = oLeft[i];
2393 if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2394 iElmLeft = ord(iElmLeft);
2395
2396 iElmRight = oRight[i];
2397 if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2398 iElmRight = ord(iElmRight);
2399
2400 if iElmLeft != iElmRight:
2401 #print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2402 return False;
2403 return True;
2404
2405
2406#
2407# Unit testing.
2408#
2409
2410# pylint: disable=missing-docstring
2411# pylint: disable=undefined-variable
2412class BuildCategoryDataTestCase(unittest.TestCase):
2413 def testIntervalSeconds(self):
2414 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2415 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2416 self.assertEqual(parseIntervalSeconds('123'), (123, None));
2417 self.assertEqual(parseIntervalSeconds(123), (123, None));
2418 self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2419 self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2420 self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2421 self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2422 self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2423 self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2424 self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2425
2426 def testZuluNormalization(self):
2427 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:34:25.000000000Z'), '2011-01-02T03:34:25.000000000Z');
2428 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25-0030'), '2011-01-02T03:34:25.000000000Z');
2429 self.assertEqual(normalizeIsoTimestampToZulu('2011-01-02T03:04:25+0030'), '2011-01-02T02:34:25.000000000Z');
2430 self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863+01:00'), '2020-03-20T19:47:39.832312000Z');
2431 self.assertEqual(normalizeIsoTimestampToZulu('2020-03-20T20:47:39,832312863-02:00'), '2020-03-20T22:47:39.832312000Z');
2432
2433 def testHasNonAsciiChars(self):
2434 self.assertEqual(hasNonAsciiCharacters(''), False);
2435 self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
2436 self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2437 self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2438 self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2439 self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2440 self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2441 self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2442 self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2443 self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2444
2445 def testAreBytesEqual(self):
2446 self.assertEqual(areBytesEqual(None, None), True);
2447 self.assertEqual(areBytesEqual(None, ''), False);
2448 self.assertEqual(areBytesEqual('', ''), True);
2449 self.assertEqual(areBytesEqual('1', '1'), True);
2450 self.assertEqual(areBytesEqual('12345', '1234'), False);
2451 self.assertEqual(areBytesEqual('1234', '1234'), True);
2452 self.assertEqual(areBytesEqual('1234', b'1234'), True);
2453 self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2454 self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2455 self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2456 self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2457 self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2458 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2459 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2460 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2461 if sys.version_info[0] >= 3:
2462 pass;
2463 else:
2464 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2465 bytearray([0x31,0x32,0x33,0x34])), True);
2466 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2467 bytearray([0x99,0x32,0x32,0x34])), False);
2468 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2469 buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2470 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2471 buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2472 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2473 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2474 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2475
2476if __name__ == '__main__':
2477 unittest.main();
2478 # not reached.
2479
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette