VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 82635

Last change on this file since 82635 was 82635, checked in by vboxsync, 5 years ago

common/utils.py: Added toUnicode method.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 77.2 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: utils.py 82635 2019-12-22 18:53:40Z vboxsync $
3# pylint: disable=too-many-lines
4
5"""
6Common Utility Functions.
7"""
8
9from __future__ import print_function;
10
11__copyright__ = \
12"""
13Copyright (C) 2012-2019 Oracle Corporation
14
15This file is part of VirtualBox Open Source Edition (OSE), as
16available from http://www.virtualbox.org. This file is free software;
17you can redistribute it and/or modify it under the terms of the GNU
18General Public License (GPL) as published by the Free Software
19Foundation, in version 2 as it comes in the "COPYING" file of the
20VirtualBox OSE distribution. VirtualBox OSE is distributed in the
21hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
22
23The contents of this file may alternatively be used under the terms
24of the Common Development and Distribution License Version 1.0
25(CDDL) only, as it comes in the "COPYING.CDDL" file of the
26VirtualBox OSE distribution, in which case the provisions of the
27CDDL are applicable instead of those of the GPL.
28
29You may elect to license modified versions of this file under the
30terms and conditions of either the GPL or the CDDL or both.
31"""
32__version__ = "$Revision: 82635 $"
33
34
35# Standard Python imports.
36import datetime;
37import os;
38import platform;
39import re;
40import stat;
41import subprocess;
42import sys;
43import time;
44import traceback;
45import unittest;
46
47if sys.platform == 'win32':
48 import ctypes;
49 import msvcrt; # pylint: disable=import-error
50 import win32api; # pylint: disable=import-error
51 import win32con; # pylint: disable=import-error
52 import win32console; # pylint: disable=import-error
53 import win32file; # pylint: disable=import-error
54 import win32process; # pylint: disable=import-error
55 import winerror; # pylint: disable=import-error
56 import pywintypes; # pylint: disable=import-error
57else:
58 import signal;
59
60# Python 3 hacks:
61if sys.version_info[0] >= 3:
62 unicode = str; # pylint: disable=redefined-builtin,invalid-name
63 xrange = range; # pylint: disable=redefined-builtin,invalid-name
64 long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67#
68# Strings.
69#
70
71def toUnicode(sString, encoding = None, errors = 'strict'):
72 """
73 A little like the python 2 unicode() function.
74 """
75 if sys.version_info[0] >= 3:
76 if isinstance(sString, bytes):
77 return str(sString, encoding if encoding else 'utf-8', errors);
78 else:
79 if not isinstance(sString, unicode):
80 return unicode(sString, encoding, errors);
81 return sString;
82
83
84
85#
86# Output.
87#
88
89def printOut(sString):
90 """
91 Outputs a string to standard output, dealing with python 2.x encoding stupidity.
92 """
93 sStreamEncoding = sys.stdout.encoding;
94 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
95 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
96 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
97 print(sString);
98 else:
99 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding));
100
101def printErr(sString):
102 """
103 Outputs a string to standard error, dealing with python 2.x encoding stupidity.
104 """
105 sStreamEncoding = sys.stderr.encoding;
106 if sStreamEncoding is None: # Files, pipes and such on 2.x. (pylint is confused here)
107 sStreamEncoding = 'US-ASCII'; # pylint: disable=redefined-variable-type
108 if sStreamEncoding == 'UTF-8' or not isinstance(sString, unicode):
109 print(sString, file = sys.stderr);
110 else:
111 print(sString.encode(sStreamEncoding, 'backslashreplace').decode(sStreamEncoding), file = sys.stderr);
112
113
114#
115# Host OS and CPU.
116#
117
118def getHostOs():
119 """
120 Gets the host OS name (short).
121
122 See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
123 """
124 sPlatform = platform.system();
125 if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
126 sPlatform = sPlatform.lower();
127 elif sPlatform == 'Windows':
128 sPlatform = 'win';
129 elif sPlatform == 'SunOS':
130 sPlatform = 'solaris';
131 else:
132 raise Exception('Unsupported platform "%s"' % (sPlatform,));
133 return sPlatform;
134
135g_sHostArch = None;
136
137def getHostArch():
138 """
139 Gets the host CPU architecture.
140
141 See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
142 """
143 global g_sHostArch;
144 if g_sHostArch is None:
145 sArch = platform.machine();
146 if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
147 sArch = 'x86';
148 elif sArch in ('AMD64', 'amd64', 'x86_64'):
149 sArch = 'amd64';
150 elif sArch == 'i86pc': # SunOS
151 if platform.architecture()[0] == '64bit':
152 sArch = 'amd64';
153 else:
154 try:
155 sArch = str(processOutputChecked(['/usr/bin/isainfo', '-n',]));
156 except:
157 pass;
158 sArch = sArch.strip();
159 if sArch != 'amd64':
160 sArch = 'x86';
161 else:
162 raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
163 g_sHostArch = sArch;
164 return g_sHostArch;
165
166
167def getHostOsDotArch():
168 """
169 Gets the 'os.arch' for the host.
170 """
171 return '%s.%s' % (getHostOs(), getHostArch());
172
173
174def isValidOs(sOs):
175 """
176 Validates the OS name.
177 """
178 if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
179 'os2', 'solaris', 'win', 'os-agnostic'):
180 return True;
181 return False;
182
183
184def isValidArch(sArch):
185 """
186 Validates the CPU architecture name.
187 """
188 if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
189 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
190 return True;
191 return False;
192
193def isValidOsDotArch(sOsDotArch):
194 """
195 Validates the 'os.arch' string.
196 """
197
198 asParts = sOsDotArch.split('.');
199 if asParts.length() != 2:
200 return False;
201 return isValidOs(asParts[0]) \
202 and isValidArch(asParts[1]);
203
204def getHostOsVersion():
205 """
206 Returns the host OS version. This is platform.release with additional
207 distro indicator on linux.
208 """
209 sVersion = platform.release();
210 sOs = getHostOs();
211 if sOs == 'linux':
212 sDist = '';
213 try:
214 # try /etc/lsb-release first to distinguish between Debian and Ubuntu
215 oFile = open('/etc/lsb-release');
216 for sLine in oFile:
217 oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
218 if oMatch is not None:
219 sDist = oMatch.group(1).strip();
220 except:
221 pass;
222 if sDist:
223 sVersion += ' / ' + sDist;
224 else:
225 asFiles = \
226 [
227 [ '/etc/debian_version', 'Debian v'],
228 [ '/etc/gentoo-release', '' ],
229 [ '/etc/oracle-release', '' ],
230 [ '/etc/redhat-release', '' ],
231 [ '/etc/SuSE-release', '' ],
232 ];
233 for sFile, sPrefix in asFiles:
234 if os.path.isfile(sFile):
235 try:
236 oFile = open(sFile);
237 sLine = oFile.readline();
238 oFile.close();
239 except:
240 continue;
241 sLine = sLine.strip()
242 if sLine:
243 sVersion += ' / ' + sPrefix + sLine;
244 break;
245
246 elif sOs == 'solaris':
247 sVersion = platform.version();
248 if os.path.isfile('/etc/release'):
249 try:
250 oFile = open('/etc/release');
251 sLast = oFile.readlines()[-1];
252 oFile.close();
253 sLast = sLast.strip();
254 if sLast:
255 sVersion += ' (' + sLast + ')';
256 except:
257 pass;
258
259 elif sOs == 'darwin':
260 sOsxVersion = platform.mac_ver()[0];
261 codenames = {"4": "Tiger",
262 "5": "Leopard",
263 "6": "Snow Leopard",
264 "7": "Lion",
265 "8": "Mountain Lion",
266 "9": "Mavericks",
267 "10": "Yosemite",
268 "11": "El Capitan",
269 "12": "Sierra",
270 "13": "High Sierra",
271 "14": "Unknown 14", }
272 sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
273
274 elif sOs == 'win':
275 class OSVersionInfoEx(ctypes.Structure):
276 """ OSVERSIONEX """
277 kaFields = [
278 ('dwOSVersionInfoSize', ctypes.c_ulong),
279 ('dwMajorVersion', ctypes.c_ulong),
280 ('dwMinorVersion', ctypes.c_ulong),
281 ('dwBuildNumber', ctypes.c_ulong),
282 ('dwPlatformId', ctypes.c_ulong),
283 ('szCSDVersion', ctypes.c_wchar*128),
284 ('wServicePackMajor', ctypes.c_ushort),
285 ('wServicePackMinor', ctypes.c_ushort),
286 ('wSuiteMask', ctypes.c_ushort),
287 ('wProductType', ctypes.c_byte),
288 ('wReserved', ctypes.c_byte)]
289 _fields_ = kaFields # pylint: disable=invalid-name
290
291 def __init__(self):
292 super(OSVersionInfoEx, self).__init__()
293 self.dwOSVersionInfoSize = ctypes.sizeof(self)
294
295 oOsVersion = OSVersionInfoEx()
296 rc = ctypes.windll.Ntdll.RtlGetVersion(ctypes.byref(oOsVersion))
297 if rc == 0:
298 # Python platform.release() is not reliable for newer server releases
299 if oOsVersion.wProductType != 1:
300 if oOsVersion.dwMajorVersion == 10 and oOsVersion.dwMinorVersion == 0:
301 sVersion = '2016Server';
302 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 3:
303 sVersion = '2012ServerR2';
304 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 2:
305 sVersion = '2012Server';
306 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 1:
307 sVersion = '2008ServerR2';
308 elif oOsVersion.dwMajorVersion == 6 and oOsVersion.dwMinorVersion == 0:
309 sVersion = '2008Server';
310 elif oOsVersion.dwMajorVersion == 5 and oOsVersion.dwMinorVersion == 2:
311 sVersion = '2003Server';
312 sVersion += ' build ' + str(oOsVersion.dwBuildNumber)
313 if oOsVersion.wServicePackMajor:
314 sVersion += ' SP' + str(oOsVersion.wServicePackMajor)
315 if oOsVersion.wServicePackMinor:
316 sVersion += '.' + str(oOsVersion.wServicePackMinor)
317
318 return sVersion;
319
320def getPresentCpuCount():
321 """
322 Gets the number of CPUs present in the system.
323
324 This differs from multiprocessor.cpu_count() and os.cpu_count() on windows in
325 that we return the active count rather than the maximum count. If we don't,
326 we will end up thinking testboxmem1 has 512 CPU threads, which it doesn't and
327 never will have.
328
329 @todo This is probably not exactly what we get on non-windows...
330 """
331
332 if getHostOs() == 'win':
333 fnGetActiveProcessorCount = getattr(ctypes.windll.kernel32, 'GetActiveProcessorCount', None);
334 if fnGetActiveProcessorCount:
335 cCpus = fnGetActiveProcessorCount(ctypes.c_ushort(0xffff));
336 if cCpus > 0:
337 return cCpus;
338
339 import multiprocessing
340 return multiprocessing.cpu_count();
341
342
343#
344# File system.
345#
346
347def openNoInherit(sFile, sMode = 'r'):
348 """
349 Wrapper around open() that tries it's best to make sure the file isn't
350 inherited by child processes.
351
352 This is a best effort thing at the moment as it doesn't synchronizes with
353 child process spawning in any way. Thus it can be subject to races in
354 multithreaded programs.
355 """
356
357 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
358 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
359 if uPythonVer >= ((3 << 16) | 4):
360 oFile = open(sFile, sMode);
361 else:
362 try:
363 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
364 except:
365 # On windows, we can use the 'N' flag introduced in Visual C++ 7.0 or 7.1 with python 2.x.
366 if getHostOs() == 'win':
367 if uPythonVer < (3 << 16):
368 offComma = sMode.find(',');
369 if offComma < 0:
370 return open(sFile, sMode + 'N');
371 return open(sFile, sMode[:offComma] + 'N' + sMode[offComma:]); # pylint: disable=bad-open-mode
372
373 # Just in case.
374 return open(sFile, sMode);
375
376 oFile = open(sFile, sMode);
377 #try:
378 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
379 #except:
380 # pass;
381 return oFile;
382
383def openNoDenyDeleteNoInherit(sFile, sMode = 'r'):
384 """
385 Wrapper around open() that tries it's best to make sure the file isn't
386 inherited by child processes.
387
388 This is a best effort thing at the moment as it doesn't synchronizes with
389 child process spawning in any way. Thus it can be subject to races in
390 multithreaded programs.
391 """
392
393 if getHostOs() == 'win':
394 # Need to use CreateFile directly to open the file so we can feed it FILE_SHARE_DELETE.
395 # pylint: disable=no-member,c-extension-no-member
396 fAccess = 0;
397 fDisposition = win32file.OPEN_EXISTING;
398 if 'r' in sMode or '+' in sMode:
399 fAccess |= win32file.GENERIC_READ;
400 if 'a' in sMode:
401 fAccess |= win32file.GENERIC_WRITE;
402 fDisposition = win32file.OPEN_ALWAYS;
403 elif 'w' in sMode:
404 fAccess = win32file.GENERIC_WRITE;
405 if '+' in sMode:
406 fDisposition = win32file.OPEN_ALWAYS;
407 fAccess |= win32file.GENERIC_READ;
408 else:
409 fDisposition = win32file.CREATE_ALWAYS;
410 if not fAccess:
411 fAccess |= win32file.GENERIC_READ;
412 fSharing = (win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE
413 | win32file.FILE_SHARE_DELETE);
414 hFile = win32file.CreateFile(sFile, fAccess, fSharing, None, fDisposition, 0, None);
415 if 'a' in sMode:
416 win32file.SetFilePointer(hFile, 0, win32file.FILE_END);
417
418 # Turn the NT handle into a CRT file descriptor.
419 hDetachedFile = hFile.Detach();
420 if fAccess == win32file.GENERIC_READ:
421 fOpen = os.O_RDONLY;
422 elif fAccess == win32file.GENERIC_WRITE:
423 fOpen = os.O_WRONLY;
424 else:
425 fOpen = os.O_RDWR;
426 # pulint: enable=no-member,c-extension-no-member
427 if 'a' in sMode:
428 fOpen |= os.O_APPEND;
429 if 'b' in sMode or 't' in sMode:
430 fOpen |= os.O_TEXT; # pylint: disable=no-member
431 fdFile = msvcrt.open_osfhandle(hDetachedFile, fOpen);
432
433 # Tell python to use this handle.
434 oFile = os.fdopen(fdFile, sMode);
435 else:
436 oFile = open(sFile, sMode);
437
438 # Python 3.4 and later automatically creates non-inherit handles. See PEP-0446.
439 uPythonVer = (sys.version_info[0] << 16) | (sys.version_info[1] & 0xffff);
440 if uPythonVer < ((3 << 16) | 4):
441 try:
442 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=import-error
443 except:
444 pass;
445 else:
446 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
447 return oFile;
448
449def noxcptReadLink(sPath, sXcptRet, sEncoding = 'utf-8'):
450 """
451 No exceptions os.readlink wrapper.
452 """
453 try:
454 sRet = os.readlink(sPath); # pylint: disable=no-member
455 except:
456 return sXcptRet;
457 if hasattr(sRet, 'decode'):
458 sRet = sRet.decode(sEncoding, 'ignore');
459 return sRet;
460
461def readFile(sFile, sMode = 'rb'):
462 """
463 Reads the entire file.
464 """
465 oFile = open(sFile, sMode);
466 sRet = oFile.read();
467 oFile.close();
468 return sRet;
469
470def noxcptReadFile(sFile, sXcptRet, sMode = 'rb', sEncoding = 'utf-8'):
471 """
472 No exceptions common.readFile wrapper.
473 """
474 try:
475 sRet = readFile(sFile, sMode);
476 except:
477 sRet = sXcptRet;
478 if sEncoding is not None and hasattr(sRet, 'decode'):
479 sRet = sRet.decode(sEncoding, 'ignore');
480 return sRet;
481
482def noxcptRmDir(sDir, oXcptRet = False):
483 """
484 No exceptions os.rmdir wrapper.
485 """
486 oRet = True;
487 try:
488 os.rmdir(sDir);
489 except:
490 oRet = oXcptRet;
491 return oRet;
492
493def noxcptDeleteFile(sFile, oXcptRet = False):
494 """
495 No exceptions os.remove wrapper.
496 """
497 oRet = True;
498 try:
499 os.remove(sFile);
500 except:
501 oRet = oXcptRet;
502 return oRet;
503
504
505def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
506 # type: (string, (string, stat) -> bool) -> bool
507 """
508 Recursively walks a directory tree, calling fnCallback for each.
509
510 fnCallback takes a full path and stat object (can be None). It
511 returns a boolean value, False stops walking and returns immediately.
512
513 Returns True or False depending on fnCallback.
514 Returns None fIgnoreExceptions is True and an exception was raised by listdir.
515 """
516 def __worker(sCurDir):
517 """ Worker for """
518 try:
519 asNames = os.listdir(sCurDir);
520 except:
521 if not fIgnoreExceptions:
522 raise;
523 return None;
524 rc = True;
525 for sName in asNames:
526 if sName not in [ '.', '..' ]:
527 sFullName = os.path.join(sCurDir, sName);
528 try: oStat = os.lstat(sFullName);
529 except: oStat = None;
530 if fnCallback(sFullName, oStat) is False:
531 return False;
532 if oStat is not None and stat.S_ISDIR(oStat.st_mode):
533 rc = __worker(sFullName);
534 if rc is False:
535 break;
536 return rc;
537
538 # Ensure unicode path here so listdir also returns unicode on windows.
539 ## @todo figure out unicode stuff on non-windows.
540 if sys.platform == 'win32':
541 sDir = unicode(sDir);
542 return __worker(sDir);
543
544
545
546def formatFileMode(uMode):
547 # type: (int) -> string
548 """
549 Format a st_mode value 'ls -la' fasion.
550 Returns string.
551 """
552 if stat.S_ISDIR(uMode): sMode = 'd';
553 elif stat.S_ISREG(uMode): sMode = '-';
554 elif stat.S_ISLNK(uMode): sMode = 'l';
555 elif stat.S_ISFIFO(uMode): sMode = 'p';
556 elif stat.S_ISCHR(uMode): sMode = 'c';
557 elif stat.S_ISBLK(uMode): sMode = 'b';
558 elif stat.S_ISSOCK(uMode): sMode = 's';
559 else: sMode = '?';
560 ## @todo sticky bits.
561 sMode += 'r' if uMode & stat.S_IRUSR else '-';
562 sMode += 'w' if uMode & stat.S_IWUSR else '-';
563 sMode += 'x' if uMode & stat.S_IXUSR else '-';
564 sMode += 'r' if uMode & stat.S_IRGRP else '-';
565 sMode += 'w' if uMode & stat.S_IWGRP else '-';
566 sMode += 'x' if uMode & stat.S_IXGRP else '-';
567 sMode += 'r' if uMode & stat.S_IROTH else '-';
568 sMode += 'w' if uMode & stat.S_IWOTH else '-';
569 sMode += 'x' if uMode & stat.S_IXOTH else '-';
570 sMode += ' ';
571 return sMode;
572
573
574def formatFileStat(oStat):
575 # type: (stat) -> string
576 """
577 Format a stat result 'ls -la' fasion (numeric IDs).
578 Returns string.
579 """
580 return '%s %3s %4s %4s %10s %s' \
581 % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
582 time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
583
584## Good buffer for file operations.
585g_cbGoodBufferSize = 256*1024;
586
587## The original shutil.copyfileobj.
588g_fnOriginalShCopyFileObj = None;
589
590def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
591 """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
592 return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
593
594def __installShUtilHacks(shutil):
595 """ Installs the shutil buffer size hacks. """
596 global g_fnOriginalShCopyFileObj;
597 if g_fnOriginalShCopyFileObj is None:
598 g_fnOriginalShCopyFileObj = shutil.copyfileobj;
599 shutil.copyfileobj = __myshutilcopyfileobj;
600 return True;
601
602
603def copyFileSimple(sFileSrc, sFileDst):
604 """
605 Wrapper around shutil.copyfile that simply copies the data of a regular file.
606 Raises exception on failure.
607 Return True for show.
608 """
609 import shutil;
610 __installShUtilHacks(shutil);
611 return shutil.copyfile(sFileSrc, sFileDst);
612
613
614def getDiskUsage(sPath):
615 """
616 Get free space of a partition that corresponds to specified sPath in MB.
617
618 Returns partition free space value in MB.
619 """
620 if platform.system() == 'Windows':
621 oCTypeFreeSpace = ctypes.c_ulonglong(0);
622 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
623 ctypes.pointer(oCTypeFreeSpace));
624 cbFreeSpace = oCTypeFreeSpace.value;
625 else:
626 oStats = os.statvfs(sPath); # pylint: disable=no-member
627 cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
628
629 # Convert to MB
630 cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
631
632 return cMbFreeSpace;
633
634
635
636#
637# SubProcess.
638#
639
640def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
641 """
642 If the "executable" is a python script, insert the python interpreter at
643 the head of the argument list so that it will work on systems which doesn't
644 support hash-bang scripts.
645 """
646
647 asArgs = dKeywordArgs.get('args');
648 if asArgs is None:
649 asArgs = aPositionalArgs[0];
650
651 if asArgs[0].endswith('.py'):
652 if sys.executable:
653 asArgs.insert(0, sys.executable);
654 else:
655 asArgs.insert(0, 'python');
656
657 # paranoia...
658 if dKeywordArgs.get('args') is not None:
659 dKeywordArgs['args'] = asArgs;
660 else:
661 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
662 return None;
663
664def processPopenSafe(*aPositionalArgs, **dKeywordArgs):
665 """
666 Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
667 """
668 if getHostOs() == 'win':
669 if dKeywordArgs.get('creationflags', 0) == 0:
670 dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
671 return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
672
673def processCall(*aPositionalArgs, **dKeywordArgs):
674 """
675 Wrapper around subprocess.call to deal with its absence in older
676 python versions.
677 Returns process exit code (see subprocess.poll).
678 """
679 assert dKeywordArgs.get('stdout') is None;
680 assert dKeywordArgs.get('stderr') is None;
681 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
682 oProcess = processPopenSafe(*aPositionalArgs, **dKeywordArgs);
683 return oProcess.wait();
684
685def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
686 """
687 Wrapper around subprocess.check_output to deal with its absense in older
688 python versions.
689
690 Extra keywords for specifying now output is to be decoded:
691 sEncoding='utf-8
692 fIgnoreEncoding=True/False
693 """
694 sEncoding = dKeywordArgs.get('sEncoding');
695 if sEncoding is not None: del dKeywordArgs['sEncoding'];
696 else: sEncoding = 'utf-8';
697
698 fIgnoreEncoding = dKeywordArgs.get('fIgnoreEncoding');
699 if fIgnoreEncoding is not None: del dKeywordArgs['fIgnoreEncoding'];
700 else: fIgnoreEncoding = True;
701
702 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
703 oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
704
705 sOutput, _ = oProcess.communicate();
706 iExitCode = oProcess.poll();
707
708 if iExitCode != 0:
709 asArgs = dKeywordArgs.get('args');
710 if asArgs is None:
711 asArgs = aPositionalArgs[0];
712 print(sOutput);
713 raise subprocess.CalledProcessError(iExitCode, asArgs);
714
715 if hasattr(sOutput, 'decode'):
716 sOutput = sOutput.decode(sEncoding, 'ignore' if fIgnoreEncoding else 'strict');
717 return sOutput;
718
719g_fOldSudo = None;
720def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
721 """
722 Adds 'sudo' (or similar) to the args parameter, whereever it is.
723 """
724
725 # Are we root?
726 fIsRoot = True;
727 try:
728 fIsRoot = os.getuid() == 0; # pylint: disable=no-member
729 except:
730 pass;
731
732 # If not, prepend sudo (non-interactive, simulate initial login).
733 if fIsRoot is not True:
734 asArgs = dKeywordArgs.get('args');
735 if asArgs is None:
736 asArgs = aPositionalArgs[0];
737
738 # Detect old sudo.
739 global g_fOldSudo;
740 if g_fOldSudo is None:
741 try:
742 sVersion = str(processOutputChecked(['sudo', '-V']));
743 except:
744 sVersion = '1.7.0';
745 sVersion = sVersion.strip().split('\n')[0];
746 sVersion = sVersion.replace('Sudo version', '').strip();
747 g_fOldSudo = len(sVersion) >= 4 \
748 and sVersion[0] == '1' \
749 and sVersion[1] == '.' \
750 and sVersion[2] <= '6' \
751 and sVersion[3] == '.';
752
753 asArgs.insert(0, 'sudo');
754 if not g_fOldSudo:
755 asArgs.insert(1, '-n');
756 if fInitialEnv and not g_fOldSudo:
757 asArgs.insert(1, '-i');
758
759 # paranoia...
760 if dKeywordArgs.get('args') is not None:
761 dKeywordArgs['args'] = asArgs;
762 else:
763 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
764 return None;
765
766
767def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
768 """
769 sudo (or similar) + subprocess.call
770 """
771 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
772 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
773 return processCall(*aPositionalArgs, **dKeywordArgs);
774
775def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
776 """
777 sudo (or similar) + subprocess.check_output.
778 """
779 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
780 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
781 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
782
783def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
784 """
785 sudo (or similar) + subprocess.check_output, except '-i' isn't used.
786 """
787 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
788 _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
789 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
790
791def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
792 """
793 sudo (or similar) + processPopenSafe.
794 """
795 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
796 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
797 return processPopenSafe(*aPositionalArgs, **dKeywordArgs);
798
799
800#
801# Generic process stuff.
802#
803
804def processInterrupt(uPid):
805 """
806 Sends a SIGINT or equivalent to interrupt the specified process.
807 Returns True on success, False on failure.
808
809 On Windows hosts this may not work unless the process happens to be a
810 process group leader.
811 """
812 if sys.platform == 'win32':
813 try:
814 win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, # pylint: disable=no-member,c-extension-no-member
815 uPid);
816 fRc = True;
817 except:
818 fRc = False;
819 else:
820 try:
821 os.kill(uPid, signal.SIGINT);
822 fRc = True;
823 except:
824 fRc = False;
825 return fRc;
826
827def sendUserSignal1(uPid):
828 """
829 Sends a SIGUSR1 or equivalent to nudge the process into shutting down
830 (VBoxSVC) or something.
831 Returns True on success, False on failure or if not supported (win).
832
833 On Windows hosts this may not work unless the process happens to be a
834 process group leader.
835 """
836 if sys.platform == 'win32':
837 fRc = False;
838 else:
839 try:
840 os.kill(uPid, signal.SIGUSR1); # pylint: disable=no-member
841 fRc = True;
842 except:
843 fRc = False;
844 return fRc;
845
846def processTerminate(uPid):
847 """
848 Terminates the process in a nice manner (SIGTERM or equivalent).
849 Returns True on success, False on failure.
850 """
851 fRc = False;
852 if sys.platform == 'win32':
853 try:
854 hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, # pylint: disable=no-member,c-extension-no-member
855 False, uPid);
856 except:
857 pass;
858 else:
859 try:
860 win32process.TerminateProcess(hProcess, # pylint: disable=no-member,c-extension-no-member
861 0x40010004); # DBG_TERMINATE_PROCESS
862 fRc = True;
863 except:
864 pass;
865 hProcess.Close(); #win32api.CloseHandle(hProcess)
866 else:
867 try:
868 os.kill(uPid, signal.SIGTERM);
869 fRc = True;
870 except:
871 pass;
872 return fRc;
873
874def processKill(uPid):
875 """
876 Terminates the process with extreme prejudice (SIGKILL).
877 Returns True on success, False on failure.
878 """
879 if sys.platform == 'win32':
880 fRc = processTerminate(uPid);
881 else:
882 try:
883 os.kill(uPid, signal.SIGKILL); # pylint: disable=no-member
884 fRc = True;
885 except:
886 fRc = False;
887 return fRc;
888
889def processKillWithNameCheck(uPid, sName):
890 """
891 Like processKill(), but checks if the process name matches before killing
892 it. This is intended for killing using potentially stale pid values.
893
894 Returns True on success, False on failure.
895 """
896
897 if processCheckPidAndName(uPid, sName) is not True:
898 return False;
899 return processKill(uPid);
900
901
902def processExists(uPid):
903 """
904 Checks if the specified process exits.
905 This will only work if we can signal/open the process.
906
907 Returns True if it positively exists, False otherwise.
908 """
909 if sys.platform == 'win32':
910 fRc = False;
911 # We try open the process for waiting since this is generally only forbidden in a very few cases.
912 try:
913 hProcess = win32api.OpenProcess(win32con.SYNCHRONIZE, # pylint: disable=no-member,c-extension-no-member
914 False, uPid);
915 except pywintypes.error as oXcpt: # pylint: disable=no-member
916 if oXcpt.winerror == winerror.ERROR_ACCESS_DENIED:
917 fRc = True;
918 except Exception as oXcpt:
919 pass;
920 else:
921 hProcess.Close();
922 fRc = True;
923 else:
924 try:
925 os.kill(uPid, 0);
926 fRc = True;
927 except: ## @todo check error code.
928 fRc = False;
929 return fRc;
930
931def processCheckPidAndName(uPid, sName):
932 """
933 Checks if a process PID and NAME matches.
934 """
935 fRc = processExists(uPid);
936 if fRc is not True:
937 return False;
938
939 if sys.platform == 'win32':
940 try:
941 from win32com.client import GetObject; # pylint: disable=import-error
942 oWmi = GetObject('winmgmts:');
943 aoProcesses = oWmi.InstancesOf('Win32_Process');
944 for oProcess in aoProcesses:
945 if long(oProcess.Properties_("ProcessId").Value) == uPid:
946 sCurName = oProcess.Properties_("Name").Value;
947 #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
948 sName = sName.lower();
949 sCurName = sCurName.lower();
950 if os.path.basename(sName) == sName:
951 sCurName = os.path.basename(sCurName);
952
953 if sCurName == sName \
954 or sCurName + '.exe' == sName \
955 or sCurName == sName + '.exe':
956 fRc = True;
957 break;
958 except:
959 #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
960 pass;
961 else:
962 if sys.platform in ('linux2', 'linux', 'linux3', 'linux4', 'linux5', 'linux6'):
963 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
964 elif sys.platform in ('sunos5',):
965 asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
966 elif sys.platform in ('darwin',):
967 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
968 else:
969 asPsCmd = None;
970
971 if asPsCmd is not None:
972 try:
973 oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
974 sCurName = oPs.communicate()[0];
975 iExitCode = oPs.wait();
976 except:
977 #reporter.logXcpt();
978 return False;
979
980 # ps fails with non-zero exit code if the pid wasn't found.
981 if iExitCode != 0:
982 return False;
983 if sCurName is None:
984 return False;
985 sCurName = sCurName.strip();
986 if not sCurName:
987 return False;
988
989 if os.path.basename(sName) == sName:
990 sCurName = os.path.basename(sCurName);
991 elif os.path.basename(sCurName) == sCurName:
992 sName = os.path.basename(sName);
993
994 if sCurName != sName:
995 return False;
996
997 fRc = True;
998 return fRc;
999
1000def processGetInfo(uPid, fSudo = False):
1001 """
1002 Tries to acquire state information of the given process.
1003
1004 Returns a string with the information on success or None on failure or
1005 if the host is not supported.
1006
1007 Note that the format of the information is host system dependent and will
1008 likely differ much between different hosts.
1009 """
1010 fRc = processExists(uPid);
1011 if fRc is not True:
1012 return None;
1013
1014 sHostOs = getHostOs();
1015 if sHostOs in [ 'linux',]:
1016 sGdb = '/usr/bin/gdb';
1017 if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
1018 if not os.path.isfile(sGdb): sGdb = 'gdb';
1019 aasCmd = [
1020 [ sGdb, '-batch',
1021 '-ex', 'set pagination off',
1022 '-ex', 'thread apply all bt',
1023 '-ex', 'info proc mapping',
1024 '-ex', 'info sharedlibrary',
1025 '-p', '%u' % (uPid,), ],
1026 ];
1027 elif sHostOs == 'darwin':
1028 # LLDB doesn't work in batch mode when attaching to a process, at least
1029 # with macOS Sierra (10.12). GDB might not be installed. Use the sample
1030 # tool instead with a 1 second duration and 1000ms sampling interval to
1031 # get one stack trace. For the process mappings use vmmap.
1032 aasCmd = [
1033 [ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
1034 [ '/usr/bin/vmmap', '%u' % (uPid,), ],
1035 ];
1036 elif sHostOs == 'solaris':
1037 aasCmd = [
1038 [ '/usr/bin/pstack', '%u' % (uPid,), ],
1039 [ '/usr/bin/pmap', '%u' % (uPid,), ],
1040 ];
1041 else:
1042 aasCmd = [];
1043
1044 sInfo = '';
1045 for asCmd in aasCmd:
1046 try:
1047 if fSudo:
1048 sThisInfo = sudoProcessOutputChecked(asCmd);
1049 else:
1050 sThisInfo = processOutputChecked(asCmd);
1051 if sThisInfo is not None:
1052 sInfo += sThisInfo;
1053 except:
1054 pass;
1055 if not sInfo:
1056 sInfo = None;
1057
1058 return sInfo;
1059
1060
1061class ProcessInfo(object):
1062 """Process info."""
1063 def __init__(self, iPid):
1064 self.iPid = iPid;
1065 self.iParentPid = None;
1066 self.sImage = None;
1067 self.sName = None;
1068 self.asArgs = None;
1069 self.sCwd = None;
1070 self.iGid = None;
1071 self.iUid = None;
1072 self.iProcGroup = None;
1073 self.iSessionId = None;
1074
1075 def loadAll(self):
1076 """Load all the info."""
1077 sOs = getHostOs();
1078 if sOs == 'linux':
1079 sProc = '/proc/%s/' % (self.iPid,);
1080 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
1081 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
1082 if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
1083 #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
1084 # sProc = '/proc/%s/' % (self.iPid,);
1085 # if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
1086 # if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
1087 else:
1088 pass;
1089 if self.sName is None and self.sImage is not None:
1090 self.sName = self.sImage;
1091
1092 def windowsGrabProcessInfo(self, oProcess):
1093 """Windows specific loadAll."""
1094 try: self.sName = oProcess.Properties_("Name").Value;
1095 except: pass;
1096 try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
1097 except: pass;
1098 try: self.asArgs = [oProcess.Properties_("CommandLine").Value]; ## @todo split it.
1099 except: pass;
1100 try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
1101 except: pass;
1102 try: self.iSessionId = oProcess.Properties_("SessionId").Value;
1103 except: pass;
1104 if self.sName is None and self.sImage is not None:
1105 self.sName = self.sImage;
1106
1107 def getBaseImageName(self):
1108 """
1109 Gets the base image name if available, use the process name if not available.
1110 Returns image/process base name or None.
1111 """
1112 sRet = self.sImage if self.sName is None else self.sName;
1113 if sRet is None:
1114 self.loadAll();
1115 sRet = self.sImage if self.sName is None else self.sName;
1116 if sRet is None:
1117 if not self.asArgs:
1118 return None;
1119 sRet = self.asArgs[0];
1120 if not sRet:
1121 return None;
1122 return os.path.basename(sRet);
1123
1124 def getBaseImageNameNoExeSuff(self):
1125 """
1126 Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
1127 """
1128 sRet = self.getBaseImageName();
1129 if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
1130 if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
1131 sRet = sRet[:-4];
1132 return sRet;
1133
1134
1135def processListAll():
1136 """
1137 Return a list of ProcessInfo objects for all the processes in the system
1138 that the current user can see.
1139 """
1140 asProcesses = [];
1141
1142 sOs = getHostOs();
1143 if sOs == 'win':
1144 from win32com.client import GetObject; # pylint: disable=import-error
1145 oWmi = GetObject('winmgmts:');
1146 aoProcesses = oWmi.InstancesOf('Win32_Process');
1147 for oProcess in aoProcesses:
1148 try:
1149 iPid = int(oProcess.Properties_("ProcessId").Value);
1150 except:
1151 continue;
1152 oMyInfo = ProcessInfo(iPid);
1153 oMyInfo.windowsGrabProcessInfo(oProcess);
1154 asProcesses.append(oMyInfo);
1155 return asProcesses;
1156
1157 if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
1158 try:
1159 asDirs = os.listdir('/proc');
1160 except:
1161 asDirs = [];
1162 for sDir in asDirs:
1163 if sDir.isdigit():
1164 asProcesses.append(ProcessInfo(int(sDir),));
1165 return asProcesses;
1166
1167 #
1168 # The other OSes parses the output from the 'ps' utility.
1169 #
1170 asPsCmd = [
1171 '/bin/ps', # 0
1172 '-A', # 1
1173 '-o', 'pid=', # 2,3
1174 '-o', 'ppid=', # 4,5
1175 '-o', 'pgid=', # 6,7
1176 '-o', 'sid=', # 8,9
1177 '-o', 'uid=', # 10,11
1178 '-o', 'gid=', # 12,13
1179 '-o', 'comm=' # 14,15
1180 ];
1181
1182 if sOs == 'darwin':
1183 assert asPsCmd[9] == 'sid=';
1184 asPsCmd[9] = 'sess=';
1185 elif sOs == 'solaris':
1186 asPsCmd[0] = '/usr/bin/ps';
1187
1188 try:
1189 sRaw = processOutputChecked(asPsCmd);
1190 except:
1191 return asProcesses;
1192
1193 for sLine in sRaw.split('\n'):
1194 sLine = sLine.lstrip();
1195 if len(sLine) < 7 or not sLine[0].isdigit():
1196 continue;
1197
1198 iField = 0;
1199 off = 0;
1200 aoFields = [None, None, None, None, None, None, None];
1201 while iField < 7:
1202 # Eat whitespace.
1203 while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
1204 off += 1;
1205
1206 # Final field / EOL.
1207 if iField == 6:
1208 aoFields[6] = sLine[off:];
1209 break;
1210 if off >= len(sLine):
1211 break;
1212
1213 # Generic field parsing.
1214 offStart = off;
1215 off += 1;
1216 while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
1217 off += 1;
1218 try:
1219 if iField != 3:
1220 aoFields[iField] = int(sLine[offStart:off]);
1221 else:
1222 aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
1223 except:
1224 pass;
1225 iField += 1;
1226
1227 if aoFields[0] is not None:
1228 oMyInfo = ProcessInfo(aoFields[0]);
1229 oMyInfo.iParentPid = aoFields[1];
1230 oMyInfo.iProcGroup = aoFields[2];
1231 oMyInfo.iSessionId = aoFields[3];
1232 oMyInfo.iUid = aoFields[4];
1233 oMyInfo.iGid = aoFields[5];
1234 oMyInfo.sName = aoFields[6];
1235 asProcesses.append(oMyInfo);
1236
1237 return asProcesses;
1238
1239
1240def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
1241 """
1242 Looks for information regarding the demise of the given process.
1243 """
1244 sOs = getHostOs();
1245 if sOs == 'darwin':
1246 #
1247 # On darwin we look for crash and diagnostic reports.
1248 #
1249 asLogDirs = [
1250 u'/Library/Logs/DiagnosticReports/',
1251 u'/Library/Logs/CrashReporter/',
1252 u'~/Library/Logs/DiagnosticReports/',
1253 u'~/Library/Logs/CrashReporter/',
1254 ];
1255 for sDir in asLogDirs:
1256 sDir = os.path.expanduser(sDir);
1257 if not os.path.isdir(sDir):
1258 continue;
1259 try:
1260 asDirEntries = os.listdir(sDir);
1261 except:
1262 continue;
1263 for sEntry in asDirEntries:
1264 # Only interested in .crash files.
1265 _, sSuff = os.path.splitext(sEntry);
1266 if sSuff != '.crash':
1267 continue;
1268
1269 # The pid can be found at the end of the first line.
1270 sFull = os.path.join(sDir, sEntry);
1271 try:
1272 oFile = open(sFull, 'r');
1273 sFirstLine = oFile.readline();
1274 oFile.close();
1275 except:
1276 continue;
1277 if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
1278 continue;
1279 offPid = len(sFirstLine) - 3;
1280 while offPid > 1 and sFirstLine[offPid - 1].isdigit():
1281 offPid -= 1;
1282 try: uReportPid = int(sFirstLine[offPid:-2]);
1283 except: continue;
1284
1285 # Does the pid we found match?
1286 if uReportPid == uPid:
1287 fnLog('Found crash report for %u: %s' % (uPid, sFull,));
1288 fnCrashFile(sFull, False);
1289 elif sOs == 'win':
1290 #
1291 # Getting WER reports would be great, however we have trouble match the
1292 # PID to those as they seems not to mention it in the brief reports.
1293 # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
1294 # location - see the windows readme for the testbox script) and what
1295 # the MSDN article lists for now.
1296 #
1297 # It's been observed on Windows server 2012 that the dump files takes
1298 # the form: <processimage>.<decimal-pid>.dmp
1299 #
1300 asDmpDirs = [
1301 u'%SystemDrive%/CrashDumps/', # Testboxes.
1302 u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
1303 u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
1304 u'%WINDIR%/ServiceProfiles/NetworkSerices/',
1305 u'%WINDIR%/ServiceProfiles/',
1306 u'%WINDIR%/System32/Config/SystemProfile/', # System services.
1307 ];
1308 sMatchSuffix = '.%u.dmp' % (uPid,);
1309
1310 for sDir in asDmpDirs:
1311 sDir = os.path.expandvars(sDir);
1312 if not os.path.isdir(sDir):
1313 continue;
1314 try:
1315 asDirEntries = os.listdir(sDir);
1316 except:
1317 continue;
1318 for sEntry in asDirEntries:
1319 if sEntry.endswith(sMatchSuffix):
1320 sFull = os.path.join(sDir, sEntry);
1321 fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1322 fnCrashFile(sFull, True);
1323
1324 else:
1325 pass; ## TODO
1326 return None;
1327
1328
1329#
1330# Time.
1331#
1332
1333#
1334# The following test case shows how time.time() only have ~ms resolution
1335# on Windows (tested W10) and why it therefore makes sense to try use
1336# performance counters.
1337#
1338# Note! We cannot use time.clock() as the timestamp must be portable across
1339# processes. See timeout testcase problem on win hosts (no logs).
1340#
1341#import sys;
1342#import time;
1343#from common import utils;
1344#
1345#atSeries = [];
1346#for i in xrange(1,160):
1347# if i == 159: time.sleep(10);
1348# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1349#
1350#tPrev = atSeries[0]
1351#for tCur in atSeries:
1352# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1353# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1354# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1355# print '';
1356# tPrev = tCur
1357#
1358#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1359#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1360#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1361
1362g_fWinUseWinPerfCounter = sys.platform == 'win32';
1363g_fpWinPerfCounterFreq = None;
1364g_oFuncwinQueryPerformanceCounter = None;
1365
1366def _winInitPerfCounter():
1367 """ Initializes the use of performance counters. """
1368 global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1369
1370 uFrequency = ctypes.c_ulonglong(0);
1371 if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1372 if uFrequency.value >= 1000:
1373 #print 'uFrequency = %s' % (uFrequency,);
1374 #print 'type(uFrequency) = %s' % (type(uFrequency),);
1375 g_fpWinPerfCounterFreq = float(uFrequency.value);
1376
1377 # Check that querying the counter works too.
1378 global g_oFuncwinQueryPerformanceCounter
1379 g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1380 uCurValue = ctypes.c_ulonglong(0);
1381 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1382 if uCurValue.value > 0:
1383 return True;
1384 g_fWinUseWinPerfCounter = False;
1385 return False;
1386
1387def _winFloatTime():
1388 """ Gets floating point time on windows. """
1389 if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1390 uCurValue = ctypes.c_ulonglong(0);
1391 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1392 return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1393 return time.time();
1394
1395def timestampNano():
1396 """
1397 Gets a nanosecond timestamp.
1398 """
1399 if g_fWinUseWinPerfCounter is True:
1400 return long(_winFloatTime() * 1000000000);
1401 return long(time.time() * 1000000000);
1402
1403def timestampMilli():
1404 """
1405 Gets a millisecond timestamp.
1406 """
1407 if g_fWinUseWinPerfCounter is True:
1408 return long(_winFloatTime() * 1000);
1409 return long(time.time() * 1000);
1410
1411def timestampSecond():
1412 """
1413 Gets a second timestamp.
1414 """
1415 if g_fWinUseWinPerfCounter is True:
1416 return long(_winFloatTime());
1417 return long(time.time());
1418
1419def secondsSinceUnixEpoch():
1420 """
1421 Returns unix time, floating point second count since 1970-01-01T00:00:00Z
1422 """
1423 ## ASSUMES This returns unix epoch time on all systems we care about...
1424 return time.time();
1425
1426def getTimePrefix():
1427 """
1428 Returns a timestamp prefix, typically used for logging. UTC.
1429 """
1430 try:
1431 oNow = datetime.datetime.utcnow();
1432 sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1433 except:
1434 sTs = 'getTimePrefix-exception';
1435 return sTs;
1436
1437def getTimePrefixAndIsoTimestamp():
1438 """
1439 Returns current UTC as log prefix and iso timestamp.
1440 """
1441 try:
1442 oNow = datetime.datetime.utcnow();
1443 sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1444 sTsIso = formatIsoTimestamp(oNow);
1445 except:
1446 sTsPrf = sTsIso = 'getTimePrefix-exception';
1447 return (sTsPrf, sTsIso);
1448
1449def formatIsoTimestamp(oNow):
1450 """Formats the datetime object as an ISO timestamp."""
1451 assert oNow.tzinfo is None;
1452 sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1453 return sTs;
1454
1455def getIsoTimestamp():
1456 """Returns the current UTC timestamp as a string."""
1457 return formatIsoTimestamp(datetime.datetime.utcnow());
1458
1459
1460def getLocalHourOfWeek():
1461 """ Local hour of week (0 based). """
1462 oNow = datetime.datetime.now();
1463 return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1464
1465
1466def formatIntervalSeconds(cSeconds):
1467 """ Format a seconds interval into a nice 01h 00m 22s string """
1468 # Two simple special cases.
1469 if cSeconds < 60:
1470 return '%ss' % (cSeconds,);
1471 if cSeconds < 3600:
1472 cMins = cSeconds // 60;
1473 cSecs = cSeconds % 60;
1474 if cSecs == 0:
1475 return '%sm' % (cMins,);
1476 return '%sm %ss' % (cMins, cSecs,);
1477
1478 # Generic and a bit slower.
1479 cDays = cSeconds // 86400;
1480 cSeconds %= 86400;
1481 cHours = cSeconds // 3600;
1482 cSeconds %= 3600;
1483 cMins = cSeconds // 60;
1484 cSecs = cSeconds % 60;
1485 sRet = '';
1486 if cDays > 0:
1487 sRet = '%sd ' % (cDays,);
1488 if cHours > 0:
1489 sRet += '%sh ' % (cHours,);
1490 if cMins > 0:
1491 sRet += '%sm ' % (cMins,);
1492 if cSecs > 0:
1493 sRet += '%ss ' % (cSecs,);
1494 assert sRet; assert sRet[-1] == ' ';
1495 return sRet[:-1];
1496
1497def formatIntervalSeconds2(oSeconds):
1498 """
1499 Flexible input version of formatIntervalSeconds for use in WUI forms where
1500 data is usually already string form.
1501 """
1502 if isinstance(oSeconds, (int, long)):
1503 return formatIntervalSeconds(oSeconds);
1504 if not isString(oSeconds):
1505 try:
1506 lSeconds = long(oSeconds);
1507 except:
1508 pass;
1509 else:
1510 if lSeconds >= 0:
1511 return formatIntervalSeconds2(lSeconds);
1512 return oSeconds;
1513
1514def parseIntervalSeconds(sString):
1515 """
1516 Reverse of formatIntervalSeconds.
1517
1518 Returns (cSeconds, sError), where sError is None on success.
1519 """
1520
1521 # We might given non-strings, just return them without any fuss.
1522 if not isString(sString):
1523 if isinstance(sString, (int, long)) or sString is None:
1524 return (sString, None);
1525 ## @todo time/date objects?
1526 return (int(sString), None);
1527
1528 # Strip it and make sure it's not empty.
1529 sString = sString.strip();
1530 if not sString:
1531 return (0, 'Empty interval string.');
1532
1533 #
1534 # Split up the input into a list of 'valueN, unitN, ...'.
1535 #
1536 # Don't want to spend too much time trying to make re.split do exactly what
1537 # I need here, so please forgive the extra pass I'm making here.
1538 #
1539 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1540 asParts = [];
1541 for sPart in asRawParts:
1542 sPart = sPart.strip();
1543 if sPart:
1544 asParts.append(sPart);
1545 if not asParts:
1546 return (0, 'Empty interval string or something?');
1547
1548 #
1549 # Process them one or two at the time.
1550 #
1551 cSeconds = 0;
1552 asErrors = [];
1553 i = 0;
1554 while i < len(asParts):
1555 sNumber = asParts[i];
1556 i += 1;
1557 if sNumber.isdigit():
1558 iNumber = int(sNumber);
1559
1560 sUnit = 's';
1561 if i < len(asParts) and not asParts[i].isdigit():
1562 sUnit = asParts[i];
1563 i += 1;
1564
1565 sUnitLower = sUnit.lower();
1566 if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1567 pass;
1568 elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1569 iNumber *= 60;
1570 elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1571 iNumber *= 3600;
1572 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1573 iNumber *= 86400;
1574 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1575 iNumber *= 7 * 86400;
1576 else:
1577 asErrors.append('Unknown unit "%s".' % (sUnit,));
1578 cSeconds += iNumber;
1579 else:
1580 asErrors.append('Bad number "%s".' % (sNumber,));
1581 return (cSeconds, None if not asErrors else ' '.join(asErrors));
1582
1583def formatIntervalHours(cHours):
1584 """ Format a hours interval into a nice 1w 2d 1h string. """
1585 # Simple special cases.
1586 if cHours < 24:
1587 return '%sh' % (cHours,);
1588
1589 # Generic and a bit slower.
1590 cWeeks = cHours / (7 * 24);
1591 cHours %= 7 * 24;
1592 cDays = cHours / 24;
1593 cHours %= 24;
1594 sRet = '';
1595 if cWeeks > 0:
1596 sRet = '%sw ' % (cWeeks,);
1597 if cDays > 0:
1598 sRet = '%sd ' % (cDays,);
1599 if cHours > 0:
1600 sRet += '%sh ' % (cHours,);
1601 assert sRet; assert sRet[-1] == ' ';
1602 return sRet[:-1];
1603
1604def parseIntervalHours(sString):
1605 """
1606 Reverse of formatIntervalHours.
1607
1608 Returns (cHours, sError), where sError is None on success.
1609 """
1610
1611 # We might given non-strings, just return them without any fuss.
1612 if not isString(sString):
1613 if isinstance(sString, (int, long)) or sString is None:
1614 return (sString, None);
1615 ## @todo time/date objects?
1616 return (int(sString), None);
1617
1618 # Strip it and make sure it's not empty.
1619 sString = sString.strip();
1620 if not sString:
1621 return (0, 'Empty interval string.');
1622
1623 #
1624 # Split up the input into a list of 'valueN, unitN, ...'.
1625 #
1626 # Don't want to spend too much time trying to make re.split do exactly what
1627 # I need here, so please forgive the extra pass I'm making here.
1628 #
1629 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1630 asParts = [];
1631 for sPart in asRawParts:
1632 sPart = sPart.strip();
1633 if sPart:
1634 asParts.append(sPart);
1635 if not asParts:
1636 return (0, 'Empty interval string or something?');
1637
1638 #
1639 # Process them one or two at the time.
1640 #
1641 cHours = 0;
1642 asErrors = [];
1643 i = 0;
1644 while i < len(asParts):
1645 sNumber = asParts[i];
1646 i += 1;
1647 if sNumber.isdigit():
1648 iNumber = int(sNumber);
1649
1650 sUnit = 'h';
1651 if i < len(asParts) and not asParts[i].isdigit():
1652 sUnit = asParts[i];
1653 i += 1;
1654
1655 sUnitLower = sUnit.lower();
1656 if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1657 pass;
1658 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1659 iNumber *= 24;
1660 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1661 iNumber *= 7 * 24;
1662 else:
1663 asErrors.append('Unknown unit "%s".' % (sUnit,));
1664 cHours += iNumber;
1665 else:
1666 asErrors.append('Bad number "%s".' % (sNumber,));
1667 return (cHours, None if not asErrors else ' '.join(asErrors));
1668
1669
1670#
1671# Introspection.
1672#
1673
1674def getCallerName(oFrame=None, iFrame=2):
1675 """
1676 Returns the name of the caller's caller.
1677 """
1678 if oFrame is None:
1679 try:
1680 raise Exception();
1681 except:
1682 oFrame = sys.exc_info()[2].tb_frame.f_back;
1683 while iFrame > 1:
1684 if oFrame is not None:
1685 oFrame = oFrame.f_back;
1686 iFrame = iFrame - 1;
1687 if oFrame is not None:
1688 sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1689 return sName;
1690 return "unknown";
1691
1692
1693def getXcptInfo(cFrames = 1):
1694 """
1695 Gets text detailing the exception. (Good for logging.)
1696 Returns list of info strings.
1697 """
1698
1699 #
1700 # Try get exception info.
1701 #
1702 try:
1703 oType, oValue, oTraceback = sys.exc_info();
1704 except:
1705 oType = oValue = oTraceback = None;
1706 if oType is not None:
1707
1708 #
1709 # Try format the info
1710 #
1711 asRet = [];
1712 try:
1713 try:
1714 asRet = asRet + traceback.format_exception_only(oType, oValue);
1715 asTraceBack = traceback.format_tb(oTraceback);
1716 if cFrames is not None and cFrames <= 1:
1717 asRet.append(asTraceBack[-1]);
1718 else:
1719 asRet.append('Traceback:')
1720 for iFrame in range(min(cFrames, len(asTraceBack))):
1721 asRet.append(asTraceBack[-iFrame - 1]);
1722 asRet.append('Stack:')
1723 asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1724 except:
1725 asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1726
1727 if not asRet:
1728 asRet.append('No exception info...');
1729 except:
1730 asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1731 else:
1732 asRet = ['Couldn\'t find exception traceback.'];
1733 return asRet;
1734
1735
1736def getObjectTypeName(oObject):
1737 """
1738 Get the type name of the given object.
1739 """
1740 if oObject is None:
1741 return 'None';
1742
1743 # Get the type object.
1744 try:
1745 oType = type(oObject);
1746 except:
1747 return 'type-throws-exception';
1748
1749 # Python 2.x only: Handle old-style object wrappers.
1750 if sys.version_info[0] < 3:
1751 try:
1752 from types import InstanceType; # pylint: disable=no-name-in-module
1753 if oType == InstanceType:
1754 oType = oObject.__class__;
1755 except:
1756 pass;
1757
1758 # Get the name.
1759 try:
1760 return oType.__name__;
1761 except:
1762 return '__type__-throws-exception';
1763
1764
1765def chmodPlusX(sFile):
1766 """
1767 Makes the specified file or directory executable.
1768 Returns success indicator, no exceptions.
1769
1770 Note! Symbolic links are followed and the target will be changed.
1771 """
1772 try:
1773 oStat = os.stat(sFile);
1774 except:
1775 return False;
1776 try:
1777 os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
1778 except:
1779 return False;
1780 return True;
1781
1782
1783#
1784# TestSuite stuff.
1785#
1786
1787def isRunningFromCheckout(cScriptDepth = 1):
1788 """
1789 Checks if we're running from the SVN checkout or not.
1790 """
1791
1792 try:
1793 sFile = __file__;
1794 cScriptDepth = 1;
1795 except:
1796 sFile = sys.argv[0];
1797
1798 sDir = os.path.abspath(sFile);
1799 while cScriptDepth >= 0:
1800 sDir = os.path.dirname(sDir);
1801 if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1802 or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1803 return True;
1804 cScriptDepth -= 1;
1805
1806 return False;
1807
1808
1809#
1810# Bourne shell argument fun.
1811#
1812
1813
1814def argsSplit(sCmdLine):
1815 """
1816 Given a bourne shell command line invocation, split it up into arguments
1817 assuming IFS is space.
1818 Returns None on syntax error.
1819 """
1820 ## @todo bourne shell argument parsing!
1821 return sCmdLine.split(' ');
1822
1823def argsGetFirst(sCmdLine):
1824 """
1825 Given a bourne shell command line invocation, get return the first argument
1826 assuming IFS is space.
1827 Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1828 """
1829 asArgs = argsSplit(sCmdLine);
1830 if not asArgs:
1831 return None;
1832
1833 return asArgs[0];
1834
1835#
1836# String helpers.
1837#
1838
1839def stricmp(sFirst, sSecond):
1840 """
1841 Compares to strings in an case insensitive fashion.
1842
1843 Python doesn't seem to have any way of doing the correctly, so this is just
1844 an approximation using lower.
1845 """
1846 if sFirst == sSecond:
1847 return 0;
1848 sLower1 = sFirst.lower();
1849 sLower2 = sSecond.lower();
1850 if sLower1 == sLower2:
1851 return 0;
1852 if sLower1 < sLower2:
1853 return -1;
1854 return 1;
1855
1856
1857def versionCompare(sVer1, sVer2):
1858 """
1859 Compares to version strings in a fashion similar to RTStrVersionCompare.
1860 """
1861
1862 ## @todo implement me!!
1863
1864 if sVer1 == sVer2:
1865 return 0;
1866 if sVer1 < sVer2:
1867 return -1;
1868 return 1;
1869
1870
1871def formatNumber(lNum, sThousandSep = ' '):
1872 """
1873 Formats a decimal number with pretty separators.
1874 """
1875 sNum = str(lNum);
1876 sRet = sNum[-3:];
1877 off = len(sNum) - 3;
1878 while off > 0:
1879 off -= 3;
1880 sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
1881 return sRet;
1882
1883
1884def formatNumberNbsp(lNum):
1885 """
1886 Formats a decimal number with pretty separators.
1887 """
1888 sRet = formatNumber(lNum);
1889 return unicode(sRet).replace(' ', u'\u00a0');
1890
1891
1892def isString(oString):
1893 """
1894 Checks if the object is a string object, hiding difference between python 2 and 3.
1895
1896 Returns True if it's a string of some kind.
1897 Returns False if not.
1898 """
1899 if sys.version_info[0] >= 3:
1900 return isinstance(oString, str);
1901 return isinstance(oString, basestring); # pylint: disable=undefined-variable
1902
1903
1904def hasNonAsciiCharacters(sText):
1905 """
1906 Returns True is specified string has non-ASCII characters, False if ASCII only.
1907 """
1908 if isString(sText):
1909 for ch in sText:
1910 if ord(ch) >= 128:
1911 return True;
1912 else:
1913 # Probably byte array or some such thing.
1914 for ch in sText:
1915 if ch >= 128 or ch < 0:
1916 return True;
1917 return False;
1918
1919
1920#
1921# Unpacking.
1922#
1923
1924def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1925 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1926 """
1927 Worker for unpackFile that deals with ZIP files, same function signature.
1928 """
1929 import zipfile
1930 if fnError is None:
1931 fnError = fnLog;
1932
1933 fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
1934
1935 # Open it.
1936 try: oZipFile = zipfile.ZipFile(sArchive, 'r')
1937 except Exception as oXcpt:
1938 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1939 return None;
1940
1941 # Extract all members.
1942 asMembers = [];
1943 try:
1944 for sMember in oZipFile.namelist():
1945 if fnFilter is None or fnFilter(sMember) is not False:
1946 if sMember.endswith('/'):
1947 os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
1948 else:
1949 oZipFile.extract(sMember, sDstDir);
1950 asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
1951 except Exception as oXcpt:
1952 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1953 asMembers = None;
1954
1955 # close it.
1956 try: oZipFile.close();
1957 except Exception as oXcpt:
1958 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1959 asMembers = None;
1960
1961 return asMembers;
1962
1963
1964## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
1965g_fTarCopyFileObjOverriddend = False;
1966
1967def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError, bufsize = None):
1968 """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
1969 _ = bufsize;
1970 if length is None:
1971 __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
1972 elif length > 0:
1973 cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
1974 for _ in xrange(cFull):
1975 abBuffer = src.read(g_cbGoodBufferSize);
1976 dst.write(abBuffer);
1977 if len(abBuffer) != g_cbGoodBufferSize:
1978 raise exception('unexpected end of source file');
1979 if cbRemainder > 0:
1980 abBuffer = src.read(cbRemainder);
1981 dst.write(abBuffer);
1982 if len(abBuffer) != cbRemainder:
1983 raise exception('unexpected end of source file');
1984
1985
1986def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1987 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1988 """
1989 Worker for unpackFile that deals with tarballs, same function signature.
1990 """
1991 import shutil;
1992 import tarfile;
1993 if fnError is None:
1994 fnError = fnLog;
1995
1996 fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
1997
1998 #
1999 # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
2000 # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
2001 # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
2002 #
2003 if True is True: # pylint: disable=comparison-with-itself
2004 __installShUtilHacks(shutil);
2005 global g_fTarCopyFileObjOverriddend;
2006 if g_fTarCopyFileObjOverriddend is False:
2007 g_fTarCopyFileObjOverriddend = True;
2008 #if sys.hexversion < 0x03060000:
2009 tarfile.copyfileobj = __mytarfilecopyfileobj;
2010
2011 #
2012 # Open it.
2013 #
2014 # Note! We not using 'r:*' because we cannot allow seeking compressed files!
2015 # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
2016 #
2017 try:
2018 if sys.hexversion >= 0x03060000:
2019 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize, copybufsize = g_cbGoodBufferSize);
2020 else:
2021 oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
2022 except Exception as oXcpt:
2023 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
2024 return None;
2025
2026 # Extract all members.
2027 asMembers = [];
2028 try:
2029 for oTarInfo in oTarFile:
2030 try:
2031 if fnFilter is None or fnFilter(oTarInfo.name) is not False:
2032 if oTarInfo.islnk():
2033 # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
2034 # in the compressed tar stream. So, fall back on shutil.copy2 instead.
2035 sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
2036 sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
2037 sParentDir = os.path.dirname(sLinkFile);
2038 try: os.unlink(sLinkFile);
2039 except: pass;
2040 if sParentDir and not os.path.exists(sParentDir):
2041 os.makedirs(sParentDir);
2042 try: os.link(sLinkTarget, sLinkFile);
2043 except: shutil.copy2(sLinkTarget, sLinkFile);
2044 else:
2045 if oTarInfo.isdir():
2046 # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
2047 oTarInfo.mode |= 0x1c0; # (octal: 0700)
2048 oTarFile.extract(oTarInfo, sDstDir);
2049 asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
2050 except Exception as oXcpt:
2051 fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
2052 for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
2053 fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
2054 for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
2055 fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
2056 asMembers = None;
2057 break;
2058 except Exception as oXcpt:
2059 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
2060 asMembers = None;
2061
2062 #
2063 # Finally, close it.
2064 #
2065 try: oTarFile.close();
2066 except Exception as oXcpt:
2067 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
2068 asMembers = None;
2069
2070 return asMembers;
2071
2072
2073def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
2074 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
2075 """
2076 Unpacks the given file if it has a know archive extension, otherwise do
2077 nothing.
2078
2079 fnLog & fnError both take a string parameter.
2080
2081 fnFilter takes a member name (string) and returns True if it's included
2082 and False if excluded.
2083
2084 Returns list of the extracted files (full path) on success.
2085 Returns empty list if not a supported archive format.
2086 Returns None on failure. Raises no exceptions.
2087 """
2088 sBaseNameLower = os.path.basename(sArchive).lower();
2089
2090 #
2091 # Zip file?
2092 #
2093 if sBaseNameLower.endswith('.zip'):
2094 return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2095
2096 #
2097 # Tarball?
2098 #
2099 if sBaseNameLower.endswith('.tar') \
2100 or sBaseNameLower.endswith('.tar.gz') \
2101 or sBaseNameLower.endswith('.tgz') \
2102 or sBaseNameLower.endswith('.tar.bz2'):
2103 return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
2104
2105 #
2106 # Cannot classify it from the name, so just return that to the caller.
2107 #
2108 fnLog('Not unpacking "%s".' % (sArchive,));
2109 return [];
2110
2111
2112#
2113# Misc.
2114#
2115def areBytesEqual(oLeft, oRight):
2116 """
2117 Compares two byte arrays, strings or whatnot.
2118
2119 returns true / false accordingly.
2120 """
2121
2122 # If both are None, consider them equal (bogus?):
2123 if oLeft is None and oRight is None:
2124 return True;
2125
2126 # If just one is None, they can't match:
2127 if oLeft is None or oRight is None:
2128 return False;
2129
2130 # If both have the same type, use the compare operator of the class:
2131 if type(oLeft) is type(oRight):
2132 #print('same type: %s' % (oLeft == oRight,));
2133 return oLeft == oRight;
2134
2135 # On the offchance that they're both strings, but of different types.
2136 if isString(oLeft) and isString(oRight):
2137 #print('string compare: %s' % (oLeft == oRight,));
2138 return oLeft == oRight;
2139
2140 #
2141 # See if byte/buffer stuff that can be compared directory. If not convert
2142 # strings to bytes.
2143 #
2144 # Note! For 2.x, we must convert both sides to the buffer type or the
2145 # comparison may fail despite it working okay in test cases.
2146 #
2147 if sys.version_info[0] >= 3:
2148 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2149 return oLeft == oRight;
2150
2151 if isString(oLeft):
2152 try: oLeft = bytes(oLeft, 'utf-8');
2153 except: pass;
2154 if isString(oRight):
2155 try: oRight = bytes(oRight, 'utf-8');
2156 except: pass;
2157 else:
2158 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2159 if isinstance(oLeft, bytearray):
2160 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2161 else:
2162 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2163 #print('buf/byte #1 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2164 return oLeft == oRight;
2165
2166 if isString(oLeft):
2167 try: oLeft = bytearray(oLeft, 'utf-8'); # pylint: disable=redefined-variable-type
2168 except: pass;
2169 if isString(oRight):
2170 try: oRight = bytearray(oRight, 'utf-8'); # pylint: disable=redefined-variable-type
2171 except: pass;
2172
2173 # Check if we now have the same type for both:
2174 if type(oLeft) is type(oRight):
2175 #print('same type now: %s' % (oLeft == oRight,));
2176 return oLeft == oRight;
2177
2178 # Check if we now have buffer/memoryview vs bytes/bytesarray again.
2179 if sys.version_info[0] >= 3:
2180 if isinstance(oLeft, (bytearray, memoryview, bytes)) and isinstance(oRight, (bytearray, memoryview, bytes)): # pylint: disable=undefined-variable
2181 return oLeft == oRight;
2182 else:
2183 if isinstance(oLeft, (bytearray, buffer)) and isinstance(oRight, (bytearray, buffer)): # pylint: disable=undefined-variable
2184 if isinstance(oLeft, bytearray):
2185 oLeft = buffer(oLeft); # pylint: disable=redefined-variable-type,undefined-variable
2186 else:
2187 oRight = buffer(oRight); # pylint: disable=redefined-variable-type,undefined-variable
2188 #print('buf/byte #2 compare: %s (%s vs %s)' % (oLeft == oRight, type(oLeft), type(oRight),));
2189 return oLeft == oRight;
2190
2191 # Do item by item comparison:
2192 if len(oLeft) != len(oRight):
2193 #print('different length: %s vs %s' % (len(oLeft), len(oRight)));
2194 return False;
2195 i = len(oLeft);
2196 while i > 0:
2197 i = i - 1;
2198
2199 iElmLeft = oLeft[i];
2200 if not isinstance(iElmLeft, int) and not isinstance(iElmLeft, long):
2201 iElmLeft = ord(iElmLeft);
2202
2203 iElmRight = oRight[i];
2204 if not isinstance(iElmRight, int) and not isinstance(iElmRight, long):
2205 iElmRight = ord(iElmRight);
2206
2207 if iElmLeft != iElmRight:
2208 #print('element %d differs: %x %x' % (i, iElmLeft, iElmRight,));
2209 return False;
2210 return True;
2211
2212
2213#
2214# Unit testing.
2215#
2216
2217# pylint: disable=missing-docstring
2218# pylint: disable=undefined-variable
2219class BuildCategoryDataTestCase(unittest.TestCase):
2220 def testIntervalSeconds(self):
2221 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
2222 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
2223 self.assertEqual(parseIntervalSeconds('123'), (123, None));
2224 self.assertEqual(parseIntervalSeconds(123), (123, None));
2225 self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
2226 self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
2227 self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
2228 self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
2229 self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
2230 self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
2231 self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
2232
2233 def testHasNonAsciiChars(self):
2234 self.assertEqual(hasNonAsciiCharacters(''), False);
2235 self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
2236 self.assertEqual(hasNonAsciiCharacters('\x80 '), True);
2237 self.assertEqual(hasNonAsciiCharacters('\x79 '), False);
2238 self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
2239 self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
2240 self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
2241 self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
2242 self.assertEqual(hasNonAsciiCharacters(b'\x20\x20\x20'), False);
2243 self.assertEqual(hasNonAsciiCharacters(b'\x20\x81\x20'), True);
2244
2245 def testAreBytesEqual(self):
2246 self.assertEqual(areBytesEqual(None, None), True);
2247 self.assertEqual(areBytesEqual(None, ''), False);
2248 self.assertEqual(areBytesEqual('', ''), True);
2249 self.assertEqual(areBytesEqual('1', '1'), True);
2250 self.assertEqual(areBytesEqual('12345', '1234'), False);
2251 self.assertEqual(areBytesEqual('1234', '1234'), True);
2252 self.assertEqual(areBytesEqual('1234', b'1234'), True);
2253 self.assertEqual(areBytesEqual(b'1234', b'1234'), True);
2254 self.assertEqual(areBytesEqual(b'1234', '1234'), True);
2255 self.assertEqual(areBytesEqual(b'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2256 self.assertEqual(areBytesEqual('1234', bytearray([0x31,0x32,0x33,0x34])), True);
2257 self.assertEqual(areBytesEqual(u'1234', bytearray([0x31,0x32,0x33,0x34])), True);
2258 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x33,0x34])), True);
2259 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), '1224'), False);
2260 self.assertEqual(areBytesEqual(bytearray([0x31,0x32,0x33,0x34]), bytearray([0x31,0x32,0x32,0x34])), False);
2261 if sys.version_info[0] >= 3:
2262 pass;
2263 else:
2264 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2265 bytearray([0x31,0x32,0x33,0x34])), True);
2266 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2267 bytearray([0x99,0x32,0x32,0x34])), False);
2268 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2269 buffer(bytearray([0x31,0x32,0x33,0x34,0x34]), 0, 4)), True);
2270 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1),
2271 buffer(bytearray([0x99,0x32,0x33,0x34,0x34]), 0, 4)), False);
2272 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), b'1234'), True);
2273 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), '1234'), True);
2274 self.assertEqual(areBytesEqual(buffer(bytearray([0x30,0x31,0x32,0x33,0x34]), 1), u'1234'), True);
2275
2276if __name__ == '__main__':
2277 unittest.main();
2278 # not reached.
2279
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette