VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 65312

Last change on this file since 65312 was 65312, checked in by vboxsync, 8 years ago

common/utils.py: Fixed processListAll for darwin (returned empty list).
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 57.9 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: utils.py 65312 2017-01-16 10:22:59Z vboxsync $
3# pylint: disable=C0302
4
5"""
6Common Utility Functions.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2016 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 65312 $"
31
32
33# Standard Python imports.
34import datetime;
35import os;
36import platform;
37import re;
38import stat;
39import subprocess;
40import sys;
41import time;
42import traceback;
43import unittest;
44
45if sys.platform == 'win32':
46 import ctypes;
47 import win32api; # pylint: disable=F0401
48 import win32con; # pylint: disable=F0401
49 import win32console; # pylint: disable=F0401
50 import win32process; # pylint: disable=F0401
51else:
52 import signal;
53
54# Python 3 hacks:
55if sys.version_info[0] >= 3:
56 unicode = str; # pylint: disable=redefined-builtin,invalid-name
57 xrange = range; # pylint: disable=redefined-builtin,invalid-name
58 long = int; # pylint: disable=redefined-builtin,invalid-name
59
60
61#
62# Host OS and CPU.
63#
64
65def getHostOs():
66 """
67 Gets the host OS name (short).
68
69 See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
70 """
71 sPlatform = platform.system();
72 if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
73 sPlatform = sPlatform.lower();
74 elif sPlatform == 'Windows':
75 sPlatform = 'win';
76 elif sPlatform == 'SunOS':
77 sPlatform = 'solaris';
78 else:
79 raise Exception('Unsupported platform "%s"' % (sPlatform,));
80 return sPlatform;
81
82g_sHostArch = None;
83
84def getHostArch():
85 """
86 Gets the host CPU architecture.
87
88 See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
89 """
90 global g_sHostArch;
91 if g_sHostArch is None:
92 sArch = platform.machine();
93 if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
94 sArch = 'x86';
95 elif sArch in ('AMD64', 'amd64', 'x86_64'):
96 sArch = 'amd64';
97 elif sArch == 'i86pc': # SunOS
98 if platform.architecture()[0] == '64bit':
99 sArch = 'amd64';
100 else:
101 try:
102 sArch = processOutputChecked(['/usr/bin/isainfo', '-n',]);
103 except:
104 pass;
105 sArch = sArch.strip();
106 if sArch != 'amd64':
107 sArch = 'x86';
108 else:
109 raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
110 g_sHostArch = sArch;
111 return g_sHostArch;
112
113
114def getHostOsDotArch():
115 """
116 Gets the 'os.arch' for the host.
117 """
118 return '%s.%s' % (getHostOs(), getHostArch());
119
120
121def isValidOs(sOs):
122 """
123 Validates the OS name.
124 """
125 if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
126 'os2', 'solaris', 'win', 'os-agnostic'):
127 return True;
128 return False;
129
130
131def isValidArch(sArch):
132 """
133 Validates the CPU architecture name.
134 """
135 if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
136 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
137 return True;
138 return False;
139
140def isValidOsDotArch(sOsDotArch):
141 """
142 Validates the 'os.arch' string.
143 """
144
145 asParts = sOsDotArch.split('.');
146 if asParts.length() != 2:
147 return False;
148 return isValidOs(asParts[0]) \
149 and isValidArch(asParts[1]);
150
151def getHostOsVersion():
152 """
153 Returns the host OS version. This is platform.release with additional
154 distro indicator on linux.
155 """
156 sVersion = platform.release();
157 sOs = getHostOs();
158 if sOs == 'linux':
159 sDist = '';
160 try:
161 # try /etc/lsb-release first to distinguish between Debian and Ubuntu
162 oFile = open('/etc/lsb-release');
163 for sLine in oFile:
164 oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
165 if oMatch is not None:
166 sDist = oMatch.group(1).strip();
167 except:
168 pass;
169 if sDist:
170 sVersion += ' / ' + sDist;
171 else:
172 asFiles = \
173 [
174 [ '/etc/debian_version', 'Debian v'],
175 [ '/etc/gentoo-release', '' ],
176 [ '/etc/oracle-release', '' ],
177 [ '/etc/redhat-release', '' ],
178 [ '/etc/SuSE-release', '' ],
179 ];
180 for sFile, sPrefix in asFiles:
181 if os.path.isfile(sFile):
182 try:
183 oFile = open(sFile);
184 sLine = oFile.readline();
185 oFile.close();
186 except:
187 continue;
188 sLine = sLine.strip()
189 if len(sLine) > 0:
190 sVersion += ' / ' + sPrefix + sLine;
191 break;
192
193 elif sOs == 'solaris':
194 sVersion = platform.version();
195 if os.path.isfile('/etc/release'):
196 try:
197 oFile = open('/etc/release');
198 sLast = oFile.readlines()[-1];
199 oFile.close();
200 sLast = sLast.strip();
201 if len(sLast) > 0:
202 sVersion += ' (' + sLast + ')';
203 except:
204 pass;
205
206 elif sOs == 'darwin':
207 sOsxVersion = platform.mac_ver()[0];
208 codenames = {"4": "Tiger",
209 "5": "Leopard",
210 "6": "Snow Leopard",
211 "7": "Lion",
212 "8": "Mountain Lion",
213 "9": "Mavericks",
214 "10": "Yosemite",
215 "11": "El Capitan",
216 "12": "Sierra",
217 "13": "Unknown 13",
218 "14": "Unknown 14", }
219 sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
220
221 return sVersion;
222
223#
224# File system.
225#
226
227def openNoInherit(sFile, sMode = 'r'):
228 """
229 Wrapper around open() that tries it's best to make sure the file isn't
230 inherited by child processes.
231
232 This is a best effort thing at the moment as it doesn't synchronizes with
233 child process spawning in any way. Thus it can be subject to races in
234 multithreaded programs.
235 """
236
237 try:
238 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=F0401
239 except:
240 return open(sFile, sMode);
241
242 oFile = open(sFile, sMode)
243 #try:
244 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
245 #except:
246 # pass;
247 return oFile;
248
249def noxcptReadLink(sPath, sXcptRet):
250 """
251 No exceptions os.readlink wrapper.
252 """
253 try:
254 sRet = os.readlink(sPath); # pylint: disable=E1101
255 except:
256 sRet = sXcptRet;
257 return sRet;
258
259def readFile(sFile, sMode = 'rb'):
260 """
261 Reads the entire file.
262 """
263 oFile = open(sFile, sMode);
264 sRet = oFile.read();
265 oFile.close();
266 return sRet;
267
268def noxcptReadFile(sFile, sXcptRet, sMode = 'rb'):
269 """
270 No exceptions common.readFile wrapper.
271 """
272 try:
273 sRet = readFile(sFile, sMode);
274 except:
275 sRet = sXcptRet;
276 return sRet;
277
278def noxcptRmDir(sDir, oXcptRet = False):
279 """
280 No exceptions os.rmdir wrapper.
281 """
282 oRet = True;
283 try:
284 os.rmdir(sDir);
285 except:
286 oRet = oXcptRet;
287 return oRet;
288
289def noxcptDeleteFile(sFile, oXcptRet = False):
290 """
291 No exceptions os.remove wrapper.
292 """
293 oRet = True;
294 try:
295 os.remove(sFile);
296 except:
297 oRet = oXcptRet;
298 return oRet;
299
300
301def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
302 # type: (string, (string, stat) -> bool) -> bool
303 """
304 Recursively walks a directory tree, calling fnCallback for each.
305
306 fnCallback takes a full path and stat object (can be None). It
307 returns a boolean value, False stops walking and returns immediately.
308
309 Returns True or False depending on fnCallback.
310 Returns None fIgnoreExceptions is True and an exception was raised by listdir.
311 """
312 def __worker(sCurDir):
313 """ Worker for """
314 try:
315 asNames = os.listdir(sCurDir);
316 except:
317 if not fIgnoreExceptions:
318 raise;
319 return None;
320 rc = True;
321 for sName in asNames:
322 if sName not in [ '.', '..' ]:
323 sFullName = os.path.join(sCurDir, sName);
324 try: oStat = os.lstat(sFullName);
325 except: oStat = None;
326 if fnCallback(sFullName, oStat) is False:
327 return False;
328 if oStat is not None and stat.S_ISDIR(oStat.st_mode):
329 rc = __worker(sFullName);
330 if rc is False:
331 break;
332 return rc;
333
334 # Ensure unicode path here so listdir also returns unicode on windows.
335 ## @todo figure out unicode stuff on non-windows.
336 if sys.platform == 'win32':
337 sDir = unicode(sDir);
338 return __worker(sDir);
339
340
341
342def formatFileMode(uMode):
343 # type: (int) -> string
344 """
345 Format a st_mode value 'ls -la' fasion.
346 Returns string.
347 """
348 if stat.S_ISDIR(uMode): sMode = 'd';
349 elif stat.S_ISREG(uMode): sMode = '-';
350 elif stat.S_ISLNK(uMode): sMode = 'l';
351 elif stat.S_ISFIFO(uMode): sMode = 'p';
352 elif stat.S_ISCHR(uMode): sMode = 'c';
353 elif stat.S_ISBLK(uMode): sMode = 'b';
354 elif stat.S_ISSOCK(uMode): sMode = 's';
355 else: sMode = '?';
356 ## @todo sticky bits.
357 sMode += 'r' if uMode & stat.S_IRUSR else '-';
358 sMode += 'w' if uMode & stat.S_IWUSR else '-';
359 sMode += 'x' if uMode & stat.S_IXUSR else '-';
360 sMode += 'r' if uMode & stat.S_IRGRP else '-';
361 sMode += 'w' if uMode & stat.S_IWGRP else '-';
362 sMode += 'x' if uMode & stat.S_IXGRP else '-';
363 sMode += 'r' if uMode & stat.S_IROTH else '-';
364 sMode += 'w' if uMode & stat.S_IWOTH else '-';
365 sMode += 'x' if uMode & stat.S_IXOTH else '-';
366 sMode += ' ';
367 return sMode;
368
369
370def formatFileStat(oStat):
371 # type: (stat) -> string
372 """
373 Format a stat result 'ls -la' fasion (numeric IDs).
374 Returns string.
375 """
376 return '%s %3s %4s %4s %10s %s' \
377 % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
378 time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
379
380## Good buffer for file operations.
381g_cbGoodBufferSize = 256*1024;
382
383## The original shutil.copyfileobj.
384g_fnOriginalShCopyFileObj = None;
385
386def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
387 """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
388 return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
389
390def __installShUtilHacks(shutil):
391 """ Installs the shutil buffer size hacks. """
392 global g_fnOriginalShCopyFileObj;
393 if g_fnOriginalShCopyFileObj is None:
394 g_fnOriginalShCopyFileObj = shutil.copyfileobj;
395 shutil.copyfileobj = __myshutilcopyfileobj;
396 return True;
397
398
399def copyFileSimple(sFileSrc, sFileDst):
400 """
401 Wrapper around shutil.copyfile that simply copies the data of a regular file.
402 Raises exception on failure.
403 Return True for show.
404 """
405 import shutil;
406 __installShUtilHacks(shutil);
407 return shutil.copyfile(sFileSrc, sFileDst);
408
409#
410# SubProcess.
411#
412
413def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
414 """
415 If the "executable" is a python script, insert the python interpreter at
416 the head of the argument list so that it will work on systems which doesn't
417 support hash-bang scripts.
418 """
419
420 asArgs = dKeywordArgs.get('args');
421 if asArgs is None:
422 asArgs = aPositionalArgs[0];
423
424 if asArgs[0].endswith('.py'):
425 if sys.executable is not None and len(sys.executable) > 0:
426 asArgs.insert(0, sys.executable);
427 else:
428 asArgs.insert(0, 'python');
429
430 # paranoia...
431 if dKeywordArgs.get('args') is not None:
432 dKeywordArgs['args'] = asArgs;
433 else:
434 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
435 return None;
436
437def processPopenSafe(*aPositionalArgs, **dKeywordArgs):
438 """
439 Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
440 """
441 if getHostOs() == 'win':
442 if dKeywordArgs.get('creationflags', 0) == 0:
443 dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
444 return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
445
446def processCall(*aPositionalArgs, **dKeywordArgs):
447 """
448 Wrapper around subprocess.call to deal with its absence in older
449 python versions.
450 Returns process exit code (see subprocess.poll).
451 """
452 assert dKeywordArgs.get('stdout') is None;
453 assert dKeywordArgs.get('stderr') is None;
454 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
455 oProcess = processPopenSafe(*aPositionalArgs, **dKeywordArgs);
456 return oProcess.wait();
457
458def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
459 """
460 Wrapper around subprocess.check_output to deal with its absense in older
461 python versions.
462 """
463 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
464 oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
465
466 sOutput, _ = oProcess.communicate();
467 iExitCode = oProcess.poll();
468
469 if iExitCode is not 0:
470 asArgs = dKeywordArgs.get('args');
471 if asArgs is None:
472 asArgs = aPositionalArgs[0];
473 print(sOutput);
474 raise subprocess.CalledProcessError(iExitCode, asArgs);
475
476 return str(sOutput); # str() make pylint happy.
477
478g_fOldSudo = None;
479def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
480 """
481 Adds 'sudo' (or similar) to the args parameter, whereever it is.
482 """
483
484 # Are we root?
485 fIsRoot = True;
486 try:
487 fIsRoot = os.getuid() == 0; # pylint: disable=E1101
488 except:
489 pass;
490
491 # If not, prepend sudo (non-interactive, simulate initial login).
492 if fIsRoot is not True:
493 asArgs = dKeywordArgs.get('args');
494 if asArgs is None:
495 asArgs = aPositionalArgs[0];
496
497 # Detect old sudo.
498 global g_fOldSudo;
499 if g_fOldSudo is None:
500 try:
501 sVersion = processOutputChecked(['sudo', '-V']);
502 except:
503 sVersion = '1.7.0';
504 sVersion = sVersion.strip().split('\n')[0];
505 sVersion = sVersion.replace('Sudo version', '').strip();
506 g_fOldSudo = len(sVersion) >= 4 \
507 and sVersion[0] == '1' \
508 and sVersion[1] == '.' \
509 and sVersion[2] <= '6' \
510 and sVersion[3] == '.';
511
512 asArgs.insert(0, 'sudo');
513 if not g_fOldSudo:
514 asArgs.insert(1, '-n');
515 if fInitialEnv and not g_fOldSudo:
516 asArgs.insert(1, '-i');
517
518 # paranoia...
519 if dKeywordArgs.get('args') is not None:
520 dKeywordArgs['args'] = asArgs;
521 else:
522 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
523 return None;
524
525
526def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
527 """
528 sudo (or similar) + subprocess.call
529 """
530 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
531 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
532 return processCall(*aPositionalArgs, **dKeywordArgs);
533
534def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
535 """
536 sudo (or similar) + subprocess.check_output.
537 """
538 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
539 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
540 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
541
542def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
543 """
544 sudo (or similar) + subprocess.check_output, except '-i' isn't used.
545 """
546 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
547 _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
548 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
549
550def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
551 """
552 sudo (or similar) + processPopenSafe.
553 """
554 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
555 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
556 return processPopenSafe(*aPositionalArgs, **dKeywordArgs);
557
558
559#
560# Generic process stuff.
561#
562
563def processInterrupt(uPid):
564 """
565 Sends a SIGINT or equivalent to interrupt the specified process.
566 Returns True on success, False on failure.
567
568 On Windows hosts this may not work unless the process happens to be a
569 process group leader.
570 """
571 if sys.platform == 'win32':
572 try:
573 win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, uPid); # pylint: disable=no-member
574 fRc = True;
575 except:
576 fRc = False;
577 else:
578 try:
579 os.kill(uPid, signal.SIGINT);
580 fRc = True;
581 except:
582 fRc = False;
583 return fRc;
584
585def sendUserSignal1(uPid):
586 """
587 Sends a SIGUSR1 or equivalent to nudge the process into shutting down
588 (VBoxSVC) or something.
589 Returns True on success, False on failure or if not supported (win).
590
591 On Windows hosts this may not work unless the process happens to be a
592 process group leader.
593 """
594 if sys.platform == 'win32':
595 fRc = False;
596 else:
597 try:
598 os.kill(uPid, signal.SIGUSR1); # pylint: disable=E1101
599 fRc = True;
600 except:
601 fRc = False;
602 return fRc;
603
604def processTerminate(uPid):
605 """
606 Terminates the process in a nice manner (SIGTERM or equivalent).
607 Returns True on success, False on failure.
608 """
609 fRc = False;
610 if sys.platform == 'win32':
611 try:
612 hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, False, uPid); # pylint: disable=no-member
613 except:
614 pass;
615 else:
616 try:
617 win32process.TerminateProcess(hProcess, 0x40010004); # DBG_TERMINATE_PROCESS # pylint: disable=no-member
618 fRc = True;
619 except:
620 pass;
621 win32api.CloseHandle(hProcess) # pylint: disable=no-member
622 else:
623 try:
624 os.kill(uPid, signal.SIGTERM);
625 fRc = True;
626 except:
627 pass;
628 return fRc;
629
630def processKill(uPid):
631 """
632 Terminates the process with extreme prejudice (SIGKILL).
633 Returns True on success, False on failure.
634 """
635 if sys.platform == 'win32':
636 fRc = processTerminate(uPid);
637 else:
638 try:
639 os.kill(uPid, signal.SIGKILL); # pylint: disable=E1101
640 fRc = True;
641 except:
642 fRc = False;
643 return fRc;
644
645def processKillWithNameCheck(uPid, sName):
646 """
647 Like processKill(), but checks if the process name matches before killing
648 it. This is intended for killing using potentially stale pid values.
649
650 Returns True on success, False on failure.
651 """
652
653 if processCheckPidAndName(uPid, sName) is not True:
654 return False;
655 return processKill(uPid);
656
657
658def processExists(uPid):
659 """
660 Checks if the specified process exits.
661 This will only work if we can signal/open the process.
662
663 Returns True if it positively exists, False otherwise.
664 """
665 if sys.platform == 'win32':
666 fRc = False;
667 try:
668 hProcess = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, False, uPid); # pylint: disable=no-member
669 except:
670 pass;
671 else:
672 win32api.CloseHandle(hProcess); # pylint: disable=no-member
673 fRc = True;
674 else:
675 try:
676 os.kill(uPid, 0);
677 fRc = True;
678 except:
679 fRc = False;
680 return fRc;
681
682def processCheckPidAndName(uPid, sName):
683 """
684 Checks if a process PID and NAME matches.
685 """
686 fRc = processExists(uPid);
687 if fRc is not True:
688 return False;
689
690 if sys.platform == 'win32':
691 try:
692 from win32com.client import GetObject; # pylint: disable=F0401
693 oWmi = GetObject('winmgmts:');
694 aoProcesses = oWmi.InstancesOf('Win32_Process');
695 for oProcess in aoProcesses:
696 if long(oProcess.Properties_("ProcessId").Value) == uPid:
697 sCurName = oProcess.Properties_("Name").Value;
698 #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
699 sName = sName.lower();
700 sCurName = sCurName.lower();
701 if os.path.basename(sName) == sName:
702 sCurName = os.path.basename(sCurName);
703
704 if sCurName == sName \
705 or sCurName + '.exe' == sName \
706 or sCurName == sName + '.exe':
707 fRc = True;
708 break;
709 except:
710 #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
711 pass;
712 else:
713 if sys.platform in ('linux2', ):
714 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
715 elif sys.platform in ('sunos5',):
716 asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
717 elif sys.platform in ('darwin',):
718 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
719 else:
720 asPsCmd = None;
721
722 if asPsCmd is not None:
723 try:
724 oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
725 sCurName = oPs.communicate()[0];
726 iExitCode = oPs.wait();
727 except:
728 #reporter.logXcpt();
729 return False;
730
731 # ps fails with non-zero exit code if the pid wasn't found.
732 if iExitCode is not 0:
733 return False;
734 if sCurName is None:
735 return False;
736 sCurName = sCurName.strip();
737 if sCurName is '':
738 return False;
739
740 if os.path.basename(sName) == sName:
741 sCurName = os.path.basename(sCurName);
742 elif os.path.basename(sCurName) == sCurName:
743 sName = os.path.basename(sName);
744
745 if sCurName != sName:
746 return False;
747
748 fRc = True;
749 return fRc;
750
751
752class ProcessInfo(object):
753 """Process info."""
754 def __init__(self, iPid):
755 self.iPid = iPid;
756 self.iParentPid = None;
757 self.sImage = None;
758 self.sName = None;
759 self.asArgs = None;
760 self.sCwd = None;
761 self.iGid = None;
762 self.iUid = None;
763 self.iProcGroup = None;
764 self.iSessionId = None;
765
766 def loadAll(self):
767 """Load all the info."""
768 sOs = getHostOs();
769 if sOs == 'linux':
770 sProc = '/proc/%s/' % (self.iPid,);
771 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
772 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
773 if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
774 #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
775 # sProc = '/proc/%s/' % (self.iPid,);
776 # if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
777 # if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
778 else:
779 pass;
780 if self.sName is None and self.sImage is not None:
781 self.sName = self.sImage;
782
783 def windowsGrabProcessInfo(self, oProcess):
784 """Windows specific loadAll."""
785 try: self.sName = oProcess.Properties_("Name").Value;
786 except: pass;
787 try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
788 except: pass;
789 try: self.asArgs = oProcess.Properties_("CommandLine").Value; ## @todo split it.
790 except: pass;
791 try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
792 except: pass;
793 try: self.iSessionId = oProcess.Properties_("SessionId").Value;
794 except: pass;
795 if self.sName is None and self.sImage is not None:
796 self.sName = self.sImage;
797
798 def getBaseImageName(self):
799 """
800 Gets the base image name if available, use the process name if not available.
801 Returns image/process base name or None.
802 """
803 sRet = self.sImage if self.sName is None else self.sName;
804 if sRet is None:
805 self.loadAll();
806 sRet = self.sImage if self.sName is None else self.sName;
807 if sRet is None:
808 if self.asArgs is None or len(self.asArgs) == 0:
809 return None;
810 sRet = self.asArgs[0];
811 if len(sRet) == 0:
812 return None;
813 return os.path.basename(sRet);
814
815 def getBaseImageNameNoExeSuff(self):
816 """
817 Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
818 """
819 sRet = self.getBaseImageName();
820 if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
821 if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
822 sRet = sRet[:-4];
823 return sRet;
824
825
826def processListAll(): # pylint: disable=R0914
827 """
828 Return a list of ProcessInfo objects for all the processes in the system
829 that the current user can see.
830 """
831 asProcesses = [];
832
833 sOs = getHostOs();
834 if sOs == 'win':
835 from win32com.client import GetObject; # pylint: disable=F0401
836 oWmi = GetObject('winmgmts:');
837 aoProcesses = oWmi.InstancesOf('Win32_Process');
838 for oProcess in aoProcesses:
839 try:
840 iPid = int(oProcess.Properties_("ProcessId").Value);
841 except:
842 continue;
843 oMyInfo = ProcessInfo(iPid);
844 oMyInfo.windowsGrabProcessInfo(oProcess);
845 asProcesses.append(oMyInfo);
846 return asProcesses;
847
848 if sOs in [ 'linux', ]: # Not solaris, ps gets more info than /proc/.
849 try:
850 asDirs = os.listdir('/proc');
851 except:
852 asDirs = [];
853 for sDir in asDirs:
854 if sDir.isdigit():
855 asProcesses.append(ProcessInfo(int(sDir),));
856 return asProcesses;
857
858 #
859 # The other OSes parses the output from the 'ps' utility.
860 #
861 asPsCmd = [
862 '/bin/ps', # 0
863 '-A', # 1
864 '-o', 'pid=', # 2,3
865 '-o', 'ppid=', # 4,5
866 '-o', 'pgid=', # 6,7
867 '-o', 'sid=', # 8,9
868 '-o', 'uid=', # 10,11
869 '-o', 'gid=', # 12,13
870 '-o', 'comm=' # 14,15
871 ];
872
873 if sOs == 'darwin':
874 assert asPsCmd[9] == 'sid=';
875 asPsCmd[9] = 'sess=';
876 elif sOs == 'solaris':
877 asPsCmd[0] = '/usr/bin/ps';
878
879 try:
880 sRaw = processOutputChecked(asPsCmd);
881 except:
882 return asProcesses;
883
884 for sLine in sRaw.split('\n'):
885 sLine = sLine.lstrip();
886 if len(sLine) < 7 or not sLine[0].isdigit():
887 continue;
888
889 iField = 0;
890 off = 0;
891 aoFields = [None, None, None, None, None, None, None];
892 while iField < 7:
893 # Eat whitespace.
894 while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
895 off += 1;
896
897 # Final field / EOL.
898 if iField == 6:
899 aoFields[6] = sLine[off:];
900 break;
901 if off >= len(sLine):
902 break;
903
904 # Generic field parsing.
905 offStart = off;
906 off += 1;
907 while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
908 off += 1;
909 try:
910 if iField != 3:
911 aoFields[iField] = int(sLine[offStart:off]);
912 else:
913 aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
914 except:
915 pass;
916 iField += 1;
917
918 if aoFields[0] is not None:
919 oMyInfo = ProcessInfo(aoFields[0]);
920 oMyInfo.iParentPid = aoFields[1];
921 oMyInfo.iProcGroup = aoFields[2];
922 oMyInfo.iSessionId = aoFields[3];
923 oMyInfo.iUid = aoFields[4];
924 oMyInfo.iGid = aoFields[5];
925 oMyInfo.sName = aoFields[6];
926 asProcesses.append(oMyInfo);
927
928 return asProcesses;
929
930
931def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
932 """
933 Looks for information regarding the demise of the given process.
934 """
935 sOs = getHostOs();
936 if sOs == 'darwin':
937 #
938 # On darwin we look for crash and diagnostic reports.
939 #
940 asLogDirs = [
941 u'/Library/Logs/DiagnosticReports/',
942 u'/Library/Logs/CrashReporter/',
943 u'~/Library/Logs/DiagnosticReports/',
944 u'~/Library/Logs/CrashReporter/',
945 ];
946 for sDir in asLogDirs:
947 sDir = os.path.expanduser(sDir);
948 if not os.path.isdir(sDir):
949 continue;
950 try:
951 asDirEntries = os.listdir(sDir);
952 except:
953 continue;
954 for sEntry in asDirEntries:
955 # Only interested in .crash files.
956 _, sSuff = os.path.splitext(sEntry);
957 if sSuff != '.crash':
958 continue;
959
960 # The pid can be found at the end of the first line.
961 sFull = os.path.join(sDir, sEntry);
962 try:
963 oFile = open(sFull, 'r');
964 sFirstLine = oFile.readline();
965 oFile.close();
966 except:
967 continue;
968 if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
969 continue;
970 offPid = len(sFirstLine) - 3;
971 while offPid > 1 and sFirstLine[offPid - 1].isdigit():
972 offPid -= 1;
973 try: uReportPid = int(sFirstLine[offPid:-2]);
974 except: continue;
975
976 # Does the pid we found match?
977 if uReportPid == uPid:
978 fnLog('Found crash report for %u: %s' % (uPid, sFull,));
979 fnCrashFile(sFull, False);
980 elif sOs == 'win':
981 #
982 # Getting WER reports would be great, however we have trouble match the
983 # PID to those as they seems not to mention it in the brief reports.
984 # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
985 # location - see the windows readme for the testbox script) and what
986 # the MSDN article lists for now.
987 #
988 # It's been observed on Windows server 2012 that the dump files takes
989 # the form: <processimage>.<decimal-pid>.dmp
990 #
991 asDmpDirs = [
992 u'%SystemDrive%/CrashDumps/', # Testboxes.
993 u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
994 u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
995 u'%WINDIR%/ServiceProfiles/NetworkSerices/',
996 u'%WINDIR%/ServiceProfiles/',
997 u'%WINDIR%/System32/Config/SystemProfile/', # System services.
998 ];
999 sMatchSuffix = '.%u.dmp' % (uPid,);
1000
1001 for sDir in asDmpDirs:
1002 sDir = os.path.expandvars(sDir);
1003 if not os.path.isdir(sDir):
1004 continue;
1005 try:
1006 asDirEntries = os.listdir(sDir);
1007 except:
1008 continue;
1009 for sEntry in asDirEntries:
1010 if sEntry.endswith(sMatchSuffix):
1011 sFull = os.path.join(sDir, sEntry);
1012 fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
1013 fnCrashFile(sFull, True);
1014
1015 else:
1016 pass; ## TODO
1017 return None;
1018
1019
1020#
1021# Time.
1022#
1023
1024#
1025# The following test case shows how time.time() only have ~ms resolution
1026# on Windows (tested W10) and why it therefore makes sense to try use
1027# performance counters.
1028#
1029# Note! We cannot use time.clock() as the timestamp must be portable across
1030# processes. See timeout testcase problem on win hosts (no logs).
1031#
1032#import sys;
1033#import time;
1034#from common import utils;
1035#
1036#atSeries = [];
1037#for i in xrange(1,160):
1038# if i == 159: time.sleep(10);
1039# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1040#
1041#tPrev = atSeries[0]
1042#for tCur in atSeries:
1043# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1044# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1045# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1046# print '';
1047# tPrev = tCur
1048#
1049#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1050#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1051#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1052
1053g_fWinUseWinPerfCounter = sys.platform == 'win32';
1054g_fpWinPerfCounterFreq = None;
1055g_oFuncwinQueryPerformanceCounter = None;
1056
1057def _winInitPerfCounter():
1058 """ Initializes the use of performance counters. """
1059 global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1060
1061 uFrequency = ctypes.c_ulonglong(0);
1062 if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1063 if uFrequency.value >= 1000:
1064 #print 'uFrequency = %s' % (uFrequency,);
1065 #print 'type(uFrequency) = %s' % (type(uFrequency),);
1066 g_fpWinPerfCounterFreq = float(uFrequency.value);
1067
1068 # Check that querying the counter works too.
1069 global g_oFuncwinQueryPerformanceCounter
1070 g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1071 uCurValue = ctypes.c_ulonglong(0);
1072 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1073 if uCurValue.value > 0:
1074 return True;
1075 g_fWinUseWinPerfCounter = False;
1076 return False;
1077
1078def _winFloatTime():
1079 """ Gets floating point time on windows. """
1080 if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1081 uCurValue = ctypes.c_ulonglong(0);
1082 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1083 return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1084 return time.time();
1085
1086
1087def timestampNano():
1088 """
1089 Gets a nanosecond timestamp.
1090 """
1091 if g_fWinUseWinPerfCounter is True:
1092 return long(_winFloatTime() * 1000000000);
1093 return long(time.time() * 1000000000);
1094
1095def timestampMilli():
1096 """
1097 Gets a millisecond timestamp.
1098 """
1099 if g_fWinUseWinPerfCounter is True:
1100 return long(_winFloatTime() * 1000);
1101 return long(time.time() * 1000);
1102
1103def timestampSecond():
1104 """
1105 Gets a second timestamp.
1106 """
1107 if g_fWinUseWinPerfCounter is True:
1108 return long(_winFloatTime());
1109 return long(time.time());
1110
1111def getTimePrefix():
1112 """
1113 Returns a timestamp prefix, typically used for logging. UTC.
1114 """
1115 try:
1116 oNow = datetime.datetime.utcnow();
1117 sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1118 except:
1119 sTs = 'getTimePrefix-exception';
1120 return sTs;
1121
1122def getTimePrefixAndIsoTimestamp():
1123 """
1124 Returns current UTC as log prefix and iso timestamp.
1125 """
1126 try:
1127 oNow = datetime.datetime.utcnow();
1128 sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1129 sTsIso = formatIsoTimestamp(oNow);
1130 except:
1131 sTsPrf = sTsIso = 'getTimePrefix-exception';
1132 return (sTsPrf, sTsIso);
1133
1134def formatIsoTimestamp(oNow):
1135 """Formats the datetime object as an ISO timestamp."""
1136 assert oNow.tzinfo is None;
1137 sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1138 return sTs;
1139
1140def getIsoTimestamp():
1141 """Returns the current UTC timestamp as a string."""
1142 return formatIsoTimestamp(datetime.datetime.utcnow());
1143
1144
1145def getLocalHourOfWeek():
1146 """ Local hour of week (0 based). """
1147 oNow = datetime.datetime.now();
1148 return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1149
1150
1151def formatIntervalSeconds(cSeconds):
1152 """ Format a seconds interval into a nice 01h 00m 22s string """
1153 # Two simple special cases.
1154 if cSeconds < 60:
1155 return '%ss' % (cSeconds,);
1156 if cSeconds < 3600:
1157 cMins = cSeconds / 60;
1158 cSecs = cSeconds % 60;
1159 if cSecs == 0:
1160 return '%sm' % (cMins,);
1161 return '%sm %ss' % (cMins, cSecs,);
1162
1163 # Generic and a bit slower.
1164 cDays = cSeconds / 86400;
1165 cSeconds %= 86400;
1166 cHours = cSeconds / 3600;
1167 cSeconds %= 3600;
1168 cMins = cSeconds / 60;
1169 cSecs = cSeconds % 60;
1170 sRet = '';
1171 if cDays > 0:
1172 sRet = '%sd ' % (cDays,);
1173 if cHours > 0:
1174 sRet += '%sh ' % (cHours,);
1175 if cMins > 0:
1176 sRet += '%sm ' % (cMins,);
1177 if cSecs > 0:
1178 sRet += '%ss ' % (cSecs,);
1179 assert len(sRet) > 0; assert sRet[-1] == ' ';
1180 return sRet[:-1];
1181
1182def formatIntervalSeconds2(oSeconds):
1183 """
1184 Flexible input version of formatIntervalSeconds for use in WUI forms where
1185 data is usually already string form.
1186 """
1187 if isinstance(oSeconds, int) or isinstance(oSeconds, long):
1188 return formatIntervalSeconds(oSeconds);
1189 if not isString(oSeconds):
1190 try:
1191 lSeconds = long(oSeconds);
1192 except:
1193 pass;
1194 else:
1195 if lSeconds >= 0:
1196 return formatIntervalSeconds2(lSeconds);
1197 return oSeconds;
1198
1199def parseIntervalSeconds(sString):
1200 """
1201 Reverse of formatIntervalSeconds.
1202
1203 Returns (cSeconds, sError), where sError is None on success.
1204 """
1205
1206 # We might given non-strings, just return them without any fuss.
1207 if not isString(sString):
1208 if isinstance(sString, int) or isinstance(sString, long) or sString is None:
1209 return (sString, None);
1210 ## @todo time/date objects?
1211 return (int(sString), None);
1212
1213 # Strip it and make sure it's not empty.
1214 sString = sString.strip();
1215 if len(sString) == 0:
1216 return (0, 'Empty interval string.');
1217
1218 #
1219 # Split up the input into a list of 'valueN, unitN, ...'.
1220 #
1221 # Don't want to spend too much time trying to make re.split do exactly what
1222 # I need here, so please forgive the extra pass I'm making here.
1223 #
1224 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1225 asParts = [];
1226 for sPart in asRawParts:
1227 sPart = sPart.strip();
1228 if len(sPart) > 0:
1229 asParts.append(sPart);
1230 if len(asParts) == 0:
1231 return (0, 'Empty interval string or something?');
1232
1233 #
1234 # Process them one or two at the time.
1235 #
1236 cSeconds = 0;
1237 asErrors = [];
1238 i = 0;
1239 while i < len(asParts):
1240 sNumber = asParts[i];
1241 i += 1;
1242 if sNumber.isdigit():
1243 iNumber = int(sNumber);
1244
1245 sUnit = 's';
1246 if i < len(asParts) and not asParts[i].isdigit():
1247 sUnit = asParts[i];
1248 i += 1;
1249
1250 sUnitLower = sUnit.lower();
1251 if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1252 pass;
1253 elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1254 iNumber *= 60;
1255 elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1256 iNumber *= 3600;
1257 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1258 iNumber *= 86400;
1259 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1260 iNumber *= 7 * 86400;
1261 else:
1262 asErrors.append('Unknown unit "%s".' % (sUnit,));
1263 cSeconds += iNumber;
1264 else:
1265 asErrors.append('Bad number "%s".' % (sNumber,));
1266 return (cSeconds, None if len(asErrors) == 0 else ' '.join(asErrors));
1267
1268def formatIntervalHours(cHours):
1269 """ Format a hours interval into a nice 1w 2d 1h string. """
1270 # Simple special cases.
1271 if cHours < 24:
1272 return '%sh' % (cHours,);
1273
1274 # Generic and a bit slower.
1275 cWeeks = cHours / (7 * 24);
1276 cHours %= 7 * 24;
1277 cDays = cHours / 24;
1278 cHours %= 24;
1279 sRet = '';
1280 if cWeeks > 0:
1281 sRet = '%sw ' % (cWeeks,);
1282 if cDays > 0:
1283 sRet = '%sd ' % (cDays,);
1284 if cHours > 0:
1285 sRet += '%sh ' % (cHours,);
1286 assert len(sRet) > 0; assert sRet[-1] == ' ';
1287 return sRet[:-1];
1288
1289def parseIntervalHours(sString):
1290 """
1291 Reverse of formatIntervalHours.
1292
1293 Returns (cHours, sError), where sError is None on success.
1294 """
1295
1296 # We might given non-strings, just return them without any fuss.
1297 if not isString(sString):
1298 if isinstance(sString, int) or isinstance(sString, long) or sString is None:
1299 return (sString, None);
1300 ## @todo time/date objects?
1301 return (int(sString), None);
1302
1303 # Strip it and make sure it's not empty.
1304 sString = sString.strip();
1305 if len(sString) == 0:
1306 return (0, 'Empty interval string.');
1307
1308 #
1309 # Split up the input into a list of 'valueN, unitN, ...'.
1310 #
1311 # Don't want to spend too much time trying to make re.split do exactly what
1312 # I need here, so please forgive the extra pass I'm making here.
1313 #
1314 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1315 asParts = [];
1316 for sPart in asRawParts:
1317 sPart = sPart.strip();
1318 if len(sPart) > 0:
1319 asParts.append(sPart);
1320 if len(asParts) == 0:
1321 return (0, 'Empty interval string or something?');
1322
1323 #
1324 # Process them one or two at the time.
1325 #
1326 cHours = 0;
1327 asErrors = [];
1328 i = 0;
1329 while i < len(asParts):
1330 sNumber = asParts[i];
1331 i += 1;
1332 if sNumber.isdigit():
1333 iNumber = int(sNumber);
1334
1335 sUnit = 'h';
1336 if i < len(asParts) and not asParts[i].isdigit():
1337 sUnit = asParts[i];
1338 i += 1;
1339
1340 sUnitLower = sUnit.lower();
1341 if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1342 pass;
1343 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1344 iNumber *= 24;
1345 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1346 iNumber *= 7 * 24;
1347 else:
1348 asErrors.append('Unknown unit "%s".' % (sUnit,));
1349 cHours += iNumber;
1350 else:
1351 asErrors.append('Bad number "%s".' % (sNumber,));
1352 return (cHours, None if len(asErrors) == 0 else ' '.join(asErrors));
1353
1354
1355#
1356# Introspection.
1357#
1358
1359def getCallerName(oFrame=None, iFrame=2):
1360 """
1361 Returns the name of the caller's caller.
1362 """
1363 if oFrame is None:
1364 try:
1365 raise Exception();
1366 except:
1367 oFrame = sys.exc_info()[2].tb_frame.f_back;
1368 while iFrame > 1:
1369 if oFrame is not None:
1370 oFrame = oFrame.f_back;
1371 iFrame = iFrame - 1;
1372 if oFrame is not None:
1373 sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1374 return sName;
1375 return "unknown";
1376
1377
1378def getXcptInfo(cFrames = 1):
1379 """
1380 Gets text detailing the exception. (Good for logging.)
1381 Returns list of info strings.
1382 """
1383
1384 #
1385 # Try get exception info.
1386 #
1387 try:
1388 oType, oValue, oTraceback = sys.exc_info();
1389 except:
1390 oType = oValue = oTraceback = None;
1391 if oType is not None:
1392
1393 #
1394 # Try format the info
1395 #
1396 asRet = [];
1397 try:
1398 try:
1399 asRet = asRet + traceback.format_exception_only(oType, oValue);
1400 asTraceBack = traceback.format_tb(oTraceback);
1401 if cFrames is not None and cFrames <= 1:
1402 asRet.append(asTraceBack[-1]);
1403 else:
1404 asRet.append('Traceback:')
1405 for iFrame in range(min(cFrames, len(asTraceBack))):
1406 asRet.append(asTraceBack[-iFrame - 1]);
1407 asRet.append('Stack:')
1408 asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1409 except:
1410 asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1411
1412 if len(asRet) == 0:
1413 asRet.append('No exception info...');
1414 except:
1415 asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1416 else:
1417 asRet = ['Couldn\'t find exception traceback.'];
1418 return asRet;
1419
1420
1421#
1422# TestSuite stuff.
1423#
1424
1425def isRunningFromCheckout(cScriptDepth = 1):
1426 """
1427 Checks if we're running from the SVN checkout or not.
1428 """
1429
1430 try:
1431 sFile = __file__;
1432 cScriptDepth = 1;
1433 except:
1434 sFile = sys.argv[0];
1435
1436 sDir = os.path.abspath(sFile);
1437 while cScriptDepth >= 0:
1438 sDir = os.path.dirname(sDir);
1439 if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1440 or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1441 return True;
1442 cScriptDepth -= 1;
1443
1444 return False;
1445
1446
1447#
1448# Bourne shell argument fun.
1449#
1450
1451
1452def argsSplit(sCmdLine):
1453 """
1454 Given a bourne shell command line invocation, split it up into arguments
1455 assuming IFS is space.
1456 Returns None on syntax error.
1457 """
1458 ## @todo bourne shell argument parsing!
1459 return sCmdLine.split(' ');
1460
1461def argsGetFirst(sCmdLine):
1462 """
1463 Given a bourne shell command line invocation, get return the first argument
1464 assuming IFS is space.
1465 Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1466 """
1467 asArgs = argsSplit(sCmdLine);
1468 if asArgs is None or len(asArgs) == 0:
1469 return None;
1470
1471 return asArgs[0];
1472
1473#
1474# String helpers.
1475#
1476
1477def stricmp(sFirst, sSecond):
1478 """
1479 Compares to strings in an case insensitive fashion.
1480
1481 Python doesn't seem to have any way of doing the correctly, so this is just
1482 an approximation using lower.
1483 """
1484 if sFirst == sSecond:
1485 return 0;
1486 sLower1 = sFirst.lower();
1487 sLower2 = sSecond.lower();
1488 if sLower1 == sLower2:
1489 return 0;
1490 if sLower1 < sLower2:
1491 return -1;
1492 return 1;
1493
1494
1495#
1496# Misc.
1497#
1498
1499def versionCompare(sVer1, sVer2):
1500 """
1501 Compares to version strings in a fashion similar to RTStrVersionCompare.
1502 """
1503
1504 ## @todo implement me!!
1505
1506 if sVer1 == sVer2:
1507 return 0;
1508 if sVer1 < sVer2:
1509 return -1;
1510 return 1;
1511
1512
1513def formatNumber(lNum, sThousandSep = ' '):
1514 """
1515 Formats a decimal number with pretty separators.
1516 """
1517 sNum = str(lNum);
1518 sRet = sNum[-3:];
1519 off = len(sNum) - 3;
1520 while off > 0:
1521 off -= 3;
1522 sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
1523 return sRet;
1524
1525
1526def formatNumberNbsp(lNum):
1527 """
1528 Formats a decimal number with pretty separators.
1529 """
1530 sRet = formatNumber(lNum);
1531 return unicode(sRet).replace(' ', u'\u00a0');
1532
1533
1534def isString(oString):
1535 """
1536 Checks if the object is a string object, hiding difference between python 2 and 3.
1537
1538 Returns True if it's a string of some kind.
1539 Returns False if not.
1540 """
1541 if sys.version_info[0] >= 3:
1542 return isinstance(oString, str);
1543 return isinstance(oString, basestring);
1544
1545
1546def hasNonAsciiCharacters(sText):
1547 """
1548 Returns True is specified string has non-ASCII characters, False if ASCII only.
1549 """
1550 sTmp = unicode(sText, errors='ignore') if isinstance(sText, str) else sText;
1551 return not all(ord(ch) < 128 for ch in sTmp);
1552
1553
1554def chmodPlusX(sFile):
1555 """
1556 Makes the specified file or directory executable.
1557 Returns success indicator, no exceptions.
1558
1559 Note! Symbolic links are followed and the target will be changed.
1560 """
1561 try:
1562 oStat = os.stat(sFile);
1563 except:
1564 return False;
1565 try:
1566 os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
1567 except:
1568 return False;
1569 return True;
1570
1571
1572def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1573 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1574 """
1575 Worker for unpackFile that deals with ZIP files, same function signature.
1576 """
1577 import zipfile
1578 if fnError is None:
1579 fnError = fnLog;
1580
1581 fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
1582
1583 # Open it.
1584 try: oZipFile = zipfile.ZipFile(sArchive, 'r')
1585 except Exception as oXcpt:
1586 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1587 return None;
1588
1589 # Extract all members.
1590 asMembers = [];
1591 try:
1592 for sMember in oZipFile.namelist():
1593 if fnFilter is None or fnFilter(sMember) is not False:
1594 if sMember.endswith('/'):
1595 os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
1596 else:
1597 oZipFile.extract(sMember, sDstDir);
1598 asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
1599 except Exception as oXcpt:
1600 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1601 asMembers = None;
1602
1603 # close it.
1604 try: oZipFile.close();
1605 except Exception as oXcpt:
1606 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1607 asMembers = None;
1608
1609 return asMembers;
1610
1611
1612## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
1613g_fTarCopyFileObjOverriddend = False;
1614
1615def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError):
1616 """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
1617 if length is None:
1618 __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
1619 elif length > 0:
1620 cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
1621 for _ in xrange(cFull):
1622 abBuffer = src.read(g_cbGoodBufferSize);
1623 dst.write(abBuffer);
1624 if len(abBuffer) != g_cbGoodBufferSize:
1625 raise exception('unexpected end of source file');
1626 if cbRemainder > 0:
1627 abBuffer = src.read(cbRemainder);
1628 dst.write(abBuffer);
1629 if len(abBuffer) != cbRemainder:
1630 raise exception('unexpected end of source file');
1631
1632
1633def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1634 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1635 """
1636 Worker for unpackFile that deals with tarballs, same function signature.
1637 """
1638 import shutil;
1639 import tarfile;
1640 if fnError is None:
1641 fnError = fnLog;
1642
1643 fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
1644
1645 #
1646 # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
1647 # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
1648 # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
1649 #
1650 if True is True:
1651 __installShUtilHacks(shutil);
1652 global g_fTarCopyFileObjOverriddend;
1653 if g_fTarCopyFileObjOverriddend is False:
1654 g_fTarCopyFileObjOverriddend = True;
1655 tarfile.copyfileobj = __mytarfilecopyfileobj;
1656
1657 #
1658 # Open it.
1659 #
1660 # Note! We not using 'r:*' because we cannot allow seeking compressed files!
1661 # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
1662 #
1663 try: oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
1664 except Exception as oXcpt:
1665 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1666 return None;
1667
1668 # Extract all members.
1669 asMembers = [];
1670 try:
1671 for oTarInfo in oTarFile:
1672 try:
1673 if fnFilter is None or fnFilter(oTarInfo.name) is not False:
1674 if oTarInfo.islnk():
1675 # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
1676 # in the compressed tar stream. So, fall back on shutil.copy2 instead.
1677 sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
1678 sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
1679 sParentDir = os.path.dirname(sLinkFile);
1680 try: os.unlink(sLinkFile);
1681 except: pass;
1682 if sParentDir is not '' and not os.path.exists(sParentDir):
1683 os.makedirs(sParentDir);
1684 try: os.link(sLinkTarget, sLinkFile);
1685 except: shutil.copy2(sLinkTarget, sLinkFile);
1686 else:
1687 if oTarInfo.isdir():
1688 # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
1689 oTarInfo.mode |= 0x1c0; # (octal: 0700)
1690 oTarFile.extract(oTarInfo, sDstDir);
1691 asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
1692 except Exception as oXcpt:
1693 fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
1694 for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
1695 fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
1696 for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
1697 fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
1698 asMembers = None;
1699 break;
1700 except Exception as oXcpt:
1701 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1702 asMembers = None;
1703
1704 #
1705 # Finally, close it.
1706 #
1707 try: oTarFile.close();
1708 except Exception as oXcpt:
1709 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1710 asMembers = None;
1711
1712 return asMembers;
1713
1714
1715def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1716 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1717 """
1718 Unpacks the given file if it has a know archive extension, otherwise do
1719 nothing.
1720
1721 fnLog & fnError both take a string parameter.
1722
1723 fnFilter takes a member name (string) and returns True if it's included
1724 and False if excluded.
1725
1726 Returns list of the extracted files (full path) on success.
1727 Returns empty list if not a supported archive format.
1728 Returns None on failure. Raises no exceptions.
1729 """
1730 sBaseNameLower = os.path.basename(sArchive).lower();
1731
1732 #
1733 # Zip file?
1734 #
1735 if sBaseNameLower.endswith('.zip'):
1736 return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
1737
1738 #
1739 # Tarball?
1740 #
1741 if sBaseNameLower.endswith('.tar') \
1742 or sBaseNameLower.endswith('.tar.gz') \
1743 or sBaseNameLower.endswith('.tgz') \
1744 or sBaseNameLower.endswith('.tar.bz2'):
1745 return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
1746
1747 #
1748 # Cannot classify it from the name, so just return that to the caller.
1749 #
1750 fnLog('Not unpacking "%s".' % (sArchive,));
1751 return [];
1752
1753
1754def getDiskUsage(sPath):
1755 """
1756 Get free space of a partition that corresponds to specified sPath in MB.
1757
1758 Returns partition free space value in MB.
1759 """
1760 if platform.system() == 'Windows':
1761 oCTypeFreeSpace = ctypes.c_ulonglong(0);
1762 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
1763 ctypes.pointer(oCTypeFreeSpace));
1764 cbFreeSpace = oCTypeFreeSpace.value;
1765 else:
1766 oStats = os.statvfs(sPath); # pylint: disable=E1101
1767 cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
1768
1769 # Convert to MB
1770 cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
1771
1772 return cMbFreeSpace;
1773
1774
1775#
1776# Unit testing.
1777#
1778
1779# pylint: disable=C0111
1780class BuildCategoryDataTestCase(unittest.TestCase):
1781 def testIntervalSeconds(self):
1782 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
1783 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
1784 self.assertEqual(parseIntervalSeconds('123'), (123, None));
1785 self.assertEqual(parseIntervalSeconds(123), (123, None));
1786 self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
1787 self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
1788 self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
1789 self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
1790 self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
1791 self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
1792 self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
1793
1794 def testHasNonAsciiChars(self):
1795 self.assertEqual(hasNonAsciiCharacters(''), False);
1796 self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
1797 self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
1798 self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
1799 self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
1800 self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
1801
1802if __name__ == '__main__':
1803 unittest.main();
1804 # not reached.
1805
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette