VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 64120

Last change on this file since 64120 was 64120, checked in by vboxsync, 8 years ago

common/utils.py: hasNonAsciiCharacters testcase.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 57.5 KB
Line 
1# -*- coding: utf-8 -*-
2# $Id: utils.py 64120 2016-10-02 06:39:19Z vboxsync $
3# pylint: disable=C0302
4
5"""
6Common Utility Functions.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2016 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 64120 $"
31
32
33# Standard Python imports.
34import datetime;
35import os;
36import platform;
37import re;
38import stat;
39import subprocess;
40import sys;
41import time;
42import traceback;
43import unittest;
44
45if sys.platform == 'win32':
46 import ctypes;
47 import win32api; # pylint: disable=F0401
48 import win32con; # pylint: disable=F0401
49 import win32console; # pylint: disable=F0401
50 import win32process; # pylint: disable=F0401
51else:
52 import signal;
53
54# Python 3 hacks:
55if sys.version_info[0] >= 3:
56 unicode = str; # pylint: disable=redefined-builtin,invalid-name
57 xrange = range; # pylint: disable=redefined-builtin,invalid-name
58 long = int; # pylint: disable=redefined-builtin,invalid-name
59
60
61#
62# Host OS and CPU.
63#
64
65def getHostOs():
66 """
67 Gets the host OS name (short).
68
69 See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
70 """
71 sPlatform = platform.system();
72 if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
73 sPlatform = sPlatform.lower();
74 elif sPlatform == 'Windows':
75 sPlatform = 'win';
76 elif sPlatform == 'SunOS':
77 sPlatform = 'solaris';
78 else:
79 raise Exception('Unsupported platform "%s"' % (sPlatform,));
80 return sPlatform;
81
82g_sHostArch = None;
83
84def getHostArch():
85 """
86 Gets the host CPU architecture.
87
88 See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
89 """
90 global g_sHostArch;
91 if g_sHostArch is None:
92 sArch = platform.machine();
93 if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
94 sArch = 'x86';
95 elif sArch in ('AMD64', 'amd64', 'x86_64'):
96 sArch = 'amd64';
97 elif sArch == 'i86pc': # SunOS
98 if platform.architecture()[0] == '64bit':
99 sArch = 'amd64';
100 else:
101 try:
102 sArch = processOutputChecked(['/usr/bin/isainfo', '-n',]);
103 except:
104 pass;
105 sArch = sArch.strip();
106 if sArch != 'amd64':
107 sArch = 'x86';
108 else:
109 raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
110 g_sHostArch = sArch;
111 return g_sHostArch;
112
113
114def getHostOsDotArch():
115 """
116 Gets the 'os.arch' for the host.
117 """
118 return '%s.%s' % (getHostOs(), getHostArch());
119
120
121def isValidOs(sOs):
122 """
123 Validates the OS name.
124 """
125 if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
126 'os2', 'solaris', 'win', 'os-agnostic'):
127 return True;
128 return False;
129
130
131def isValidArch(sArch):
132 """
133 Validates the CPU architecture name.
134 """
135 if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
136 'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
137 return True;
138 return False;
139
140def isValidOsDotArch(sOsDotArch):
141 """
142 Validates the 'os.arch' string.
143 """
144
145 asParts = sOsDotArch.split('.');
146 if asParts.length() != 2:
147 return False;
148 return isValidOs(asParts[0]) \
149 and isValidArch(asParts[1]);
150
151def getHostOsVersion():
152 """
153 Returns the host OS version. This is platform.release with additional
154 distro indicator on linux.
155 """
156 sVersion = platform.release();
157 sOs = getHostOs();
158 if sOs == 'linux':
159 sDist = '';
160 try:
161 # try /etc/lsb-release first to distinguish between Debian and Ubuntu
162 oFile = open('/etc/lsb-release');
163 for sLine in oFile:
164 oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
165 if oMatch is not None:
166 sDist = oMatch.group(1).strip();
167 except:
168 pass;
169 if sDist:
170 sVersion += ' / ' + sDist;
171 else:
172 asFiles = \
173 [
174 [ '/etc/debian_version', 'Debian v'],
175 [ '/etc/gentoo-release', '' ],
176 [ '/etc/oracle-release', '' ],
177 [ '/etc/redhat-release', '' ],
178 [ '/etc/SuSE-release', '' ],
179 ];
180 for sFile, sPrefix in asFiles:
181 if os.path.isfile(sFile):
182 try:
183 oFile = open(sFile);
184 sLine = oFile.readline();
185 oFile.close();
186 except:
187 continue;
188 sLine = sLine.strip()
189 if len(sLine) > 0:
190 sVersion += ' / ' + sPrefix + sLine;
191 break;
192
193 elif sOs == 'solaris':
194 sVersion = platform.version();
195 if os.path.isfile('/etc/release'):
196 try:
197 oFile = open('/etc/release');
198 sLast = oFile.readlines()[-1];
199 oFile.close();
200 sLast = sLast.strip();
201 if len(sLast) > 0:
202 sVersion += ' (' + sLast + ')';
203 except:
204 pass;
205
206 elif sOs == 'darwin':
207 sOsxVersion = platform.mac_ver()[0];
208 codenames = {"4": "Tiger",
209 "5": "Leopard",
210 "6": "Snow Leopard",
211 "7": "Lion",
212 "8": "Mountain Lion",
213 "9": "Mavericks",
214 "10": "Yosemite",
215 "11": "El Capitan",
216 "12": "Sierra",
217 "13": "Unknown 13",
218 "14": "Unknown 14", }
219 sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'
220
221 return sVersion;
222
223#
224# File system.
225#
226
227def openNoInherit(sFile, sMode = 'r'):
228 """
229 Wrapper around open() that tries it's best to make sure the file isn't
230 inherited by child processes.
231
232 This is a best effort thing at the moment as it doesn't synchronizes with
233 child process spawning in any way. Thus it can be subject to races in
234 multithreaded programs.
235 """
236
237 try:
238 from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=F0401
239 except:
240 return open(sFile, sMode);
241
242 oFile = open(sFile, sMode)
243 #try:
244 fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
245 #except:
246 # pass;
247 return oFile;
248
249def noxcptReadLink(sPath, sXcptRet):
250 """
251 No exceptions os.readlink wrapper.
252 """
253 try:
254 sRet = os.readlink(sPath); # pylint: disable=E1101
255 except:
256 sRet = sXcptRet;
257 return sRet;
258
259def readFile(sFile, sMode = 'rb'):
260 """
261 Reads the entire file.
262 """
263 oFile = open(sFile, sMode);
264 sRet = oFile.read();
265 oFile.close();
266 return sRet;
267
268def noxcptReadFile(sFile, sXcptRet, sMode = 'rb'):
269 """
270 No exceptions common.readFile wrapper.
271 """
272 try:
273 sRet = readFile(sFile, sMode);
274 except:
275 sRet = sXcptRet;
276 return sRet;
277
278def noxcptRmDir(sDir, oXcptRet = False):
279 """
280 No exceptions os.rmdir wrapper.
281 """
282 oRet = True;
283 try:
284 os.rmdir(sDir);
285 except:
286 oRet = oXcptRet;
287 return oRet;
288
289def noxcptDeleteFile(sFile, oXcptRet = False):
290 """
291 No exceptions os.remove wrapper.
292 """
293 oRet = True;
294 try:
295 os.remove(sFile);
296 except:
297 oRet = oXcptRet;
298 return oRet;
299
300
301def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
302 # type: (string, (string, stat) -> bool) -> bool
303 """
304 Recursively walks a directory tree, calling fnCallback for each.
305
306 fnCallback takes a full path and stat object (can be None). It
307 returns a boolean value, False stops walking and returns immediately.
308
309 Returns True or False depending on fnCallback.
310 Returns None fIgnoreExceptions is True and an exception was raised by listdir.
311 """
312 def __worker(sCurDir):
313 """ Worker for """
314 try:
315 asNames = os.listdir(sCurDir);
316 except:
317 if not fIgnoreExceptions:
318 raise;
319 return None;
320 rc = True;
321 for sName in asNames:
322 if sName not in [ '.', '..' ]:
323 sFullName = os.path.join(sCurDir, sName);
324 try: oStat = os.lstat(sFullName);
325 except: oStat = None;
326 if fnCallback(sFullName, oStat) is False:
327 return False;
328 if oStat is not None and stat.S_ISDIR(oStat.st_mode):
329 rc = __worker(sFullName);
330 if rc is False:
331 break;
332 return rc;
333
334 # Ensure unicode path here so listdir also returns unicode on windows.
335 ## @todo figure out unicode stuff on non-windows.
336 if sys.platform == 'win32':
337 sDir = unicode(sDir);
338 return __worker(sDir);
339
340
341
342def formatFileMode(uMode):
343 # type: (int) -> string
344 """
345 Format a st_mode value 'ls -la' fasion.
346 Returns string.
347 """
348 if stat.S_ISDIR(uMode): sMode = 'd';
349 elif stat.S_ISREG(uMode): sMode = '-';
350 elif stat.S_ISLNK(uMode): sMode = 'l';
351 elif stat.S_ISFIFO(uMode): sMode = 'p';
352 elif stat.S_ISCHR(uMode): sMode = 'c';
353 elif stat.S_ISBLK(uMode): sMode = 'b';
354 elif stat.S_ISSOCK(uMode): sMode = 's';
355 else: sMode = '?';
356 ## @todo sticky bits.
357 sMode += 'r' if uMode & stat.S_IRUSR else '-';
358 sMode += 'w' if uMode & stat.S_IWUSR else '-';
359 sMode += 'x' if uMode & stat.S_IXUSR else '-';
360 sMode += 'r' if uMode & stat.S_IRGRP else '-';
361 sMode += 'w' if uMode & stat.S_IWGRP else '-';
362 sMode += 'x' if uMode & stat.S_IXGRP else '-';
363 sMode += 'r' if uMode & stat.S_IROTH else '-';
364 sMode += 'w' if uMode & stat.S_IWOTH else '-';
365 sMode += 'x' if uMode & stat.S_IXOTH else '-';
366 sMode += ' ';
367 return sMode;
368
369
370def formatFileStat(oStat):
371 # type: (stat) -> string
372 """
373 Format a stat result 'ls -la' fasion (numeric IDs).
374 Returns string.
375 """
376 return '%s %3s %4s %4s %10s %s' \
377 % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
378 time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );
379
380## Good buffer for file operations.
381g_cbGoodBufferSize = 256*1024;
382
383## The original shutil.copyfileobj.
384g_fnOriginalShCopyFileObj = None;
385
386def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
387 """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
388 return g_fnOriginalShCopyFileObj(fsrc, fdst, length);
389
390def __installShUtilHacks(shutil):
391 """ Installs the shutil buffer size hacks. """
392 global g_fnOriginalShCopyFileObj;
393 if g_fnOriginalShCopyFileObj is None:
394 g_fnOriginalShCopyFileObj = shutil.copyfileobj;
395 shutil.copyfileobj = __myshutilcopyfileobj;
396 return True;
397
398
399def copyFileSimple(sFileSrc, sFileDst):
400 """
401 Wrapper around shutil.copyfile that simply copies the data of a regular file.
402 Raises exception on failure.
403 Return True for show.
404 """
405 import shutil;
406 __installShUtilHacks(shutil);
407 return shutil.copyfile(sFileSrc, sFileDst);
408
409#
410# SubProcess.
411#
412
413def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
414 """
415 If the "executable" is a python script, insert the python interpreter at
416 the head of the argument list so that it will work on systems which doesn't
417 support hash-bang scripts.
418 """
419
420 asArgs = dKeywordArgs.get('args');
421 if asArgs is None:
422 asArgs = aPositionalArgs[0];
423
424 if asArgs[0].endswith('.py'):
425 if sys.executable is not None and len(sys.executable) > 0:
426 asArgs.insert(0, sys.executable);
427 else:
428 asArgs.insert(0, 'python');
429
430 # paranoia...
431 if dKeywordArgs.get('args') is not None:
432 dKeywordArgs['args'] = asArgs;
433 else:
434 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
435 return None;
436
437def processCall(*aPositionalArgs, **dKeywordArgs):
438 """
439 Wrapper around subprocess.call to deal with its absense in older
440 python versions.
441 Returns process exit code (see subprocess.poll).
442 """
443 assert dKeywordArgs.get('stdout') is None;
444 assert dKeywordArgs.get('stderr') is None;
445 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
446 oProcess = subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
447 return oProcess.wait();
448
449def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
450 """
451 Wrapper around subprocess.check_output to deal with its absense in older
452 python versions.
453 """
454 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
455 oProcess = subprocess.Popen(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);
456
457 sOutput, _ = oProcess.communicate();
458 iExitCode = oProcess.poll();
459
460 if iExitCode is not 0:
461 asArgs = dKeywordArgs.get('args');
462 if asArgs is None:
463 asArgs = aPositionalArgs[0];
464 print(sOutput);
465 raise subprocess.CalledProcessError(iExitCode, asArgs);
466
467 return str(sOutput); # str() make pylint happy.
468
469g_fOldSudo = None;
470def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
471 """
472 Adds 'sudo' (or similar) to the args parameter, whereever it is.
473 """
474
475 # Are we root?
476 fIsRoot = True;
477 try:
478 fIsRoot = os.getuid() == 0; # pylint: disable=E1101
479 except:
480 pass;
481
482 # If not, prepend sudo (non-interactive, simulate initial login).
483 if fIsRoot is not True:
484 asArgs = dKeywordArgs.get('args');
485 if asArgs is None:
486 asArgs = aPositionalArgs[0];
487
488 # Detect old sudo.
489 global g_fOldSudo;
490 if g_fOldSudo is None:
491 try:
492 sVersion = processOutputChecked(['sudo', '-V']);
493 except:
494 sVersion = '1.7.0';
495 sVersion = sVersion.strip().split('\n')[0];
496 sVersion = sVersion.replace('Sudo version', '').strip();
497 g_fOldSudo = len(sVersion) >= 4 \
498 and sVersion[0] == '1' \
499 and sVersion[1] == '.' \
500 and sVersion[2] <= '6' \
501 and sVersion[3] == '.';
502
503 asArgs.insert(0, 'sudo');
504 if not g_fOldSudo:
505 asArgs.insert(1, '-n');
506 if fInitialEnv and not g_fOldSudo:
507 asArgs.insert(1, '-i');
508
509 # paranoia...
510 if dKeywordArgs.get('args') is not None:
511 dKeywordArgs['args'] = asArgs;
512 else:
513 aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
514 return None;
515
516
517def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
518 """
519 sudo (or similar) + subprocess.call
520 """
521 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
522 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
523 return processCall(*aPositionalArgs, **dKeywordArgs);
524
525def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
526 """
527 sudo (or similar) + subprocess.check_output.
528 """
529 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
530 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
531 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
532
533def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
534 """
535 sudo (or similar) + subprocess.check_output, except '-i' isn't used.
536 """
537 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
538 _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
539 return processOutputChecked(*aPositionalArgs, **dKeywordArgs);
540
541def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
542 """
543 sudo (or similar) + subprocess.Popen.
544 """
545 _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
546 _sudoFixArguments(aPositionalArgs, dKeywordArgs);
547 return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);
548
549
550#
551# Generic process stuff.
552#
553
554def processInterrupt(uPid):
555 """
556 Sends a SIGINT or equivalent to interrupt the specified process.
557 Returns True on success, False on failure.
558
559 On Windows hosts this may not work unless the process happens to be a
560 process group leader.
561 """
562 if sys.platform == 'win32':
563 try:
564 win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, uPid); # pylint: disable=no-member
565 fRc = True;
566 except:
567 fRc = False;
568 else:
569 try:
570 os.kill(uPid, signal.SIGINT);
571 fRc = True;
572 except:
573 fRc = False;
574 return fRc;
575
576def sendUserSignal1(uPid):
577 """
578 Sends a SIGUSR1 or equivalent to nudge the process into shutting down
579 (VBoxSVC) or something.
580 Returns True on success, False on failure or if not supported (win).
581
582 On Windows hosts this may not work unless the process happens to be a
583 process group leader.
584 """
585 if sys.platform == 'win32':
586 fRc = False;
587 else:
588 try:
589 os.kill(uPid, signal.SIGUSR1); # pylint: disable=E1101
590 fRc = True;
591 except:
592 fRc = False;
593 return fRc;
594
595def processTerminate(uPid):
596 """
597 Terminates the process in a nice manner (SIGTERM or equivalent).
598 Returns True on success, False on failure.
599 """
600 fRc = False;
601 if sys.platform == 'win32':
602 try:
603 hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, False, uPid); # pylint: disable=no-member
604 except:
605 pass;
606 else:
607 try:
608 win32process.TerminateProcess(hProcess, 0x40010004); # DBG_TERMINATE_PROCESS # pylint: disable=no-member
609 fRc = True;
610 except:
611 pass;
612 win32api.CloseHandle(hProcess) # pylint: disable=no-member
613 else:
614 try:
615 os.kill(uPid, signal.SIGTERM);
616 fRc = True;
617 except:
618 pass;
619 return fRc;
620
621def processKill(uPid):
622 """
623 Terminates the process with extreme prejudice (SIGKILL).
624 Returns True on success, False on failure.
625 """
626 if sys.platform == 'win32':
627 fRc = processTerminate(uPid);
628 else:
629 try:
630 os.kill(uPid, signal.SIGKILL); # pylint: disable=E1101
631 fRc = True;
632 except:
633 fRc = False;
634 return fRc;
635
636def processKillWithNameCheck(uPid, sName):
637 """
638 Like processKill(), but checks if the process name matches before killing
639 it. This is intended for killing using potentially stale pid values.
640
641 Returns True on success, False on failure.
642 """
643
644 if processCheckPidAndName(uPid, sName) is not True:
645 return False;
646 return processKill(uPid);
647
648
649def processExists(uPid):
650 """
651 Checks if the specified process exits.
652 This will only work if we can signal/open the process.
653
654 Returns True if it positively exists, False otherwise.
655 """
656 if sys.platform == 'win32':
657 fRc = False;
658 try:
659 hProcess = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, False, uPid); # pylint: disable=no-member
660 except:
661 pass;
662 else:
663 win32api.CloseHandle(hProcess); # pylint: disable=no-member
664 fRc = True;
665 else:
666 try:
667 os.kill(uPid, 0);
668 fRc = True;
669 except:
670 fRc = False;
671 return fRc;
672
673def processCheckPidAndName(uPid, sName):
674 """
675 Checks if a process PID and NAME matches.
676 """
677 fRc = processExists(uPid);
678 if fRc is not True:
679 return False;
680
681 if sys.platform == 'win32':
682 try:
683 from win32com.client import GetObject; # pylint: disable=F0401
684 oWmi = GetObject('winmgmts:');
685 aoProcesses = oWmi.InstancesOf('Win32_Process');
686 for oProcess in aoProcesses:
687 if long(oProcess.Properties_("ProcessId").Value) == uPid:
688 sCurName = oProcess.Properties_("Name").Value;
689 #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
690 sName = sName.lower();
691 sCurName = sCurName.lower();
692 if os.path.basename(sName) == sName:
693 sCurName = os.path.basename(sCurName);
694
695 if sCurName == sName \
696 or sCurName + '.exe' == sName \
697 or sCurName == sName + '.exe':
698 fRc = True;
699 break;
700 except:
701 #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
702 pass;
703 else:
704 if sys.platform in ('linux2', ):
705 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
706 elif sys.platform in ('sunos5',):
707 asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
708 elif sys.platform in ('darwin',):
709 asPsCmd = ['/bin/ps', '-p', '%u' % (uPid,), '-o', 'ucomm='];
710 else:
711 asPsCmd = None;
712
713 if asPsCmd is not None:
714 try:
715 oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
716 sCurName = oPs.communicate()[0];
717 iExitCode = oPs.wait();
718 except:
719 #reporter.logXcpt();
720 return False;
721
722 # ps fails with non-zero exit code if the pid wasn't found.
723 if iExitCode is not 0:
724 return False;
725 if sCurName is None:
726 return False;
727 sCurName = sCurName.strip();
728 if sCurName is '':
729 return False;
730
731 if os.path.basename(sName) == sName:
732 sCurName = os.path.basename(sCurName);
733 elif os.path.basename(sCurName) == sCurName:
734 sName = os.path.basename(sName);
735
736 if sCurName != sName:
737 return False;
738
739 fRc = True;
740 return fRc;
741
742
743class ProcessInfo(object):
744 """Process info."""
745 def __init__(self, iPid):
746 self.iPid = iPid;
747 self.iParentPid = None;
748 self.sImage = None;
749 self.sName = None;
750 self.asArgs = None;
751 self.sCwd = None;
752 self.iGid = None;
753 self.iUid = None;
754 self.iProcGroup = None;
755 self.iSessionId = None;
756
757 def loadAll(self):
758 """Load all the info."""
759 sOs = getHostOs();
760 if sOs == 'linux':
761 sProc = '/proc/%s/' % (self.iPid,);
762 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
763 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'cwd', None);
764 if self.asArgs is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
765 elif sOs == 'solaris':
766 sProc = '/proc/%s/' % (self.iPid,);
767 if self.sImage is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
768 if self.sCwd is None: self.sCwd = noxcptReadLink(sProc + 'path/cwd', None);
769 else:
770 pass;
771 if self.sName is None and self.sImage is not None:
772 self.sName = self.sImage;
773
774 def windowsGrabProcessInfo(self, oProcess):
775 """Windows specific loadAll."""
776 try: self.sName = oProcess.Properties_("Name").Value;
777 except: pass;
778 try: self.sImage = oProcess.Properties_("ExecutablePath").Value;
779 except: pass;
780 try: self.asArgs = oProcess.Properties_("CommandLine").Value; ## @todo split it.
781 except: pass;
782 try: self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
783 except: pass;
784 try: self.iSessionId = oProcess.Properties_("SessionId").Value;
785 except: pass;
786 if self.sName is None and self.sImage is not None:
787 self.sName = self.sImage;
788
789 def getBaseImageName(self):
790 """
791 Gets the base image name if available, use the process name if not available.
792 Returns image/process base name or None.
793 """
794 sRet = self.sImage if self.sName is None else self.sName;
795 if sRet is None:
796 self.loadAll();
797 sRet = self.sImage if self.sName is None else self.sName;
798 if sRet is None:
799 if self.asArgs is None or len(self.asArgs) == 0:
800 return None;
801 sRet = self.asArgs[0];
802 if len(sRet) == 0:
803 return None;
804 return os.path.basename(sRet);
805
806 def getBaseImageNameNoExeSuff(self):
807 """
808 Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
809 """
810 sRet = self.getBaseImageName();
811 if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
812 if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
813 sRet = sRet[:-4];
814 return sRet;
815
816
817def processListAll(): # pylint: disable=R0914
818 """
819 Return a list of ProcessInfo objects for all the processes in the system
820 that the current user can see.
821 """
822 asProcesses = [];
823
824 sOs = getHostOs();
825 if sOs == 'win':
826 from win32com.client import GetObject; # pylint: disable=F0401
827 oWmi = GetObject('winmgmts:');
828 aoProcesses = oWmi.InstancesOf('Win32_Process');
829 for oProcess in aoProcesses:
830 try:
831 iPid = int(oProcess.Properties_("ProcessId").Value);
832 except:
833 continue;
834 oMyInfo = ProcessInfo(iPid);
835 oMyInfo.windowsGrabProcessInfo(oProcess);
836 asProcesses.append(oMyInfo);
837
838 elif sOs in [ 'linux', 'solaris' ]:
839 try:
840 asDirs = os.listdir('/proc');
841 except:
842 asDirs = [];
843 for sDir in asDirs:
844 if sDir.isdigit():
845 asProcesses.append(ProcessInfo(int(sDir),));
846
847 elif sOs == 'darwin':
848 # Try our best to parse ps output. (Not perfect but does the job most of the time.)
849 try:
850 sRaw = processOutputChecked([ '/bin/ps', '-A',
851 '-o', 'pid=',
852 '-o', 'ppid=',
853 '-o', 'pgid=',
854 '-o', 'sess=',
855 '-o', 'uid=',
856 '-o', 'gid=',
857 '-o', 'comm=' ]);
858 except:
859 return asProcesses;
860
861 for sLine in sRaw.split('\n'):
862 sLine = sLine.lstrip();
863 if len(sLine) < 7 or not sLine[0].isdigit():
864 continue;
865
866 iField = 0;
867 off = 0;
868 aoFields = [None, None, None, None, None, None, None];
869 while iField < 7:
870 # Eat whitespace.
871 while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
872 off += 1;
873
874 # Final field / EOL.
875 if iField == 6:
876 aoFields[6] = sLine[off:];
877 break;
878 if off >= len(sLine):
879 break;
880
881 # Generic field parsing.
882 offStart = off;
883 off += 1;
884 while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
885 off += 1;
886 try:
887 if iField != 3:
888 aoFields[iField] = int(sLine[offStart:off]);
889 else:
890 aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
891 except:
892 pass;
893 iField += 1;
894
895 if aoFields[0] is not None:
896 oMyInfo = ProcessInfo(aoFields[0]);
897 oMyInfo.iParentPid = aoFields[1];
898 oMyInfo.iProcGroup = aoFields[2];
899 oMyInfo.iSessionId = aoFields[3];
900 oMyInfo.iUid = aoFields[4];
901 oMyInfo.iGid = aoFields[5];
902 oMyInfo.sName = aoFields[6];
903 asProcesses.append(oMyInfo);
904
905 return asProcesses;
906
907
908def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
909 """
910 Looks for information regarding the demise of the given process.
911 """
912 sOs = getHostOs();
913 if sOs == 'darwin':
914 #
915 # On darwin we look for crash and diagnostic reports.
916 #
917 asLogDirs = [
918 u'/Library/Logs/DiagnosticReports/',
919 u'/Library/Logs/CrashReporter/',
920 u'~/Library/Logs/DiagnosticReports/',
921 u'~/Library/Logs/CrashReporter/',
922 ];
923 for sDir in asLogDirs:
924 sDir = os.path.expanduser(sDir);
925 if not os.path.isdir(sDir):
926 continue;
927 try:
928 asDirEntries = os.listdir(sDir);
929 except:
930 continue;
931 for sEntry in asDirEntries:
932 # Only interested in .crash files.
933 _, sSuff = os.path.splitext(sEntry);
934 if sSuff != '.crash':
935 continue;
936
937 # The pid can be found at the end of the first line.
938 sFull = os.path.join(sDir, sEntry);
939 try:
940 oFile = open(sFull, 'r');
941 sFirstLine = oFile.readline();
942 oFile.close();
943 except:
944 continue;
945 if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
946 continue;
947 offPid = len(sFirstLine) - 3;
948 while offPid > 1 and sFirstLine[offPid - 1].isdigit():
949 offPid -= 1;
950 try: uReportPid = int(sFirstLine[offPid:-2]);
951 except: continue;
952
953 # Does the pid we found match?
954 if uReportPid == uPid:
955 fnLog('Found crash report for %u: %s' % (uPid, sFull,));
956 fnCrashFile(sFull, False);
957 elif sOs == 'win':
958 #
959 # Getting WER reports would be great, however we have trouble match the
960 # PID to those as they seems not to mention it in the brief reports.
961 # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
962 # location - see the windows readme for the testbox script) and what
963 # the MSDN article lists for now.
964 #
965 # It's been observed on Windows server 2012 that the dump files takes
966 # the form: <processimage>.<decimal-pid>.dmp
967 #
968 asDmpDirs = [
969 u'%SystemDrive%/CrashDumps/', # Testboxes.
970 u'%LOCALAPPDATA%/CrashDumps/', # MSDN example.
971 u'%WINDIR%/ServiceProfiles/LocalServices/', # Local and network service.
972 u'%WINDIR%/ServiceProfiles/NetworkSerices/',
973 u'%WINDIR%/ServiceProfiles/',
974 u'%WINDIR%/System32/Config/SystemProfile/', # System services.
975 ];
976 sMatchSuffix = '.%u.dmp' % (uPid,);
977
978 for sDir in asDmpDirs:
979 sDir = os.path.expandvars(sDir);
980 if not os.path.isdir(sDir):
981 continue;
982 try:
983 asDirEntries = os.listdir(sDir);
984 except:
985 continue;
986 for sEntry in asDirEntries:
987 if sEntry.endswith(sMatchSuffix):
988 sFull = os.path.join(sDir, sEntry);
989 fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
990 fnCrashFile(sFull, True);
991
992 else:
993 pass; ## TODO
994 return None;
995
996
997#
998# Time.
999#
1000
1001#
1002# The following test case shows how time.time() only have ~ms resolution
1003# on Windows (tested W10) and why it therefore makes sense to try use
1004# performance counters.
1005#
1006# Note! We cannot use time.clock() as the timestamp must be portable across
1007# processes. See timeout testcase problem on win hosts (no logs).
1008#
1009#import sys;
1010#import time;
1011#from common import utils;
1012#
1013#atSeries = [];
1014#for i in xrange(1,160):
1015# if i == 159: time.sleep(10);
1016# atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
1017#
1018#tPrev = atSeries[0]
1019#for tCur in atSeries:
1020# print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
1021# print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
1022# print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
1023# print '';
1024# tPrev = tCur
1025#
1026#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
1027#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
1028#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);
1029
1030g_fWinUseWinPerfCounter = sys.platform == 'win32';
1031g_fpWinPerfCounterFreq = None;
1032g_oFuncwinQueryPerformanceCounter = None;
1033
1034def _winInitPerfCounter():
1035 """ Initializes the use of performance counters. """
1036 global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter
1037
1038 uFrequency = ctypes.c_ulonglong(0);
1039 if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
1040 if uFrequency.value >= 1000:
1041 #print 'uFrequency = %s' % (uFrequency,);
1042 #print 'type(uFrequency) = %s' % (type(uFrequency),);
1043 g_fpWinPerfCounterFreq = float(uFrequency.value);
1044
1045 # Check that querying the counter works too.
1046 global g_oFuncwinQueryPerformanceCounter
1047 g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
1048 uCurValue = ctypes.c_ulonglong(0);
1049 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1050 if uCurValue.value > 0:
1051 return True;
1052 g_fWinUseWinPerfCounter = False;
1053 return False;
1054
1055def _winFloatTime():
1056 """ Gets floating point time on windows. """
1057 if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
1058 uCurValue = ctypes.c_ulonglong(0);
1059 if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
1060 return float(uCurValue.value) / g_fpWinPerfCounterFreq;
1061 return time.time();
1062
1063
1064def timestampNano():
1065 """
1066 Gets a nanosecond timestamp.
1067 """
1068 if g_fWinUseWinPerfCounter is True:
1069 return long(_winFloatTime() * 1000000000);
1070 return long(time.time() * 1000000000);
1071
1072def timestampMilli():
1073 """
1074 Gets a millisecond timestamp.
1075 """
1076 if g_fWinUseWinPerfCounter is True:
1077 return long(_winFloatTime() * 1000);
1078 return long(time.time() * 1000);
1079
1080def timestampSecond():
1081 """
1082 Gets a second timestamp.
1083 """
1084 if g_fWinUseWinPerfCounter is True:
1085 return long(_winFloatTime());
1086 return long(time.time());
1087
1088def getTimePrefix():
1089 """
1090 Returns a timestamp prefix, typically used for logging. UTC.
1091 """
1092 try:
1093 oNow = datetime.datetime.utcnow();
1094 sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1095 except:
1096 sTs = 'getTimePrefix-exception';
1097 return sTs;
1098
1099def getTimePrefixAndIsoTimestamp():
1100 """
1101 Returns current UTC as log prefix and iso timestamp.
1102 """
1103 try:
1104 oNow = datetime.datetime.utcnow();
1105 sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
1106 sTsIso = formatIsoTimestamp(oNow);
1107 except:
1108 sTsPrf = sTsIso = 'getTimePrefix-exception';
1109 return (sTsPrf, sTsIso);
1110
1111def formatIsoTimestamp(oNow):
1112 """Formats the datetime object as an ISO timestamp."""
1113 assert oNow.tzinfo is None;
1114 sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
1115 return sTs;
1116
1117def getIsoTimestamp():
1118 """Returns the current UTC timestamp as a string."""
1119 return formatIsoTimestamp(datetime.datetime.utcnow());
1120
1121
1122def getLocalHourOfWeek():
1123 """ Local hour of week (0 based). """
1124 oNow = datetime.datetime.now();
1125 return (oNow.isoweekday() - 1) * 24 + oNow.hour;
1126
1127
1128def formatIntervalSeconds(cSeconds):
1129 """ Format a seconds interval into a nice 01h 00m 22s string """
1130 # Two simple special cases.
1131 if cSeconds < 60:
1132 return '%ss' % (cSeconds,);
1133 if cSeconds < 3600:
1134 cMins = cSeconds / 60;
1135 cSecs = cSeconds % 60;
1136 if cSecs == 0:
1137 return '%sm' % (cMins,);
1138 return '%sm %ss' % (cMins, cSecs,);
1139
1140 # Generic and a bit slower.
1141 cDays = cSeconds / 86400;
1142 cSeconds %= 86400;
1143 cHours = cSeconds / 3600;
1144 cSeconds %= 3600;
1145 cMins = cSeconds / 60;
1146 cSecs = cSeconds % 60;
1147 sRet = '';
1148 if cDays > 0:
1149 sRet = '%sd ' % (cDays,);
1150 if cHours > 0:
1151 sRet += '%sh ' % (cHours,);
1152 if cMins > 0:
1153 sRet += '%sm ' % (cMins,);
1154 if cSecs > 0:
1155 sRet += '%ss ' % (cSecs,);
1156 assert len(sRet) > 0; assert sRet[-1] == ' ';
1157 return sRet[:-1];
1158
1159def formatIntervalSeconds2(oSeconds):
1160 """
1161 Flexible input version of formatIntervalSeconds for use in WUI forms where
1162 data is usually already string form.
1163 """
1164 if isinstance(oSeconds, int) or isinstance(oSeconds, long):
1165 return formatIntervalSeconds(oSeconds);
1166 if not isString(oSeconds):
1167 try:
1168 lSeconds = long(oSeconds);
1169 except:
1170 pass;
1171 else:
1172 if lSeconds >= 0:
1173 return formatIntervalSeconds2(lSeconds);
1174 return oSeconds;
1175
1176def parseIntervalSeconds(sString):
1177 """
1178 Reverse of formatIntervalSeconds.
1179
1180 Returns (cSeconds, sError), where sError is None on success.
1181 """
1182
1183 # We might given non-strings, just return them without any fuss.
1184 if not isString(sString):
1185 if isinstance(sString, int) or isinstance(sString, long) or sString is None:
1186 return (sString, None);
1187 ## @todo time/date objects?
1188 return (int(sString), None);
1189
1190 # Strip it and make sure it's not empty.
1191 sString = sString.strip();
1192 if len(sString) == 0:
1193 return (0, 'Empty interval string.');
1194
1195 #
1196 # Split up the input into a list of 'valueN, unitN, ...'.
1197 #
1198 # Don't want to spend too much time trying to make re.split do exactly what
1199 # I need here, so please forgive the extra pass I'm making here.
1200 #
1201 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1202 asParts = [];
1203 for sPart in asRawParts:
1204 sPart = sPart.strip();
1205 if len(sPart) > 0:
1206 asParts.append(sPart);
1207 if len(asParts) == 0:
1208 return (0, 'Empty interval string or something?');
1209
1210 #
1211 # Process them one or two at the time.
1212 #
1213 cSeconds = 0;
1214 asErrors = [];
1215 i = 0;
1216 while i < len(asParts):
1217 sNumber = asParts[i];
1218 i += 1;
1219 if sNumber.isdigit():
1220 iNumber = int(sNumber);
1221
1222 sUnit = 's';
1223 if i < len(asParts) and not asParts[i].isdigit():
1224 sUnit = asParts[i];
1225 i += 1;
1226
1227 sUnitLower = sUnit.lower();
1228 if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
1229 pass;
1230 elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
1231 iNumber *= 60;
1232 elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1233 iNumber *= 3600;
1234 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1235 iNumber *= 86400;
1236 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1237 iNumber *= 7 * 86400;
1238 else:
1239 asErrors.append('Unknown unit "%s".' % (sUnit,));
1240 cSeconds += iNumber;
1241 else:
1242 asErrors.append('Bad number "%s".' % (sNumber,));
1243 return (cSeconds, None if len(asErrors) == 0 else ' '.join(asErrors));
1244
1245def formatIntervalHours(cHours):
1246 """ Format a hours interval into a nice 1w 2d 1h string. """
1247 # Simple special cases.
1248 if cHours < 24:
1249 return '%sh' % (cHours,);
1250
1251 # Generic and a bit slower.
1252 cWeeks = cHours / (7 * 24);
1253 cHours %= 7 * 24;
1254 cDays = cHours / 24;
1255 cHours %= 24;
1256 sRet = '';
1257 if cWeeks > 0:
1258 sRet = '%sw ' % (cWeeks,);
1259 if cDays > 0:
1260 sRet = '%sd ' % (cDays,);
1261 if cHours > 0:
1262 sRet += '%sh ' % (cHours,);
1263 assert len(sRet) > 0; assert sRet[-1] == ' ';
1264 return sRet[:-1];
1265
1266def parseIntervalHours(sString):
1267 """
1268 Reverse of formatIntervalHours.
1269
1270 Returns (cHours, sError), where sError is None on success.
1271 """
1272
1273 # We might given non-strings, just return them without any fuss.
1274 if not isString(sString):
1275 if isinstance(sString, int) or isinstance(sString, long) or sString is None:
1276 return (sString, None);
1277 ## @todo time/date objects?
1278 return (int(sString), None);
1279
1280 # Strip it and make sure it's not empty.
1281 sString = sString.strip();
1282 if len(sString) == 0:
1283 return (0, 'Empty interval string.');
1284
1285 #
1286 # Split up the input into a list of 'valueN, unitN, ...'.
1287 #
1288 # Don't want to spend too much time trying to make re.split do exactly what
1289 # I need here, so please forgive the extra pass I'm making here.
1290 #
1291 asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
1292 asParts = [];
1293 for sPart in asRawParts:
1294 sPart = sPart.strip();
1295 if len(sPart) > 0:
1296 asParts.append(sPart);
1297 if len(asParts) == 0:
1298 return (0, 'Empty interval string or something?');
1299
1300 #
1301 # Process them one or two at the time.
1302 #
1303 cHours = 0;
1304 asErrors = [];
1305 i = 0;
1306 while i < len(asParts):
1307 sNumber = asParts[i];
1308 i += 1;
1309 if sNumber.isdigit():
1310 iNumber = int(sNumber);
1311
1312 sUnit = 'h';
1313 if i < len(asParts) and not asParts[i].isdigit():
1314 sUnit = asParts[i];
1315 i += 1;
1316
1317 sUnitLower = sUnit.lower();
1318 if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
1319 pass;
1320 elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
1321 iNumber *= 24;
1322 elif sUnitLower in [ 'w', 'week', 'weeks' ]:
1323 iNumber *= 7 * 24;
1324 else:
1325 asErrors.append('Unknown unit "%s".' % (sUnit,));
1326 cHours += iNumber;
1327 else:
1328 asErrors.append('Bad number "%s".' % (sNumber,));
1329 return (cHours, None if len(asErrors) == 0 else ' '.join(asErrors));
1330
1331
1332#
1333# Introspection.
1334#
1335
1336def getCallerName(oFrame=None, iFrame=2):
1337 """
1338 Returns the name of the caller's caller.
1339 """
1340 if oFrame is None:
1341 try:
1342 raise Exception();
1343 except:
1344 oFrame = sys.exc_info()[2].tb_frame.f_back;
1345 while iFrame > 1:
1346 if oFrame is not None:
1347 oFrame = oFrame.f_back;
1348 iFrame = iFrame - 1;
1349 if oFrame is not None:
1350 sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
1351 return sName;
1352 return "unknown";
1353
1354
1355def getXcptInfo(cFrames = 1):
1356 """
1357 Gets text detailing the exception. (Good for logging.)
1358 Returns list of info strings.
1359 """
1360
1361 #
1362 # Try get exception info.
1363 #
1364 try:
1365 oType, oValue, oTraceback = sys.exc_info();
1366 except:
1367 oType = oValue = oTraceback = None;
1368 if oType is not None:
1369
1370 #
1371 # Try format the info
1372 #
1373 asRet = [];
1374 try:
1375 try:
1376 asRet = asRet + traceback.format_exception_only(oType, oValue);
1377 asTraceBack = traceback.format_tb(oTraceback);
1378 if cFrames is not None and cFrames <= 1:
1379 asRet.append(asTraceBack[-1]);
1380 else:
1381 asRet.append('Traceback:')
1382 for iFrame in range(min(cFrames, len(asTraceBack))):
1383 asRet.append(asTraceBack[-iFrame - 1]);
1384 asRet.append('Stack:')
1385 asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
1386 except:
1387 asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));
1388
1389 if len(asRet) == 0:
1390 asRet.append('No exception info...');
1391 except:
1392 asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
1393 else:
1394 asRet = ['Couldn\'t find exception traceback.'];
1395 return asRet;
1396
1397
1398#
1399# TestSuite stuff.
1400#
1401
1402def isRunningFromCheckout(cScriptDepth = 1):
1403 """
1404 Checks if we're running from the SVN checkout or not.
1405 """
1406
1407 try:
1408 sFile = __file__;
1409 cScriptDepth = 1;
1410 except:
1411 sFile = sys.argv[0];
1412
1413 sDir = os.path.abspath(sFile);
1414 while cScriptDepth >= 0:
1415 sDir = os.path.dirname(sDir);
1416 if os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
1417 or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
1418 return True;
1419 cScriptDepth -= 1;
1420
1421 return False;
1422
1423
1424#
1425# Bourne shell argument fun.
1426#
1427
1428
1429def argsSplit(sCmdLine):
1430 """
1431 Given a bourne shell command line invocation, split it up into arguments
1432 assuming IFS is space.
1433 Returns None on syntax error.
1434 """
1435 ## @todo bourne shell argument parsing!
1436 return sCmdLine.split(' ');
1437
1438def argsGetFirst(sCmdLine):
1439 """
1440 Given a bourne shell command line invocation, get return the first argument
1441 assuming IFS is space.
1442 Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
1443 """
1444 asArgs = argsSplit(sCmdLine);
1445 if asArgs is None or len(asArgs) == 0:
1446 return None;
1447
1448 return asArgs[0];
1449
1450#
1451# String helpers.
1452#
1453
1454def stricmp(sFirst, sSecond):
1455 """
1456 Compares to strings in an case insensitive fashion.
1457
1458 Python doesn't seem to have any way of doing the correctly, so this is just
1459 an approximation using lower.
1460 """
1461 if sFirst == sSecond:
1462 return 0;
1463 sLower1 = sFirst.lower();
1464 sLower2 = sSecond.lower();
1465 if sLower1 == sLower2:
1466 return 0;
1467 if sLower1 < sLower2:
1468 return -1;
1469 return 1;
1470
1471
1472#
1473# Misc.
1474#
1475
1476def versionCompare(sVer1, sVer2):
1477 """
1478 Compares to version strings in a fashion similar to RTStrVersionCompare.
1479 """
1480
1481 ## @todo implement me!!
1482
1483 if sVer1 == sVer2:
1484 return 0;
1485 if sVer1 < sVer2:
1486 return -1;
1487 return 1;
1488
1489
1490def formatNumber(lNum, sThousandSep = ' '):
1491 """
1492 Formats a decimal number with pretty separators.
1493 """
1494 sNum = str(lNum);
1495 sRet = sNum[-3:];
1496 off = len(sNum) - 3;
1497 while off > 0:
1498 off -= 3;
1499 sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
1500 return sRet;
1501
1502
1503def formatNumberNbsp(lNum):
1504 """
1505 Formats a decimal number with pretty separators.
1506 """
1507 sRet = formatNumber(lNum);
1508 return unicode(sRet).replace(' ', u'\u00a0');
1509
1510
1511def isString(oString):
1512 """
1513 Checks if the object is a string object, hiding difference between python 2 and 3.
1514
1515 Returns True if it's a string of some kind.
1516 Returns False if not.
1517 """
1518 if sys.version_info[0] >= 3:
1519 return isinstance(oString, str);
1520 return isinstance(oString, basestring);
1521
1522
1523def hasNonAsciiCharacters(sText):
1524 """
1525 Returns True is specified string has non-ASCII characters, False if ASCII only.
1526 """
1527 sTmp = unicode(sText, errors='ignore') if isinstance(sText, str) else sText;
1528 return not all(ord(ch) < 128 for ch in sTmp);
1529
1530
1531def chmodPlusX(sFile):
1532 """
1533 Makes the specified file or directory executable.
1534 Returns success indicator, no exceptions.
1535
1536 Note! Symbolic links are followed and the target will be changed.
1537 """
1538 try:
1539 oStat = os.stat(sFile);
1540 except:
1541 return False;
1542 try:
1543 os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
1544 except:
1545 return False;
1546 return True;
1547
1548
1549def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1550 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1551 """
1552 Worker for unpackFile that deals with ZIP files, same function signature.
1553 """
1554 import zipfile
1555 if fnError is None:
1556 fnError = fnLog;
1557
1558 fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));
1559
1560 # Open it.
1561 try: oZipFile = zipfile.ZipFile(sArchive, 'r')
1562 except Exception as oXcpt:
1563 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1564 return None;
1565
1566 # Extract all members.
1567 asMembers = [];
1568 try:
1569 for sMember in oZipFile.namelist():
1570 if fnFilter is None or fnFilter(sMember) is not False:
1571 if sMember.endswith('/'):
1572 os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
1573 else:
1574 oZipFile.extract(sMember, sDstDir);
1575 asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
1576 except Exception as oXcpt:
1577 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1578 asMembers = None;
1579
1580 # close it.
1581 try: oZipFile.close();
1582 except Exception as oXcpt:
1583 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1584 asMembers = None;
1585
1586 return asMembers;
1587
1588
1589## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
1590g_fTarCopyFileObjOverriddend = False;
1591
1592def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError):
1593 """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
1594 if length is None:
1595 __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
1596 elif length > 0:
1597 cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
1598 for _ in xrange(cFull):
1599 abBuffer = src.read(g_cbGoodBufferSize);
1600 dst.write(abBuffer);
1601 if len(abBuffer) != g_cbGoodBufferSize:
1602 raise exception('unexpected end of source file');
1603 if cbRemainder > 0:
1604 abBuffer = src.read(cbRemainder);
1605 dst.write(abBuffer);
1606 if len(abBuffer) != cbRemainder:
1607 raise exception('unexpected end of source file');
1608
1609
1610def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1611 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1612 """
1613 Worker for unpackFile that deals with tarballs, same function signature.
1614 """
1615 import shutil;
1616 import tarfile;
1617 if fnError is None:
1618 fnError = fnLog;
1619
1620 fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));
1621
1622 #
1623 # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
1624 # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
1625 # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
1626 #
1627 if True is True:
1628 __installShUtilHacks(shutil);
1629 global g_fTarCopyFileObjOverriddend;
1630 if g_fTarCopyFileObjOverriddend is False:
1631 g_fTarCopyFileObjOverriddend = True;
1632 tarfile.copyfileobj = __mytarfilecopyfileobj;
1633
1634 #
1635 # Open it.
1636 #
1637 # Note! We not using 'r:*' because we cannot allow seeking compressed files!
1638 # That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
1639 #
1640 try: oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
1641 except Exception as oXcpt:
1642 fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
1643 return None;
1644
1645 # Extract all members.
1646 asMembers = [];
1647 try:
1648 for oTarInfo in oTarFile:
1649 try:
1650 if fnFilter is None or fnFilter(oTarInfo.name) is not False:
1651 if oTarInfo.islnk():
1652 # Links are trouble, especially on Windows. We must avoid the falling that will end up seeking
1653 # in the compressed tar stream. So, fall back on shutil.copy2 instead.
1654 sLinkFile = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
1655 sLinkTarget = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
1656 sParentDir = os.path.dirname(sLinkFile);
1657 try: os.unlink(sLinkFile);
1658 except: pass;
1659 if sParentDir is not '' and not os.path.exists(sParentDir):
1660 os.makedirs(sParentDir);
1661 try: os.link(sLinkTarget, sLinkFile);
1662 except: shutil.copy2(sLinkTarget, sLinkFile);
1663 else:
1664 if oTarInfo.isdir():
1665 # Just make sure the user (we) got full access to dirs. Don't bother getting it 100% right.
1666 oTarInfo.mode |= 0x1c0; # (octal: 0700)
1667 oTarFile.extract(oTarInfo, sDstDir);
1668 asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
1669 except Exception as oXcpt:
1670 fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
1671 for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
1672 fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
1673 for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
1674 fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
1675 asMembers = None;
1676 break;
1677 except Exception as oXcpt:
1678 fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
1679 asMembers = None;
1680
1681 #
1682 # Finally, close it.
1683 #
1684 try: oTarFile.close();
1685 except Exception as oXcpt:
1686 fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
1687 asMembers = None;
1688
1689 return asMembers;
1690
1691
1692def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
1693 # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
1694 """
1695 Unpacks the given file if it has a know archive extension, otherwise do
1696 nothing.
1697
1698 fnLog & fnError both take a string parameter.
1699
1700 fnFilter takes a member name (string) and returns True if it's included
1701 and False if excluded.
1702
1703 Returns list of the extracted files (full path) on success.
1704 Returns empty list if not a supported archive format.
1705 Returns None on failure. Raises no exceptions.
1706 """
1707 sBaseNameLower = os.path.basename(sArchive).lower();
1708
1709 #
1710 # Zip file?
1711 #
1712 if sBaseNameLower.endswith('.zip'):
1713 return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
1714
1715 #
1716 # Tarball?
1717 #
1718 if sBaseNameLower.endswith('.tar') \
1719 or sBaseNameLower.endswith('.tar.gz') \
1720 or sBaseNameLower.endswith('.tgz') \
1721 or sBaseNameLower.endswith('.tar.bz2'):
1722 return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);
1723
1724 #
1725 # Cannot classify it from the name, so just return that to the caller.
1726 #
1727 fnLog('Not unpacking "%s".' % (sArchive,));
1728 return [];
1729
1730
1731def getDiskUsage(sPath):
1732 """
1733 Get free space of a partition that corresponds to specified sPath in MB.
1734
1735 Returns partition free space value in MB.
1736 """
1737 if platform.system() == 'Windows':
1738 oCTypeFreeSpace = ctypes.c_ulonglong(0);
1739 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
1740 ctypes.pointer(oCTypeFreeSpace));
1741 cbFreeSpace = oCTypeFreeSpace.value;
1742 else:
1743 oStats = os.statvfs(sPath); # pylint: disable=E1101
1744 cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;
1745
1746 # Convert to MB
1747 cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);
1748
1749 return cMbFreeSpace;
1750
1751
1752#
1753# Unit testing.
1754#
1755
1756# pylint: disable=C0111
1757class BuildCategoryDataTestCase(unittest.TestCase):
1758 def testIntervalSeconds(self):
1759 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
1760 self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
1761 self.assertEqual(parseIntervalSeconds('123'), (123, None));
1762 self.assertEqual(parseIntervalSeconds(123), (123, None));
1763 self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
1764 self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
1765 self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
1766 self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
1767 self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
1768 self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
1769 self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));
1770
1771 def testHasNonAsciiChars(self):
1772 self.assertEqual(hasNonAsciiCharacters(''), False);
1773 self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
1774 self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
1775 self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
1776 self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
1777 self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);
1778
1779if __name__ == '__main__':
1780 unittest.main();
1781 # not reached.
1782
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette