source: vbox/trunk/src/VBox/ValidationKit/common/utils.py@ 65754

# -*- coding: utf-8 -*-
# $Id: utils.py 65375 2017-01-19 15:00:19Z vboxsync $
# pylint: disable=C0302

"""
Common Utility Functions.
"""

__copyright__ = \
"""
Copyright (C) 2012-2016 Oracle Corporation

This file is part of VirtualBox Open Source Edition (OSE), as
available from http://www.virtualbox.org. This file is free software;
you can redistribute it and/or modify it under the terms of the GNU
General Public License (GPL) as published by the Free Software
Foundation, in version 2 as it comes in the "COPYING" file of the
VirtualBox OSE distribution. VirtualBox OSE is distributed in the
hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.

The contents of this file may alternatively be used under the terms
of the Common Development and Distribution License Version 1.0
(CDDL) only, as it comes in the "COPYING.CDDL" file of the
VirtualBox OSE distribution, in which case the provisions of the
CDDL are applicable instead of those of the GPL.

You may elect to license modified versions of this file under the
terms and conditions of either the GPL or the CDDL or both.
"""
__version__ = "$Revision: 65375 $"


# Standard Python imports.
import datetime;
import os;
import platform;
import re;
import stat;
import subprocess;
import sys;
import time;
import traceback;
import unittest;

if sys.platform == 'win32':
    import ctypes;
    import win32api;            # pylint: disable=F0401
    import win32con;            # pylint: disable=F0401
    import win32console;        # pylint: disable=F0401
    import win32process;        # pylint: disable=F0401
else:
    import signal;

# Python 3 hacks:
if sys.version_info[0] >= 3:
    unicode = str;  # pylint: disable=redefined-builtin,invalid-name
    xrange = range; # pylint: disable=redefined-builtin,invalid-name
    long = int;     # pylint: disable=redefined-builtin,invalid-name


#
# Host OS and CPU.
#

def getHostOs():
    """
    Gets the host OS name (short).

    See the KBUILD_OSES variable in kBuild/header.kmk for possible return values.
    """
    sPlatform = platform.system();
    if sPlatform in ('Linux', 'Darwin', 'Solaris', 'FreeBSD', 'NetBSD', 'OpenBSD'):
        sPlatform = sPlatform.lower();
    elif sPlatform == 'Windows':
        sPlatform = 'win';
    elif sPlatform == 'SunOS':
        sPlatform = 'solaris';
    else:
        raise Exception('Unsupported platform "%s"' % (sPlatform,));
    return sPlatform;

g_sHostArch = None;

def getHostArch():
    """
    Gets the host CPU architecture.

    See the KBUILD_ARCHES variable in kBuild/header.kmk for possible return values.
    """
    global g_sHostArch;
    if g_sHostArch is None:
        sArch = platform.machine();
        if sArch in ('i386', 'i486', 'i586', 'i686', 'i786', 'i886', 'x86'):
            sArch = 'x86';
        elif sArch in ('AMD64', 'amd64', 'x86_64'):
            sArch = 'amd64';
        elif sArch == 'i86pc': # SunOS
            if platform.architecture()[0] == '64bit':
                sArch = 'amd64';
            else:
                try:
                    sArch = processOutputChecked(['/usr/bin/isainfo', '-n',]);
                except:
                    pass;
                sArch = sArch.strip();
                if sArch != 'amd64':
                    sArch = 'x86';
        else:
            raise Exception('Unsupported architecture/machine "%s"' % (sArch,));
        g_sHostArch = sArch;
    return g_sHostArch;


def getHostOsDotArch():
    """
    Gets the 'os.arch' for the host.
    """
    return '%s.%s' % (getHostOs(), getHostArch());


def isValidOs(sOs):
    """
    Validates the OS name.
    """
    if sOs in ('darwin', 'dos', 'dragonfly', 'freebsd', 'haiku', 'l4', 'linux', 'netbsd', 'nt', 'openbsd', \
               'os2', 'solaris', 'win', 'os-agnostic'):
        return True;
    return False;


def isValidArch(sArch):
    """
    Validates the CPU architecture name.
    """
    if sArch in ('x86', 'amd64', 'sparc32', 'sparc64', 's390', 's390x', 'ppc32', 'ppc64', \
               'mips32', 'mips64', 'ia64', 'hppa32', 'hppa64', 'arm', 'alpha'):
        return True;
    return False;

def isValidOsDotArch(sOsDotArch):
    """
    Validates the 'os.arch' string.
    """

    asParts = sOsDotArch.split('.');
    if asParts.length() != 2:
        return False;
    return isValidOs(asParts[0]) \
       and isValidArch(asParts[1]);

def getHostOsVersion():
    """
    Returns the host OS version.  This is platform.release with additional
    distro indicator on linux.
    """
    sVersion = platform.release();
    sOs = getHostOs();
    if sOs == 'linux':
        sDist = '';
        try:
            # try /etc/lsb-release first to distinguish between Debian and Ubuntu
            oFile = open('/etc/lsb-release');
            for sLine in oFile:
                oMatch = re.search(r'(?:DISTRIB_DESCRIPTION\s*=)\s*"*(.*)"', sLine);
                if oMatch is not None:
                    sDist = oMatch.group(1).strip();
        except:
            pass;
        if sDist:
            sVersion += ' / ' + sDist;
        else:
            asFiles = \
            [
                [ '/etc/debian_version', 'Debian v'],
                [ '/etc/gentoo-release', '' ],
                [ '/etc/oracle-release', '' ],
                [ '/etc/redhat-release', '' ],
                [ '/etc/SuSE-release', '' ],
            ];
            for sFile, sPrefix in asFiles:
                if os.path.isfile(sFile):
                    try:
                        oFile = open(sFile);
                        sLine = oFile.readline();
                        oFile.close();
                    except:
                        continue;
                    sLine = sLine.strip()
                    if len(sLine) > 0:
                        sVersion += ' / ' + sPrefix + sLine;
                    break;

    elif sOs == 'solaris':
        sVersion = platform.version();
        if os.path.isfile('/etc/release'):
            try:
                oFile = open('/etc/release');
                sLast = oFile.readlines()[-1];
                oFile.close();
                sLast = sLast.strip();
                if len(sLast) > 0:
                    sVersion += ' (' + sLast + ')';
            except:
                pass;

    elif sOs == 'darwin':
        sOsxVersion = platform.mac_ver()[0];
        codenames = {"4": "Tiger",
                     "5": "Leopard",
                     "6": "Snow Leopard",
                     "7": "Lion",
                     "8": "Mountain Lion",
                     "9": "Mavericks",
                     "10": "Yosemite",
                     "11": "El Capitan",
                     "12": "Sierra",
                     "13": "Unknown 13",
                     "14": "Unknown 14", }
        sVersion += ' / OS X ' + sOsxVersion + ' (' + codenames[sOsxVersion.split('.')[1]] + ')'

    return sVersion;

#
# File system.
#

def openNoInherit(sFile, sMode = 'r'):
    """
    Wrapper around open() that tries it's best to make sure the file isn't
    inherited by child processes.

    This is a best effort thing at the moment as it doesn't synchronizes with
    child process spawning in any way.  Thus it can be subject to races in
    multithreaded programs.
    """

    try:
        from fcntl import FD_CLOEXEC, F_GETFD, F_SETFD, fcntl; # pylint: disable=F0401
    except:
        return open(sFile, sMode);

    oFile = open(sFile, sMode)
    #try:
    fcntl(oFile, F_SETFD, fcntl(oFile, F_GETFD) | FD_CLOEXEC);
    #except:
    #    pass;
    return oFile;

def noxcptReadLink(sPath, sXcptRet):
    """
    No exceptions os.readlink wrapper.
    """
    try:
        sRet = os.readlink(sPath); # pylint: disable=E1101
    except:
        sRet = sXcptRet;
    return sRet;

def readFile(sFile, sMode = 'rb'):
    """
    Reads the entire file.
    """
    oFile = open(sFile, sMode);
    sRet = oFile.read();
    oFile.close();
    return sRet;

def noxcptReadFile(sFile, sXcptRet, sMode = 'rb'):
    """
    No exceptions common.readFile wrapper.
    """
    try:
        sRet = readFile(sFile, sMode);
    except:
        sRet = sXcptRet;
    return sRet;

def noxcptRmDir(sDir, oXcptRet = False):
    """
    No exceptions os.rmdir wrapper.
    """
    oRet = True;
    try:
        os.rmdir(sDir);
    except:
        oRet = oXcptRet;
    return oRet;

def noxcptDeleteFile(sFile, oXcptRet = False):
    """
    No exceptions os.remove wrapper.
    """
    oRet = True;
    try:
        os.remove(sFile);
    except:
        oRet = oXcptRet;
    return oRet;


def dirEnumerateTree(sDir, fnCallback, fIgnoreExceptions = True):
    # type: (string, (string, stat) -> bool) -> bool
    """
    Recursively walks a directory tree, calling fnCallback for each.

    fnCallback takes a full path and stat object (can be None).  It
    returns a boolean value, False stops walking and returns immediately.

    Returns True or False depending on fnCallback.
    Returns None fIgnoreExceptions is True and an exception was raised by listdir.
    """
    def __worker(sCurDir):
        """ Worker for """
        try:
            asNames = os.listdir(sCurDir);
        except:
            if not fIgnoreExceptions:
                raise;
            return None;
        rc = True;
        for sName in asNames:
            if sName not in [ '.', '..' ]:
                sFullName = os.path.join(sCurDir, sName);
                try:    oStat = os.lstat(sFullName);
                except: oStat = None;
                if fnCallback(sFullName, oStat) is False:
                    return False;
                if oStat is not None and stat.S_ISDIR(oStat.st_mode):
                    rc =  __worker(sFullName);
                    if rc is False:
                        break;
        return rc;

    # Ensure unicode path here so listdir also returns unicode on windows.
    ## @todo figure out unicode stuff on non-windows.
    if sys.platform == 'win32':
        sDir = unicode(sDir);
    return __worker(sDir);



def formatFileMode(uMode):
    # type: (int) -> string
    """
    Format a st_mode value 'ls -la' fasion.
    Returns string.
    """
    if   stat.S_ISDIR(uMode):   sMode = 'd';
    elif stat.S_ISREG(uMode):   sMode = '-';
    elif stat.S_ISLNK(uMode):   sMode = 'l';
    elif stat.S_ISFIFO(uMode):  sMode = 'p';
    elif stat.S_ISCHR(uMode):   sMode = 'c';
    elif stat.S_ISBLK(uMode):   sMode = 'b';
    elif stat.S_ISSOCK(uMode):  sMode = 's';
    else:                       sMode = '?';
    ## @todo sticky bits.
    sMode += 'r' if uMode & stat.S_IRUSR else '-';
    sMode += 'w' if uMode & stat.S_IWUSR else '-';
    sMode += 'x' if uMode & stat.S_IXUSR else '-';
    sMode += 'r' if uMode & stat.S_IRGRP else '-';
    sMode += 'w' if uMode & stat.S_IWGRP else '-';
    sMode += 'x' if uMode & stat.S_IXGRP else '-';
    sMode += 'r' if uMode & stat.S_IROTH else '-';
    sMode += 'w' if uMode & stat.S_IWOTH else '-';
    sMode += 'x' if uMode & stat.S_IXOTH else '-';
    sMode += ' ';
    return sMode;


def formatFileStat(oStat):
    # type: (stat) -> string
    """
    Format a stat result 'ls -la' fasion (numeric IDs).
    Returns string.
    """
    return '%s %3s %4s %4s %10s %s' \
          % (formatFileMode(oStat.st_mode), oStat.st_nlink, oStat.st_uid, oStat.st_gid, oStat.st_size,
             time.strftime('%Y-%m-%d %H:%M', time.localtime(oStat.st_mtime)), );

## Good buffer for file operations.
g_cbGoodBufferSize = 256*1024;

## The original shutil.copyfileobj.
g_fnOriginalShCopyFileObj = None;

def __myshutilcopyfileobj(fsrc, fdst, length = g_cbGoodBufferSize):
    """ shutil.copyfileobj with different length default value (16384 is slow with python 2.7 on windows). """
    return g_fnOriginalShCopyFileObj(fsrc, fdst, length);

def __installShUtilHacks(shutil):
    """ Installs the shutil buffer size hacks. """
    global g_fnOriginalShCopyFileObj;
    if g_fnOriginalShCopyFileObj is None:
        g_fnOriginalShCopyFileObj = shutil.copyfileobj;
        shutil.copyfileobj = __myshutilcopyfileobj;
    return True;


def copyFileSimple(sFileSrc, sFileDst):
    """
    Wrapper around shutil.copyfile that simply copies the data of a regular file.
    Raises exception on failure.
    Return True for show.
    """
    import shutil;
    __installShUtilHacks(shutil);
    return shutil.copyfile(sFileSrc, sFileDst);

#
# SubProcess.
#

def _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs):
    """
    If the "executable" is a python script, insert the python interpreter at
    the head of the argument list so that it will work on systems which doesn't
    support hash-bang scripts.
    """

    asArgs = dKeywordArgs.get('args');
    if asArgs is None:
        asArgs = aPositionalArgs[0];

    if asArgs[0].endswith('.py'):
        if sys.executable is not None  and  len(sys.executable) > 0:
            asArgs.insert(0, sys.executable);
        else:
            asArgs.insert(0, 'python');

        # paranoia...
        if dKeywordArgs.get('args') is not None:
            dKeywordArgs['args'] = asArgs;
        else:
            aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
    return None;

def processPopenSafe(*aPositionalArgs, **dKeywordArgs):
    """
    Wrapper for subprocess.Popen that's Ctrl-C safe on windows.
    """
    if getHostOs() == 'win':
        if dKeywordArgs.get('creationflags', 0) == 0:
            dKeywordArgs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP;
    return subprocess.Popen(*aPositionalArgs, **dKeywordArgs);

def processCall(*aPositionalArgs, **dKeywordArgs):
    """
    Wrapper around subprocess.call to deal with its absence in older
    python versions.
    Returns process exit code (see subprocess.poll).
    """
    assert dKeywordArgs.get('stdout') is None;
    assert dKeywordArgs.get('stderr') is None;
    _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
    oProcess = processPopenSafe(*aPositionalArgs, **dKeywordArgs);
    return oProcess.wait();

def processOutputChecked(*aPositionalArgs, **dKeywordArgs):
    """
    Wrapper around subprocess.check_output to deal with its absense in older
    python versions.
    """
    _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
    oProcess = processPopenSafe(stdout=subprocess.PIPE, *aPositionalArgs, **dKeywordArgs);

    sOutput, _ = oProcess.communicate();
    iExitCode  = oProcess.poll();

    if iExitCode is not 0:
        asArgs = dKeywordArgs.get('args');
        if asArgs is None:
            asArgs = aPositionalArgs[0];
        print(sOutput);
        raise subprocess.CalledProcessError(iExitCode, asArgs);

    return str(sOutput); # str() make pylint happy.

g_fOldSudo = None;
def _sudoFixArguments(aPositionalArgs, dKeywordArgs, fInitialEnv = True):
    """
    Adds 'sudo' (or similar) to the args parameter, whereever it is.
    """

    # Are we root?
    fIsRoot = True;
    try:
        fIsRoot = os.getuid() == 0; # pylint: disable=E1101
    except:
        pass;

    # If not, prepend sudo (non-interactive, simulate initial login).
    if fIsRoot is not True:
        asArgs = dKeywordArgs.get('args');
        if asArgs is None:
            asArgs = aPositionalArgs[0];

        # Detect old sudo.
        global g_fOldSudo;
        if g_fOldSudo is None:
            try:
                sVersion = processOutputChecked(['sudo', '-V']);
            except:
                sVersion = '1.7.0';
            sVersion = sVersion.strip().split('\n')[0];
            sVersion = sVersion.replace('Sudo version', '').strip();
            g_fOldSudo = len(sVersion) >= 4 \
                     and sVersion[0] == '1' \
                     and sVersion[1] == '.' \
                     and sVersion[2] <= '6' \
                     and sVersion[3] == '.';

        asArgs.insert(0, 'sudo');
        if not g_fOldSudo:
            asArgs.insert(1, '-n');
        if fInitialEnv and not g_fOldSudo:
            asArgs.insert(1, '-i');

        # paranoia...
        if dKeywordArgs.get('args') is not None:
            dKeywordArgs['args'] = asArgs;
        else:
            aPositionalArgs = (asArgs,) + aPositionalArgs[1:];
    return None;


def sudoProcessCall(*aPositionalArgs, **dKeywordArgs):
    """
    sudo (or similar) + subprocess.call
    """
    _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
    _sudoFixArguments(aPositionalArgs, dKeywordArgs);
    return processCall(*aPositionalArgs, **dKeywordArgs);

def sudoProcessOutputChecked(*aPositionalArgs, **dKeywordArgs):
    """
    sudo (or similar) + subprocess.check_output.
    """
    _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
    _sudoFixArguments(aPositionalArgs, dKeywordArgs);
    return processOutputChecked(*aPositionalArgs, **dKeywordArgs);

def sudoProcessOutputCheckedNoI(*aPositionalArgs, **dKeywordArgs):
    """
    sudo (or similar) + subprocess.check_output, except '-i' isn't used.
    """
    _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
    _sudoFixArguments(aPositionalArgs, dKeywordArgs, False);
    return processOutputChecked(*aPositionalArgs, **dKeywordArgs);

def sudoProcessPopen(*aPositionalArgs, **dKeywordArgs):
    """
    sudo (or similar) + processPopenSafe.
    """
    _processFixPythonInterpreter(aPositionalArgs, dKeywordArgs);
    _sudoFixArguments(aPositionalArgs, dKeywordArgs);
    return processPopenSafe(*aPositionalArgs, **dKeywordArgs);


#
# Generic process stuff.
#

def processInterrupt(uPid):
    """
    Sends a SIGINT or equivalent to interrupt the specified process.
    Returns True on success, False on failure.

    On Windows hosts this may not work unless the process happens to be a
    process group leader.
    """
    if sys.platform == 'win32':
        try:
            win32console.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, uPid);             # pylint: disable=no-member
            fRc = True;
        except:
            fRc = False;
    else:
        try:
            os.kill(uPid, signal.SIGINT);
            fRc = True;
        except:
            fRc = False;
    return fRc;

def sendUserSignal1(uPid):
    """
    Sends a SIGUSR1 or equivalent to nudge the process into shutting down
    (VBoxSVC) or something.
    Returns True on success, False on failure or if not supported (win).

    On Windows hosts this may not work unless the process happens to be a
    process group leader.
    """
    if sys.platform == 'win32':
        fRc = False;
    else:
        try:
            os.kill(uPid, signal.SIGUSR1); # pylint: disable=E1101
            fRc = True;
        except:
            fRc = False;
    return fRc;

def processTerminate(uPid):
    """
    Terminates the process in a nice manner (SIGTERM or equivalent).
    Returns True on success, False on failure.
    """
    fRc = False;
    if sys.platform == 'win32':
        try:
            hProcess = win32api.OpenProcess(win32con.PROCESS_TERMINATE, False, uPid);           # pylint: disable=no-member
        except:
            pass;
        else:
            try:
                win32process.TerminateProcess(hProcess, 0x40010004); # DBG_TERMINATE_PROCESS    # pylint: disable=no-member
                fRc = True;
            except:
                pass;
            win32api.CloseHandle(hProcess)                                                      # pylint: disable=no-member
    else:
        try:
            os.kill(uPid, signal.SIGTERM);
            fRc = True;
        except:
            pass;
    return fRc;

def processKill(uPid):
    """
    Terminates the process with extreme prejudice (SIGKILL).
    Returns True on success, False on failure.
    """
    if sys.platform == 'win32':
        fRc = processTerminate(uPid);
    else:
        try:
            os.kill(uPid, signal.SIGKILL); # pylint: disable=E1101
            fRc = True;
        except:
            fRc = False;
    return fRc;

def processKillWithNameCheck(uPid, sName):
    """
    Like processKill(), but checks if the process name matches before killing
    it.  This is intended for killing using potentially stale pid values.

    Returns True on success, False on failure.
    """

    if processCheckPidAndName(uPid, sName) is not True:
        return False;
    return processKill(uPid);


def processExists(uPid):
    """
    Checks if the specified process exits.
    This will only work if we can signal/open the process.

    Returns True if it positively exists, False otherwise.
    """
    if sys.platform == 'win32':
        fRc = False;
        try:
            hProcess = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, False, uPid);   # pylint: disable=no-member
        except:
            pass;
        else:
            win32api.CloseHandle(hProcess);                                                     # pylint: disable=no-member
            fRc = True;
    else:
        try:
            os.kill(uPid, 0);
            fRc = True;
        except:
            fRc = False;
    return fRc;

def processCheckPidAndName(uPid, sName):
    """
    Checks if a process PID and NAME matches.
    """
    fRc = processExists(uPid);
    if fRc is not True:
        return False;

    if sys.platform == 'win32':
        try:
            from win32com.client import GetObject; # pylint: disable=F0401
            oWmi = GetObject('winmgmts:');
            aoProcesses = oWmi.InstancesOf('Win32_Process');
            for oProcess in aoProcesses:
                if long(oProcess.Properties_("ProcessId").Value) == uPid:
                    sCurName = oProcess.Properties_("Name").Value;
                    #reporter.log2('uPid=%s sName=%s sCurName=%s' % (uPid, sName, sCurName));
                    sName    = sName.lower();
                    sCurName = sCurName.lower();
                    if os.path.basename(sName) == sName:
                        sCurName = os.path.basename(sCurName);

                    if   sCurName == sName \
                      or sCurName + '.exe' == sName \
                      or sCurName == sName  + '.exe':
                        fRc = True;
                    break;
        except:
            #reporter.logXcpt('uPid=%s sName=%s' % (uPid, sName));
            pass;
    else:
        if sys.platform in ('linux2', ):
            asPsCmd = ['/bin/ps',     '-p', '%u' % (uPid,), '-o', 'fname='];
        elif sys.platform in ('sunos5',):
            asPsCmd = ['/usr/bin/ps', '-p', '%u' % (uPid,), '-o', 'fname='];
        elif sys.platform in ('darwin',):
            asPsCmd = ['/bin/ps',     '-p', '%u' % (uPid,), '-o', 'ucomm='];
        else:
            asPsCmd = None;

        if asPsCmd is not None:
            try:
                oPs = subprocess.Popen(asPsCmd, stdout=subprocess.PIPE);
                sCurName = oPs.communicate()[0];
                iExitCode = oPs.wait();
            except:
                #reporter.logXcpt();
                return False;

            # ps fails with non-zero exit code if the pid wasn't found.
            if iExitCode is not 0:
                return False;
            if sCurName is None:
                return False;
            sCurName = sCurName.strip();
            if sCurName is '':
                return False;

            if os.path.basename(sName) == sName:
                sCurName = os.path.basename(sCurName);
            elif os.path.basename(sCurName) == sCurName:
                sName = os.path.basename(sName);

            if sCurName != sName:
                return False;

            fRc = True;
    return fRc;

def processGetInfo(uPid, fSudo = False):
    """
    Tries to acquire state information of the given process.

    Returns a string with the information on success or None on failure or
    if the host is not supported.

    Note that the format of the information is host system dependent and will
    likely differ much between different hosts.
    """
    fRc = processExists(uPid);
    if fRc is not True:
        return None;

    sHostOs = getHostOs();
    if sHostOs in [ 'linux',]:
        sGdb = '/usr/bin/gdb';
        if not os.path.isfile(sGdb): sGdb = '/usr/local/bin/gdb';
        if not os.path.isfile(sGdb): sGdb = 'gdb';
        aasCmd = [
            [ sGdb, '-batch',
              '-ex', 'thread apply all bt',
              '-ex', 'info proc mapping',
              '-ex', 'info sharedlibrary',
              '-p', '%u' % (uPid,), ],
        ];
    elif sHostOs == 'darwin':
        # LLDB doesn't work in batch mode when attaching to a process, at least
        # with macOS Sierra (10.12). GDB might not be installed. Use the sample
        # tool instead with a 1 second duration and 1000ms sampling interval to
        # get one stack trace.  For the process mappings use vmmap.
        aasCmd = [
            [ '/usr/bin/sample', '-mayDie', '%u' % (uPid,), '1', '1000', ],
            [ '/usr/bin/vmmap', '%u' % (uPid,), ],
        ];
    elif sHostOs == 'solaris':
        aasCmd = [
            [ '/usr/bin/pstack', '%u' % (uPid,), ],
            [ '/usr/bin/pmap', '%u' % (uPid,), ],
        ];
    elif sHostOs == 'win':
        # There are quite a few possibilities on Windows where to find CDB.
        asCdbPathsCheck = [
                              'c:\\Program Files\\Debugging Tools for Windows',
                              'c:\\Program Files\\Debugging Tools for Windows (x64)',
                              'c:\\Program Files\\Debugging Tools for Windows (x86)',
                              'c:\\Program Files\\Windows Kits',
                              'c:\\Program Files\\Windows Kits (x64)',
                              'c:\\Program Files\\Windows Kits (x86)',
                              'c:\\Programme\\Debugging Tools for Windows',
                              'c:\\Programme\\Debugging Tools for Windows (x64)',
                              'c:\\Programme\\Debugging Tools for Windows (x86)',
                              'c:\\Programme\\Windows Kits',
                              'c:\\Programme\\Windows Kits (x64)',
                              'c:\\Programme\\Windows Kits (x86)'
                          ];

        sWinCdb = 'cdb'; # CDB must be in the path; better than nothing if we can't find anything in the paths above.
        for sPath in asCdbPathsCheck:
            fFound = False;
            try:
                for sDirPath, _, asFiles in os.walk(sPath):
                    if 'cdb.exe' in asFiles:
                        sWinCdb = os.path.join(sDirPath, 'cdb.exe');
                        fFound = True;
                        break;
            except:
                pass;

            if fFound:
                break;

        #
        # The commands used to gather the information are quite cryptic so they are explained
        # below:
        #     ~* f    -> Freeze all threads
        #     ~* k    -> Acquire stack traces for all threads
        #     lm -v   -> List of loaded modules, verbose
        #     ~* u    -> Unfreeze all threads
        #     .detach -> Detach from target process
        #     q       -> Quit CDB
        #
        aasCmd = [sWinCdb, '-p', '%u' % (uPid,), '-c', '~* f; ~* k; lm -v; ~* u; .detach; q'];
    else:
        aasCmd = [];

    sInfo = '';
    for asCmd in aasCmd:
        try:
            if fSudo:
                sThisInfo = sudoProcessOutputChecked(asCmd);
            else:
                sThisInfo = processOutputChecked(asCmd);
            if sThisInfo is not None:
                sInfo += sThisInfo;
        except:
            pass;
    if not sInfo:
        sInfo = None;

    return sInfo;


class ProcessInfo(object):
    """Process info."""
    def __init__(self, iPid):
        self.iPid       = iPid;
        self.iParentPid = None;
        self.sImage     = None;
        self.sName      = None;
        self.asArgs     = None;
        self.sCwd       = None;
        self.iGid       = None;
        self.iUid       = None;
        self.iProcGroup = None;
        self.iSessionId = None;

    def loadAll(self):
        """Load all the info."""
        sOs = getHostOs();
        if sOs == 'linux':
            sProc = '/proc/%s/' % (self.iPid,);
            if self.sImage   is None: self.sImage = noxcptReadLink(sProc + 'exe', None);
            if self.sCwd     is None: self.sCwd   = noxcptReadLink(sProc + 'cwd', None);
            if self.asArgs   is None: self.asArgs = noxcptReadFile(sProc + 'cmdline', '').split('\x00');
        #elif sOs == 'solaris': - doesn't work for root processes, suid proces, and other stuff.
        #    sProc = '/proc/%s/' % (self.iPid,);
        #    if self.sImage   is None: self.sImage = noxcptReadLink(sProc + 'path/a.out', None);
        #    if self.sCwd     is None: self.sCwd   = noxcptReadLink(sProc + 'path/cwd', None);
        else:
            pass;
        if self.sName is None and self.sImage is not None:
            self.sName = self.sImage;

    def windowsGrabProcessInfo(self, oProcess):
        """Windows specific loadAll."""
        try:    self.sName      = oProcess.Properties_("Name").Value;
        except: pass;
        try:    self.sImage     = oProcess.Properties_("ExecutablePath").Value;
        except: pass;
        try:    self.asArgs     = oProcess.Properties_("CommandLine").Value; ## @todo split it.
        except: pass;
        try:    self.iParentPid = oProcess.Properties_("ParentProcessId").Value;
        except: pass;
        try:    self.iSessionId = oProcess.Properties_("SessionId").Value;
        except: pass;
        if self.sName is None and self.sImage is not None:
            self.sName = self.sImage;

    def getBaseImageName(self):
        """
        Gets the base image name if available, use the process name if not available.
        Returns image/process base name or None.
        """
        sRet = self.sImage if self.sName is None else self.sName;
        if sRet is None:
            self.loadAll();
            sRet = self.sImage if self.sName is None else self.sName;
            if sRet is None:
                if self.asArgs is None or len(self.asArgs) == 0:
                    return None;
                sRet = self.asArgs[0];
                if len(sRet) == 0:
                    return None;
        return os.path.basename(sRet);

    def getBaseImageNameNoExeSuff(self):
        """
        Same as getBaseImageName, except any '.exe' or similar suffix is stripped.
        """
        sRet = self.getBaseImageName();
        if sRet is not None and len(sRet) > 4 and sRet[-4] == '.':
            if (sRet[-4:]).lower() in [ '.exe', '.com', '.msc', '.vbs', '.cmd', '.bat' ]:
                sRet = sRet[:-4];
        return sRet;


def processListAll(): # pylint: disable=R0914
    """
    Return a list of ProcessInfo objects for all the processes in the system
    that the current user can see.
    """
    asProcesses = [];

    sOs = getHostOs();
    if sOs == 'win':
        from win32com.client import GetObject; # pylint: disable=F0401
        oWmi = GetObject('winmgmts:');
        aoProcesses = oWmi.InstancesOf('Win32_Process');
        for oProcess in aoProcesses:
            try:
                iPid = int(oProcess.Properties_("ProcessId").Value);
            except:
                continue;
            oMyInfo = ProcessInfo(iPid);
            oMyInfo.windowsGrabProcessInfo(oProcess);
            asProcesses.append(oMyInfo);
        return asProcesses;

    if sOs in [ 'linux', ]:  # Not solaris, ps gets more info than /proc/.
        try:
            asDirs = os.listdir('/proc');
        except:
            asDirs = [];
        for sDir in asDirs:
            if sDir.isdigit():
                asProcesses.append(ProcessInfo(int(sDir),));
        return asProcesses;

    #
    # The other OSes parses the output from the 'ps' utility.
    #
    asPsCmd = [
        '/bin/ps',          # 0
        '-A',               # 1
        '-o', 'pid=',       # 2,3
        '-o', 'ppid=',      # 4,5
        '-o', 'pgid=',      # 6,7
        '-o', 'sid=',       # 8,9
        '-o', 'uid=',       # 10,11
        '-o', 'gid=',       # 12,13
        '-o', 'comm='       # 14,15
    ];

    if sOs == 'darwin':
        assert asPsCmd[9] == 'sid=';
        asPsCmd[9] = 'sess=';
    elif sOs == 'solaris':
        asPsCmd[0] = '/usr/bin/ps';

    try:
        sRaw = processOutputChecked(asPsCmd);
    except:
        return asProcesses;

    for sLine in sRaw.split('\n'):
        sLine = sLine.lstrip();
        if len(sLine) < 7 or not sLine[0].isdigit():
            continue;

        iField   = 0;
        off      = 0;
        aoFields = [None, None, None, None, None, None, None];
        while iField < 7:
            # Eat whitespace.
            while off < len(sLine) and (sLine[off] == ' ' or sLine[off] == '\t'):
                off += 1;

            # Final field / EOL.
            if iField == 6:
                aoFields[6] = sLine[off:];
                break;
            if off >= len(sLine):
                break;

            # Generic field parsing.
            offStart = off;
            off += 1;
            while off < len(sLine) and sLine[off] != ' ' and sLine[off] != '\t':
                off += 1;
            try:
                if iField != 3:
                    aoFields[iField] = int(sLine[offStart:off]);
                else:
                    aoFields[iField] = long(sLine[offStart:off], 16); # sess is a hex address.
            except:
                pass;
            iField += 1;

        if aoFields[0] is not None:
            oMyInfo = ProcessInfo(aoFields[0]);
            oMyInfo.iParentPid = aoFields[1];
            oMyInfo.iProcGroup = aoFields[2];
            oMyInfo.iSessionId = aoFields[3];
            oMyInfo.iUid       = aoFields[4];
            oMyInfo.iGid       = aoFields[5];
            oMyInfo.sName      = aoFields[6];
            asProcesses.append(oMyInfo);

    return asProcesses;


def processCollectCrashInfo(uPid, fnLog, fnCrashFile):
    """
    Looks for information regarding the demise of the given process.
    """
    sOs = getHostOs();
    if sOs == 'darwin':
        #
        # On darwin we look for crash and diagnostic reports.
        #
        asLogDirs = [
            u'/Library/Logs/DiagnosticReports/',
            u'/Library/Logs/CrashReporter/',
            u'~/Library/Logs/DiagnosticReports/',
            u'~/Library/Logs/CrashReporter/',
        ];
        for sDir in asLogDirs:
            sDir = os.path.expanduser(sDir);
            if not os.path.isdir(sDir):
                continue;
            try:
                asDirEntries = os.listdir(sDir);
            except:
                continue;
            for sEntry in asDirEntries:
                # Only interested in .crash files.
                _, sSuff = os.path.splitext(sEntry);
                if sSuff != '.crash':
                    continue;

                # The pid can be found at the end of the first line.
                sFull = os.path.join(sDir, sEntry);
                try:
                    oFile = open(sFull, 'r');
                    sFirstLine = oFile.readline();
                    oFile.close();
                except:
                    continue;
                if len(sFirstLine) <= 4 or sFirstLine[-2] != ']':
                    continue;
                offPid = len(sFirstLine) - 3;
                while offPid > 1 and sFirstLine[offPid - 1].isdigit():
                    offPid -= 1;
                try:    uReportPid = int(sFirstLine[offPid:-2]);
                except: continue;

                # Does the pid we found match?
                if uReportPid == uPid:
                    fnLog('Found crash report for %u: %s' % (uPid, sFull,));
                    fnCrashFile(sFull, False);
    elif sOs == 'win':
        #
        # Getting WER reports would be great, however we have trouble match the
        # PID to those as they seems not to mention it in the brief reports.
        # Instead we'll just look for crash dumps in C:\CrashDumps (our custom
        # location - see the windows readme for the testbox script) and what
        # the MSDN article lists for now.
        #
        # It's been observed on Windows server 2012 that the dump files takes
        # the form: <processimage>.<decimal-pid>.dmp
        #
        asDmpDirs = [
            u'%SystemDrive%/CrashDumps/',                   # Testboxes.
            u'%LOCALAPPDATA%/CrashDumps/',                  # MSDN example.
            u'%WINDIR%/ServiceProfiles/LocalServices/',     # Local and network service.
            u'%WINDIR%/ServiceProfiles/NetworkSerices/',
            u'%WINDIR%/ServiceProfiles/',
            u'%WINDIR%/System32/Config/SystemProfile/',     # System services.
        ];
        sMatchSuffix = '.%u.dmp' % (uPid,);

        for sDir in asDmpDirs:
            sDir = os.path.expandvars(sDir);
            if not os.path.isdir(sDir):
                continue;
            try:
                asDirEntries = os.listdir(sDir);
            except:
                continue;
            for sEntry in asDirEntries:
                if sEntry.endswith(sMatchSuffix):
                    sFull = os.path.join(sDir, sEntry);
                    fnLog('Found crash dump for %u: %s' % (uPid, sFull,));
                    fnCrashFile(sFull, True);

    else:
        pass; ## TODO
    return None;


#
# Time.
#

#
# The following test case shows how time.time() only have ~ms resolution
# on Windows (tested W10) and why it therefore makes sense to try use
# performance counters.
#
# Note! We cannot use time.clock() as the timestamp must be portable across
#       processes.  See timeout testcase problem on win hosts (no logs).
#
#import sys;
#import time;
#from common import utils;
#
#atSeries = [];
#for i in xrange(1,160):
#    if i == 159: time.sleep(10);
#    atSeries.append((utils.timestampNano(), long(time.clock() * 1000000000), long(time.time() * 1000000000)));
#
#tPrev = atSeries[0]
#for tCur in atSeries:
#    print 't1=%+22u, %u' % (tCur[0], tCur[0] - tPrev[0]);
#    print 't2=%+22u, %u' % (tCur[1], tCur[1] - tPrev[1]);
#    print 't3=%+22u, %u' % (tCur[2], tCur[2] - tPrev[2]);
#    print '';
#    tPrev = tCur
#
#print 't1=%u' % (atSeries[-1][0] - atSeries[0][0]);
#print 't2=%u' % (atSeries[-1][1] - atSeries[0][1]);
#print 't3=%u' % (atSeries[-1][2] - atSeries[0][2]);

g_fWinUseWinPerfCounter           = sys.platform == 'win32';
g_fpWinPerfCounterFreq            = None;
g_oFuncwinQueryPerformanceCounter = None;

def _winInitPerfCounter():
    """ Initializes the use of performance counters. """
    global g_fWinUseWinPerfCounter, g_fpWinPerfCounterFreq, g_oFuncwinQueryPerformanceCounter

    uFrequency = ctypes.c_ulonglong(0);
    if ctypes.windll.kernel32.QueryPerformanceFrequency(ctypes.byref(uFrequency)):
        if uFrequency.value >= 1000:
            #print 'uFrequency = %s' % (uFrequency,);
            #print 'type(uFrequency) = %s' % (type(uFrequency),);
            g_fpWinPerfCounterFreq = float(uFrequency.value);

            # Check that querying the counter works too.
            global g_oFuncwinQueryPerformanceCounter
            g_oFuncwinQueryPerformanceCounter = ctypes.windll.kernel32.QueryPerformanceCounter;
            uCurValue = ctypes.c_ulonglong(0);
            if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
                if uCurValue.value > 0:
                    return True;
    g_fWinUseWinPerfCounter = False;
    return False;

def _winFloatTime():
    """ Gets floating point time on windows. """
    if g_fpWinPerfCounterFreq is not None or _winInitPerfCounter():
        uCurValue = ctypes.c_ulonglong(0);
        if g_oFuncwinQueryPerformanceCounter(ctypes.byref(uCurValue)):
            return float(uCurValue.value) / g_fpWinPerfCounterFreq;
    return time.time();


def timestampNano():
    """
    Gets a nanosecond timestamp.
    """
    if g_fWinUseWinPerfCounter is True:
        return long(_winFloatTime() * 1000000000);
    return long(time.time() * 1000000000);

def timestampMilli():
    """
    Gets a millisecond timestamp.
    """
    if g_fWinUseWinPerfCounter is True:
        return long(_winFloatTime() * 1000);
    return long(time.time() * 1000);

def timestampSecond():
    """
    Gets a second timestamp.
    """
    if g_fWinUseWinPerfCounter is True:
        return long(_winFloatTime());
    return long(time.time());

def getTimePrefix():
    """
    Returns a timestamp prefix, typically used for logging. UTC.
    """
    try:
        oNow = datetime.datetime.utcnow();
        sTs = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
    except:
        sTs = 'getTimePrefix-exception';
    return sTs;

def getTimePrefixAndIsoTimestamp():
    """
    Returns current UTC as log prefix and iso timestamp.
    """
    try:
        oNow = datetime.datetime.utcnow();
        sTsPrf = '%02u:%02u:%02u.%06u' % (oNow.hour, oNow.minute, oNow.second, oNow.microsecond);
        sTsIso = formatIsoTimestamp(oNow);
    except:
        sTsPrf = sTsIso = 'getTimePrefix-exception';
    return (sTsPrf, sTsIso);

def formatIsoTimestamp(oNow):
    """Formats the datetime object as an ISO timestamp."""
    assert oNow.tzinfo is None;
    sTs = '%s.%09uZ' % (oNow.strftime('%Y-%m-%dT%H:%M:%S'), oNow.microsecond * 1000);
    return sTs;

def getIsoTimestamp():
    """Returns the current UTC timestamp as a string."""
    return formatIsoTimestamp(datetime.datetime.utcnow());


def getLocalHourOfWeek():
    """ Local hour of week (0 based). """
    oNow = datetime.datetime.now();
    return (oNow.isoweekday() - 1) * 24 + oNow.hour;


def formatIntervalSeconds(cSeconds):
    """ Format a seconds interval into a nice 01h 00m 22s string  """
    # Two simple special cases.
    if cSeconds < 60:
        return '%ss' % (cSeconds,);
    if cSeconds < 3600:
        cMins = cSeconds / 60;
        cSecs = cSeconds % 60;
        if cSecs == 0:
            return '%sm' % (cMins,);
        return '%sm %ss' % (cMins, cSecs,);

    # Generic and a bit slower.
    cDays     = cSeconds / 86400;
    cSeconds %= 86400;
    cHours    = cSeconds / 3600;
    cSeconds %= 3600;
    cMins     = cSeconds / 60;
    cSecs     = cSeconds % 60;
    sRet = '';
    if cDays > 0:
        sRet = '%sd ' % (cDays,);
    if cHours > 0:
        sRet += '%sh ' % (cHours,);
    if cMins > 0:
        sRet += '%sm ' % (cMins,);
    if cSecs > 0:
        sRet += '%ss ' % (cSecs,);
    assert len(sRet) > 0; assert sRet[-1] == ' ';
    return sRet[:-1];

def formatIntervalSeconds2(oSeconds):
    """
    Flexible input version of formatIntervalSeconds for use in WUI forms where
    data is usually already string form.
    """
    if isinstance(oSeconds, int) or isinstance(oSeconds, long):
        return formatIntervalSeconds(oSeconds);
    if not isString(oSeconds):
        try:
            lSeconds = long(oSeconds);
        except:
            pass;
        else:
            if lSeconds >= 0:
                return formatIntervalSeconds2(lSeconds);
    return oSeconds;

def parseIntervalSeconds(sString):
    """
    Reverse of formatIntervalSeconds.

    Returns (cSeconds, sError), where sError is None on success.
    """

    # We might given non-strings, just return them without any fuss.
    if not isString(sString):
        if isinstance(sString, int) or isinstance(sString, long) or  sString is None:
            return (sString, None);
        ## @todo time/date objects?
        return (int(sString), None);

    # Strip it and make sure it's not empty.
    sString = sString.strip();
    if len(sString) == 0:
        return (0, 'Empty interval string.');

    #
    # Split up the input into a list of 'valueN, unitN, ...'.
    #
    # Don't want to spend too much time trying to make re.split do exactly what
    # I need here, so please forgive the extra pass I'm making here.
    #
    asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
    asParts    = [];
    for sPart in asRawParts:
        sPart = sPart.strip();
        if len(sPart) > 0:
            asParts.append(sPart);
    if len(asParts) == 0:
        return (0, 'Empty interval string or something?');

    #
    # Process them one or two at the time.
    #
    cSeconds   = 0;
    asErrors   = [];
    i          = 0;
    while i < len(asParts):
        sNumber = asParts[i];
        i += 1;
        if sNumber.isdigit():
            iNumber = int(sNumber);

            sUnit = 's';
            if i < len(asParts) and not asParts[i].isdigit():
                sUnit = asParts[i];
                i += 1;

            sUnitLower = sUnit.lower();
            if sUnitLower in [ 's', 'se', 'sec', 'second', 'seconds' ]:
                pass;
            elif sUnitLower in [ 'm', 'mi', 'min', 'minute', 'minutes' ]:
                iNumber *= 60;
            elif sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
                iNumber *= 3600;
            elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
                iNumber *= 86400;
            elif sUnitLower in [ 'w', 'week', 'weeks' ]:
                iNumber *= 7 * 86400;
            else:
                asErrors.append('Unknown unit "%s".' % (sUnit,));
            cSeconds += iNumber;
        else:
            asErrors.append('Bad number "%s".' % (sNumber,));
    return (cSeconds, None if len(asErrors) == 0 else ' '.join(asErrors));

def formatIntervalHours(cHours):
    """ Format a hours interval into a nice 1w 2d 1h string. """
    # Simple special cases.
    if cHours < 24:
        return '%sh' % (cHours,);

    # Generic and a bit slower.
    cWeeks    = cHours / (7 * 24);
    cHours   %= 7 * 24;
    cDays     = cHours / 24;
    cHours   %= 24;
    sRet = '';
    if cWeeks > 0:
        sRet = '%sw ' % (cWeeks,);
    if cDays > 0:
        sRet = '%sd ' % (cDays,);
    if cHours > 0:
        sRet += '%sh ' % (cHours,);
    assert len(sRet) > 0; assert sRet[-1] == ' ';
    return sRet[:-1];

def parseIntervalHours(sString):
    """
    Reverse of formatIntervalHours.

    Returns (cHours, sError), where sError is None on success.
    """

    # We might given non-strings, just return them without any fuss.
    if not isString(sString):
        if isinstance(sString, int) or isinstance(sString, long) or  sString is None:
            return (sString, None);
        ## @todo time/date objects?
        return (int(sString), None);

    # Strip it and make sure it's not empty.
    sString = sString.strip();
    if len(sString) == 0:
        return (0, 'Empty interval string.');

    #
    # Split up the input into a list of 'valueN, unitN, ...'.
    #
    # Don't want to spend too much time trying to make re.split do exactly what
    # I need here, so please forgive the extra pass I'm making here.
    #
    asRawParts = re.split(r'\s*([0-9]+)\s*([^0-9,;]*)[\s,;]*', sString);
    asParts    = [];
    for sPart in asRawParts:
        sPart = sPart.strip();
        if len(sPart) > 0:
            asParts.append(sPart);
    if len(asParts) == 0:
        return (0, 'Empty interval string or something?');

    #
    # Process them one or two at the time.
    #
    cHours     = 0;
    asErrors   = [];
    i          = 0;
    while i < len(asParts):
        sNumber = asParts[i];
        i += 1;
        if sNumber.isdigit():
            iNumber = int(sNumber);

            sUnit = 'h';
            if i < len(asParts) and not asParts[i].isdigit():
                sUnit = asParts[i];
                i += 1;

            sUnitLower = sUnit.lower();
            if sUnitLower in [ 'h', 'ho', 'hou', 'hour', 'hours' ]:
                pass;
            elif sUnitLower in [ 'd', 'da', 'day', 'days' ]:
                iNumber *= 24;
            elif sUnitLower in [ 'w', 'week', 'weeks' ]:
                iNumber *= 7 * 24;
            else:
                asErrors.append('Unknown unit "%s".' % (sUnit,));
            cHours += iNumber;
        else:
            asErrors.append('Bad number "%s".' % (sNumber,));
    return (cHours, None if len(asErrors) == 0 else ' '.join(asErrors));


#
# Introspection.
#

def getCallerName(oFrame=None, iFrame=2):
    """
    Returns the name of the caller's caller.
    """
    if oFrame is None:
        try:
            raise Exception();
        except:
            oFrame = sys.exc_info()[2].tb_frame.f_back;
        while iFrame > 1:
            if oFrame is not None:
                oFrame = oFrame.f_back;
            iFrame = iFrame - 1;
    if oFrame is not None:
        sName = '%s:%u' % (oFrame.f_code.co_name, oFrame.f_lineno);
        return sName;
    return "unknown";


def getXcptInfo(cFrames = 1):
    """
    Gets text detailing the exception. (Good for logging.)
    Returns list of info strings.
    """

    #
    # Try get exception info.
    #
    try:
        oType, oValue, oTraceback = sys.exc_info();
    except:
        oType = oValue = oTraceback = None;
    if oType is not None:

        #
        # Try format the info
        #
        asRet = [];
        try:
            try:
                asRet = asRet + traceback.format_exception_only(oType, oValue);
                asTraceBack = traceback.format_tb(oTraceback);
                if cFrames is not None and cFrames <= 1:
                    asRet.append(asTraceBack[-1]);
                else:
                    asRet.append('Traceback:')
                    for iFrame in range(min(cFrames, len(asTraceBack))):
                        asRet.append(asTraceBack[-iFrame - 1]);
                    asRet.append('Stack:')
                    asRet = asRet + traceback.format_stack(oTraceback.tb_frame.f_back, cFrames);
            except:
                asRet.append('internal-error: Hit exception #2! %s' % (traceback.format_exc(),));

            if len(asRet) == 0:
                asRet.append('No exception info...');
        except:
            asRet.append('internal-error: Hit exception! %s' % (traceback.format_exc(),));
    else:
        asRet = ['Couldn\'t find exception traceback.'];
    return asRet;


#
# TestSuite stuff.
#

def isRunningFromCheckout(cScriptDepth = 1):
    """
    Checks if we're running from the SVN checkout or not.
    """

    try:
        sFile = __file__;
        cScriptDepth = 1;
    except:
        sFile = sys.argv[0];

    sDir = os.path.abspath(sFile);
    while cScriptDepth >= 0:
        sDir = os.path.dirname(sDir);
        if    os.path.exists(os.path.join(sDir, 'Makefile.kmk')) \
           or os.path.exists(os.path.join(sDir, 'Makefile.kup')):
            return True;
        cScriptDepth -= 1;

    return False;


#
# Bourne shell argument fun.
#


def argsSplit(sCmdLine):
    """
    Given a bourne shell command line invocation, split it up into arguments
    assuming IFS is space.
    Returns None on syntax error.
    """
    ## @todo bourne shell argument parsing!
    return sCmdLine.split(' ');

def argsGetFirst(sCmdLine):
    """
    Given a bourne shell command line invocation, get return the first argument
    assuming IFS is space.
    Returns None on invalid syntax, otherwise the parsed and unescaped argv[0] string.
    """
    asArgs = argsSplit(sCmdLine);
    if asArgs is None  or  len(asArgs) == 0:
        return None;

    return asArgs[0];

#
# String helpers.
#

def stricmp(sFirst, sSecond):
    """
    Compares to strings in an case insensitive fashion.

    Python doesn't seem to have any way of doing the correctly, so this is just
    an approximation using lower.
    """
    if sFirst == sSecond:
        return 0;
    sLower1 = sFirst.lower();
    sLower2 = sSecond.lower();
    if sLower1 == sLower2:
        return 0;
    if sLower1 < sLower2:
        return -1;
    return 1;


#
# Misc.
#

def versionCompare(sVer1, sVer2):
    """
    Compares to version strings in a fashion similar to RTStrVersionCompare.
    """

    ## @todo implement me!!

    if sVer1 == sVer2:
        return 0;
    if sVer1 < sVer2:
        return -1;
    return 1;


def formatNumber(lNum, sThousandSep = ' '):
    """
    Formats a decimal number with pretty separators.
    """
    sNum = str(lNum);
    sRet = sNum[-3:];
    off  = len(sNum) - 3;
    while off > 0:
        off -= 3;
        sRet = sNum[(off if off >= 0 else 0):(off + 3)] + sThousandSep + sRet;
    return sRet;


def formatNumberNbsp(lNum):
    """
    Formats a decimal number with pretty separators.
    """
    sRet = formatNumber(lNum);
    return unicode(sRet).replace(' ', u'\u00a0');


def isString(oString):
    """
    Checks if the object is a string object, hiding difference between python 2 and 3.

    Returns True if it's a string of some kind.
    Returns False if not.
    """
    if sys.version_info[0] >= 3:
        return isinstance(oString, str);
    return isinstance(oString, basestring);


def hasNonAsciiCharacters(sText):
    """
    Returns True is specified string has non-ASCII characters, False if ASCII only.
    """
    sTmp = unicode(sText, errors='ignore') if isinstance(sText, str) else sText;
    return not all(ord(ch) < 128 for ch in sTmp);


def chmodPlusX(sFile):
    """
    Makes the specified file or directory executable.
    Returns success indicator, no exceptions.

    Note! Symbolic links are followed and the target will be changed.
    """
    try:
        oStat = os.stat(sFile);
    except:
        return False;
    try:
        os.chmod(sFile, oStat.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH);
    except:
        return False;
    return True;


def unpackZipFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
    # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
    """
    Worker for unpackFile that deals with ZIP files, same function signature.
    """
    import zipfile
    if fnError is None:
        fnError = fnLog;

    fnLog('Unzipping "%s" to "%s"...' % (sArchive, sDstDir));

    # Open it.
    try: oZipFile = zipfile.ZipFile(sArchive, 'r')
    except Exception as oXcpt:
        fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
        return None;

    # Extract all members.
    asMembers = [];
    try:
        for sMember in oZipFile.namelist():
            if fnFilter is None  or  fnFilter(sMember) is not False:
                if sMember.endswith('/'):
                    os.makedirs(os.path.join(sDstDir, sMember.replace('/', os.path.sep)), 0x1fd); # octal: 0775 (python 3/2)
                else:
                    oZipFile.extract(sMember, sDstDir);
                asMembers.append(os.path.join(sDstDir, sMember.replace('/', os.path.sep)));
    except Exception as oXcpt:
        fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
        asMembers = None;

    # close it.
    try: oZipFile.close();
    except Exception as oXcpt:
        fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
        asMembers = None;

    return asMembers;


## Set if we've replaced tarfile.copyfileobj with __mytarfilecopyfileobj already.
g_fTarCopyFileObjOverriddend = False;

def __mytarfilecopyfileobj(src, dst, length = None, exception = OSError):
    """ tarfile.copyfileobj with different buffer size (16384 is slow on windows). """
    if length is None:
        __myshutilcopyfileobj(src, dst, g_cbGoodBufferSize);
    elif length > 0:
        cFull, cbRemainder = divmod(length, g_cbGoodBufferSize);
        for _ in xrange(cFull):
            abBuffer = src.read(g_cbGoodBufferSize);
            dst.write(abBuffer);
            if len(abBuffer) != g_cbGoodBufferSize:
                raise exception('unexpected end of source file');
        if cbRemainder > 0:
            abBuffer = src.read(cbRemainder);
            dst.write(abBuffer);
            if len(abBuffer) != cbRemainder:
                raise exception('unexpected end of source file');


def unpackTarFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
    # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
    """
    Worker for unpackFile that deals with tarballs, same function signature.
    """
    import shutil;
    import tarfile;
    if fnError is None:
        fnError = fnLog;

    fnLog('Untarring "%s" to "%s"...' % (sArchive, sDstDir));

    #
    # Default buffer sizes of 16384 bytes is causing too many syscalls on Windows.
    # 60%+ speedup for python 2.7 and 50%+ speedup for python 3.5, both on windows with PDBs.
    # 20%+ speedup for python 2.7 and 15%+ speedup for python 3.5, both on windows skipping PDBs.
    #
    if True is True:
        __installShUtilHacks(shutil);
        global g_fTarCopyFileObjOverriddend;
        if g_fTarCopyFileObjOverriddend is False:
            g_fTarCopyFileObjOverriddend = True;
            tarfile.copyfileobj = __mytarfilecopyfileobj;

    #
    # Open it.
    #
    # Note! We not using 'r:*' because we cannot allow seeking compressed files!
    #       That's how we got a 13 min unpack time for VBoxAll on windows (hardlinked pdb).
    #
    try: oTarFile = tarfile.open(sArchive, 'r|*', bufsize = g_cbGoodBufferSize);
    except Exception as oXcpt:
        fnError('Error opening "%s" for unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt,));
        return None;

    # Extract all members.
    asMembers = [];
    try:
        for oTarInfo in oTarFile:
            try:
                if fnFilter is None  or  fnFilter(oTarInfo.name) is not False:
                    if oTarInfo.islnk():
                        # Links are trouble, especially on Windows.  We must avoid the falling that will end up seeking
                        # in the compressed tar stream.  So, fall back on shutil.copy2 instead.
                        sLinkFile     = os.path.join(sDstDir, oTarInfo.name.rstrip('/').replace('/', os.path.sep));
                        sLinkTarget   = os.path.join(sDstDir, oTarInfo.linkname.rstrip('/').replace('/', os.path.sep));
                        sParentDir    = os.path.dirname(sLinkFile);
                        try:    os.unlink(sLinkFile);
                        except: pass;
                        if sParentDir is not ''  and  not os.path.exists(sParentDir):
                            os.makedirs(sParentDir);
                        try:    os.link(sLinkTarget, sLinkFile);
                        except: shutil.copy2(sLinkTarget, sLinkFile);
                    else:
                        if oTarInfo.isdir():
                            # Just make sure the user (we) got full access to dirs.  Don't bother getting it 100% right.
                            oTarInfo.mode |= 0x1c0; # (octal: 0700)
                        oTarFile.extract(oTarInfo, sDstDir);
                    asMembers.append(os.path.join(sDstDir, oTarInfo.name.replace('/', os.path.sep)));
            except Exception as oXcpt:
                fnError('Error unpacking "%s" member "%s" into "%s": %s' % (sArchive, oTarInfo.name, sDstDir, oXcpt));
                for sAttr in [ 'name', 'linkname', 'type', 'mode', 'size', 'mtime', 'uid', 'uname', 'gid', 'gname' ]:
                    fnError('Info: %8s=%s' % (sAttr, getattr(oTarInfo, sAttr),));
                for sFn in [ 'isdir', 'isfile', 'islnk', 'issym' ]:
                    fnError('Info: %8s=%s' % (sFn, getattr(oTarInfo, sFn)(),));
                asMembers = None;
                break;
    except Exception as oXcpt:
        fnError('Error unpacking "%s" into "%s": %s' % (sArchive, sDstDir, oXcpt));
        asMembers = None;

    #
    # Finally, close it.
    #
    try: oTarFile.close();
    except Exception as oXcpt:
        fnError('Error closing "%s" after unpacking into "%s": %s' % (sArchive, sDstDir, oXcpt));
        asMembers = None;

    return asMembers;


def unpackFile(sArchive, sDstDir, fnLog, fnError = None, fnFilter = None):
    # type: (string, string, (string) -> None, (string) -> None, (string) -> bool) -> list[string]
    """
    Unpacks the given file if it has a know archive extension, otherwise do
    nothing.

    fnLog & fnError both take a string parameter.

    fnFilter takes a member name (string) and returns True if it's included
    and False if excluded.

    Returns list of the extracted files (full path) on success.
    Returns empty list if not a supported archive format.
    Returns None on failure.  Raises no exceptions.
    """
    sBaseNameLower = os.path.basename(sArchive).lower();

    #
    # Zip file?
    #
    if sBaseNameLower.endswith('.zip'):
        return unpackZipFile(sArchive, sDstDir, fnLog, fnError, fnFilter);

    #
    # Tarball?
    #
    if   sBaseNameLower.endswith('.tar') \
      or sBaseNameLower.endswith('.tar.gz') \
      or sBaseNameLower.endswith('.tgz') \
      or sBaseNameLower.endswith('.tar.bz2'):
        return unpackTarFile(sArchive, sDstDir, fnLog, fnError, fnFilter);

    #
    # Cannot classify it from the name, so just return that to the caller.
    #
    fnLog('Not unpacking "%s".' % (sArchive,));
    return [];


def getDiskUsage(sPath):
    """
    Get free space of a partition that corresponds to specified sPath in MB.

    Returns partition free space value in MB.
    """
    if platform.system() == 'Windows':
        oCTypeFreeSpace = ctypes.c_ulonglong(0);
        ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(sPath), None, None,
                                                   ctypes.pointer(oCTypeFreeSpace));
        cbFreeSpace = oCTypeFreeSpace.value;
    else:
        oStats = os.statvfs(sPath); # pylint: disable=E1101
        cbFreeSpace = long(oStats.f_frsize) * oStats.f_bfree;

    # Convert to MB
    cMbFreeSpace = long(cbFreeSpace) / (1024 * 1024);

    return cMbFreeSpace;


#
# Unit testing.
#

# pylint: disable=C0111
class BuildCategoryDataTestCase(unittest.TestCase):
    def testIntervalSeconds(self):
        self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(3600)), (3600, None));
        self.assertEqual(parseIntervalSeconds(formatIntervalSeconds(1209438593)), (1209438593, None));
        self.assertEqual(parseIntervalSeconds('123'), (123, None));
        self.assertEqual(parseIntervalSeconds(123), (123, None));
        self.assertEqual(parseIntervalSeconds(99999999999), (99999999999, None));
        self.assertEqual(parseIntervalSeconds(''), (0, 'Empty interval string.'));
        self.assertEqual(parseIntervalSeconds('1X2'), (3, 'Unknown unit "X".'));
        self.assertEqual(parseIntervalSeconds('1 Y3'), (4, 'Unknown unit "Y".'));
        self.assertEqual(parseIntervalSeconds('1 Z 4'), (5, 'Unknown unit "Z".'));
        self.assertEqual(parseIntervalSeconds('1 hour 2m 5second'), (3725, None));
        self.assertEqual(parseIntervalSeconds('1 hour,2m ; 5second'), (3725, None));

    def testHasNonAsciiChars(self):
        self.assertEqual(hasNonAsciiCharacters(''), False);
        self.assertEqual(hasNonAsciiCharacters('asdfgebASDFKJ@#$)(!@#UNASDFKHB*&$%&)@#(!)@(#!(#$&*#$&%*Y@#$IQWN---00;'), False);
        self.assertEqual(hasNonAsciiCharacters(u'12039889y!@#$%^&*()0-0asjdkfhoiuyweasdfASDFnvV'), False);
        self.assertEqual(hasNonAsciiCharacters(u'\u0079'), False);
        self.assertEqual(hasNonAsciiCharacters(u'\u0080'), True);
        self.assertEqual(hasNonAsciiCharacters(u'\u0081 \u0100'), True);

if __name__ == '__main__':
    unittest.main();
    # not reached.