Index: trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py
===================================================================
--- trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py	(revision 61282)
+++ trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py	(revision 61282)
@@ -0,0 +1,240 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# $Id$
+# pylint: disable=C0301
+
+"""
+Virtual Test Sheriff.
+
+Duties:
+    - Try to a assign failure reasons to recently failed tests.
+    - Reboot or disable bad test boxes.
+
+"""
+
+__copyright__ = \
+"""
+Copyright (C) 2012-2016 Oracle Corporation
+
+This file is part of VirtualBox Open Source Edition (OSE), as
+available from http://www.virtualbox.org. This file is free software;
+you can redistribute it and/or modify it under the terms of the GNU
+General Public License (GPL) as published by the Free Software
+Foundation, in version 2 as it comes in the "COPYING" file of the
+VirtualBox OSE distribution. VirtualBox OSE is distributed in the
+hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
+
+The contents of this file may alternatively be used under the terms
+of the Common Development and Distribution License Version 1.0
+(CDDL) only, as it comes in the "COPYING.CDDL" file of the
+VirtualBox OSE distribution, in which case the provisions of the
+CDDL are applicable instead of those of the GPL.
+
+You may elect to license modified versions of this file under the
+terms and conditions of either the GPL or the CDDL or both.
+"""
+__version__ = "$Revision$"
+
+
+# Standard python imports
+import sys;
+import os;
+from optparse import OptionParser;
+
+# Add Test Manager's modules path
+g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
+sys.path.append(g_ksTestManagerDir);
+
+# Test Manager imports
+from testmanager.core.db            import TMDatabaseConnection;
+from testmanager.core.testbox       import TestBoxLogic, TestBoxData;
+from testmanager.core.testset       import TestSetLogic, TestSetData;
+from testmanager.core.testresults   import TestResultLogic;
+from testmanager.core.useraccount   import UserAccountLogic;
+
+
+class VirtualTestSheriff(object): # pylint: disable=R0903
+    """
+    Add build info into Test Manager database.
+    """
+
+    ## The user account for the virtual sheriff.
+    ksLoginName = 'vsheriff';
+
+
+    def __init__(self):
+        """
+        Parse command line.
+        """
+        self.oDb = None;
+        self.tsNow = None;
+        self.oResultLogic = None;
+        self.oTestSetLogic = None;
+        self.oLogin = None;
+        self.uidSelf = -1;
+
+        oParser = OptionParser();
+        oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0,
+                           help = 'When to start specified as hours relative to current time.  Defauls is right now.', );
+        oParser.add_option('--hours-period', dest = 'cPeriodInHours', metavar = '<period-in-hours>', default = 2,
+                           help = 'Work period specified in hours.  Defauls is 2 hours.');
+        oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
+                           help = 'Whether to commit the findings to the database. Default is a dry run.');
+        oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
+                           help = 'Quiet execution');
+        oParser.add_option('-l', '--log', dest = 'sBuildLogPath', metavar = '<url>',
+                           help = 'URL to the build logs (optional).');
+        oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
+                           help = 'Enables debug mode.');
+
+        (self.oConfig, _) = oParser.parse_args();
+
+    def eprint(self, sText):
+        """
+        Prints error messages.
+        Returns 1 (for exit code usage.)
+        """
+        print 'error: %s' % (sText,);
+        return 1;
+
+    def dprint(self, sText):
+        """
+        Prints debug info.
+        """
+        if not self.oConfig.fDebug:
+            print 'debug: %s' % (sText, );
+        return 0;
+
+    def vprint(self, sText):
+        """
+        Prints verbose info.
+        """
+        if not self.oConfig.fQuiet:
+            print 'info: %s' % (sText,);
+        return 0;
+
+
+    def badTestBoxManagement(self):
+        """
+        Looks for bad test boxes and first tries once to reboot them then disables them.
+        """
+        rcExit = 0;
+
+        #
+        # We skip this entirely if we're running in the past and not in harmless debug mode.
+        #
+        if    self.oConfig.cStartHoursAgo != 0 \
+          and (not self.oConfig.fDebug or self.oConfig.fRealRun):
+            return rcExit;
+        tsNow = self.oConfig.tsNow if self.oConfig.fDebug else None;
+        oTestBoxLogic = TestBoxLogic(self.oDb);
+
+        #
+        # Get list of bad test boxes for given period and check them out individually.
+        #
+        aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds();
+        for idTestBox in aidBadTestBoxes:
+            # Skip if the testbox is already disabled or has a pending reboot command.
+            try:
+                oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
+            except Exception as oXcpt:
+                rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
+                continue;
+            if not oTestBox.fEnabled:
+                self.dprint('badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
+                            % ( idTestBox, oTestBox.sName, ));
+                continue;
+            if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
+                self.dprint('badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
+                            % ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
+                continue;
+
+            # Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
+            aoSets  = self.oTestSetLogic.fetchResultForTestBox(idTestBox, cHoursBack = 2, tsNow = tsNow);
+            cOkay      = 0;
+            cBad       = 0;
+            iFirstOkay = len(aoSets);
+            for iSet, oSet in enumerate(aoSets):
+                if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
+                    cBad += 1;
+                else:
+                    ## @todo maybe check the elapsed time here, it could still be a bad run.
+                    iFirstOkay = iSet;
+                    cOkay += 1;
+                if iSet > 10:
+                    break;
+
+            # We react if there are two or more bad-testbox statuses at the head of the
+            # history and at least three in the last 10 results.
+            if iFirstOkay >= 2 and cBad > 2:
+                if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = 2, tsNow = tsNow):
+                    self.vprint('Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u'
+                                % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+                    if self.oConfig.fRealRun:
+                        try:
+                            oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
+                                                         sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
+                                                                  % (iFirstOkay, cBad, cOkay),);
+                        except Exception as oXcpt:
+                            rcExit = self.eprint('Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
+                else:
+                    self.vprint('Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u'
+                                % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+                    if self.oConfig.fRealRun:
+                        try:
+                            oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
+                                                        sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
+                                                                 % (iFirstOkay, cBad, cOkay),);
+                        except Exception as oXcpt:
+                            rcExit = self.eprint('Error rebooting testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
+        return rcExit;
+
+
+    def reasoningFailures(self):
+        """
+        Guess the reason for failures.
+        """
+
+        return 0;
+
+
+    def main(self):
+        """
+        The 'main' function.
+        Return exit code (0, 1, etc).
+        """
+        # Database stuff.
+        self.oDb = TMDatabaseConnection()
+        self.oResultLogic = TestResultLogic(self.oDb);
+        self.oTestSetLogic = TestSetLogic(self.oDb);
+
+        # Get a fix on our 'now' before we do anything..
+        self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
+        self.tsNow = self.oDb.fetchOne();
+
+        # If we're suppost to commit anything we need to get our user ID.
+        rcExit = 0;
+        if self.oConfig.fRealRun:
+            self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
+            if self.oLogin is None:
+                rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
+            else:
+                self.uidSelf = self.oLogin.uid;
+
+        if rcExit == 0:
+            # Do the stuff.
+            rcExit  = self.badTestBoxManagement();
+            rcExit2 = self.reasoningFailures();
+            if rcExit == 0:
+                rcExit = rcExit2;
+
+        # Cleanup.
+        self.oTestSetLogic = None;
+        self.oResultLogic = None;
+        self.oDb.close();
+        self.oDb = None;
+        return rcExit;
+
+if __name__ == '__main__':
+    sys.exit(VirtualTestSheriff().main());
+
Index: trunk/src/VBox/ValidationKit/testmanager/core/base.py
===================================================================
--- trunk/src/VBox/ValidationKit/testmanager/core/base.py	(revision 61280)
+++ trunk/src/VBox/ValidationKit/testmanager/core/base.py	(revision 61282)
@@ -89,4 +89,13 @@
     """
     Database row is in use and cannot be deleted.
+    Used by ModelLogicBase decendants.
+    """
+    pass;
+
+
+class TMInFligthCollision(TMExceptionBase):
+    """
+    Database update failed because someone else had already made changes to
+    the data there.
     Used by ModelLogicBase decendants.
     """
@@ -1159,4 +1168,24 @@
         return self._oDb;
 
+    def _dbRowsToModelDataList(self, oModelDataType, aaoRows = None):
+        """
+        Helper for conerting a simple fetch into a list of ModelDataType python objects.
+
+        If aaoRows is None, we'll fetchAll from the database ourselves.
+
+        The oModelDataType must be a class derived from ModelDataBase and implement
+        the initFormDbRow method.
+
+        Returns a list of oModelDataType instances.
+        """
+        assert issubclass(oModelDataType, ModelDataBase);
+        aoRet = [];
+        if aaoRows is None:
+            aaoRows = self._oDb.fetchAll();
+        for aoRow in aaoRows:
+            aoRet.append(oModelDataType().initFromDbRow(aoRow));
+        return aoRet;
+
+
 
 class AttributeChangeEntry(object): # pylint: disable=R0903
Index: trunk/src/VBox/ValidationKit/testmanager/core/testbox.py
===================================================================
--- trunk/src/VBox/ValidationKit/testmanager/core/testbox.py	(revision 61280)
+++ trunk/src/VBox/ValidationKit/testmanager/core/testbox.py	(revision 61282)
@@ -34,5 +34,5 @@
 
 # Validation Kit imports.
-from testmanager.core.base  import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase, \
+from testmanager.core.base  import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMExceptionBase, TMInFligthCollision, \
                                    TMInvalidData, TMTooManyRows, TMRowNotFound, ChangeLogEntry, AttributeChangeEntry;
 
@@ -731,9 +731,11 @@
         return idGenTestBox;
 
-    def setCommand(self, idTestBox, sOldCommand, sNewCommand, uidAuthor = None, fCommit = False):
+    def setCommand(self, idTestBox, sOldCommand, sNewCommand, uidAuthor = None, fCommit = False, sComment = None,
+                   fNoRollbackOnInFlightCollision = False):
         """
         Sets or resets the pending command on a testbox.
         Returns (idGenTestBox, tsEffective) of the new row.
         """
+        _ = sComment;
         try:
             # Would be easier to do this using an insert or update hook, I think. Much easier.
@@ -745,4 +747,6 @@
                               'RETURNING tsExpire\n',
                               (idTestBox, sOldCommand,));
+            if self._oDb.getRowCount() == 0:
+                raise TMInFligthCollision();
             tsEffective = self._oDb.fetchOne()[0];
 
@@ -825,4 +829,9 @@
             if fCommit is True:
                 self._oDb.commit();
+
+        except TMInFligthCollision: # This is pretty stupid, but don't want to touch testboxcontroller.py now.
+            if not fNoRollbackOnInFlightCollision:
+                self._oDb.rollback();
+            raise;
         except:
             self._oDb.rollback();
@@ -879,4 +888,63 @@
 
 
+    #
+    # The virtual test sheriff interface.
+    #
+
+    def hasTestBoxRecentlyBeenRebooted(self, idTestBox, cHoursBack = 2, tsNow = None):
+        """
+        Checks if the testbox has been rebooted in the specified time period.
+
+        This does not include already pending reboots, though under some
+        circumstances it may.  These being the test box entry being edited for
+        other reasons.
+
+        Returns True / False.
+        """
+        if tsNow is None:
+            tsNow = self._oDb.getCurrentTimestamp();
+        self._oDb.execute('SELECT COUNT(idTestBox)\n'
+                          'FROM   TestBoxes\n'
+                          'WHERE  idTestBox      = %s\n'
+                          '   AND tsExpire       < %s\n'
+                          '   AND tsExpire      >= %s - interval \'%u hours\'\n'
+                          '   AND enmPendingCmd IN (%s, %s)\n'
+                          , ( idTestBox, tsNow, tsNow, cHoursBack,
+                              TestBoxData.ksTestBoxCmd_Reboot, TestBoxData.ksTestBoxCmd_UpgradeAndReboot, ));
+        return self._oDb.fetchOne()[0] > 0;
+
+
+    def rebootTestBox(self, idTestBox, uidAuthor, sComment, sOldCommand = TestBoxData.ksTestBoxCmd_None, fCommit = False):
+        """
+        Issues a reboot command for the given test box.
+        Return True on succes, False on in-flight collision.
+        May raise DB exception with rollback on other trouble.
+        """
+        try:
+            self.setCommand(idTestBox, sOldCommand, TestBoxData.ksTestBoxCmd_Reboot,
+                            uidAuthor = uidAuthor, fCommit = fCommit, sComment = sComment,
+                            fNoRollbackOnInFlightCollision = True);
+        except TMInFligthCollision:
+            return False;
+        except:
+            raise;
+        return True;
+
+
+    def disableTestBox(self, idTestBox, uidAuthor, sComment, fCommit = False):
+        """
+        Disables the given test box.
+
+        Raises exception on trouble, without rollback.
+        """
+        oTestBox = TestBoxData().initFromDbWithId(self._oDb, idTestBox);
+        if oTestBox.fEnabled:
+            oTestBox.fEnabled = False;
+            if sComment is not None:
+                _ = sComment; # oTestBox.sComment = sComment;
+            self.editEntry(oTestBox, uidAuthor = uidAuthor, fCommit = fCommit);
+        return None;
+
+
 #
 # Unit testing.
Index: trunk/src/VBox/ValidationKit/testmanager/core/testresults.py
===================================================================
--- trunk/src/VBox/ValidationKit/testmanager/core/testresults.py	(revision 61280)
+++ trunk/src/VBox/ValidationKit/testmanager/core/testresults.py	(revision 61282)
@@ -1857,4 +1857,6 @@
 
 
+
+
 #
 # Unit testing.
Index: trunk/src/VBox/ValidationKit/testmanager/core/testset.py
===================================================================
--- trunk/src/VBox/ValidationKit/testmanager/core/testset.py	(revision 61280)
+++ trunk/src/VBox/ValidationKit/testmanager/core/testset.py	(revision 61282)
@@ -229,4 +229,6 @@
         return oFile;
 
+
+
 class TestSetLogic(ModelLogicBase):
     """
@@ -637,4 +639,43 @@
 
 
+    #
+    # The virtual test sheriff interface.
+    #
+
+    def fetchBadTestBoxIds(self, cHoursBack = 2, tsNow = None):
+        """
+        Fetches a list of test box IDs which returned bad-testbox statuses in the
+        given period (tsDone).
+        """
+        if tsNow is None:
+            tsNow = self._oDb.getCurrentTimestamp();
+        self._oDb.execute('SELECT DISTINCT idTestBox\n'
+                          'FROM   TestSets\n'
+                          'WHERE  TestSets.enmStatus = \'bad-testbox\'\n'
+                          '   AND tsDone           <= %s\n'
+                          '   AND tsDone            > (%s - interval \'%s hours\')\n'
+                          , ( tsNow, tsNow, cHoursBack,));
+        return [aoRow[0] for aoRow in self._oDb.fetchAll()];
+
+    def fetchResultForTestBox(self, idTestBox, cHoursBack = 2, tsNow = None):
+        """
+        Fetches the TestSet rows for idTestBox for the given period (tsDone), w/o running ones.
+
+        Returns list of TestSetData sorted by tsDone in descending order.
+        """
+        if tsNow is None:
+            tsNow = self._oDb.getCurrentTimestamp();
+        self._oDb.execute('SELECT *\n'
+                          'FROM   TestSets\n'
+                          'WHERE  TestSets.idTestBox = %s\n'
+                          '   AND tsDone IS NOT NULL\n'
+                          '   AND tsDone           <= %s\n'
+                          '   AND tsDone            > (%s - interval \'%s hours\')\n'
+                          'ORDER by tsDone DESC\n'
+                          , ( idTestBox, tsNow, tsNow, cHoursBack,));
+        return self._dbRowsToModelDataList(TestSetData);
+
+
+
 #
 # Unit testing.
