Changeset 77645 in vbox for trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py

Timestamp:

Mar 11, 2019 10:03:19 AM (6 years ago)

Author:

vboxsync

Message:

vsheriff: enable automatic rebooting/disabling of testboxes which are failing a lot, with email alert

File:

• trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py (modified) (3 diffs)

trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py

@@ -480 +480 @@
             # history and at least three in the last 10 results.
             if iFirstOkay >= 2 and cBad > 2:
-                # Frank: For now don't reboot boxes automatically
-                if True or oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
-                    self.vprint(u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u'
-                                % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+                if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
+                    sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
+                             % (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
+                    self.vprint(sComment);
+                    self.sendEmailAlert(self.uidSelf, sComment);
                     if self.oConfig.fRealRun is True:
                         try:
@@ -492 +493 @@
                             rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
                 else:
-                    self.vprint(u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u'
-                                % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+                    sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
+                             % (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
+                    self.vprint(sComment);
+                    self.sendEmailAlert(self.uidSelf, sComment);
                     if self.oConfig.fRealRun is True:
                         try:
@@ -504 +507 @@
                 self.dprint(u'badTestBoxManagement: #%u (%s) looks ok:  iFirstOkay=%u cBad=%u cOkay=%u'
                             % ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
+
         #
         # Reset hanged testboxes