virtual_test_sheriff.py@ 78717

Last change on this file since 78717 was 78717, checked in by vboxsync, 6 years ago
vsheriff: Detect a sporadic hardening error.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 71.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: virtual_test_sheriff.py 78717 2019-05-24 11:24:07Z vboxsync $
4	# pylint: disable=C0301
5
6	"""
7	Virtual Test Sheriff.
8
9	Duties:
10	- Try to a assign failure reasons to recently failed tests.
11	- Reboot or disable bad test boxes.
12
13	"""
14
15	from __future__ import print_function;
16
17	__copyright__ = \
18	"""
19	Copyright (C) 2012-2019 Oracle Corporation
20
21	This file is part of VirtualBox Open Source Edition (OSE), as
22	available from http://www.virtualbox.org. This file is free software;
23	you can redistribute it and/or modify it under the terms of the GNU
24	General Public License (GPL) as published by the Free Software
25	Foundation, in version 2 as it comes in the "COPYING" file of the
26	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
27	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
32	VirtualBox OSE distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37	"""
38	__version__ = "$Revision: 78717 $"
39
40
41	# Standard python imports
42	import sys;
43	import os;
44	import hashlib;
45	import subprocess;
46	import smtplib
47	from email.mime.multipart import MIMEMultipart
48	from email.mime.text import MIMEText
49	from email.utils import COMMASPACE
50
51	if sys.version_info[0] >= 3:
52	from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module
53	else:
54	from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module
55	from optparse import OptionParser; # pylint: disable=deprecated-module
56	from PIL import Image; # pylint: disable=import-error
57
58	# Add Test Manager's modules path
59	g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
60	sys.path.append(g_ksTestManagerDir);
61
62	# Test Manager imports
63	from testmanager.core.db import TMDatabaseConnection;
64	from testmanager.core.build import BuildDataEx;
65	from testmanager.core.failurereason import FailureReasonLogic;
66	from testmanager.core.testbox import TestBoxLogic, TestBoxData;
67	from testmanager.core.testcase import TestCaseDataEx;
68	from testmanager.core.testgroup import TestGroupData;
69	from testmanager.core.testset import TestSetLogic, TestSetData;
70	from testmanager.core.testresults import TestResultLogic, TestResultFileData;
71	from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData;
72	from testmanager.core.useraccount import UserAccountLogic;
73	from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \
74	g_ksAlertSubject, g_asAlertList, g_ksLomPassword;
75
76	# Python 3 hacks:
77	if sys.version_info[0] >= 3:
78	xrange = range; # pylint: disable=redefined-builtin,invalid-name
79
80
81	class VirtualTestSheriffCaseFile(object):
82	"""
83	A failure investigation case file.
84
85	"""
86
87
88	## Max log file we'll read into memory. (256 MB)
89	kcbMaxLogRead = 0x10000000;
90
91	def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase):
92	self.oSheriff = oSheriff;
93	self.oTestSet = oTestSet; # TestSetData
94	self.oTree = oTree; # TestResultDataEx
95	self.oBuild = oBuild; # BuildDataEx
96	self.oTestBox = oTestBox; # TestBoxData
97	self.oTestGroup = oTestGroup; # TestGroupData
98	self.oTestCase = oTestCase; # TestCaseDataEx
99	self.sMainLog = ''; # The main log file. Empty string if not accessible.
100
101	# Generate a case file name.
102	self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,)
103	self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \
104	% ( self.oTestSet.idTestSet,
105	self.oTestCase.sName,
106	self.oTestBox.sName,
107	self.oTestBox.sOs,
108	self.oTestBox.sOsVersion,
109	self.oTestBox.sCpuArch,
110	self.oTestBox.sCpuName,
111	self.oTestBox.sCpuVendor,
112	self.oBuild.oCat.sProduct,
113	self.oBuild.oCat.sBranch,
114	self.oBuild.oCat.sType,
115	self.oBuild.iRevision, );
116
117	# Investigation notes.
118	self.tReason = None; # None or one of the ktReason_XXX constants.
119	self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult.
120	self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult.
121
122	#
123	# Reason.
124	#
125
126	def noteReason(self, tReason):
127	""" Notes down a possible reason. """
128	self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,));
129	self.tReason = tReason;
130	return True;
131
132	def noteReasonForId(self, tReason, idTestResult, sComment = None):
133	""" Notes down a possible reason for a specific test result. """
134	self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s'
135	% (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason,
136	(u' (%s)' % (sComment,)) if sComment is not None else ''));
137	self.dReasonForResultId[idTestResult] = tReason;
138	if sComment is not None:
139	self.dCommentForResultId[idTestResult] = sComment;
140	return True;
141
142
143	#
144	# Test classification.
145	#
146
147	def isVBoxTest(self):
148	""" Test classification: VirtualBox (using the build) """
149	return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ];
150
151	def isVBoxUnitTest(self):
152	""" Test case classification: The unit test doing all our testcase/*.cpp stuff. """
153	return self.isVBoxTest() \
154	and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests');
155
156	def isVBoxInstallTest(self):
157	""" Test case classification: VirtualBox Guest installation test. """
158	return self.isVBoxTest() \
159	and self.oTestCase.sName.lower().startswith('install:');
160
161	def isVBoxUSBTest(self):
162	""" Test case classification: VirtualBox USB test. """
163	return self.isVBoxTest() \
164	and self.oTestCase.sName.lower().startswith('usb:');
165
166	def isVBoxStorageTest(self):
167	""" Test case classification: VirtualBox Storage test. """
168	return self.isVBoxTest() \
169	and self.oTestCase.sName.lower().startswith('storage:');
170
171	def isVBoxGAsTest(self):
172	""" Test case classification: VirtualBox Guest Additions test. """
173	return self.isVBoxTest() \
174	and self.oTestCase.sName.lower().startswith('ga\'s tests');
175
176	def isVBoxAPITest(self):
177	""" Test case classification: VirtualBox API test. """
178	return self.isVBoxTest() \
179	and self.oTestCase.sName.lower().startswith('api:');
180
181	def isVBoxBenchmarkTest(self):
182	""" Test case classification: VirtualBox Benchmark test. """
183	return self.isVBoxTest() \
184	and self.oTestCase.sName.lower().startswith('benchmark:');
185
186	def isVBoxSmokeTest(self):
187	""" Test case classification: Smoke test. """
188	return self.isVBoxTest() \
189	and self.oTestCase.sName.lower().startswith('smoketest');
190
191
192	#
193	# Utility methods.
194	#
195
196	def getMainLog(self):
197	"""
198	Tries to read the main log file since this will be the first source of information.
199	"""
200	if self.sMainLog:
201	return self.sMainLog;
202	(oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb');
203	if oFile is not None:
204	try:
205	self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
206	except Exception as oXcpt:
207	self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,))
208	self.sMainLog = '';
209	else:
210	self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,));
211	return self.sMainLog;
212
213	def getLogFile(self, oFile):
214	"""
215	Tries to read the given file as a utf-8 log file.
216	oFile is a TestFileDataEx instance.
217	Returns empty string if problems opening or reading the file.
218	"""
219	sContent = '';
220	(oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
221	if oFile is not None:
222	try:
223	sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
224	except Exception as oXcpt:
225	self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,))
226	else:
227	self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,));
228	return sContent;
229
230	def getScreenshotSha256(self, oFile):
231	"""
232	Tries to read the given screenshot file, uncompress it, and do SHA-2
233	on the raw pixels.
234	Returns SHA-2 digest string on success, None on failure.
235	"""
236	(oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
237	try:
238	abImageFile = oImgFile.read();
239	except Exception as oXcpt:
240	self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,))
241	else:
242	try:
243	oImage = Image.open(StringIO(abImageFile));
244	except Exception as oXcpt:
245	self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,))
246	else:
247	try:
248	oHash = hashlib.sha256();
249	oHash.update(oImage.tostring());
250	except Exception as oXcpt:
251	self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,))
252	else:
253	return oHash.hexdigest();
254	return None;
255
256
257
258	def isSingleTestFailure(self):
259	"""
260	Figure out if this is a single test failing or if it's one of the
261	more complicated ones.
262	"""
263	if self.oTree.cErrors == 1:
264	return True;
265	if self.oTree.deepCountErrorContributers() <= 1:
266	return True;
267	return False;
268
269
270
271	class VirtualTestSheriff(object): # pylint: disable=R0903
272	"""
273	Add build info into Test Manager database.
274	"""
275
276	## The user account for the virtual sheriff.
277	ksLoginName = 'vsheriff';
278
279	def __init__(self):
280	"""
281	Parse command line.
282	"""
283	self.oDb = None;
284	self.tsNow = None;
285	self.oTestResultLogic = None;
286	self.oTestSetLogic = None;
287	self.oFailureReasonLogic = None; # FailureReasonLogic;
288	self.oTestResultFailureLogic = None; # TestResultFailureLogic
289	self.oLogin = None;
290	self.uidSelf = -1;
291	self.oLogFile = None;
292	self.asBsodReasons = [];
293	self.asUnitTestReasons = [];
294
295	oParser = OptionParser();
296	oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int',
297	help = 'When to start specified as hours relative to current time. Defauls is right now.', );
298	oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int',
299	help = 'Work period specified in hours. Defauls is 2 hours.');
300	oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
301	help = 'Whether to commit the findings to the database. Default is a dry run.');
302	oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
303	help = 'Quiet execution');
304	oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None,
305	help = 'Where to log messages.');
306	oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
307	help = 'Enables debug mode.');
308
309	(self.oConfig, _) = oParser.parse_args();
310
311	if self.oConfig.sLogFile:
312	self.oLogFile = open(self.oConfig.sLogFile, "a");
313	self.oLogFile.write('VirtualTestSheriff: $Revision: 78717 $ \n');
314
315
316	def eprint(self, sText):
317	"""
318	Prints error messages.
319	Returns 1 (for exit code usage.)
320	"""
321	print('error: %s' % (sText,));
322	if self.oLogFile is not None:
323	self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8'));
324	return 1;
325
326	def dprint(self, sText):
327	"""
328	Prints debug info.
329	"""
330	if self.oConfig.fDebug:
331	if not self.oConfig.fQuiet:
332	print('debug: %s' % (sText, ));
333	if self.oLogFile is not None:
334	self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8'));
335	return 0;
336
337	def vprint(self, sText):
338	"""
339	Prints verbose info.
340	"""
341	if not self.oConfig.fQuiet:
342	print('info: %s' % (sText,));
343	if self.oLogFile is not None:
344	self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8'));
345	return 0;
346
347	def getFailureReason(self, tReason):
348	""" Gets the failure reason object for tReason. """
349	return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]);
350
351	def selfCheck(self):
352	""" Does some self checks, looking up things we expect to be in the database and such. """
353	rcExit = 0;
354	for sAttr in dir(self.__class__):
355	if sAttr.startswith('ktReason_'):
356	tReason = getattr(self.__class__, sAttr);
357	oFailureReason = self.getFailureReason(tReason);
358	if oFailureReason is None:
359	rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!'
360	% (tReason[1], tReason[0],));
361
362	# Check the user account as well.
363	if self.oLogin is None:
364	oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
365	if oLogin is None:
366	rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
367	return rcExit;
368
369	def sendEmailAlert(self, uidAuthor, sBodyText):
370	"""
371	Sends email alert.
372	"""
373
374	# Get author email
375	self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,));
376	sFrom = self.oDb.fetchOne();
377	if sFrom is not None:
378	sFrom = sFrom[0];
379	else:
380	sFrom = g_ksAlertFrom;
381
382	# Gather recipient list.
383	asEmailList = [];
384	for sUser in g_asAlertList:
385	self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,));
386	sEmail = self.oDb.fetchOne();
387	if sEmail:
388	asEmailList.append(sEmail[0]);
389	if not asEmailList:
390	return self.eprint('No email addresses to send alter to!');
391
392	# Compose the message.
393	oMsg = MIMEMultipart();
394	oMsg['From'] = sFrom;
395	oMsg['To'] = COMMASPACE.join(asEmailList);
396	oMsg['Subject'] = g_ksAlertSubject;
397	oMsg.attach(MIMEText(sBodyText, 'plain'))
398
399	# Try send it.
400	try:
401	oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort);
402	oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string())
403	oSMTP.quit()
404	except smtplib.SMTPException as oXcpt:
405	return self.eprint('Failed to send mail: %s' % (oXcpt,));
406
407	return 0;
408
409	def badTestBoxManagement(self):
410	"""
411	Looks for bad test boxes and first tries once to reboot them then disables them.
412	"""
413	rcExit = 0;
414
415	#
416	# We skip this entirely if we're running in the past and not in harmless debug mode.
417	#
418	if self.oConfig.cStartHoursAgo != 0 \
419	and (not self.oConfig.fDebug or self.oConfig.fRealRun):
420	return rcExit;
421	tsNow = self.tsNow if self.oConfig.fDebug else None;
422	cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2;
423	oTestBoxLogic = TestBoxLogic(self.oDb);
424
425	#
426	# Generate a list of failures reasons we consider bad-testbox behavior.
427	#
428	aidFailureReasons = [
429	self.getFailureReason(self.ktReason_Host_DriverNotLoaded).idFailureReason,
430	self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason,
431	self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason,
432	self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason,
433	];
434
435	#
436	# Get list of bad test boxes for given period and check them out individually.
437	#
438	aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow,
439	aidFailureReasons = aidFailureReasons);
440	for idTestBox in aidBadTestBoxes:
441	# Skip if the testbox is already disabled or has a pending reboot command.
442	try:
443	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
444	except Exception as oXcpt:
445	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
446	continue;
447	if not oTestBox.fEnabled:
448	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
449	% ( idTestBox, oTestBox.sName, ));
450	continue;
451	if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
452	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
453	% ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
454	continue;
455
456	# Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
457	aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow);
458	cOkay = 0;
459	cBad = 0;
460	iFirstOkay = len(aoSets);
461	for iSet, oSet in enumerate(aoSets):
462	if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
463	cBad += 1;
464	else:
465	# Check for bad failure reasons.
466	oFailure = None;
467	if oSet.enmStatus in TestSetData.kasBadTestStatuses:
468	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oSet.idTestSet)
469	aoFailedResults = oTree.getListOfFailures();
470	for oFailedResult in aoFailedResults:
471	oFailure = self.oTestResultFailureLogic.getById(oFailedResult.idTestResult);
472	if oFailure is not None and oFailure.idFailureReason in aidFailureReasons:
473	break;
474	else:
475	oFailure = None;
476	if oFailure is not None:
477	cBad += 1;
478	else:
479	# This is an okay test result then.
480	## @todo maybe check the elapsed time here, it could still be a bad run?
481	cOkay += 1;
482	if iFirstOkay > iSet:
483	iFirstOkay = iSet;
484	if iSet > 10:
485	break;
486
487	# We react if there are two or more bad-testbox statuses at the head of the
488	# history and at least three in the last 10 results.
489	if iFirstOkay >= 2 and cBad > 2:
490	if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
491	sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
492	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
493	self.vprint(sComment);
494	self.sendEmailAlert(self.uidSelf, sComment);
495	if self.oConfig.fRealRun is True:
496	try:
497	oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
498	sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
499	% (iFirstOkay, cBad, cOkay),);
500	except Exception as oXcpt:
501	rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
502	else:
503	sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
504	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
505	self.vprint(sComment);
506	self.sendEmailAlert(self.uidSelf, sComment);
507	if self.oConfig.fRealRun is True:
508	try:
509	oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
510	sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
511	% (iFirstOkay, cBad, cOkay),);
512	except Exception as oXcpt:
513	rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
514	else:
515	self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u'
516	% ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
517
518	#
519	# Reset hanged testboxes
520	#
521	cStatusTimeoutMins = 10;
522
523	self.oDb.execute('SELECT TestBoxStatuses.idTestBox\n'
524	' FROM TestBoxStatuses, TestBoxes\n'
525	' WHERE TestBoxStatuses.tsUpdated >= (CURRENT_TIMESTAMP - interval \'%s hours\')\n'
526	' AND TestBoxStatuses.tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')\n'
527	' AND TestBoxStatuses.idTestBox = TestBoxes.idTestBox\n'
528	' AND Testboxes.tsExpire = \'infinity\'::timestamp', (cHoursBack,cStatusTimeoutMins));
529	for idTestBox in self.oDb.fetchAll():
530	idTestBox = idTestBox[0];
531	try:
532	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
533	except Exception as oXcpt:
534	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
535	continue;
536	# Skip if the testbox is already disabled, already reset or there's no iLOM
537	if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0:
538	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
539	% ( idTestBox, oTestBox.sName, ));
540	continue;
541	## @todo get iLOM credentials from a table?
542	sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,);
543	try:
544	oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True);
545	sStdout = oPs.communicate()[0];
546	iRC = oPs.wait();
547
548	oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,);
549	oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True);
550
551	sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout);
552	self.vprint(sComment);
553	self.sendEmailAlert(self.uidSelf, sComment);
554
555	except Exception as oXcpt:
556	rcExit = self.eprint(u'Error resetting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
557
558	return rcExit;
559
560
561	## @name Failure reasons we know.
562	## @{
563	ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' );
564	ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' );
565	ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' );
566	ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' );
567	ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' );
568	ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' );
569	ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' );
570	ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' );
571	ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' );
572	ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' );
573	ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' );
574	ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' );
575	ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' );
576	ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' );
577	ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' );
578	ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' );
579	ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' );
580	ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' );
581	ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' );
582	ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' );
583	ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' );
584	ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' );
585	ktReason_Host_Install_Hang = ( 'Host', 'Install hang' );
586	ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' );
587	ktReason_Host_TSTInfo_Accuracy_OOR = ( 'Host', 'TSTInfo accuracy out of range' );
588	ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' );
589	ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' );
590	ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' );
591	ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' );
592	ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' );
593	ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' );
594	ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' );
595	ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' );
596	ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' );
597	ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' );
598	ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' );
599	ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' );
600	ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' );
601	ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' );
602	ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' );
603	ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' );
604	ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' );
605	ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' );
606	ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' );
607	ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' );
608	ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' );
609	ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' );
610	ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' );
611	ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' );
612	ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' );
613	ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' );
614	## @}
615
616	## BSOD category.
617	ksBsodCategory = 'BSOD';
618	## Special reason indicating that the flesh and blood sheriff has work to do.
619	ksBsodAddNew = 'Add new BSOD';
620
621	## Unit test category.
622	ksUnitTestCategory = 'Unit';
623	## Special reason indicating that the flesh and blood sheriff has work to do.
624	ksUnitTestAddNew = 'Add new';
625
626	## Used for indica that we shouldn't report anything for this test result ID and
627	## consider promoting the previous error to test set level if it's the only one.
628	ktHarmless = ( 'Probably', 'Caused by previous error' );
629
630
631	def caseClosed(self, oCaseFile):
632	"""
633	Reports the findings in the case and closes it.
634	"""
635	#
636	# Log it and create a dReasonForReasultId we can use below.
637	#
638	dCommentForResultId = oCaseFile.dCommentForResultId;
639	if oCaseFile.dReasonForResultId:
640	# Must weed out ktHarmless.
641	dReasonForResultId = {};
642	for idKey, tReason in oCaseFile.dReasonForResultId.items():
643	if tReason is not self.ktHarmless:
644	dReasonForResultId[idKey] = tReason;
645	if not dReasonForResultId:
646	self.vprint(u'TODO: Closing %s without a real reason, only %s.'
647	% (oCaseFile.sName, oCaseFile.dReasonForResultId));
648	return False;
649
650	# Try promote to single reason.
651	atValues = dReasonForResultId.values();
652	fSingleReason = True;
653	if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult:
654	self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],));
655	elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]):
656	self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0]));
657	else:
658	fSingleReason = False;
659	if fSingleReason:
660	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], };
661	if dCommentForResultId:
662	dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], };
663	elif oCaseFile.tReason is not None:
664	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, };
665	else:
666	self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,));
667	return False;
668
669	self.vprint(u'Closing %s with following reason%s: %s'
670	% ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, ));
671
672	#
673	# Add the test failure reason record(s).
674	#
675	for idTestResult, tReason in dReasonForResultId.items():
676	oFailureReason = self.getFailureReason(tReason);
677	if oFailureReason is not None:
678	sComment = 'Set by $Revision: 78717 $' # Handy for reverting later.
679	if idTestResult in dCommentForResultId:
680	sComment += ': ' + dCommentForResultId[idTestResult];
681
682	oAdd = TestResultFailureData();
683	oAdd.initFromValues(idTestResult = idTestResult,
684	idFailureReason = oFailureReason.idFailureReason,
685	uidAuthor = self.uidSelf,
686	idTestSet = oCaseFile.oTestSet.idTestSet,
687	sComment = sComment,);
688	if self.oConfig.fRealRun:
689	try:
690	self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True);
691	except Exception as oXcpt:
692	self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s'
693	% (oXcpt, oAdd, oCaseFile.sLongName,));
694	else:
695	self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],));
696	return True;
697
698	#
699	# Tools for assiting log parsing.
700	#
701
702	@staticmethod
703	def matchFollowedByLines(sStr, off, asFollowingLines):
704	""" Worker for isThisFollowedByTheseLines. """
705
706	# Advance off to the end of the line.
707	off = sStr.find('\n', off);
708	if off < 0:
709	return False;
710	off += 1;
711
712	# Match each string with the subsequent lines.
713	for iLine, sLine in enumerate(asFollowingLines):
714	offEnd = sStr.find('\n', off);
715	if offEnd < 0:
716	return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0;
717	if sLine and sStr.find(sLine, off, offEnd) < 0:
718	return False;
719
720	# next line.
721	off = offEnd + 1;
722
723	return True;
724
725	@staticmethod
726	def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines):
727	"""
728	Looks for a line contining sFirst which is then followed by lines
729	with the strings in asFollowingLines. (No newline chars anywhere!)
730	Returns True / False.
731	"""
732	off = sStr.find(sFirst, 0);
733	while off >= 0:
734	if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines):
735	return True;
736	off = sStr.find(sFirst, off + 1);
737	return False;
738
739	@staticmethod
740	def findAndReturnRestOfLine(sHaystack, sNeedle):
741	"""
742	Looks for sNeedle in sHaystack.
743	Returns The text following the needle up to the end of the line.
744	Returns None if not found.
745	"""
746	if sHaystack is None:
747	return None;
748	off = sHaystack.find(sNeedle);
749	if off < 0:
750	return None;
751	off += len(sNeedle)
752	offEol = sHaystack.find('\n', off);
753	if offEol < 0:
754	offEol = len(sHaystack);
755	return sHaystack[off:offEol]
756
757	@staticmethod
758	def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle):
759	"""
760	Looks for sNeedle in zeroe or more haystacks (asHaystack).
761	Returns The text following the first needed found up to the end of the line.
762	Returns None if not found.
763	"""
764	for sHaystack in asHaystacks:
765	sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle);
766	if sRet is not None:
767	return sRet;
768	return None;
769
770
771	#
772	# The investigative units.
773	#
774
775	katSimpleInstallUninstallMainLogReasons = [
776	# ( Whether to stop on hit, reason tuple, needle text. )
777	( False, ktReason_Host_LeftoverService,
778	'SERVICE_NAME: vbox' ),
779	];
780
781	kdatSimpleInstallUninstallMainLogReasonsPerOs = {
782	'darwin': [
783	# ( Whether to stop on hit, reason tuple, needle text. )
784	( True, ktReason_Host_DriverNotUnloading,
785	'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ),
786	],
787	'linux': [
788	# ( Whether to stop on hit, reason tuple, needle text. )
789	( True, ktReason_Host_DriverNotCompilable,
790	'This system is not currently set up to build kernel modules' ),
791	( True, ktReason_Host_DriverNotCompilable,
792	'This system is currently not set up to build kernel modules' ),
793	( True, ktReason_Host_InstallationFailed,
794	'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ),
795	( True, ktReason_Host_DriverNotUnloading,
796	'Cannot unload module vboxdrv'),
797	],
798	'solaris': [
799	# ( Whether to stop on hit, reason tuple, needle text. )
800	( True, ktReason_Host_InstallationFailed,
801	'svcadm: Couldn\'t bind to svc.configd.' ),
802	( True, ktReason_Host_InstallationFailed,
803	'pkgadd: ERROR: postinstall script did not complete successfully' ),
804	( True, ktReason_Host_DriverNotUnloading,
805	'can\'t unload the module: Device busy' ),
806	],
807	};
808
809
810	def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall):
811	"""
812	Investigates an install or uninstall failure.
813
814	We lump the two together since the installation typically also performs
815	an uninstall first and will be seeing similar issues to the uninstall.
816	"""
817
818	if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut:
819	oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult)
820	return True;
821
822	atSimple = self.katSimpleInstallUninstallMainLogReasons;
823	if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs:
824	atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple;
825
826	fFoundSomething = False;
827	for fStopOnHit, tReason, sNeedle in atSimple:
828	if sResultLog.find(sNeedle) > 0:
829	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
830	if fStopOnHit:
831	return True;
832	fFoundSomething = True;
833
834	return fFoundSomething if fFoundSomething else None;
835
836
837	def investigateBadTestBox(self, oCaseFile):
838	"""
839	Checks out bad-testbox statuses.
840	"""
841	_ = oCaseFile;
842	return False;
843
844
845	def investigateVBoxUnitTest(self, oCaseFile):
846	"""
847	Checks out a VBox unittest problem.
848	"""
849
850	#
851	# Process simple test case failures first, using their name as reason.
852	# We do the reason management just like for BSODs.
853	#
854	cRelevantOnes = 0;
855	sMainLog = oCaseFile.getMainLog();
856	aoFailedResults = oCaseFile.oTree.getListOfFailures();
857	for oFailedResult in aoFailedResults:
858	if oFailedResult is oCaseFile.oTree:
859	self.vprint('TODO: toplevel failure');
860	cRelevantOnes += 1
861
862	elif oFailedResult.sName == 'Installing VirtualBox':
863	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
864	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
865	cRelevantOnes += 1
866
867	elif oFailedResult.sName == 'Uninstalling VirtualBox':
868	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
869	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
870	cRelevantOnes += 1
871
872	elif oFailedResult.oParent is not None:
873	# Get the 2nd level node because that's where we'll find the unit test name.
874	while oFailedResult.oParent.oParent is not None:
875	oFailedResult = oFailedResult.oParent;
876
877	# Only report a failure once.
878	if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId:
879	sKey = oFailedResult.sName;
880	if sKey.startswith('testcase/'):
881	sKey = sKey[9:];
882	if sKey in self.asUnitTestReasons:
883	tReason = ( self.ksUnitTestCategory, sKey );
884	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
885	else:
886	self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons));
887	tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew );
888	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey);
889	cRelevantOnes += 1
890	else:
891	self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,));
892
893	#
894	# If we've caught all the relevant ones by now, report the result.
895	#
896	if len(oCaseFile.dReasonForResultId) >= cRelevantOnes:
897	return self.caseClosed(oCaseFile);
898	return False;
899
900	def extractGuestCpuStack(self, sInfoText):
901	"""
902	Extracts the guest CPU stacks from the input file.
903
904	Returns a dictionary keyed by the CPU number, value being a list of
905	raw stack lines (no header).
906	Returns empty dictionary if no stacks where found.
907	"""
908	dRet = {};
909	off = 0;
910	while True:
911	# Find the stack.
912	offStart = sInfoText.find('=== start guest stack VCPU ', off);
913	if offStart < 0:
914	break;
915	offEnd = sInfoText.find('=== end guest stack', offStart + 20);
916	if offEnd >= 0:
917	offEnd += 3;
918	else:
919	offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20);
920	if offEnd < 0:
921	offEnd = len(sInfoText);
922
923	sStack = sInfoText[offStart : offEnd];
924	sStack = sStack.replace('\r',''); # paranoia
925	asLines = sStack.split('\n');
926
927	# Figure the CPU.
928	asWords = asLines[0].split();
929	if len(asWords) < 6 or not asWords[5].isdigit():
930	break;
931	iCpu = int(asWords[5]);
932
933	# Add it and advance.
934	dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]]
935	off = offEnd;
936	return dRet;
937
938	def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs):
939	""" Investigates kvm_lock_spinning deadlocks """
940	#
941	# Extract the stacks. We need more than one CPU to create a deadlock.
942	#
943	dStacks = self.extractGuestCpuStack(sInfoText);
944	self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),));
945	if len(dStacks) >= 2:
946	#
947	# Examin each of the stacks. Each must have kvm_lock_spinning in
948	# one of the first three entries.
949	#
950	cHits = 0;
951	for iCpu in dStacks:
952	asBacktrace = dStacks[iCpu];
953	for iFrame in xrange(min(3, len(asBacktrace))):
954	if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0:
955	cHits += 1;
956	break;
957	self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),));
958	if cHits == len(dStacks):
959	return (True, self.ktReason_VMM_kvm_lock_spinning);
960
961	_ = dLogs; _ = oCaseFile;
962	return (False, None);
963
964	def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs):
965	""" Investigates HalReturnToFirmware hangs """
966	del oCaseFile
967	del sInfoText
968	del dLogs
969	# hope that's sufficient
970	return (True, self.ktReason_Unknown_HalReturnToFirmware);
971
972	## Things we search a main or VM log for to figure out why something went bust.
973	katSimpleMainAndVmLogReasons = [
974	# ( Whether to stop on hit, reason tuple, needle text. )
975	( False, ktReason_Guru_Generic, 'GuruMeditation' ),
976	( False, ktReason_Guru_Generic, 'Guru Meditation' ),
977	( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ),
978	( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ),
979	( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ),
980	( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ),
981	( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ),
982	( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ),
983	( True, ktReason_Networking_Nonexistent_host_nic,
984	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
985	( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ),
986	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
987	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
988	( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ),
989	( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ),
990	( True, ktReason_Unknown_File_Not_Found,
991	'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ),
992	( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now.
993	'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ),
994	( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ),
995	( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ),
996	( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ),
997	( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ),
998	( True, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ),
999	( True, ktReason_Unknown_VM_Start_Error, 'error: failed to open session for' ),
1000	( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ),
1001	];
1002
1003	## Things we search a VBoxHardening.log file for to figure out why something went bust.
1004	katSimpleVBoxHardeningLogReasons = [
1005	# ( Whether to stop on hit, reason tuple, needle text. )
1006	( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ),
1007	( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ),
1008	( True, ktReason_Host_TSTInfo_Accuracy_OOR, 'RTCRTSPTSTINFO::Accuracy::Millis: Out of range' ),
1009	];
1010
1011	## Things we search a kernel.log file for to figure out why something went bust.
1012	katSimpleKernelLogReasons = [
1013	# ( Whether to stop on hit, reason tuple, needle text. )
1014	( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ),
1015	( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn''t work' ),
1016	( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ),
1017	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ),
1018	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ),
1019	];
1020
1021	## Things we search the _RIGHT_ _STRIPPED_ vgatext for.
1022	katSimpleVgaTextReasons = [
1023	# ( Whether to stop on hit, reason tuple, needle text. )
1024	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1025	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ),
1026	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1027	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n"
1028	"...trying to set up timer (IRQ0) through the 8259A ... failed.\n"
1029	"...trying to set up timer as Virtual Wire IRQ... failed.\n"
1030	"...trying to set up timer as ExtINT IRQ... failed :(.\n"
1031	"Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n"
1032	"and send a report. Then try booting with the 'noapic' option\n"
1033	"\n" ),
1034	( True, ktReason_OSInstall_GRUB_hang,
1035	"-----\nGRUB Loading stage2..\n\n\n\n" ),
1036	( True, ktReason_OSInstall_GRUB_hang,
1037	"-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent
1038	( True, ktReason_OSInstall_GRUB_hang,
1039	"-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent
1040	( True, ktReason_OSInstall_GRUB_hang,
1041	"-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again
1042	( True, ktReason_OSInstall_Udev_hang,
1043	"\nStarting udev:\n\n\n\n" ),
1044	( True, ktReason_OSInstall_Udev_hang,
1045	"\nStarting udev:\n------" ),
1046	( True, ktReason_Panic_BootManagerC000000F,
1047	"Windows failed to start. A recent hardware or software change might be the" ),
1048	( True, ktReason_BootManager_Image_corrupt,
1049	"BOOTMGR image is corrupt. The system cannot boot." ),
1050	];
1051
1052	## Things we search for in the info.txt file. Require handlers for now.
1053	katInfoTextHandlers = [
1054	# ( Trigger text, handler method )
1055	( "kvm_lock_spinning", investigateInfoKvmLockSpinning ),
1056	( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ),
1057	];
1058
1059	## Mapping screenshot/failure SHA-256 hashes to failure reasons.
1060	katSimpleScreenshotHashReasons = [
1061	# ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output )
1062	( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ),
1063	( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ),
1064	( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ),
1065	( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ),
1066	( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ),
1067	( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ),
1068	( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ),
1069	( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ),
1070	( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ),
1071	( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ),
1072	( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ),
1073	];
1074
1075	def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog):
1076	"""
1077	Investigates a failed VM run.
1078	"""
1079
1080	def investigateLogSet():
1081	"""
1082	Investigates the current set of VM related logs.
1083	"""
1084	self.dprint('investigateLogSet: lengths: result log %u, VM log %u, kernel log %u, vga text %u, info text %u'
1085	% ( len(sResultLog if sResultLog else ''),
1086	len(sVMLog if sVMLog else ''),
1087	len(sKrnlLog if sKrnlLog else ''),
1088	len(sVgaText if sVgaText else ''),
1089	len(sInfoText if sInfoText else ''), ));
1090
1091	#self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,));
1092	#self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,));
1093	#self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,));
1094	#self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,));
1095	#self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,));
1096
1097	# TODO: more
1098
1099	#
1100	# Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this.
1101	#
1102	sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ],
1103	'GIM: HyperV: Guest indicates a fatal condition! P0=');
1104	if sDetails is not None:
1105	# P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 "
1106	sKey = sDetails.split(' ', 1)[0];
1107	try: sKey = '0x%08X' % (int(sKey, 16),);
1108	except: pass;
1109	if sKey in self.asBsodReasons:
1110	tReason = ( self.ksBsodCategory, sKey );
1111	elif sKey.lower() in self.asBsodReasons: # just in case.
1112	tReason = ( self.ksBsodCategory, sKey.lower() );
1113	else:
1114	self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons));
1115	tReason = ( self.ksBsodCategory, self.ksBsodAddNew );
1116	return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip());
1117
1118	#
1119	# Look for linux panic.
1120	#
1121	if sKrnlLog is not None:
1122	for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons:
1123	if sKrnlLog.find(sNeedle) > 0:
1124	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1125	if fStopOnHit:
1126	return True;
1127	fFoundSomething = True;
1128
1129	#
1130	# Loop thru the simple stuff.
1131	#
1132	fFoundSomething = False;
1133	for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons:
1134	if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0):
1135	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1136	if fStopOnHit:
1137	return True;
1138	fFoundSomething = True;
1139
1140	# Continue with vga text.
1141	if sVgaText:
1142	for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons:
1143	if sVgaText.find(sNeedle) > 0:
1144	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1145	if fStopOnHit:
1146	return True;
1147	fFoundSomething = True;
1148	_ = sInfoText;
1149
1150	# Continue with screen hashes.
1151	if sScreenHash is not None:
1152	for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons:
1153	if sScreenHash == sHash:
1154	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1155	if fStopOnHit:
1156	return True;
1157	fFoundSomething = True;
1158
1159	# Check VBoxHardening.log.
1160	if sNtHardLog is not None:
1161	for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons:
1162	if sNtHardLog.find(sNeedle) > 0:
1163	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1164	if fStopOnHit:
1165	return True;
1166	fFoundSomething = True;
1167
1168	#
1169	# Complicated stuff.
1170	#
1171	dLogs = {
1172	'sVMLog': sVMLog,
1173	'sNtHardLog': sNtHardLog,
1174	'sScreenHash': sScreenHash,
1175	'sKrnlLog': sKrnlLog,
1176	'sVgaText': sVgaText,
1177	'sInfoText': sInfoText,
1178	};
1179
1180	# info.txt.
1181	if sInfoText:
1182	for sNeedle, fnHandler in self.katInfoTextHandlers:
1183	if sInfoText.find(sNeedle) > 0:
1184	(fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs);
1185	if tReason is not None:
1186	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1187	if fStop:
1188	return True;
1189	fFoundSomething = True;
1190
1191	#
1192	# Check for repeated reboots...
1193	#
1194	if sVMLog is not None:
1195	cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\'');
1196	if cResets > 10:
1197	return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult,
1198	sComment = 'Counted %s reboots' % (cResets,));
1199
1200	return fFoundSomething;
1201
1202	#
1203	# Check if we got any VM or/and kernel logs. Treat them as sets in
1204	# case we run multiple VMs here (this is of course ASSUMING they
1205	# appear in the order that terminateVmBySession uploads them).
1206	#
1207	cTimes = 0;
1208	sVMLog = None;
1209	sNtHardLog = None;
1210	sScreenHash = None;
1211	sKrnlLog = None;
1212	sVgaText = None;
1213	sInfoText = None;
1214	for oFile in oFailedResult.aoFiles:
1215	if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm:
1216	if 'VBoxHardening.log' not in oFile.sFile:
1217	if sVMLog is not None:
1218	if investigateLogSet() is True:
1219	return True;
1220	cTimes += 1;
1221	sInfoText = None;
1222	sVgaText = None;
1223	sKrnlLog = None;
1224	sScreenHash = None;
1225	sNtHardLog = None;
1226	sVMLog = oCaseFile.getLogFile(oFile);
1227	else:
1228	sNtHardLog = oCaseFile.getLogFile(oFile);
1229	elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel:
1230	sKrnlLog = oCaseFile.getLogFile(oFile);
1231	elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText:
1232	sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]);
1233	elif oFile.sKind == TestResultFileData.ksKind_InfoCollection:
1234	sInfoText = oCaseFile.getLogFile(oFile);
1235	elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure:
1236	sScreenHash = oCaseFile.getScreenshotSha256(oFile);
1237	if sScreenHash is not None:
1238	sScreenHash = sScreenHash.lower();
1239	self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,));
1240
1241	if ( sVMLog is not None \
1242	or sNtHardLog is not None \
1243	or cTimes == 0) \
1244	and investigateLogSet() is True:
1245	return True;
1246
1247	return None;
1248
1249
1250	def isResultFromVMRun(self, oFailedResult, sResultLog):
1251	"""
1252	Checks if this result and corresponding log snippet looks like a VM run.
1253	"""
1254
1255	# Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log.
1256	if sResultLog.find(' startVm') > 0:
1257	return True;
1258
1259	# Any other indicators? No?
1260	_ = oFailedResult;
1261	return False;
1262
1263	def investigateVBoxVMTest(self, oCaseFile, fSingleVM):
1264	"""
1265	Checks out a VBox VM test.
1266
1267	This is generic investigation of a test running one or more VMs, like
1268	for example a smoke test or a guest installation test.
1269
1270	The fSingleVM parameter is a hint, which probably won't come in useful.
1271	"""
1272	_ = fSingleVM;
1273
1274	#
1275	# Get a list of test result failures we should be looking into and the main log.
1276	#
1277	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1278	sMainLog = oCaseFile.getMainLog();
1279
1280	#
1281	# There are a set of errors ending up on the top level result record.
1282	# Should deal with these first.
1283	#
1284	if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree:
1285	# Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always
1286	# be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to
1287	# ignore other failures in the test if we're not a little bit careful here.
1288	if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0:
1289	oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11);
1290	return self.caseClosed(oCaseFile);
1291
1292	# Hang after starting VBoxSVC (e.g. idTestSet=136307258)
1293	if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at',
1294	(' Timeout: ', ' Attempting to abort child...',) ):
1295	if sMainLog.find('* glibc detected * /') > 0:
1296	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption);
1297	else:
1298	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang);
1299	return self.caseClosed(oCaseFile);
1300
1301	# Look for heap corruption without visible hang.
1302	if sMainLog.find('* glibc detected * /') > 0 \
1303	or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374
1304	oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption);
1305	return self.caseClosed(oCaseFile);
1306
1307	# Out of memory w/ timeout.
1308	if sMainLog.find('sErrId=HostMemoryLow') > 0:
1309	oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow);
1310	return self.caseClosed(oCaseFile);
1311
1312	# Stale files like vts_rm.exe (windows).
1313	offEnd = sMainLog.rfind('* The test driver exits successfully. *');
1314	if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0:
1315	oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files);
1316	return self.caseClosed(oCaseFile);
1317
1318	#
1319	# XPCOM screwup
1320	#
1321	if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0:
1322	oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build);
1323	return self.caseClosed(oCaseFile);
1324
1325	#
1326	# Go thru each failed result.
1327	#
1328	for oFailedResult in aoFailedResults:
1329	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1330	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1331	if oFailedResult.sName == 'Installing VirtualBox':
1332	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1333
1334	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1335	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1336
1337	elif self.isResultFromVMRun(oFailedResult, sResultLog):
1338	self.investigateVMResult(oCaseFile, oFailedResult, sResultLog);
1339
1340	elif sResultLog.find('most likely not unique') > 0:
1341	oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult)
1342	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1343	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1344
1345	elif sResultLog.find('The machine is not mutable (state is ') > 0:
1346	self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem');
1347	oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult);
1348
1349	elif sResultLog.find('** error: no action was specified') > 0 \
1350	or sResultLog.find('(len(self._asXml, asText))') > 0:
1351	oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult);
1352
1353	else:
1354	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1355	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1356
1357	#
1358	# Report home and close the case if we got them all, otherwise log it.
1359	#
1360	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1361	return self.caseClosed(oCaseFile);
1362
1363	if oCaseFile.dReasonForResultId:
1364	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1365	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1366	else:
1367	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1368	return False;
1369
1370
1371	def reasoningFailures(self):
1372	"""
1373	Guess the reason for failures.
1374	"""
1375	#
1376	# Get a list of failed test sets without any assigned failure reason.
1377	#
1378	cGot = 0;
1379	aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack, tsNow = self.tsNow);
1380	for oTestSet in aoTestSets:
1381	self.dprint(u'');
1382	self.dprint(u'reasoningFailures: Checking out test set #%u, status %s' % ( oTestSet.idTestSet, oTestSet.enmStatus,))
1383
1384	#
1385	# Open a case file and assign it to the right investigator.
1386	#
1387	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet);
1388	oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated);
1389	oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox);
1390	oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated);
1391	oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig);
1392
1393	oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase);
1394
1395	if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
1396	self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,));
1397	fRc = self.investigateBadTestBox(oCaseFile);
1398
1399	elif oCaseFile.isVBoxUnitTest():
1400	self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,));
1401	fRc = self.investigateVBoxUnitTest(oCaseFile);
1402
1403	elif oCaseFile.isVBoxInstallTest():
1404	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1405	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1406
1407	elif oCaseFile.isVBoxUSBTest():
1408	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1409	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1410
1411	elif oCaseFile.isVBoxStorageTest():
1412	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1413	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1414
1415	elif oCaseFile.isVBoxGAsTest():
1416	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1417	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1418
1419	elif oCaseFile.isVBoxAPITest():
1420	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1421	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1422
1423	elif oCaseFile.isVBoxBenchmarkTest():
1424	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1425	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1426
1427	elif oCaseFile.isVBoxSmokeTest():
1428	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1429	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1430
1431	else:
1432	self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,));
1433	fRc = False;
1434	cGot += fRc is True;
1435
1436	self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), ));
1437	return 0;
1438
1439
1440	def main(self):
1441	"""
1442	The 'main' function.
1443	Return exit code (0, 1, etc).
1444	"""
1445	# Database stuff.
1446	self.oDb = TMDatabaseConnection()
1447	self.oTestResultLogic = TestResultLogic(self.oDb);
1448	self.oTestSetLogic = TestSetLogic(self.oDb);
1449	self.oFailureReasonLogic = FailureReasonLogic(self.oDb);
1450	self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb);
1451	self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory);
1452	self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory);
1453
1454	# Get a fix on our 'now' before we do anything..
1455	self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
1456	self.tsNow = self.oDb.fetchOne();
1457
1458	# If we're suppost to commit anything we need to get our user ID.
1459	rcExit = 0;
1460	if self.oConfig.fRealRun:
1461	self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
1462	if self.oLogin is None:
1463	rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
1464	else:
1465	self.uidSelf = self.oLogin.uid;
1466
1467	#
1468	# Do the stuff.
1469	#
1470	if rcExit == 0:
1471	rcExit = self.selfCheck();
1472	if rcExit == 0:
1473	rcExit = self.badTestBoxManagement();
1474	rcExit2 = self.reasoningFailures();
1475	if rcExit == 0:
1476	rcExit = rcExit2;
1477	# Redo the bad testbox management after failure reasons have been assigned (got timing issues).
1478	if rcExit == 0:
1479	rcExit = self.badTestBoxManagement();
1480
1481	# Cleanup.
1482	self.oFailureReasonLogic = None;
1483	self.oTestResultFailureLogic = None;
1484	self.oTestSetLogic = None;
1485	self.oTestResultLogic = None;
1486	self.oDb.close();
1487	self.oDb = None;
1488	if self.oLogFile is not None:
1489	self.oLogFile.close();
1490	self.oLogFile = None;
1491	return rcExit;
1492
1493	if __name__ == '__main__':
1494	sys.exit(VirtualTestSheriff().main());
1495

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py@ 78717

Download in other formats: