virtual_test_sheriff.py@ 78726

Last change on this file since 78726 was 78726, checked in by vboxsync, 6 years ago
vsheriff: Correctly identify GA and serial port tests and pass them to investigateVBoxVMTest. Added --testset option for targetly testing changes against a few specific testsets. Don't stop looking on ktReason_Unknown_VM_Start_Error as we might get better match when going over the hardening log, for instance.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 72.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: virtual_test_sheriff.py 78726 2019-05-24 13:20:21Z vboxsync $
4	# pylint: disable=C0301
5
6	"""
7	Virtual Test Sheriff.
8
9	Duties:
10	- Try to a assign failure reasons to recently failed tests.
11	- Reboot or disable bad test boxes.
12
13	"""
14
15	from __future__ import print_function;
16
17	__copyright__ = \
18	"""
19	Copyright (C) 2012-2019 Oracle Corporation
20
21	This file is part of VirtualBox Open Source Edition (OSE), as
22	available from http://www.virtualbox.org. This file is free software;
23	you can redistribute it and/or modify it under the terms of the GNU
24	General Public License (GPL) as published by the Free Software
25	Foundation, in version 2 as it comes in the "COPYING" file of the
26	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
27	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
32	VirtualBox OSE distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37	"""
38	__version__ = "$Revision: 78726 $"
39
40
41	# Standard python imports
42	import sys;
43	import os;
44	import hashlib;
45	import subprocess;
46	import smtplib
47	from email.mime.multipart import MIMEMultipart
48	from email.mime.text import MIMEText
49	from email.utils import COMMASPACE
50
51	if sys.version_info[0] >= 3:
52	from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module
53	else:
54	from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module
55	from optparse import OptionParser; # pylint: disable=deprecated-module
56	from PIL import Image; # pylint: disable=import-error
57
58	# Add Test Manager's modules path
59	g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
60	sys.path.append(g_ksTestManagerDir);
61
62	# Test Manager imports
63	from testmanager.core.db import TMDatabaseConnection;
64	from testmanager.core.build import BuildDataEx;
65	from testmanager.core.failurereason import FailureReasonLogic;
66	from testmanager.core.testbox import TestBoxLogic, TestBoxData;
67	from testmanager.core.testcase import TestCaseDataEx;
68	from testmanager.core.testgroup import TestGroupData;
69	from testmanager.core.testset import TestSetLogic, TestSetData;
70	from testmanager.core.testresults import TestResultLogic, TestResultFileData;
71	from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData;
72	from testmanager.core.useraccount import UserAccountLogic;
73	from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \
74	g_ksAlertSubject, g_asAlertList, g_ksLomPassword;
75
76	# Python 3 hacks:
77	if sys.version_info[0] >= 3:
78	xrange = range; # pylint: disable=redefined-builtin,invalid-name
79
80
81	class VirtualTestSheriffCaseFile(object):
82	"""
83	A failure investigation case file.
84
85	"""
86
87
88	## Max log file we'll read into memory. (256 MB)
89	kcbMaxLogRead = 0x10000000;
90
91	def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase):
92	self.oSheriff = oSheriff;
93	self.oTestSet = oTestSet; # TestSetData
94	self.oTree = oTree; # TestResultDataEx
95	self.oBuild = oBuild; # BuildDataEx
96	self.oTestBox = oTestBox; # TestBoxData
97	self.oTestGroup = oTestGroup; # TestGroupData
98	self.oTestCase = oTestCase; # TestCaseDataEx
99	self.sMainLog = ''; # The main log file. Empty string if not accessible.
100
101	# Generate a case file name.
102	self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,)
103	self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \
104	% ( self.oTestSet.idTestSet,
105	self.oTestCase.sName,
106	self.oTestBox.sName,
107	self.oTestBox.sOs,
108	self.oTestBox.sOsVersion,
109	self.oTestBox.sCpuArch,
110	self.oTestBox.sCpuName,
111	self.oTestBox.sCpuVendor,
112	self.oBuild.oCat.sProduct,
113	self.oBuild.oCat.sBranch,
114	self.oBuild.oCat.sType,
115	self.oBuild.iRevision, );
116
117	# Investigation notes.
118	self.tReason = None; # None or one of the ktReason_XXX constants.
119	self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult.
120	self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult.
121
122	#
123	# Reason.
124	#
125
126	def noteReason(self, tReason):
127	""" Notes down a possible reason. """
128	self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,));
129	self.tReason = tReason;
130	return True;
131
132	def noteReasonForId(self, tReason, idTestResult, sComment = None):
133	""" Notes down a possible reason for a specific test result. """
134	self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s'
135	% (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason,
136	(u' (%s)' % (sComment,)) if sComment is not None else ''));
137	self.dReasonForResultId[idTestResult] = tReason;
138	if sComment is not None:
139	self.dCommentForResultId[idTestResult] = sComment;
140	return True;
141
142
143	#
144	# Test classification.
145	#
146
147	def isVBoxTest(self):
148	""" Test classification: VirtualBox (using the build) """
149	return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ];
150
151	def isVBoxUnitTest(self):
152	""" Test case classification: The unit test doing all our testcase/*.cpp stuff. """
153	return self.isVBoxTest() \
154	and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests');
155
156	def isVBoxInstallTest(self):
157	""" Test case classification: VirtualBox Guest installation test. """
158	return self.isVBoxTest() \
159	and self.oTestCase.sName.lower().startswith('install:');
160
161	def isVBoxUSBTest(self):
162	""" Test case classification: VirtualBox USB test. """
163	return self.isVBoxTest() \
164	and self.oTestCase.sName.lower().startswith('usb:');
165
166	def isVBoxStorageTest(self):
167	""" Test case classification: VirtualBox Storage test. """
168	return self.isVBoxTest() \
169	and self.oTestCase.sName.lower().startswith('storage:');
170
171	def isVBoxGAsTest(self):
172	""" Test case classification: VirtualBox Guest Additions test. """
173	return self.isVBoxTest() \
174	and ( self.oTestCase.sName.lower().startswith('guest additions')
175	or self.oTestCase.sName.lower().startswith('ga\'s tests'));
176
177	def isVBoxAPITest(self):
178	""" Test case classification: VirtualBox API test. """
179	return self.isVBoxTest() \
180	and self.oTestCase.sName.lower().startswith('api:');
181
182	def isVBoxBenchmarkTest(self):
183	""" Test case classification: VirtualBox Benchmark test. """
184	return self.isVBoxTest() \
185	and self.oTestCase.sName.lower().startswith('benchmark:');
186
187	def isVBoxSmokeTest(self):
188	""" Test case classification: Smoke test. """
189	return self.isVBoxTest() \
190	and self.oTestCase.sName.lower().startswith('smoketest');
191
192	def isVBoxSerialTest(self):
193	""" Test case classification: Smoke test. """
194	return self.isVBoxTest() \
195	and self.oTestCase.sName.lower().startswith('serial:');
196
197
198	#
199	# Utility methods.
200	#
201
202	def getMainLog(self):
203	"""
204	Tries to read the main log file since this will be the first source of information.
205	"""
206	if self.sMainLog:
207	return self.sMainLog;
208	(oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb');
209	if oFile is not None:
210	try:
211	self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
212	except Exception as oXcpt:
213	self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,))
214	self.sMainLog = '';
215	else:
216	self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,));
217	return self.sMainLog;
218
219	def getLogFile(self, oFile):
220	"""
221	Tries to read the given file as a utf-8 log file.
222	oFile is a TestFileDataEx instance.
223	Returns empty string if problems opening or reading the file.
224	"""
225	sContent = '';
226	(oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
227	if oFile is not None:
228	try:
229	sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
230	except Exception as oXcpt:
231	self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,))
232	else:
233	self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,));
234	return sContent;
235
236	def getScreenshotSha256(self, oFile):
237	"""
238	Tries to read the given screenshot file, uncompress it, and do SHA-2
239	on the raw pixels.
240	Returns SHA-2 digest string on success, None on failure.
241	"""
242	(oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
243	try:
244	abImageFile = oImgFile.read();
245	except Exception as oXcpt:
246	self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,))
247	else:
248	try:
249	oImage = Image.open(StringIO(abImageFile));
250	except Exception as oXcpt:
251	self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,))
252	else:
253	try:
254	oHash = hashlib.sha256();
255	oHash.update(oImage.tostring());
256	except Exception as oXcpt:
257	self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,))
258	else:
259	return oHash.hexdigest();
260	return None;
261
262
263
264	def isSingleTestFailure(self):
265	"""
266	Figure out if this is a single test failing or if it's one of the
267	more complicated ones.
268	"""
269	if self.oTree.cErrors == 1:
270	return True;
271	if self.oTree.deepCountErrorContributers() <= 1:
272	return True;
273	return False;
274
275
276
277	class VirtualTestSheriff(object): # pylint: disable=R0903
278	"""
279	Add build info into Test Manager database.
280	"""
281
282	## The user account for the virtual sheriff.
283	ksLoginName = 'vsheriff';
284
285	def __init__(self):
286	"""
287	Parse command line.
288	"""
289	self.oDb = None;
290	self.tsNow = None;
291	self.oTestResultLogic = None;
292	self.oTestSetLogic = None;
293	self.oFailureReasonLogic = None; # FailureReasonLogic;
294	self.oTestResultFailureLogic = None; # TestResultFailureLogic
295	self.oLogin = None;
296	self.uidSelf = -1;
297	self.oLogFile = None;
298	self.asBsodReasons = [];
299	self.asUnitTestReasons = [];
300
301	oParser = OptionParser();
302	oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int',
303	help = 'When to start specified as hours relative to current time. Defauls is right now.', );
304	oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int',
305	help = 'Work period specified in hours. Defauls is 2 hours.');
306	oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
307	help = 'Whether to commit the findings to the database. Default is a dry run.');
308	oParser.add_option('--testset', dest = 'aidTestSets', metavar = '<id>', default = [], type = 'int', action = 'append',
309	help = 'Only investigate this one. Accumulates IDs when repeated.');
310	oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
311	help = 'Quiet execution');
312	oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None,
313	help = 'Where to log messages.');
314	oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
315	help = 'Enables debug mode.');
316
317	(self.oConfig, _) = oParser.parse_args();
318
319	if self.oConfig.sLogFile:
320	self.oLogFile = open(self.oConfig.sLogFile, "a");
321	self.oLogFile.write('VirtualTestSheriff: $Revision: 78726 $ \n');
322
323
324	def eprint(self, sText):
325	"""
326	Prints error messages.
327	Returns 1 (for exit code usage.)
328	"""
329	print('error: %s' % (sText,));
330	if self.oLogFile is not None:
331	self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8'));
332	return 1;
333
334	def dprint(self, sText):
335	"""
336	Prints debug info.
337	"""
338	if self.oConfig.fDebug:
339	if not self.oConfig.fQuiet:
340	print('debug: %s' % (sText, ));
341	if self.oLogFile is not None:
342	self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8'));
343	return 0;
344
345	def vprint(self, sText):
346	"""
347	Prints verbose info.
348	"""
349	if not self.oConfig.fQuiet:
350	print('info: %s' % (sText,));
351	if self.oLogFile is not None:
352	self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8'));
353	return 0;
354
355	def getFailureReason(self, tReason):
356	""" Gets the failure reason object for tReason. """
357	return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]);
358
359	def selfCheck(self):
360	""" Does some self checks, looking up things we expect to be in the database and such. """
361	rcExit = 0;
362	for sAttr in dir(self.__class__):
363	if sAttr.startswith('ktReason_'):
364	tReason = getattr(self.__class__, sAttr);
365	oFailureReason = self.getFailureReason(tReason);
366	if oFailureReason is None:
367	rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!'
368	% (tReason[1], tReason[0],));
369
370	# Check the user account as well.
371	if self.oLogin is None:
372	oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
373	if oLogin is None:
374	rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
375	return rcExit;
376
377	def sendEmailAlert(self, uidAuthor, sBodyText):
378	"""
379	Sends email alert.
380	"""
381
382	# Get author email
383	self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,));
384	sFrom = self.oDb.fetchOne();
385	if sFrom is not None:
386	sFrom = sFrom[0];
387	else:
388	sFrom = g_ksAlertFrom;
389
390	# Gather recipient list.
391	asEmailList = [];
392	for sUser in g_asAlertList:
393	self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,));
394	sEmail = self.oDb.fetchOne();
395	if sEmail:
396	asEmailList.append(sEmail[0]);
397	if not asEmailList:
398	return self.eprint('No email addresses to send alter to!');
399
400	# Compose the message.
401	oMsg = MIMEMultipart();
402	oMsg['From'] = sFrom;
403	oMsg['To'] = COMMASPACE.join(asEmailList);
404	oMsg['Subject'] = g_ksAlertSubject;
405	oMsg.attach(MIMEText(sBodyText, 'plain'))
406
407	# Try send it.
408	try:
409	oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort);
410	oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string())
411	oSMTP.quit()
412	except smtplib.SMTPException as oXcpt:
413	return self.eprint('Failed to send mail: %s' % (oXcpt,));
414
415	return 0;
416
417	def badTestBoxManagement(self):
418	"""
419	Looks for bad test boxes and first tries once to reboot them then disables them.
420	"""
421	rcExit = 0;
422
423	#
424	# We skip this entirely if we're running in the past and not in harmless debug mode.
425	#
426	if self.oConfig.cStartHoursAgo != 0 \
427	and (not self.oConfig.fDebug or self.oConfig.fRealRun):
428	return rcExit;
429	tsNow = self.tsNow if self.oConfig.fDebug else None;
430	cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2;
431	oTestBoxLogic = TestBoxLogic(self.oDb);
432
433	#
434	# Generate a list of failures reasons we consider bad-testbox behavior.
435	#
436	aidFailureReasons = [
437	self.getFailureReason(self.ktReason_Host_DriverNotLoaded).idFailureReason,
438	self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason,
439	self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason,
440	self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason,
441	];
442
443	#
444	# Get list of bad test boxes for given period and check them out individually.
445	#
446	aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow,
447	aidFailureReasons = aidFailureReasons);
448	for idTestBox in aidBadTestBoxes:
449	# Skip if the testbox is already disabled or has a pending reboot command.
450	try:
451	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
452	except Exception as oXcpt:
453	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
454	continue;
455	if not oTestBox.fEnabled:
456	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
457	% ( idTestBox, oTestBox.sName, ));
458	continue;
459	if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
460	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
461	% ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
462	continue;
463
464	# Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
465	aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow);
466	cOkay = 0;
467	cBad = 0;
468	iFirstOkay = len(aoSets);
469	for iSet, oSet in enumerate(aoSets):
470	if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
471	cBad += 1;
472	else:
473	# Check for bad failure reasons.
474	oFailure = None;
475	if oSet.enmStatus in TestSetData.kasBadTestStatuses:
476	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oSet.idTestSet)
477	aoFailedResults = oTree.getListOfFailures();
478	for oFailedResult in aoFailedResults:
479	oFailure = self.oTestResultFailureLogic.getById(oFailedResult.idTestResult);
480	if oFailure is not None and oFailure.idFailureReason in aidFailureReasons:
481	break;
482	else:
483	oFailure = None;
484	if oFailure is not None:
485	cBad += 1;
486	else:
487	# This is an okay test result then.
488	## @todo maybe check the elapsed time here, it could still be a bad run?
489	cOkay += 1;
490	if iFirstOkay > iSet:
491	iFirstOkay = iSet;
492	if iSet > 10:
493	break;
494
495	# We react if there are two or more bad-testbox statuses at the head of the
496	# history and at least three in the last 10 results.
497	if iFirstOkay >= 2 and cBad > 2:
498	if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
499	sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
500	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
501	self.vprint(sComment);
502	self.sendEmailAlert(self.uidSelf, sComment);
503	if self.oConfig.fRealRun is True:
504	try:
505	oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
506	sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
507	% (iFirstOkay, cBad, cOkay),);
508	except Exception as oXcpt:
509	rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
510	else:
511	sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
512	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
513	self.vprint(sComment);
514	self.sendEmailAlert(self.uidSelf, sComment);
515	if self.oConfig.fRealRun is True:
516	try:
517	oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
518	sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
519	% (iFirstOkay, cBad, cOkay),);
520	except Exception as oXcpt:
521	rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
522	else:
523	self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u'
524	% ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
525
526	#
527	# Reset hanged testboxes
528	#
529	cStatusTimeoutMins = 10;
530
531	self.oDb.execute('SELECT TestBoxStatuses.idTestBox\n'
532	' FROM TestBoxStatuses, TestBoxes\n'
533	' WHERE TestBoxStatuses.tsUpdated >= (CURRENT_TIMESTAMP - interval \'%s hours\')\n'
534	' AND TestBoxStatuses.tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')\n'
535	' AND TestBoxStatuses.idTestBox = TestBoxes.idTestBox\n'
536	' AND Testboxes.tsExpire = \'infinity\'::timestamp', (cHoursBack,cStatusTimeoutMins));
537	for idTestBox in self.oDb.fetchAll():
538	idTestBox = idTestBox[0];
539	try:
540	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
541	except Exception as oXcpt:
542	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
543	continue;
544	# Skip if the testbox is already disabled, already reset or there's no iLOM
545	if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0:
546	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
547	% ( idTestBox, oTestBox.sName, ));
548	continue;
549	## @todo get iLOM credentials from a table?
550	sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,);
551	try:
552	oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True);
553	sStdout = oPs.communicate()[0];
554	iRC = oPs.wait();
555
556	oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,);
557	oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True);
558
559	sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout);
560	self.vprint(sComment);
561	self.sendEmailAlert(self.uidSelf, sComment);
562
563	except Exception as oXcpt:
564	rcExit = self.eprint(u'Error resetting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
565
566	return rcExit;
567
568
569	## @name Failure reasons we know.
570	## @{
571	ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' );
572	ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' );
573	ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' );
574	ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' );
575	ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' );
576	ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' );
577	ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' );
578	ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' );
579	ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' );
580	ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' );
581	ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' );
582	ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' );
583	ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' );
584	ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' );
585	ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' );
586	ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' );
587	ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' );
588	ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' );
589	ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' );
590	ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' );
591	ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' );
592	ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' );
593	ktReason_Host_Install_Hang = ( 'Host', 'Install hang' );
594	ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' );
595	ktReason_Host_TSTInfo_Accuracy_OOR = ( 'Host', 'TSTInfo accuracy out of range' );
596	ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' );
597	ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' );
598	ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' );
599	ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' );
600	ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' );
601	ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' );
602	ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' );
603	ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' );
604	ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' );
605	ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' );
606	ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' );
607	ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' );
608	ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' );
609	ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' );
610	ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' );
611	ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' );
612	ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' );
613	ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' );
614	ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' );
615	ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' );
616	ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' );
617	ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' );
618	ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' );
619	ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' );
620	ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' );
621	ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' );
622	## @}
623
624	## BSOD category.
625	ksBsodCategory = 'BSOD';
626	## Special reason indicating that the flesh and blood sheriff has work to do.
627	ksBsodAddNew = 'Add new BSOD';
628
629	## Unit test category.
630	ksUnitTestCategory = 'Unit';
631	## Special reason indicating that the flesh and blood sheriff has work to do.
632	ksUnitTestAddNew = 'Add new';
633
634	## Used for indica that we shouldn't report anything for this test result ID and
635	## consider promoting the previous error to test set level if it's the only one.
636	ktHarmless = ( 'Probably', 'Caused by previous error' );
637
638
639	def caseClosed(self, oCaseFile):
640	"""
641	Reports the findings in the case and closes it.
642	"""
643	#
644	# Log it and create a dReasonForReasultId we can use below.
645	#
646	dCommentForResultId = oCaseFile.dCommentForResultId;
647	if oCaseFile.dReasonForResultId:
648	# Must weed out ktHarmless.
649	dReasonForResultId = {};
650	for idKey, tReason in oCaseFile.dReasonForResultId.items():
651	if tReason is not self.ktHarmless:
652	dReasonForResultId[idKey] = tReason;
653	if not dReasonForResultId:
654	self.vprint(u'TODO: Closing %s without a real reason, only %s.'
655	% (oCaseFile.sName, oCaseFile.dReasonForResultId));
656	return False;
657
658	# Try promote to single reason.
659	atValues = dReasonForResultId.values();
660	fSingleReason = True;
661	if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult:
662	self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],));
663	elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]):
664	self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0]));
665	else:
666	fSingleReason = False;
667	if fSingleReason:
668	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], };
669	if dCommentForResultId:
670	dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], };
671	elif oCaseFile.tReason is not None:
672	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, };
673	else:
674	self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,));
675	return False;
676
677	self.vprint(u'Closing %s with following reason%s: %s'
678	% ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, ));
679
680	#
681	# Add the test failure reason record(s).
682	#
683	for idTestResult, tReason in dReasonForResultId.items():
684	oFailureReason = self.getFailureReason(tReason);
685	if oFailureReason is not None:
686	sComment = 'Set by $Revision: 78726 $' # Handy for reverting later.
687	if idTestResult in dCommentForResultId:
688	sComment += ': ' + dCommentForResultId[idTestResult];
689
690	oAdd = TestResultFailureData();
691	oAdd.initFromValues(idTestResult = idTestResult,
692	idFailureReason = oFailureReason.idFailureReason,
693	uidAuthor = self.uidSelf,
694	idTestSet = oCaseFile.oTestSet.idTestSet,
695	sComment = sComment,);
696	if self.oConfig.fRealRun:
697	try:
698	self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True);
699	except Exception as oXcpt:
700	self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s'
701	% (oXcpt, oAdd, oCaseFile.sLongName,));
702	else:
703	self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],));
704	return True;
705
706	#
707	# Tools for assiting log parsing.
708	#
709
710	@staticmethod
711	def matchFollowedByLines(sStr, off, asFollowingLines):
712	""" Worker for isThisFollowedByTheseLines. """
713
714	# Advance off to the end of the line.
715	off = sStr.find('\n', off);
716	if off < 0:
717	return False;
718	off += 1;
719
720	# Match each string with the subsequent lines.
721	for iLine, sLine in enumerate(asFollowingLines):
722	offEnd = sStr.find('\n', off);
723	if offEnd < 0:
724	return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0;
725	if sLine and sStr.find(sLine, off, offEnd) < 0:
726	return False;
727
728	# next line.
729	off = offEnd + 1;
730
731	return True;
732
733	@staticmethod
734	def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines):
735	"""
736	Looks for a line contining sFirst which is then followed by lines
737	with the strings in asFollowingLines. (No newline chars anywhere!)
738	Returns True / False.
739	"""
740	off = sStr.find(sFirst, 0);
741	while off >= 0:
742	if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines):
743	return True;
744	off = sStr.find(sFirst, off + 1);
745	return False;
746
747	@staticmethod
748	def findAndReturnRestOfLine(sHaystack, sNeedle):
749	"""
750	Looks for sNeedle in sHaystack.
751	Returns The text following the needle up to the end of the line.
752	Returns None if not found.
753	"""
754	if sHaystack is None:
755	return None;
756	off = sHaystack.find(sNeedle);
757	if off < 0:
758	return None;
759	off += len(sNeedle)
760	offEol = sHaystack.find('\n', off);
761	if offEol < 0:
762	offEol = len(sHaystack);
763	return sHaystack[off:offEol]
764
765	@staticmethod
766	def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle):
767	"""
768	Looks for sNeedle in zeroe or more haystacks (asHaystack).
769	Returns The text following the first needed found up to the end of the line.
770	Returns None if not found.
771	"""
772	for sHaystack in asHaystacks:
773	sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle);
774	if sRet is not None:
775	return sRet;
776	return None;
777
778
779	#
780	# The investigative units.
781	#
782
783	katSimpleInstallUninstallMainLogReasons = [
784	# ( Whether to stop on hit, reason tuple, needle text. )
785	( False, ktReason_Host_LeftoverService,
786	'SERVICE_NAME: vbox' ),
787	];
788
789	kdatSimpleInstallUninstallMainLogReasonsPerOs = {
790	'darwin': [
791	# ( Whether to stop on hit, reason tuple, needle text. )
792	( True, ktReason_Host_DriverNotUnloading,
793	'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ),
794	],
795	'linux': [
796	# ( Whether to stop on hit, reason tuple, needle text. )
797	( True, ktReason_Host_DriverNotCompilable,
798	'This system is not currently set up to build kernel modules' ),
799	( True, ktReason_Host_DriverNotCompilable,
800	'This system is currently not set up to build kernel modules' ),
801	( True, ktReason_Host_InstallationFailed,
802	'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ),
803	( True, ktReason_Host_DriverNotUnloading,
804	'Cannot unload module vboxdrv'),
805	],
806	'solaris': [
807	# ( Whether to stop on hit, reason tuple, needle text. )
808	( True, ktReason_Host_InstallationFailed,
809	'svcadm: Couldn\'t bind to svc.configd.' ),
810	( True, ktReason_Host_InstallationFailed,
811	'pkgadd: ERROR: postinstall script did not complete successfully' ),
812	( True, ktReason_Host_DriverNotUnloading,
813	'can\'t unload the module: Device busy' ),
814	],
815	};
816
817
818	def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall):
819	"""
820	Investigates an install or uninstall failure.
821
822	We lump the two together since the installation typically also performs
823	an uninstall first and will be seeing similar issues to the uninstall.
824	"""
825
826	if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut:
827	oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult)
828	return True;
829
830	atSimple = self.katSimpleInstallUninstallMainLogReasons;
831	if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs:
832	atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple;
833
834	fFoundSomething = False;
835	for fStopOnHit, tReason, sNeedle in atSimple:
836	if sResultLog.find(sNeedle) > 0:
837	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
838	if fStopOnHit:
839	return True;
840	fFoundSomething = True;
841
842	return fFoundSomething if fFoundSomething else None;
843
844
845	def investigateBadTestBox(self, oCaseFile):
846	"""
847	Checks out bad-testbox statuses.
848	"""
849	_ = oCaseFile;
850	return False;
851
852
853	def investigateVBoxUnitTest(self, oCaseFile):
854	"""
855	Checks out a VBox unittest problem.
856	"""
857
858	#
859	# Process simple test case failures first, using their name as reason.
860	# We do the reason management just like for BSODs.
861	#
862	cRelevantOnes = 0;
863	sMainLog = oCaseFile.getMainLog();
864	aoFailedResults = oCaseFile.oTree.getListOfFailures();
865	for oFailedResult in aoFailedResults:
866	if oFailedResult is oCaseFile.oTree:
867	self.vprint('TODO: toplevel failure');
868	cRelevantOnes += 1
869
870	elif oFailedResult.sName == 'Installing VirtualBox':
871	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
872	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
873	cRelevantOnes += 1
874
875	elif oFailedResult.sName == 'Uninstalling VirtualBox':
876	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
877	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
878	cRelevantOnes += 1
879
880	elif oFailedResult.oParent is not None:
881	# Get the 2nd level node because that's where we'll find the unit test name.
882	while oFailedResult.oParent.oParent is not None:
883	oFailedResult = oFailedResult.oParent;
884
885	# Only report a failure once.
886	if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId:
887	sKey = oFailedResult.sName;
888	if sKey.startswith('testcase/'):
889	sKey = sKey[9:];
890	if sKey in self.asUnitTestReasons:
891	tReason = ( self.ksUnitTestCategory, sKey );
892	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
893	else:
894	self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons));
895	tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew );
896	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey);
897	cRelevantOnes += 1
898	else:
899	self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,));
900
901	#
902	# If we've caught all the relevant ones by now, report the result.
903	#
904	if len(oCaseFile.dReasonForResultId) >= cRelevantOnes:
905	return self.caseClosed(oCaseFile);
906	return False;
907
908	def extractGuestCpuStack(self, sInfoText):
909	"""
910	Extracts the guest CPU stacks from the input file.
911
912	Returns a dictionary keyed by the CPU number, value being a list of
913	raw stack lines (no header).
914	Returns empty dictionary if no stacks where found.
915	"""
916	dRet = {};
917	off = 0;
918	while True:
919	# Find the stack.
920	offStart = sInfoText.find('=== start guest stack VCPU ', off);
921	if offStart < 0:
922	break;
923	offEnd = sInfoText.find('=== end guest stack', offStart + 20);
924	if offEnd >= 0:
925	offEnd += 3;
926	else:
927	offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20);
928	if offEnd < 0:
929	offEnd = len(sInfoText);
930
931	sStack = sInfoText[offStart : offEnd];
932	sStack = sStack.replace('\r',''); # paranoia
933	asLines = sStack.split('\n');
934
935	# Figure the CPU.
936	asWords = asLines[0].split();
937	if len(asWords) < 6 or not asWords[5].isdigit():
938	break;
939	iCpu = int(asWords[5]);
940
941	# Add it and advance.
942	dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]]
943	off = offEnd;
944	return dRet;
945
946	def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs):
947	""" Investigates kvm_lock_spinning deadlocks """
948	#
949	# Extract the stacks. We need more than one CPU to create a deadlock.
950	#
951	dStacks = self.extractGuestCpuStack(sInfoText);
952	self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),));
953	if len(dStacks) >= 2:
954	#
955	# Examin each of the stacks. Each must have kvm_lock_spinning in
956	# one of the first three entries.
957	#
958	cHits = 0;
959	for iCpu in dStacks:
960	asBacktrace = dStacks[iCpu];
961	for iFrame in xrange(min(3, len(asBacktrace))):
962	if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0:
963	cHits += 1;
964	break;
965	self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),));
966	if cHits == len(dStacks):
967	return (True, self.ktReason_VMM_kvm_lock_spinning);
968
969	_ = dLogs; _ = oCaseFile;
970	return (False, None);
971
972	def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs):
973	""" Investigates HalReturnToFirmware hangs """
974	del oCaseFile
975	del sInfoText
976	del dLogs
977	# hope that's sufficient
978	return (True, self.ktReason_Unknown_HalReturnToFirmware);
979
980	## Things we search a main or VM log for to figure out why something went bust.
981	katSimpleMainAndVmLogReasons = [
982	# ( Whether to stop on hit, reason tuple, needle text. )
983	( False, ktReason_Guru_Generic, 'GuruMeditation' ),
984	( False, ktReason_Guru_Generic, 'Guru Meditation' ),
985	( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ),
986	( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ),
987	( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ),
988	( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ),
989	( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ),
990	( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ),
991	( True, ktReason_Networking_Nonexistent_host_nic,
992	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
993	( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ),
994	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
995	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
996	( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ),
997	( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ),
998	( True, ktReason_Unknown_File_Not_Found,
999	'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ),
1000	( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now.
1001	'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ),
1002	( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ),
1003	( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ),
1004	( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ),
1005	( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ),
1006	( False, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ),
1007	( False, ktReason_Unknown_VM_Start_Error, 'error: failed to open session for' ),
1008	( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ),
1009	];
1010
1011	## Things we search a VBoxHardening.log file for to figure out why something went bust.
1012	katSimpleVBoxHardeningLogReasons = [
1013	# ( Whether to stop on hit, reason tuple, needle text. )
1014	( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ),
1015	( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ),
1016	( True, ktReason_Host_TSTInfo_Accuracy_OOR, 'RTCRTSPTSTINFO::Accuracy::Millis: Out of range' ),
1017	];
1018
1019	## Things we search a kernel.log file for to figure out why something went bust.
1020	katSimpleKernelLogReasons = [
1021	# ( Whether to stop on hit, reason tuple, needle text. )
1022	( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ),
1023	( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn''t work' ),
1024	( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ),
1025	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ),
1026	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ),
1027	];
1028
1029	## Things we search the _RIGHT_ _STRIPPED_ vgatext for.
1030	katSimpleVgaTextReasons = [
1031	# ( Whether to stop on hit, reason tuple, needle text. )
1032	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1033	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ),
1034	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1035	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n"
1036	"...trying to set up timer (IRQ0) through the 8259A ... failed.\n"
1037	"...trying to set up timer as Virtual Wire IRQ... failed.\n"
1038	"...trying to set up timer as ExtINT IRQ... failed :(.\n"
1039	"Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n"
1040	"and send a report. Then try booting with the 'noapic' option\n"
1041	"\n" ),
1042	( True, ktReason_OSInstall_GRUB_hang,
1043	"-----\nGRUB Loading stage2..\n\n\n\n" ),
1044	( True, ktReason_OSInstall_GRUB_hang,
1045	"-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent
1046	( True, ktReason_OSInstall_GRUB_hang,
1047	"-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent
1048	( True, ktReason_OSInstall_GRUB_hang,
1049	"-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again
1050	( True, ktReason_OSInstall_Udev_hang,
1051	"\nStarting udev:\n\n\n\n" ),
1052	( True, ktReason_OSInstall_Udev_hang,
1053	"\nStarting udev:\n------" ),
1054	( True, ktReason_Panic_BootManagerC000000F,
1055	"Windows failed to start. A recent hardware or software change might be the" ),
1056	( True, ktReason_BootManager_Image_corrupt,
1057	"BOOTMGR image is corrupt. The system cannot boot." ),
1058	];
1059
1060	## Things we search for in the info.txt file. Require handlers for now.
1061	katInfoTextHandlers = [
1062	# ( Trigger text, handler method )
1063	( "kvm_lock_spinning", investigateInfoKvmLockSpinning ),
1064	( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ),
1065	];
1066
1067	## Mapping screenshot/failure SHA-256 hashes to failure reasons.
1068	katSimpleScreenshotHashReasons = [
1069	# ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output )
1070	( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ),
1071	( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ),
1072	( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ),
1073	( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ),
1074	( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ),
1075	( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ),
1076	( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ),
1077	( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ),
1078	( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ),
1079	( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ),
1080	( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ),
1081	];
1082
1083	def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog):
1084	"""
1085	Investigates a failed VM run.
1086	"""
1087
1088	def investigateLogSet():
1089	"""
1090	Investigates the current set of VM related logs.
1091	"""
1092	self.dprint('investigateLogSet: lengths: result log %u, VM log %u, kernel log %u, vga text %u, info text %u'
1093	% ( len(sResultLog if sResultLog else ''),
1094	len(sVMLog if sVMLog else ''),
1095	len(sKrnlLog if sKrnlLog else ''),
1096	len(sVgaText if sVgaText else ''),
1097	len(sInfoText if sInfoText else ''), ));
1098
1099	#self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,));
1100	#self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,));
1101	#self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,));
1102	#self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,));
1103	#self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,));
1104
1105	# TODO: more
1106
1107	#
1108	# Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this.
1109	#
1110	sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ],
1111	'GIM: HyperV: Guest indicates a fatal condition! P0=');
1112	if sDetails is not None:
1113	# P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 "
1114	sKey = sDetails.split(' ', 1)[0];
1115	try: sKey = '0x%08X' % (int(sKey, 16),);
1116	except: pass;
1117	if sKey in self.asBsodReasons:
1118	tReason = ( self.ksBsodCategory, sKey );
1119	elif sKey.lower() in self.asBsodReasons: # just in case.
1120	tReason = ( self.ksBsodCategory, sKey.lower() );
1121	else:
1122	self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons));
1123	tReason = ( self.ksBsodCategory, self.ksBsodAddNew );
1124	return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip());
1125
1126	#
1127	# Look for linux panic.
1128	#
1129	if sKrnlLog is not None:
1130	for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons:
1131	if sKrnlLog.find(sNeedle) > 0:
1132	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1133	if fStopOnHit:
1134	return True;
1135	fFoundSomething = True;
1136
1137	#
1138	# Loop thru the simple stuff.
1139	#
1140	fFoundSomething = False;
1141	for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons:
1142	if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0):
1143	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1144	if fStopOnHit:
1145	return True;
1146	fFoundSomething = True;
1147
1148	# Continue with vga text.
1149	if sVgaText:
1150	for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons:
1151	if sVgaText.find(sNeedle) > 0:
1152	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1153	if fStopOnHit:
1154	return True;
1155	fFoundSomething = True;
1156	_ = sInfoText;
1157
1158	# Continue with screen hashes.
1159	if sScreenHash is not None:
1160	for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons:
1161	if sScreenHash == sHash:
1162	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1163	if fStopOnHit:
1164	return True;
1165	fFoundSomething = True;
1166
1167	# Check VBoxHardening.log.
1168	if sNtHardLog is not None:
1169	for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons:
1170	if sNtHardLog.find(sNeedle) > 0:
1171	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1172	if fStopOnHit:
1173	return True;
1174	fFoundSomething = True;
1175
1176	#
1177	# Complicated stuff.
1178	#
1179	dLogs = {
1180	'sVMLog': sVMLog,
1181	'sNtHardLog': sNtHardLog,
1182	'sScreenHash': sScreenHash,
1183	'sKrnlLog': sKrnlLog,
1184	'sVgaText': sVgaText,
1185	'sInfoText': sInfoText,
1186	};
1187
1188	# info.txt.
1189	if sInfoText:
1190	for sNeedle, fnHandler in self.katInfoTextHandlers:
1191	if sInfoText.find(sNeedle) > 0:
1192	(fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs);
1193	if tReason is not None:
1194	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1195	if fStop:
1196	return True;
1197	fFoundSomething = True;
1198
1199	#
1200	# Check for repeated reboots...
1201	#
1202	if sVMLog is not None:
1203	cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\'');
1204	if cResets > 10:
1205	return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult,
1206	sComment = 'Counted %s reboots' % (cResets,));
1207
1208	return fFoundSomething;
1209
1210	#
1211	# Check if we got any VM or/and kernel logs. Treat them as sets in
1212	# case we run multiple VMs here (this is of course ASSUMING they
1213	# appear in the order that terminateVmBySession uploads them).
1214	#
1215	cTimes = 0;
1216	sVMLog = None;
1217	sNtHardLog = None;
1218	sScreenHash = None;
1219	sKrnlLog = None;
1220	sVgaText = None;
1221	sInfoText = None;
1222	for oFile in oFailedResult.aoFiles:
1223	if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm:
1224	if 'VBoxHardening.log' not in oFile.sFile:
1225	if sVMLog is not None:
1226	if investigateLogSet() is True:
1227	return True;
1228	cTimes += 1;
1229	sInfoText = None;
1230	sVgaText = None;
1231	sKrnlLog = None;
1232	sScreenHash = None;
1233	sNtHardLog = None;
1234	sVMLog = oCaseFile.getLogFile(oFile);
1235	else:
1236	sNtHardLog = oCaseFile.getLogFile(oFile);
1237	elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel:
1238	sKrnlLog = oCaseFile.getLogFile(oFile);
1239	elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText:
1240	sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]);
1241	elif oFile.sKind == TestResultFileData.ksKind_InfoCollection:
1242	sInfoText = oCaseFile.getLogFile(oFile);
1243	elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure:
1244	sScreenHash = oCaseFile.getScreenshotSha256(oFile);
1245	if sScreenHash is not None:
1246	sScreenHash = sScreenHash.lower();
1247	self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,));
1248
1249	if ( sVMLog is not None \
1250	or sNtHardLog is not None \
1251	or cTimes == 0) \
1252	and investigateLogSet() is True:
1253	return True;
1254
1255	return None;
1256
1257
1258	def isResultFromVMRun(self, oFailedResult, sResultLog):
1259	"""
1260	Checks if this result and corresponding log snippet looks like a VM run.
1261	"""
1262
1263	# Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log.
1264	if sResultLog.find(' startVm') > 0:
1265	return True;
1266
1267	# Any other indicators? No?
1268	_ = oFailedResult;
1269	return False;
1270
1271	def investigateVBoxVMTest(self, oCaseFile, fSingleVM):
1272	"""
1273	Checks out a VBox VM test.
1274
1275	This is generic investigation of a test running one or more VMs, like
1276	for example a smoke test or a guest installation test.
1277
1278	The fSingleVM parameter is a hint, which probably won't come in useful.
1279	"""
1280	_ = fSingleVM;
1281
1282	#
1283	# Get a list of test result failures we should be looking into and the main log.
1284	#
1285	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1286	sMainLog = oCaseFile.getMainLog();
1287
1288	#
1289	# There are a set of errors ending up on the top level result record.
1290	# Should deal with these first.
1291	#
1292	if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree:
1293	# Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always
1294	# be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to
1295	# ignore other failures in the test if we're not a little bit careful here.
1296	if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0:
1297	oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11);
1298	return self.caseClosed(oCaseFile);
1299
1300	# Hang after starting VBoxSVC (e.g. idTestSet=136307258)
1301	if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at',
1302	(' Timeout: ', ' Attempting to abort child...',) ):
1303	if sMainLog.find('* glibc detected * /') > 0:
1304	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption);
1305	else:
1306	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang);
1307	return self.caseClosed(oCaseFile);
1308
1309	# Look for heap corruption without visible hang.
1310	if sMainLog.find('* glibc detected * /') > 0 \
1311	or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374
1312	oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption);
1313	return self.caseClosed(oCaseFile);
1314
1315	# Out of memory w/ timeout.
1316	if sMainLog.find('sErrId=HostMemoryLow') > 0:
1317	oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow);
1318	return self.caseClosed(oCaseFile);
1319
1320	# Stale files like vts_rm.exe (windows).
1321	offEnd = sMainLog.rfind('* The test driver exits successfully. *');
1322	if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0:
1323	oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files);
1324	return self.caseClosed(oCaseFile);
1325
1326	#
1327	# XPCOM screwup
1328	#
1329	if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0:
1330	oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build);
1331	return self.caseClosed(oCaseFile);
1332
1333	#
1334	# Go thru each failed result.
1335	#
1336	for oFailedResult in aoFailedResults:
1337	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1338	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1339	if oFailedResult.sName == 'Installing VirtualBox':
1340	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1341
1342	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1343	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1344
1345	elif self.isResultFromVMRun(oFailedResult, sResultLog):
1346	self.investigateVMResult(oCaseFile, oFailedResult, sResultLog);
1347
1348	elif sResultLog.find('most likely not unique') > 0:
1349	oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult)
1350	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1351	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1352
1353	elif sResultLog.find('The machine is not mutable (state is ') > 0:
1354	self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem');
1355	oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult);
1356
1357	elif sResultLog.find('** error: no action was specified') > 0 \
1358	or sResultLog.find('(len(self._asXml, asText))') > 0:
1359	oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult);
1360
1361	else:
1362	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1363	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1364
1365	#
1366	# Report home and close the case if we got them all, otherwise log it.
1367	#
1368	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1369	return self.caseClosed(oCaseFile);
1370
1371	if oCaseFile.dReasonForResultId:
1372	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1373	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1374	else:
1375	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1376	return False;
1377
1378
1379	def reasoningFailures(self):
1380	"""
1381	Guess the reason for failures.
1382	"""
1383	#
1384	# Get a list of failed test sets without any assigned failure reason.
1385	#
1386	cGot = 0;
1387	if self.oConfig.aidTestSets is None or len(self.oConfig.aidTestSets) == 0:
1388	aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack,
1389	tsNow = self.tsNow);
1390	else:
1391	aoTestSets = [self.oTestSetLogic.getById(idTestSet) for idTestSet in self.oConfig.aidTestSets];
1392	for oTestSet in aoTestSets:
1393	self.dprint(u'----------------------------------- #%u, status %s -----------------------------------'
1394	% ( oTestSet.idTestSet, oTestSet.enmStatus,));
1395
1396	#
1397	# Open a case file and assign it to the right investigator.
1398	#
1399	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet);
1400	oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated);
1401	oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox);
1402	oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated);
1403	oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig);
1404
1405	oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase);
1406
1407	if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
1408	self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,));
1409	fRc = self.investigateBadTestBox(oCaseFile);
1410
1411	elif oCaseFile.isVBoxUnitTest():
1412	self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,));
1413	fRc = self.investigateVBoxUnitTest(oCaseFile);
1414
1415	elif oCaseFile.isVBoxInstallTest():
1416	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1417	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1418
1419	elif oCaseFile.isVBoxUSBTest():
1420	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1421	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1422
1423	elif oCaseFile.isVBoxStorageTest():
1424	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1425	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1426
1427	elif oCaseFile.isVBoxGAsTest():
1428	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1429	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1430
1431	elif oCaseFile.isVBoxAPITest():
1432	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1433	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1434
1435	elif oCaseFile.isVBoxBenchmarkTest():
1436	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1437	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1438
1439	elif oCaseFile.isVBoxSmokeTest():
1440	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1441	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1442
1443	elif oCaseFile.isVBoxSerialTest():
1444	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1445	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1446
1447	else:
1448	self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,));
1449	fRc = False;
1450	cGot += fRc is True;
1451
1452	self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), ));
1453	return 0;
1454
1455
1456	def main(self):
1457	"""
1458	The 'main' function.
1459	Return exit code (0, 1, etc).
1460	"""
1461	# Database stuff.
1462	self.oDb = TMDatabaseConnection()
1463	self.oTestResultLogic = TestResultLogic(self.oDb);
1464	self.oTestSetLogic = TestSetLogic(self.oDb);
1465	self.oFailureReasonLogic = FailureReasonLogic(self.oDb);
1466	self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb);
1467	self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory);
1468	self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory);
1469
1470	# Get a fix on our 'now' before we do anything..
1471	self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
1472	self.tsNow = self.oDb.fetchOne();
1473
1474	# If we're suppost to commit anything we need to get our user ID.
1475	rcExit = 0;
1476	if self.oConfig.fRealRun:
1477	self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
1478	if self.oLogin is None:
1479	rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
1480	else:
1481	self.uidSelf = self.oLogin.uid;
1482
1483	#
1484	# Do the stuff.
1485	#
1486	if rcExit == 0:
1487	rcExit = self.selfCheck();
1488	if rcExit == 0:
1489	rcExit = self.badTestBoxManagement();
1490	rcExit2 = self.reasoningFailures();
1491	if rcExit == 0:
1492	rcExit = rcExit2;
1493	# Redo the bad testbox management after failure reasons have been assigned (got timing issues).
1494	if rcExit == 0:
1495	rcExit = self.badTestBoxManagement();
1496
1497	# Cleanup.
1498	self.oFailureReasonLogic = None;
1499	self.oTestResultFailureLogic = None;
1500	self.oTestSetLogic = None;
1501	self.oTestResultLogic = None;
1502	self.oDb.close();
1503	self.oDb = None;
1504	if self.oLogFile is not None:
1505	self.oLogFile.close();
1506	self.oLogFile = None;
1507	return rcExit;
1508
1509	if __name__ == '__main__':
1510	sys.exit(VirtualTestSheriff().main());
1511

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py@ 78726

Download in other formats: