virtual_test_sheriff.py@ 78728

Last change on this file since 78728 was 78728, checked in by vboxsync, 6 years ago
vsheriff: Logging.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 72.6 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: virtual_test_sheriff.py 78728 2019-05-24 13:24:30Z vboxsync $
4	# pylint: disable=C0301
5
6	"""
7	Virtual Test Sheriff.
8
9	Duties:
10	- Try to a assign failure reasons to recently failed tests.
11	- Reboot or disable bad test boxes.
12
13	"""
14
15	from __future__ import print_function;
16
17	__copyright__ = \
18	"""
19	Copyright (C) 2012-2019 Oracle Corporation
20
21	This file is part of VirtualBox Open Source Edition (OSE), as
22	available from http://www.virtualbox.org. This file is free software;
23	you can redistribute it and/or modify it under the terms of the GNU
24	General Public License (GPL) as published by the Free Software
25	Foundation, in version 2 as it comes in the "COPYING" file of the
26	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
27	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
32	VirtualBox OSE distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37	"""
38	__version__ = "$Revision: 78728 $"
39
40
41	# Standard python imports
42	import sys;
43	import os;
44	import hashlib;
45	import subprocess;
46	import smtplib
47	from email.mime.multipart import MIMEMultipart
48	from email.mime.text import MIMEText
49	from email.utils import COMMASPACE
50
51	if sys.version_info[0] >= 3:
52	from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module
53	else:
54	from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module
55	from optparse import OptionParser; # pylint: disable=deprecated-module
56	from PIL import Image; # pylint: disable=import-error
57
58	# Add Test Manager's modules path
59	g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
60	sys.path.append(g_ksTestManagerDir);
61
62	# Test Manager imports
63	from testmanager.core.db import TMDatabaseConnection;
64	from testmanager.core.build import BuildDataEx;
65	from testmanager.core.failurereason import FailureReasonLogic;
66	from testmanager.core.testbox import TestBoxLogic, TestBoxData;
67	from testmanager.core.testcase import TestCaseDataEx;
68	from testmanager.core.testgroup import TestGroupData;
69	from testmanager.core.testset import TestSetLogic, TestSetData;
70	from testmanager.core.testresults import TestResultLogic, TestResultFileData;
71	from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData;
72	from testmanager.core.useraccount import UserAccountLogic;
73	from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \
74	g_ksAlertSubject, g_asAlertList, g_ksLomPassword;
75
76	# Python 3 hacks:
77	if sys.version_info[0] >= 3:
78	xrange = range; # pylint: disable=redefined-builtin,invalid-name
79
80
81	class VirtualTestSheriffCaseFile(object):
82	"""
83	A failure investigation case file.
84
85	"""
86
87
88	## Max log file we'll read into memory. (256 MB)
89	kcbMaxLogRead = 0x10000000;
90
91	def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase):
92	self.oSheriff = oSheriff;
93	self.oTestSet = oTestSet; # TestSetData
94	self.oTree = oTree; # TestResultDataEx
95	self.oBuild = oBuild; # BuildDataEx
96	self.oTestBox = oTestBox; # TestBoxData
97	self.oTestGroup = oTestGroup; # TestGroupData
98	self.oTestCase = oTestCase; # TestCaseDataEx
99	self.sMainLog = ''; # The main log file. Empty string if not accessible.
100
101	# Generate a case file name.
102	self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,)
103	self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \
104	% ( self.oTestSet.idTestSet,
105	self.oTestCase.sName,
106	self.oTestBox.sName,
107	self.oTestBox.sOs,
108	self.oTestBox.sOsVersion,
109	self.oTestBox.sCpuArch,
110	self.oTestBox.sCpuName,
111	self.oTestBox.sCpuVendor,
112	self.oBuild.oCat.sProduct,
113	self.oBuild.oCat.sBranch,
114	self.oBuild.oCat.sType,
115	self.oBuild.iRevision, );
116
117	# Investigation notes.
118	self.tReason = None; # None or one of the ktReason_XXX constants.
119	self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult.
120	self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult.
121
122	#
123	# Reason.
124	#
125
126	def noteReason(self, tReason):
127	""" Notes down a possible reason. """
128	self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,));
129	self.tReason = tReason;
130	return True;
131
132	def noteReasonForId(self, tReason, idTestResult, sComment = None):
133	""" Notes down a possible reason for a specific test result. """
134	self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s'
135	% (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason,
136	(u' (%s)' % (sComment,)) if sComment is not None else ''));
137	self.dReasonForResultId[idTestResult] = tReason;
138	if sComment is not None:
139	self.dCommentForResultId[idTestResult] = sComment;
140	return True;
141
142
143	#
144	# Test classification.
145	#
146
147	def isVBoxTest(self):
148	""" Test classification: VirtualBox (using the build) """
149	return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ];
150
151	def isVBoxUnitTest(self):
152	""" Test case classification: The unit test doing all our testcase/*.cpp stuff. """
153	return self.isVBoxTest() \
154	and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests');
155
156	def isVBoxInstallTest(self):
157	""" Test case classification: VirtualBox Guest installation test. """
158	return self.isVBoxTest() \
159	and self.oTestCase.sName.lower().startswith('install:');
160
161	def isVBoxUSBTest(self):
162	""" Test case classification: VirtualBox USB test. """
163	return self.isVBoxTest() \
164	and self.oTestCase.sName.lower().startswith('usb:');
165
166	def isVBoxStorageTest(self):
167	""" Test case classification: VirtualBox Storage test. """
168	return self.isVBoxTest() \
169	and self.oTestCase.sName.lower().startswith('storage:');
170
171	def isVBoxGAsTest(self):
172	""" Test case classification: VirtualBox Guest Additions test. """
173	return self.isVBoxTest() \
174	and ( self.oTestCase.sName.lower().startswith('guest additions')
175	or self.oTestCase.sName.lower().startswith('ga\'s tests'));
176
177	def isVBoxAPITest(self):
178	""" Test case classification: VirtualBox API test. """
179	return self.isVBoxTest() \
180	and self.oTestCase.sName.lower().startswith('api:');
181
182	def isVBoxBenchmarkTest(self):
183	""" Test case classification: VirtualBox Benchmark test. """
184	return self.isVBoxTest() \
185	and self.oTestCase.sName.lower().startswith('benchmark:');
186
187	def isVBoxSmokeTest(self):
188	""" Test case classification: Smoke test. """
189	return self.isVBoxTest() \
190	and self.oTestCase.sName.lower().startswith('smoketest');
191
192	def isVBoxSerialTest(self):
193	""" Test case classification: Smoke test. """
194	return self.isVBoxTest() \
195	and self.oTestCase.sName.lower().startswith('serial:');
196
197
198	#
199	# Utility methods.
200	#
201
202	def getMainLog(self):
203	"""
204	Tries to read the main log file since this will be the first source of information.
205	"""
206	if self.sMainLog:
207	return self.sMainLog;
208	(oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb');
209	if oFile is not None:
210	try:
211	self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
212	except Exception as oXcpt:
213	self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,))
214	self.sMainLog = '';
215	else:
216	self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,));
217	return self.sMainLog;
218
219	def getLogFile(self, oFile):
220	"""
221	Tries to read the given file as a utf-8 log file.
222	oFile is a TestFileDataEx instance.
223	Returns empty string if problems opening or reading the file.
224	"""
225	sContent = '';
226	(oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
227	if oFile is not None:
228	try:
229	sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
230	except Exception as oXcpt:
231	self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,))
232	else:
233	self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,));
234	return sContent;
235
236	def getScreenshotSha256(self, oFile):
237	"""
238	Tries to read the given screenshot file, uncompress it, and do SHA-2
239	on the raw pixels.
240	Returns SHA-2 digest string on success, None on failure.
241	"""
242	(oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
243	try:
244	abImageFile = oImgFile.read();
245	except Exception as oXcpt:
246	self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,))
247	else:
248	try:
249	oImage = Image.open(StringIO(abImageFile));
250	except Exception as oXcpt:
251	self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,))
252	else:
253	try:
254	oHash = hashlib.sha256();
255	oHash.update(oImage.tostring());
256	except Exception as oXcpt:
257	self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,))
258	else:
259	return oHash.hexdigest();
260	return None;
261
262
263
264	def isSingleTestFailure(self):
265	"""
266	Figure out if this is a single test failing or if it's one of the
267	more complicated ones.
268	"""
269	if self.oTree.cErrors == 1:
270	return True;
271	if self.oTree.deepCountErrorContributers() <= 1:
272	return True;
273	return False;
274
275
276
277	class VirtualTestSheriff(object): # pylint: disable=R0903
278	"""
279	Add build info into Test Manager database.
280	"""
281
282	## The user account for the virtual sheriff.
283	ksLoginName = 'vsheriff';
284
285	def __init__(self):
286	"""
287	Parse command line.
288	"""
289	self.oDb = None;
290	self.tsNow = None;
291	self.oTestResultLogic = None;
292	self.oTestSetLogic = None;
293	self.oFailureReasonLogic = None; # FailureReasonLogic;
294	self.oTestResultFailureLogic = None; # TestResultFailureLogic
295	self.oLogin = None;
296	self.uidSelf = -1;
297	self.oLogFile = None;
298	self.asBsodReasons = [];
299	self.asUnitTestReasons = [];
300
301	oParser = OptionParser();
302	oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int',
303	help = 'When to start specified as hours relative to current time. Defauls is right now.', );
304	oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int',
305	help = 'Work period specified in hours. Defauls is 2 hours.');
306	oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
307	help = 'Whether to commit the findings to the database. Default is a dry run.');
308	oParser.add_option('--testset', dest = 'aidTestSets', metavar = '<id>', default = [], type = 'int', action = 'append',
309	help = 'Only investigate this one. Accumulates IDs when repeated.');
310	oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
311	help = 'Quiet execution');
312	oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None,
313	help = 'Where to log messages.');
314	oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
315	help = 'Enables debug mode.');
316
317	(self.oConfig, _) = oParser.parse_args();
318
319	if self.oConfig.sLogFile:
320	self.oLogFile = open(self.oConfig.sLogFile, "a");
321	self.oLogFile.write('VirtualTestSheriff: $Revision: 78728 $ \n');
322
323
324	def eprint(self, sText):
325	"""
326	Prints error messages.
327	Returns 1 (for exit code usage.)
328	"""
329	print('error: %s' % (sText,));
330	if self.oLogFile is not None:
331	self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8'));
332	return 1;
333
334	def dprint(self, sText):
335	"""
336	Prints debug info.
337	"""
338	if self.oConfig.fDebug:
339	if not self.oConfig.fQuiet:
340	print('debug: %s' % (sText, ));
341	if self.oLogFile is not None:
342	self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8'));
343	return 0;
344
345	def vprint(self, sText):
346	"""
347	Prints verbose info.
348	"""
349	if not self.oConfig.fQuiet:
350	print('info: %s' % (sText,));
351	if self.oLogFile is not None:
352	self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8'));
353	return 0;
354
355	def getFailureReason(self, tReason):
356	""" Gets the failure reason object for tReason. """
357	return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]);
358
359	def selfCheck(self):
360	""" Does some self checks, looking up things we expect to be in the database and such. """
361	rcExit = 0;
362	for sAttr in dir(self.__class__):
363	if sAttr.startswith('ktReason_'):
364	tReason = getattr(self.__class__, sAttr);
365	oFailureReason = self.getFailureReason(tReason);
366	if oFailureReason is None:
367	rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!'
368	% (tReason[1], tReason[0],));
369
370	# Check the user account as well.
371	if self.oLogin is None:
372	oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
373	if oLogin is None:
374	rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
375	return rcExit;
376
377	def sendEmailAlert(self, uidAuthor, sBodyText):
378	"""
379	Sends email alert.
380	"""
381
382	# Get author email
383	self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,));
384	sFrom = self.oDb.fetchOne();
385	if sFrom is not None:
386	sFrom = sFrom[0];
387	else:
388	sFrom = g_ksAlertFrom;
389
390	# Gather recipient list.
391	asEmailList = [];
392	for sUser in g_asAlertList:
393	self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,));
394	sEmail = self.oDb.fetchOne();
395	if sEmail:
396	asEmailList.append(sEmail[0]);
397	if not asEmailList:
398	return self.eprint('No email addresses to send alter to!');
399
400	# Compose the message.
401	oMsg = MIMEMultipart();
402	oMsg['From'] = sFrom;
403	oMsg['To'] = COMMASPACE.join(asEmailList);
404	oMsg['Subject'] = g_ksAlertSubject;
405	oMsg.attach(MIMEText(sBodyText, 'plain'))
406
407	# Try send it.
408	try:
409	oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort);
410	oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string())
411	oSMTP.quit()
412	except smtplib.SMTPException as oXcpt:
413	return self.eprint('Failed to send mail: %s' % (oXcpt,));
414
415	return 0;
416
417	def badTestBoxManagement(self):
418	"""
419	Looks for bad test boxes and first tries once to reboot them then disables them.
420	"""
421	rcExit = 0;
422
423	#
424	# We skip this entirely if we're running in the past and not in harmless debug mode.
425	#
426	if self.oConfig.cStartHoursAgo != 0 \
427	and (not self.oConfig.fDebug or self.oConfig.fRealRun):
428	return rcExit;
429	tsNow = self.tsNow if self.oConfig.fDebug else None;
430	cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2;
431	oTestBoxLogic = TestBoxLogic(self.oDb);
432
433	#
434	# Generate a list of failures reasons we consider bad-testbox behavior.
435	#
436	aidFailureReasons = [
437	self.getFailureReason(self.ktReason_Host_DriverNotLoaded).idFailureReason,
438	self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason,
439	self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason,
440	self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason,
441	];
442
443	#
444	# Get list of bad test boxes for given period and check them out individually.
445	#
446	aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow,
447	aidFailureReasons = aidFailureReasons);
448	for idTestBox in aidBadTestBoxes:
449	# Skip if the testbox is already disabled or has a pending reboot command.
450	try:
451	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
452	except Exception as oXcpt:
453	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
454	continue;
455	if not oTestBox.fEnabled:
456	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
457	% ( idTestBox, oTestBox.sName, ));
458	continue;
459	if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
460	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
461	% ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
462	continue;
463
464	# Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
465	aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow);
466	cOkay = 0;
467	cBad = 0;
468	iFirstOkay = len(aoSets);
469	for iSet, oSet in enumerate(aoSets):
470	if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
471	cBad += 1;
472	else:
473	# Check for bad failure reasons.
474	oFailure = None;
475	if oSet.enmStatus in TestSetData.kasBadTestStatuses:
476	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oSet.idTestSet)
477	aoFailedResults = oTree.getListOfFailures();
478	for oFailedResult in aoFailedResults:
479	oFailure = self.oTestResultFailureLogic.getById(oFailedResult.idTestResult);
480	if oFailure is not None and oFailure.idFailureReason in aidFailureReasons:
481	break;
482	else:
483	oFailure = None;
484	if oFailure is not None:
485	cBad += 1;
486	else:
487	# This is an okay test result then.
488	## @todo maybe check the elapsed time here, it could still be a bad run?
489	cOkay += 1;
490	if iFirstOkay > iSet:
491	iFirstOkay = iSet;
492	if iSet > 10:
493	break;
494
495	# We react if there are two or more bad-testbox statuses at the head of the
496	# history and at least three in the last 10 results.
497	if iFirstOkay >= 2 and cBad > 2:
498	if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
499	sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
500	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
501	self.vprint(sComment);
502	self.sendEmailAlert(self.uidSelf, sComment);
503	if self.oConfig.fRealRun is True:
504	try:
505	oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
506	sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
507	% (iFirstOkay, cBad, cOkay),);
508	except Exception as oXcpt:
509	rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
510	else:
511	sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
512	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
513	self.vprint(sComment);
514	self.sendEmailAlert(self.uidSelf, sComment);
515	if self.oConfig.fRealRun is True:
516	try:
517	oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
518	sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
519	% (iFirstOkay, cBad, cOkay),);
520	except Exception as oXcpt:
521	rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
522	else:
523	self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u'
524	% ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
525
526	#
527	# Reset hanged testboxes
528	#
529	cStatusTimeoutMins = 10;
530
531	self.oDb.execute('SELECT TestBoxStatuses.idTestBox\n'
532	' FROM TestBoxStatuses, TestBoxes\n'
533	' WHERE TestBoxStatuses.tsUpdated >= (CURRENT_TIMESTAMP - interval \'%s hours\')\n'
534	' AND TestBoxStatuses.tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')\n'
535	' AND TestBoxStatuses.idTestBox = TestBoxes.idTestBox\n'
536	' AND Testboxes.tsExpire = \'infinity\'::timestamp', (cHoursBack,cStatusTimeoutMins));
537	for idTestBox in self.oDb.fetchAll():
538	idTestBox = idTestBox[0];
539	try:
540	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
541	except Exception as oXcpt:
542	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
543	continue;
544	# Skip if the testbox is already disabled, already reset or there's no iLOM
545	if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0:
546	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
547	% ( idTestBox, oTestBox.sName, ));
548	continue;
549	## @todo get iLOM credentials from a table?
550	sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,);
551	try:
552	oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True);
553	sStdout = oPs.communicate()[0];
554	iRC = oPs.wait();
555
556	oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,);
557	oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True);
558
559	sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout);
560	self.vprint(sComment);
561	self.sendEmailAlert(self.uidSelf, sComment);
562
563	except Exception as oXcpt:
564	rcExit = self.eprint(u'Error resetting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
565
566	return rcExit;
567
568
569	## @name Failure reasons we know.
570	## @{
571	ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' );
572	ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' );
573	ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' );
574	ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' );
575	ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' );
576	ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' );
577	ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' );
578	ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' );
579	ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' );
580	ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' );
581	ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' );
582	ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' );
583	ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' );
584	ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' );
585	ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' );
586	ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' );
587	ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' );
588	ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' );
589	ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' );
590	ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' );
591	ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' );
592	ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' );
593	ktReason_Host_Install_Hang = ( 'Host', 'Install hang' );
594	ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' );
595	ktReason_Host_TSTInfo_Accuracy_OOR = ( 'Host', 'TSTInfo accuracy out of range' );
596	ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' );
597	ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' );
598	ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' );
599	ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' );
600	ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' );
601	ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' );
602	ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' );
603	ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' );
604	ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' );
605	ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' );
606	ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' );
607	ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' );
608	ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' );
609	ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' );
610	ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' );
611	ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' );
612	ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' );
613	ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' );
614	ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' );
615	ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' );
616	ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' );
617	ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' );
618	ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' );
619	ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' );
620	ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' );
621	ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' );
622	## @}
623
624	## BSOD category.
625	ksBsodCategory = 'BSOD';
626	## Special reason indicating that the flesh and blood sheriff has work to do.
627	ksBsodAddNew = 'Add new BSOD';
628
629	## Unit test category.
630	ksUnitTestCategory = 'Unit';
631	## Special reason indicating that the flesh and blood sheriff has work to do.
632	ksUnitTestAddNew = 'Add new';
633
634	## Used for indica that we shouldn't report anything for this test result ID and
635	## consider promoting the previous error to test set level if it's the only one.
636	ktHarmless = ( 'Probably', 'Caused by previous error' );
637
638
639	def caseClosed(self, oCaseFile):
640	"""
641	Reports the findings in the case and closes it.
642	"""
643	#
644	# Log it and create a dReasonForReasultId we can use below.
645	#
646	dCommentForResultId = oCaseFile.dCommentForResultId;
647	if oCaseFile.dReasonForResultId:
648	# Must weed out ktHarmless.
649	dReasonForResultId = {};
650	for idKey, tReason in oCaseFile.dReasonForResultId.items():
651	if tReason is not self.ktHarmless:
652	dReasonForResultId[idKey] = tReason;
653	if not dReasonForResultId:
654	self.vprint(u'TODO: Closing %s without a real reason, only %s.'
655	% (oCaseFile.sName, oCaseFile.dReasonForResultId));
656	return False;
657
658	# Try promote to single reason.
659	atValues = dReasonForResultId.values();
660	fSingleReason = True;
661	if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult:
662	self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],));
663	elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]):
664	self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0]));
665	else:
666	fSingleReason = False;
667	if fSingleReason:
668	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], };
669	if dCommentForResultId:
670	dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], };
671	elif oCaseFile.tReason is not None:
672	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, };
673	else:
674	self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,));
675	return False;
676
677	self.vprint(u'Closing %s with following reason%s: %s'
678	% ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, ));
679
680	#
681	# Add the test failure reason record(s).
682	#
683	for idTestResult, tReason in dReasonForResultId.items():
684	oFailureReason = self.getFailureReason(tReason);
685	if oFailureReason is not None:
686	sComment = 'Set by $Revision: 78728 $' # Handy for reverting later.
687	if idTestResult in dCommentForResultId:
688	sComment += ': ' + dCommentForResultId[idTestResult];
689
690	oAdd = TestResultFailureData();
691	oAdd.initFromValues(idTestResult = idTestResult,
692	idFailureReason = oFailureReason.idFailureReason,
693	uidAuthor = self.uidSelf,
694	idTestSet = oCaseFile.oTestSet.idTestSet,
695	sComment = sComment,);
696	if self.oConfig.fRealRun:
697	try:
698	self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True);
699	except Exception as oXcpt:
700	self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s'
701	% (oXcpt, oAdd, oCaseFile.sLongName,));
702	else:
703	self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],));
704	return True;
705
706	#
707	# Tools for assiting log parsing.
708	#
709
710	@staticmethod
711	def matchFollowedByLines(sStr, off, asFollowingLines):
712	""" Worker for isThisFollowedByTheseLines. """
713
714	# Advance off to the end of the line.
715	off = sStr.find('\n', off);
716	if off < 0:
717	return False;
718	off += 1;
719
720	# Match each string with the subsequent lines.
721	for iLine, sLine in enumerate(asFollowingLines):
722	offEnd = sStr.find('\n', off);
723	if offEnd < 0:
724	return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0;
725	if sLine and sStr.find(sLine, off, offEnd) < 0:
726	return False;
727
728	# next line.
729	off = offEnd + 1;
730
731	return True;
732
733	@staticmethod
734	def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines):
735	"""
736	Looks for a line contining sFirst which is then followed by lines
737	with the strings in asFollowingLines. (No newline chars anywhere!)
738	Returns True / False.
739	"""
740	off = sStr.find(sFirst, 0);
741	while off >= 0:
742	if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines):
743	return True;
744	off = sStr.find(sFirst, off + 1);
745	return False;
746
747	@staticmethod
748	def findAndReturnRestOfLine(sHaystack, sNeedle):
749	"""
750	Looks for sNeedle in sHaystack.
751	Returns The text following the needle up to the end of the line.
752	Returns None if not found.
753	"""
754	if sHaystack is None:
755	return None;
756	off = sHaystack.find(sNeedle);
757	if off < 0:
758	return None;
759	off += len(sNeedle)
760	offEol = sHaystack.find('\n', off);
761	if offEol < 0:
762	offEol = len(sHaystack);
763	return sHaystack[off:offEol]
764
765	@staticmethod
766	def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle):
767	"""
768	Looks for sNeedle in zeroe or more haystacks (asHaystack).
769	Returns The text following the first needed found up to the end of the line.
770	Returns None if not found.
771	"""
772	for sHaystack in asHaystacks:
773	sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle);
774	if sRet is not None:
775	return sRet;
776	return None;
777
778
779	#
780	# The investigative units.
781	#
782
783	katSimpleInstallUninstallMainLogReasons = [
784	# ( Whether to stop on hit, reason tuple, needle text. )
785	( False, ktReason_Host_LeftoverService,
786	'SERVICE_NAME: vbox' ),
787	];
788
789	kdatSimpleInstallUninstallMainLogReasonsPerOs = {
790	'darwin': [
791	# ( Whether to stop on hit, reason tuple, needle text. )
792	( True, ktReason_Host_DriverNotUnloading,
793	'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ),
794	],
795	'linux': [
796	# ( Whether to stop on hit, reason tuple, needle text. )
797	( True, ktReason_Host_DriverNotCompilable,
798	'This system is not currently set up to build kernel modules' ),
799	( True, ktReason_Host_DriverNotCompilable,
800	'This system is currently not set up to build kernel modules' ),
801	( True, ktReason_Host_InstallationFailed,
802	'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ),
803	( True, ktReason_Host_DriverNotUnloading,
804	'Cannot unload module vboxdrv'),
805	],
806	'solaris': [
807	# ( Whether to stop on hit, reason tuple, needle text. )
808	( True, ktReason_Host_InstallationFailed,
809	'svcadm: Couldn\'t bind to svc.configd.' ),
810	( True, ktReason_Host_InstallationFailed,
811	'pkgadd: ERROR: postinstall script did not complete successfully' ),
812	( True, ktReason_Host_DriverNotUnloading,
813	'can\'t unload the module: Device busy' ),
814	],
815	};
816
817
818	def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall):
819	"""
820	Investigates an install or uninstall failure.
821
822	We lump the two together since the installation typically also performs
823	an uninstall first and will be seeing similar issues to the uninstall.
824	"""
825
826	if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut:
827	oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult)
828	return True;
829
830	atSimple = self.katSimpleInstallUninstallMainLogReasons;
831	if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs:
832	atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple;
833
834	fFoundSomething = False;
835	for fStopOnHit, tReason, sNeedle in atSimple:
836	if sResultLog.find(sNeedle) > 0:
837	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
838	if fStopOnHit:
839	return True;
840	fFoundSomething = True;
841
842	return fFoundSomething if fFoundSomething else None;
843
844
845	def investigateBadTestBox(self, oCaseFile):
846	"""
847	Checks out bad-testbox statuses.
848	"""
849	_ = oCaseFile;
850	return False;
851
852
853	def investigateVBoxUnitTest(self, oCaseFile):
854	"""
855	Checks out a VBox unittest problem.
856	"""
857
858	#
859	# Process simple test case failures first, using their name as reason.
860	# We do the reason management just like for BSODs.
861	#
862	cRelevantOnes = 0;
863	sMainLog = oCaseFile.getMainLog();
864	aoFailedResults = oCaseFile.oTree.getListOfFailures();
865	for oFailedResult in aoFailedResults:
866	if oFailedResult is oCaseFile.oTree:
867	self.vprint('TODO: toplevel failure');
868	cRelevantOnes += 1
869
870	elif oFailedResult.sName == 'Installing VirtualBox':
871	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
872	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
873	cRelevantOnes += 1
874
875	elif oFailedResult.sName == 'Uninstalling VirtualBox':
876	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
877	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
878	cRelevantOnes += 1
879
880	elif oFailedResult.oParent is not None:
881	# Get the 2nd level node because that's where we'll find the unit test name.
882	while oFailedResult.oParent.oParent is not None:
883	oFailedResult = oFailedResult.oParent;
884
885	# Only report a failure once.
886	if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId:
887	sKey = oFailedResult.sName;
888	if sKey.startswith('testcase/'):
889	sKey = sKey[9:];
890	if sKey in self.asUnitTestReasons:
891	tReason = ( self.ksUnitTestCategory, sKey );
892	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
893	else:
894	self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons));
895	tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew );
896	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey);
897	cRelevantOnes += 1
898	else:
899	self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,));
900
901	#
902	# If we've caught all the relevant ones by now, report the result.
903	#
904	if len(oCaseFile.dReasonForResultId) >= cRelevantOnes:
905	return self.caseClosed(oCaseFile);
906	return False;
907
908	def extractGuestCpuStack(self, sInfoText):
909	"""
910	Extracts the guest CPU stacks from the input file.
911
912	Returns a dictionary keyed by the CPU number, value being a list of
913	raw stack lines (no header).
914	Returns empty dictionary if no stacks where found.
915	"""
916	dRet = {};
917	off = 0;
918	while True:
919	# Find the stack.
920	offStart = sInfoText.find('=== start guest stack VCPU ', off);
921	if offStart < 0:
922	break;
923	offEnd = sInfoText.find('=== end guest stack', offStart + 20);
924	if offEnd >= 0:
925	offEnd += 3;
926	else:
927	offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20);
928	if offEnd < 0:
929	offEnd = len(sInfoText);
930
931	sStack = sInfoText[offStart : offEnd];
932	sStack = sStack.replace('\r',''); # paranoia
933	asLines = sStack.split('\n');
934
935	# Figure the CPU.
936	asWords = asLines[0].split();
937	if len(asWords) < 6 or not asWords[5].isdigit():
938	break;
939	iCpu = int(asWords[5]);
940
941	# Add it and advance.
942	dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]]
943	off = offEnd;
944	return dRet;
945
946	def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs):
947	""" Investigates kvm_lock_spinning deadlocks """
948	#
949	# Extract the stacks. We need more than one CPU to create a deadlock.
950	#
951	dStacks = self.extractGuestCpuStack(sInfoText);
952	self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),));
953	if len(dStacks) >= 2:
954	#
955	# Examin each of the stacks. Each must have kvm_lock_spinning in
956	# one of the first three entries.
957	#
958	cHits = 0;
959	for iCpu in dStacks:
960	asBacktrace = dStacks[iCpu];
961	for iFrame in xrange(min(3, len(asBacktrace))):
962	if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0:
963	cHits += 1;
964	break;
965	self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),));
966	if cHits == len(dStacks):
967	return (True, self.ktReason_VMM_kvm_lock_spinning);
968
969	_ = dLogs; _ = oCaseFile;
970	return (False, None);
971
972	def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs):
973	""" Investigates HalReturnToFirmware hangs """
974	del oCaseFile
975	del sInfoText
976	del dLogs
977	# hope that's sufficient
978	return (True, self.ktReason_Unknown_HalReturnToFirmware);
979
980	## Things we search a main or VM log for to figure out why something went bust.
981	katSimpleMainAndVmLogReasons = [
982	# ( Whether to stop on hit, reason tuple, needle text. )
983	( False, ktReason_Guru_Generic, 'GuruMeditation' ),
984	( False, ktReason_Guru_Generic, 'Guru Meditation' ),
985	( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ),
986	( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ),
987	( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ),
988	( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ),
989	( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ),
990	( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ),
991	( True, ktReason_Networking_Nonexistent_host_nic,
992	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
993	( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ),
994	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
995	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
996	( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ),
997	( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ),
998	( True, ktReason_Unknown_File_Not_Found,
999	'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ),
1000	( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now.
1001	'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ),
1002	( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ),
1003	( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ),
1004	( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ),
1005	( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ),
1006	( False, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ),
1007	( False, ktReason_Unknown_VM_Start_Error, 'error: failed to open session for' ),
1008	( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ),
1009	];
1010
1011	## Things we search a VBoxHardening.log file for to figure out why something went bust.
1012	katSimpleVBoxHardeningLogReasons = [
1013	# ( Whether to stop on hit, reason tuple, needle text. )
1014	( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ),
1015	( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ),
1016	( True, ktReason_Host_TSTInfo_Accuracy_OOR, 'RTCRTSPTSTINFO::Accuracy::Millis: Out of range' ),
1017	];
1018
1019	## Things we search a kernel.log file for to figure out why something went bust.
1020	katSimpleKernelLogReasons = [
1021	# ( Whether to stop on hit, reason tuple, needle text. )
1022	( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ),
1023	( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn''t work' ),
1024	( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ),
1025	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ),
1026	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ),
1027	];
1028
1029	## Things we search the _RIGHT_ _STRIPPED_ vgatext for.
1030	katSimpleVgaTextReasons = [
1031	# ( Whether to stop on hit, reason tuple, needle text. )
1032	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1033	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ),
1034	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1035	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n"
1036	"...trying to set up timer (IRQ0) through the 8259A ... failed.\n"
1037	"...trying to set up timer as Virtual Wire IRQ... failed.\n"
1038	"...trying to set up timer as ExtINT IRQ... failed :(.\n"
1039	"Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n"
1040	"and send a report. Then try booting with the 'noapic' option\n"
1041	"\n" ),
1042	( True, ktReason_OSInstall_GRUB_hang,
1043	"-----\nGRUB Loading stage2..\n\n\n\n" ),
1044	( True, ktReason_OSInstall_GRUB_hang,
1045	"-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent
1046	( True, ktReason_OSInstall_GRUB_hang,
1047	"-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent
1048	( True, ktReason_OSInstall_GRUB_hang,
1049	"-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again
1050	( True, ktReason_OSInstall_Udev_hang,
1051	"\nStarting udev:\n\n\n\n" ),
1052	( True, ktReason_OSInstall_Udev_hang,
1053	"\nStarting udev:\n------" ),
1054	( True, ktReason_Panic_BootManagerC000000F,
1055	"Windows failed to start. A recent hardware or software change might be the" ),
1056	( True, ktReason_BootManager_Image_corrupt,
1057	"BOOTMGR image is corrupt. The system cannot boot." ),
1058	];
1059
1060	## Things we search for in the info.txt file. Require handlers for now.
1061	katInfoTextHandlers = [
1062	# ( Trigger text, handler method )
1063	( "kvm_lock_spinning", investigateInfoKvmLockSpinning ),
1064	( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ),
1065	];
1066
1067	## Mapping screenshot/failure SHA-256 hashes to failure reasons.
1068	katSimpleScreenshotHashReasons = [
1069	# ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output )
1070	( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ),
1071	( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ),
1072	( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ),
1073	( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ),
1074	( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ),
1075	( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ),
1076	( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ),
1077	( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ),
1078	( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ),
1079	( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ),
1080	( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ),
1081	];
1082
1083	def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog):
1084	"""
1085	Investigates a failed VM run.
1086	"""
1087
1088	def investigateLogSet():
1089	"""
1090	Investigates the current set of VM related logs.
1091	"""
1092	self.dprint('investigateLogSet: log lengths: result %u, VM %u, kernel %u, vga text %u, info text %u, hard %u'
1093	% ( len(sResultLog if sResultLog else ''),
1094	len(sVMLog if sVMLog else ''),
1095	len(sKrnlLog if sKrnlLog else ''),
1096	len(sVgaText if sVgaText else ''),
1097	len(sInfoText if sInfoText else ''),
1098	len(sNtHardLog if sNtHardLog else ''), ));
1099
1100	#self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,));
1101	#self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,));
1102	#self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,));
1103	#self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,));
1104	#self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,));
1105	#self.dprint(u'hard.txt<<<\n%s\n<<<\n' % (sNtHardLog,));
1106
1107	# TODO: more
1108
1109	#
1110	# Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this.
1111	#
1112	sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ],
1113	'GIM: HyperV: Guest indicates a fatal condition! P0=');
1114	if sDetails is not None:
1115	# P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 "
1116	sKey = sDetails.split(' ', 1)[0];
1117	try: sKey = '0x%08X' % (int(sKey, 16),);
1118	except: pass;
1119	if sKey in self.asBsodReasons:
1120	tReason = ( self.ksBsodCategory, sKey );
1121	elif sKey.lower() in self.asBsodReasons: # just in case.
1122	tReason = ( self.ksBsodCategory, sKey.lower() );
1123	else:
1124	self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons));
1125	tReason = ( self.ksBsodCategory, self.ksBsodAddNew );
1126	return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip());
1127
1128	#
1129	# Look for linux panic.
1130	#
1131	if sKrnlLog is not None:
1132	for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons:
1133	if sKrnlLog.find(sNeedle) > 0:
1134	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1135	if fStopOnHit:
1136	return True;
1137	fFoundSomething = True;
1138
1139	#
1140	# Loop thru the simple stuff.
1141	#
1142	fFoundSomething = False;
1143	for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons:
1144	if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0):
1145	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1146	if fStopOnHit:
1147	return True;
1148	fFoundSomething = True;
1149
1150	# Continue with vga text.
1151	if sVgaText:
1152	for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons:
1153	if sVgaText.find(sNeedle) > 0:
1154	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1155	if fStopOnHit:
1156	return True;
1157	fFoundSomething = True;
1158	_ = sInfoText;
1159
1160	# Continue with screen hashes.
1161	if sScreenHash is not None:
1162	for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons:
1163	if sScreenHash == sHash:
1164	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1165	if fStopOnHit:
1166	return True;
1167	fFoundSomething = True;
1168
1169	# Check VBoxHardening.log.
1170	if sNtHardLog is not None:
1171	for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons:
1172	if sNtHardLog.find(sNeedle) > 0:
1173	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1174	if fStopOnHit:
1175	return True;
1176	fFoundSomething = True;
1177
1178	#
1179	# Complicated stuff.
1180	#
1181	dLogs = {
1182	'sVMLog': sVMLog,
1183	'sNtHardLog': sNtHardLog,
1184	'sScreenHash': sScreenHash,
1185	'sKrnlLog': sKrnlLog,
1186	'sVgaText': sVgaText,
1187	'sInfoText': sInfoText,
1188	};
1189
1190	# info.txt.
1191	if sInfoText:
1192	for sNeedle, fnHandler in self.katInfoTextHandlers:
1193	if sInfoText.find(sNeedle) > 0:
1194	(fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs);
1195	if tReason is not None:
1196	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1197	if fStop:
1198	return True;
1199	fFoundSomething = True;
1200
1201	#
1202	# Check for repeated reboots...
1203	#
1204	if sVMLog is not None:
1205	cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\'');
1206	if cResets > 10:
1207	return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult,
1208	sComment = 'Counted %s reboots' % (cResets,));
1209
1210	return fFoundSomething;
1211
1212	#
1213	# Check if we got any VM or/and kernel logs. Treat them as sets in
1214	# case we run multiple VMs here (this is of course ASSUMING they
1215	# appear in the order that terminateVmBySession uploads them).
1216	#
1217	cTimes = 0;
1218	sVMLog = None;
1219	sNtHardLog = None;
1220	sScreenHash = None;
1221	sKrnlLog = None;
1222	sVgaText = None;
1223	sInfoText = None;
1224	for oFile in oFailedResult.aoFiles:
1225	if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm:
1226	if 'VBoxHardening.log' not in oFile.sFile:
1227	if sVMLog is not None:
1228	if investigateLogSet() is True:
1229	return True;
1230	cTimes += 1;
1231	sInfoText = None;
1232	sVgaText = None;
1233	sKrnlLog = None;
1234	sScreenHash = None;
1235	sNtHardLog = None;
1236	sVMLog = oCaseFile.getLogFile(oFile);
1237	else:
1238	sNtHardLog = oCaseFile.getLogFile(oFile);
1239	elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel:
1240	sKrnlLog = oCaseFile.getLogFile(oFile);
1241	elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText:
1242	sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]);
1243	elif oFile.sKind == TestResultFileData.ksKind_InfoCollection:
1244	sInfoText = oCaseFile.getLogFile(oFile);
1245	elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure:
1246	sScreenHash = oCaseFile.getScreenshotSha256(oFile);
1247	if sScreenHash is not None:
1248	sScreenHash = sScreenHash.lower();
1249	self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,));
1250
1251	if ( sVMLog is not None \
1252	or sNtHardLog is not None \
1253	or cTimes == 0) \
1254	and investigateLogSet() is True:
1255	return True;
1256
1257	return None;
1258
1259
1260	def isResultFromVMRun(self, oFailedResult, sResultLog):
1261	"""
1262	Checks if this result and corresponding log snippet looks like a VM run.
1263	"""
1264
1265	# Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log.
1266	if sResultLog.find(' startVm') > 0:
1267	return True;
1268
1269	# Any other indicators? No?
1270	_ = oFailedResult;
1271	return False;
1272
1273	def investigateVBoxVMTest(self, oCaseFile, fSingleVM):
1274	"""
1275	Checks out a VBox VM test.
1276
1277	This is generic investigation of a test running one or more VMs, like
1278	for example a smoke test or a guest installation test.
1279
1280	The fSingleVM parameter is a hint, which probably won't come in useful.
1281	"""
1282	_ = fSingleVM;
1283
1284	#
1285	# Get a list of test result failures we should be looking into and the main log.
1286	#
1287	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1288	sMainLog = oCaseFile.getMainLog();
1289
1290	#
1291	# There are a set of errors ending up on the top level result record.
1292	# Should deal with these first.
1293	#
1294	if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree:
1295	# Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always
1296	# be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to
1297	# ignore other failures in the test if we're not a little bit careful here.
1298	if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0:
1299	oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11);
1300	return self.caseClosed(oCaseFile);
1301
1302	# Hang after starting VBoxSVC (e.g. idTestSet=136307258)
1303	if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at',
1304	(' Timeout: ', ' Attempting to abort child...',) ):
1305	if sMainLog.find('* glibc detected * /') > 0:
1306	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption);
1307	else:
1308	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang);
1309	return self.caseClosed(oCaseFile);
1310
1311	# Look for heap corruption without visible hang.
1312	if sMainLog.find('* glibc detected * /') > 0 \
1313	or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374
1314	oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption);
1315	return self.caseClosed(oCaseFile);
1316
1317	# Out of memory w/ timeout.
1318	if sMainLog.find('sErrId=HostMemoryLow') > 0:
1319	oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow);
1320	return self.caseClosed(oCaseFile);
1321
1322	# Stale files like vts_rm.exe (windows).
1323	offEnd = sMainLog.rfind('* The test driver exits successfully. *');
1324	if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0:
1325	oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files);
1326	return self.caseClosed(oCaseFile);
1327
1328	#
1329	# XPCOM screwup
1330	#
1331	if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0:
1332	oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build);
1333	return self.caseClosed(oCaseFile);
1334
1335	#
1336	# Go thru each failed result.
1337	#
1338	for oFailedResult in aoFailedResults:
1339	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1340	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1341	if oFailedResult.sName == 'Installing VirtualBox':
1342	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1343
1344	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1345	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1346
1347	elif self.isResultFromVMRun(oFailedResult, sResultLog):
1348	self.investigateVMResult(oCaseFile, oFailedResult, sResultLog);
1349
1350	elif sResultLog.find('most likely not unique') > 0:
1351	oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult)
1352	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1353	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1354
1355	elif sResultLog.find('The machine is not mutable (state is ') > 0:
1356	self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem');
1357	oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult);
1358
1359	elif sResultLog.find('** error: no action was specified') > 0 \
1360	or sResultLog.find('(len(self._asXml, asText))') > 0:
1361	oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult);
1362
1363	else:
1364	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1365	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1366
1367	#
1368	# Report home and close the case if we got them all, otherwise log it.
1369	#
1370	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1371	return self.caseClosed(oCaseFile);
1372
1373	if oCaseFile.dReasonForResultId:
1374	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1375	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1376	else:
1377	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1378	return False;
1379
1380
1381	def reasoningFailures(self):
1382	"""
1383	Guess the reason for failures.
1384	"""
1385	#
1386	# Get a list of failed test sets without any assigned failure reason.
1387	#
1388	cGot = 0;
1389	if self.oConfig.aidTestSets is None or len(self.oConfig.aidTestSets) == 0:
1390	aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack,
1391	tsNow = self.tsNow);
1392	else:
1393	aoTestSets = [self.oTestSetLogic.getById(idTestSet) for idTestSet in self.oConfig.aidTestSets];
1394	for oTestSet in aoTestSets:
1395	self.dprint(u'----------------------------------- #%u, status %s -----------------------------------'
1396	% ( oTestSet.idTestSet, oTestSet.enmStatus,));
1397
1398	#
1399	# Open a case file and assign it to the right investigator.
1400	#
1401	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet);
1402	oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated);
1403	oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox);
1404	oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated);
1405	oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig);
1406
1407	oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase);
1408
1409	if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
1410	self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,));
1411	fRc = self.investigateBadTestBox(oCaseFile);
1412
1413	elif oCaseFile.isVBoxUnitTest():
1414	self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,));
1415	fRc = self.investigateVBoxUnitTest(oCaseFile);
1416
1417	elif oCaseFile.isVBoxInstallTest():
1418	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1419	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1420
1421	elif oCaseFile.isVBoxUSBTest():
1422	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1423	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1424
1425	elif oCaseFile.isVBoxStorageTest():
1426	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1427	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1428
1429	elif oCaseFile.isVBoxGAsTest():
1430	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1431	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1432
1433	elif oCaseFile.isVBoxAPITest():
1434	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1435	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1436
1437	elif oCaseFile.isVBoxBenchmarkTest():
1438	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1439	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1440
1441	elif oCaseFile.isVBoxSmokeTest():
1442	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1443	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1444
1445	elif oCaseFile.isVBoxSerialTest():
1446	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1447	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1448
1449	else:
1450	self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,));
1451	fRc = False;
1452	cGot += fRc is True;
1453
1454	self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), ));
1455	return 0;
1456
1457
1458	def main(self):
1459	"""
1460	The 'main' function.
1461	Return exit code (0, 1, etc).
1462	"""
1463	# Database stuff.
1464	self.oDb = TMDatabaseConnection()
1465	self.oTestResultLogic = TestResultLogic(self.oDb);
1466	self.oTestSetLogic = TestSetLogic(self.oDb);
1467	self.oFailureReasonLogic = FailureReasonLogic(self.oDb);
1468	self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb);
1469	self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory);
1470	self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory);
1471
1472	# Get a fix on our 'now' before we do anything..
1473	self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
1474	self.tsNow = self.oDb.fetchOne();
1475
1476	# If we're suppost to commit anything we need to get our user ID.
1477	rcExit = 0;
1478	if self.oConfig.fRealRun:
1479	self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
1480	if self.oLogin is None:
1481	rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
1482	else:
1483	self.uidSelf = self.oLogin.uid;
1484
1485	#
1486	# Do the stuff.
1487	#
1488	if rcExit == 0:
1489	rcExit = self.selfCheck();
1490	if rcExit == 0:
1491	rcExit = self.badTestBoxManagement();
1492	rcExit2 = self.reasoningFailures();
1493	if rcExit == 0:
1494	rcExit = rcExit2;
1495	# Redo the bad testbox management after failure reasons have been assigned (got timing issues).
1496	if rcExit == 0:
1497	rcExit = self.badTestBoxManagement();
1498
1499	# Cleanup.
1500	self.oFailureReasonLogic = None;
1501	self.oTestResultFailureLogic = None;
1502	self.oTestSetLogic = None;
1503	self.oTestResultLogic = None;
1504	self.oDb.close();
1505	self.oDb = None;
1506	if self.oLogFile is not None:
1507	self.oLogFile.close();
1508	self.oLogFile = None;
1509	return rcExit;
1510
1511	if __name__ == '__main__':
1512	sys.exit(VirtualTestSheriff().main());
1513

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py@ 78728

Download in other formats: