virtual_test_sheriff.py@ 79092

Last change on this file since 79092 was 79092, checked in by vboxsync, 6 years ago
ValKit,++: Pylint 2.3.1 adjustments.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 76.8 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: virtual_test_sheriff.py 79092 2019-06-11 15:26:40Z vboxsync $
4	# pylint: disable=line-too-long
5
6	"""
7	Virtual Test Sheriff.
8
9	Duties:
10	- Try to a assign failure reasons to recently failed tests.
11	- Reboot or disable bad test boxes.
12
13	"""
14
15	from __future__ import print_function;
16
17	__copyright__ = \
18	"""
19	Copyright (C) 2012-2019 Oracle Corporation
20
21	This file is part of VirtualBox Open Source Edition (OSE), as
22	available from http://www.virtualbox.org. This file is free software;
23	you can redistribute it and/or modify it under the terms of the GNU
24	General Public License (GPL) as published by the Free Software
25	Foundation, in version 2 as it comes in the "COPYING" file of the
26	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
27	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
32	VirtualBox OSE distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37	"""
38	__version__ = "$Revision: 79092 $"
39
40
41	# Standard python imports
42	import sys;
43	import os;
44	import hashlib;
45	import subprocess;
46	import smtplib
47	from email.mime.multipart import MIMEMultipart
48	from email.mime.text import MIMEText
49	from email.utils import COMMASPACE
50
51	if sys.version_info[0] >= 3:
52	from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias
53	else:
54	from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias
55	from optparse import OptionParser; # pylint: disable=deprecated-module
56	from PIL import Image; # pylint: disable=import-error
57
58	# Add Test Manager's modules path
59	g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
60	sys.path.append(g_ksTestManagerDir);
61
62	# Test Manager imports
63	from testmanager.core.db import TMDatabaseConnection;
64	from testmanager.core.build import BuildDataEx;
65	from testmanager.core.failurereason import FailureReasonLogic;
66	from testmanager.core.testbox import TestBoxLogic, TestBoxData;
67	from testmanager.core.testcase import TestCaseDataEx;
68	from testmanager.core.testgroup import TestGroupData;
69	from testmanager.core.testset import TestSetLogic, TestSetData;
70	from testmanager.core.testresults import TestResultLogic, TestResultFileData;
71	from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData;
72	from testmanager.core.useraccount import UserAccountLogic;
73	from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \
74	g_ksAlertSubject, g_asAlertList, g_ksLomPassword;
75
76	# Python 3 hacks:
77	if sys.version_info[0] >= 3:
78	xrange = range; # pylint: disable=redefined-builtin,invalid-name
79
80
81	class VirtualTestSheriffCaseFile(object):
82	"""
83	A failure investigation case file.
84
85	"""
86
87
88	## Max log file we'll read into memory. (256 MB)
89	kcbMaxLogRead = 0x10000000;
90
91	def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase):
92	self.oSheriff = oSheriff;
93	self.oTestSet = oTestSet; # TestSetData
94	self.oTree = oTree; # TestResultDataEx
95	self.oBuild = oBuild; # BuildDataEx
96	self.oTestBox = oTestBox; # TestBoxData
97	self.oTestGroup = oTestGroup; # TestGroupData
98	self.oTestCase = oTestCase; # TestCaseDataEx
99	self.sMainLog = ''; # The main log file. Empty string if not accessible.
100
101	# Generate a case file name.
102	self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,)
103	self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \
104	% ( self.oTestSet.idTestSet,
105	self.oTestCase.sName,
106	self.oTestBox.sName,
107	self.oTestBox.sOs,
108	self.oTestBox.sOsVersion,
109	self.oTestBox.sCpuArch,
110	self.oTestBox.sCpuName,
111	self.oTestBox.sCpuVendor,
112	self.oBuild.oCat.sProduct,
113	self.oBuild.oCat.sBranch,
114	self.oBuild.oCat.sType,
115	self.oBuild.iRevision, );
116
117	# Investigation notes.
118	self.tReason = None; # None or one of the ktReason_XXX constants.
119	self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult.
120	self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult.
121
122	#
123	# Reason.
124	#
125
126	def noteReason(self, tReason):
127	""" Notes down a possible reason. """
128	self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,));
129	self.tReason = tReason;
130	return True;
131
132	def noteReasonForId(self, tReason, idTestResult, sComment = None):
133	""" Notes down a possible reason for a specific test result. """
134	self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s'
135	% (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason,
136	(u' (%s)' % (sComment,)) if sComment is not None else ''));
137	self.dReasonForResultId[idTestResult] = tReason;
138	if sComment is not None:
139	self.dCommentForResultId[idTestResult] = sComment;
140	return True;
141
142
143	#
144	# Test classification.
145	#
146
147	def isVBoxTest(self):
148	""" Test classification: VirtualBox (using the build) """
149	return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ];
150
151	def isVBoxUnitTest(self):
152	""" Test case classification: The unit test doing all our testcase/*.cpp stuff. """
153	return self.isVBoxTest() \
154	and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests');
155
156	def isVBoxInstallTest(self):
157	""" Test case classification: VirtualBox Guest installation test. """
158	return self.isVBoxTest() \
159	and self.oTestCase.sName.lower().startswith('install:');
160
161	def isVBoxUSBTest(self):
162	""" Test case classification: VirtualBox USB test. """
163	return self.isVBoxTest() \
164	and self.oTestCase.sName.lower().startswith('usb:');
165
166	def isVBoxStorageTest(self):
167	""" Test case classification: VirtualBox Storage test. """
168	return self.isVBoxTest() \
169	and self.oTestCase.sName.lower().startswith('storage:');
170
171	def isVBoxGAsTest(self):
172	""" Test case classification: VirtualBox Guest Additions test. """
173	return self.isVBoxTest() \
174	and ( self.oTestCase.sName.lower().startswith('guest additions')
175	or self.oTestCase.sName.lower().startswith('ga\'s tests'));
176
177	def isVBoxAPITest(self):
178	""" Test case classification: VirtualBox API test. """
179	return self.isVBoxTest() \
180	and self.oTestCase.sName.lower().startswith('api:');
181
182	def isVBoxBenchmarkTest(self):
183	""" Test case classification: VirtualBox Benchmark test. """
184	return self.isVBoxTest() \
185	and self.oTestCase.sName.lower().startswith('benchmark:');
186
187	def isVBoxSmokeTest(self):
188	""" Test case classification: Smoke test. """
189	return self.isVBoxTest() \
190	and self.oTestCase.sName.lower().startswith('smoketest');
191
192	def isVBoxSerialTest(self):
193	""" Test case classification: Smoke test. """
194	return self.isVBoxTest() \
195	and self.oTestCase.sName.lower().startswith('serial:');
196
197
198	#
199	# Utility methods.
200	#
201
202	def getMainLog(self):
203	"""
204	Tries to read the main log file since this will be the first source of information.
205	"""
206	if self.sMainLog:
207	return self.sMainLog;
208	(oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb');
209	if oFile is not None:
210	try:
211	self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
212	except Exception as oXcpt:
213	self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,))
214	self.sMainLog = '';
215	else:
216	self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,));
217	return self.sMainLog;
218
219	def getLogFile(self, oFile):
220	"""
221	Tries to read the given file as a utf-8 log file.
222	oFile is a TestFileDataEx instance.
223	Returns empty string if problems opening or reading the file.
224	"""
225	sContent = '';
226	(oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
227	if oFile is not None:
228	try:
229	sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
230	except Exception as oXcpt:
231	self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,))
232	else:
233	self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,));
234	return sContent;
235
236	def getScreenshotSha256(self, oFile):
237	"""
238	Tries to read the given screenshot file, uncompress it, and do SHA-2
239	on the raw pixels.
240	Returns SHA-2 digest string on success, None on failure.
241	"""
242	(oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
243	try:
244	abImageFile = oImgFile.read();
245	except Exception as oXcpt:
246	self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,))
247	else:
248	try:
249	oImage = Image.open(StringIO(abImageFile));
250	except Exception as oXcpt:
251	self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,))
252	else:
253	try:
254	oHash = hashlib.sha256();
255	oHash.update(oImage.tostring());
256	except Exception as oXcpt:
257	self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,))
258	else:
259	return oHash.hexdigest();
260	return None;
261
262
263
264	def isSingleTestFailure(self):
265	"""
266	Figure out if this is a single test failing or if it's one of the
267	more complicated ones.
268	"""
269	if self.oTree.cErrors == 1:
270	return True;
271	if self.oTree.deepCountErrorContributers() <= 1:
272	return True;
273	return False;
274
275
276
277	class VirtualTestSheriff(object): # pylint: disable=too-few-public-methods
278	"""
279	Add build info into Test Manager database.
280	"""
281
282	## The user account for the virtual sheriff.
283	ksLoginName = 'vsheriff';
284
285	def __init__(self):
286	"""
287	Parse command line.
288	"""
289	self.oDb = None;
290	self.tsNow = None;
291	self.oTestResultLogic = None;
292	self.oTestSetLogic = None;
293	self.oFailureReasonLogic = None; # FailureReasonLogic;
294	self.oTestResultFailureLogic = None; # TestResultFailureLogic
295	self.oLogin = None;
296	self.uidSelf = -1;
297	self.oLogFile = None;
298	self.asBsodReasons = [];
299	self.asUnitTestReasons = [];
300
301	oParser = OptionParser();
302	oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int',
303	help = 'When to start specified as hours relative to current time. Defauls is right now.', );
304	oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int',
305	help = 'Work period specified in hours. Defauls is 2 hours.');
306	oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
307	help = 'Whether to commit the findings to the database. Default is a dry run.');
308	oParser.add_option('--testset', dest = 'aidTestSets', metavar = '<id>', default = [], type = 'int', action = 'append',
309	help = 'Only investigate this one. Accumulates IDs when repeated.');
310	oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
311	help = 'Quiet execution');
312	oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None,
313	help = 'Where to log messages.');
314	oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
315	help = 'Enables debug mode.');
316
317	(self.oConfig, _) = oParser.parse_args();
318
319	if self.oConfig.sLogFile:
320	self.oLogFile = open(self.oConfig.sLogFile, "a");
321	self.oLogFile.write('VirtualTestSheriff: $Revision: 79092 $ \n');
322
323
324	def eprint(self, sText):
325	"""
326	Prints error messages.
327	Returns 1 (for exit code usage.)
328	"""
329	print('error: %s' % (sText,));
330	if self.oLogFile is not None:
331	self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8'));
332	return 1;
333
334	def dprint(self, sText):
335	"""
336	Prints debug info.
337	"""
338	if self.oConfig.fDebug:
339	if not self.oConfig.fQuiet:
340	print('debug: %s' % (sText, ));
341	if self.oLogFile is not None:
342	self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8'));
343	return 0;
344
345	def vprint(self, sText):
346	"""
347	Prints verbose info.
348	"""
349	if not self.oConfig.fQuiet:
350	print('info: %s' % (sText,));
351	if self.oLogFile is not None:
352	self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8'));
353	return 0;
354
355	def getFailureReason(self, tReason):
356	""" Gets the failure reason object for tReason. """
357	return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]);
358
359	def selfCheck(self):
360	""" Does some self checks, looking up things we expect to be in the database and such. """
361	rcExit = 0;
362	for sAttr in dir(self.__class__):
363	if sAttr.startswith('ktReason_'):
364	tReason = getattr(self.__class__, sAttr);
365	oFailureReason = self.getFailureReason(tReason);
366	if oFailureReason is None:
367	rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!'
368	% (tReason[1], tReason[0],));
369
370	# Check the user account as well.
371	if self.oLogin is None:
372	oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
373	if oLogin is None:
374	rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
375	return rcExit;
376
377	def sendEmailAlert(self, uidAuthor, sBodyText):
378	"""
379	Sends email alert.
380	"""
381
382	# Get author email
383	self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,));
384	sFrom = self.oDb.fetchOne();
385	if sFrom is not None:
386	sFrom = sFrom[0];
387	else:
388	sFrom = g_ksAlertFrom;
389
390	# Gather recipient list.
391	asEmailList = [];
392	for sUser in g_asAlertList:
393	self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,));
394	sEmail = self.oDb.fetchOne();
395	if sEmail:
396	asEmailList.append(sEmail[0]);
397	if not asEmailList:
398	return self.eprint('No email addresses to send alter to!');
399
400	# Compose the message.
401	oMsg = MIMEMultipart();
402	oMsg['From'] = sFrom;
403	oMsg['To'] = COMMASPACE.join(asEmailList);
404	oMsg['Subject'] = g_ksAlertSubject;
405	oMsg.attach(MIMEText(sBodyText, 'plain'))
406
407	# Try send it.
408	try:
409	oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort);
410	oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string())
411	oSMTP.quit()
412	except smtplib.SMTPException as oXcpt:
413	return self.eprint('Failed to send mail: %s' % (oXcpt,));
414
415	return 0;
416
417	def badTestBoxManagement(self):
418	"""
419	Looks for bad test boxes and first tries once to reboot them then disables them.
420	"""
421	rcExit = 0;
422
423	#
424	# We skip this entirely if we're running in the past and not in harmless debug mode.
425	#
426	if self.oConfig.cStartHoursAgo != 0 \
427	and (not self.oConfig.fDebug or self.oConfig.fRealRun):
428	return rcExit;
429	tsNow = self.tsNow if self.oConfig.fDebug else None;
430	cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2;
431	oTestBoxLogic = TestBoxLogic(self.oDb);
432
433	#
434	# Generate a list of failures reasons we consider bad-testbox behavior.
435	#
436	aidFailureReasons = [
437	self.getFailureReason(self.ktReason_Host_DriverNotLoaded).idFailureReason,
438	self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason,
439	self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason,
440	self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason,
441	];
442
443	#
444	# Get list of bad test boxes for given period and check them out individually.
445	#
446	aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow,
447	aidFailureReasons = aidFailureReasons);
448	for idTestBox in aidBadTestBoxes:
449	# Skip if the testbox is already disabled or has a pending reboot command.
450	try:
451	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
452	except Exception as oXcpt:
453	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
454	continue;
455	if not oTestBox.fEnabled:
456	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
457	% ( idTestBox, oTestBox.sName, ));
458	continue;
459	if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
460	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
461	% ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
462	continue;
463
464	# Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
465	aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow);
466	cOkay = 0;
467	cBad = 0;
468	iFirstOkay = len(aoSets);
469	for iSet, oSet in enumerate(aoSets):
470	if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
471	cBad += 1;
472	else:
473	# Check for bad failure reasons.
474	oFailure = None;
475	if oSet.enmStatus in TestSetData.kasBadTestStatuses:
476	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oSet.idTestSet)
477	aoFailedResults = oTree.getListOfFailures();
478	for oFailedResult in aoFailedResults:
479	oFailure = self.oTestResultFailureLogic.getById(oFailedResult.idTestResult);
480	if oFailure is not None and oFailure.idFailureReason in aidFailureReasons:
481	break;
482	else:
483	oFailure = None;
484	if oFailure is not None:
485	cBad += 1;
486	else:
487	# This is an okay test result then.
488	## @todo maybe check the elapsed time here, it could still be a bad run?
489	cOkay += 1;
490	if iFirstOkay > iSet:
491	iFirstOkay = iSet;
492	if iSet > 10:
493	break;
494
495	# We react if there are two or more bad-testbox statuses at the head of the
496	# history and at least three in the last 10 results.
497	if iFirstOkay >= 2 and cBad > 2:
498	if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
499	sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
500	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
501	self.vprint(sComment);
502	self.sendEmailAlert(self.uidSelf, sComment);
503	if self.oConfig.fRealRun is True:
504	try:
505	oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
506	sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
507	% (iFirstOkay, cBad, cOkay),);
508	except Exception as oXcpt:
509	rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
510	else:
511	sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
512	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
513	self.vprint(sComment);
514	self.sendEmailAlert(self.uidSelf, sComment);
515	if self.oConfig.fRealRun is True:
516	try:
517	oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
518	sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
519	% (iFirstOkay, cBad, cOkay),);
520	except Exception as oXcpt:
521	rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
522	else:
523	self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u'
524	% ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
525
526	#
527	# Reset hanged testboxes
528	#
529	cStatusTimeoutMins = 10;
530
531	self.oDb.execute('SELECT TestBoxStatuses.idTestBox\n'
532	' FROM TestBoxStatuses, TestBoxes\n'
533	' WHERE TestBoxStatuses.tsUpdated >= (CURRENT_TIMESTAMP - interval \'%s hours\')\n'
534	' AND TestBoxStatuses.tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')\n'
535	' AND TestBoxStatuses.idTestBox = TestBoxes.idTestBox\n'
536	' AND Testboxes.tsExpire = \'infinity\'::timestamp', (cHoursBack,cStatusTimeoutMins));
537	for idTestBox in self.oDb.fetchAll():
538	idTestBox = idTestBox[0];
539	try:
540	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
541	except Exception as oXcpt:
542	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
543	continue;
544	# Skip if the testbox is already disabled, already reset or there's no iLOM
545	if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0:
546	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
547	% ( idTestBox, oTestBox.sName, ));
548	continue;
549	## @todo get iLOM credentials from a table?
550	sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,);
551	try:
552	oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True);
553	sStdout = oPs.communicate()[0];
554	iRC = oPs.wait();
555
556	oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,);
557	oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True);
558
559	sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout);
560	self.vprint(sComment);
561	self.sendEmailAlert(self.uidSelf, sComment);
562
563	except Exception as oXcpt:
564	rcExit = self.eprint(u'Error resetting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
565
566	return rcExit;
567
568
569	## @name Failure reasons we know.
570	## @{
571	ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' );
572	ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' );
573	ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' );
574	ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' );
575	ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' );
576	ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' );
577	ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' );
578	ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' );
579	ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' );
580	ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' );
581	ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' );
582	ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' );
583	ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' );
584	ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' );
585	ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' );
586	ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' );
587	ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' );
588	ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' );
589	ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' );
590	ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' );
591	ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' );
592	ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' );
593	ktReason_Host_Install_Hang = ( 'Host', 'Install hang' );
594	ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' );
595	ktReason_Host_TSTInfo_Accuracy_OOR = ( 'Host', 'TSTInfo accuracy out of range' );
596	ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' );
597	ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' );
598	ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' );
599	ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' );
600	ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' );
601	ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' );
602	ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' );
603	ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' );
604	ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' );
605	ktReason_API_std_bad_alloc = ( 'API / (XP)COM', 'std::bad_alloc' );
606	ktReason_API_Digest_Mismatch = ( 'API / (XP)COM', 'Digest mismatch' );
607	ktReason_API_MoveVM_SharingViolation = ( 'API / (XP)COM', 'MoveVM sharing violation' );
608	ktReason_API_MoveVM_InvalidParameter = ( 'API / (XP)COM', 'MoveVM invalid parameter' );
609	ktReason_API_Open_Session_Failed = ( 'API / (XP)COM', 'Open session failed' );
610	ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' );
611	ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' );
612	ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' );
613	ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' );
614	ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' );
615	ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' );
616	ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' );
617	ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' );
618	ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' );
619	ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' );
620	ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' );
621	ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' );
622	ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' );
623	ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' );
624	ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' );
625	ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' );
626	ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' );
627	## @}
628
629	## BSOD category.
630	ksBsodCategory = 'BSOD';
631	## Special reason indicating that the flesh and blood sheriff has work to do.
632	ksBsodAddNew = 'Add new BSOD';
633
634	## Unit test category.
635	ksUnitTestCategory = 'Unit';
636	## Special reason indicating that the flesh and blood sheriff has work to do.
637	ksUnitTestAddNew = 'Add new';
638
639	## Used for indica that we shouldn't report anything for this test result ID and
640	## consider promoting the previous error to test set level if it's the only one.
641	ktHarmless = ( 'Probably', 'Caused by previous error' );
642
643
644	def caseClosed(self, oCaseFile):
645	"""
646	Reports the findings in the case and closes it.
647	"""
648	#
649	# Log it and create a dReasonForReasultId we can use below.
650	#
651	dCommentForResultId = oCaseFile.dCommentForResultId;
652	if oCaseFile.dReasonForResultId:
653	# Must weed out ktHarmless.
654	dReasonForResultId = {};
655	for idKey, tReason in oCaseFile.dReasonForResultId.items():
656	if tReason is not self.ktHarmless:
657	dReasonForResultId[idKey] = tReason;
658	if not dReasonForResultId:
659	self.vprint(u'TODO: Closing %s without a real reason, only %s.'
660	% (oCaseFile.sName, oCaseFile.dReasonForResultId));
661	return False;
662
663	# Try promote to single reason.
664	atValues = dReasonForResultId.values();
665	fSingleReason = True;
666	if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult:
667	self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],));
668	elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]):
669	self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0]));
670	else:
671	fSingleReason = False;
672	if fSingleReason:
673	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], };
674	if dCommentForResultId:
675	dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], };
676	elif oCaseFile.tReason is not None:
677	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, };
678	else:
679	self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,));
680	return False;
681
682	self.vprint(u'Closing %s with following reason%s: %s'
683	% ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, ));
684
685	#
686	# Add the test failure reason record(s).
687	#
688	for idTestResult, tReason in dReasonForResultId.items():
689	oFailureReason = self.getFailureReason(tReason);
690	if oFailureReason is not None:
691	sComment = 'Set by $Revision: 79092 $' # Handy for reverting later.
692	if idTestResult in dCommentForResultId:
693	sComment += ': ' + dCommentForResultId[idTestResult];
694
695	oAdd = TestResultFailureData();
696	oAdd.initFromValues(idTestResult = idTestResult,
697	idFailureReason = oFailureReason.idFailureReason,
698	uidAuthor = self.uidSelf,
699	idTestSet = oCaseFile.oTestSet.idTestSet,
700	sComment = sComment,);
701	if self.oConfig.fRealRun:
702	try:
703	self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True);
704	except Exception as oXcpt:
705	self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s'
706	% (oXcpt, oAdd, oCaseFile.sLongName,));
707	else:
708	self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],));
709	return True;
710
711	#
712	# Tools for assiting log parsing.
713	#
714
715	@staticmethod
716	def matchFollowedByLines(sStr, off, asFollowingLines):
717	""" Worker for isThisFollowedByTheseLines. """
718
719	# Advance off to the end of the line.
720	off = sStr.find('\n', off);
721	if off < 0:
722	return False;
723	off += 1;
724
725	# Match each string with the subsequent lines.
726	for iLine, sLine in enumerate(asFollowingLines):
727	offEnd = sStr.find('\n', off);
728	if offEnd < 0:
729	return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0;
730	if sLine and sStr.find(sLine, off, offEnd) < 0:
731	return False;
732
733	# next line.
734	off = offEnd + 1;
735
736	return True;
737
738	@staticmethod
739	def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines):
740	"""
741	Looks for a line contining sFirst which is then followed by lines
742	with the strings in asFollowingLines. (No newline chars anywhere!)
743	Returns True / False.
744	"""
745	off = sStr.find(sFirst, 0);
746	while off >= 0:
747	if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines):
748	return True;
749	off = sStr.find(sFirst, off + 1);
750	return False;
751
752	@staticmethod
753	def findAndReturnRestOfLine(sHaystack, sNeedle):
754	"""
755	Looks for sNeedle in sHaystack.
756	Returns The text following the needle up to the end of the line.
757	Returns None if not found.
758	"""
759	if sHaystack is None:
760	return None;
761	off = sHaystack.find(sNeedle);
762	if off < 0:
763	return None;
764	off += len(sNeedle)
765	offEol = sHaystack.find('\n', off);
766	if offEol < 0:
767	offEol = len(sHaystack);
768	return sHaystack[off:offEol]
769
770	@staticmethod
771	def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle):
772	"""
773	Looks for sNeedle in zeroe or more haystacks (asHaystack).
774	Returns The text following the first needed found up to the end of the line.
775	Returns None if not found.
776	"""
777	for sHaystack in asHaystacks:
778	sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle);
779	if sRet is not None:
780	return sRet;
781	return None;
782
783
784	#
785	# The investigative units.
786	#
787
788	katSimpleInstallUninstallMainLogReasons = [
789	# ( Whether to stop on hit, reason tuple, needle text. )
790	( False, ktReason_Host_LeftoverService,
791	'SERVICE_NAME: vbox' ),
792	];
793
794	kdatSimpleInstallUninstallMainLogReasonsPerOs = {
795	'darwin': [
796	# ( Whether to stop on hit, reason tuple, needle text. )
797	( True, ktReason_Host_DriverNotUnloading,
798	'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ),
799	],
800	'linux': [
801	# ( Whether to stop on hit, reason tuple, needle text. )
802	( True, ktReason_Host_DriverNotCompilable,
803	'This system is not currently set up to build kernel modules' ),
804	( True, ktReason_Host_DriverNotCompilable,
805	'This system is currently not set up to build kernel modules' ),
806	( True, ktReason_Host_InstallationFailed,
807	'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ),
808	( True, ktReason_Host_DriverNotUnloading,
809	'Cannot unload module vboxdrv'),
810	],
811	'solaris': [
812	# ( Whether to stop on hit, reason tuple, needle text. )
813	( True, ktReason_Host_InstallationFailed,
814	'svcadm: Couldn\'t bind to svc.configd.' ),
815	( True, ktReason_Host_InstallationFailed,
816	'pkgadd: ERROR: postinstall script did not complete successfully' ),
817	( True, ktReason_Host_DriverNotUnloading,
818	'can\'t unload the module: Device busy' ),
819	],
820	};
821
822
823	def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall):
824	"""
825	Investigates an install or uninstall failure.
826
827	We lump the two together since the installation typically also performs
828	an uninstall first and will be seeing similar issues to the uninstall.
829	"""
830
831	if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut:
832	oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult)
833	return True;
834
835	atSimple = self.katSimpleInstallUninstallMainLogReasons;
836	if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs:
837	atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple;
838
839	fFoundSomething = False;
840	for fStopOnHit, tReason, sNeedle in atSimple:
841	if sResultLog.find(sNeedle) > 0:
842	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
843	if fStopOnHit:
844	return True;
845	fFoundSomething = True;
846
847	return fFoundSomething if fFoundSomething else None;
848
849
850	def investigateBadTestBox(self, oCaseFile):
851	"""
852	Checks out bad-testbox statuses.
853	"""
854	_ = oCaseFile;
855	return False;
856
857
858	def investigateVBoxUnitTest(self, oCaseFile):
859	"""
860	Checks out a VBox unittest problem.
861	"""
862
863	#
864	# Process simple test case failures first, using their name as reason.
865	# We do the reason management just like for BSODs.
866	#
867	cRelevantOnes = 0;
868	sMainLog = oCaseFile.getMainLog();
869	aoFailedResults = oCaseFile.oTree.getListOfFailures();
870	for oFailedResult in aoFailedResults:
871	if oFailedResult is oCaseFile.oTree:
872	self.vprint('TODO: toplevel failure');
873	cRelevantOnes += 1
874
875	elif oFailedResult.sName == 'Installing VirtualBox':
876	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
877	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
878	cRelevantOnes += 1
879
880	elif oFailedResult.sName == 'Uninstalling VirtualBox':
881	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
882	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
883	cRelevantOnes += 1
884
885	elif oFailedResult.oParent is not None:
886	# Get the 2nd level node because that's where we'll find the unit test name.
887	while oFailedResult.oParent.oParent is not None:
888	oFailedResult = oFailedResult.oParent;
889
890	# Only report a failure once.
891	if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId:
892	sKey = oFailedResult.sName;
893	if sKey.startswith('testcase/'):
894	sKey = sKey[9:];
895	if sKey in self.asUnitTestReasons:
896	tReason = ( self.ksUnitTestCategory, sKey );
897	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
898	else:
899	self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons));
900	tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew );
901	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey);
902	cRelevantOnes += 1
903	else:
904	self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,));
905
906	#
907	# If we've caught all the relevant ones by now, report the result.
908	#
909	if len(oCaseFile.dReasonForResultId) >= cRelevantOnes:
910	return self.caseClosed(oCaseFile);
911	return False;
912
913	def extractGuestCpuStack(self, sInfoText):
914	"""
915	Extracts the guest CPU stacks from the input file.
916
917	Returns a dictionary keyed by the CPU number, value being a list of
918	raw stack lines (no header).
919	Returns empty dictionary if no stacks where found.
920	"""
921	dRet = {};
922	off = 0;
923	while True:
924	# Find the stack.
925	offStart = sInfoText.find('=== start guest stack VCPU ', off);
926	if offStart < 0:
927	break;
928	offEnd = sInfoText.find('=== end guest stack', offStart + 20);
929	if offEnd >= 0:
930	offEnd += 3;
931	else:
932	offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20);
933	if offEnd < 0:
934	offEnd = len(sInfoText);
935
936	sStack = sInfoText[offStart : offEnd];
937	sStack = sStack.replace('\r',''); # paranoia
938	asLines = sStack.split('\n');
939
940	# Figure the CPU.
941	asWords = asLines[0].split();
942	if len(asWords) < 6 or not asWords[5].isdigit():
943	break;
944	iCpu = int(asWords[5]);
945
946	# Add it and advance.
947	dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]]
948	off = offEnd;
949	return dRet;
950
951	def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs):
952	""" Investigates kvm_lock_spinning deadlocks """
953	#
954	# Extract the stacks. We need more than one CPU to create a deadlock.
955	#
956	dStacks = self.extractGuestCpuStack(sInfoText);
957	self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),));
958	if len(dStacks) >= 2:
959	#
960	# Examin each of the stacks. Each must have kvm_lock_spinning in
961	# one of the first three entries.
962	#
963	cHits = 0;
964	for iCpu in dStacks:
965	asBacktrace = dStacks[iCpu];
966	for iFrame in xrange(min(3, len(asBacktrace))):
967	if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0:
968	cHits += 1;
969	break;
970	self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),));
971	if cHits == len(dStacks):
972	return (True, self.ktReason_VMM_kvm_lock_spinning);
973
974	_ = dLogs; _ = oCaseFile;
975	return (False, None);
976
977	def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs):
978	""" Investigates HalReturnToFirmware hangs """
979	del oCaseFile
980	del sInfoText
981	del dLogs
982	# hope that's sufficient
983	return (True, self.ktReason_Unknown_HalReturnToFirmware);
984
985	## Things we search a main or VM log for to figure out why something went bust.
986	katSimpleMainAndVmLogReasons = [
987	# ( Whether to stop on hit, reason tuple, needle text. )
988	( False, ktReason_Guru_Generic, 'GuruMeditation' ),
989	( False, ktReason_Guru_Generic, 'Guru Meditation' ),
990	( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ),
991	( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ),
992	( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ),
993	( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ),
994	( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ),
995	( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ),
996	( True, ktReason_Networking_Nonexistent_host_nic,
997	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
998	( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ),
999	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
1000	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
1001	( True, ktReason_API_std_bad_alloc, 'Unexpected exception: std::bad_alloc' ),
1002	( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ),
1003	( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ),
1004	( True, ktReason_Unknown_File_Not_Found,
1005	'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ),
1006	( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now.
1007	'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ),
1008	( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ),
1009	( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ),
1010	( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ),
1011	( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ),
1012	( False, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ),
1013	( False, ktReason_Unknown_VM_Start_Error, 'error: failed to open session for' ),
1014	( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ),
1015	];
1016
1017	## Things we search a VBoxHardening.log file for to figure out why something went bust.
1018	katSimpleVBoxHardeningLogReasons = [
1019	# ( Whether to stop on hit, reason tuple, needle text. )
1020	( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ),
1021	( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ),
1022	( True, ktReason_Host_TSTInfo_Accuracy_OOR, 'RTCRTSPTSTINFO::Accuracy::Millis: Out of range' ),
1023	];
1024
1025	## Things we search a kernel.log file for to figure out why something went bust.
1026	katSimpleKernelLogReasons = [
1027	# ( Whether to stop on hit, reason tuple, needle text. )
1028	( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ),
1029	( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn\'t work' ),
1030	( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ),
1031	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ),
1032	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ),
1033	];
1034
1035	## Things we search the _RIGHT_ _STRIPPED_ vgatext for.
1036	katSimpleVgaTextReasons = [
1037	# ( Whether to stop on hit, reason tuple, needle text. )
1038	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1039	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ),
1040	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1041	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n"
1042	"...trying to set up timer (IRQ0) through the 8259A ... failed.\n"
1043	"...trying to set up timer as Virtual Wire IRQ... failed.\n"
1044	"...trying to set up timer as ExtINT IRQ... failed :(.\n"
1045	"Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n"
1046	"and send a report. Then try booting with the 'noapic' option\n"
1047	"\n" ),
1048	( True, ktReason_OSInstall_GRUB_hang,
1049	"-----\nGRUB Loading stage2..\n\n\n\n" ),
1050	( True, ktReason_OSInstall_GRUB_hang,
1051	"-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent
1052	( True, ktReason_OSInstall_GRUB_hang,
1053	"-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent
1054	( True, ktReason_OSInstall_GRUB_hang,
1055	"-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again
1056	( True, ktReason_OSInstall_Udev_hang,
1057	"\nStarting udev:\n\n\n\n" ),
1058	( True, ktReason_OSInstall_Udev_hang,
1059	"\nStarting udev:\n------" ),
1060	( True, ktReason_Panic_BootManagerC000000F,
1061	"Windows failed to start. A recent hardware or software change might be the" ),
1062	( True, ktReason_BootManager_Image_corrupt,
1063	"BOOTMGR image is corrupt. The system cannot boot." ),
1064	];
1065
1066	## Things we search for in the info.txt file. Require handlers for now.
1067	katInfoTextHandlers = [
1068	# ( Trigger text, handler method )
1069	( "kvm_lock_spinning", investigateInfoKvmLockSpinning ),
1070	( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ),
1071	];
1072
1073	## Mapping screenshot/failure SHA-256 hashes to failure reasons.
1074	katSimpleScreenshotHashReasons = [
1075	# ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output )
1076	( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ),
1077	( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ),
1078	( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ),
1079	( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ),
1080	( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ),
1081	( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ),
1082	( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ),
1083	( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ),
1084	( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ),
1085	( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ),
1086	( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ),
1087	];
1088
1089	def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog):
1090	"""
1091	Investigates a failed VM run.
1092	"""
1093
1094	def investigateLogSet():
1095	"""
1096	Investigates the current set of VM related logs.
1097	"""
1098	self.dprint('investigateLogSet: log lengths: result %u, VM %u, kernel %u, vga text %u, info text %u, hard %u'
1099	% ( len(sResultLog if sResultLog else ''),
1100	len(sVMLog if sVMLog else ''),
1101	len(sKrnlLog if sKrnlLog else ''),
1102	len(sVgaText if sVgaText else ''),
1103	len(sInfoText if sInfoText else ''),
1104	len(sNtHardLog if sNtHardLog else ''), ));
1105
1106	#self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,));
1107	#self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,));
1108	#self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,));
1109	#self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,));
1110	#self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,));
1111	#self.dprint(u'hard.txt<<<\n%s\n<<<\n' % (sNtHardLog,));
1112
1113	# TODO: more
1114
1115	#
1116	# Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this.
1117	#
1118	sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ],
1119	'GIM: HyperV: Guest indicates a fatal condition! P0=');
1120	if sDetails is not None:
1121	# P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 "
1122	sKey = sDetails.split(' ', 1)[0];
1123	try: sKey = '0x%08X' % (int(sKey, 16),);
1124	except: pass;
1125	if sKey in self.asBsodReasons:
1126	tReason = ( self.ksBsodCategory, sKey );
1127	elif sKey.lower() in self.asBsodReasons: # just in case.
1128	tReason = ( self.ksBsodCategory, sKey.lower() );
1129	else:
1130	self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons));
1131	tReason = ( self.ksBsodCategory, self.ksBsodAddNew );
1132	return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip());
1133
1134	#
1135	# Look for linux panic.
1136	#
1137	if sKrnlLog is not None:
1138	for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons:
1139	if sKrnlLog.find(sNeedle) > 0:
1140	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1141	if fStopOnHit:
1142	return True;
1143	fFoundSomething = True;
1144
1145	#
1146	# Loop thru the simple stuff.
1147	#
1148	fFoundSomething = False;
1149	for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons:
1150	if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0):
1151	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1152	if fStopOnHit:
1153	return True;
1154	fFoundSomething = True;
1155
1156	# Continue with vga text.
1157	if sVgaText:
1158	for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons:
1159	if sVgaText.find(sNeedle) > 0:
1160	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1161	if fStopOnHit:
1162	return True;
1163	fFoundSomething = True;
1164	_ = sInfoText;
1165
1166	# Continue with screen hashes.
1167	if sScreenHash is not None:
1168	for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons:
1169	if sScreenHash == sHash:
1170	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1171	if fStopOnHit:
1172	return True;
1173	fFoundSomething = True;
1174
1175	# Check VBoxHardening.log.
1176	if sNtHardLog is not None:
1177	for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons:
1178	if sNtHardLog.find(sNeedle) > 0:
1179	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1180	if fStopOnHit:
1181	return True;
1182	fFoundSomething = True;
1183
1184	#
1185	# Complicated stuff.
1186	#
1187	dLogs = {
1188	'sVMLog': sVMLog,
1189	'sNtHardLog': sNtHardLog,
1190	'sScreenHash': sScreenHash,
1191	'sKrnlLog': sKrnlLog,
1192	'sVgaText': sVgaText,
1193	'sInfoText': sInfoText,
1194	};
1195
1196	# info.txt.
1197	if sInfoText:
1198	for sNeedle, fnHandler in self.katInfoTextHandlers:
1199	if sInfoText.find(sNeedle) > 0:
1200	(fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs);
1201	if tReason is not None:
1202	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1203	if fStop:
1204	return True;
1205	fFoundSomething = True;
1206
1207	#
1208	# Check for repeated reboots...
1209	#
1210	if sVMLog is not None:
1211	cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\'');
1212	if cResets > 10:
1213	return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult,
1214	sComment = 'Counted %s reboots' % (cResets,));
1215
1216	return fFoundSomething;
1217
1218	#
1219	# Check if we got any VM or/and kernel logs. Treat them as sets in
1220	# case we run multiple VMs here (this is of course ASSUMING they
1221	# appear in the order that terminateVmBySession uploads them).
1222	#
1223	cTimes = 0;
1224	sVMLog = None;
1225	sNtHardLog = None;
1226	sScreenHash = None;
1227	sKrnlLog = None;
1228	sVgaText = None;
1229	sInfoText = None;
1230	for oFile in oFailedResult.aoFiles:
1231	if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm:
1232	if 'VBoxHardening.log' not in oFile.sFile:
1233	if sVMLog is not None:
1234	if investigateLogSet() is True:
1235	return True;
1236	cTimes += 1;
1237	sInfoText = None;
1238	sVgaText = None;
1239	sKrnlLog = None;
1240	sScreenHash = None;
1241	sNtHardLog = None;
1242	sVMLog = oCaseFile.getLogFile(oFile);
1243	else:
1244	sNtHardLog = oCaseFile.getLogFile(oFile);
1245	elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel:
1246	sKrnlLog = oCaseFile.getLogFile(oFile);
1247	elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText:
1248	sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]);
1249	elif oFile.sKind == TestResultFileData.ksKind_InfoCollection:
1250	sInfoText = oCaseFile.getLogFile(oFile);
1251	elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure:
1252	sScreenHash = oCaseFile.getScreenshotSha256(oFile);
1253	if sScreenHash is not None:
1254	sScreenHash = sScreenHash.lower();
1255	self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,));
1256
1257	if ( sVMLog is not None \
1258	or sNtHardLog is not None \
1259	or cTimes == 0) \
1260	and investigateLogSet() is True:
1261	return True;
1262
1263	return None;
1264
1265
1266	def isResultFromVMRun(self, oFailedResult, sResultLog):
1267	"""
1268	Checks if this result and corresponding log snippet looks like a VM run.
1269	"""
1270
1271	# Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log.
1272	if sResultLog.find(' startVm') > 0:
1273	return True;
1274
1275	# Any other indicators? No?
1276	_ = oFailedResult;
1277	return False;
1278
1279	def investigateVBoxVMTest(self, oCaseFile, fSingleVM):
1280	"""
1281	Checks out a VBox VM test.
1282
1283	This is generic investigation of a test running one or more VMs, like
1284	for example a smoke test or a guest installation test.
1285
1286	The fSingleVM parameter is a hint, which probably won't come in useful.
1287	"""
1288	_ = fSingleVM;
1289
1290	#
1291	# Get a list of test result failures we should be looking into and the main log.
1292	#
1293	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1294	sMainLog = oCaseFile.getMainLog();
1295
1296	#
1297	# There are a set of errors ending up on the top level result record.
1298	# Should deal with these first.
1299	#
1300	if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree:
1301	# Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always
1302	# be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to
1303	# ignore other failures in the test if we're not a little bit careful here.
1304	if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0:
1305	oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11);
1306	return self.caseClosed(oCaseFile);
1307
1308	# Hang after starting VBoxSVC (e.g. idTestSet=136307258)
1309	if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at',
1310	(' Timeout: ', ' Attempting to abort child...',) ):
1311	if sMainLog.find('* glibc detected * /') > 0:
1312	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption);
1313	else:
1314	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang);
1315	return self.caseClosed(oCaseFile);
1316
1317	# Look for heap corruption without visible hang.
1318	if sMainLog.find('* glibc detected * /') > 0 \
1319	or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374
1320	oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption);
1321	return self.caseClosed(oCaseFile);
1322
1323	# Out of memory w/ timeout.
1324	if sMainLog.find('sErrId=HostMemoryLow') > 0:
1325	oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow);
1326	return self.caseClosed(oCaseFile);
1327
1328	# Stale files like vts_rm.exe (windows).
1329	offEnd = sMainLog.rfind('* The test driver exits successfully. *');
1330	if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0:
1331	oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files);
1332	return self.caseClosed(oCaseFile);
1333
1334	#
1335	# XPCOM screwup
1336	#
1337	if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0:
1338	oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build);
1339	return self.caseClosed(oCaseFile);
1340
1341	#
1342	# Go thru each failed result.
1343	#
1344	for oFailedResult in aoFailedResults:
1345	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1346	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1347	if oFailedResult.sName == 'Installing VirtualBox':
1348	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1349
1350	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1351	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1352
1353	elif self.isResultFromVMRun(oFailedResult, sResultLog):
1354	self.investigateVMResult(oCaseFile, oFailedResult, sResultLog);
1355
1356	elif sResultLog.find('most likely not unique') > 0:
1357	oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult)
1358	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1359	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1360
1361	elif sResultLog.find('The machine is not mutable (state is ') > 0:
1362	self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem');
1363	oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult);
1364
1365	elif sResultLog.find('** error: no action was specified') > 0 \
1366	or sResultLog.find('(len(self._asXml, asText))') > 0:
1367	oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult);
1368
1369	else:
1370	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1371	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1372
1373	#
1374	# Report home and close the case if we got them all, otherwise log it.
1375	#
1376	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1377	return self.caseClosed(oCaseFile);
1378
1379	if oCaseFile.dReasonForResultId:
1380	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1381	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1382	else:
1383	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1384	return False;
1385
1386
1387	## Things we search a main log for to figure out why something in the API test went bust.
1388	katSimpleApiMainLogReasons = [
1389	# ( Whether to stop on hit, reason tuple, needle text. )
1390	( True, ktReason_Networking_Nonexistent_host_nic,
1391	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
1392	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
1393	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
1394	( True, ktReason_API_std_bad_alloc, 'Unexpected exception: std::bad_alloc' ),
1395	( True, ktReason_API_Digest_Mismatch, 'Digest mismatch (VERR_NOT_EQUAL)' ),
1396	( True, ktReason_API_MoveVM_SharingViolation, 'rc=VBOX_E_IPRT_ERROR text="Could not copy the log file ' ),
1397	( True, ktReason_API_MoveVM_InvalidParameter,
1398	'rc=VBOX_E_IPRT_ERROR text="Could not copy the setting file ' ),
1399	( True, ktReason_API_Open_Session_Failed, 'error: failed to open session for' ),
1400	];
1401
1402	def investigateVBoxApiTest(self, oCaseFile):
1403	"""
1404	Checks out a VBox API test.
1405	"""
1406
1407	#
1408	# Get a list of test result failures we should be looking into and the main log.
1409	#
1410	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1411	sMainLog = oCaseFile.getMainLog();
1412
1413	#
1414	# Go thru each failed result.
1415	#
1416	for oFailedResult in aoFailedResults:
1417	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1418	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1419	if oFailedResult.sName == 'Installing VirtualBox':
1420	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1421
1422	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1423	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1424
1425	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1426	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1427
1428	else:
1429	fFoundSomething = False;
1430	for fStopOnHit, tReason, sNeedle in self.katSimpleApiMainLogReasons:
1431	if sResultLog.find(sNeedle) > 0:
1432	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1433	fFoundSomething = True;
1434	if fStopOnHit:
1435	break;
1436	if fFoundSomething:
1437	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1438	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1439
1440	#
1441	# Report home and close the case if we got them all, otherwise log it.
1442	#
1443	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1444	return self.caseClosed(oCaseFile);
1445
1446	if oCaseFile.dReasonForResultId:
1447	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1448	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1449	else:
1450	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1451	return False;
1452
1453
1454	def reasoningFailures(self):
1455	"""
1456	Guess the reason for failures.
1457	"""
1458	#
1459	# Get a list of failed test sets without any assigned failure reason.
1460	#
1461	cGot = 0;
1462	if not self.oConfig.aidTestSets:
1463	aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack,
1464	tsNow = self.tsNow);
1465	else:
1466	aoTestSets = [self.oTestSetLogic.getById(idTestSet) for idTestSet in self.oConfig.aidTestSets];
1467	for oTestSet in aoTestSets:
1468	self.dprint(u'----------------------------------- #%u, status %s -----------------------------------'
1469	% ( oTestSet.idTestSet, oTestSet.enmStatus,));
1470
1471	#
1472	# Open a case file and assign it to the right investigator.
1473	#
1474	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet);
1475	oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated);
1476	oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox);
1477	oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated);
1478	oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig);
1479
1480	oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase);
1481
1482	if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
1483	self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,));
1484	fRc = self.investigateBadTestBox(oCaseFile);
1485
1486	elif oCaseFile.isVBoxUnitTest():
1487	self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,));
1488	fRc = self.investigateVBoxUnitTest(oCaseFile);
1489
1490	elif oCaseFile.isVBoxInstallTest():
1491	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1492	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1493
1494	elif oCaseFile.isVBoxUSBTest():
1495	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1496	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1497
1498	elif oCaseFile.isVBoxStorageTest():
1499	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1500	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1501
1502	elif oCaseFile.isVBoxGAsTest():
1503	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1504	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1505
1506	elif oCaseFile.isVBoxAPITest():
1507	self.dprint(u'investigateVBoxApiTest is taking over %s.' % (oCaseFile.sLongName,));
1508	fRc = self.investigateVBoxApiTest(oCaseFile);
1509
1510	elif oCaseFile.isVBoxBenchmarkTest():
1511	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1512	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1513
1514	elif oCaseFile.isVBoxSmokeTest():
1515	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1516	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1517
1518	elif oCaseFile.isVBoxSerialTest():
1519	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1520	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1521
1522	else:
1523	self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,));
1524	fRc = False;
1525	cGot += fRc is True;
1526
1527	self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), ));
1528	return 0;
1529
1530
1531	def main(self):
1532	"""
1533	The 'main' function.
1534	Return exit code (0, 1, etc).
1535	"""
1536	# Database stuff.
1537	self.oDb = TMDatabaseConnection()
1538	self.oTestResultLogic = TestResultLogic(self.oDb);
1539	self.oTestSetLogic = TestSetLogic(self.oDb);
1540	self.oFailureReasonLogic = FailureReasonLogic(self.oDb);
1541	self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb);
1542	self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory);
1543	self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory);
1544
1545	# Get a fix on our 'now' before we do anything..
1546	self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
1547	self.tsNow = self.oDb.fetchOne();
1548
1549	# If we're suppost to commit anything we need to get our user ID.
1550	rcExit = 0;
1551	if self.oConfig.fRealRun:
1552	self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
1553	if self.oLogin is None:
1554	rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
1555	else:
1556	self.uidSelf = self.oLogin.uid;
1557
1558	#
1559	# Do the stuff.
1560	#
1561	if rcExit == 0:
1562	rcExit = self.selfCheck();
1563	if rcExit == 0:
1564	rcExit = self.badTestBoxManagement();
1565	rcExit2 = self.reasoningFailures();
1566	if rcExit == 0:
1567	rcExit = rcExit2;
1568	# Redo the bad testbox management after failure reasons have been assigned (got timing issues).
1569	if rcExit == 0:
1570	rcExit = self.badTestBoxManagement();
1571
1572	# Cleanup.
1573	self.oFailureReasonLogic = None;
1574	self.oTestResultFailureLogic = None;
1575	self.oTestSetLogic = None;
1576	self.oTestResultLogic = None;
1577	self.oDb.close();
1578	self.oDb = None;
1579	if self.oLogFile is not None:
1580	self.oLogFile.close();
1581	self.oLogFile = None;
1582	return rcExit;
1583
1584	if __name__ == '__main__':
1585	sys.exit(VirtualTestSheriff().main());
1586

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py@ 79092

Download in other formats: