virtual_test_sheriff.py@ 79918

Last change on this file since 79918 was 79918, checked in by vboxsync, 5 years ago
vsheriff: Categorize unattended install tests with the old install tests. bugref:9151
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 77.0 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: virtual_test_sheriff.py 79918 2019-07-22 10:11:05Z vboxsync $
4	# pylint: disable=line-too-long
5
6	"""
7	Virtual Test Sheriff.
8
9	Duties:
10	- Try to a assign failure reasons to recently failed tests.
11	- Reboot or disable bad test boxes.
12
13	"""
14
15	from __future__ import print_function;
16
17	__copyright__ = \
18	"""
19	Copyright (C) 2012-2019 Oracle Corporation
20
21	This file is part of VirtualBox Open Source Edition (OSE), as
22	available from http://www.virtualbox.org. This file is free software;
23	you can redistribute it and/or modify it under the terms of the GNU
24	General Public License (GPL) as published by the Free Software
25	Foundation, in version 2 as it comes in the "COPYING" file of the
26	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
27	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
32	VirtualBox OSE distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37	"""
38	__version__ = "$Revision: 79918 $"
39
40
41	# Standard python imports
42	import sys;
43	import os;
44	import hashlib;
45	import subprocess;
46	import smtplib
47	from email.mime.multipart import MIMEMultipart
48	from email.mime.text import MIMEText
49	from email.utils import COMMASPACE
50
51	if sys.version_info[0] >= 3:
52	from io import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias
53	else:
54	from StringIO import StringIO as StringIO; # pylint: disable=import-error,no-name-in-module,useless-import-alias
55	from optparse import OptionParser; # pylint: disable=deprecated-module
56	from PIL import Image; # pylint: disable=import-error
57
58	# Add Test Manager's modules path
59	g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
60	sys.path.append(g_ksTestManagerDir);
61
62	# Test Manager imports
63	from testmanager.core.db import TMDatabaseConnection;
64	from testmanager.core.build import BuildDataEx;
65	from testmanager.core.failurereason import FailureReasonLogic;
66	from testmanager.core.testbox import TestBoxLogic, TestBoxData;
67	from testmanager.core.testcase import TestCaseDataEx;
68	from testmanager.core.testgroup import TestGroupData;
69	from testmanager.core.testset import TestSetLogic, TestSetData;
70	from testmanager.core.testresults import TestResultLogic, TestResultFileData;
71	from testmanager.core.testresultfailures import TestResultFailureLogic, TestResultFailureData;
72	from testmanager.core.useraccount import UserAccountLogic;
73	from testmanager.config import g_ksSmtpHost, g_kcSmtpPort, g_ksAlertFrom, \
74	g_ksAlertSubject, g_asAlertList, g_ksLomPassword;
75
76	# Python 3 hacks:
77	if sys.version_info[0] >= 3:
78	xrange = range; # pylint: disable=redefined-builtin,invalid-name
79
80
81	class VirtualTestSheriffCaseFile(object):
82	"""
83	A failure investigation case file.
84
85	"""
86
87
88	## Max log file we'll read into memory. (256 MB)
89	kcbMaxLogRead = 0x10000000;
90
91	def __init__(self, oSheriff, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase):
92	self.oSheriff = oSheriff;
93	self.oTestSet = oTestSet; # TestSetData
94	self.oTree = oTree; # TestResultDataEx
95	self.oBuild = oBuild; # BuildDataEx
96	self.oTestBox = oTestBox; # TestBoxData
97	self.oTestGroup = oTestGroup; # TestGroupData
98	self.oTestCase = oTestCase; # TestCaseDataEx
99	self.sMainLog = ''; # The main log file. Empty string if not accessible.
100
101	# Generate a case file name.
102	self.sName = '#%u: %s' % (self.oTestSet.idTestSet, self.oTestCase.sName,)
103	self.sLongName = '#%u: "%s" on "%s" running %s %s (%s), "%s" by %s, using %s %s %s r%u' \
104	% ( self.oTestSet.idTestSet,
105	self.oTestCase.sName,
106	self.oTestBox.sName,
107	self.oTestBox.sOs,
108	self.oTestBox.sOsVersion,
109	self.oTestBox.sCpuArch,
110	self.oTestBox.sCpuName,
111	self.oTestBox.sCpuVendor,
112	self.oBuild.oCat.sProduct,
113	self.oBuild.oCat.sBranch,
114	self.oBuild.oCat.sType,
115	self.oBuild.iRevision, );
116
117	# Investigation notes.
118	self.tReason = None; # None or one of the ktReason_XXX constants.
119	self.dReasonForResultId = {}; # Reason assignments indexed by idTestResult.
120	self.dCommentForResultId = {}; # Comment assignments indexed by idTestResult.
121
122	#
123	# Reason.
124	#
125
126	def noteReason(self, tReason):
127	""" Notes down a possible reason. """
128	self.oSheriff.dprint(u'noteReason: %s -> %s' % (self.tReason, tReason,));
129	self.tReason = tReason;
130	return True;
131
132	def noteReasonForId(self, tReason, idTestResult, sComment = None):
133	""" Notes down a possible reason for a specific test result. """
134	self.oSheriff.dprint(u'noteReasonForId: %u: %s -> %s%s'
135	% (idTestResult, self.dReasonForResultId.get(idTestResult, None), tReason,
136	(u' (%s)' % (sComment,)) if sComment is not None else ''));
137	self.dReasonForResultId[idTestResult] = tReason;
138	if sComment is not None:
139	self.dCommentForResultId[idTestResult] = sComment;
140	return True;
141
142
143	#
144	# Test classification.
145	#
146
147	def isVBoxTest(self):
148	""" Test classification: VirtualBox (using the build) """
149	return self.oBuild.oCat.sProduct.lower() in [ 'virtualbox', 'vbox' ];
150
151	def isVBoxUnitTest(self):
152	""" Test case classification: The unit test doing all our testcase/*.cpp stuff. """
153	return self.isVBoxTest() \
154	and (self.oTestCase.sName.lower() == 'unit tests' or self.oTestCase.sName.lower() == 'misc: unit tests');
155
156	def isVBoxInstallTest(self):
157	""" Test case classification: VirtualBox Guest installation test. """
158	return self.isVBoxTest() \
159	and self.oTestCase.sName.lower().startswith('install:');
160
161	def isVBoxUnattendedInstallTest(self):
162	""" Test case classification: VirtualBox Guest installation test. """
163	return self.isVBoxTest() \
164	and self.oTestCase.sName.lower().startswith('uinstall:');
165
166	def isVBoxUSBTest(self):
167	""" Test case classification: VirtualBox USB test. """
168	return self.isVBoxTest() \
169	and self.oTestCase.sName.lower().startswith('usb:');
170
171	def isVBoxStorageTest(self):
172	""" Test case classification: VirtualBox Storage test. """
173	return self.isVBoxTest() \
174	and self.oTestCase.sName.lower().startswith('storage:');
175
176	def isVBoxGAsTest(self):
177	""" Test case classification: VirtualBox Guest Additions test. """
178	return self.isVBoxTest() \
179	and ( self.oTestCase.sName.lower().startswith('guest additions')
180	or self.oTestCase.sName.lower().startswith('ga\'s tests'));
181
182	def isVBoxAPITest(self):
183	""" Test case classification: VirtualBox API test. """
184	return self.isVBoxTest() \
185	and self.oTestCase.sName.lower().startswith('api:');
186
187	def isVBoxBenchmarkTest(self):
188	""" Test case classification: VirtualBox Benchmark test. """
189	return self.isVBoxTest() \
190	and self.oTestCase.sName.lower().startswith('benchmark:');
191
192	def isVBoxSmokeTest(self):
193	""" Test case classification: Smoke test. """
194	return self.isVBoxTest() \
195	and self.oTestCase.sName.lower().startswith('smoketest');
196
197	def isVBoxSerialTest(self):
198	""" Test case classification: Smoke test. """
199	return self.isVBoxTest() \
200	and self.oTestCase.sName.lower().startswith('serial:');
201
202
203	#
204	# Utility methods.
205	#
206
207	def getMainLog(self):
208	"""
209	Tries to read the main log file since this will be the first source of information.
210	"""
211	if self.sMainLog:
212	return self.sMainLog;
213	(oFile, oSizeOrError, _) = self.oTestSet.openFile('main.log', 'rb');
214	if oFile is not None:
215	try:
216	self.sMainLog = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
217	except Exception as oXcpt:
218	self.oSheriff.vprint(u'Error reading main log file: %s' % (oXcpt,))
219	self.sMainLog = '';
220	else:
221	self.oSheriff.vprint(u'Error opening main log file: %s' % (oSizeOrError,));
222	return self.sMainLog;
223
224	def getLogFile(self, oFile):
225	"""
226	Tries to read the given file as a utf-8 log file.
227	oFile is a TestFileDataEx instance.
228	Returns empty string if problems opening or reading the file.
229	"""
230	sContent = '';
231	(oFile, oSizeOrError, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
232	if oFile is not None:
233	try:
234	sContent = oFile.read(min(self.kcbMaxLogRead, oSizeOrError)).decode('utf-8', 'replace');
235	except Exception as oXcpt:
236	self.oSheriff.vprint(u'Error reading the "%s" log file: %s' % (oFile.sFile, oXcpt,))
237	else:
238	self.oSheriff.vprint(u'Error opening the "%s" log file: %s' % (oFile.sFile, oSizeOrError,));
239	return sContent;
240
241	def getScreenshotSha256(self, oFile):
242	"""
243	Tries to read the given screenshot file, uncompress it, and do SHA-2
244	on the raw pixels.
245	Returns SHA-2 digest string on success, None on failure.
246	"""
247	(oImgFile, _, _) = self.oTestSet.openFile(oFile.sFile, 'rb');
248	try:
249	abImageFile = oImgFile.read();
250	except Exception as oXcpt:
251	self.oSheriff.vprint(u'Error reading the "%s" image file: %s' % (oFile.sFile, oXcpt,))
252	else:
253	try:
254	oImage = Image.open(StringIO(abImageFile));
255	except Exception as oXcpt:
256	self.oSheriff.vprint(u'Error opening the "%s" image bytes using PIL.Image.open: %s' % (oFile.sFile, oXcpt,))
257	else:
258	try:
259	oHash = hashlib.sha256();
260	oHash.update(oImage.tostring());
261	except Exception as oXcpt:
262	self.oSheriff.vprint(u'Error hashing the uncompressed image bytes for "%s": %s' % (oFile.sFile, oXcpt,))
263	else:
264	return oHash.hexdigest();
265	return None;
266
267
268
269	def isSingleTestFailure(self):
270	"""
271	Figure out if this is a single test failing or if it's one of the
272	more complicated ones.
273	"""
274	if self.oTree.cErrors == 1:
275	return True;
276	if self.oTree.deepCountErrorContributers() <= 1:
277	return True;
278	return False;
279
280
281
282	class VirtualTestSheriff(object): # pylint: disable=too-few-public-methods
283	"""
284	Add build info into Test Manager database.
285	"""
286
287	## The user account for the virtual sheriff.
288	ksLoginName = 'vsheriff';
289
290	def __init__(self):
291	"""
292	Parse command line.
293	"""
294	self.oDb = None;
295	self.tsNow = None;
296	self.oTestResultLogic = None;
297	self.oTestSetLogic = None;
298	self.oFailureReasonLogic = None; # FailureReasonLogic;
299	self.oTestResultFailureLogic = None; # TestResultFailureLogic
300	self.oLogin = None;
301	self.uidSelf = -1;
302	self.oLogFile = None;
303	self.asBsodReasons = [];
304	self.asUnitTestReasons = [];
305
306	oParser = OptionParser();
307	oParser.add_option('--start-hours-ago', dest = 'cStartHoursAgo', metavar = '<hours>', default = 0, type = 'int',
308	help = 'When to start specified as hours relative to current time. Defauls is right now.', );
309	oParser.add_option('--hours-period', dest = 'cHoursBack', metavar = '<period-in-hours>', default = 2, type = 'int',
310	help = 'Work period specified in hours. Defauls is 2 hours.');
311	oParser.add_option('--real-run-back', dest = 'fRealRun', action = 'store_true', default = False,
312	help = 'Whether to commit the findings to the database. Default is a dry run.');
313	oParser.add_option('--testset', dest = 'aidTestSets', metavar = '<id>', default = [], type = 'int', action = 'append',
314	help = 'Only investigate this one. Accumulates IDs when repeated.');
315	oParser.add_option('-q', '--quiet', dest = 'fQuiet', action = 'store_true', default = False,
316	help = 'Quiet execution');
317	oParser.add_option('-l', '--log', dest = 'sLogFile', metavar = '<logfile>', default = None,
318	help = 'Where to log messages.');
319	oParser.add_option('--debug', dest = 'fDebug', action = 'store_true', default = False,
320	help = 'Enables debug mode.');
321
322	(self.oConfig, _) = oParser.parse_args();
323
324	if self.oConfig.sLogFile:
325	self.oLogFile = open(self.oConfig.sLogFile, "a");
326	self.oLogFile.write('VirtualTestSheriff: $Revision: 79918 $ \n');
327
328
329	def eprint(self, sText):
330	"""
331	Prints error messages.
332	Returns 1 (for exit code usage.)
333	"""
334	print('error: %s' % (sText,));
335	if self.oLogFile is not None:
336	self.oLogFile.write((u'error: %s\n' % (sText,)).encode('utf-8'));
337	return 1;
338
339	def dprint(self, sText):
340	"""
341	Prints debug info.
342	"""
343	if self.oConfig.fDebug:
344	if not self.oConfig.fQuiet:
345	print('debug: %s' % (sText, ));
346	if self.oLogFile is not None:
347	self.oLogFile.write((u'debug: %s\n' % (sText,)).encode('utf-8'));
348	return 0;
349
350	def vprint(self, sText):
351	"""
352	Prints verbose info.
353	"""
354	if not self.oConfig.fQuiet:
355	print('info: %s' % (sText,));
356	if self.oLogFile is not None:
357	self.oLogFile.write((u'info: %s\n' % (sText,)).encode('utf-8'));
358	return 0;
359
360	def getFailureReason(self, tReason):
361	""" Gets the failure reason object for tReason. """
362	return self.oFailureReasonLogic.cachedLookupByNameAndCategory(tReason[1], tReason[0]);
363
364	def selfCheck(self):
365	""" Does some self checks, looking up things we expect to be in the database and such. """
366	rcExit = 0;
367	for sAttr in dir(self.__class__):
368	if sAttr.startswith('ktReason_'):
369	tReason = getattr(self.__class__, sAttr);
370	oFailureReason = self.getFailureReason(tReason);
371	if oFailureReason is None:
372	rcExit = self.eprint(u'Failed to find failure reason "%s" in category "%s" in the database!'
373	% (tReason[1], tReason[0],));
374
375	# Check the user account as well.
376	if self.oLogin is None:
377	oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
378	if oLogin is None:
379	rcExit = self.eprint(u'Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
380	return rcExit;
381
382	def sendEmailAlert(self, uidAuthor, sBodyText):
383	"""
384	Sends email alert.
385	"""
386
387	# Get author email
388	self.oDb.execute('SELECT sEmail FROM Users WHERE uid=%s', (uidAuthor,));
389	sFrom = self.oDb.fetchOne();
390	if sFrom is not None:
391	sFrom = sFrom[0];
392	else:
393	sFrom = g_ksAlertFrom;
394
395	# Gather recipient list.
396	asEmailList = [];
397	for sUser in g_asAlertList:
398	self.oDb.execute('SELECT sEmail FROM Users WHERE sUsername=%s', (sUser,));
399	sEmail = self.oDb.fetchOne();
400	if sEmail:
401	asEmailList.append(sEmail[0]);
402	if not asEmailList:
403	return self.eprint('No email addresses to send alter to!');
404
405	# Compose the message.
406	oMsg = MIMEMultipart();
407	oMsg['From'] = sFrom;
408	oMsg['To'] = COMMASPACE.join(asEmailList);
409	oMsg['Subject'] = g_ksAlertSubject;
410	oMsg.attach(MIMEText(sBodyText, 'plain'))
411
412	# Try send it.
413	try:
414	oSMTP = smtplib.SMTP(g_ksSmtpHost, g_kcSmtpPort);
415	oSMTP.sendmail(sFrom, asEmailList, oMsg.as_string())
416	oSMTP.quit()
417	except smtplib.SMTPException as oXcpt:
418	return self.eprint('Failed to send mail: %s' % (oXcpt,));
419
420	return 0;
421
422	def badTestBoxManagement(self):
423	"""
424	Looks for bad test boxes and first tries once to reboot them then disables them.
425	"""
426	rcExit = 0;
427
428	#
429	# We skip this entirely if we're running in the past and not in harmless debug mode.
430	#
431	if self.oConfig.cStartHoursAgo != 0 \
432	and (not self.oConfig.fDebug or self.oConfig.fRealRun):
433	return rcExit;
434	tsNow = self.tsNow if self.oConfig.fDebug else None;
435	cHoursBack = self.oConfig.cHoursBack if self.oConfig.fDebug else 2;
436	oTestBoxLogic = TestBoxLogic(self.oDb);
437
438	#
439	# Generate a list of failures reasons we consider bad-testbox behavior.
440	#
441	aidFailureReasons = [
442	self.getFailureReason(self.ktReason_Host_DriverNotLoaded).idFailureReason,
443	self.getFailureReason(self.ktReason_Host_DriverNotUnloading).idFailureReason,
444	self.getFailureReason(self.ktReason_Host_DriverNotCompilable).idFailureReason,
445	self.getFailureReason(self.ktReason_Host_InstallationFailed).idFailureReason,
446	];
447
448	#
449	# Get list of bad test boxes for given period and check them out individually.
450	#
451	aidBadTestBoxes = self.oTestSetLogic.fetchBadTestBoxIds(cHoursBack = cHoursBack, tsNow = tsNow,
452	aidFailureReasons = aidFailureReasons);
453	for idTestBox in aidBadTestBoxes:
454	# Skip if the testbox is already disabled or has a pending reboot command.
455	try:
456	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
457	except Exception as oXcpt:
458	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
459	continue;
460	if not oTestBox.fEnabled:
461	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
462	% ( idTestBox, oTestBox.sName, ));
463	continue;
464	if oTestBox.enmPendingCmd != TestBoxData.ksTestBoxCmd_None:
465	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has a command pending: %s'
466	% ( idTestBox, oTestBox.sName, oTestBox.enmPendingCmd));
467	continue;
468
469	# Get the most recent testsets for this box (descending on tsDone) and see how bad it is.
470	aoSets = self.oTestSetLogic.fetchSetsForTestBox(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow);
471	cOkay = 0;
472	cBad = 0;
473	iFirstOkay = len(aoSets);
474	for iSet, oSet in enumerate(aoSets):
475	if oSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
476	cBad += 1;
477	else:
478	# Check for bad failure reasons.
479	oFailure = None;
480	if oSet.enmStatus in TestSetData.kasBadTestStatuses:
481	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oSet.idTestSet)
482	aoFailedResults = oTree.getListOfFailures();
483	for oFailedResult in aoFailedResults:
484	oFailure = self.oTestResultFailureLogic.getById(oFailedResult.idTestResult);
485	if oFailure is not None and oFailure.idFailureReason in aidFailureReasons:
486	break;
487	else:
488	oFailure = None;
489	if oFailure is not None:
490	cBad += 1;
491	else:
492	# This is an okay test result then.
493	## @todo maybe check the elapsed time here, it could still be a bad run?
494	cOkay += 1;
495	if iFirstOkay > iSet:
496	iFirstOkay = iSet;
497	if iSet > 10:
498	break;
499
500	# We react if there are two or more bad-testbox statuses at the head of the
501	# history and at least three in the last 10 results.
502	if iFirstOkay >= 2 and cBad > 2:
503	if oTestBoxLogic.hasTestBoxRecentlyBeenRebooted(idTestBox, cHoursBack = cHoursBack, tsNow = tsNow):
504	sComment = u'Disabling testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
505	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
506	self.vprint(sComment);
507	self.sendEmailAlert(self.uidSelf, sComment);
508	if self.oConfig.fRealRun is True:
509	try:
510	oTestBoxLogic.disableTestBox(idTestBox, self.uidSelf, fCommit = True,
511	sComment = 'Automatically disabled (iFirstOkay=%u cBad=%u cOkay=%u)'
512	% (iFirstOkay, cBad, cOkay),);
513	except Exception as oXcpt:
514	rcExit = self.eprint(u'Error disabling testbox #%u (%u): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
515	else:
516	sComment = u'Rebooting testbox #%u (%s) - iFirstOkay=%u cBad=%u cOkay=%u' \
517	% (idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay);
518	self.vprint(sComment);
519	self.sendEmailAlert(self.uidSelf, sComment);
520	if self.oConfig.fRealRun is True:
521	try:
522	oTestBoxLogic.rebootTestBox(idTestBox, self.uidSelf, fCommit = True,
523	sComment = 'Automatically rebooted (iFirstOkay=%u cBad=%u cOkay=%u)'
524	% (iFirstOkay, cBad, cOkay),);
525	except Exception as oXcpt:
526	rcExit = self.eprint(u'Error rebooting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
527	else:
528	self.dprint(u'badTestBoxManagement: #%u (%s) looks ok: iFirstOkay=%u cBad=%u cOkay=%u'
529	% ( idTestBox, oTestBox.sName, iFirstOkay, cBad, cOkay));
530
531	#
532	# Reset hanged testboxes
533	#
534	cStatusTimeoutMins = 10;
535
536	self.oDb.execute('SELECT TestBoxStatuses.idTestBox\n'
537	' FROM TestBoxStatuses, TestBoxes\n'
538	' WHERE TestBoxStatuses.tsUpdated >= (CURRENT_TIMESTAMP - interval \'%s hours\')\n'
539	' AND TestBoxStatuses.tsUpdated < (CURRENT_TIMESTAMP - interval \'%s minutes\')\n'
540	' AND TestBoxStatuses.idTestBox = TestBoxes.idTestBox\n'
541	' AND Testboxes.tsExpire = \'infinity\'::timestamp', (cHoursBack,cStatusTimeoutMins));
542	for idTestBox in self.oDb.fetchAll():
543	idTestBox = idTestBox[0];
544	try:
545	oTestBox = TestBoxData().initFromDbWithId(self.oDb, idTestBox);
546	except Exception as oXcpt:
547	rcExit = self.eprint('Failed to get data for test box #%u in badTestBoxManagement: %s' % (idTestBox, oXcpt,));
548	continue;
549	# Skip if the testbox is already disabled, already reset or there's no iLOM
550	if not oTestBox.fEnabled or oTestBox.ipLom is None or oTestBox.sComment is not None and oTestBox.sComment.find('Automatically reset') >= 0:
551	self.dprint(u'badTestBoxManagement: Skipping test box #%u (%s) as it has been disabled already.'
552	% ( idTestBox, oTestBox.sName, ));
553	continue;
554	## @todo get iLOM credentials from a table?
555	sCmd = 'sshpass -p%s ssh -oStrictHostKeyChecking=no root@%s show /SP && reset /SYS' % (g_ksLomPassword, oTestBox.ipLom,);
556	try:
557	oPs = subprocess.Popen(sCmd, stdout=subprocess.PIPE, shell=True);
558	sStdout = oPs.communicate()[0];
559	iRC = oPs.wait();
560
561	oTestBox.sComment = 'Automatically reset (iRC=%u sStdout=%s)' % (iRC, sStdout,);
562	oTestBoxLogic.editEntry(oTestBox, self.uidSelf, fCommit = True);
563
564	sComment = u'Reset testbox #%u (%s) - iRC=%u sStduot=%s' % ( idTestBox, oTestBox.sName, iRC, sStdout);
565	self.vprint(sComment);
566	self.sendEmailAlert(self.uidSelf, sComment);
567
568	except Exception as oXcpt:
569	rcExit = self.eprint(u'Error resetting testbox #%u (%s): %s\n' % (idTestBox, oTestBox.sName, oXcpt,));
570
571	return rcExit;
572
573
574	## @name Failure reasons we know.
575	## @{
576	ktReason_BSOD_Recovery = ( 'BSOD', 'Recovery' );
577	ktReason_BSOD_Automatic_Repair = ( 'BSOD', 'Automatic Repair' );
578	ktReason_BSOD_0000007F = ( 'BSOD', '0x0000007F' );
579	ktReason_BSOD_000000D1 = ( 'BSOD', '0x000000D1' );
580	ktReason_BSOD_C0000225 = ( 'BSOD', '0xC0000225 (boot)' );
581	ktReason_Guru_Generic = ( 'Guru Meditations', 'Generic Guru Meditation' );
582	ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_INSTR_NOT_IMPLEMENTED' );
583	ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED = ( 'Guru Meditations', 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' );
584	ktReason_Guru_VERR_TRPM_DONT_PANIC = ( 'Guru Meditations', 'VERR_TRPM_DONT_PANIC' );
585	ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED = ( 'Guru Meditations', 'VERR_PGM_PHYS_PAGE_RESERVED' );
586	ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE = ( 'Guru Meditations', 'VERR_VMX_INVALID_GUEST_STATE' );
587	ktReason_Guru_VINF_EM_TRIPLE_FAULT = ( 'Guru Meditations', 'VINF_EM_TRIPLE_FAULT' );
588	ktReason_Host_HostMemoryLow = ( 'Host', 'HostMemoryLow' );
589	ktReason_Host_DriverNotLoaded = ( 'Host', 'Driver not loaded' );
590	ktReason_Host_DriverNotUnloading = ( 'Host', 'Driver not unloading' );
591	ktReason_Host_DriverNotCompilable = ( 'Host', 'Driver not compilable' );
592	ktReason_Host_InstallationFailed = ( 'Host', 'Installation failed' );
593	ktReason_Host_NotSignedWithBuildCert = ( 'Host', 'Not signed with build cert' );
594	ktReason_Host_DoubleFreeHeap = ( 'Host', 'Double free or corruption' );
595	ktReason_Host_LeftoverService = ( 'Host', 'Leftover service' );
596	ktReason_Host_Reboot_OSX_Watchdog_Timeout = ( 'Host Reboot', 'OSX Watchdog Timeout' );
597	ktReason_Host_Modprobe_Failed = ( 'Host', 'Modprobe failed' );
598	ktReason_Host_Install_Hang = ( 'Host', 'Install hang' );
599	ktReason_Host_NetworkMisconfiguration = ( 'Host', 'Network misconfiguration' );
600	ktReason_Host_TSTInfo_Accuracy_OOR = ( 'Host', 'TSTInfo accuracy out of range' );
601	ktReason_Networking_Nonexistent_host_nic = ( 'Networking', 'Nonexistent host networking interface' );
602	ktReason_OSInstall_GRUB_hang = ( 'O/S Install', 'GRUB hang' );
603	ktReason_OSInstall_Udev_hang = ( 'O/S Install', 'udev hang' );
604	ktReason_OSInstall_Sata_no_BM = ( 'O/S Install', 'SATA busmaster bit not set' );
605	ktReason_Panic_BootManagerC000000F = ( 'Panic', 'Hardware Changed' );
606	ktReason_Panic_MP_BIOS_IO_APIC = ( 'Panic', 'MP-BIOS/IO-APIC' );
607	ktReason_Panic_HugeMemory = ( 'Panic', 'Huge memory assertion' );
608	ktReason_Panic_IOAPICDoesntWork = ( 'Panic', 'IO-APIC and timer does not work' );
609	ktReason_Panic_TxUnitHang = ( 'Panic', 'Tx Unit Hang' );
610	ktReason_API_std_bad_alloc = ( 'API / (XP)COM', 'std::bad_alloc' );
611	ktReason_API_Digest_Mismatch = ( 'API / (XP)COM', 'Digest mismatch' );
612	ktReason_API_MoveVM_SharingViolation = ( 'API / (XP)COM', 'MoveVM sharing violation' );
613	ktReason_API_MoveVM_InvalidParameter = ( 'API / (XP)COM', 'MoveVM invalid parameter' );
614	ktReason_API_Open_Session_Failed = ( 'API / (XP)COM', 'Open session failed' );
615	ktReason_XPCOM_Exit_Minus_11 = ( 'API / (XP)COM', 'exit -11' );
616	ktReason_XPCOM_VBoxSVC_Hang = ( 'API / (XP)COM', 'VBoxSVC hang' );
617	ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption = ( 'API / (XP)COM', 'VBoxSVC hang + heap corruption' );
618	ktReason_XPCOM_NS_ERROR_CALL_FAILED = ( 'API / (XP)COM', 'NS_ERROR_CALL_FAILED' );
619	ktReason_BootManager_Image_corrupt = ( 'Unknown', 'BOOTMGR Image corrupt' );
620	ktReason_Unknown_Heap_Corruption = ( 'Unknown', 'Heap corruption' );
621	ktReason_Unknown_Reboot_Loop = ( 'Unknown', 'Reboot loop' );
622	ktReason_Unknown_File_Not_Found = ( 'Unknown', 'File not found' );
623	ktReason_Unknown_HalReturnToFirmware = ( 'Unknown', 'HalReturnToFirmware' );
624	ktReason_Unknown_VM_Crash = ( 'Unknown', 'VM crash' );
625	ktReason_Unknown_VM_Start_Error = ( 'Unknown', 'VM Start Error' );
626	ktReason_Unknown_VM_Runtime_Error = ( 'Unknown', 'VM Runtime Error' );
627	ktReason_VMM_kvm_lock_spinning = ( 'VMM', 'kvm_lock_spinning' );
628	ktReason_Ignore_Buggy_Test_Driver = ( 'Ignore', 'Buggy test driver' );
629	ktReason_Ignore_Stale_Files = ( 'Ignore', 'Stale files' );
630	ktReason_Buggy_Build_Broken_Build = ( 'Broken Build', 'Buggy build' );
631	ktReason_GuestBug_CompizVBoxQt = ( 'Guest Bug', 'Compiz + VirtualBox Qt GUI crash' );
632	## @}
633
634	## BSOD category.
635	ksBsodCategory = 'BSOD';
636	## Special reason indicating that the flesh and blood sheriff has work to do.
637	ksBsodAddNew = 'Add new BSOD';
638
639	## Unit test category.
640	ksUnitTestCategory = 'Unit';
641	## Special reason indicating that the flesh and blood sheriff has work to do.
642	ksUnitTestAddNew = 'Add new';
643
644	## Used for indica that we shouldn't report anything for this test result ID and
645	## consider promoting the previous error to test set level if it's the only one.
646	ktHarmless = ( 'Probably', 'Caused by previous error' );
647
648
649	def caseClosed(self, oCaseFile):
650	"""
651	Reports the findings in the case and closes it.
652	"""
653	#
654	# Log it and create a dReasonForReasultId we can use below.
655	#
656	dCommentForResultId = oCaseFile.dCommentForResultId;
657	if oCaseFile.dReasonForResultId:
658	# Must weed out ktHarmless.
659	dReasonForResultId = {};
660	for idKey, tReason in oCaseFile.dReasonForResultId.items():
661	if tReason is not self.ktHarmless:
662	dReasonForResultId[idKey] = tReason;
663	if not dReasonForResultId:
664	self.vprint(u'TODO: Closing %s without a real reason, only %s.'
665	% (oCaseFile.sName, oCaseFile.dReasonForResultId));
666	return False;
667
668	# Try promote to single reason.
669	atValues = dReasonForResultId.values();
670	fSingleReason = True;
671	if len(dReasonForResultId) == 1 and dReasonForResultId.keys()[0] != oCaseFile.oTestSet.idTestResult:
672	self.dprint(u'Promoting single reason to whole set: %s' % (atValues[0],));
673	elif len(dReasonForResultId) > 1 and len(atValues) == atValues.count(atValues[0]):
674	self.dprint(u'Merged %d reasons to a single one: %s' % (len(atValues), atValues[0]));
675	else:
676	fSingleReason = False;
677	if fSingleReason:
678	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: atValues[0], };
679	if dCommentForResultId:
680	dCommentForResultId = { oCaseFile.oTestSet.idTestResult: dCommentForResultId.values()[0], };
681	elif oCaseFile.tReason is not None:
682	dReasonForResultId = { oCaseFile.oTestSet.idTestResult: oCaseFile.tReason, };
683	else:
684	self.vprint(u'Closing %s without a reason - this should not happen!' % (oCaseFile.sName,));
685	return False;
686
687	self.vprint(u'Closing %s with following reason%s: %s'
688	% ( oCaseFile.sName, 's' if dReasonForResultId > 0 else '', dReasonForResultId, ));
689
690	#
691	# Add the test failure reason record(s).
692	#
693	for idTestResult, tReason in dReasonForResultId.items():
694	oFailureReason = self.getFailureReason(tReason);
695	if oFailureReason is not None:
696	sComment = 'Set by $Revision: 79918 $' # Handy for reverting later.
697	if idTestResult in dCommentForResultId:
698	sComment += ': ' + dCommentForResultId[idTestResult];
699
700	oAdd = TestResultFailureData();
701	oAdd.initFromValues(idTestResult = idTestResult,
702	idFailureReason = oFailureReason.idFailureReason,
703	uidAuthor = self.uidSelf,
704	idTestSet = oCaseFile.oTestSet.idTestSet,
705	sComment = sComment,);
706	if self.oConfig.fRealRun:
707	try:
708	self.oTestResultFailureLogic.addEntry(oAdd, self.uidSelf, fCommit = True);
709	except Exception as oXcpt:
710	self.eprint(u'caseClosed: Exception "%s" while adding reason %s for %s'
711	% (oXcpt, oAdd, oCaseFile.sLongName,));
712	else:
713	self.eprint(u'caseClosed: Cannot locate failure reason: %s / %s' % ( tReason[0], tReason[1],));
714	return True;
715
716	#
717	# Tools for assiting log parsing.
718	#
719
720	@staticmethod
721	def matchFollowedByLines(sStr, off, asFollowingLines):
722	""" Worker for isThisFollowedByTheseLines. """
723
724	# Advance off to the end of the line.
725	off = sStr.find('\n', off);
726	if off < 0:
727	return False;
728	off += 1;
729
730	# Match each string with the subsequent lines.
731	for iLine, sLine in enumerate(asFollowingLines):
732	offEnd = sStr.find('\n', off);
733	if offEnd < 0:
734	return iLine + 1 == len(asFollowingLines) and sStr.find(sLine, off) < 0;
735	if sLine and sStr.find(sLine, off, offEnd) < 0:
736	return False;
737
738	# next line.
739	off = offEnd + 1;
740
741	return True;
742
743	@staticmethod
744	def isThisFollowedByTheseLines(sStr, sFirst, asFollowingLines):
745	"""
746	Looks for a line contining sFirst which is then followed by lines
747	with the strings in asFollowingLines. (No newline chars anywhere!)
748	Returns True / False.
749	"""
750	off = sStr.find(sFirst, 0);
751	while off >= 0:
752	if VirtualTestSheriff.matchFollowedByLines(sStr, off, asFollowingLines):
753	return True;
754	off = sStr.find(sFirst, off + 1);
755	return False;
756
757	@staticmethod
758	def findAndReturnRestOfLine(sHaystack, sNeedle):
759	"""
760	Looks for sNeedle in sHaystack.
761	Returns The text following the needle up to the end of the line.
762	Returns None if not found.
763	"""
764	if sHaystack is None:
765	return None;
766	off = sHaystack.find(sNeedle);
767	if off < 0:
768	return None;
769	off += len(sNeedle)
770	offEol = sHaystack.find('\n', off);
771	if offEol < 0:
772	offEol = len(sHaystack);
773	return sHaystack[off:offEol]
774
775	@staticmethod
776	def findInAnyAndReturnRestOfLine(asHaystacks, sNeedle):
777	"""
778	Looks for sNeedle in zeroe or more haystacks (asHaystack).
779	Returns The text following the first needed found up to the end of the line.
780	Returns None if not found.
781	"""
782	for sHaystack in asHaystacks:
783	sRet = VirtualTestSheriff.findAndReturnRestOfLine(sHaystack, sNeedle);
784	if sRet is not None:
785	return sRet;
786	return None;
787
788
789	#
790	# The investigative units.
791	#
792
793	katSimpleInstallUninstallMainLogReasons = [
794	# ( Whether to stop on hit, reason tuple, needle text. )
795	( False, ktReason_Host_LeftoverService,
796	'SERVICE_NAME: vbox' ),
797	];
798
799	kdatSimpleInstallUninstallMainLogReasonsPerOs = {
800	'darwin': [
801	# ( Whether to stop on hit, reason tuple, needle text. )
802	( True, ktReason_Host_DriverNotUnloading,
803	'Can\'t remove kext org.virtualbox.kext.VBoxDrv; services failed to terminate - 0xe00002c7' ),
804	],
805	'linux': [
806	# ( Whether to stop on hit, reason tuple, needle text. )
807	( True, ktReason_Host_DriverNotCompilable,
808	'This system is not currently set up to build kernel modules' ),
809	( True, ktReason_Host_DriverNotCompilable,
810	'This system is currently not set up to build kernel modules' ),
811	( True, ktReason_Host_InstallationFailed,
812	'vboxdrv.sh: failed: Look at /var/log/vbox-install.log to find out what went wrong.' ),
813	( True, ktReason_Host_DriverNotUnloading,
814	'Cannot unload module vboxdrv'),
815	],
816	'solaris': [
817	# ( Whether to stop on hit, reason tuple, needle text. )
818	( True, ktReason_Host_InstallationFailed,
819	'svcadm: Couldn\'t bind to svc.configd.' ),
820	( True, ktReason_Host_InstallationFailed,
821	'pkgadd: ERROR: postinstall script did not complete successfully' ),
822	( True, ktReason_Host_DriverNotUnloading,
823	'can\'t unload the module: Device busy' ),
824	],
825	};
826
827
828	def investigateInstallUninstallFailure(self, oCaseFile, oFailedResult, sResultLog, fInstall):
829	"""
830	Investigates an install or uninstall failure.
831
832	We lump the two together since the installation typically also performs
833	an uninstall first and will be seeing similar issues to the uninstall.
834	"""
835
836	if fInstall and oFailedResult.enmStatus == TestSetData.ksTestStatus_TimedOut:
837	oCaseFile.noteReasonForId(self.ktReason_Host_Install_Hang, oFailedResult.idTestResult)
838	return True;
839
840	atSimple = self.katSimpleInstallUninstallMainLogReasons;
841	if oCaseFile.oTestBox.sOs in self.kdatSimpleInstallUninstallMainLogReasonsPerOs:
842	atSimple = self.kdatSimpleInstallUninstallMainLogReasonsPerOs[oCaseFile.oTestBox.sOs] + atSimple;
843
844	fFoundSomething = False;
845	for fStopOnHit, tReason, sNeedle in atSimple:
846	if sResultLog.find(sNeedle) > 0:
847	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
848	if fStopOnHit:
849	return True;
850	fFoundSomething = True;
851
852	return fFoundSomething if fFoundSomething else None;
853
854
855	def investigateBadTestBox(self, oCaseFile):
856	"""
857	Checks out bad-testbox statuses.
858	"""
859	_ = oCaseFile;
860	return False;
861
862
863	def investigateVBoxUnitTest(self, oCaseFile):
864	"""
865	Checks out a VBox unittest problem.
866	"""
867
868	#
869	# Process simple test case failures first, using their name as reason.
870	# We do the reason management just like for BSODs.
871	#
872	cRelevantOnes = 0;
873	sMainLog = oCaseFile.getMainLog();
874	aoFailedResults = oCaseFile.oTree.getListOfFailures();
875	for oFailedResult in aoFailedResults:
876	if oFailedResult is oCaseFile.oTree:
877	self.vprint('TODO: toplevel failure');
878	cRelevantOnes += 1
879
880	elif oFailedResult.sName == 'Installing VirtualBox':
881	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
882	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
883	cRelevantOnes += 1
884
885	elif oFailedResult.sName == 'Uninstalling VirtualBox':
886	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
887	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
888	cRelevantOnes += 1
889
890	elif oFailedResult.oParent is not None:
891	# Get the 2nd level node because that's where we'll find the unit test name.
892	while oFailedResult.oParent.oParent is not None:
893	oFailedResult = oFailedResult.oParent;
894
895	# Only report a failure once.
896	if oFailedResult.idTestResult not in oCaseFile.dReasonForResultId:
897	sKey = oFailedResult.sName;
898	if sKey.startswith('testcase/'):
899	sKey = sKey[9:];
900	if sKey in self.asUnitTestReasons:
901	tReason = ( self.ksUnitTestCategory, sKey );
902	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
903	else:
904	self.dprint(u'Unit test failure "%s" not found in %s;' % (sKey, self.asUnitTestReasons));
905	tReason = ( self.ksUnitTestCategory, self.ksUnitTestAddNew );
906	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sKey);
907	cRelevantOnes += 1
908	else:
909	self.vprint(u'Internal error: expected oParent to NOT be None for %s' % (oFailedResult,));
910
911	#
912	# If we've caught all the relevant ones by now, report the result.
913	#
914	if len(oCaseFile.dReasonForResultId) >= cRelevantOnes:
915	return self.caseClosed(oCaseFile);
916	return False;
917
918	def extractGuestCpuStack(self, sInfoText):
919	"""
920	Extracts the guest CPU stacks from the input file.
921
922	Returns a dictionary keyed by the CPU number, value being a list of
923	raw stack lines (no header).
924	Returns empty dictionary if no stacks where found.
925	"""
926	dRet = {};
927	off = 0;
928	while True:
929	# Find the stack.
930	offStart = sInfoText.find('=== start guest stack VCPU ', off);
931	if offStart < 0:
932	break;
933	offEnd = sInfoText.find('=== end guest stack', offStart + 20);
934	if offEnd >= 0:
935	offEnd += 3;
936	else:
937	offEnd = sInfoText.find('=== start guest stack VCPU', offStart + 20);
938	if offEnd < 0:
939	offEnd = len(sInfoText);
940
941	sStack = sInfoText[offStart : offEnd];
942	sStack = sStack.replace('\r',''); # paranoia
943	asLines = sStack.split('\n');
944
945	# Figure the CPU.
946	asWords = asLines[0].split();
947	if len(asWords) < 6 or not asWords[5].isdigit():
948	break;
949	iCpu = int(asWords[5]);
950
951	# Add it and advance.
952	dRet[iCpu] = [sLine.rstrip() for sLine in asLines[2:-1]]
953	off = offEnd;
954	return dRet;
955
956	def investigateInfoKvmLockSpinning(self, oCaseFile, sInfoText, dLogs):
957	""" Investigates kvm_lock_spinning deadlocks """
958	#
959	# Extract the stacks. We need more than one CPU to create a deadlock.
960	#
961	dStacks = self.extractGuestCpuStack(sInfoText);
962	self.dprint('kvm_lock_spinning: found %s stacks' % (len(dStacks),));
963	if len(dStacks) >= 2:
964	#
965	# Examin each of the stacks. Each must have kvm_lock_spinning in
966	# one of the first three entries.
967	#
968	cHits = 0;
969	for iCpu in dStacks:
970	asBacktrace = dStacks[iCpu];
971	for iFrame in xrange(min(3, len(asBacktrace))):
972	if asBacktrace[iFrame].find('kvm_lock_spinning') >= 0:
973	cHits += 1;
974	break;
975	self.dprint('kvm_lock_spinning: %s/%s hits' % (cHits, len(dStacks),));
976	if cHits == len(dStacks):
977	return (True, self.ktReason_VMM_kvm_lock_spinning);
978
979	_ = dLogs; _ = oCaseFile;
980	return (False, None);
981
982	def investigateInfoHalReturnToFirmware(self, oCaseFile, sInfoText, dLogs):
983	""" Investigates HalReturnToFirmware hangs """
984	del oCaseFile
985	del sInfoText
986	del dLogs
987	# hope that's sufficient
988	return (True, self.ktReason_Unknown_HalReturnToFirmware);
989
990	## Things we search a main or VM log for to figure out why something went bust.
991	katSimpleMainAndVmLogReasons = [
992	# ( Whether to stop on hit, reason tuple, needle text. )
993	( False, ktReason_Guru_Generic, 'GuruMeditation' ),
994	( False, ktReason_Guru_Generic, 'Guru Meditation' ),
995	( True, ktReason_Guru_VERR_IEM_INSTR_NOT_IMPLEMENTED, 'VERR_IEM_INSTR_NOT_IMPLEMENTED' ),
996	( True, ktReason_Guru_VERR_IEM_ASPECT_NOT_IMPLEMENTED, 'VERR_IEM_ASPECT_NOT_IMPLEMENTED' ),
997	( True, ktReason_Guru_VERR_TRPM_DONT_PANIC, 'VERR_TRPM_DONT_PANIC' ),
998	( True, ktReason_Guru_VERR_PGM_PHYS_PAGE_RESERVED, 'VERR_PGM_PHYS_PAGE_RESERVED' ),
999	( True, ktReason_Guru_VERR_VMX_INVALID_GUEST_STATE, 'VERR_VMX_INVALID_GUEST_STATE' ),
1000	( True, ktReason_Guru_VINF_EM_TRIPLE_FAULT, 'VINF_EM_TRIPLE_FAULT' ),
1001	( True, ktReason_Networking_Nonexistent_host_nic,
1002	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
1003	( True, ktReason_Host_Reboot_OSX_Watchdog_Timeout, ': "OSX Watchdog Timeout: ' ),
1004	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
1005	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
1006	( True, ktReason_API_std_bad_alloc, 'Unexpected exception: std::bad_alloc' ),
1007	( True, ktReason_Host_HostMemoryLow, 'HostMemoryLow' ),
1008	( True, ktReason_Host_HostMemoryLow, 'Failed to procure handy pages; rc=VERR_NO_MEMORY' ),
1009	( True, ktReason_Unknown_File_Not_Found,
1010	'Error: failed to start machine. Error message: File not found. (VERR_FILE_NOT_FOUND)' ),
1011	( True, ktReason_Unknown_File_Not_Found, # lump it in with file-not-found for now.
1012	'Error: failed to start machine. Error message: Not supported. (VERR_NOT_SUPPORTED)' ),
1013	( False, ktReason_Unknown_VM_Crash, 'txsDoConnectViaTcp: Machine state: Aborted' ),
1014	( True, ktReason_Host_Modprobe_Failed, 'Kernel driver not installed' ),
1015	( True, ktReason_OSInstall_Sata_no_BM, 'PCHS=14128/14134/8224' ),
1016	( True, ktReason_Host_DoubleFreeHeap, 'double free or corruption' ),
1017	( False, ktReason_Unknown_VM_Start_Error, 'VMSetError: ' ),
1018	( False, ktReason_Unknown_VM_Start_Error, 'error: failed to open session for' ),
1019	( False, ktReason_Unknown_VM_Runtime_Error, 'Console: VM runtime error: fatal=true' ),
1020	];
1021
1022	## Things we search a VBoxHardening.log file for to figure out why something went bust.
1023	katSimpleVBoxHardeningLogReasons = [
1024	# ( Whether to stop on hit, reason tuple, needle text. )
1025	( True, ktReason_Host_DriverNotLoaded, 'Error opening VBoxDrvStub: STATUS_OBJECT_NAME_NOT_FOUND' ),
1026	( True, ktReason_Host_NotSignedWithBuildCert, 'Not signed with the build certificate' ),
1027	( True, ktReason_Host_TSTInfo_Accuracy_OOR, 'RTCRTSPTSTINFO::Accuracy::Millis: Out of range' ),
1028	];
1029
1030	## Things we search a kernel.log file for to figure out why something went bust.
1031	katSimpleKernelLogReasons = [
1032	# ( Whether to stop on hit, reason tuple, needle text. )
1033	( True, ktReason_Panic_HugeMemory, 'mm/huge_memory.c:1988' ),
1034	( True, ktReason_Panic_IOAPICDoesntWork, 'IO-APIC + timer doesn\'t work' ),
1035	( True, ktReason_Panic_TxUnitHang, 'Detected Tx Unit Hang' ),
1036	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libQt5CoreVBox' ),
1037	( True, ktReason_GuestBug_CompizVBoxQt, 'error 4 in libgtk-3' ),
1038	];
1039
1040	## Things we search the _RIGHT_ _STRIPPED_ vgatext for.
1041	katSimpleVgaTextReasons = [
1042	# ( Whether to stop on hit, reason tuple, needle text. )
1043	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1044	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n\n" ),
1045	( True, ktReason_Panic_MP_BIOS_IO_APIC,
1046	"..MP-BIOS bug: 8254 timer not connected to IO-APIC\n"
1047	"...trying to set up timer (IRQ0) through the 8259A ... failed.\n"
1048	"...trying to set up timer as Virtual Wire IRQ... failed.\n"
1049	"...trying to set up timer as ExtINT IRQ... failed :(.\n"
1050	"Kernel panic - not syncing: IO-APIC + timer doesn't work! Boot with apic=debug\n"
1051	"and send a report. Then try booting with the 'noapic' option\n"
1052	"\n" ),
1053	( True, ktReason_OSInstall_GRUB_hang,
1054	"-----\nGRUB Loading stage2..\n\n\n\n" ),
1055	( True, ktReason_OSInstall_GRUB_hang,
1056	"-----\nGRUB Loading stage2...\n\n\n\n" ), # the 3 dot hang appears to be less frequent
1057	( True, ktReason_OSInstall_GRUB_hang,
1058	"-----\nGRUB Loading stage2....\n\n\n\n" ), # the 4 dot hang appears to be very infrequent
1059	( True, ktReason_OSInstall_GRUB_hang,
1060	"-----\nGRUB Loading stage2.....\n\n\n\n" ), # the 5 dot hang appears to be more frequent again
1061	( True, ktReason_OSInstall_Udev_hang,
1062	"\nStarting udev:\n\n\n\n" ),
1063	( True, ktReason_OSInstall_Udev_hang,
1064	"\nStarting udev:\n------" ),
1065	( True, ktReason_Panic_BootManagerC000000F,
1066	"Windows failed to start. A recent hardware or software change might be the" ),
1067	( True, ktReason_BootManager_Image_corrupt,
1068	"BOOTMGR image is corrupt. The system cannot boot." ),
1069	];
1070
1071	## Things we search for in the info.txt file. Require handlers for now.
1072	katInfoTextHandlers = [
1073	# ( Trigger text, handler method )
1074	( "kvm_lock_spinning", investigateInfoKvmLockSpinning ),
1075	( "HalReturnToFirmware", investigateInfoHalReturnToFirmware ),
1076	];
1077
1078	## Mapping screenshot/failure SHA-256 hashes to failure reasons.
1079	katSimpleScreenshotHashReasons = [
1080	# ( Whether to stop on hit, reason tuple, lowercased sha-256 of PIL.Image.tostring output )
1081	( True, ktReason_BSOD_Recovery, '576f8e38d62b311cac7e3dc3436a0d0b9bd8cfd7fa9c43aafa95631520a45eac' ),
1082	( True, ktReason_BSOD_Automatic_Repair, 'c6a72076cc619937a7a39cfe9915b36d94cee0d4e3ce5ce061485792dcee2749' ),
1083	( True, ktReason_BSOD_Automatic_Repair, '26c4d8a724ff2c5e1051f3d5b650dbda7b5fdee0aa3e3c6059797f7484a515df' ),
1084	( True, ktReason_BSOD_0000007F, '57e1880619e13042a87100e7a38c8974b85ce3866501be621bea0cc696bb2c63' ),
1085	( True, ktReason_BSOD_000000D1, '134621281f00a3f8aeeb7660064bffbf6187ed56d5852142328d0bcb18ef0ede' ),
1086	( True, ktReason_BSOD_000000D1, '279f11258150c9d2fef041eca65501f3141da8df39256d8f6377e897e3b45a93' ),
1087	( True, ktReason_BSOD_C0000225, 'bd13a144be9dcdfb16bc863ff4c8f02a86e263c174f2cd5ffd27ca5f3aa31789' ),
1088	( True, ktReason_BSOD_C0000225, '8348b465e7ee9e59dd4e785880c57fd8677de05d11ac21e786bfde935307b42f' ),
1089	( True, ktReason_BSOD_C0000225, '1316e1fc818a73348412788e6910b8c016f237d8b4e15b20caf4a866f7a7840e' ),
1090	( True, ktReason_BSOD_C0000225, '54e0acbff365ce20a85abbe42bcd53647b8b9e80c68e45b2cd30e86bf177a0b5' ),
1091	( True, ktReason_BSOD_C0000225, '50fec50b5199923fa48b3f3e782687cc381e1c8a788ebda14e6a355fbe3bb1b3' ),
1092	];
1093
1094	def investigateVMResult(self, oCaseFile, oFailedResult, sResultLog):
1095	"""
1096	Investigates a failed VM run.
1097	"""
1098
1099	def investigateLogSet():
1100	"""
1101	Investigates the current set of VM related logs.
1102	"""
1103	self.dprint('investigateLogSet: log lengths: result %u, VM %u, kernel %u, vga text %u, info text %u, hard %u'
1104	% ( len(sResultLog if sResultLog else ''),
1105	len(sVMLog if sVMLog else ''),
1106	len(sKrnlLog if sKrnlLog else ''),
1107	len(sVgaText if sVgaText else ''),
1108	len(sInfoText if sInfoText else ''),
1109	len(sNtHardLog if sNtHardLog else ''), ));
1110
1111	#self.dprint(u'main.log<<<\n%s\n<<<\n' % (sResultLog,));
1112	#self.dprint(u'vbox.log<<<\n%s\n<<<\n' % (sVMLog,));
1113	#self.dprint(u'krnl.log<<<\n%s\n<<<\n' % (sKrnlLog,));
1114	#self.dprint(u'vgatext.txt<<<\n%s\n<<<\n' % (sVgaText,));
1115	#self.dprint(u'info.txt<<<\n%s\n<<<\n' % (sInfoText,));
1116	#self.dprint(u'hard.txt<<<\n%s\n<<<\n' % (sNtHardLog,));
1117
1118	# TODO: more
1119
1120	#
1121	# Look for BSODs. Some stupid stupid inconsistencies in reason and log messages here, so don't try prettify this.
1122	#
1123	sDetails = self.findInAnyAndReturnRestOfLine([ sVMLog, sResultLog ],
1124	'GIM: HyperV: Guest indicates a fatal condition! P0=');
1125	if sDetails is not None:
1126	# P0=%#RX64 P1=%#RX64 P2=%#RX64 P3=%#RX64 P4=%#RX64 "
1127	sKey = sDetails.split(' ', 1)[0];
1128	try: sKey = '0x%08X' % (int(sKey, 16),);
1129	except: pass;
1130	if sKey in self.asBsodReasons:
1131	tReason = ( self.ksBsodCategory, sKey );
1132	elif sKey.lower() in self.asBsodReasons: # just in case.
1133	tReason = ( self.ksBsodCategory, sKey.lower() );
1134	else:
1135	self.dprint(u'BSOD "%s" not found in %s;' % (sKey, self.asBsodReasons));
1136	tReason = ( self.ksBsodCategory, self.ksBsodAddNew );
1137	return oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult, sComment = sDetails.strip());
1138
1139	#
1140	# Look for linux panic.
1141	#
1142	if sKrnlLog is not None:
1143	for fStopOnHit, tReason, sNeedle in self.katSimpleKernelLogReasons:
1144	if sKrnlLog.find(sNeedle) > 0:
1145	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1146	if fStopOnHit:
1147	return True;
1148	fFoundSomething = True;
1149
1150	#
1151	# Loop thru the simple stuff.
1152	#
1153	fFoundSomething = False;
1154	for fStopOnHit, tReason, sNeedle in self.katSimpleMainAndVmLogReasons:
1155	if sResultLog.find(sNeedle) > 0 or (sVMLog is not None and sVMLog.find(sNeedle) > 0):
1156	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1157	if fStopOnHit:
1158	return True;
1159	fFoundSomething = True;
1160
1161	# Continue with vga text.
1162	if sVgaText:
1163	for fStopOnHit, tReason, sNeedle in self.katSimpleVgaTextReasons:
1164	if sVgaText.find(sNeedle) > 0:
1165	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1166	if fStopOnHit:
1167	return True;
1168	fFoundSomething = True;
1169	_ = sInfoText;
1170
1171	# Continue with screen hashes.
1172	if sScreenHash is not None:
1173	for fStopOnHit, tReason, sHash in self.katSimpleScreenshotHashReasons:
1174	if sScreenHash == sHash:
1175	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1176	if fStopOnHit:
1177	return True;
1178	fFoundSomething = True;
1179
1180	# Check VBoxHardening.log.
1181	if sNtHardLog is not None:
1182	for fStopOnHit, tReason, sNeedle in self.katSimpleVBoxHardeningLogReasons:
1183	if sNtHardLog.find(sNeedle) > 0:
1184	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1185	if fStopOnHit:
1186	return True;
1187	fFoundSomething = True;
1188
1189	#
1190	# Complicated stuff.
1191	#
1192	dLogs = {
1193	'sVMLog': sVMLog,
1194	'sNtHardLog': sNtHardLog,
1195	'sScreenHash': sScreenHash,
1196	'sKrnlLog': sKrnlLog,
1197	'sVgaText': sVgaText,
1198	'sInfoText': sInfoText,
1199	};
1200
1201	# info.txt.
1202	if sInfoText:
1203	for sNeedle, fnHandler in self.katInfoTextHandlers:
1204	if sInfoText.find(sNeedle) > 0:
1205	(fStop, tReason) = fnHandler(self, oCaseFile, sInfoText, dLogs);
1206	if tReason is not None:
1207	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1208	if fStop:
1209	return True;
1210	fFoundSomething = True;
1211
1212	#
1213	# Check for repeated reboots...
1214	#
1215	if sVMLog is not None:
1216	cResets = sVMLog.count('Changing the VM state from \'RUNNING\' to \'RESETTING\'');
1217	if cResets > 10:
1218	return oCaseFile.noteReasonForId(self.ktReason_Unknown_Reboot_Loop, oFailedResult.idTestResult,
1219	sComment = 'Counted %s reboots' % (cResets,));
1220
1221	return fFoundSomething;
1222
1223	#
1224	# Check if we got any VM or/and kernel logs. Treat them as sets in
1225	# case we run multiple VMs here (this is of course ASSUMING they
1226	# appear in the order that terminateVmBySession uploads them).
1227	#
1228	cTimes = 0;
1229	sVMLog = None;
1230	sNtHardLog = None;
1231	sScreenHash = None;
1232	sKrnlLog = None;
1233	sVgaText = None;
1234	sInfoText = None;
1235	for oFile in oFailedResult.aoFiles:
1236	if oFile.sKind == TestResultFileData.ksKind_LogReleaseVm:
1237	if 'VBoxHardening.log' not in oFile.sFile:
1238	if sVMLog is not None:
1239	if investigateLogSet() is True:
1240	return True;
1241	cTimes += 1;
1242	sInfoText = None;
1243	sVgaText = None;
1244	sKrnlLog = None;
1245	sScreenHash = None;
1246	sNtHardLog = None;
1247	sVMLog = oCaseFile.getLogFile(oFile);
1248	else:
1249	sNtHardLog = oCaseFile.getLogFile(oFile);
1250	elif oFile.sKind == TestResultFileData.ksKind_LogGuestKernel:
1251	sKrnlLog = oCaseFile.getLogFile(oFile);
1252	elif oFile.sKind == TestResultFileData.ksKind_InfoVgaText:
1253	sVgaText = '\n'.join([sLine.rstrip() for sLine in oCaseFile.getLogFile(oFile).split('\n')]);
1254	elif oFile.sKind == TestResultFileData.ksKind_InfoCollection:
1255	sInfoText = oCaseFile.getLogFile(oFile);
1256	elif oFile.sKind == TestResultFileData.ksKind_ScreenshotFailure:
1257	sScreenHash = oCaseFile.getScreenshotSha256(oFile);
1258	if sScreenHash is not None:
1259	sScreenHash = sScreenHash.lower();
1260	self.vprint(u'%s %s' % ( sScreenHash, oFile.sFile,));
1261
1262	if ( sVMLog is not None \
1263	or sNtHardLog is not None \
1264	or cTimes == 0) \
1265	and investigateLogSet() is True:
1266	return True;
1267
1268	return None;
1269
1270
1271	def isResultFromVMRun(self, oFailedResult, sResultLog):
1272	"""
1273	Checks if this result and corresponding log snippet looks like a VM run.
1274	"""
1275
1276	# Look for startVmEx/ startVmAndConnectToTxsViaTcp and similar output in the log.
1277	if sResultLog.find(' startVm') > 0:
1278	return True;
1279
1280	# Any other indicators? No?
1281	_ = oFailedResult;
1282	return False;
1283
1284	def investigateVBoxVMTest(self, oCaseFile, fSingleVM):
1285	"""
1286	Checks out a VBox VM test.
1287
1288	This is generic investigation of a test running one or more VMs, like
1289	for example a smoke test or a guest installation test.
1290
1291	The fSingleVM parameter is a hint, which probably won't come in useful.
1292	"""
1293	_ = fSingleVM;
1294
1295	#
1296	# Get a list of test result failures we should be looking into and the main log.
1297	#
1298	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1299	sMainLog = oCaseFile.getMainLog();
1300
1301	#
1302	# There are a set of errors ending up on the top level result record.
1303	# Should deal with these first.
1304	#
1305	if len(aoFailedResults) == 1 and aoFailedResults[0] == oCaseFile.oTree:
1306	# Check if we've just got that XPCOM client smoke test shutdown issue. This will currently always
1307	# be reported on the top result because vboxinstall.py doesn't add an error for it. It is easy to
1308	# ignore other failures in the test if we're not a little bit careful here.
1309	if sMainLog.find('vboxinstaller: Exit code: -11 (') > 0:
1310	oCaseFile.noteReason(self.ktReason_XPCOM_Exit_Minus_11);
1311	return self.caseClosed(oCaseFile);
1312
1313	# Hang after starting VBoxSVC (e.g. idTestSet=136307258)
1314	if self.isThisFollowedByTheseLines(sMainLog, 'oVBoxMgr=<vboxapi.VirtualBoxManager object at',
1315	(' Timeout: ', ' Attempting to abort child...',) ):
1316	if sMainLog.find('* glibc detected * /') > 0:
1317	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang_Plus_Heap_Corruption);
1318	else:
1319	oCaseFile.noteReason(self.ktReason_XPCOM_VBoxSVC_Hang);
1320	return self.caseClosed(oCaseFile);
1321
1322	# Look for heap corruption without visible hang.
1323	if sMainLog.find('* glibc detected * /') > 0 \
1324	or sMainLog.find("-1073740940") > 0: # STATUS_HEAP_CORRUPTION / 0xc0000374
1325	oCaseFile.noteReason(self.ktReason_Unknown_Heap_Corruption);
1326	return self.caseClosed(oCaseFile);
1327
1328	# Out of memory w/ timeout.
1329	if sMainLog.find('sErrId=HostMemoryLow') > 0:
1330	oCaseFile.noteReason(self.ktReason_Host_HostMemoryLow);
1331	return self.caseClosed(oCaseFile);
1332
1333	# Stale files like vts_rm.exe (windows).
1334	offEnd = sMainLog.rfind('* The test driver exits successfully. *');
1335	if offEnd > 0 and sMainLog.find('[Error 145] The directory is not empty: ', offEnd) > 0:
1336	oCaseFile.noteReason(self.ktReason_Ignore_Stale_Files);
1337	return self.caseClosed(oCaseFile);
1338
1339	#
1340	# XPCOM screwup
1341	#
1342	if sMainLog.find('AttributeError: \'NoneType\' object has no attribute \'addObserver\'') > 0:
1343	oCaseFile.noteReason(self.ktReason_Buggy_Build_Broken_Build);
1344	return self.caseClosed(oCaseFile);
1345
1346	#
1347	# Go thru each failed result.
1348	#
1349	for oFailedResult in aoFailedResults:
1350	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1351	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1352	if oFailedResult.sName == 'Installing VirtualBox':
1353	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1354
1355	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1356	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1357
1358	elif self.isResultFromVMRun(oFailedResult, sResultLog):
1359	self.investigateVMResult(oCaseFile, oFailedResult, sResultLog);
1360
1361	elif sResultLog.find('most likely not unique') > 0:
1362	oCaseFile.noteReasonForId(self.ktReason_Host_NetworkMisconfiguration, oFailedResult.idTestResult)
1363	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1364	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1365
1366	elif sResultLog.find('The machine is not mutable (state is ') > 0:
1367	self.vprint('Ignoring "machine not mutable" error as it is probably due to an earlier problem');
1368	oCaseFile.noteReasonForId(self.ktHarmless, oFailedResult.idTestResult);
1369
1370	elif sResultLog.find('** error: no action was specified') > 0 \
1371	or sResultLog.find('(len(self._asXml, asText))') > 0:
1372	oCaseFile.noteReasonForId(self.ktReason_Ignore_Buggy_Test_Driver, oFailedResult.idTestResult);
1373
1374	else:
1375	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1376	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1377
1378	#
1379	# Report home and close the case if we got them all, otherwise log it.
1380	#
1381	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1382	return self.caseClosed(oCaseFile);
1383
1384	if oCaseFile.dReasonForResultId:
1385	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1386	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1387	else:
1388	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1389	return False;
1390
1391
1392	## Things we search a main log for to figure out why something in the API test went bust.
1393	katSimpleApiMainLogReasons = [
1394	# ( Whether to stop on hit, reason tuple, needle text. )
1395	( True, ktReason_Networking_Nonexistent_host_nic,
1396	'rc=E_FAIL text="Nonexistent host networking interface, name \'eth0\' (VERR_INTERNAL_ERROR)"' ),
1397	( False, ktReason_XPCOM_NS_ERROR_CALL_FAILED,
1398	'Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))' ),
1399	( True, ktReason_API_std_bad_alloc, 'Unexpected exception: std::bad_alloc' ),
1400	( True, ktReason_API_Digest_Mismatch, 'Digest mismatch (VERR_NOT_EQUAL)' ),
1401	( True, ktReason_API_MoveVM_SharingViolation, 'rc=VBOX_E_IPRT_ERROR text="Could not copy the log file ' ),
1402	( True, ktReason_API_MoveVM_InvalidParameter,
1403	'rc=VBOX_E_IPRT_ERROR text="Could not copy the setting file ' ),
1404	( True, ktReason_API_Open_Session_Failed, 'error: failed to open session for' ),
1405	];
1406
1407	def investigateVBoxApiTest(self, oCaseFile):
1408	"""
1409	Checks out a VBox API test.
1410	"""
1411
1412	#
1413	# Get a list of test result failures we should be looking into and the main log.
1414	#
1415	aoFailedResults = oCaseFile.oTree.getListOfFailures();
1416	sMainLog = oCaseFile.getMainLog();
1417
1418	#
1419	# Go thru each failed result.
1420	#
1421	for oFailedResult in aoFailedResults:
1422	self.dprint(u'Looking at test result #%u - %s' % (oFailedResult.idTestResult, oFailedResult.getFullName(),));
1423	sResultLog = TestSetData.extractLogSectionElapsed(sMainLog, oFailedResult.tsCreated, oFailedResult.tsElapsed);
1424	if oFailedResult.sName == 'Installing VirtualBox':
1425	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = True)
1426
1427	elif oFailedResult.sName == 'Uninstalling VirtualBox':
1428	self.investigateInstallUninstallFailure(oCaseFile, oFailedResult, sResultLog, fInstall = False)
1429
1430	elif sResultLog.find('Exception: 0x800706be (Call to remote object failed (NS_ERROR_CALL_FAILED))') > 0:
1431	oCaseFile.noteReasonForId(self.ktReason_XPCOM_NS_ERROR_CALL_FAILED, oFailedResult.idTestResult);
1432
1433	else:
1434	fFoundSomething = False;
1435	for fStopOnHit, tReason, sNeedle in self.katSimpleApiMainLogReasons:
1436	if sResultLog.find(sNeedle) > 0:
1437	oCaseFile.noteReasonForId(tReason, oFailedResult.idTestResult);
1438	fFoundSomething = True;
1439	if fStopOnHit:
1440	break;
1441	if fFoundSomething:
1442	self.vprint(u'TODO: Cannot place idTestResult=%u - %s' % (oFailedResult.idTestResult, oFailedResult.sName,));
1443	self.dprint(u'%s + %s <<\n%s\n<<' % (oFailedResult.tsCreated, oFailedResult.tsElapsed, sResultLog,));
1444
1445	#
1446	# Report home and close the case if we got them all, otherwise log it.
1447	#
1448	if len(oCaseFile.dReasonForResultId) >= len(aoFailedResults):
1449	return self.caseClosed(oCaseFile);
1450
1451	if oCaseFile.dReasonForResultId:
1452	self.vprint(u'TODO: Got %u out of %u - close, but no cigar. :-/'
1453	% (len(oCaseFile.dReasonForResultId), len(aoFailedResults)));
1454	else:
1455	self.vprint(u'XXX: Could not figure out anything at all! :-(');
1456	return False;
1457
1458
1459	def reasoningFailures(self):
1460	"""
1461	Guess the reason for failures.
1462	"""
1463	#
1464	# Get a list of failed test sets without any assigned failure reason.
1465	#
1466	cGot = 0;
1467	if not self.oConfig.aidTestSets:
1468	aoTestSets = self.oTestSetLogic.fetchFailedSetsWithoutReason(cHoursBack = self.oConfig.cHoursBack,
1469	tsNow = self.tsNow);
1470	else:
1471	aoTestSets = [self.oTestSetLogic.getById(idTestSet) for idTestSet in self.oConfig.aidTestSets];
1472	for oTestSet in aoTestSets:
1473	self.dprint(u'----------------------------------- #%u, status %s -----------------------------------'
1474	% ( oTestSet.idTestSet, oTestSet.enmStatus,));
1475
1476	#
1477	# Open a case file and assign it to the right investigator.
1478	#
1479	(oTree, _ ) = self.oTestResultLogic.fetchResultTree(oTestSet.idTestSet);
1480	oBuild = BuildDataEx().initFromDbWithId( self.oDb, oTestSet.idBuild, oTestSet.tsCreated);
1481	oTestBox = TestBoxData().initFromDbWithGenId( self.oDb, oTestSet.idGenTestBox);
1482	oTestGroup = TestGroupData().initFromDbWithId( self.oDb, oTestSet.idTestGroup, oTestSet.tsCreated);
1483	oTestCase = TestCaseDataEx().initFromDbWithGenId( self.oDb, oTestSet.idGenTestCase, oTestSet.tsConfig);
1484
1485	oCaseFile = VirtualTestSheriffCaseFile(self, oTestSet, oTree, oBuild, oTestBox, oTestGroup, oTestCase);
1486
1487	if oTestSet.enmStatus == TestSetData.ksTestStatus_BadTestBox:
1488	self.dprint(u'investigateBadTestBox is taking over %s.' % (oCaseFile.sLongName,));
1489	fRc = self.investigateBadTestBox(oCaseFile);
1490
1491	elif oCaseFile.isVBoxUnitTest():
1492	self.dprint(u'investigateVBoxUnitTest is taking over %s.' % (oCaseFile.sLongName,));
1493	fRc = self.investigateVBoxUnitTest(oCaseFile);
1494
1495	elif oCaseFile.isVBoxInstallTest() or oCaseFile.isVBoxUnattendedInstallTest():
1496	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1497	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1498
1499	elif oCaseFile.isVBoxUSBTest():
1500	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1501	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1502
1503	elif oCaseFile.isVBoxStorageTest():
1504	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1505	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1506
1507	elif oCaseFile.isVBoxGAsTest():
1508	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1509	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = True);
1510
1511	elif oCaseFile.isVBoxAPITest():
1512	self.dprint(u'investigateVBoxApiTest is taking over %s.' % (oCaseFile.sLongName,));
1513	fRc = self.investigateVBoxApiTest(oCaseFile);
1514
1515	elif oCaseFile.isVBoxBenchmarkTest():
1516	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1517	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1518
1519	elif oCaseFile.isVBoxSmokeTest():
1520	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1521	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1522
1523	elif oCaseFile.isVBoxSerialTest():
1524	self.dprint(u'investigateVBoxVMTest is taking over %s.' % (oCaseFile.sLongName,));
1525	fRc = self.investigateVBoxVMTest(oCaseFile, fSingleVM = False);
1526
1527	else:
1528	self.vprint(u'reasoningFailures: Unable to classify test set: %s' % (oCaseFile.sLongName,));
1529	fRc = False;
1530	cGot += fRc is True;
1531
1532	self.vprint(u'reasoningFailures: Got %u out of %u' % (cGot, len(aoTestSets), ));
1533	return 0;
1534
1535
1536	def main(self):
1537	"""
1538	The 'main' function.
1539	Return exit code (0, 1, etc).
1540	"""
1541	# Database stuff.
1542	self.oDb = TMDatabaseConnection()
1543	self.oTestResultLogic = TestResultLogic(self.oDb);
1544	self.oTestSetLogic = TestSetLogic(self.oDb);
1545	self.oFailureReasonLogic = FailureReasonLogic(self.oDb);
1546	self.oTestResultFailureLogic = TestResultFailureLogic(self.oDb);
1547	self.asBsodReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksBsodCategory);
1548	self.asUnitTestReasons = self.oFailureReasonLogic.fetchForSheriffByNamedCategory(self.ksUnitTestCategory);
1549
1550	# Get a fix on our 'now' before we do anything..
1551	self.oDb.execute('SELECT CURRENT_TIMESTAMP - interval \'%s hours\'', (self.oConfig.cStartHoursAgo,));
1552	self.tsNow = self.oDb.fetchOne();
1553
1554	# If we're suppost to commit anything we need to get our user ID.
1555	rcExit = 0;
1556	if self.oConfig.fRealRun:
1557	self.oLogin = UserAccountLogic(self.oDb).tryFetchAccountByLoginName(VirtualTestSheriff.ksLoginName);
1558	if self.oLogin is None:
1559	rcExit = self.eprint('Cannot find my user account "%s"!' % (VirtualTestSheriff.ksLoginName,));
1560	else:
1561	self.uidSelf = self.oLogin.uid;
1562
1563	#
1564	# Do the stuff.
1565	#
1566	if rcExit == 0:
1567	rcExit = self.selfCheck();
1568	if rcExit == 0:
1569	rcExit = self.badTestBoxManagement();
1570	rcExit2 = self.reasoningFailures();
1571	if rcExit == 0:
1572	rcExit = rcExit2;
1573	# Redo the bad testbox management after failure reasons have been assigned (got timing issues).
1574	if rcExit == 0:
1575	rcExit = self.badTestBoxManagement();
1576
1577	# Cleanup.
1578	self.oFailureReasonLogic = None;
1579	self.oTestResultFailureLogic = None;
1580	self.oTestSetLogic = None;
1581	self.oTestResultLogic = None;
1582	self.oDb.close();
1583	self.oDb = None;
1584	if self.oLogFile is not None:
1585	self.oLogFile.close();
1586	self.oLogFile = None;
1587	return rcExit;
1588
1589	if __name__ == '__main__':
1590	sys.exit(VirtualTestSheriff().main());
1591

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/batch/virtual_test_sheriff.py@ 79918

Download in other formats: