tdUnitTest1.py@ 97288

Last change on this file since 97288 was 97288, checked in by vboxsync, 2 years ago

ValidationKit/tests/unittests/tdUnitTest1.py: Skip checking for black list / buggy tests list etc. when executing with --only-whitelist. This allows a unit test being black listed AND white listed at the same time, so that it only gets executed when --only-whitelist is being used (but skipped on the rest). Handy for unit tests which require a full-blown OS (e.g. a guest VM) instead of a (test) box which only has minimal dependencies.

Added testcase/tstClipboardMockHGCM, so that it only executes remotely (nested) on the guest VMS for the reasons above.

Property svn:eol-style set to LF
Property svn:executable set to *
Property svn:keywords set to Author Date Id Revision

File size: 53.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: tdUnitTest1.py 97288 2022-10-25 07:13:52Z vboxsync $
4
5	"""
6	VirtualBox Validation Kit - Unit Tests.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2022 Oracle and/or its affiliates.
12
13	This file is part of VirtualBox base platform packages, as
14	available from https://www.virtualbox.org.
15
16	This program is free software; you can redistribute it and/or
17	modify it under the terms of the GNU General Public License
18	as published by the Free Software Foundation, in version 3 of the
19	License.
20
21	This program is distributed in the hope that it will be useful, but
22	WITHOUT ANY WARRANTY; without even the implied warranty of
23	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	General Public License for more details.
25
26	You should have received a copy of the GNU General Public License
27	along with this program; if not, see <https://www.gnu.org/licenses>.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32	in the VirtualBox distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37
38	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39	"""
40	__version__ = "$Revision: 97288 $"
41
42
43	# Standard Python imports.
44	import os
45	import sys
46	import re
47
48
49	# Only the main script needs to modify the path.
50	try: __file__
51	except: __file__ = sys.argv[0];
52	g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
53	sys.path.append(g_ksValidationKitDir)
54
55	# Validation Kit imports.
56	from common import utils;
57	from testdriver import base;
58	from testdriver import reporter;
59	from testdriver import vbox;
60	from testdriver import vboxcon;
61
62
63	class tdUnitTest1(vbox.TestDriver):
64	"""
65	Unit Tests.
66	"""
67
68	## The temporary exclude list.
69	## @note This shall be empty before we release 4.3!
70	kdTestCasesBuggyPerOs = {
71	'darwin': {
72	'testcase/tstX86-1': '', # 'FSTP M32R, ST0' fails; no idea why.
73	},
74	'linux': {
75	'testcase/tstRTFileAio': '', # See xTracker #8035.
76	},
77	'linux.amd64': {
78	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
79	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
80	},
81	'solaris': {
82	'testcase/tstIntNet-1': '', # Fails opening rge0, probably a generic issue figuring which nic to use.
83	'testcase/tstIprtList': '', # Crashes in the multithreaded test, I think.
84	'testcase/tstRTCritSect': '', # Fairness/whatever issue here.
85	'testcase/tstRTR0MemUserKernelDriver': '', # Failes when kernel to kernel buffers.
86	'testcase/tstRTSemRW': '', # line 338: RTSemRWReleaseRead(hSemRW): got VERR_ACCESS_DENIED
87	'testcase/tstRTStrAlloc': '', # VERR_NO_STR_MEMORY!
88	'testcase/tstRTFileQuerySize-1': '', # VERR_DEV_IO_ERROR on /dev/null!
89	},
90	'solaris.amd64': {
91	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
92	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
93	},
94	'win': {
95	'testcase/tstFile': '', # ??
96	'testcase/tstIntNet-1': '', # possibly same issue as solaris.
97	'testcase/tstMouseImpl': '', # STATUS_ACCESS_VIOLATION
98	'testcase/tstRTR0ThreadPreemptionDriver': '', # ??
99	'testcase/tstRTPath': '<4.3.51r89894',
100	'testcase/tstRTPipe': '', # ??
101	'testcase/tstRTR0MemUserKernelDriver': '', # ??
102	'testcase/tstRTR0SemMutexDriver': '', # ??
103	'testcase/tstRTStrAlloc': '', # ??
104	'testcase/tstRTStrFormat': '', # ??
105	'testcase/tstRTSystemQueryOsInfo': '', # ??
106	'testcase/tstRTTemp': '', # ??
107	'testcase/tstRTTime': '', # ??
108	'testcase/tstTime-2': '', # Total time differs too much! ... delta=-10859859
109	'testcase/tstTime-4': '', # Needs to be converted to DLL; ditto for tstTime-2.
110	'testcase/tstUtf8': '', # ??
111	'testcase/tstVMMR0CallHost-2': '', # STATUS_STACK_OVERFLOW
112	'testcase/tstX86-1': '', # Fails on win.x86.
113	'tscpasswd': '', # ??
114	'tstVMREQ': '', # ?? Same as darwin.x86?
115	},
116	'win.x86': {
117	'testcase/tstRTR0TimerDriver': '', # See xTracker #8041.
118	}
119	};
120
121	kdTestCasesBuggy = {
122	'testcase/tstGuestPropSvc': '', # GET_NOTIFICATION fails on testboxlin5.de.oracle.com and others.
123	'testcase/tstRTProcCreateEx': '', # Seen failing on wei01-b6ka-9.de.oracle.com.
124	'testcase/tstTimer': '', # Sometimes fails on linux, not important atm.
125	'testcase/tstGIP-2': '', # 2015-09-10: Fails regularly. E.g. TestSetID 2744205 (testboxsh2),
126	# 2743961 (wei01-b6kc-6). The responsible engineer should reenable
127	# it once it has been fixed.
128	};
129
130	## The permanent exclude list.
131	# @note Stripped of extensions!
132	kdTestCasesBlackList = {
133	'testcase/tstClipboardX11Smoke': '', # (Old naming, deprecated) Needs X, not available on all test boxes.
134	'testcase/tstClipboardGH-X11Smoke': '', # (New name) Ditto.
135	'testcase/tstClipboardMockHGCM': '', # Ditto.
136	'tstClipboardQt': '', # Is interactive and needs Qt, needed for Qt clipboard bugfixing.
137	'testcase/tstClipboardQt': '', # In case it moves here.
138	'testcase/tstFileLock': '',
139	'testcase/tstDisasm-2': '', # without parameters it will disassembler 1GB starting from 0
140	'testcase/tstFileAppendWin-1': '',
141	'testcase/tstDir': '', # useless without parameters
142	'testcase/tstDir-2': '', # useless without parameters
143	'testcase/tstGlobalConfig': '',
144	'testcase/tstHostHardwareLinux': '', # must be killed with CTRL-C
145	'testcase/tstHttp': '', # Talks to outside servers.
146	'testcase/tstRTHttp': '', # parameters required
147	'testcase/tstLdr-2': '', # parameters required
148	'testcase/tstLdr-3': '', # parameters required
149	'testcase/tstLdr': '', # parameters required
150	'testcase/tstLdrLoad': '', # parameters required
151	'testcase/tstMove': '', # parameters required
152	'testcase/tstRTR0Timer': '', # loads 'tstRTR0Timer.r0'
153	'testcase/tstRTR0ThreadDriver': '', # loads 'tstRTR0Thread.r0'
154	'testcase/tstRunTestcases': '', # that's a script like this one
155	'testcase/tstRTReqPool': '', # fails sometimes, testcase buggy
156	'testcase/tstRTS3': '', # parameters required
157	'testcase/tstSDL': '', # graphics test
158	'testcase/tstSupLoadModule': '', # Needs parameters and vboxdrv access. Covered elsewhere.
159	'testcase/tstSeamlessX11': '', # graphics test
160	'testcase/tstTime-3': '', # parameters required
161	'testcase/tstVBoxControl': '', # works only inside a guest
162	'testcase/tstVDCopy': '', # parameters required
163	'testcase/tstVDFill': '', # parameters required
164	'tstAnimate': '', # parameters required
165	'testcase/tstAPI': '', # user interaction required
166	'tstCollector': '', # takes forever
167	'testcase/tstHeadless': '', # parameters required
168	'tstHeadless': '', # parameters required
169	'tstMicroRC': '', # required by tstMicro
170	'tstVBoxDbg': '', # interactive test
171	'testcase/tstTestServMgr': '', # some strange xpcom18a4 test, does not work
172	'tstTestServMgr': '', # some strange xpcom18a4 test, does not work
173	'tstPDMAsyncCompletion': '', # parameters required
174	'testcase/tstXptDump': '', # parameters required
175	'tstXptDump': '', # parameters required
176	'testcase/tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
177	'tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
178	'testcase/tstSimpleTypeLib': '', # parameters required
179	'tstSimpleTypeLib': '', # parameters required
180	'testcase/tstTestAtoms': '', # additional test file (words.txt) required
181	'tstTestAtoms': '', # additional test file (words.txt) required
182	'testcase/tstXptLink': '', # parameters required
183	'tstXptLink': '', # parameters required
184	'tstXPCOMCGlue': '', # user interaction required
185	'testcase/tstXPCOMCGlue': '', # user interaction required
186	'testcase/tstCAPIGlue': '', # user interaction required
187	'testcase/tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
188	'tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
189	'testcase/tstRTFilesystem': '', # parameters required
190	'testcase/tstRTDvm': '', # parameters required
191	'tstSSLCertDownloads': '', # Obsolete.
192	# later
193	'testcase/tstIntNetR0': '', # RTSPINLOCK_FLAGS_INTERRUPT_SAFE == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE
194	# slow stuff
195	'testcase/tstAvl': '', # SLOW!
196	'testcase/tstRTAvl': '', # SLOW! (new name)
197	'testcase/tstVD': '', # 8GB fixed-sized vmdk
198	# failed or hang
199	'testcase/tstCryptoPkcs7Verify': '', # hang
200	'tstOVF': '', # hang (only ancient version, now in new place)
201	'testcase/tstRTLockValidator': '', # Lock validation is not enabled for critical sections
202	'testcase/tstGuestControlSvc': '', # failed: line 288: testHost(&svcTable):
203	# expected VINF_SUCCESS, got VERR_NOT_FOUND
204	'testcase/tstRTMemEf': '', # failed w/o error message
205	'testcase/tstSupSem': '', # failed: SRE Timeout Accuracy (ms) : FAILED (1 errors)
206	'testcase/tstCryptoPkcs7Sign': '', # failed: 29330:
207	# error:02001002:lib(2):func(1):reason(2):NA:0:fopen('server.pem': '','r')
208	'testcase/tstCompressionBenchmark': '', # failed: error: RTZipBlockCompress failed
209	# for 'RTZipBlock/LZJB' (#4): VERR_NOT_SUPPORTED
210	'tstPDMAsyncCompletionStress': '', # VERR_INVALID_PARAMETER (cbSize = 0)
211	'tstMicro': '', # doesn't work on solaris, fix later if we care.
212	'tstVMM-HwAccm': '', # failed: Only checked AMD-V on linux
213	'tstVMM-HM': '', # failed: Only checked AMD-V on linux
214	'tstVMMFork': '', # failed: xtracker 6171
215	'tstTestFactory': '', # some strange xpcom18a4 test, does not work
216	'testcase/tstRTSemXRoads': '', # sporadically failed: Traffic - 8 threads per direction, 10 sec :
217	# FAILED (8 errors)
218	'tstVBoxAPILinux': '', # creates VirtualBox directories for root user because of sudo
219	# (should be in vbox)
220	'testcase/tstVMStructDTrace': '', # This is a D-script generator.
221	'tstVMStructRC': '', # This is a C-code generator.
222	'tstDeviceStructSizeRC': '', # This is a C-code generator.
223	'testcase/tstTSC': '', # Doesn't test anything and might fail with HT or/and too many cores.
224	'testcase/tstOpenUSBDev': '', # Not a useful testcase.
225	'testcase/tstX86-1': '', # Really more guest side.
226	'testcase/tstX86-FpuSaveRestore': '', # Experiments, could be useful for the guest not the host.
227	'tstAsmStructsRC': '', # Testcase run during build time (fails to find libstdc++.so.6 on some
228	# Solaris testboxes).
229	};
230
231	# Suffix exclude list.
232	kasSuffixBlackList = [
233	'.r0',
234	'.gc',
235	'.debug',
236	'.rel',
237	'.sys',
238	'.ko',
239	'.o',
240	'.obj',
241	'.lib',
242	'.a',
243	'.so',
244	'.dll',
245	'.dylib',
246	'.tmp',
247	'.log',
248	'.py',
249	'.pyc',
250	'.pyo',
251	'.pdb',
252	'.dSYM',
253	'.sym',
254	'.template',
255	'.expected',
256	'.expect',
257	];
258
259	# White list, which contains tests considered to be safe to execute,
260	# even on remote targets (guests).
261	#
262	# When --only-whitelist is specified, this is the only list being checked for.
263	kdTestCasesWhiteList = {
264	'testcase/tstFile': '',
265	'testcase/tstFileLock': '',
266	'testcase/tstClipboardMockHGCM': '', # Requires X on Linux OSes. Execute on remote targets only (guests).
267	'testcase/tstRTLocalIpc': '',
268	'testcase/tstRTPathQueryInfo': '',
269	'testcase/tstRTPipe': '',
270	'testcase/tstRTProcCreateEx': '',
271	'testcase/tstRTProcCreatePrf': '',
272	'testcase/tstRTProcIsRunningByName': '',
273	'testcase/tstRTProcQueryUsername': '',
274	'testcase/tstRTProcWait': '',
275	'testcase/tstTime-2': '',
276	'testcase/tstTime-3': '',
277	'testcase/tstTime-4': '',
278	'testcase/tstTimer': '',
279	'testcase/tstThread-1': '',
280	'testcase/tstUtf8': ''
281	};
282
283	# Test dependency list -- libraries.
284	# Needed in order to execute testcases on remote targets which don't have a VBox installation present.
285	kdTestCaseDepsLibs = [
286	"VBoxRT"
287	];
288
289	## The exclude list.
290	# @note Stripped extensions!
291	kasHardened = [
292	"testcase/tstIntNet-1",
293	"testcase/tstR0ThreadPreemptionDriver", # VBox 4.3
294	"testcase/tstRTR0ThreadPreemptionDriver",
295	"testcase/tstRTR0MemUserKernelDriver",
296	"testcase/tstRTR0SemMutexDriver",
297	"testcase/tstRTR0TimerDriver",
298	"testcase/tstRTR0ThreadDriver",
299	'testcase/tstRTR0DbgKrnlInfoDriver',
300	"tstInt",
301	"tstPDMQueue", # Comment in testcase says its driverless, but it needs driver access.
302	"tstVMM",
303	"tstVMMFork",
304	"tstVMREQ",
305	'testcase/tstCFGM',
306	'testcase/tstContiguous',
307	'testcase/tstGetPagingMode',
308	'testcase/tstGIP-2',
309	'testcase/tstInit',
310	'testcase/tstLow',
311	'testcase/tstMMHyperHeap',
312	'testcase/tstPage',
313	'testcase/tstPin',
314	'testcase/tstRTTime', 'testcase/tstTime', # GIP test case.
315	'testcase/tstRTTime-2', 'testcase/tstTime-2', # GIP test case.
316	'testcase/tstRTTime-4', 'testcase/tstTime-4', # GIP test case.
317	'testcase/tstSSM',
318	'testcase/tstSupSem-Zombie',
319	]
320
321	## Argument lists
322	kdArguments = {
323	'testcase/tstbntest': [ '-out', os.devnull, ], # Very noisy.
324	};
325
326
327	## Status code translations.
328	## @{
329	kdExitCodeNames = {
330	0: 'RTEXITCODE_SUCCESS',
331	1: 'RTEXITCODE_FAILURE',
332	2: 'RTEXITCODE_SYNTAX',
333	3: 'RTEXITCODE_INIT',
334	4: 'RTEXITCODE_SKIPPED',
335	};
336	kdExitCodeNamesWin = {
337	-1073741515: 'STATUS_DLL_NOT_FOUND',
338	-1073741512: 'STATUS_ORDINAL_NOT_FOUND',
339	-1073741511: 'STATUS_ENTRYPOINT_NOT_FOUND',
340	-1073741502: 'STATUS_DLL_INIT_FAILED',
341	-1073741500: 'STATUS_UNHANDLED_EXCEPTION',
342	-1073741499: 'STATUS_APP_INIT_FAILURE',
343	-1073741819: 'STATUS_ACCESS_VIOLATION',
344	-1073741571: 'STATUS_STACK_OVERFLOW',
345	};
346	## @}
347
348	def __init__(self):
349	"""
350	Reinitialize child class instance.
351	"""
352	vbox.TestDriver.__init__(self);
353
354	# We need to set a default test VM set here -- otherwise the test
355	# driver base class won't let us use the "--test-vms" switch.
356	#
357	# See the "--local" switch in self.parseOption().
358	self.oTestVmSet = self.oTestVmManager.getSmokeVmSet('nat');
359
360	# Selected NIC attachment.
361	self.sNicAttachment = '';
362
363	# Session handling stuff.
364	# Only needed for remote tests executed by TxS.
365	self.oSession = None;
366	self.oTxsSession = None;
367
368	self.sVBoxInstallRoot = None;
369
370	# Testing mode being used:
371	# "local": Execute unit tests locally (same host, default).
372	# "remote-copy": Copies unit tests from host to the remote, then executing it.
373	# "remote-exec": Executes unit tests right on the remote from a given source.
374	self.sMode = 'local';
375
376	self.cSkipped = 0;
377	self.cPassed = 0;
378	self.cFailed = 0;
379
380	# The source directory where our unit tests live.
381	# This most likely is our out/ or some staging directory and
382	# also acts the source for copying over the testcases to a remote target.
383	self.sUnitTestsPathSrc = None;
384
385	# Array of environment variables with NAME=VAL entries
386	# to be applied for testcases.
387	#
388	# This is also needed for testcases which are being executed remotely.
389	self.asEnv = [];
390
391	# The destination directory our unit tests live when being
392	# copied over to a remote target (via TxS).
393	self.sUnitTestsPathDst = None;
394
395	# Will be determined before executing the actual unit tests.
396	self.sExeSuff = '';
397
398	self.aiVBoxVer = (4, 3, 0, 0);
399
400	# For testing testcase logic.
401	self.fDryRun = False;
402	self.fOnlyWhiteList = False;
403
404	@staticmethod
405	def _sanitizePath(sPath):
406	"""
407	Does a little bit of sanitizing a given path by removing quoting, if any.
408
409	This is needed because handed-in paths via command line arguments can contain variables like "${CDROM}"
410	which might need to get processed by TXS on the guest side first.
411
412	Returns the sanitized path.
413	"""
414	if sPath is None: # Keep uninitialized strings as-is.
415	return None;
416	return sPath.strip('\"').strip('\'');
417
418	def _detectPaths(self):
419	"""
420	Internal worker for actionVerify and actionExecute that detects paths.
421
422	This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
423	"""
424
425	reporter.log2('Detecting paths ...');
426
427	#
428	# Do some sanity checking first.
429	#
430	if self.sMode == 'remote-exec' \
431	and not self.sUnitTestsPathSrc: # There is no way we can figure this out automatically.
432	reporter.error('Unit tests source must be specified explicitly for selected mode!');
433	return False;
434
435	#
436	# We need a VBox install (/ build) to test.
437	#
438	if False is True: ## @todo r=andy WTF?
439	if not self.importVBoxApi():
440	reporter.error('Unabled to import the VBox Python API.');
441	return False;
442	else:
443	self._detectBuild();
444	if self.oBuild is None:
445	reporter.error('Unabled to detect the VBox build.');
446	return False;
447
448	#
449	# Where are the files installed?
450	# Solaris requires special handling because of it's multi arch subdirs.
451	#
452	if not self.sVBoxInstallRoot:
453	self.sVBoxInstallRoot = self.oBuild.sInstallPath;
454	if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
455	sArchDir = utils.getHostArch();
456	if sArchDir == 'x86': sArchDir = 'i386';
457	self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
458
459	## @todo r=andy Make sure the install root really exists and is accessible.
460
461	# Add the installation root to the PATH on windows so we can get DLLs from it.
462	if utils.getHostOs() == 'win':
463	sPathName = 'PATH';
464	if not sPathName in os.environ:
465	sPathName = 'Path';
466	sPath = os.environ.get(sPathName, '.');
467	if sPath and sPath[-1] != ';':
468	sPath += ';';
469	os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
470	else:
471	reporter.log2('VBox installation root already set to "%s"' % (self.sVBoxInstallRoot));
472
473	self.sVBoxInstallRoot = self._sanitizePath(self.sVBoxInstallRoot);
474
475	#
476	# The unittests are generally not installed, so look for them.
477	#
478	if not self.sUnitTestsPathSrc:
479	sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
480	asCandidates = [
481	self.oBuild.sInstallPath,
482	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), self.oBuild.sType, sBinOrDist),
483	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'release', sBinOrDist),
484	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug', sBinOrDist),
485	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict', sBinOrDist),
486	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt', sBinOrDist),
487	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'profile', sBinOrDist),
488	os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
489	os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
490	os.path.join(self.sScratchPath, sBinOrDist),
491	];
492	if utils.getHostOs() == 'darwin':
493	for i in range(1, len(asCandidates)):
494	asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
495
496	for sCandidat in asCandidates:
497	if os.path.exists(os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff)):
498	self.sUnitTestsPathSrc = sCandidat;
499	break;
500
501	if not self.sUnitTestsPathSrc:
502	reporter.error('Unable to find unit test source dir. Candidates: %s' % (asCandidates,));
503	return False;
504	else:
505	reporter.log2('Unit test source dir already set to "%s"' % (self.sUnitTestsPathSrc));
506
507	self.sUnitTestsPathSrc = self._sanitizePath(self.sUnitTestsPathSrc);
508
509	return True;
510
511	#
512	# Overridden methods.
513	#
514
515	def showUsage(self):
516	"""
517	Shows the testdriver usage.
518	"""
519	fRc = vbox.TestDriver.showUsage(self);
520	reporter.log('');
521	reporter.log('Unit Test #1 options:');
522	reporter.log(' --dryrun');
523	reporter.log(' Performs a dryrun (no tests being executed).');
524	reporter.log(' --mode <local\|remote-copy\|remote-exec>');
525	reporter.log(' Specifies the test execution mode:');
526	reporter.log(' local: Locally on the same machine.');
527	reporter.log(' remote-copy: On remote (guest) by copying them from the local source.');
528	reporter.log(' remote-exec: On remote (guest) directly (needs unit test source).');
529	reporter.log(' --only-whitelist');
530	reporter.log(' Only processes the white list.');
531	reporter.log(' --quick');
532	reporter.log(' Very selective testing.');
533	reporter.log(' --unittest-source <dir>');
534	reporter.log(' Sets the unit test source to <dir>.');
535	reporter.log(' Also used for remote execution.');
536	reporter.log(' --vbox-install-root <dir>');
537	reporter.log(' Sets the VBox install root to <dir>.');
538	reporter.log(' Also used for remote execution.');
539	return fRc;
540
541	def parseOption(self, asArgs, iArg):
542	"""
543	Parses the testdriver arguments from the command line.
544	"""
545	if asArgs[iArg] == '--dryrun':
546	self.fDryRun = True;
547	elif asArgs[iArg] == '--mode':
548	iArg += 1;
549	if iArg >= len(asArgs):
550	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
551	if asArgs[iArg] == 'local' \
552	or asArgs[iArg] == 'remote-copy' \
553	or asArgs[iArg] == 'remote-exec':
554	self.sMode = asArgs[iArg];
555	else:
556	raise base.InvalidOption('Argument "%s" invalid' % (asArgs[iArg]));
557	elif asArgs[iArg] == '--unittest-source':
558	iArg += 1;
559	if iArg >= len(asArgs):
560	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
561	self.sUnitTestsPathSrc = asArgs[iArg];
562	elif asArgs[iArg] == '--only-whitelist':
563	self.fOnlyWhiteList = True;
564	elif asArgs[iArg] == '--quick':
565	self.fOnlyWhiteList = True;
566	elif asArgs[iArg] == '--vbox-install-root':
567	iArg += 1;
568	if iArg >= len(asArgs):
569	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
570	self.sVBoxInstallRoot = asArgs[iArg];
571	else:
572	return vbox.TestDriver.parseOption(self, asArgs, iArg);
573	return iArg + 1;
574
575	def actionVerify(self):
576	if not self._detectPaths():
577	return False;
578
579	if self.oTestVmSet:
580	return vbox.TestDriver.actionVerify(self);
581
582	return True;
583
584	def actionConfig(self):
585	# Make sure vboxapi has been imported so we can use the constants.
586	if not self.importVBoxApi():
587	return False;
588
589	# Do the configuring.
590	if self.sMode.startswith('remote'):
591	if self.sNicAttachment == 'nat': eNic0AttachType = vboxcon.NetworkAttachmentType_NAT;
592	elif self.sNicAttachment == 'bridged': eNic0AttachType = vboxcon.NetworkAttachmentType_Bridged;
593	else: eNic0AttachType = None;
594
595	# Make sure to mount the Validation Kit .ISO so that TxS has the chance
596	# to update itself.
597	#
598	# This is necessary as a lot of our test VMs nowadays have a very old TxS
599	# installed which don't understand commands like uploading files to the guest.
600	# Uploading files is needed for this test driver, however.
601	#
602	## @todo Get rid of this as soon as we create test VMs in a descriptive (automated) manner.
603	return self.oTestVmSet.actionConfig(self, eNic0AttachType = eNic0AttachType, \
604	sDvdImage = self.sVBoxValidationKitIso);
605
606	return True;
607
608	def actionExecute(self):
609	# Make sure vboxapi has been imported so we can execute the driver without going thru
610	# a former configuring step.
611	if not self.importVBoxApi():
612	return False;
613	if not self._detectPaths():
614	return False;
615	reporter.log2('Unit test source path is "%s"\n' % self.sUnitTestsPathSrc);
616
617	if not self.sUnitTestsPathDst:
618	self.sUnitTestsPathDst = self.sScratchPath;
619	reporter.log2('Unit test destination path is "%s"\n' % self.sUnitTestsPathDst);
620
621	if self.sMode.startswith('remote'): # Run on a test VM (guest).
622	if self.fpApiVer < 7.0: ## @todo Needs Validation Kit .ISO tweaking (including the unit tests) first.
623	reporter.log('Remote unit tests for non-trunk builds skipped.');
624	fRc = True;
625	else:
626	assert self.oTestVmSet is not None;
627	fRc = self.oTestVmSet.actionExecute(self, self.testOneVmConfig);
628	else: # Run locally (host).
629	self._figureVersion();
630	self._makeEnvironmentChanges();
631	fRc = self._testRunUnitTests(None);
632
633	return fRc;
634
635	#
636	# Test execution helpers.
637	#
638
639	def _testRunUnitTests(self, oTestVm):
640	"""
641	Main function to execute all unit tests.
642	"""
643
644	# Determine executable suffix based on selected execution mode.
645	if self.sMode.startswith('remote'): # Run on a test VM (guest).
646	if oTestVm.isWindows():
647	self.sExeSuff = '.exe';
648	else:
649	self.sExeSuff = '';
650	else:
651	self.sExeSuff = '.exe' if utils.getHostOs() in [ 'win', 'dos', 'os2' ] else '';
652
653	self._testRunUnitTestsSet(oTestVm, r'^tst*', 'testcase');
654	self._testRunUnitTestsSet(oTestVm, r'^tst*', '.');
655
656	fRc = self.cFailed == 0;
657
658	reporter.log('');
659	if self.fDryRun:
660	reporter.log('*********************************************************');
661	reporter.log('DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN');
662	reporter.log('*********************************************************');
663	reporter.log('*********************************************************');
664	reporter.log(' Target: %s' % (oTestVm.sVmName if oTestVm else 'local'));
665	reporter.log(' Mode: %s' % (self.sMode));
666	reporter.log('Unit tests source: %s %s' % (self.sUnitTestsPathSrc, \
667	'(on remote)' if self.sMode == 'remote-exec' else ''));
668	reporter.log('VBox install root: %s %s' % (self.sVBoxInstallRoot, \
669	'(on remote)' if self.sMode == 'remote-exec' else ''));
670	reporter.log('*********************************************************');
671	reporter.log('*** PASSED: %d' % self.cPassed);
672	reporter.log('*** FAILED: %d' % self.cFailed);
673	reporter.log('*** SKIPPED: %d' % self.cSkipped);
674	reporter.log('*** TOTAL: %d' % (self.cPassed + self.cFailed + self.cSkipped));
675
676	return fRc;
677
678
679	def testOneVmConfig(self, oVM, oTestVm):
680	"""
681	Runs the specified VM thru test #1.
682	"""
683
684	# Simple test.
685	self.logVmInfo(oVM);
686
687	if not self.fDryRun:
688	# Try waiting for a bit longer (5 minutes) until the CD is available to avoid running into timeouts.
689	self.oSession, self.oTxsSession = self.startVmAndConnectToTxsViaTcp(oTestVm.sVmName, \
690	fCdWait = not self.fDryRun, \
691	cMsCdWait = 5 * 60 * 1000);
692	if self.oSession is None:
693	return False;
694
695	self.addTask(self.oTxsSession);
696
697	# Determine the unit tests destination path.
698	self.sUnitTestsPathDst = oTestVm.pathJoin(self.getGuestTempDir(oTestVm), 'testUnitTests');
699
700	# Run the unit tests.
701	self._testRunUnitTests(oTestVm);
702
703	# Cleanup.
704	if self.oSession is not None:
705	self.removeTask(self.oTxsSession);
706	self.terminateVmBySession(self.oSession);
707	return True;
708
709	#
710	# Test execution helpers.
711	#
712
713	def _figureVersion(self):
714	""" Tries to figure which VBox version this is, setting self.aiVBoxVer. """
715	try:
716	sVer = utils.processOutputChecked(['VBoxManage', '--version'])
717
718	sVer = sVer.strip();
719	sVer = re.sub(r'_BETA.*r', '.', sVer);
720	sVer = re.sub(r'_ALPHA.*r', '.', sVer);
721	sVer = re.sub(r'_RC.*r', '.', sVer);
722	sVer = re.sub('_SPB', '', sVer)
723	sVer = sVer.replace('r', '.');
724
725	self.aiVBoxVer = [int(sComp) for sComp in sVer.split('.')];
726
727	reporter.log('VBox version: %s' % (self.aiVBoxVer,));
728	except:
729	reporter.logXcpt();
730	return False;
731	return True;
732
733	def _compareVersion(self, aiVer):
734	"""
735	Compares the give version string with the vbox version string,
736	returning a result similar to C strcmp(). aiVer is on the right side.
737	"""
738	cComponents = min(len(self.aiVBoxVer), len(aiVer));
739	for i in range(cComponents):
740	if self.aiVBoxVer[i] < aiVer[i]:
741	return -1;
742	if self.aiVBoxVer[i] > aiVer[i]:
743	return 1;
744	return len(self.aiVBoxVer) - len(aiVer);
745
746	def _isExcluded(self, sTest, dExclList):
747	""" Checks if the testcase is excluded or not. """
748	if sTest in dExclList:
749	sFullExpr = dExclList[sTest].replace(' ', '').strip();
750	if sFullExpr == '':
751	return True;
752
753	# Consider each exclusion expression. These are generally ranges,
754	# either open ended or closed: "<4.3.51r12345", ">=4.3.0 && <=4.3.4".
755	asExprs = sFullExpr.split(';');
756	for sExpr in asExprs:
757
758	# Split it on the and operator and process each sub expression.
759	fResult = True;
760	for sSubExpr in sExpr.split('&&'):
761	# Split out the comparison operator and the version value.
762	if sSubExpr.startswith('<=') or sSubExpr.startswith('>='):
763	sOp = sSubExpr[:2];
764	sValue = sSubExpr[2:];
765	elif sSubExpr.startswith('<') or sSubExpr.startswith('>') or sSubExpr.startswith('='):
766	sOp = sSubExpr[:1];
767	sValue = sSubExpr[1:];
768	else:
769	sOp = sValue = '';
770
771	# Convert the version value, making sure we've got a valid one.
772	try: aiValue = [int(sComp) for sComp in sValue.replace('r', '.').split('.')];
773	except: aiValue = ();
774	if not aiValue or len(aiValue) > 4:
775	reporter.error('Invalid exclusion expression for %s: "%s" [%s]' % (sTest, sSubExpr, dExclList[sTest]));
776	return True;
777
778	# Do the compare.
779	iCmp = self._compareVersion(aiValue);
780	if sOp == '>=' and iCmp < 0:
781	fResult = False;
782	elif sOp == '>' and iCmp <= 0:
783	fResult = False;
784	elif sOp == '<' and iCmp >= 0:
785	fResult = False;
786	elif sOp == '>=' and iCmp < 0:
787	fResult = False;
788	reporter.log2('iCmp=%s; %s %s %s -> %s' % (iCmp, self.aiVBoxVer, sOp, aiValue, fResult));
789
790	# Did the expression match?
791	if fResult:
792	return True;
793
794	return False;
795
796	def _sudoExecuteSync(self, asArgs):
797	"""
798	Executes a sudo child process synchronously.
799	Returns True if the process executed successfully and returned 0,
800	otherwise False is returned.
801	"""
802	reporter.log2('Executing [sudo]: %s' % (asArgs, ));
803	if self.sMode.startswith('remote'):
804	iRc = -1; ## @todo Not used remotely yet.
805	else:
806	try:
807	iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False);
808	except:
809	reporter.errorXcpt();
810	return False;
811	reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs));
812	return iRc == 0;
813
814
815	def _logExpandString(self, sString, cVerbosity = 2):
816	"""
817	Expands a given string by asking TxS on the guest side and logs it.
818	Uses log level 2 by default.
819
820	No-op if no TxS involved.
821	"""
822	if reporter.getVerbosity() < cVerbosity \
823	or self.oTxsSession is None:
824	return;
825	sStringExp = self.oTxsSession.syncExpandString(sString);
826	if not sStringExp:
827	return;
828	reporter.log2('_logExpandString: "%s" -> "%s"' % (sString, sStringExp));
829
830	def _wrapPathExists(self, sPath):
831	"""
832	Creates the directory specified sPath (including parents).
833	"""
834	reporter.log2('_wrapPathExists: %s' % (sPath,));
835	if self.fDryRun:
836	return True;
837	fRc = False;
838	if self.sMode.startswith('remote'):
839	self._logExpandString(sPath);
840	fRc = self.oTxsSession.syncIsDir(sPath, fIgnoreErrors = True);
841	if not fRc:
842	fRc = self.oTxsSession.syncIsFile(sPath, fIgnoreErrors = True);
843	else:
844	fRc = os.path.exists(sPath);
845	return fRc;
846
847	def _wrapMkDir(self, sPath):
848	"""
849	Creates the directory specified sPath (including parents).
850	"""
851	reporter.log2('_wrapMkDir: %s' % (sPath,));
852	if self.fDryRun:
853	return True;
854	fRc = True;
855	if self.sMode.startswith('remote'):
856	fRc = self.oTxsSession.syncMkDirPath(sPath, fMode = 0o755);
857	else:
858	if utils.getHostOs() in [ 'win', 'os2' ]:
859	os.makedirs(sPath, 0o755);
860	else:
861	fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
862	if not fRc:
863	reporter.log('Failed to create dir "%s".' % (sPath,));
864	return fRc;
865
866	def _wrapCopyFile(self, sSrc, sDst, iMode):
867	"""
868	Copies a file.
869	"""
870	reporter.log2('_wrapCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
871	if self.fDryRun:
872	return True;
873	fRc = True;
874	if self.sMode.startswith('remote'):
875	self._logExpandString(sSrc);
876	self._logExpandString(sDst);
877	if self.sMode == 'remote-exec':
878	self.oTxsSession.syncCopyFile(sSrc, sDst, iMode);
879	else:
880	fRc = self.oTxsSession.syncUploadFile(sSrc, sDst);
881	if fRc:
882	fRc = self.oTxsSession.syncChMod(sDst, iMode);
883	else:
884	if utils.getHostOs() in [ 'win', 'os2' ]:
885	utils.copyFileSimple(sSrc, sDst);
886	os.chmod(sDst, iMode);
887	else:
888	fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
889	if fRc:
890	fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
891	if fRc is not True:
892	raise Exception('Failed to chmod "%s".' % (sDst,));
893	if not fRc:
894	reporter.log('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
895	return fRc;
896
897	def _wrapDeleteFile(self, sPath):
898	"""
899	Deletes a file.
900	"""
901	reporter.log2('_wrapDeleteFile: %s' % (sPath,));
902	if self.fDryRun:
903	return True;
904	fRc = True;
905	if self.sMode.startswith('remote'):
906	if self.oTxsSession.syncIsFile(sPath):
907	fRc = self.oTxsSession.syncRmFile(sPath, fIgnoreErrors = True);
908	else:
909	if os.path.exists(sPath):
910	if utils.getHostOs() in [ 'win', 'os2' ]:
911	os.remove(sPath);
912	else:
913	fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
914	if not fRc:
915	reporter.log('Failed to remove "%s".' % (sPath,));
916	return fRc;
917
918	def _wrapRemoveDir(self, sPath):
919	"""
920	Removes a directory.
921	"""
922	reporter.log2('_wrapRemoveDir: %s' % (sPath,));
923	if self.fDryRun:
924	return True;
925	fRc = True;
926	if self.sMode.startswith('remote'):
927	if self.oTxsSession.syncIsDir(sPath):
928	fRc = self.oTxsSession.syncRmDir(sPath, fIgnoreErrors = True);
929	else:
930	if os.path.exists(sPath):
931	if utils.getHostOs() in [ 'win', 'os2' ]:
932	os.rmdir(sPath);
933	else:
934	fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
935	if not fRc:
936	reporter.log('Failed to remove "%s".' % (sPath,));
937	return fRc;
938
939	def _envSet(self, sName, sValue):
940	if self.sMode.startswith('remote'):
941	# For remote execution we cache the environment block and pass it
942	# right when the process execution happens.
943	self.asEnv.append([ sName, sValue ]);
944	else:
945	os.environ[sName] = sValue;
946	return True;
947
948	def _executeTestCase(self, oTestVm, sName, sFullPath, sTestCaseSubDir, oDevNull): # pylint: disable=too-many-locals,too-many-statements
949	"""
950	Executes a test case.
951	"""
952
953	fSkipped = False;
954
955	#
956	# If hardening is enabled, some test cases and their dependencies
957	# needs to be copied to and execute from the source
958	# directory in order to work. They also have to be executed as
959	# root, i.e. via sudo.
960	#
961	fHardened = False;
962	fCopyToRemote = False;
963	fCopyDeps = False;
964	asFilesToRemove = []; # Stuff to clean up.
965	asDirsToRemove = []; # Ditto.
966
967	if sName in self.kasHardened \
968	and self.sUnitTestsPathSrc != self.sVBoxInstallRoot:
969	fHardened = True;
970
971	if self.sMode.startswith('remote'):
972	fCopyToRemote = True;
973	fCopyDeps = True;
974
975	if fHardened \
976	or fCopyToRemote:
977	if fCopyToRemote:
978	sDstDir = os.path.join(self.sUnitTestsPathDst, sTestCaseSubDir);
979	else:
980	sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
981	if not self._wrapPathExists(sDstDir):
982	self._wrapMkDir(sDstDir);
983	asDirsToRemove.append(sDstDir);
984
985	sSrc = sFullPath + self.sExeSuff;
986	# If the testcase source does not exist for whatever reason, just mark it as skipped
987	# instead of reporting an error.
988	if not self._wrapPathExists(sSrc):
989	self.cSkipped += 1;
990	fSkipped = True;
991	return fSkipped;
992
993	sDst = os.path.join(sDstDir, os.path.basename(sFullPath) + self.sExeSuff);
994	fModeExe = 0;
995	fModeDeps = 0;
996	if not oTestVm \
997	or (oTestVm and not oTestVm.isWindows()): ## @todo NT4 does not like the chmod. Investigate this!
998	fModeExe = 0o755;
999	fModeDeps = 0o644;
1000	self._wrapCopyFile(sSrc, sDst, fModeExe);
1001	asFilesToRemove.append(sDst);
1002
1003	# Copy required dependencies to destination.
1004	if fCopyDeps:
1005	for sLib in self.kdTestCaseDepsLibs:
1006	for sSuff in [ '.dll', '.so', '.dylib' ]:
1007	assert self.sVBoxInstallRoot is not None;
1008	sSrc = os.path.join(self.sVBoxInstallRoot, sLib + sSuff);
1009	if self._wrapPathExists(sSrc):
1010	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1011	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1012	asFilesToRemove.append(sDst);
1013
1014	# Copy any associated .dll/.so/.dylib.
1015	for sSuff in [ '.dll', '.so', '.dylib' ]:
1016	sSrc = os.path.splitext(sFullPath)[0] + sSuff;
1017	if os.path.exists(sSrc):
1018	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1019	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1020	asFilesToRemove.append(sDst);
1021
1022	# Copy any associated .r0, .rc and .gc modules.
1023	offDriver = sFullPath.rfind('Driver')
1024	if offDriver > 0:
1025	for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
1026	sSrc = sFullPath[:offDriver] + sSuff;
1027	if os.path.exists(sSrc):
1028	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1029	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1030	asFilesToRemove.append(sDst);
1031
1032	sFullPath = os.path.join(sDstDir, os.path.basename(sFullPath));
1033
1034	#
1035	# Set up arguments and environment.
1036	#
1037	asArgs = [sFullPath + self.sExeSuff,]
1038	if sName in self.kdArguments:
1039	asArgs.extend(self.kdArguments[sName]);
1040
1041	sXmlFile = os.path.join(self.sUnitTestsPathDst, 'result.xml');
1042
1043	self._envSet('IPRT_TEST_OMIT_TOP_TEST', '1');
1044	self._envSet('IPRT_TEST_FILE', sXmlFile);
1045
1046	if self._wrapPathExists(sXmlFile):
1047	try: os.unlink(sXmlFile);
1048	except: self._wrapDeleteFile(sXmlFile);
1049
1050	#
1051	# Execute the test case.
1052	#
1053	# Windows is confusing output. Trying a few things to get rid of this.
1054	# First, flush both stderr and stdout before running the child. Second,
1055	# assign the child stderr to stdout. If this doesn't help, we'll have
1056	# to capture the child output.
1057	#
1058	reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
1059	try: sys.stdout.flush();
1060	except: pass;
1061	try: sys.stderr.flush();
1062	except: pass;
1063
1064	iRc = 0;
1065
1066	if not self.fDryRun:
1067	if fCopyToRemote:
1068	fRc = self.txsRunTest(self.oTxsSession, sName, cMsTimeout = 30 * 60 * 1000, sExecName = asArgs[0], \
1069	asArgs = asArgs, asAddEnv = self.asEnv, fCheckSessionStatus = True);
1070	if fRc:
1071	iRc = 0;
1072	else:
1073	(_, sOpcode, abPayload) = self.oTxsSession.getLastReply();
1074	if sOpcode.startswith('PROC NOK '): # Extract process rc.
1075	iRc = abPayload[0]; # ASSUMES 8-bit rc for now.
1076	if iRc == 0: # Might happen if the testcase misses some dependencies. Set it to -42 then.
1077	iRc = -42;
1078	else:
1079	iRc = -1; ## @todo
1080	else:
1081	oChild = None;
1082	try:
1083	if fHardened:
1084	oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1085	else:
1086	oChild = utils.processPopenSafe(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1087	except:
1088	if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
1089	]:
1090	reporter.logXcpt();
1091	fSkipped = True;
1092	else:
1093	reporter.errorXcpt();
1094	iRc = 1023;
1095	oChild = None;
1096
1097	if oChild is not None:
1098	self.pidFileAdd(oChild.pid, sName, fSudo = fHardened);
1099	iRc = oChild.wait();
1100	self.pidFileRemove(oChild.pid);
1101	#
1102	# Clean up
1103	#
1104	for sPath in asFilesToRemove:
1105	self._wrapDeleteFile(sPath);
1106	for sPath in asDirsToRemove:
1107	self._wrapRemoveDir(sPath);
1108
1109	#
1110	# Report.
1111	#
1112	if os.path.exists(sXmlFile):
1113	reporter.addSubXmlFile(sXmlFile);
1114	if fHardened:
1115	self._wrapDeleteFile(sXmlFile);
1116	else:
1117	os.unlink(sXmlFile);
1118
1119	if iRc == 0:
1120	reporter.log('*** %s: exit code %d' % (sFullPath, iRc));
1121	self.cPassed += 1;
1122
1123	elif iRc == 4: # RTEXITCODE_SKIPPED
1124	reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFullPath, iRc));
1125	fSkipped = True;
1126	self.cSkipped += 1;
1127
1128	elif fSkipped:
1129	reporter.log('*** %s: exit code %d (Skipped)' % (sFullPath, iRc));
1130	self.cSkipped += 1;
1131
1132	else:
1133	sName = self.kdExitCodeNames.get(iRc, '');
1134	if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
1135	sName = self.kdExitCodeNamesWin[iRc];
1136	if sName != '':
1137	sName = ' (%s)' % (sName);
1138
1139	if iRc != 1:
1140	reporter.testFailure('Exit status: %d%s' % (iRc, sName));
1141	reporter.log( '!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
1142	else:
1143	reporter.error('!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
1144	self.cFailed += 1;
1145
1146	return fSkipped;
1147
1148	def _testRunUnitTestsSet(self, oTestVm, sTestCasePattern, sTestCaseSubDir):
1149	"""
1150	Run subset of the unit tests set.
1151	"""
1152
1153	# Open /dev/null for use as stdin further down.
1154	try:
1155	oDevNull = open(os.path.devnull, 'w+'); # pylint: disable=consider-using-with
1156	except:
1157	oDevNull = None;
1158
1159	# Determin the host OS specific exclusion lists.
1160	dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
1161	dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
1162
1163	## @todo Add filtering for more specific OSes (like OL server, doesn't have X installed) by adding a separate
1164	# black list + using utils.getHostOsVersion().
1165
1166	#
1167	# Process the file list and run everything looking like a testcase.
1168	#
1169	if not self.fOnlyWhiteList:
1170	if self.sMode == 'local' \
1171	or self.sMode == 'remote-copy':
1172	asFiles = sorted(os.listdir(os.path.join(self.sUnitTestsPathSrc, sTestCaseSubDir)));
1173	else: # 'remote-exec'
1174	## @todo Implement remote file enumeration / directory listing.
1175	reporter.error('Sorry, no remote file enumeration implemented yet!\nUse --only-whitelist instead.');
1176	return;
1177	else:
1178	# Transform our dict into a list, where the keys are the list elements.
1179	asFiles = list(self.kdTestCasesWhiteList.keys());
1180	# Make sure to only keep the list item's base name so that the iteration down below works
1181	# with our white list without any additional modification.
1182	asFiles = [os.path.basename(s) for s in asFiles];
1183
1184	for sFilename in asFiles:
1185	# Separate base and suffix and morph the base into something we
1186	# can use for reporting and array lookups.
1187	sBaseName = os.path.basename(sFilename);
1188	sName, sSuffix = os.path.splitext(sBaseName);
1189	if sTestCaseSubDir != '.':
1190	sName = sTestCaseSubDir + '/' + sName;
1191
1192	reporter.log2('sTestCasePattern=%s, sBaseName=%s, sName=%s, sSuffix=%s, sFileName=%s' \
1193	% (sTestCasePattern, sBaseName, sName, sSuffix, sFilename,));
1194
1195	# Process white list first, if set.
1196	if self.fOnlyWhiteList \
1197	and not self._isExcluded(sName, self.kdTestCasesWhiteList):
1198	# (No testStart/Done or accounting here!)
1199	reporter.log('%s: SKIPPED (not in white list)' % (sName,));
1200	continue;
1201
1202	# Basic exclusion.
1203	if not re.match(sTestCasePattern, sBaseName) \
1204	or sSuffix in self.kasSuffixBlackList:
1205	reporter.log2('"%s" is not a test case.' % (sName,));
1206	continue;
1207
1208	# When not only processing the white list, do some more checking first.
1209	if not self.fOnlyWhiteList:
1210	# Check if the testcase is black listed or buggy before executing it.
1211	if self._isExcluded(sName, self.kdTestCasesBlackList):
1212	# (No testStart/Done or accounting here!)
1213	reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
1214	continue;
1215
1216	elif self._isExcluded(sName, self.kdTestCasesBuggy):
1217	reporter.testStart(sName);
1218	reporter.log('%s: Skipping, buggy in general.' % (sName,));
1219	reporter.testDone(fSkipped = True);
1220	self.cSkipped += 1;
1221	continue;
1222
1223	elif self._isExcluded(sName, dTestCasesBuggyForHostOs):
1224	reporter.testStart(sName);
1225	reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
1226	reporter.testDone(fSkipped = True);
1227	self.cSkipped += 1;
1228	continue;
1229	else:
1230	# Passed the white list check already above.
1231	pass;
1232
1233	sFullPath = os.path.normpath(os.path.join(self.sUnitTestsPathSrc, os.path.join(sTestCaseSubDir, sFilename)));
1234	reporter.testStart(sName);
1235	try:
1236	fSkipped = self._executeTestCase(oTestVm, sName, sFullPath, sTestCaseSubDir, oDevNull);
1237	except:
1238	reporter.errorXcpt('!*!');
1239	self.cFailed += 1;
1240	fSkipped = False;
1241	reporter.testDone(fSkipped);
1242
1243
1244	if __name__ == '__main__':
1245	sys.exit(tdUnitTest1().main(sys.argv))

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/tests/unittests/tdUnitTest1.py@ 97288

Download in other formats: