tdUnitTest1.py@ 97488

Last change on this file since 97488 was 97488, checked in by vboxsync, 2 years ago
ValKit/tdUnitTest1.py: Added a kludge for making tstAsmStructs work with in an asan build.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 53.8 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: tdUnitTest1.py 97488 2022-11-09 21:00:05Z vboxsync $
4
5	"""
6	VirtualBox Validation Kit - Unit Tests.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2022 Oracle and/or its affiliates.
12
13	This file is part of VirtualBox base platform packages, as
14	available from https://www.virtualbox.org.
15
16	This program is free software; you can redistribute it and/or
17	modify it under the terms of the GNU General Public License
18	as published by the Free Software Foundation, in version 3 of the
19	License.
20
21	This program is distributed in the hope that it will be useful, but
22	WITHOUT ANY WARRANTY; without even the implied warranty of
23	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	General Public License for more details.
25
26	You should have received a copy of the GNU General Public License
27	along with this program; if not, see <https://www.gnu.org/licenses>.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32	in the VirtualBox distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37
38	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39	"""
40	__version__ = "$Revision: 97488 $"
41
42
43	# Standard Python imports.
44	import os
45	import sys
46	import re
47
48
49	# Only the main script needs to modify the path.
50	try: __file__
51	except: __file__ = sys.argv[0];
52	g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
53	sys.path.append(g_ksValidationKitDir)
54
55	# Validation Kit imports.
56	from common import utils;
57	from testdriver import base;
58	from testdriver import reporter;
59	from testdriver import vbox;
60	from testdriver import vboxcon;
61
62
63	class tdUnitTest1(vbox.TestDriver):
64	"""
65	Unit Tests.
66	"""
67
68	## The temporary exclude list.
69	## @note This shall be empty before we release 4.3!
70	kdTestCasesBuggyPerOs = {
71	'darwin': {
72	'testcase/tstX86-1': '', # 'FSTP M32R, ST0' fails; no idea why.
73	},
74	'linux': {
75	'testcase/tstRTFileAio': '', # See xTracker #8035.
76	},
77	'linux.amd64': {
78	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
79	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
80	},
81	'solaris': {
82	'testcase/tstIntNet-1': '', # Fails opening rge0, probably a generic issue figuring which nic to use.
83	'testcase/tstIprtList': '', # Crashes in the multithreaded test, I think.
84	'testcase/tstRTCritSect': '', # Fairness/whatever issue here.
85	'testcase/tstRTR0MemUserKernelDriver': '', # Failes when kernel to kernel buffers.
86	'testcase/tstRTSemRW': '', # line 338: RTSemRWReleaseRead(hSemRW): got VERR_ACCESS_DENIED
87	'testcase/tstRTStrAlloc': '', # VERR_NO_STR_MEMORY!
88	'testcase/tstRTFileQuerySize-1': '', # VERR_DEV_IO_ERROR on /dev/null!
89	},
90	'solaris.amd64': {
91	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
92	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
93	},
94	'win': {
95	'testcase/tstFile': '', # ??
96	'testcase/tstIntNet-1': '', # possibly same issue as solaris.
97	'testcase/tstMouseImpl': '', # STATUS_ACCESS_VIOLATION
98	'testcase/tstRTR0ThreadPreemptionDriver': '', # ??
99	'testcase/tstRTPath': '<4.3.51r89894',
100	'testcase/tstRTPipe': '', # ??
101	'testcase/tstRTR0MemUserKernelDriver': '', # ??
102	'testcase/tstRTR0SemMutexDriver': '', # ??
103	'testcase/tstRTStrAlloc': '', # ??
104	'testcase/tstRTStrFormat': '', # ??
105	'testcase/tstRTSystemQueryOsInfo': '', # ??
106	'testcase/tstRTTemp': '', # ??
107	'testcase/tstRTTime': '', # ??
108	'testcase/tstTime-2': '', # Total time differs too much! ... delta=-10859859
109	'testcase/tstTime-4': '', # Needs to be converted to DLL; ditto for tstTime-2.
110	'testcase/tstUtf8': '', # ??
111	'testcase/tstVMMR0CallHost-2': '', # STATUS_STACK_OVERFLOW
112	'testcase/tstX86-1': '', # Fails on win.x86.
113	'tscpasswd': '', # ??
114	'tstVMREQ': '', # ?? Same as darwin.x86?
115	},
116	'win.x86': {
117	'testcase/tstRTR0TimerDriver': '', # See xTracker #8041.
118	}
119	};
120
121	kdTestCasesBuggy = {
122	'testcase/tstGuestPropSvc': '', # GET_NOTIFICATION fails on testboxlin5.de.oracle.com and others.
123	'testcase/tstRTProcCreateEx': '', # Seen failing on wei01-b6ka-9.de.oracle.com.
124	'testcase/tstTimer': '', # Sometimes fails on linux, not important atm.
125	'testcase/tstGIP-2': '', # 2015-09-10: Fails regularly. E.g. TestSetID 2744205 (testboxsh2),
126	# 2743961 (wei01-b6kc-6). The responsible engineer should reenable
127	# it once it has been fixed.
128	};
129
130	## The permanent exclude list.
131	# @note Stripped of extensions!
132	kdTestCasesBlackList = {
133	'testcase/tstClipboardX11Smoke': '', # (Old naming, deprecated) Needs X, not available on all test boxes.
134	'testcase/tstClipboardGH-X11Smoke': '', # (New name) Ditto.
135	'testcase/tstClipboardMockHGCM': '', # Ditto.
136	'tstClipboardQt': '', # Is interactive and needs Qt, needed for Qt clipboard bugfixing.
137	'testcase/tstClipboardQt': '', # In case it moves here.
138	'testcase/tstFileLock': '',
139	'testcase/tstDisasm-2': '', # without parameters it will disassembler 1GB starting from 0
140	'testcase/tstFileAppendWin-1': '',
141	'testcase/tstDir': '', # useless without parameters
142	'testcase/tstDir-2': '', # useless without parameters
143	'testcase/tstGlobalConfig': '',
144	'testcase/tstHostHardwareLinux': '', # must be killed with CTRL-C
145	'testcase/tstHttp': '', # Talks to outside servers.
146	'testcase/tstRTHttp': '', # parameters required
147	'testcase/tstLdr-2': '', # parameters required
148	'testcase/tstLdr-3': '', # parameters required
149	'testcase/tstLdr': '', # parameters required
150	'testcase/tstLdrLoad': '', # parameters required
151	'testcase/tstMove': '', # parameters required
152	'testcase/tstRTR0Timer': '', # loads 'tstRTR0Timer.r0'
153	'testcase/tstRTR0ThreadDriver': '', # loads 'tstRTR0Thread.r0'
154	'testcase/tstRunTestcases': '', # that's a script like this one
155	'testcase/tstRTReqPool': '', # fails sometimes, testcase buggy
156	'testcase/tstRTS3': '', # parameters required
157	'testcase/tstSDL': '', # graphics test
158	'testcase/tstSupLoadModule': '', # Needs parameters and vboxdrv access. Covered elsewhere.
159	'testcase/tstSeamlessX11': '', # graphics test
160	'testcase/tstTime-3': '', # parameters required
161	'testcase/tstVBoxControl': '', # works only inside a guest
162	'testcase/tstVDCopy': '', # parameters required
163	'testcase/tstVDFill': '', # parameters required
164	'tstAnimate': '', # parameters required
165	'testcase/tstAPI': '', # user interaction required
166	'tstCollector': '', # takes forever
167	'testcase/tstHeadless': '', # parameters required
168	'tstHeadless': '', # parameters required
169	'tstMicroRC': '', # required by tstMicro
170	'tstVBoxDbg': '', # interactive test
171	'testcase/tstTestServMgr': '', # some strange xpcom18a4 test, does not work
172	'tstTestServMgr': '', # some strange xpcom18a4 test, does not work
173	'tstPDMAsyncCompletion': '', # parameters required
174	'testcase/tstXptDump': '', # parameters required
175	'tstXptDump': '', # parameters required
176	'testcase/tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
177	'tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
178	'testcase/tstSimpleTypeLib': '', # parameters required
179	'tstSimpleTypeLib': '', # parameters required
180	'testcase/tstTestAtoms': '', # additional test file (words.txt) required
181	'tstTestAtoms': '', # additional test file (words.txt) required
182	'testcase/tstXptLink': '', # parameters required
183	'tstXptLink': '', # parameters required
184	'tstXPCOMCGlue': '', # user interaction required
185	'testcase/tstXPCOMCGlue': '', # user interaction required
186	'testcase/tstCAPIGlue': '', # user interaction required
187	'testcase/tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
188	'tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
189	'testcase/tstRTFilesystem': '', # parameters required
190	'testcase/tstRTDvm': '', # parameters required
191	'tstSSLCertDownloads': '', # Obsolete.
192	# later
193	'testcase/tstIntNetR0': '', # RTSPINLOCK_FLAGS_INTERRUPT_SAFE == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE
194	# slow stuff
195	'testcase/tstAvl': '', # SLOW!
196	'testcase/tstRTAvl': '', # SLOW! (new name)
197	'testcase/tstVD': '', # 8GB fixed-sized vmdk
198	# failed or hang
199	'testcase/tstCryptoPkcs7Verify': '', # hang
200	'tstOVF': '', # hang (only ancient version, now in new place)
201	'testcase/tstRTLockValidator': '', # Lock validation is not enabled for critical sections
202	'testcase/tstGuestControlSvc': '', # failed: line 288: testHost(&svcTable):
203	# expected VINF_SUCCESS, got VERR_NOT_FOUND
204	'testcase/tstRTMemEf': '', # failed w/o error message
205	'testcase/tstSupSem': '', # failed: SRE Timeout Accuracy (ms) : FAILED (1 errors)
206	'testcase/tstCryptoPkcs7Sign': '', # failed: 29330:
207	# error:02001002:lib(2):func(1):reason(2):NA:0:fopen('server.pem': '','r')
208	'testcase/tstCompressionBenchmark': '', # failed: error: RTZipBlockCompress failed
209	# for 'RTZipBlock/LZJB' (#4): VERR_NOT_SUPPORTED
210	'tstPDMAsyncCompletionStress': '', # VERR_INVALID_PARAMETER (cbSize = 0)
211	'tstMicro': '', # doesn't work on solaris, fix later if we care.
212	'tstVMM-HwAccm': '', # failed: Only checked AMD-V on linux
213	'tstVMM-HM': '', # failed: Only checked AMD-V on linux
214	'tstVMMFork': '', # failed: xtracker 6171
215	'tstTestFactory': '', # some strange xpcom18a4 test, does not work
216	'testcase/tstRTSemXRoads': '', # sporadically failed: Traffic - 8 threads per direction, 10 sec :
217	# FAILED (8 errors)
218	'tstVBoxAPILinux': '', # creates VirtualBox directories for root user because of sudo
219	# (should be in vbox)
220	'testcase/tstVMStructDTrace': '', # This is a D-script generator.
221	'tstVMStructRC': '', # This is a C-code generator.
222	'tstDeviceStructSizeRC': '', # This is a C-code generator.
223	'testcase/tstTSC': '', # Doesn't test anything and might fail with HT or/and too many cores.
224	'testcase/tstOpenUSBDev': '', # Not a useful testcase.
225	'testcase/tstX86-1': '', # Really more guest side.
226	'testcase/tstX86-FpuSaveRestore': '', # Experiments, could be useful for the guest not the host.
227	'tstAsmStructsRC': '', # Testcase run during build time (fails to find libstdc++.so.6 on some
228	# Solaris testboxes).
229	};
230
231	# Suffix exclude list.
232	kasSuffixBlackList = [
233	'.r0',
234	'.gc',
235	'.debug',
236	'.rel',
237	'.sys',
238	'.ko',
239	'.o',
240	'.obj',
241	'.lib',
242	'.a',
243	'.so',
244	'.dll',
245	'.dylib',
246	'.tmp',
247	'.log',
248	'.py',
249	'.pyc',
250	'.pyo',
251	'.pdb',
252	'.dSYM',
253	'.sym',
254	'.template',
255	'.expected',
256	'.expect',
257	];
258
259	# White list, which contains tests considered to be safe to execute,
260	# even on remote targets (guests).
261	#
262	# When --only-whitelist is specified, this is the only list being checked for.
263	kdTestCasesWhiteList = {
264	'testcase/tstFile': '',
265	'testcase/tstFileLock': '',
266	'testcase/tstClipboardMockHGCM': '', # Requires X on Linux OSes. Execute on remote targets only (guests).
267	'testcase/tstRTLocalIpc': '',
268	'testcase/tstRTPathQueryInfo': '',
269	'testcase/tstRTPipe': '',
270	'testcase/tstRTProcCreateEx': '',
271	'testcase/tstRTProcCreatePrf': '',
272	'testcase/tstRTProcIsRunningByName': '',
273	'testcase/tstRTProcQueryUsername': '',
274	'testcase/tstRTProcWait': '',
275	'testcase/tstTime-2': '',
276	'testcase/tstTime-3': '',
277	'testcase/tstTime-4': '',
278	'testcase/tstTimer': '',
279	'testcase/tstThread-1': '',
280	'testcase/tstUtf8': ''
281	};
282
283	# Test dependency list -- libraries.
284	# Needed in order to execute testcases on remote targets which don't have a VBox installation present.
285	kdTestCaseDepsLibs = [
286	"VBoxRT"
287	];
288
289	## The exclude list.
290	# @note Stripped extensions!
291	kasHardened = [
292	"testcase/tstIntNet-1",
293	"testcase/tstR0ThreadPreemptionDriver", # VBox 4.3
294	"testcase/tstRTR0ThreadPreemptionDriver",
295	"testcase/tstRTR0MemUserKernelDriver",
296	"testcase/tstRTR0SemMutexDriver",
297	"testcase/tstRTR0TimerDriver",
298	"testcase/tstRTR0ThreadDriver",
299	'testcase/tstRTR0DbgKrnlInfoDriver',
300	"tstInt",
301	"tstPDMQueue", # Comment in testcase says its driverless, but it needs driver access.
302	"tstVMM",
303	"tstVMMFork",
304	"tstVMREQ",
305	'testcase/tstCFGM',
306	'testcase/tstContiguous',
307	'testcase/tstGetPagingMode',
308	'testcase/tstGIP-2',
309	'testcase/tstInit',
310	'testcase/tstLow',
311	'testcase/tstMMHyperHeap',
312	'testcase/tstPage',
313	'testcase/tstPin',
314	'testcase/tstRTTime', 'testcase/tstTime', # GIP test case.
315	'testcase/tstRTTime-2', 'testcase/tstTime-2', # GIP test case.
316	'testcase/tstRTTime-4', 'testcase/tstTime-4', # GIP test case.
317	'testcase/tstSSM',
318	'testcase/tstSupSem-Zombie',
319	]
320
321	## Argument lists
322	kdArguments = {
323	'testcase/tstbntest': [ '-out', os.devnull, ], # Very noisy.
324	};
325
326
327	## Status code translations.
328	## @{
329	kdExitCodeNames = {
330	0: 'RTEXITCODE_SUCCESS',
331	1: 'RTEXITCODE_FAILURE',
332	2: 'RTEXITCODE_SYNTAX',
333	3: 'RTEXITCODE_INIT',
334	4: 'RTEXITCODE_SKIPPED',
335	};
336	kdExitCodeNamesWin = {
337	-1073741515: 'STATUS_DLL_NOT_FOUND',
338	-1073741512: 'STATUS_ORDINAL_NOT_FOUND',
339	-1073741511: 'STATUS_ENTRYPOINT_NOT_FOUND',
340	-1073741502: 'STATUS_DLL_INIT_FAILED',
341	-1073741500: 'STATUS_UNHANDLED_EXCEPTION',
342	-1073741499: 'STATUS_APP_INIT_FAILURE',
343	-1073741819: 'STATUS_ACCESS_VIOLATION',
344	-1073741571: 'STATUS_STACK_OVERFLOW',
345	};
346	## @}
347
348	def __init__(self):
349	"""
350	Reinitialize child class instance.
351	"""
352	vbox.TestDriver.__init__(self);
353
354	# We need to set a default test VM set here -- otherwise the test
355	# driver base class won't let us use the "--test-vms" switch.
356	#
357	# See the "--local" switch in self.parseOption().
358	self.oTestVmSet = self.oTestVmManager.getSmokeVmSet('nat');
359
360	# Selected NIC attachment.
361	self.sNicAttachment = '';
362
363	# Session handling stuff.
364	# Only needed for remote tests executed by TxS.
365	self.oSession = None;
366	self.oTxsSession = None;
367
368	self.sVBoxInstallRoot = None;
369
370	## Testing mode being used:
371	# "local": Execute unit tests locally (same host, default).
372	# "remote-copy": Copies unit tests from host to the remote, then executing it.
373	# "remote-exec": Executes unit tests right on the remote from a given source.
374	self.sMode = 'local';
375
376	self.cSkipped = 0;
377	self.cPassed = 0;
378	self.cFailed = 0;
379
380	# The source directory where our unit tests live.
381	# This most likely is our out/ or some staging directory and
382	# also acts the source for copying over the testcases to a remote target.
383	self.sUnitTestsPathSrc = None;
384
385	# Array of environment variables with NAME=VAL entries
386	# to be applied for testcases.
387	#
388	# This is also needed for testcases which are being executed remotely.
389	self.asEnv = [];
390
391	# The destination directory our unit tests live when being
392	# copied over to a remote target (via TxS).
393	self.sUnitTestsPathDst = None;
394
395	# Will be determined before executing the actual unit tests.
396	self.sExeSuff = '';
397
398	self.aiVBoxVer = (4, 3, 0, 0);
399
400	# For testing testcase logic.
401	self.fDryRun = False;
402	self.fOnlyWhiteList = False;
403
404	@staticmethod
405	def _sanitizePath(sPath):
406	"""
407	Does a little bit of sanitizing a given path by removing quoting, if any.
408
409	This is needed because handed-in paths via command line arguments can contain variables like "${CDROM}"
410	which might need to get processed by TXS on the guest side first.
411
412	Returns the sanitized path.
413	"""
414	if sPath is None: # Keep uninitialized strings as-is.
415	return None;
416	return sPath.strip('\"').strip('\'');
417
418	def _detectPaths(self):
419	"""
420	Internal worker for actionVerify and actionExecute that detects paths.
421
422	This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
423	"""
424
425	reporter.log2('Detecting paths ...');
426
427	#
428	# Do some sanity checking first.
429	#
430	if self.sMode == 'remote-exec' and not self.sUnitTestsPathSrc: # There is no way we can figure this out automatically.
431	reporter.error('Unit tests source must be specified explicitly for selected mode!');
432	return False;
433
434	#
435	# We need a VBox install (/ build) to test.
436	#
437	if False is True: ## @todo r=andy WTF?
438	if not self.importVBoxApi():
439	reporter.error('Unabled to import the VBox Python API.');
440	return False;
441	else:
442	self._detectBuild();
443	if self.oBuild is None:
444	reporter.error('Unabled to detect the VBox build.');
445	return False;
446
447	#
448	# Where are the files installed?
449	# Solaris requires special handling because of it's multi arch subdirs.
450	#
451	if not self.sVBoxInstallRoot:
452	self.sVBoxInstallRoot = self.oBuild.sInstallPath;
453	if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
454	sArchDir = utils.getHostArch();
455	if sArchDir == 'x86': sArchDir = 'i386';
456	self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
457
458	## @todo r=andy Make sure the install root really exists and is accessible.
459
460	# Add the installation root to the PATH on windows so we can get DLLs from it.
461	if utils.getHostOs() == 'win':
462	sPathName = 'PATH';
463	if not sPathName in os.environ:
464	sPathName = 'Path';
465	sPath = os.environ.get(sPathName, '.');
466	if sPath and sPath[-1] != ';':
467	sPath += ';';
468	os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
469	else:
470	reporter.log2('VBox installation root already set to "%s"' % (self.sVBoxInstallRoot));
471
472	self.sVBoxInstallRoot = self._sanitizePath(self.sVBoxInstallRoot);
473
474	#
475	# The unittests are generally not installed, so look for them.
476	#
477	if not self.sUnitTestsPathSrc:
478	sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
479	asCandidates = [
480	self.oBuild.sInstallPath,
481	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), self.oBuild.sType, sBinOrDist),
482	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'release', sBinOrDist),
483	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug', sBinOrDist),
484	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict', sBinOrDist),
485	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt', sBinOrDist),
486	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'profile', sBinOrDist),
487	os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
488	os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
489	os.path.join(self.sScratchPath, sBinOrDist),
490	];
491	if utils.getHostOs() == 'darwin':
492	for i in range(1, len(asCandidates)):
493	asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
494
495	for sCandidat in asCandidates:
496	if os.path.exists(os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff)):
497	self.sUnitTestsPathSrc = sCandidat;
498	break;
499
500	if not self.sUnitTestsPathSrc:
501	reporter.error('Unable to find unit test source dir. Candidates: %s' % (asCandidates,));
502	return False;
503	else:
504	reporter.log2('Unit test source dir already set to "%s"' % (self.sUnitTestsPathSrc));
505
506	self.sUnitTestsPathSrc = self._sanitizePath(self.sUnitTestsPathSrc);
507
508	return True;
509
510	#
511	# Overridden methods.
512	#
513
514	def showUsage(self):
515	"""
516	Shows the testdriver usage.
517	"""
518	fRc = vbox.TestDriver.showUsage(self);
519	reporter.log('');
520	reporter.log('Unit Test #1 options:');
521	reporter.log(' --dryrun');
522	reporter.log(' Performs a dryrun (no tests being executed).');
523	reporter.log(' --mode <local\|remote-copy\|remote-exec>');
524	reporter.log(' Specifies the test execution mode:');
525	reporter.log(' local: Locally on the same machine.');
526	reporter.log(' remote-copy: On remote (guest) by copying them from the local source.');
527	reporter.log(' remote-exec: On remote (guest) directly (needs unit test source).');
528	reporter.log(' --only-whitelist');
529	reporter.log(' Only processes the white list.');
530	reporter.log(' --quick');
531	reporter.log(' Very selective testing.');
532	reporter.log(' --unittest-source <dir>');
533	reporter.log(' Sets the unit test source to <dir>.');
534	reporter.log(' Also used for remote execution.');
535	reporter.log(' --vbox-install-root <dir>');
536	reporter.log(' Sets the VBox install root to <dir>.');
537	reporter.log(' Also used for remote execution.');
538	return fRc;
539
540	def parseOption(self, asArgs, iArg):
541	"""
542	Parses the testdriver arguments from the command line.
543	"""
544	if asArgs[iArg] == '--dryrun':
545	self.fDryRun = True;
546	elif asArgs[iArg] == '--mode':
547	iArg += 1;
548	if iArg >= len(asArgs):
549	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
550	if asArgs[iArg] in ('local', 'remote-copy', 'remote-exec',):
551	self.sMode = asArgs[iArg];
552	else:
553	raise base.InvalidOption('Argument "%s" invalid' % (asArgs[iArg]));
554	elif asArgs[iArg] == '--unittest-source':
555	iArg += 1;
556	if iArg >= len(asArgs):
557	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
558	self.sUnitTestsPathSrc = asArgs[iArg];
559	elif asArgs[iArg] == '--only-whitelist':
560	self.fOnlyWhiteList = True;
561	elif asArgs[iArg] == '--quick':
562	self.fOnlyWhiteList = True;
563	elif asArgs[iArg] == '--vbox-install-root':
564	iArg += 1;
565	if iArg >= len(asArgs):
566	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
567	self.sVBoxInstallRoot = asArgs[iArg];
568	else:
569	return vbox.TestDriver.parseOption(self, asArgs, iArg);
570	return iArg + 1;
571
572	def actionVerify(self):
573	if not self._detectPaths():
574	return False;
575
576	if self.oTestVmSet:
577	return vbox.TestDriver.actionVerify(self);
578
579	return True;
580
581	def actionConfig(self):
582	# Make sure vboxapi has been imported so we can use the constants.
583	if not self.importVBoxApi():
584	return False;
585
586	# Do the configuring.
587	if self.isRemoteMode():
588	if self.sNicAttachment == 'nat': eNic0AttachType = vboxcon.NetworkAttachmentType_NAT;
589	elif self.sNicAttachment == 'bridged': eNic0AttachType = vboxcon.NetworkAttachmentType_Bridged;
590	else: eNic0AttachType = None;
591
592	# Make sure to mount the Validation Kit .ISO so that TxS has the chance
593	# to update itself.
594	#
595	# This is necessary as a lot of our test VMs nowadays have a very old TxS
596	# installed which don't understand commands like uploading files to the guest.
597	# Uploading files is needed for this test driver, however.
598	#
599	## @todo Get rid of this as soon as we create test VMs in a descriptive (automated) manner.
600	return self.oTestVmSet.actionConfig(self, eNic0AttachType = eNic0AttachType,
601	sDvdImage = self.sVBoxValidationKitIso);
602
603	return True;
604
605	def actionExecute(self):
606	# Make sure vboxapi has been imported so we can execute the driver without going thru
607	# a former configuring step.
608	if not self.importVBoxApi():
609	return False;
610	if not self._detectPaths():
611	return False;
612	reporter.log2('Unit test source path is "%s"\n' % self.sUnitTestsPathSrc);
613
614	if not self.sUnitTestsPathDst:
615	self.sUnitTestsPathDst = self.sScratchPath;
616	reporter.log2('Unit test destination path is "%s"\n' % self.sUnitTestsPathDst);
617
618	if self.isRemoteMode(): # Run on a test VM (guest).
619	if self.fpApiVer < 7.0: ## @todo Needs Validation Kit .ISO tweaking (including the unit tests) first.
620	reporter.log('Remote unit tests for non-trunk builds skipped.');
621	fRc = True;
622	else:
623	assert self.oTestVmSet is not None;
624	fRc = self.oTestVmSet.actionExecute(self, self.testOneVmConfig);
625	else: # Run locally (host).
626	self._figureVersion();
627	self._makeEnvironmentChanges();
628
629	# If this is an ASAN build and we're on linux, make sure we've got
630	# libasan.so.N in the LD_LIBRARY_PATH or stuff w/o a RPATH entry
631	# pointing to /opt/VirtualBox will fail (like tstAsmStructs).
632	if self.getBuildType() == 'asan' and utils.getHostOs() in ('linux',):
633	sLdLibraryPath = '';
634	if 'LD_LIBRARY_PATH' in os.environ:
635	sLdLibraryPath = os.environ['LD_LIBRARY_PATH'] + ':';
636	sLdLibraryPath += self.oBuild.sInstallPath;
637	os.environ['LD_LIBRARY_PATH'] = sLdLibraryPath;
638
639	fRc = self._testRunUnitTests(None);
640
641	return fRc;
642
643	#
644	# Misc.
645	#
646	def isRemoteMode(self):
647	""" Predicate method for checking if in any remote mode. """
648	return self.sMode.startswith('remote');
649
650	#
651	# Test execution helpers.
652	#
653
654	def _testRunUnitTests(self, oTestVm):
655	"""
656	Main function to execute all unit tests.
657	"""
658
659	# Determine executable suffix based on selected execution mode.
660	if self.isRemoteMode(): # Run on a test VM (guest).
661	if oTestVm.isWindows():
662	self.sExeSuff = '.exe';
663	else:
664	self.sExeSuff = '';
665	else:
666	self.sExeSuff = '.exe' if utils.getHostOs() in [ 'win', 'dos', 'os2' ] else '';
667
668	self._testRunUnitTestsSet(oTestVm, r'^tst*', 'testcase');
669	self._testRunUnitTestsSet(oTestVm, r'^tst*', '.');
670
671	fRc = self.cFailed == 0;
672
673	reporter.log('');
674	if self.fDryRun:
675	reporter.log('*********************************************************');
676	reporter.log('DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN');
677	reporter.log('*********************************************************');
678	reporter.log('*********************************************************');
679	reporter.log(' Target: %s' % (oTestVm.sVmName if oTestVm else 'local',));
680	reporter.log(' Mode: %s' % (self.sMode,));
681	reporter.log('Unit tests source: %s %s'
682	% (self.sUnitTestsPathSrc, '(on remote)' if self.sMode == 'remote-exec' else '',));
683	reporter.log('VBox install root: %s %s'
684	% (self.sVBoxInstallRoot, '(on remote)' if self.sMode == 'remote-exec' else '',));
685	reporter.log('*********************************************************');
686	reporter.log('*** PASSED: %d' % (self.cPassed,));
687	reporter.log('*** FAILED: %d' % (self.cFailed,));
688	reporter.log('*** SKIPPED: %d' % (self.cSkipped,));
689	reporter.log('*** TOTAL: %d' % (self.cPassed + self.cFailed + self.cSkipped,));
690
691	return fRc;
692
693
694	def testOneVmConfig(self, oVM, oTestVm):
695	"""
696	Runs the specified VM thru test #1.
697	"""
698
699	# Simple test.
700	self.logVmInfo(oVM);
701
702	if not self.fDryRun:
703	# Try waiting for a bit longer (5 minutes) until the CD is available to avoid running into timeouts.
704	self.oSession, self.oTxsSession = self.startVmAndConnectToTxsViaTcp(oTestVm.sVmName,
705	fCdWait = not self.fDryRun,
706	cMsCdWait = 5 * 60 * 1000);
707	if self.oSession is None:
708	return False;
709
710	self.addTask(self.oTxsSession);
711
712	# Determine the unit tests destination path.
713	self.sUnitTestsPathDst = oTestVm.pathJoin(self.getGuestTempDir(oTestVm), 'testUnitTests');
714
715	# Run the unit tests.
716	self._testRunUnitTests(oTestVm);
717
718	# Cleanup.
719	if self.oSession is not None:
720	self.removeTask(self.oTxsSession);
721	self.terminateVmBySession(self.oSession);
722	return True;
723
724	#
725	# Test execution helpers.
726	#
727
728	def _figureVersion(self):
729	""" Tries to figure which VBox version this is, setting self.aiVBoxVer. """
730	try:
731	sVer = utils.processOutputChecked(['VBoxManage', '--version'])
732
733	sVer = sVer.strip();
734	sVer = re.sub(r'_BETA.*r', '.', sVer);
735	sVer = re.sub(r'_ALPHA.*r', '.', sVer);
736	sVer = re.sub(r'_RC.*r', '.', sVer);
737	sVer = re.sub('_SPB', '', sVer)
738	sVer = sVer.replace('r', '.');
739
740	self.aiVBoxVer = [int(sComp) for sComp in sVer.split('.')];
741
742	reporter.log('VBox version: %s' % (self.aiVBoxVer,));
743	except:
744	reporter.logXcpt();
745	return False;
746	return True;
747
748	def _compareVersion(self, aiVer):
749	"""
750	Compares the give version string with the vbox version string,
751	returning a result similar to C strcmp(). aiVer is on the right side.
752	"""
753	cComponents = min(len(self.aiVBoxVer), len(aiVer));
754	for i in range(cComponents):
755	if self.aiVBoxVer[i] < aiVer[i]:
756	return -1;
757	if self.aiVBoxVer[i] > aiVer[i]:
758	return 1;
759	return len(self.aiVBoxVer) - len(aiVer);
760
761	def _isExcluded(self, sTest, dExclList):
762	""" Checks if the testcase is excluded or not. """
763	if sTest in dExclList:
764	sFullExpr = dExclList[sTest].replace(' ', '').strip();
765	if sFullExpr == '':
766	return True;
767
768	# Consider each exclusion expression. These are generally ranges,
769	# either open ended or closed: "<4.3.51r12345", ">=4.3.0 && <=4.3.4".
770	asExprs = sFullExpr.split(';');
771	for sExpr in asExprs:
772
773	# Split it on the and operator and process each sub expression.
774	fResult = True;
775	for sSubExpr in sExpr.split('&&'):
776	# Split out the comparison operator and the version value.
777	if sSubExpr.startswith('<=') or sSubExpr.startswith('>='):
778	sOp = sSubExpr[:2];
779	sValue = sSubExpr[2:];
780	elif sSubExpr.startswith('<') or sSubExpr.startswith('>') or sSubExpr.startswith('='):
781	sOp = sSubExpr[:1];
782	sValue = sSubExpr[1:];
783	else:
784	sOp = sValue = '';
785
786	# Convert the version value, making sure we've got a valid one.
787	try: aiValue = [int(sComp) for sComp in sValue.replace('r', '.').split('.')];
788	except: aiValue = ();
789	if not aiValue or len(aiValue) > 4:
790	reporter.error('Invalid exclusion expression for %s: "%s" [%s]' % (sTest, sSubExpr, dExclList[sTest]));
791	return True;
792
793	# Do the compare.
794	iCmp = self._compareVersion(aiValue);
795	if sOp == '>=' and iCmp < 0:
796	fResult = False;
797	elif sOp == '>' and iCmp <= 0:
798	fResult = False;
799	elif sOp == '<' and iCmp >= 0:
800	fResult = False;
801	elif sOp == '>=' and iCmp < 0:
802	fResult = False;
803	reporter.log2('iCmp=%s; %s %s %s -> %s' % (iCmp, self.aiVBoxVer, sOp, aiValue, fResult));
804
805	# Did the expression match?
806	if fResult:
807	return True;
808
809	return False;
810
811	def _sudoExecuteSync(self, asArgs):
812	"""
813	Executes a sudo child process synchronously.
814	Returns True if the process executed successfully and returned 0,
815	otherwise False is returned.
816	"""
817	reporter.log2('Executing [sudo]: %s' % (asArgs, ));
818	if self.isRemoteMode():
819	iRc = -1; ## @todo Not used remotely yet.
820	else:
821	try:
822	iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False);
823	except:
824	reporter.errorXcpt();
825	return False;
826	reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs));
827	return iRc == 0;
828
829
830	def _logExpandString(self, sString, cVerbosity = 2):
831	"""
832	Expands a given string by asking TxS on the guest side and logs it.
833	Uses log level 2 by default.
834
835	No-op if no TxS involved.
836	"""
837	if reporter.getVerbosity() < cVerbosity or self.oTxsSession is None:
838	return;
839	sStringExp = self.oTxsSession.syncExpandString(sString);
840	if not sStringExp:
841	return;
842	reporter.log2('_logExpandString: "%s" -> "%s"' % (sString, sStringExp));
843
844	def _wrapPathExists(self, sPath):
845	"""
846	Creates the directory specified sPath (including parents).
847	"""
848	reporter.log2('_wrapPathExists: %s' % (sPath,));
849	if self.fDryRun:
850	return True;
851	fRc = False;
852	if self.isRemoteMode():
853	self._logExpandString(sPath);
854	fRc = self.oTxsSession.syncIsDir(sPath, fIgnoreErrors = True);
855	if not fRc:
856	fRc = self.oTxsSession.syncIsFile(sPath, fIgnoreErrors = True);
857	else:
858	fRc = os.path.exists(sPath);
859	return fRc;
860
861	def _wrapMkDir(self, sPath):
862	"""
863	Creates the directory specified sPath (including parents).
864	"""
865	reporter.log2('_wrapMkDir: %s' % (sPath,));
866	if self.fDryRun:
867	return True;
868	fRc = True;
869	if self.isRemoteMode():
870	fRc = self.oTxsSession.syncMkDirPath(sPath, fMode = 0o755);
871	else:
872	if utils.getHostOs() in [ 'win', 'os2' ]:
873	os.makedirs(sPath, 0o755);
874	else:
875	fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
876	if not fRc:
877	reporter.log('Failed to create dir "%s".' % (sPath,));
878	return fRc;
879
880	def _wrapCopyFile(self, sSrc, sDst, iMode):
881	"""
882	Copies a file.
883	"""
884	reporter.log2('_wrapCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
885	if self.fDryRun:
886	return True;
887	fRc = True;
888	if self.isRemoteMode():
889	self._logExpandString(sSrc);
890	self._logExpandString(sDst);
891	if self.sMode == 'remote-exec':
892	self.oTxsSession.syncCopyFile(sSrc, sDst, iMode);
893	else:
894	fRc = self.oTxsSession.syncUploadFile(sSrc, sDst);
895	if fRc:
896	fRc = self.oTxsSession.syncChMod(sDst, iMode);
897	else:
898	if utils.getHostOs() in [ 'win', 'os2' ]:
899	utils.copyFileSimple(sSrc, sDst);
900	os.chmod(sDst, iMode);
901	else:
902	fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
903	if fRc:
904	fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
905	if fRc is not True:
906	raise Exception('Failed to chmod "%s".' % (sDst,));
907	if not fRc:
908	reporter.log('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
909	return fRc;
910
911	def _wrapDeleteFile(self, sPath):
912	"""
913	Deletes a file.
914	"""
915	reporter.log2('_wrapDeleteFile: %s' % (sPath,));
916	if self.fDryRun:
917	return True;
918	fRc = True;
919	if self.isRemoteMode():
920	if self.oTxsSession.syncIsFile(sPath):
921	fRc = self.oTxsSession.syncRmFile(sPath, fIgnoreErrors = True);
922	else:
923	if os.path.exists(sPath):
924	if utils.getHostOs() in [ 'win', 'os2' ]:
925	os.remove(sPath);
926	else:
927	fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
928	if not fRc:
929	reporter.log('Failed to remove "%s".' % (sPath,));
930	return fRc;
931
932	def _wrapRemoveDir(self, sPath):
933	"""
934	Removes a directory.
935	"""
936	reporter.log2('_wrapRemoveDir: %s' % (sPath,));
937	if self.fDryRun:
938	return True;
939	fRc = True;
940	if self.isRemoteMode():
941	if self.oTxsSession.syncIsDir(sPath):
942	fRc = self.oTxsSession.syncRmDir(sPath, fIgnoreErrors = True);
943	else:
944	if os.path.exists(sPath):
945	if utils.getHostOs() in [ 'win', 'os2' ]:
946	os.rmdir(sPath);
947	else:
948	fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
949	if not fRc:
950	reporter.log('Failed to remove "%s".' % (sPath,));
951	return fRc;
952
953	def _envSet(self, sName, sValue):
954	if self.isRemoteMode():
955	# For remote execution we cache the environment block and pass it
956	# right when the process execution happens.
957	self.asEnv.append([ sName, sValue ]);
958	else:
959	os.environ[sName] = sValue;
960	return True;
961
962	def _executeTestCase(self, oTestVm, sName, sFullPath, sTestCaseSubDir, oDevNull): # pylint: disable=too-many-locals,too-many-statements
963	"""
964	Executes a test case.
965	"""
966
967	fSkipped = False;
968
969	#
970	# If hardening is enabled, some test cases and their dependencies
971	# needs to be copied to and execute from the source
972	# directory in order to work. They also have to be executed as
973	# root, i.e. via sudo.
974	#
975	fHardened = sName in self.kasHardened and self.sUnitTestsPathSrc != self.sVBoxInstallRoot;
976	fCopyToRemote = self.isRemoteMode();
977	fCopyDeps = self.isRemoteMode();
978	asFilesToRemove = []; # Stuff to clean up.
979	asDirsToRemove = []; # Ditto.
980
981	if fHardened or fCopyToRemote:
982	if fCopyToRemote:
983	sDstDir = os.path.join(self.sUnitTestsPathDst, sTestCaseSubDir);
984	else:
985	sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
986	if not self._wrapPathExists(sDstDir):
987	self._wrapMkDir(sDstDir);
988	asDirsToRemove.append(sDstDir);
989
990	sSrc = sFullPath + self.sExeSuff;
991	# If the testcase source does not exist for whatever reason, just mark it as skipped
992	# instead of reporting an error.
993	if not self._wrapPathExists(sSrc):
994	self.cSkipped += 1;
995	fSkipped = True;
996	return fSkipped;
997
998	sDst = os.path.join(sDstDir, os.path.basename(sFullPath) + self.sExeSuff);
999	fModeExe = 0;
1000	fModeDeps = 0;
1001	if not oTestVm or (oTestVm and not oTestVm.isWindows()): ## @todo NT4 does not like the chmod. Investigate this!
1002	fModeExe = 0o755;
1003	fModeDeps = 0o644;
1004	self._wrapCopyFile(sSrc, sDst, fModeExe);
1005	asFilesToRemove.append(sDst);
1006
1007	# Copy required dependencies to destination.
1008	if fCopyDeps:
1009	for sLib in self.kdTestCaseDepsLibs:
1010	for sSuff in [ '.dll', '.so', '.dylib' ]:
1011	assert self.sVBoxInstallRoot is not None;
1012	sSrc = os.path.join(self.sVBoxInstallRoot, sLib + sSuff);
1013	if self._wrapPathExists(sSrc):
1014	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1015	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1016	asFilesToRemove.append(sDst);
1017
1018	# Copy any associated .dll/.so/.dylib.
1019	for sSuff in [ '.dll', '.so', '.dylib' ]:
1020	sSrc = os.path.splitext(sFullPath)[0] + sSuff;
1021	if os.path.exists(sSrc):
1022	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1023	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1024	asFilesToRemove.append(sDst);
1025
1026	# Copy any associated .r0, .rc and .gc modules.
1027	offDriver = sFullPath.rfind('Driver')
1028	if offDriver > 0:
1029	for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
1030	sSrc = sFullPath[:offDriver] + sSuff;
1031	if os.path.exists(sSrc):
1032	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1033	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1034	asFilesToRemove.append(sDst);
1035
1036	sFullPath = os.path.join(sDstDir, os.path.basename(sFullPath));
1037
1038	#
1039	# Set up arguments and environment.
1040	#
1041	asArgs = [sFullPath + self.sExeSuff,]
1042	if sName in self.kdArguments:
1043	asArgs.extend(self.kdArguments[sName]);
1044
1045	sXmlFile = os.path.join(self.sUnitTestsPathDst, 'result.xml');
1046
1047	self._envSet('IPRT_TEST_OMIT_TOP_TEST', '1');
1048	self._envSet('IPRT_TEST_FILE', sXmlFile);
1049
1050	if self._wrapPathExists(sXmlFile):
1051	try: os.unlink(sXmlFile);
1052	except: self._wrapDeleteFile(sXmlFile);
1053
1054	#
1055	# Execute the test case.
1056	#
1057	# Windows is confusing output. Trying a few things to get rid of this.
1058	# First, flush both stderr and stdout before running the child. Second,
1059	# assign the child stderr to stdout. If this doesn't help, we'll have
1060	# to capture the child output.
1061	#
1062	reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
1063	try: sys.stdout.flush();
1064	except: pass;
1065	try: sys.stderr.flush();
1066	except: pass;
1067
1068	iRc = 0;
1069
1070	if not self.fDryRun:
1071	if fCopyToRemote:
1072	fRc = self.txsRunTest(self.oTxsSession, sName, cMsTimeout = 30 * 60 * 1000, sExecName = asArgs[0],
1073	asArgs = asArgs, asAddEnv = self.asEnv, fCheckSessionStatus = True);
1074	if fRc:
1075	iRc = 0;
1076	else:
1077	(_, sOpcode, abPayload) = self.oTxsSession.getLastReply();
1078	if sOpcode.startswith('PROC NOK '): # Extract process rc.
1079	iRc = abPayload[0]; # ASSUMES 8-bit rc for now.
1080	if iRc == 0: # Might happen if the testcase misses some dependencies. Set it to -42 then.
1081	iRc = -42;
1082	else:
1083	iRc = -1; ## @todo
1084	else:
1085	oChild = None;
1086	try:
1087	if fHardened:
1088	oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1089	else:
1090	oChild = utils.processPopenSafe(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1091	except:
1092	if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
1093	]:
1094	reporter.logXcpt();
1095	fSkipped = True;
1096	else:
1097	reporter.errorXcpt();
1098	iRc = 1023;
1099	oChild = None;
1100
1101	if oChild is not None:
1102	self.pidFileAdd(oChild.pid, sName, fSudo = fHardened);
1103	iRc = oChild.wait();
1104	self.pidFileRemove(oChild.pid);
1105	#
1106	# Clean up
1107	#
1108	for sPath in asFilesToRemove:
1109	self._wrapDeleteFile(sPath);
1110	for sPath in asDirsToRemove:
1111	self._wrapRemoveDir(sPath);
1112
1113	#
1114	# Report.
1115	#
1116	if os.path.exists(sXmlFile):
1117	reporter.addSubXmlFile(sXmlFile);
1118	if fHardened:
1119	self._wrapDeleteFile(sXmlFile);
1120	else:
1121	os.unlink(sXmlFile);
1122
1123	if iRc == 0:
1124	reporter.log('*** %s: exit code %d' % (sFullPath, iRc));
1125	self.cPassed += 1;
1126
1127	elif iRc == 4: # RTEXITCODE_SKIPPED
1128	reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFullPath, iRc));
1129	fSkipped = True;
1130	self.cSkipped += 1;
1131
1132	elif fSkipped:
1133	reporter.log('*** %s: exit code %d (Skipped)' % (sFullPath, iRc));
1134	self.cSkipped += 1;
1135
1136	else:
1137	sName = self.kdExitCodeNames.get(iRc, '');
1138	if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
1139	sName = self.kdExitCodeNamesWin[iRc];
1140	if sName != '':
1141	sName = ' (%s)' % (sName);
1142
1143	if iRc != 1:
1144	reporter.testFailure('Exit status: %d%s' % (iRc, sName));
1145	reporter.log( '!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
1146	else:
1147	reporter.error('!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
1148	self.cFailed += 1;
1149
1150	return fSkipped;
1151
1152	def _testRunUnitTestsSet(self, oTestVm, sTestCasePattern, sTestCaseSubDir):
1153	"""
1154	Run subset of the unit tests set.
1155	"""
1156
1157	# Open /dev/null for use as stdin further down.
1158	try:
1159	oDevNull = open(os.path.devnull, 'w+'); # pylint: disable=consider-using-with
1160	except:
1161	oDevNull = None;
1162
1163	# Determin the host OS specific exclusion lists.
1164	dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
1165	dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
1166
1167	## @todo Add filtering for more specific OSes (like OL server, doesn't have X installed) by adding a separate
1168	# black list + using utils.getHostOsVersion().
1169
1170	#
1171	# Process the file list and run everything looking like a testcase.
1172	#
1173	if not self.fOnlyWhiteList:
1174	if self.sMode in ('local', 'remote-copy'):
1175	asFiles = sorted(os.listdir(os.path.join(self.sUnitTestsPathSrc, sTestCaseSubDir)));
1176	else: # 'remote-exec'
1177	## @todo Implement remote file enumeration / directory listing.
1178	reporter.error('Sorry, no remote file enumeration implemented yet!\nUse --only-whitelist instead.');
1179	return;
1180	else:
1181	# Transform our dict into a list, where the keys are the list elements.
1182	asFiles = list(self.kdTestCasesWhiteList.keys());
1183	# Make sure to only keep the list item's base name so that the iteration down below works
1184	# with our white list without any additional modification.
1185	asFiles = [os.path.basename(s) for s in asFiles];
1186
1187	for sFilename in asFiles:
1188	# Separate base and suffix and morph the base into something we
1189	# can use for reporting and array lookups.
1190	sBaseName = os.path.basename(sFilename);
1191	sName, sSuffix = os.path.splitext(sBaseName);
1192	if sTestCaseSubDir != '.':
1193	sName = sTestCaseSubDir + '/' + sName;
1194
1195	reporter.log2('sTestCasePattern=%s, sBaseName=%s, sName=%s, sSuffix=%s, sFileName=%s'
1196	% (sTestCasePattern, sBaseName, sName, sSuffix, sFilename,));
1197
1198	# Process white list first, if set.
1199	if self.fOnlyWhiteList and not self._isExcluded(sName, self.kdTestCasesWhiteList):
1200	# (No testStart/Done or accounting here!)
1201	reporter.log('%s: SKIPPED (not in white list)' % (sName,));
1202	continue;
1203
1204	# Basic exclusion.
1205	if not re.match(sTestCasePattern, sBaseName) or sSuffix in self.kasSuffixBlackList:
1206	reporter.log2('"%s" is not a test case.' % (sName,));
1207	continue;
1208
1209	# When not only processing the white list, do some more checking first.
1210	if not self.fOnlyWhiteList:
1211	# Check if the testcase is black listed or buggy before executing it.
1212	if self._isExcluded(sName, self.kdTestCasesBlackList):
1213	# (No testStart/Done or accounting here!)
1214	reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
1215	continue;
1216
1217	if self._isExcluded(sName, self.kdTestCasesBuggy):
1218	reporter.testStart(sName);
1219	reporter.log('%s: Skipping, buggy in general.' % (sName,));
1220	reporter.testDone(fSkipped = True);
1221	self.cSkipped += 1;
1222	continue;
1223
1224	if self._isExcluded(sName, dTestCasesBuggyForHostOs):
1225	reporter.testStart(sName);
1226	reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
1227	reporter.testDone(fSkipped = True);
1228	self.cSkipped += 1;
1229	continue;
1230	else:
1231	# Passed the white list check already above.
1232	pass;
1233
1234	sFullPath = os.path.normpath(os.path.join(self.sUnitTestsPathSrc, os.path.join(sTestCaseSubDir, sFilename)));
1235	reporter.testStart(sName);
1236	try:
1237	fSkipped = self._executeTestCase(oTestVm, sName, sFullPath, sTestCaseSubDir, oDevNull);
1238	except:
1239	reporter.errorXcpt('!*!');
1240	self.cFailed += 1;
1241	fSkipped = False;
1242	reporter.testDone(fSkipped);
1243
1244
1245	if __name__ == '__main__':
1246	sys.exit(tdUnitTest1().main(sys.argv))

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/tests/unittests/tdUnitTest1.py@ 97488

Download in other formats: