tdUnitTest1.py@ 99104

Last change on this file since 99104 was 99104, checked in by vboxsync, 23 months ago
ValidationKit/tdUnitTest1: Exclude a few testcases on darwin in general due to being a driverless package and darwin.arm64 specifically
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 57.8 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: tdUnitTest1.py 99104 2023-03-22 08:16:25Z vboxsync $
4
5	"""
6	VirtualBox Validation Kit - Unit Tests.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2023 Oracle and/or its affiliates.
12
13	This file is part of VirtualBox base platform packages, as
14	available from https://www.virtualbox.org.
15
16	This program is free software; you can redistribute it and/or
17	modify it under the terms of the GNU General Public License
18	as published by the Free Software Foundation, in version 3 of the
19	License.
20
21	This program is distributed in the hope that it will be useful, but
22	WITHOUT ANY WARRANTY; without even the implied warranty of
23	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	General Public License for more details.
25
26	You should have received a copy of the GNU General Public License
27	along with this program; if not, see <https://www.gnu.org/licenses>.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32	in the VirtualBox distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37
38	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39	"""
40	__version__ = "$Revision: 99104 $"
41
42
43	# Standard Python imports.
44	import os
45	import sys
46	import re
47
48
49	# Only the main script needs to modify the path.
50	try: __file__ # pylint: disable=used-before-assignment
51	except: __file__ = sys.argv[0];
52	g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
53	sys.path.append(g_ksValidationKitDir)
54
55	# Validation Kit imports.
56	from common import utils;
57	from testdriver import base;
58	from testdriver import reporter;
59	from testdriver import vbox;
60	from testdriver import vboxcon;
61
62
63	class tdUnitTest1(vbox.TestDriver):
64	"""
65	Unit Tests.
66	"""
67
68	## The temporary exclude list.
69	## @note This shall be empty before we release 4.3!
70	kdTestCasesBuggyPerOs = {
71	'darwin': {
72	'testcase/tstX86-1': '', # 'FSTP M32R, ST0' fails; no idea why.
73	'testcase/tstLow': '>=7.0.0', # Driverless package.
74	'testcase/tstPin': '>=7.0.0', # Driverless package.
75	'testcase/tstIntNet-1': '>=7.0.0', # Driverless package.
76	},
77	'darwin.arm64': {
78	'testcase/tstRTDarwinMachKernel': '', # Not supported on arm64 right now (and not required due to driverless).
79	'testcase/tstAsmStructs': '', # Fails on arm64 due to different sizes, also not required as there is no
80	# assembly code which needs to match with structs.
81	'testcase/tstDarwinKeyboard': '', # Fails for unknown reason.
82	}
83	'linux': {
84	'testcase/tstRTFileAio': '', # See xTracker #8035.
85	},
86	'linux.amd64': {
87	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
88	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
89	},
90	'solaris': {
91	'testcase/tstIntNet-1': '', # Fails opening rge0, probably a generic issue figuring which nic to use.
92	'testcase/tstIprtList': '', # Crashes in the multithreaded test, I think.
93	'testcase/tstRTCritSect': '', # Fairness/whatever issue here.
94	'testcase/tstRTR0MemUserKernelDriver': '', # Failes when kernel to kernel buffers.
95	'testcase/tstRTSemRW': '', # line 338: RTSemRWReleaseRead(hSemRW): got VERR_ACCESS_DENIED
96	'testcase/tstRTStrAlloc': '', # VERR_NO_STR_MEMORY!
97	'testcase/tstRTFileQuerySize-1': '', # VERR_DEV_IO_ERROR on /dev/null!
98	},
99	'solaris.amd64': {
100	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
101	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
102	},
103	'win': {
104	'testcase/tstFile': '', # ??
105	'testcase/tstIntNet-1': '', # possibly same issue as solaris.
106	'testcase/tstMouseImpl': '', # STATUS_ACCESS_VIOLATION
107	'testcase/tstRTR0ThreadPreemptionDriver': '', # ??
108	'testcase/tstRTPath': '<4.3.51r89894',
109	'testcase/tstRTPipe': '', # ??
110	'testcase/tstRTR0MemUserKernelDriver': '', # ??
111	'testcase/tstRTR0SemMutexDriver': '', # ??
112	'testcase/tstRTStrAlloc': '', # ??
113	'testcase/tstRTStrFormat': '', # ??
114	'testcase/tstRTSystemQueryOsInfo': '', # ??
115	'testcase/tstRTTemp': '', # ??
116	'testcase/tstRTTime': '', # ??
117	'testcase/tstTime-2': '', # Total time differs too much! ... delta=-10859859
118	'testcase/tstTime-4': '', # Needs to be converted to DLL; ditto for tstTime-2.
119	'testcase/tstUtf8': '', # ??
120	'testcase/tstVMMR0CallHost-2': '', # STATUS_STACK_OVERFLOW
121	'testcase/tstX86-1': '', # Fails on win.x86.
122	'tscpasswd': '', # ??
123	'tstVMREQ': '', # ?? Same as darwin.x86?
124	},
125	'win.x86': {
126	'testcase/tstRTR0TimerDriver': '', # See xTracker #8041.
127	}
128	};
129
130	kdTestCasesBuggy = {
131	'testcase/tstGuestPropSvc': '', # GET_NOTIFICATION fails on testboxlin5.de.oracle.com and others.
132	'testcase/tstRTProcCreateEx': '', # Seen failing on wei01-b6ka-9.de.oracle.com.
133	'testcase/tstTimer': '', # Sometimes fails on linux, not important atm.
134	'testcase/tstGIP-2': '', # 2015-09-10: Fails regularly. E.g. TestSetID 2744205 (testboxsh2),
135	# 2743961 (wei01-b6kc-6). The responsible engineer should reenable
136	# it once it has been fixed.
137	};
138
139	## The permanent exclude list.
140	# @note Stripped of extensions!
141	kdTestCasesBlackList = {
142	'testcase/tstClipboardX11Smoke': '', # (Old naming, deprecated) Needs X, not available on all test boxes.
143	'testcase/tstClipboardGH-X11Smoke': '', # (New name) Ditto.
144	'testcase/tstClipboardMockHGCM': '', # Ditto.
145	'tstClipboardQt': '', # Is interactive and needs Qt, needed for Qt clipboard bugfixing.
146	'testcase/tstClipboardQt': '', # In case it moves here.
147	'tstDragAndDropQt': '', # Is interactive and needs Qt, needed for Qt drag'n drop bugfixing.
148	'testcase/tstDragAndDropQt': '', # In case it moves here.
149	'testcase/tstFileLock': '',
150	'testcase/tstDisasm-2': '', # without parameters it will disassembler 1GB starting from 0
151	'testcase/tstFileAppendWin-1': '',
152	'testcase/tstDir': '', # useless without parameters
153	'testcase/tstDir-2': '', # useless without parameters
154	'testcase/tstGlobalConfig': '',
155	'testcase/tstHostHardwareLinux': '', # must be killed with CTRL-C
156	'testcase/tstHttp': '', # Talks to outside servers.
157	'testcase/tstRTHttp': '', # parameters required
158	'testcase/tstLdr-2': '', # parameters required
159	'testcase/tstLdr-3': '', # parameters required
160	'testcase/tstLdr': '', # parameters required
161	'testcase/tstLdrLoad': '', # parameters required
162	'testcase/tstMove': '', # parameters required
163	'testcase/tstRTR0Timer': '', # loads 'tstRTR0Timer.r0'
164	'testcase/tstRTR0ThreadDriver': '', # loads 'tstRTR0Thread.r0'
165	'testcase/tstRunTestcases': '', # that's a script like this one
166	'testcase/tstRTReqPool': '', # fails sometimes, testcase buggy
167	'testcase/tstRTS3': '', # parameters required
168	'testcase/tstSDL': '', # graphics test
169	'testcase/tstSupLoadModule': '', # Needs parameters and vboxdrv access. Covered elsewhere.
170	'testcase/tstSeamlessX11': '', # graphics test
171	'testcase/tstTime-3': '', # parameters required
172	'testcase/tstVBoxControl': '', # works only inside a guest
173	'testcase/tstVDCopy': '', # parameters required
174	'testcase/tstVDFill': '', # parameters required
175	'tstAnimate': '', # parameters required
176	'testcase/tstAPI': '', # user interaction required
177	'tstCollector': '', # takes forever
178	'testcase/tstHeadless': '', # parameters required
179	'tstHeadless': '', # parameters required
180	'tstMicroRC': '', # required by tstMicro
181	'tstVBoxDbg': '', # interactive test
182	'testcase/tstTestServMgr': '', # some strange xpcom18a4 test, does not work
183	'tstTestServMgr': '', # some strange xpcom18a4 test, does not work
184	'tstPDMAsyncCompletion': '', # parameters required
185	'testcase/tstXptDump': '', # parameters required
186	'tstXptDump': '', # parameters required
187	'testcase/tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
188	'tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
189	'testcase/tstSimpleTypeLib': '', # parameters required
190	'tstSimpleTypeLib': '', # parameters required
191	'testcase/tstTestAtoms': '', # additional test file (words.txt) required
192	'tstTestAtoms': '', # additional test file (words.txt) required
193	'testcase/tstXptLink': '', # parameters required
194	'tstXptLink': '', # parameters required
195	'tstXPCOMCGlue': '', # user interaction required
196	'testcase/tstXPCOMCGlue': '', # user interaction required
197	'testcase/tstCAPIGlue': '', # user interaction required
198	'testcase/tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
199	'tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
200	'testcase/tstRTFilesystem': '', # parameters required
201	'testcase/tstRTDvm': '', # parameters required
202	'tstSSLCertDownloads': '', # Obsolete.
203	# later
204	'testcase/tstIntNetR0': '', # RTSPINLOCK_FLAGS_INTERRUPT_SAFE == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE
205	# slow stuff
206	'testcase/tstAvl': '', # SLOW!
207	'testcase/tstRTAvl': '', # SLOW! (new name)
208	'testcase/tstVD': '', # 8GB fixed-sized vmdk
209	# failed or hang
210	'testcase/tstCryptoPkcs7Verify': '', # hang
211	'tstOVF': '', # hang (only ancient version, now in new place)
212	'testcase/tstRTLockValidator': '', # Lock validation is not enabled for critical sections
213	'testcase/tstGuestControlSvc': '', # failed: line 288: testHost(&svcTable):
214	# expected VINF_SUCCESS, got VERR_NOT_FOUND
215	'testcase/tstRTMemEf': '', # failed w/o error message
216	'testcase/tstSupSem': '', # failed: SRE Timeout Accuracy (ms) : FAILED (1 errors)
217	'testcase/tstCryptoPkcs7Sign': '', # failed: 29330:
218	# error:02001002:lib(2):func(1):reason(2):NA:0:fopen('server.pem': '','r')
219	'testcase/tstCompressionBenchmark': '', # failed: error: RTZipBlockCompress failed
220	# for 'RTZipBlock/LZJB' (#4): VERR_NOT_SUPPORTED
221	'tstPDMAsyncCompletionStress': '', # VERR_INVALID_PARAMETER (cbSize = 0)
222	'tstMicro': '', # doesn't work on solaris, fix later if we care.
223	'tstVMM-HwAccm': '', # failed: Only checked AMD-V on linux
224	'tstVMM-HM': '', # failed: Only checked AMD-V on linux
225	'tstVMMFork': '', # failed: xtracker 6171
226	'tstTestFactory': '', # some strange xpcom18a4 test, does not work
227	'testcase/tstRTSemXRoads': '', # sporadically failed: Traffic - 8 threads per direction, 10 sec :
228	# FAILED (8 errors)
229	'tstVBoxAPILinux': '', # creates VirtualBox directories for root user because of sudo
230	# (should be in vbox)
231	'testcase/tstVMStructDTrace': '', # This is a D-script generator.
232	'tstVMStructRC': '', # This is a C-code generator.
233	'tstDeviceStructSizeRC': '', # This is a C-code generator.
234	'testcase/tstTSC': '', # Doesn't test anything and might fail with HT or/and too many cores.
235	'testcase/tstOpenUSBDev': '', # Not a useful testcase.
236	'testcase/tstX86-1': '', # Really more guest side.
237	'testcase/tstX86-FpuSaveRestore': '', # Experiments, could be useful for the guest not the host.
238	'tstAsmStructsRC': '', # Testcase run during build time (fails to find libstdc++.so.6 on some
239	# Solaris testboxes).
240	};
241
242	# Suffix exclude list.
243	kasSuffixBlackList = [
244	'.r0',
245	'.gc',
246	'.debug',
247	'.rel',
248	'.sys',
249	'.ko',
250	'.o',
251	'.obj',
252	'.lib',
253	'.a',
254	'.so',
255	'.dll',
256	'.dylib',
257	'.tmp',
258	'.log',
259	'.py',
260	'.pyc',
261	'.pyo',
262	'.pdb',
263	'.dSYM',
264	'.sym',
265	'.template',
266	'.expected',
267	'.expect',
268	];
269
270	# White list, which contains tests considered to be safe to execute,
271	# even on remote targets (guests).
272	#
273	# When --only-whitelist is specified, this is the only list being checked for.
274	kdTestCasesWhiteList = {
275	'testcase/tstFile': '',
276	'testcase/tstFileLock': '',
277	'testcase/tstClipboardMockHGCM': '', # Requires X on Linux OSes. Execute on remote targets only (guests).
278	'testcase/tstRTLocalIpc': '',
279	'testcase/tstRTPathQueryInfo': '',
280	'testcase/tstRTPipe': '',
281	'testcase/tstRTProcCreateEx': '',
282	'testcase/tstRTProcCreatePrf': '',
283	'testcase/tstRTProcIsRunningByName': '',
284	'testcase/tstRTProcQueryUsername': '',
285	'testcase/tstRTProcWait': '',
286	'testcase/tstTime-2': '',
287	'testcase/tstTime-3': '',
288	'testcase/tstTime-4': '',
289	'testcase/tstTimer': '',
290	'testcase/tstThread-1': '',
291	'testcase/tstUtf8': ''
292	};
293
294	# Test dependency list -- libraries.
295	# Needed in order to execute testcases on remote targets which don't have a VBox installation present.
296	kdTestCaseDepsLibs = [
297	"VBoxRT"
298	];
299
300	## The exclude list.
301	# @note Stripped extensions!
302	kasHardened = [
303	"testcase/tstIntNet-1",
304	"testcase/tstR0ThreadPreemptionDriver", # VBox 4.3
305	"testcase/tstRTR0ThreadPreemptionDriver",
306	"testcase/tstRTR0MemUserKernelDriver",
307	"testcase/tstRTR0SemMutexDriver",
308	"testcase/tstRTR0TimerDriver",
309	"testcase/tstRTR0ThreadDriver",
310	'testcase/tstRTR0DbgKrnlInfoDriver',
311	"tstInt",
312	"tstPDMQueue", # Comment in testcase says its driverless, but it needs driver access.
313	"tstVMM",
314	"tstVMMFork",
315	"tstVMREQ",
316	'testcase/tstCFGM',
317	'testcase/tstContiguous',
318	'testcase/tstGetPagingMode',
319	'testcase/tstGIP-2',
320	'testcase/tstInit',
321	'testcase/tstLow',
322	'testcase/tstMMHyperHeap',
323	'testcase/tstPage',
324	'testcase/tstPin',
325	'testcase/tstRTTime', 'testcase/tstTime', # GIP test case.
326	'testcase/tstRTTime-2', 'testcase/tstTime-2', # GIP test case.
327	'testcase/tstRTTime-4', 'testcase/tstTime-4', # GIP test case.
328	'testcase/tstSSM',
329	'testcase/tstSupSem-Zombie',
330	]
331
332	## Argument lists
333	kdArguments = {
334	'testcase/tstbntest': [ '-out', os.devnull, ], # Very noisy.
335	};
336
337
338	## Status code translations.
339	## @{
340	kdExitCodeNames = {
341	0: 'RTEXITCODE_SUCCESS',
342	1: 'RTEXITCODE_FAILURE',
343	2: 'RTEXITCODE_SYNTAX',
344	3: 'RTEXITCODE_INIT',
345	4: 'RTEXITCODE_SKIPPED',
346	};
347	kdExitCodeNamesWin = {
348	-1073741515: 'STATUS_DLL_NOT_FOUND',
349	-1073741512: 'STATUS_ORDINAL_NOT_FOUND',
350	-1073741511: 'STATUS_ENTRYPOINT_NOT_FOUND',
351	-1073741502: 'STATUS_DLL_INIT_FAILED',
352	-1073741500: 'STATUS_UNHANDLED_EXCEPTION',
353	-1073741499: 'STATUS_APP_INIT_FAILURE',
354	-1073741819: 'STATUS_ACCESS_VIOLATION',
355	-1073741571: 'STATUS_STACK_OVERFLOW',
356	};
357	## @}
358
359	def __init__(self):
360	"""
361	Reinitialize child class instance.
362	"""
363	vbox.TestDriver.__init__(self);
364
365	# We need to set a default test VM set here -- otherwise the test
366	# driver base class won't let us use the "--test-vms" switch.
367	#
368	# See the "--local" switch in self.parseOption().
369	self.oTestVmSet = self.oTestVmManager.getSmokeVmSet('nat');
370
371	# Selected NIC attachment.
372	self.sNicAttachment = '';
373
374	# Session handling stuff.
375	# Only needed for remote tests executed by TxS.
376	self.oSession = None;
377	self.oTxsSession = None;
378
379	self.sVBoxInstallRoot = None;
380
381	## Testing mode being used:
382	# "local": Execute unit tests locally (same host, default).
383	# "remote-copy": Copies unit tests from host to the remote, then executing it.
384	# "remote-exec": Executes unit tests right on the remote from a given source.
385	## @todo r=bird: 'remote-exec' and 'remote-copy' are confusing. We're presumably executing the test remotely in both
386	## cases, the different being that in the latter case we copy from the valkit iso rather than uploading the test files.
387	## That's hardly clear from the names or the explanation.
388	self.sMode = 'local';
389
390	self.cSkipped = 0;
391	self.cPassed = 0;
392	self.cFailed = 0;
393
394	## The source directory where our unit tests live.
395	# This most likely is our out/ or some staging directory and
396	# also acts the source for copying over the testcases to a remote target.
397	self.sUnitTestsPathSrc = None;
398
399	# The destination directory our unit tests live when being
400	# copied over to a remote target (via TxS).
401	self.sUnitTestsPathDst = None;
402
403	# The executable suffix to use for the executing the actual testcases.
404	# Will be re-set when executing the testcases on a remote (VM) once we know
405	# what type of suffix to use then (based on guest OS).
406	self.sExeSuff = base.exeSuff();
407
408	self.aiVBoxVer = (4, 3, 0, 0);
409
410	# For testing testcase logic.
411	self.fDryRun = False;
412	self.fOnlyWhiteList = False;
413
414	@staticmethod
415	def _sanitizePath(sPath):
416	"""
417	Does a little bit of sanitizing a given path by removing quoting, if any.
418
419	This is needed because handed-in paths via command line arguments can contain variables like "${CDROM}"
420	which might need to get processed by TXS on the guest side first.
421
422	Returns the sanitized path.
423	"""
424	if sPath is None: # Keep uninitialized strings as-is.
425	return None;
426	return sPath.strip('\"').strip('\'');
427
428	def _detectPaths(self):
429	"""
430	Internal worker for actionVerify and actionExecute that detects paths.
431
432	This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
433	"""
434
435	reporter.log2('Detecting paths ...');
436
437	#
438	# We need a VBox install (/ build) to test.
439	#
440	if False is True: ## @todo r=andy ?? # pylint: disable=comparison-of-constants
441	if not self.importVBoxApi():
442	reporter.error('Unabled to import the VBox Python API.');
443	return False;
444	else:
445	self._detectBuild();
446	if self.oBuild is None:
447	reporter.error('Unabled to detect the VBox build.');
448	return False;
449
450	#
451	# Where are the files installed?
452	# Solaris requires special handling because of it's multi arch subdirs.
453	#
454	if not self.sVBoxInstallRoot and self.sMode == 'remote-exec':
455	self.sVBoxInstallRoot = '${CDROM}/${OS}/${ARCH}';
456
457	elif not self.sVBoxInstallRoot:
458	self.sVBoxInstallRoot = self.oBuild.sInstallPath;
459	if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
460	sArchDir = utils.getHostArch();
461	if sArchDir == 'x86': sArchDir = 'i386';
462	self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
463
464	## @todo r=andy Make sure the install root really exists and is accessible.
465
466	# Add the installation root to the PATH on windows so we can get DLLs from it.
467	if utils.getHostOs() == 'win':
468	sPathName = 'PATH';
469	if not sPathName in os.environ:
470	sPathName = 'Path';
471	sPath = os.environ.get(sPathName, '.');
472	if sPath and sPath[-1] != ';':
473	sPath += ';';
474	os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
475	else:
476	reporter.log2('VBox installation root already set to "%s"' % (self.sVBoxInstallRoot));
477
478	self.sVBoxInstallRoot = self._sanitizePath(self.sVBoxInstallRoot);
479
480	#
481	# The unittests are generally not installed, so look for them.
482	#
483	if not self.sUnitTestsPathSrc and self.sMode == 'remote-exec':
484	self.sUnitTestsPathSrc = '${CDROM}/testcase/${OS}/${ARCH}';
485
486	elif not self.sUnitTestsPathSrc:
487	sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
488	asCandidates = [
489	self.oBuild.sInstallPath,
490	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), self.oBuild.sType, sBinOrDist),
491	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'release', sBinOrDist),
492	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug', sBinOrDist),
493	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict', sBinOrDist),
494	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt', sBinOrDist),
495	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'profile', sBinOrDist),
496	os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
497	os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
498	os.path.join(self.sScratchPath, sBinOrDist),
499	];
500	if utils.getHostOs() == 'darwin':
501	for i in range(1, len(asCandidates)):
502	asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
503
504	for sCandidat in asCandidates:
505	# The path of tstVMStructSize acts as a beacon to know where all other testcases are.
506	sFileBeacon = os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff);
507	reporter.log2('Searching for "%s" ...' % sFileBeacon);
508	if os.path.exists(sFileBeacon):
509	self.sUnitTestsPathSrc = sCandidat;
510	break
511
512	if self.sUnitTestsPathSrc:
513	reporter.log('Unit test source dir path: ', self.sUnitTestsPathSrc)
514	else:
515	reporter.error('Unable to find unit test source dir. Candidates: %s' % (asCandidates,));
516	if reporter.getVerbosity() >= 2:
517	reporter.log('Contents of "%s"' % self.sScratchPath);
518	for paths, dirs, files in os.walk(self.sScratchPath):
519	reporter.log('{} {} {}'.format(repr(paths), repr(dirs), repr(files)));
520	return False
521
522	else:
523	reporter.log2('Unit test source dir already set to "%s"' % (self.sUnitTestsPathSrc))
524	reporter.log('Unit test source dir path: ', self.sUnitTestsPathSrc)
525
526	self.sUnitTestsPathSrc = self._sanitizePath(self.sUnitTestsPathSrc);
527
528	return True;
529
530	#
531	# Overridden methods.
532	#
533
534	def showUsage(self):
535	"""
536	Shows the testdriver usage.
537	"""
538	fRc = vbox.TestDriver.showUsage(self);
539	reporter.log('');
540	reporter.log('Unit Test #1 options:');
541	reporter.log(' --dryrun');
542	reporter.log(' Performs a dryrun (no tests being executed).');
543	reporter.log(' --mode <local\|remote-copy\|remote-exec>');
544	reporter.log(' Specifies the test execution mode:');
545	reporter.log(' local: Locally on the same machine.');
546	reporter.log(' remote-copy: On remote (guest) by copying them from the local source. (BORKED!)');
547	reporter.log(' remote-exec: On remote (guest) directly (needs unit test source).');
548	reporter.log(' --only-whitelist');
549	reporter.log(' Only processes the white list.');
550	reporter.log(' --quick');
551	reporter.log(' Very selective testing.');
552	reporter.log(' --unittest-source <dir>');
553	reporter.log(' Sets the unit test source to <dir>.');
554	reporter.log(' Also used for remote execution.');
555	reporter.log(' --vbox-install-root <dir>');
556	reporter.log(' Sets the VBox install root to <dir>.');
557	reporter.log(' Also used for remote execution.');
558	return fRc;
559
560	def parseOption(self, asArgs, iArg):
561	"""
562	Parses the testdriver arguments from the command line.
563	"""
564	if asArgs[iArg] == '--dryrun':
565	self.fDryRun = True;
566	elif asArgs[iArg] == '--mode':
567	iArg += 1;
568	if iArg >= len(asArgs):
569	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
570	if asArgs[iArg] in ('local', 'remote-copy', 'remote-exec',):
571	self.sMode = asArgs[iArg];
572	else:
573	raise base.InvalidOption('Argument "%s" invalid' % (asArgs[iArg]));
574	elif asArgs[iArg] == '--unittest-source':
575	iArg += 1;
576	if iArg >= len(asArgs):
577	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
578	self.sUnitTestsPathSrc = asArgs[iArg];
579	elif asArgs[iArg] == '--only-whitelist':
580	self.fOnlyWhiteList = True;
581	elif asArgs[iArg] == '--quick':
582	self.fOnlyWhiteList = True;
583	elif asArgs[iArg] == '--vbox-install-root':
584	iArg += 1;
585	if iArg >= len(asArgs):
586	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
587	self.sVBoxInstallRoot = asArgs[iArg];
588	else:
589	return vbox.TestDriver.parseOption(self, asArgs, iArg);
590	return iArg + 1;
591
592	def actionVerify(self):
593	if not self._detectPaths():
594	return False;
595
596	if self.oTestVmSet:
597	return vbox.TestDriver.actionVerify(self);
598
599	return True;
600
601	def actionConfig(self):
602	# Make sure vboxapi has been imported so we can use the constants.
603	if not self.importVBoxApi():
604	return False;
605
606	# Do the configuring.
607	if self.isRemoteMode():
608	if self.sNicAttachment == 'nat': eNic0AttachType = vboxcon.NetworkAttachmentType_NAT;
609	elif self.sNicAttachment == 'bridged': eNic0AttachType = vboxcon.NetworkAttachmentType_Bridged;
610	else: eNic0AttachType = None;
611
612	# Make sure to mount the Validation Kit .ISO so that TxS has the chance
613	# to update itself.
614	#
615	# This is necessary as a lot of our test VMs nowadays have a very old TxS
616	# installed which don't understand commands like uploading files to the guest.
617	# Uploading files is needed for this test driver, however.
618	#
619	## @todo Get rid of this as soon as we create test VMs in a descriptive (automated) manner.
620	return self.oTestVmSet.actionConfig(self, eNic0AttachType = eNic0AttachType,
621	sDvdImage = self.sVBoxValidationKitIso);
622
623	return True;
624
625	def actionExecute(self):
626	# Make sure vboxapi has been imported so we can execute the driver without going thru
627	# a former configuring step.
628	if not self.importVBoxApi():
629	return False;
630	if not self._detectPaths():
631	return False;
632	reporter.log2('Unit test source path is "%s"\n' % self.sUnitTestsPathSrc);
633
634	if not self.sUnitTestsPathDst:
635	self.sUnitTestsPathDst = self.sScratchPath;
636	reporter.log2('Unit test destination path is "%s"\n' % self.sUnitTestsPathDst);
637
638	if self.isRemoteMode(): # Run on a test VM (guest).
639	if self.fpApiVer < 7.0: ## @todo Needs Validation Kit .ISO tweaking (including the unit tests) first.
640	reporter.log('Remote unit tests for non-trunk builds skipped.');
641	fRc = True;
642	else:
643	assert self.oTestVmSet is not None;
644	fRc = self.oTestVmSet.actionExecute(self, self.testOneVmConfig);
645	else: # Run locally (host).
646	self._figureVersion();
647	self._makeEnvironmentChanges();
648
649	# If this is an ASAN build and we're on linux, make sure we've got
650	# libasan.so.N in the LD_LIBRARY_PATH or stuff w/o a RPATH entry
651	# pointing to /opt/VirtualBox will fail (like tstAsmStructs).
652	if self.getBuildType() == 'asan' and utils.getHostOs() in ('linux',):
653	sLdLibraryPath = '';
654	if 'LD_LIBRARY_PATH' in os.environ:
655	sLdLibraryPath = os.environ['LD_LIBRARY_PATH'] + ':';
656	sLdLibraryPath += self.oBuild.sInstallPath;
657	os.environ['LD_LIBRARY_PATH'] = sLdLibraryPath;
658
659	fRc = self._testRunUnitTests(None);
660
661	return fRc;
662
663	#
664	# Misc.
665	#
666	def isRemoteMode(self):
667	""" Predicate method for checking if in any remote mode. """
668	return self.sMode.startswith('remote');
669
670	#
671	# Test execution helpers.
672	#
673
674	def _testRunUnitTests(self, oTestVm):
675	"""
676	Main function to execute all unit tests.
677	"""
678
679	# Determine executable suffix based on selected execution mode.
680	if self.isRemoteMode(): # Run on a test VM (guest).
681	if oTestVm.isWindows():
682	self.sExeSuff = '.exe';
683	else:
684	self.sExeSuff = '';
685	else:
686	# For local tests this already is set in __init__
687	pass;
688
689	self._testRunUnitTestsSet(oTestVm, r'^tst*', 'testcase');
690	self._testRunUnitTestsSet(oTestVm, r'^tst*', '.');
691
692	fRc = self.cFailed == 0;
693
694	reporter.log('');
695	if self.fDryRun:
696	reporter.log('*********************************************************');
697	reporter.log('DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN');
698	reporter.log('*********************************************************');
699	reporter.log('*********************************************************');
700	reporter.log(' Target: %s' % (oTestVm.sVmName if oTestVm else 'local',));
701	reporter.log(' Mode: %s' % (self.sMode,));
702	reporter.log(' Exe suffix: %s' % (self.sExeSuff,));
703	reporter.log('Unit tests source: %s %s'
704	% (self.sUnitTestsPathSrc, '(on remote)' if self.sMode == 'remote-exec' else '',));
705	reporter.log('VBox install root: %s %s'
706	% (self.sVBoxInstallRoot, '(on remote)' if self.sMode == 'remote-exec' else '',));
707	reporter.log('*********************************************************');
708	reporter.log('*** PASSED: %d' % (self.cPassed,));
709	reporter.log('*** FAILED: %d' % (self.cFailed,));
710	reporter.log('*** SKIPPED: %d' % (self.cSkipped,));
711	reporter.log('*** TOTAL: %d' % (self.cPassed + self.cFailed + self.cSkipped,));
712
713	return fRc;
714
715
716	def testOneVmConfig(self, oVM, oTestVm):
717	"""
718	Runs the specified VM thru test #1.
719	"""
720
721	# Simple test.
722	self.logVmInfo(oVM);
723
724	if not self.fDryRun:
725	# Try waiting for a bit longer (5 minutes) until the CD is available to avoid running into timeouts.
726	self.oSession, self.oTxsSession = self.startVmAndConnectToTxsViaTcp(oTestVm.sVmName,
727	fCdWait = not self.fDryRun,
728	cMsCdWait = 5 * 60 * 1000);
729	if self.oSession is None:
730	return False;
731
732	self.addTask(self.oTxsSession);
733
734	# Determine the unit tests destination path.
735	self.sUnitTestsPathDst = oTestVm.pathJoin(self.getGuestTempDir(oTestVm), 'testUnitTests');
736
737	# Run the unit tests.
738	self._testRunUnitTests(oTestVm);
739
740	# Cleanup.
741	if self.oSession is not None:
742	self.removeTask(self.oTxsSession);
743	self.terminateVmBySession(self.oSession);
744	return True;
745
746	#
747	# Test execution helpers.
748	#
749
750	def _figureVersion(self):
751	""" Tries to figure which VBox version this is, setting self.aiVBoxVer. """
752	try:
753	sVer = utils.processOutputChecked(['VBoxManage', '--version'])
754
755	sVer = sVer.strip();
756	sVer = re.sub(r'_BETA.*r', '.', sVer);
757	sVer = re.sub(r'_ALPHA.*r', '.', sVer);
758	sVer = re.sub(r'_RC.*r', '.', sVer);
759	sVer = re.sub('_SPB', '', sVer)
760	sVer = sVer.replace('r', '.');
761
762	self.aiVBoxVer = [int(sComp) for sComp in sVer.split('.')];
763
764	reporter.log('VBox version: %s' % (self.aiVBoxVer,));
765	except:
766	reporter.logXcpt();
767	return False;
768	return True;
769
770	def _compareVersion(self, aiVer):
771	"""
772	Compares the give version string with the vbox version string,
773	returning a result similar to C strcmp(). aiVer is on the right side.
774	"""
775	cComponents = min(len(self.aiVBoxVer), len(aiVer));
776	for i in range(cComponents):
777	if self.aiVBoxVer[i] < aiVer[i]:
778	return -1;
779	if self.aiVBoxVer[i] > aiVer[i]:
780	return 1;
781	return len(self.aiVBoxVer) - len(aiVer);
782
783	def _isExcluded(self, sTest, dExclList):
784	""" Checks if the testcase is excluded or not. """
785	if sTest in dExclList:
786	sFullExpr = dExclList[sTest].replace(' ', '').strip();
787	if sFullExpr == '':
788	return True;
789
790	# Consider each exclusion expression. These are generally ranges,
791	# either open ended or closed: "<4.3.51r12345", ">=4.3.0 && <=4.3.4".
792	asExprs = sFullExpr.split(';');
793	for sExpr in asExprs:
794
795	# Split it on the and operator and process each sub expression.
796	fResult = True;
797	for sSubExpr in sExpr.split('&&'):
798	# Split out the comparison operator and the version value.
799	if sSubExpr.startswith('<=') or sSubExpr.startswith('>='):
800	sOp = sSubExpr[:2];
801	sValue = sSubExpr[2:];
802	elif sSubExpr.startswith('<') or sSubExpr.startswith('>') or sSubExpr.startswith('='):
803	sOp = sSubExpr[:1];
804	sValue = sSubExpr[1:];
805	else:
806	sOp = sValue = '';
807
808	# Convert the version value, making sure we've got a valid one.
809	try: aiValue = [int(sComp) for sComp in sValue.replace('r', '.').split('.')];
810	except: aiValue = ();
811	if not aiValue or len(aiValue) > 4:
812	reporter.error('Invalid exclusion expression for %s: "%s" [%s]' % (sTest, sSubExpr, dExclList[sTest]));
813	return True;
814
815	# Do the compare.
816	iCmp = self._compareVersion(aiValue);
817	if sOp == '>=' and iCmp < 0:
818	fResult = False;
819	elif sOp == '>' and iCmp <= 0:
820	fResult = False;
821	elif sOp == '<' and iCmp >= 0:
822	fResult = False;
823	elif sOp == '>=' and iCmp < 0:
824	fResult = False;
825	reporter.log2('iCmp=%s; %s %s %s -> %s' % (iCmp, self.aiVBoxVer, sOp, aiValue, fResult));
826
827	# Did the expression match?
828	if fResult:
829	return True;
830
831	return False;
832
833	def _sudoExecuteSync(self, asArgs):
834	"""
835	Executes a sudo child process synchronously.
836	Returns True if the process executed successfully and returned 0,
837	otherwise False is returned.
838	"""
839	reporter.log2('Executing [sudo]: %s' % (asArgs, ));
840	if self.isRemoteMode():
841	iRc = -1; ## @todo Not used remotely yet.
842	else:
843	try:
844	iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False);
845	except:
846	reporter.errorXcpt();
847	return False;
848	reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs));
849	return iRc == 0;
850
851
852	def _logExpandString(self, sString, cVerbosity = 2):
853	"""
854	Expands a given string by asking TxS on the guest side and logs it.
855	Uses log level 2 by default.
856
857	No-op if no TxS involved.
858	"""
859	if reporter.getVerbosity() < cVerbosity or self.oTxsSession is None:
860	return;
861	sStringExp = self.oTxsSession.syncExpandString(sString);
862	if not sStringExp:
863	return;
864	reporter.log2('_logExpandString: "%s" -> "%s"' % (sString, sStringExp));
865
866	def _wrapPathExists(self, sPath):
867	"""
868	Creates the directory specified sPath (including parents).
869	"""
870	reporter.log2('_wrapPathExists: %s' % (sPath,));
871	if self.fDryRun:
872	return True;
873	fRc = False;
874	if self.isRemoteMode():
875	self._logExpandString(sPath);
876	fRc = self.oTxsSession.syncIsDir(sPath, fIgnoreErrors = True);
877	if not fRc:
878	fRc = self.oTxsSession.syncIsFile(sPath, fIgnoreErrors = True);
879	else:
880	fRc = os.path.exists(sPath);
881	return fRc;
882
883	def _wrapMkDir(self, sPath):
884	"""
885	Creates the directory specified sPath (including parents).
886	"""
887	reporter.log2('_wrapMkDir: %s' % (sPath,));
888	if self.fDryRun:
889	return True;
890	fRc = True;
891	if self.isRemoteMode():
892	fRc = self.oTxsSession.syncMkDirPath(sPath, fMode = 0o755);
893	else:
894	if utils.getHostOs() in [ 'win', 'os2' ]:
895	os.makedirs(sPath, 0o755);
896	else:
897	fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
898	if not fRc:
899	reporter.log('Failed to create dir "%s".' % (sPath,));
900	return fRc;
901
902	def _wrapCopyFile(self, sSrc, sDst, iMode):
903	"""
904	Copies a file.
905	"""
906	reporter.log2('_wrapCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
907	if self.fDryRun:
908	return True;
909	fRc = True;
910	if self.isRemoteMode():
911	self._logExpandString(sSrc);
912	self._logExpandString(sDst);
913	if self.sMode == 'remote-exec':
914	self.oTxsSession.syncCopyFile(sSrc, sDst, iMode);
915	else:
916	fRc = self.oTxsSession.syncUploadFile(sSrc, sDst);
917	if fRc:
918	fRc = self.oTxsSession.syncChMod(sDst, iMode);
919	else:
920	if utils.getHostOs() in [ 'win', 'os2' ]:
921	utils.copyFileSimple(sSrc, sDst);
922	os.chmod(sDst, iMode);
923	else:
924	fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
925	if fRc:
926	fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
927	if fRc is not True:
928	raise Exception('Failed to chmod "%s".' % (sDst,));
929	if not fRc:
930	reporter.log('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
931	return fRc;
932
933	def _wrapDeleteFile(self, sPath):
934	"""
935	Deletes a file.
936	"""
937	reporter.log2('_wrapDeleteFile: %s' % (sPath,));
938	if self.fDryRun:
939	return True;
940	fRc = True;
941	if self.isRemoteMode():
942	if self.oTxsSession.syncIsFile(sPath):
943	fRc = self.oTxsSession.syncRmFile(sPath, fIgnoreErrors = True);
944	else:
945	if os.path.exists(sPath):
946	if utils.getHostOs() in [ 'win', 'os2' ]:
947	os.remove(sPath);
948	else:
949	fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
950	if not fRc:
951	reporter.log('Failed to remove "%s".' % (sPath,));
952	return fRc;
953
954	def _wrapRemoveDir(self, sPath):
955	"""
956	Removes a directory.
957	"""
958	reporter.log2('_wrapRemoveDir: %s' % (sPath,));
959	if self.fDryRun:
960	return True;
961	fRc = True;
962	if self.isRemoteMode():
963	if self.oTxsSession.syncIsDir(sPath):
964	fRc = self.oTxsSession.syncRmDir(sPath, fIgnoreErrors = True);
965	else:
966	if os.path.exists(sPath):
967	if utils.getHostOs() in [ 'win', 'os2' ]:
968	os.rmdir(sPath);
969	else:
970	fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
971	if not fRc:
972	reporter.log('Failed to remove "%s".' % (sPath,));
973	return fRc;
974
975	def _executeTestCase(self, oTestVm, sName, sFilePathAbs, sTestCaseSubDir, oDevNull): # pylint: disable=too-many-locals,too-many-statements
976	"""
977	Executes a test case.
978
979	sFilePathAbs contains the absolute path (including OS-dependent executable suffix) of the testcase.
980
981	Returns @c true if testcase was skipped, or @c if not.
982	"""
983
984	fSkipped = False;
985
986	#
987	# If hardening is enabled, some test cases and their dependencies needs
988	# to be copied to and execute from the source directory in order to
989	# work. They also have to be executed as root, i.e. via sudo.
990	#
991	# When executing test remotely we must also copy stuff over to the
992	# remote location. Currently there is no diferent between remote-copy
993	# and remote-exec here, which it would be nice if Andy could explain...
994	#
995	## @todo r=bird: Please explain + fix ^^.
996	fHardened = sName in self.kasHardened and self.sUnitTestsPathSrc != self.sVBoxInstallRoot;
997	asFilesToRemove = []; # Stuff to clean up.
998	asDirsToRemove = []; # Ditto.
999
1000	if fHardened or self.isRemoteMode():
1001	if self.isRemoteMode():
1002	sDstDir = os.path.join(self.sUnitTestsPathDst, sTestCaseSubDir);
1003	else:
1004	sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
1005	if not self._wrapPathExists(sDstDir):
1006	self._wrapMkDir(sDstDir);
1007	asDirsToRemove.append(sDstDir);
1008
1009	sSrc = sFilePathAbs;
1010	# If the testcase source does not exist for whatever reason, just mark it as skipped
1011	# instead of reporting an error.
1012	if not self._wrapPathExists(sSrc): ## @todo r=bird: This doesn't add up for me with the two remote modes.
1013	self.cSkipped += 1; ## It seems to presuppose that we're in remote-exec mode as _wrapPathExists
1014	fSkipped = True; ## does not differentiate between the two remote modes and will always check
1015	return fSkipped; ## the path on the remote side.
1016
1017	sDst = os.path.join(sDstDir, os.path.basename(sFilePathAbs));
1018	fModeExe = 0;
1019	fModeDeps = 0;
1020	if not oTestVm or (oTestVm and not oTestVm.isWindows()): ## @todo NT4 does not like the chmod. Investigate this!
1021	fModeExe = 0o755;
1022	fModeDeps = 0o644;
1023	self._wrapCopyFile(sSrc, sDst, fModeExe);
1024	asFilesToRemove.append(sDst);
1025
1026	# Copy required dependencies to destination .
1027	# Note! The testcases are statically linked, so there are no VBoxRT.dll/so/dylib
1028	# to copy here. This code can be currently be ignored.
1029	if self.isRemoteMode():
1030	for sLib in self.kdTestCaseDepsLibs:
1031	for sSuff in [ '.dll', '.so', '.dylib' ]:
1032	assert self.sVBoxInstallRoot is not None;
1033	sSrc = os.path.join(self.sVBoxInstallRoot, sLib + sSuff);
1034	if self._wrapPathExists(sSrc):
1035	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1036	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1037	asFilesToRemove.append(sDst);
1038
1039	## @todo r=bird: The next two are check for _local_ files matching the remote path when in remote-mode.
1040	## It makes for very confusing reading and is a potential for trouble.
1041
1042	# Copy any associated .dll/.so/.dylib.
1043	for sSuff in [ '.dll', '.so', '.dylib' ]:
1044	sSrc = os.path.splitext(sFilePathAbs)[0] + sSuff;
1045	if os.path.exists(sSrc):
1046	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1047	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1048	asFilesToRemove.append(sDst);
1049
1050	# Copy any associated .r0, .rc and .gc modules.
1051	offDriver = sFilePathAbs.rfind('Driver')
1052	if offDriver > 0:
1053	for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
1054	sSrc = sFilePathAbs[:offDriver] + sSuff;
1055	if os.path.exists(sSrc):
1056	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1057	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1058	asFilesToRemove.append(sDst);
1059
1060	sFilePathAbs = os.path.join(sDstDir, os.path.basename(sFilePathAbs));
1061
1062	#
1063	# Set up arguments.
1064	#
1065	asArgs = [sFilePathAbs,]
1066	if sName in self.kdArguments:
1067	asArgs.extend(self.kdArguments[sName]);
1068
1069	#
1070	# Set up the environment.
1071	#
1072	# - We set IPRT_TEST_OMIT_TOP_TEST to avoid the unnecessary top-test
1073	# entry when running the inner tests, as it'll just add an unnecessary
1074	# result nesting.
1075	#
1076	# - IPRT_TEST_FILE is set to a result.xml file when running locally.
1077	# This is not necessary when executing via TxS as it sets IPRT_TEST_PIPE,
1078	# which overrides IPRT_TEST_FILE, to collect the XML output.
1079	#
1080	dEnvChanges = {
1081	'IPRT_TEST_OMIT_TOP_TEST': '1',
1082	};
1083
1084	sXmlFile = os.path.join(self.sUnitTestsPathDst, 'result.xml') if not self.isRemoteMode() else None;
1085	if sXmlFile:
1086	dEnvChanges['IPRT_TEST_FILE'] = sXmlFile;
1087	if self._wrapPathExists(sXmlFile):
1088	try: os.unlink(sXmlFile);
1089	except: self._wrapDeleteFile(sXmlFile);
1090
1091	#
1092	# Execute the test case.
1093	#
1094	# Windows is confusing output. Trying a few things to get rid of this.
1095	# First, flush both stderr and stdout before running the child. Second,
1096	# assign the child stderr to stdout. If this doesn't help, we'll have
1097	# to capture the child output.
1098	#
1099	reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
1100	try: sys.stdout.flush();
1101	except: pass;
1102	try: sys.stderr.flush();
1103	except: pass;
1104
1105	iRc = 0;
1106
1107	if not self.fDryRun:
1108	if self.isRemoteMode():
1109	asRemoteEnvChg = ['%s=%s' % (sKey, sValue) for sKey, sValue in utils.iteritems(dEnvChanges)];
1110
1111	fRc = self.txsRunTest(self.oTxsSession, sName, cMsTimeout = 30 * 60 * 1000, sExecName = asArgs[0],
1112	asArgs = asArgs, asAddEnv = asRemoteEnvChg, fCheckSessionStatus = True);
1113	if fRc:
1114	iRc = 0;
1115	else:
1116	(_, sOpcode, abPayload) = self.oTxsSession.getLastReply();
1117	if sOpcode.startswith('PROC NOK '): # Extract process rc.
1118	iRc = abPayload[0]; # ASSUMES 8-bit rc for now.
1119	if iRc == 0: # Might happen if the testcase misses some dependencies. Set it to -42 then.
1120	iRc = -42;
1121	else:
1122	iRc = -1; ## @todo
1123	else:
1124	for sKey, sValue in utils.iteritems(dEnvChanges):
1125	os.environ[sKey] = sValue;
1126
1127	oChild = None;
1128	try:
1129	if fHardened:
1130	oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1131	else:
1132	oChild = utils.processPopenSafe(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1133	except:
1134	if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
1135	]:
1136	reporter.logXcpt();
1137	fSkipped = True;
1138	else:
1139	reporter.errorXcpt();
1140	iRc = 1023;
1141	oChild = None;
1142
1143	if oChild is not None:
1144	self.pidFileAdd(oChild.pid, sName, fSudo = fHardened);
1145	iRc = oChild.wait();
1146	self.pidFileRemove(oChild.pid);
1147	#
1148	# Clean up
1149	#
1150	for sPath in asFilesToRemove:
1151	self._wrapDeleteFile(sPath);
1152	for sPath in asDirsToRemove:
1153	self._wrapRemoveDir(sPath);
1154
1155	#
1156	# Report (sXmlFile is None when in remote mode).
1157	#
1158	if sXmlFile and os.path.exists(sXmlFile):
1159	reporter.addSubXmlFile(sXmlFile);
1160	if fHardened:
1161	self._wrapDeleteFile(sXmlFile);
1162	else:
1163	os.unlink(sXmlFile);
1164
1165	if iRc == 0:
1166	reporter.log('*** %s: exit code %d' % (sFilePathAbs, iRc));
1167	self.cPassed += 1;
1168
1169	elif iRc == 4: # RTEXITCODE_SKIPPED
1170	reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFilePathAbs, iRc));
1171	fSkipped = True;
1172	self.cSkipped += 1;
1173
1174	elif fSkipped:
1175	reporter.log('*** %s: exit code %d (Skipped)' % (sFilePathAbs, iRc));
1176	self.cSkipped += 1;
1177
1178	else:
1179	sName = self.kdExitCodeNames.get(iRc, '');
1180	if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
1181	sName = self.kdExitCodeNamesWin[iRc];
1182	if sName != '':
1183	sName = ' (%s)' % (sName);
1184
1185	if iRc != 1:
1186	reporter.testFailure('Exit status: %d%s' % (iRc, sName));
1187	reporter.log( '!*! %s: exit code %d%s' % (sFilePathAbs, iRc, sName));
1188	else:
1189	reporter.error('!*! %s: exit code %d%s' % (sFilePathAbs, iRc, sName));
1190	self.cFailed += 1;
1191
1192	return fSkipped;
1193
1194	def _testRunUnitTestsSet(self, oTestVm, sTestCasePattern, sTestCaseSubDir):
1195	"""
1196	Run subset of the unit tests set.
1197	"""
1198
1199	# Open /dev/null for use as stdin further down.
1200	try:
1201	oDevNull = open(os.path.devnull, 'w+'); # pylint: disable=consider-using-with,unspecified-encoding
1202	except:
1203	oDevNull = None;
1204
1205	# Determin the host OS specific exclusion lists.
1206	dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
1207	dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
1208
1209	## @todo Add filtering for more specific OSes (like OL server, doesn't have X installed) by adding a separate
1210	# black list + using utils.getHostOsVersion().
1211
1212	#
1213	# Process the file list and run everything looking like a testcase.
1214	#
1215	if not self.fOnlyWhiteList:
1216	if self.sMode in ('local', 'remote-copy'):
1217	asFiles = sorted(os.listdir(os.path.join(self.sUnitTestsPathSrc, sTestCaseSubDir)));
1218	else: # 'remote-exec'
1219	## @todo Implement remote file enumeration / directory listing.
1220	reporter.error('Sorry, no remote file enumeration implemented yet!\nUse --only-whitelist instead.');
1221	return;
1222	else:
1223	# Transform our dict into a list, where the keys are the list elements.
1224	asFiles = list(self.kdTestCasesWhiteList.keys());
1225	# Make sure to only keep the list item's base name so that the iteration down below works
1226	# with our white list without any additional modification.
1227	asFiles = [os.path.basename(s) for s in asFiles];
1228
1229	for sFilename in asFiles:
1230	# When executing in remote execution mode, make sure to append the executable suffix here, as
1231	# the (white / black) lists do not contain any OS-specific executable suffixes.
1232	if self.sMode == 'remote-exec':
1233	sFilename = sFilename + self.sExeSuff;
1234	# Separate base and suffix and morph the base into something we
1235	# can use for reporting and array lookups.
1236	sBaseName = os.path.basename(sFilename);
1237	sName, sSuffix = os.path.splitext(sBaseName);
1238	if sTestCaseSubDir != '.':
1239	sName = sTestCaseSubDir + '/' + sName;
1240
1241	reporter.log2('sTestCasePattern=%s, sBaseName=%s, sName=%s, sSuffix=%s, sFileName=%s'
1242	% (sTestCasePattern, sBaseName, sName, sSuffix, sFilename,));
1243
1244	# Process white list first, if set.
1245	if self.fOnlyWhiteList \
1246	and not self._isExcluded(sName, self.kdTestCasesWhiteList):
1247	# (No testStart/Done or accounting here!)
1248	reporter.log('%s: SKIPPED (not in white list)' % (sName,));
1249	continue;
1250
1251	# Basic exclusion.
1252	if not re.match(sTestCasePattern, sBaseName) \
1253	or sSuffix in self.kasSuffixBlackList:
1254	reporter.log2('"%s" is not a test case.' % (sName,));
1255	continue;
1256
1257	# When not only processing the white list, do some more checking first.
1258	if not self.fOnlyWhiteList:
1259	# Check if the testcase is black listed or buggy before executing it.
1260	if self._isExcluded(sName, self.kdTestCasesBlackList):
1261	# (No testStart/Done or accounting here!)
1262	reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
1263	continue;
1264
1265	if self._isExcluded(sName, self.kdTestCasesBuggy):
1266	reporter.testStart(sName);
1267	reporter.log('%s: Skipping, buggy in general.' % (sName,));
1268	reporter.testDone(fSkipped = True);
1269	self.cSkipped += 1;
1270	continue;
1271
1272	if self._isExcluded(sName, dTestCasesBuggyForHostOs):
1273	reporter.testStart(sName);
1274	reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
1275	reporter.testDone(fSkipped = True);
1276	self.cSkipped += 1;
1277	continue;
1278	else:
1279	# Passed the white list check already above.
1280	pass;
1281
1282	sFilePathAbs = os.path.normpath(os.path.join(self.sUnitTestsPathSrc, os.path.join(sTestCaseSubDir, sFilename)));
1283	reporter.log2('sFilePathAbs=%s\n' % (sFilePathAbs,));
1284	reporter.testStart(sName);
1285	try:
1286	fSkipped = self._executeTestCase(oTestVm, sName, sFilePathAbs, sTestCaseSubDir, oDevNull);
1287	except:
1288	reporter.errorXcpt('!*!');
1289	self.cFailed += 1;
1290	fSkipped = False;
1291	reporter.testDone(fSkipped);
1292
1293
1294	if __name__ == '__main__':
1295	sys.exit(tdUnitTest1().main(sys.argv))

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/tests/unittests/tdUnitTest1.py@ 99104

Download in other formats: