tdUnitTest1.py@ 103389

Last change on this file since 103389 was 103389, checked in by vboxsync, 11 months ago
Validation Kit/tdUnitTest1: Resolved some @todos. The execution syntax for remote tests changed from "remote-exec" to "remote". "remote-copy" has been removed.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 59.4 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: tdUnitTest1.py 103389 2024-02-15 14:24:19Z vboxsync $
4
5	"""
6	VirtualBox Validation Kit - Unit Tests.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2023 Oracle and/or its affiliates.
12
13	This file is part of VirtualBox base platform packages, as
14	available from https://www.virtualbox.org.
15
16	This program is free software; you can redistribute it and/or
17	modify it under the terms of the GNU General Public License
18	as published by the Free Software Foundation, in version 3 of the
19	License.
20
21	This program is distributed in the hope that it will be useful, but
22	WITHOUT ANY WARRANTY; without even the implied warranty of
23	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	General Public License for more details.
25
26	You should have received a copy of the GNU General Public License
27	along with this program; if not, see <https://www.gnu.org/licenses>.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32	in the VirtualBox distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37
38	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39	"""
40	__version__ = "$Revision: 103389 $"
41
42
43	# Standard Python imports.
44	import os
45	import sys
46	import re
47
48
49	# Only the main script needs to modify the path.
50	try: __file__ # pylint: disable=used-before-assignment
51	except: __file__ = sys.argv[0];
52	g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
53	sys.path.append(g_ksValidationKitDir)
54
55	# Validation Kit imports.
56	from common import utils;
57	from testdriver import base;
58	from testdriver import reporter;
59	from testdriver import vbox;
60	from testdriver import vboxcon;
61
62
63	class tdUnitTest1(vbox.TestDriver):
64	"""
65	Unit Tests.
66	"""
67
68	## The temporary exclude list.
69	## @note This shall be empty before we release 4.3!
70	kdTestCasesBuggyPerOs = {
71	'darwin': {
72	'testcase/tstX86-1': '', # 'FSTP M32R, ST0' fails; no idea why.
73	'testcase/tstLow': '>=7.0.0', # Driverless package.
74	'testcase/tstPin': '>=7.0.0', # Driverless package.
75	'testcase/tstIntNet-1': '>=7.0.0', # Driverless package.
76	'testcase/tstRTR0DbgKrnlInfoDriver': '>=7.0.0', # Driverless package.
77	'testcase/tstRTR0MemUserKernelDriver': '>=7.0.0', # Driverless package.
78	'testcase/tstRTR0SemMutexDriver': '>=7.0.0', # Driverless package.
79	'testcase/tstRTR0ThreadPreemptionDriver': '>=7.0.0', # Driverless package.
80	'testcase/tstRTR0TimerDriver': '>=7.0.0', # Driverless package.
81	'testcase/tstDarwinKeyboard': '', # Fails for unknown reason.
82	'testcase/tstVBoxAPIXPCOM': '', # Can't instantiate the VirtualBox object
83	# (binary would need moving to the VirtualBox installation
84	# directory, merely a compile time test anyway)
85	},
86	'darwin.arm64': {
87	'testcase/tstRTDarwinMachKernel': '', # Not supported on arm64 right now (and not required due to driverless).
88	'testcase/tstAsmStructs': '', # Fails on arm64 due to different sizes, also not required as there is no
89	# assembly code which needs to match with structs.
90	'testcase/tstRTTime': '', # Needs more work first.
91	},
92	'linux': {
93	'testcase/tstRTFileAio': '', # See xTracker #8035.
94	},
95	'linux.amd64': {
96	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
97	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
98	},
99	'solaris': {
100	'testcase/tstIntNet-1': '', # Fails opening rge0, probably a generic issue figuring which nic to use.
101	'testcase/tstIprtList': '', # Crashes in the multithreaded test, I think.
102	'testcase/tstRTCritSect': '', # Fairness/whatever issue here.
103	'testcase/tstRTR0MemUserKernelDriver': '', # Failes when kernel to kernel buffers.
104	'testcase/tstRTSemRW': '', # line 338: RTSemRWReleaseRead(hSemRW): got VERR_ACCESS_DENIED
105	'testcase/tstRTStrAlloc': '', # VERR_NO_STR_MEMORY!
106	'testcase/tstRTFileQuerySize-1': '', # VERR_DEV_IO_ERROR on /dev/null!
107	'testcase/tstLow' : '', # VERR_NOT_SUPPORTED - allocating kernel memory with physical backing
108	# below 4GB (RTR0MemObjAllocLow) for running code (fExecutable=true)
109	# isn't implemented.
110	'testcase/tstContiguous' : '', # VERR_NOT_SUPPORTED - allocating kernel memory with contiguous physical
111	# backing below 4GB (RTR0MemObjAllocCont) for running code
112	# (fExecutable=true) isn't implemented.
113	'tstPDMQueue' : '' # VERR_NOT_SUPPORTED - running without the support driver (vboxdrv) isn't
114	# supported on Solaris (VMCREATE_F_DRIVERLESS/SUPR3INIT_F_DRIVERLESS).
115	},
116	'solaris.amd64': {
117	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
118	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
119	},
120	'win': {
121	'testcase/tstFile': '', # ??
122	'testcase/tstIntNet-1': '', # possibly same issue as solaris.
123	'testcase/tstMouseImpl': '', # STATUS_ACCESS_VIOLATION
124	'testcase/tstRTR0ThreadPreemptionDriver': '', # ??
125	'testcase/tstRTPath': '<4.3.51r89894',
126	'testcase/tstRTPipe': '', # ??
127	'testcase/tstRTR0MemUserKernelDriver': '', # ??
128	'testcase/tstRTR0SemMutexDriver': '', # ??
129	'testcase/tstRTStrAlloc': '', # ??
130	'testcase/tstRTStrFormat': '', # ??
131	'testcase/tstRTSystemQueryOsInfo': '', # ??
132	'testcase/tstRTTemp': '', # ??
133	'testcase/tstRTTime': '', # ??
134	'testcase/tstTime-2': '', # Total time differs too much! ... delta=-10859859
135	'testcase/tstTime-4': '', # Needs to be converted to DLL; ditto for tstTime-2.
136	'testcase/tstUtf8': '', # ??
137	'testcase/tstVMMR0CallHost-2': '', # STATUS_STACK_OVERFLOW
138	'testcase/tstX86-1': '', # Fails on win.x86.
139	'tscpasswd': '', # ??
140	'tstVMREQ': '', # ?? Same as darwin.x86?
141	},
142	'win.x86': {
143	'testcase/tstRTR0TimerDriver': '', # See xTracker #8041.
144	}
145	};
146
147	kdTestCasesBuggy = {
148	'testcase/tstGuestPropSvc': '', # GET_NOTIFICATION fails on testboxlin5.de.oracle.com and others.
149	'testcase/tstTimer': '', # Sometimes fails on linux, not important atm.
150	'testcase/tstGIP-2': '', # 2015-09-10: Fails regularly. E.g. TestSetID 2744205 (testboxsh2),
151	# 2743961 (wei01-b6kc-6). The responsible engineer should reenable
152	# it once it has been fixed.
153	};
154
155	## The permanent exclude list.
156	# @note Stripped of extensions!
157	kdTestCasesBlackList = {
158	'testcase/tstClipboardX11Smoke': '', # (Old naming, deprecated) Needs X, not available on all test boxes.
159	'testcase/tstClipboardGH-X11Smoke': '', # (New name) Ditto.
160	'testcase/tstClipboardHttpServerX11': '', # Ditto.
161	'testcase/tstClipboardMockHGCM': '', # Ditto.
162	'tstClipboardQt': '', # Is interactive and needs Qt, needed for Qt clipboard bugfixing.
163	'testcase/tstClipboardQt': '', # In case it moves here.
164	'tstDragAndDropQt': '', # Is interactive and needs Qt, needed for Qt drag'n drop bugfixing.
165	'testcase/tstDragAndDropQt': '', # In case it moves here.
166	'testcase/tstFileLock': '',
167	'testcase/tstDisasm-2': '', # without parameters it will disassembler 1GB starting from 0
168	'testcase/tstFileAppendWin-1': '',
169	'testcase/tstDir': '', # useless without parameters
170	'testcase/tstDir-2': '', # useless without parameters
171	'testcase/tstGlobalConfig': '',
172	'testcase/tstHostHardwareLinux': '', # must be killed with CTRL-C
173	'testcase/tstHttp': '', # Talks to outside servers.
174	'testcase/tstRTHttp': '', # parameters required
175	'testcase/tstLdr-2': '', # parameters required
176	'testcase/tstLdr-3': '', # parameters required
177	'testcase/tstLdr': '', # parameters required
178	'testcase/tstLdrLoad': '', # parameters required
179	'testcase/tstMove': '', # parameters required
180	'testcase/tstRTR0Timer': '', # loads 'tstRTR0Timer.r0'
181	'testcase/tstRTR0ThreadDriver': '', # loads 'tstRTR0Thread.r0'
182	'testcase/tstRunTestcases': '', # that's a script like this one
183	'testcase/tstRTReqPool': '', # fails sometimes, testcase buggy
184	'testcase/tstRTS3': '', # parameters required
185	'testcase/tstSDL': '', # graphics test
186	'testcase/tstSupLoadModule': '', # Needs parameters and vboxdrv access. Covered elsewhere.
187	'testcase/tstSeamlessX11': '', # graphics test
188	'testcase/tstTime-3': '', # parameters required
189	'testcase/tstVBoxControl': '', # works only inside a guest
190	'testcase/tstVDCopy': '', # parameters required
191	'testcase/tstVDFill': '', # parameters required
192	'tstAnimate': '', # parameters required
193	'testcase/tstAPI': '', # user interaction required
194	'tstCollector': '', # takes forever
195	'testcase/tstHeadless': '', # parameters required
196	'tstHeadless': '', # parameters required
197	'tstMicroRC': '', # required by tstMicro
198	'tstVBoxDbg': '', # interactive test
199	'testcase/tstTestServMgr': '', # some strange xpcom18a4 test, does not work
200	'tstTestServMgr': '', # some strange xpcom18a4 test, does not work
201	'tstPDMAsyncCompletion': '', # parameters required
202	'testcase/tstXptDump': '', # parameters required
203	'tstXptDump': '', # parameters required
204	'testcase/tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
205	'tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
206	'testcase/tstSimpleTypeLib': '', # parameters required
207	'tstSimpleTypeLib': '', # parameters required
208	'testcase/tstTestAtoms': '', # additional test file (words.txt) required
209	'tstTestAtoms': '', # additional test file (words.txt) required
210	'testcase/tstXptLink': '', # parameters required
211	'tstXptLink': '', # parameters required
212	'tstXPCOMCGlue': '', # user interaction required
213	'testcase/tstXPCOMCGlue': '', # user interaction required
214	'testcase/tstCAPIGlue': '', # user interaction required
215	'testcase/tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
216	'tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
217	'testcase/tstRTFilesystem': '', # parameters required
218	'testcase/tstRTDvm': '', # parameters required
219	'tstSSLCertDownloads': '', # Obsolete.
220	# later
221	'testcase/tstIntNetR0': '', # RTSPINLOCK_FLAGS_INTERRUPT_SAFE == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE
222	# slow stuff
223	'testcase/tstAvl': '', # SLOW!
224	'testcase/tstRTAvl': '', # SLOW! (new name)
225	'testcase/tstVD': '', # 8GB fixed-sized vmdk
226	# failed or hang
227	'testcase/tstCryptoPkcs7Verify': '', # hang
228	'tstOVF': '', # hang (only ancient version, now in new place)
229	'testcase/tstRTLockValidator': '', # Lock validation is not enabled for critical sections
230	'testcase/tstGuestControlSvc': '', # failed: line 288: testHost(&svcTable):
231	# expected VINF_SUCCESS, got VERR_NOT_FOUND
232	'testcase/tstRTMemEf': '', # failed w/o error message
233	'testcase/tstSupSem': '', # failed: SRE Timeout Accuracy (ms) : FAILED (1 errors)
234	'testcase/tstCryptoPkcs7Sign': '', # failed: 29330:
235	# error:02001002:lib(2):func(1):reason(2):NA:0:fopen('server.pem': '','r')
236	'testcase/tstCompressionBenchmark': '', # failed: error: RTZipBlockCompress failed
237	# for 'RTZipBlock/LZJB' (#4): VERR_NOT_SUPPORTED
238	'tstPDMAsyncCompletionStress': '', # VERR_INVALID_PARAMETER (cbSize = 0)
239	'tstMicro': '', # doesn't work on solaris, fix later if we care.
240	'tstVMM-HwAccm': '', # failed: Only checked AMD-V on linux
241	'tstVMM-HM': '', # failed: Only checked AMD-V on linux
242	'tstVMMFork': '', # failed: xtracker 6171
243	'tstTestFactory': '', # some strange xpcom18a4 test, does not work
244	'testcase/tstRTSemXRoads': '', # sporadically failed: Traffic - 8 threads per direction, 10 sec :
245	# FAILED (8 errors)
246	'tstVBoxAPILinux': '', # creates VirtualBox directories for root user because of sudo
247	# (should be in vbox)
248	'testcase/tstVMStructDTrace': '', # This is a D-script generator.
249	'tstVMStructRC': '', # This is a C-code generator.
250	'tstDeviceStructSizeRC': '', # This is a C-code generator.
251	'testcase/tstTSC': '', # Doesn't test anything and might fail with HT or/and too many cores.
252	'testcase/tstOpenUSBDev': '', # Not a useful testcase.
253	'testcase/tstX86-1': '', # Really more guest side.
254	'testcase/tstX86-FpuSaveRestore': '', # Experiments, could be useful for the guest not the host.
255	'tstAsmStructsRC': '', # Testcase run during build time (fails to find libstdc++.so.6 on some
256	# Solaris testboxes).
257	};
258
259	## The permanent exclude list for ASAN builds because they trigger false-positives.
260	# @note Stripped of extensions!
261	kdTestCasesBlackListAsan = {
262	'testcase/tstVMMR0CallHost-1': '', # Triggers a stack overflow error on linux.amd64
263	}
264
265	# Suffix exclude list.
266	kasSuffixBlackList = [
267	'.r0',
268	'.gc',
269	'.debug',
270	'.rel',
271	'.sys',
272	'.ko',
273	'.o',
274	'.obj',
275	'.lib',
276	'.a',
277	'.so',
278	'.dll',
279	'.dylib',
280	'.tmp',
281	'.log',
282	'.py',
283	'.pyc',
284	'.pyo',
285	'.pdb',
286	'.dSYM',
287	'.sym',
288	'.template',
289	'.expected',
290	'.expect',
291	];
292
293	# White list, which contains tests considered to be safe to execute,
294	# even on remote targets (guests).
295	#
296	# When --only-whitelist is specified, this is the only list being checked for.
297	kdTestCasesWhiteList = {
298	'testcase/tstFile': '',
299	'testcase/tstFileLock': '',
300	'testcase/tstClipboardMockHGCM': '', # Requires X on Linux OSes. Execute on remote targets only (guests).
301	'testcase/tstRTFsQueries': '',
302	'testcase/tstRTLocalIpc': '',
303	'testcase/tstRTPathQueryInfo': '',
304	'testcase/tstRTPipe': '',
305	'testcase/tstRTProcCreateEx': '',
306	'testcase/tstRTProcCreatePrf': '',
307	'testcase/tstRTProcIsRunningByName': '',
308	'testcase/tstRTProcQueryUsername': '',
309	'testcase/tstRTProcWait': '',
310	'testcase/tstTime-2': '',
311	'testcase/tstTime-3': '',
312	'testcase/tstTime-4': '',
313	'testcase/tstTimer': '',
314	'testcase/tstThread-1': '',
315	'testcase/tstUtf8': ''
316	};
317
318	# Test dependency list -- libraries.
319	# Needed in order to execute testcases on remote targets which don't have a VBox installation present.
320	kdTestCaseDepsLibs = [
321	"VBoxRT"
322	];
323
324	## The exclude list.
325	# @note Stripped extensions!
326	kasHardened = [
327	"testcase/tstIntNet-1",
328	"testcase/tstR0ThreadPreemptionDriver", # VBox 4.3
329	"testcase/tstRTR0ThreadPreemptionDriver",
330	"testcase/tstRTR0MemUserKernelDriver",
331	"testcase/tstRTR0SemMutexDriver",
332	"testcase/tstRTR0TimerDriver",
333	"testcase/tstRTR0ThreadDriver",
334	'testcase/tstRTR0DbgKrnlInfoDriver',
335	"tstInt",
336	"tstPDMQueue", # Comment in testcase says its driverless, but it needs driver access.
337	"tstVMM",
338	"tstVMMFork",
339	"tstVMREQ",
340	'testcase/tstCFGM',
341	'testcase/tstContiguous',
342	'testcase/tstGetPagingMode',
343	'testcase/tstGIP-2',
344	'testcase/tstInit',
345	'testcase/tstLow',
346	'testcase/tstMMHyperHeap',
347	'testcase/tstPage',
348	'testcase/tstPin',
349	'testcase/tstRTTime', 'testcase/tstTime', # GIP test case.
350	'testcase/tstRTTime-2', 'testcase/tstTime-2', # GIP test case.
351	'testcase/tstRTTime-4', 'testcase/tstTime-4', # GIP test case.
352	'testcase/tstSSM',
353	'testcase/tstSupSem-Zombie',
354	]
355
356	## Argument lists
357	kdArguments = {
358	'testcase/tstbntest': [ '-out', os.devnull, ], # Very noisy.
359	};
360
361
362	## Status code translations.
363	## @{
364	kdExitCodeNames = {
365	0: 'RTEXITCODE_SUCCESS',
366	1: 'RTEXITCODE_FAILURE',
367	2: 'RTEXITCODE_SYNTAX',
368	3: 'RTEXITCODE_INIT',
369	4: 'RTEXITCODE_SKIPPED',
370	};
371	kdExitCodeNamesWin = {
372	-1073741515: 'STATUS_DLL_NOT_FOUND',
373	-1073741512: 'STATUS_ORDINAL_NOT_FOUND',
374	-1073741511: 'STATUS_ENTRYPOINT_NOT_FOUND',
375	-1073741502: 'STATUS_DLL_INIT_FAILED',
376	-1073741500: 'STATUS_UNHANDLED_EXCEPTION',
377	-1073741499: 'STATUS_APP_INIT_FAILURE',
378	-1073741819: 'STATUS_ACCESS_VIOLATION',
379	-1073741571: 'STATUS_STACK_OVERFLOW',
380	};
381	## @}
382
383	def __init__(self):
384	"""
385	Reinitialize child class instance.
386	"""
387	vbox.TestDriver.__init__(self);
388
389	# We need to set a default test VM set here -- otherwise the test
390	# driver base class won't let us use the "--test-vms" switch.
391	#
392	# See the "--local" switch in self.parseOption().
393	self.oTestVmSet = self.oTestVmManager.getSmokeVmSet('nat');
394
395	# Selected NIC attachment.
396	self.sNicAttachment = '';
397
398	# Session handling stuff.
399	# Only needed for remote tests executed by TxS.
400	self.oSession = None;
401	self.oTxsSession = None;
402
403	# The VirtualBox installation root directory.
404	self.sVBoxInstallRoot = None;
405
406	## Testing mode being used:
407	# "local": Execute unit tests locally (same host, default).
408	# "remote": Executes unit tests right on the remote from a given source.
409	self.sMode = 'local';
410
411	self.cSkipped = 0;
412	self.cPassed = 0;
413	self.cFailed = 0;
414
415	## The source directory where our unit tests live.
416	#
417	# For local mode this is our out/ or some staging directory and
418	# also acts the source for copying over the testcases to a remote target.
419	#
420	# For remote remote this is the ${CDROM} directory where we ship the included
421	# testcases on the Validation Kit ISO.
422	self.sUnitTestsPathSrc = None;
423
424	# The destination directory our unit tests live when being
425	# copied over to a remote target (via TxS).
426	self.sUnitTestsPathDst = None;
427
428	# The executable suffix to use for the executing the actual testcases.
429	# Will be re-set when executing the testcases on a remote (VM) once we know
430	# what type of suffix to use then (based on guest OS).
431	self.sExeSuff = base.exeSuff();
432
433	self.aiVBoxVer = (4, 3, 0, 0);
434
435	# For testing testcase logic.
436	self.fDryRun = False;
437	self.fOnlyWhiteList = False;
438
439	@staticmethod
440	def _sanitizePath(sPath):
441	"""
442	Does a little bit of sanitizing a given path by removing quoting, if any.
443
444	This is needed because handed-in paths via command line arguments can contain variables like "${CDROM}"
445	which might need to get processed by TXS on the guest side first.
446
447	Returns the sanitized path.
448	"""
449	if sPath is None: # Keep uninitialized strings as-is.
450	return None;
451	return sPath.strip('\"').strip('\'');
452
453	def _detectPaths(self):
454	"""
455	Internal worker for actionVerify and actionExecute that detects paths.
456
457	This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
458	"""
459
460	reporter.log2('Detecting paths (mode is "%s") ...' % ("remot" if self.isRemoteMode() else "local",));
461
462	#
463	# We need a VBox install (/ build) to test.
464	#
465	if not self._detectBuild():
466	reporter.error('Unabled to detect the VBox build.');
467	return False;
468
469	#
470	# Where are the files installed?
471	# Solaris requires special handling because of it's multi arch subdirs.
472	#
473	if not self.sVBoxInstallRoot and self.isRemoteMode():
474	self.sVBoxInstallRoot = '${CDROM}/${OS}/${ARCH}';
475
476	elif not self.sVBoxInstallRoot:
477	self.sVBoxInstallRoot = self.oBuild.sInstallPath;
478	if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
479	sArchDir = utils.getHostArch();
480	if sArchDir == 'x86': sArchDir = 'i386';
481	self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
482
483	## @todo r=andy Make sure the install root really exists and is accessible.
484
485	# Add the installation root to the PATH on windows so we can get DLLs from it.
486	if utils.getHostOs() == 'win':
487	sPathName = 'PATH';
488	if not sPathName in os.environ:
489	sPathName = 'Path';
490	sPath = os.environ.get(sPathName, '.');
491	if sPath and sPath[-1] != ';':
492	sPath += ';';
493	os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
494	else:
495	reporter.log2('VBox installation root already set to "%s"' % (self.sVBoxInstallRoot));
496
497	reporter.log('VBox installation root path: %s' % (self.sVBoxInstallRoot,));
498
499	self.sVBoxInstallRoot = self._sanitizePath(self.sVBoxInstallRoot);
500
501	#
502	# The unittests are generally not installed, so look for them.
503	#
504	if not self.sUnitTestsPathSrc and self.isRemoteMode():
505	self.sUnitTestsPathSrc = '${CDROM}/testcase/${OS}/${ARCH}';
506
507	elif not self.sUnitTestsPathSrc:
508	sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
509	asCandidates = [
510	self.oBuild.sInstallPath,
511	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), self.oBuild.sType, sBinOrDist),
512	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'release', sBinOrDist),
513	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug', sBinOrDist),
514	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict', sBinOrDist),
515	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt', sBinOrDist),
516	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'profile', sBinOrDist),
517	os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
518	os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
519	os.path.join(self.sScratchPath, sBinOrDist),
520	];
521	if utils.getHostOs() == 'darwin':
522	for i in range(1, len(asCandidates)):
523	asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
524
525	for sCandidat in asCandidates:
526	# The path of tstVMStructSize acts as a beacon to know where all other testcases are.
527	sFileBeacon = os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff);
528	reporter.log2('Searching for "%s" ...' % sFileBeacon);
529	if os.path.exists(sFileBeacon):
530	self.sUnitTestsPathSrc = sCandidat;
531	break
532
533	if self.sUnitTestsPathSrc:
534	reporter.log('Unit test source dir path: ', self.sUnitTestsPathSrc)
535	else:
536	reporter.error('Unable to find unit test source dir. Candidates: %s' % (asCandidates,));
537	if reporter.getVerbosity() >= 2:
538	reporter.log('Contents of "%s"' % self.sScratchPath);
539	for paths, dirs, files in os.walk(self.sScratchPath):
540	reporter.log('{} {} {}'.format(repr(paths), repr(dirs), repr(files)));
541	return False
542
543	else:
544	reporter.log2('Unit test source dir already set to "%s"' % (self.sUnitTestsPathSrc))
545
546	reporter.log('Unit test source dir path: %s' % (self.sUnitTestsPathSrc,));
547
548	self.sUnitTestsPathSrc = self._sanitizePath(self.sUnitTestsPathSrc);
549
550	return True;
551
552	#
553	# Overridden methods.
554	#
555
556	def showUsage(self):
557	"""
558	Shows the testdriver usage.
559	"""
560	fRc = vbox.TestDriver.showUsage(self);
561	reporter.log('');
562	reporter.log('Unit Test #1 options:');
563	reporter.log(' --dryrun');
564	reporter.log(' Performs a dryrun (no tests being executed).');
565	reporter.log(' --mode <local\|remote>');
566	reporter.log(' Specifies the test execution mode:');
567	reporter.log(' local: Locally on the same machine.');
568	reporter.log(' remote: On remote (guest) directly (needs unit test source).');
569	reporter.log(' --only-whitelist');
570	reporter.log(' Only processes the white list.');
571	reporter.log(' --quick');
572	reporter.log(' Very selective testing.');
573	reporter.log(' --unittest-source <dir>');
574	reporter.log(' Sets the unit test source to <dir>.');
575	reporter.log(' Also used for remote execution.');
576	reporter.log(' --vbox-install-root <dir>');
577	reporter.log(' Sets the VBox install root to <dir>.');
578	reporter.log(' Also used for remote execution.');
579	return fRc;
580
581	def parseOption(self, asArgs, iArg):
582	"""
583	Parses the testdriver arguments from the command line.
584	"""
585	if asArgs[iArg] == '--dryrun':
586	self.fDryRun = True;
587	elif asArgs[iArg] == '--mode':
588	iArg += 1;
589	if iArg >= len(asArgs):
590	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
591	if asArgs[iArg] in ('local', 'remote',):
592	self.sMode = asArgs[iArg];
593	else:
594	raise base.InvalidOption('Argument "%s" invalid' % (asArgs[iArg]));
595	elif asArgs[iArg] == '--unittest-source':
596	iArg += 1;
597	if iArg >= len(asArgs):
598	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
599	self.sUnitTestsPathSrc = asArgs[iArg];
600	elif asArgs[iArg] == '--only-whitelist':
601	self.fOnlyWhiteList = True;
602	elif asArgs[iArg] == '--quick':
603	self.fOnlyWhiteList = True;
604	elif asArgs[iArg] == '--vbox-install-root':
605	iArg += 1;
606	if iArg >= len(asArgs):
607	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
608	self.sVBoxInstallRoot = asArgs[iArg];
609	else:
610	return vbox.TestDriver.parseOption(self, asArgs, iArg);
611	return iArg + 1;
612
613	def actionVerify(self):
614	if not self._detectPaths():
615	return False;
616
617	if self.oTestVmSet:
618	return vbox.TestDriver.actionVerify(self);
619
620	return True;
621
622	def actionConfig(self):
623	# Make sure vboxapi has been imported so we can use the constants.
624	reporter.log2('actionConfig started\n')
625	if not self.importVBoxApi():
626	return False
627
628	# Do the configuring.
629	if self.isRemoteMode():
630	if self.sNicAttachment == 'nat': eNic0AttachType = vboxcon.NetworkAttachmentType_NAT;
631	elif self.sNicAttachment == 'bridged': eNic0AttachType = vboxcon.NetworkAttachmentType_Bridged;
632	else: eNic0AttachType = None;
633
634	# Make sure to mount the Validation Kit .ISO so that TxS has the chance
635	# to update itself.
636	#
637	# This is necessary as a lot of our test VMs nowadays have a very old TxS
638	# installed which don't understand commands like uploading files to the guest.
639	# Uploading files is needed for this test driver, however.
640	#
641	## @todo Get rid of this as soon as we create test VMs in a descriptive (automated) manner.
642	return self.oTestVmSet.actionConfig(self, eNic0AttachType = eNic0AttachType,
643	sDvdImage = self.sVBoxValidationKitIso);
644	reporter.log2('actionConfig finished\n')
645
646	return True;
647
648	def actionExecute(self):
649	# Make sure vboxapi has been imported so we can execute the driver without going thru
650	# a former configuring step.
651	reporter.log2('actionExecute started\n')
652	if not self.importVBoxApi():
653	reporter.log2('failed to import VBox API while actionExecute\n')
654	return False;
655	if not self._detectPaths():
656	return False;
657	reporter.log2('Unit test source path is "%s"\n' % self.sUnitTestsPathSrc);
658
659	if not self.sUnitTestsPathDst:
660	self.sUnitTestsPathDst = self.sScratchPath;
661	reporter.log2('Unit test destination path is "%s"\n' % self.sUnitTestsPathDst);
662
663	if self.isRemoteMode(): # Run on a test VM (guest).
664	if self.fpApiVer < 7.0: ## @todo Needs Validation Kit .ISO tweaking (including the unit tests) first.
665	reporter.log('Remote unit tests for non-trunk builds skipped.');
666	fRc = True;
667	else:
668	assert self.oTestVmSet is not None;
669	fRc = self.oTestVmSet.actionExecute(self, self.testOneVmConfig);
670	else: # Run locally (host).
671	self._figureVersion();
672	self._makeEnvironmentChanges();
673
674	# If this is an ASAN build and we're on linux, make sure we've got
675	# libasan.so.N in the LD_LIBRARY_PATH or stuff w/o a RPATH entry
676	# pointing to /opt/VirtualBox will fail (like tstAsmStructs).
677	if self.getBuildType() == 'asan' and utils.getHostOs() in ('linux',):
678	sLdLibraryPath = '';
679	if 'LD_LIBRARY_PATH' in os.environ:
680	sLdLibraryPath = os.environ['LD_LIBRARY_PATH'] + ':';
681	sLdLibraryPath += self.oBuild.sInstallPath;
682	os.environ['LD_LIBRARY_PATH'] = sLdLibraryPath;
683
684	fRc = self._testRunUnitTests(None);
685	reporter.log2('actionExecute finished\n')
686
687	return fRc;
688
689	#
690	# Misc.
691	#
692	def isRemoteMode(self):
693	""" Predicate method for checking if in any remote mode. """
694	return self.sMode.startswith('remote');
695
696	#
697	# Test execution helpers.
698	#
699
700	def _testRunUnitTests(self, oTestVm):
701	"""
702	Main function to execute all unit tests.
703	"""
704
705	# Determine executable suffix based on selected execution mode.
706	if self.isRemoteMode(): # Run on a test VM (guest).
707	if oTestVm.isWindows():
708	self.sExeSuff = '.exe';
709	else:
710	self.sExeSuff = '';
711	else:
712	# For local tests this already is set in __init__
713	pass;
714
715	self._testRunUnitTestsSet(oTestVm, r'^tst*', 'testcase');
716	self._testRunUnitTestsSet(oTestVm, r'^tst*', '.');
717
718	fRc = self.cFailed == 0;
719
720	reporter.log('');
721	if self.fDryRun:
722	reporter.log('*********************************************************');
723	reporter.log('DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN');
724	reporter.log('*********************************************************');
725	reporter.log('*********************************************************');
726	reporter.log(' Target: %s' % (oTestVm.sVmName if oTestVm else 'local',));
727	reporter.log(' Mode: %s' % (self.sMode,));
728	reporter.log(' Exe suffix: %s' % (self.sExeSuff,));
729	reporter.log('Unit tests source: %s %s'
730	% (self.sUnitTestsPathSrc, '(on remote)' if self.isRemoteMode() else '',));
731	reporter.log('VBox install root: %s %s'
732	% (self.sVBoxInstallRoot, '(on remote)' if self.isRemoteMode() else '',));
733	reporter.log('*********************************************************');
734	reporter.log('*** PASSED: %d' % (self.cPassed,));
735	reporter.log('*** FAILED: %d' % (self.cFailed,));
736	reporter.log('*** SKIPPED: %d' % (self.cSkipped,));
737	reporter.log('*** TOTAL: %d' % (self.cPassed + self.cFailed + self.cSkipped,));
738
739	return fRc;
740
741
742	def testOneVmConfig(self, oVM, oTestVm):
743	"""
744	Runs the specified VM thru test #1.
745	"""
746
747	# Simple test.
748	self.logVmInfo(oVM);
749
750	if not self.fDryRun:
751	# Try waiting for a bit longer (5 minutes) until the CD is available to avoid running into timeouts.
752	self.oSession, self.oTxsSession = self.startVmAndConnectToTxsViaTcp(oTestVm.sVmName,
753	fCdWait = not self.fDryRun,
754	cMsCdWait = 5 * 60 * 1000);
755	if self.oSession is None:
756	return False;
757
758	self.addTask(self.oTxsSession);
759
760	# Determine the unit tests destination path.
761	self.sUnitTestsPathDst = oTestVm.pathJoin(self.getGuestTempDir(oTestVm), 'testUnitTests');
762
763	# Run the unit tests.
764	self._testRunUnitTests(oTestVm);
765
766	# Cleanup.
767	if self.oSession is not None:
768	self.removeTask(self.oTxsSession);
769	self.terminateVmBySession(self.oSession);
770	return True;
771
772	#
773	# Test execution helpers.
774	#
775
776	def _figureVersion(self):
777	""" Tries to figure which VBox version this is, setting self.aiVBoxVer. """
778	try:
779	oVBox = self.oVBoxMgr.getVirtualBox();
780	sVer = oVBox.version;
781	sVer += 'r' + str(self.uRevision);
782
783	sVer = sVer.strip();
784	sVer = re.sub(r'_BETA.*r', '.', sVer);
785	sVer = re.sub(r'_ALPHA.*r', '.', sVer);
786	sVer = re.sub(r'_RC.*r', '.', sVer);
787	sVer = re.sub('_SPB', '', sVer)
788	sVer = sVer.replace('r', '.');
789
790	self.aiVBoxVer = [int(sComp) for sComp in sVer.split('.')];
791
792	reporter.log('VBox version: %s' % (self.aiVBoxVer,));
793	except:
794	reporter.logXcpt();
795	return False;
796	return True;
797
798	def _compareVersion(self, aiVer):
799	"""
800	Compares the give version string with the vbox version string,
801	returning a result similar to C strcmp(). aiVer is on the right side.
802	"""
803	cComponents = min(len(self.aiVBoxVer), len(aiVer));
804	for i in range(cComponents):
805	if self.aiVBoxVer[i] < aiVer[i]:
806	return -1;
807	if self.aiVBoxVer[i] > aiVer[i]:
808	return 1;
809	return len(self.aiVBoxVer) - len(aiVer);
810
811	def _isExcluded(self, sTest, dExclList):
812	""" Checks if the testcase is excluded or not. """
813	if sTest in dExclList:
814	sFullExpr = dExclList[sTest].replace(' ', '').strip();
815	if sFullExpr == '':
816	return True;
817
818	# Consider each exclusion expression. These are generally ranges,
819	# either open ended or closed: "<4.3.51r12345", ">=4.3.0 && <=4.3.4".
820	asExprs = sFullExpr.split(';');
821	for sExpr in asExprs:
822
823	# Split it on the and operator and process each sub expression.
824	fResult = True;
825	for sSubExpr in sExpr.split('&&'):
826	# Split out the comparison operator and the version value.
827	if sSubExpr.startswith('<=') or sSubExpr.startswith('>='):
828	sOp = sSubExpr[:2];
829	sValue = sSubExpr[2:];
830	elif sSubExpr.startswith('<') or sSubExpr.startswith('>') or sSubExpr.startswith('='):
831	sOp = sSubExpr[:1];
832	sValue = sSubExpr[1:];
833	else:
834	sOp = sValue = '';
835
836	# Convert the version value, making sure we've got a valid one.
837	try: aiValue = [int(sComp) for sComp in sValue.replace('r', '.').split('.')];
838	except: aiValue = ();
839	if not aiValue or len(aiValue) > 4:
840	reporter.error('Invalid exclusion expression for %s: "%s" [%s]' % (sTest, sSubExpr, dExclList[sTest]));
841	return True;
842
843	# Do the compare.
844	iCmp = self._compareVersion(aiValue);
845	if sOp == '>=' and iCmp < 0:
846	fResult = False;
847	elif sOp == '>' and iCmp <= 0:
848	fResult = False;
849	elif sOp == '<' and iCmp >= 0:
850	fResult = False;
851	elif sOp == '>=' and iCmp < 0:
852	fResult = False;
853	reporter.log2('iCmp=%s; %s %s %s -> %s' % (iCmp, self.aiVBoxVer, sOp, aiValue, fResult));
854
855	# Did the expression match?
856	if fResult:
857	return True;
858
859	return False;
860
861	def _sudoExecuteSync(self, asArgs):
862	"""
863	Executes a sudo child process synchronously.
864	Returns True if the process executed successfully and returned 0,
865	otherwise False is returned.
866	"""
867	reporter.log2('Executing [sudo]: %s' % (asArgs, ));
868	if self.isRemoteMode():
869	iRc = -1; ## @todo Not used remotely yet.
870	else:
871	try:
872	iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False);
873	except:
874	reporter.errorXcpt();
875	return False;
876	reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs));
877	return iRc == 0;
878
879
880	def _logExpandString(self, sString, cVerbosity = 2):
881	"""
882	Expands a given string by asking TxS on the guest side and logs it.
883	Uses log level 2 by default.
884
885	No-op if no TxS involved.
886	"""
887	if reporter.getVerbosity() < cVerbosity or self.oTxsSession is None:
888	return;
889	sStringExp = self.oTxsSession.syncExpandString(sString);
890	if not sStringExp:
891	return;
892	reporter.log2('_logExpandString: "%s" -> "%s"' % (sString, sStringExp));
893
894	def _wrapPathExists(self, sPath):
895	"""
896	Creates the directory specified sPath (including parents).
897	"""
898	reporter.log2('_wrapPathExists: %s' % (sPath,));
899	if self.fDryRun:
900	return True;
901	fRc = False;
902	if self.isRemoteMode():
903	self._logExpandString(sPath);
904	fRc = self.oTxsSession.syncIsDir(sPath, fIgnoreErrors = True);
905	if not fRc:
906	fRc = self.oTxsSession.syncIsFile(sPath, fIgnoreErrors = True);
907	else:
908	fRc = os.path.exists(sPath);
909	return fRc;
910
911	def _wrapMkDir(self, sPath):
912	"""
913	Creates the directory specified sPath (including parents).
914	"""
915	reporter.log2('_wrapMkDir: %s' % (sPath,));
916	if self.fDryRun:
917	return True;
918	fRc = True;
919	if self.isRemoteMode():
920	fRc = self.oTxsSession.syncMkDirPath(sPath, fMode = 0o755);
921	else:
922	if utils.getHostOs() in [ 'win', 'os2' ]:
923	os.makedirs(sPath, 0o755);
924	else:
925	fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
926	if not fRc:
927	reporter.log('Failed to create dir "%s".' % (sPath,));
928	return fRc;
929
930	def _wrapCopyFile(self, sSrc, sDst, iMode):
931	"""
932	Copies a file.
933	"""
934	reporter.log2('_wrapCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
935	if self.fDryRun:
936	return True;
937	fRc = True;
938	if self.isRemoteMode():
939	self._logExpandString(sSrc);
940	self._logExpandString(sDst);
941	if self.isRemoteMode():
942	self.oTxsSession.syncCopyFile(sSrc, sDst, iMode);
943	else:
944	fRc = self.oTxsSession.syncUploadFile(sSrc, sDst);
945	if fRc:
946	fRc = self.oTxsSession.syncChMod(sDst, iMode);
947	else:
948	if utils.getHostOs() in [ 'win', 'os2' ]:
949	utils.copyFileSimple(sSrc, sDst);
950	os.chmod(sDst, iMode);
951	else:
952	fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
953	if fRc:
954	fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
955	if fRc is not True:
956	raise Exception('Failed to chmod "%s".' % (sDst,));
957	if not fRc:
958	reporter.log('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
959	return fRc;
960
961	def _wrapDeleteFile(self, sPath):
962	"""
963	Deletes a file.
964	"""
965	reporter.log2('_wrapDeleteFile: %s' % (sPath,));
966	if self.fDryRun:
967	return True;
968	fRc = True;
969	if self.isRemoteMode():
970	if self.oTxsSession.syncIsFile(sPath):
971	fRc = self.oTxsSession.syncRmFile(sPath, fIgnoreErrors = True);
972	else:
973	if os.path.exists(sPath):
974	if utils.getHostOs() in [ 'win', 'os2' ]:
975	os.remove(sPath);
976	else:
977	fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
978	if not fRc:
979	reporter.log('Failed to remove "%s".' % (sPath,));
980	return fRc;
981
982	def _wrapRemoveDir(self, sPath):
983	"""
984	Removes a directory.
985	"""
986	reporter.log2('_wrapRemoveDir: %s' % (sPath,));
987	if self.fDryRun:
988	return True;
989	fRc = True;
990	if self.isRemoteMode():
991	if self.oTxsSession.syncIsDir(sPath):
992	fRc = self.oTxsSession.syncRmDir(sPath, fIgnoreErrors = True);
993	else:
994	if os.path.exists(sPath):
995	if utils.getHostOs() in [ 'win', 'os2' ]:
996	os.rmdir(sPath);
997	else:
998	fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
999	if not fRc:
1000	reporter.log('Failed to remove "%s".' % (sPath,));
1001	return fRc;
1002
1003	def _executeTestCase(self, oTestVm, sName, sFilePathAbs, sTestCaseSubDir, oDevNull): # pylint: disable=too-many-locals,too-many-statements
1004	"""
1005	Executes a test case.
1006
1007	sFilePathAbs contains the absolute path (including OS-dependent executable suffix) of the testcase.
1008
1009	Returns @c true if testcase was skipped, or @c if not.
1010	"""
1011
1012	fSkipped = False;
1013
1014	#
1015	# If hardening is enabled, some test cases and their dependencies needs
1016	# to be copied to and execute from the source directory in order to
1017	# work. They also have to be executed as root, i.e. via sudo.
1018	#
1019	fHardened = sName in self.kasHardened and self.sUnitTestsPathSrc != self.sVBoxInstallRoot;
1020	asFilesToRemove = []; # Stuff to clean up.
1021	asDirsToRemove = []; # Ditto.
1022
1023	if fHardened or self.isRemoteMode():
1024	if self.isRemoteMode():
1025	sDstDir = os.path.join(self.sUnitTestsPathDst, sTestCaseSubDir);
1026	else:
1027	sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
1028	if not self._wrapPathExists(sDstDir):
1029	self._wrapMkDir(sDstDir);
1030	asDirsToRemove.append(sDstDir);
1031
1032	sSrc = sFilePathAbs;
1033	# If the testcase source does not exist for whatever reason, just mark it as skipped
1034	# instead of reporting an error.
1035	if not self._wrapPathExists(sSrc):
1036	self.cSkipped += 1;
1037	return True;
1038
1039	sDst = os.path.join(sDstDir, os.path.basename(sFilePathAbs));
1040	fModeExe = 0;
1041	fModeDeps = 0;
1042	if not oTestVm or (oTestVm and not oTestVm.isWindows()): ## @todo NT4 does not like the chmod. Investigate this!
1043	fModeExe = 0o755;
1044	fModeDeps = 0o644;
1045	self._wrapCopyFile(sSrc, sDst, fModeExe);
1046	asFilesToRemove.append(sDst);
1047
1048	# Copy required dependencies to destination.
1049	# Note! The testcases are statically linked, so there are no VBoxRT.dll/so/dylib
1050	# to copy here. This code can be currently be ignored.
1051	if self.isRemoteMode():
1052	for sLib in self.kdTestCaseDepsLibs:
1053	for sSuff in [ '.dll', '.so', '.dylib' ]:
1054	assert self.sVBoxInstallRoot is not None;
1055	sSrc = os.path.join(self.sVBoxInstallRoot, sLib + sSuff);
1056	if self._wrapPathExists(sSrc):
1057	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1058	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1059	asFilesToRemove.append(sDst);
1060
1061	## @todo r=bird: The next two are checks for _local_ files matching the remote path when in remote-mode.
1062	## It makes for very confusing reading and is a potential for trouble.
1063
1064	# Copy any associated .dll/.so/.dylib.
1065	for sSuff in [ '.dll', '.so', '.dylib' ]:
1066	sSrc = os.path.splitext(sFilePathAbs)[0] + sSuff;
1067	if os.path.exists(sSrc):
1068	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1069	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1070	asFilesToRemove.append(sDst);
1071
1072	# Copy any associated .r0, .rc and .gc modules.
1073	offDriver = sFilePathAbs.rfind('Driver')
1074	if offDriver > 0:
1075	for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
1076	sSrc = sFilePathAbs[:offDriver] + sSuff;
1077	if os.path.exists(sSrc):
1078	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1079	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1080	asFilesToRemove.append(sDst);
1081
1082	sFilePathAbs = os.path.join(sDstDir, os.path.basename(sFilePathAbs));
1083
1084	#
1085	# Set up arguments.
1086	#
1087	asArgs = [sFilePathAbs,]
1088	if sName in self.kdArguments:
1089	asArgs.extend(self.kdArguments[sName]);
1090
1091	#
1092	# Set up the environment.
1093	#
1094	# - We set IPRT_TEST_OMIT_TOP_TEST to avoid the unnecessary top-test
1095	# entry when running the inner tests, as it'll just add an unnecessary
1096	# result nesting.
1097	#
1098	# - IPRT_TEST_FILE is set to a result.xml file when running locally.
1099	# This is not necessary when executing via TxS as it sets IPRT_TEST_PIPE,
1100	# which overrides IPRT_TEST_FILE, to collect the XML output.
1101	#
1102	dEnvChanges = {
1103	'IPRT_TEST_OMIT_TOP_TEST': '1',
1104	};
1105
1106	sXmlFile = os.path.join(self.sUnitTestsPathDst, 'result.xml') if not self.isRemoteMode() else None;
1107	if sXmlFile:
1108	dEnvChanges['IPRT_TEST_FILE'] = sXmlFile;
1109	if self._wrapPathExists(sXmlFile):
1110	try: os.unlink(sXmlFile);
1111	except: self._wrapDeleteFile(sXmlFile);
1112
1113	#
1114	# Execute the test case.
1115	#
1116	# Windows is confusing output. Trying a few things to get rid of this.
1117	# First, flush both stderr and stdout before running the child. Second,
1118	# assign the child stderr to stdout. If this doesn't help, we'll have
1119	# to capture the child output.
1120	#
1121	reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
1122	try: sys.stdout.flush();
1123	except: pass;
1124	try: sys.stderr.flush();
1125	except: pass;
1126
1127	iRc = 0;
1128
1129	if not self.fDryRun:
1130	if self.isRemoteMode():
1131	asRemoteEnvChg = ['%s=%s' % (sKey, sValue) for sKey, sValue in utils.iteritems(dEnvChanges)];
1132
1133	fRc = self.txsRunTest(self.oTxsSession, sName, cMsTimeout = 30 * 60 * 1000, sExecName = asArgs[0],
1134	asArgs = asArgs, asAddEnv = asRemoteEnvChg, fCheckSessionStatus = True);
1135	if fRc:
1136	iRc = 0;
1137	else:
1138	(_, sOpcode, abPayload) = self.oTxsSession.getLastReply();
1139	if sOpcode.startswith('PROC NOK '): # Extract process rc.
1140	iRc = abPayload[0]; # ASSUMES 8-bit rc for now.
1141	if iRc == 0: # Might happen if the testcase misses some dependencies. Set it to -42 then.
1142	iRc = -42;
1143	else:
1144	iRc = -1; ## @todo
1145	else:
1146	for sKey, sValue in utils.iteritems(dEnvChanges):
1147	os.environ[sKey] = sValue;
1148
1149	oChild = None;
1150	try:
1151	if fHardened:
1152	oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1153	else:
1154	oChild = utils.processPopenSafe(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1155	except:
1156	if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
1157	]:
1158	reporter.logXcpt();
1159	fSkipped = True;
1160	else:
1161	reporter.errorXcpt();
1162	iRc = 1023;
1163	oChild = None;
1164
1165	if oChild is not None:
1166	self.pidFileAdd(oChild.pid, sName, fSudo = fHardened);
1167	iRc = oChild.wait();
1168	self.pidFileRemove(oChild.pid);
1169	#
1170	# Clean up
1171	#
1172	for sPath in asFilesToRemove:
1173	self._wrapDeleteFile(sPath);
1174	for sPath in asDirsToRemove:
1175	self._wrapRemoveDir(sPath);
1176
1177	#
1178	# Report (sXmlFile is None when in remote mode).
1179	#
1180	if sXmlFile and os.path.exists(sXmlFile):
1181	reporter.addSubXmlFile(sXmlFile);
1182	if fHardened:
1183	self._wrapDeleteFile(sXmlFile);
1184	else:
1185	os.unlink(sXmlFile);
1186
1187	if iRc == 0:
1188	reporter.log('*** %s: exit code %d' % (sFilePathAbs, iRc));
1189	self.cPassed += 1;
1190
1191	elif iRc == 4: # RTEXITCODE_SKIPPED
1192	reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFilePathAbs, iRc));
1193	fSkipped = True;
1194	self.cSkipped += 1;
1195
1196	elif fSkipped:
1197	reporter.log('*** %s: exit code %d (Skipped)' % (sFilePathAbs, iRc));
1198	self.cSkipped += 1;
1199
1200	else:
1201	sName = self.kdExitCodeNames.get(iRc, '');
1202	if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
1203	sName = self.kdExitCodeNamesWin[iRc];
1204	if sName != '':
1205	sName = ' (%s)' % (sName);
1206
1207	if iRc != 1:
1208	reporter.testFailure('Exit status: %d%s' % (iRc, sName));
1209	reporter.log( '!*! %s: exit code %d%s' % (sFilePathAbs, iRc, sName));
1210	else:
1211	reporter.error('!*! %s: exit code %d%s' % (sFilePathAbs, iRc, sName));
1212	self.cFailed += 1;
1213
1214	return fSkipped;
1215
1216	def _testRunUnitTestsSet(self, oTestVm, sTestCasePattern, sTestCaseSubDir):
1217	"""
1218	Run subset of the unit tests set.
1219	"""
1220
1221	# Open /dev/null for use as stdin further down.
1222	try:
1223	oDevNull = open(os.path.devnull, 'w+'); # pylint: disable=consider-using-with,unspecified-encoding
1224	except:
1225	oDevNull = None;
1226
1227	# Determin the host OS specific exclusion lists.
1228	dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
1229	dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
1230
1231	## @todo Add filtering for more specific OSes (like OL server, doesn't have X installed) by adding a separate
1232	# black list + using utils.getHostOsVersion().
1233
1234	#
1235	# Process the file list and run everything looking like a testcase.
1236	#
1237	if not self.fOnlyWhiteList:
1238	if not self.isRemoteMode():
1239	asFiles = sorted(os.listdir(os.path.join(self.sUnitTestsPathSrc, sTestCaseSubDir)));
1240	else: # 'remote'
1241	## @todo Implement remote file enumeration / directory listing.
1242	reporter.error('Sorry, no remote file enumeration implemented yet!\nUse --only-whitelist instead.');
1243	return;
1244	else:
1245	# Transform our dict into a list, where the keys are the list elements.
1246	asFiles = list(self.kdTestCasesWhiteList.keys());
1247	# Make sure to only keep the list item's base name so that the iteration down below works
1248	# with our white list without any additional modification.
1249	asFiles = [os.path.basename(s) for s in asFiles];
1250
1251	for sFilename in asFiles:
1252	# When executing in remote execution mode, make sure to append the executable suffix here, as
1253	# the (white / black) lists do not contain any OS-specific executable suffixes.
1254	if self.isRemoteMode():
1255	sFilename = sFilename + self.sExeSuff;
1256	# Separate base and suffix and morph the base into something we
1257	# can use for reporting and array lookups.
1258	sBaseName = os.path.basename(sFilename);
1259	sName, sSuffix = os.path.splitext(sBaseName);
1260	if sTestCaseSubDir != '.':
1261	sName = sTestCaseSubDir + '/' + sName;
1262
1263	reporter.log2('sTestCasePattern=%s, sBaseName=%s, sName=%s, sSuffix=%s, sFileName=%s'
1264	% (sTestCasePattern, sBaseName, sName, sSuffix, sFilename,));
1265
1266	# Process white list first, if set.
1267	if self.fOnlyWhiteList \
1268	and not self._isExcluded(sName, self.kdTestCasesWhiteList):
1269	# (No testStart/Done or accounting here!)
1270	reporter.log('%s: SKIPPED (not in white list)' % (sName,));
1271	continue;
1272
1273	# Basic exclusion.
1274	if not re.match(sTestCasePattern, sBaseName) \
1275	or sSuffix in self.kasSuffixBlackList:
1276	reporter.log2('"%s" is not a test case.' % (sName,));
1277	continue;
1278
1279	# When not only processing the white list, do some more checking first.
1280	if not self.fOnlyWhiteList:
1281	# Check if the testcase is black listed or buggy before executing it.
1282	if self._isExcluded(sName, self.kdTestCasesBlackList):
1283	# (No testStart/Done or accounting here!)
1284	reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
1285	continue;
1286
1287	if self._isExcluded(sName, self.kdTestCasesBuggy):
1288	reporter.testStart(sName);
1289	reporter.log('%s: Skipping, buggy in general.' % (sName,));
1290	reporter.testDone(fSkipped = True);
1291	self.cSkipped += 1;
1292	continue;
1293
1294	# Some testcases don't work with ASAN.
1295	if self.getBuildType() == 'asan' \
1296	and self._isExcluded(sName, self.kdTestCasesBlackListAsan):
1297	# (No testStart/Done or accounting here!)
1298	reporter.log('%s: SKIPPED (blacklisted ASAN)' % (sName,));
1299	continue;
1300
1301	if self._isExcluded(sName, dTestCasesBuggyForHostOs):
1302	reporter.testStart(sName);
1303	reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
1304	reporter.testDone(fSkipped = True);
1305	self.cSkipped += 1;
1306	continue;
1307	else:
1308	# Passed the white list check already above.
1309	pass;
1310
1311	sFilePathAbs = os.path.normpath(os.path.join(self.sUnitTestsPathSrc, os.path.join(sTestCaseSubDir, sFilename)));
1312	reporter.log2('sFilePathAbs=%s\n' % (sFilePathAbs,));
1313	reporter.testStart(sName);
1314	try:
1315	fSkipped = self._executeTestCase(oTestVm, sName, sFilePathAbs, sTestCaseSubDir, oDevNull);
1316	except:
1317	reporter.errorXcpt('!*!');
1318	self.cFailed += 1;
1319	fSkipped = False;
1320	reporter.testDone(fSkipped);
1321
1322
1323	if __name__ == '__main__':
1324	sys.exit(tdUnitTest1().main(sys.argv))

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/tests/unittests/tdUnitTest1.py@ 103389

Download in other formats: