tdUnitTest1.py@ 99675

Last change on this file since 99675 was 99675, checked in by vboxsync, 21 months ago
ValidationKit/tdUnitTest1: Blacklist tstVBoxAPIXPCOM on macOS because it would need to be moved to the installation directory to work actually
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 58.2 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: tdUnitTest1.py 99675 2023-05-08 14:07:04Z vboxsync $
4
5	"""
6	VirtualBox Validation Kit - Unit Tests.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2023 Oracle and/or its affiliates.
12
13	This file is part of VirtualBox base platform packages, as
14	available from https://www.virtualbox.org.
15
16	This program is free software; you can redistribute it and/or
17	modify it under the terms of the GNU General Public License
18	as published by the Free Software Foundation, in version 3 of the
19	License.
20
21	This program is distributed in the hope that it will be useful, but
22	WITHOUT ANY WARRANTY; without even the implied warranty of
23	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24	General Public License for more details.
25
26	You should have received a copy of the GNU General Public License
27	along with this program; if not, see <https://www.gnu.org/licenses>.
28
29	The contents of this file may alternatively be used under the terms
30	of the Common Development and Distribution License Version 1.0
31	(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32	in the VirtualBox distribution, in which case the provisions of the
33	CDDL are applicable instead of those of the GPL.
34
35	You may elect to license modified versions of this file under the
36	terms and conditions of either the GPL or the CDDL or both.
37
38	SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39	"""
40	__version__ = "$Revision: 99675 $"
41
42
43	# Standard Python imports.
44	import os
45	import sys
46	import re
47
48
49	# Only the main script needs to modify the path.
50	try: __file__ # pylint: disable=used-before-assignment
51	except: __file__ = sys.argv[0];
52	g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
53	sys.path.append(g_ksValidationKitDir)
54
55	# Validation Kit imports.
56	from common import utils;
57	from testdriver import base;
58	from testdriver import reporter;
59	from testdriver import vbox;
60	from testdriver import vboxcon;
61
62
63	class tdUnitTest1(vbox.TestDriver):
64	"""
65	Unit Tests.
66	"""
67
68	## The temporary exclude list.
69	## @note This shall be empty before we release 4.3!
70	kdTestCasesBuggyPerOs = {
71	'darwin': {
72	'testcase/tstX86-1': '', # 'FSTP M32R, ST0' fails; no idea why.
73	'testcase/tstLow': '>=7.0.0', # Driverless package.
74	'testcase/tstPin': '>=7.0.0', # Driverless package.
75	'testcase/tstIntNet-1': '>=7.0.0', # Driverless package.
76	'testcase/tstVBoxAPIXPCOM': '', # Can't instantiate the VirtualBox object (binary would need moving to the
77	# VirtualBox installation directory, merely a compile time test anyway)
78	},
79	'darwin.arm64': {
80	'testcase/tstRTDarwinMachKernel': '', # Not supported on arm64 right now (and not required due to driverless).
81	'testcase/tstAsmStructs': '', # Fails on arm64 due to different sizes, also not required as there is no
82	# assembly code which needs to match with structs.
83	'testcase/tstDarwinKeyboard': '', # Fails for unknown reason.
84	'testcase/tstIntNet-1': '', # Not supported on arm64 right now.
85	'testcase/tstLow': '', # Ditto.
86	'testcase/tstPin': '', # Ditto.
87	'testcase/tstRTTime': '', # Needs more work first.
88	},
89	'linux': {
90	'testcase/tstRTFileAio': '', # See xTracker #8035.
91	},
92	'linux.amd64': {
93	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
94	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
95	},
96	'solaris': {
97	'testcase/tstIntNet-1': '', # Fails opening rge0, probably a generic issue figuring which nic to use.
98	'testcase/tstIprtList': '', # Crashes in the multithreaded test, I think.
99	'testcase/tstRTCritSect': '', # Fairness/whatever issue here.
100	'testcase/tstRTR0MemUserKernelDriver': '', # Failes when kernel to kernel buffers.
101	'testcase/tstRTSemRW': '', # line 338: RTSemRWReleaseRead(hSemRW): got VERR_ACCESS_DENIED
102	'testcase/tstRTStrAlloc': '', # VERR_NO_STR_MEMORY!
103	'testcase/tstRTFileQuerySize-1': '', # VERR_DEV_IO_ERROR on /dev/null!
104	},
105	'solaris.amd64': {
106	'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
107	# rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
108	},
109	'win': {
110	'testcase/tstFile': '', # ??
111	'testcase/tstIntNet-1': '', # possibly same issue as solaris.
112	'testcase/tstMouseImpl': '', # STATUS_ACCESS_VIOLATION
113	'testcase/tstRTR0ThreadPreemptionDriver': '', # ??
114	'testcase/tstRTPath': '<4.3.51r89894',
115	'testcase/tstRTPipe': '', # ??
116	'testcase/tstRTR0MemUserKernelDriver': '', # ??
117	'testcase/tstRTR0SemMutexDriver': '', # ??
118	'testcase/tstRTStrAlloc': '', # ??
119	'testcase/tstRTStrFormat': '', # ??
120	'testcase/tstRTSystemQueryOsInfo': '', # ??
121	'testcase/tstRTTemp': '', # ??
122	'testcase/tstRTTime': '', # ??
123	'testcase/tstTime-2': '', # Total time differs too much! ... delta=-10859859
124	'testcase/tstTime-4': '', # Needs to be converted to DLL; ditto for tstTime-2.
125	'testcase/tstUtf8': '', # ??
126	'testcase/tstVMMR0CallHost-2': '', # STATUS_STACK_OVERFLOW
127	'testcase/tstX86-1': '', # Fails on win.x86.
128	'tscpasswd': '', # ??
129	'tstVMREQ': '', # ?? Same as darwin.x86?
130	},
131	'win.x86': {
132	'testcase/tstRTR0TimerDriver': '', # See xTracker #8041.
133	}
134	};
135
136	kdTestCasesBuggy = {
137	'testcase/tstGuestPropSvc': '', # GET_NOTIFICATION fails on testboxlin5.de.oracle.com and others.
138	'testcase/tstTimer': '', # Sometimes fails on linux, not important atm.
139	'testcase/tstGIP-2': '', # 2015-09-10: Fails regularly. E.g. TestSetID 2744205 (testboxsh2),
140	# 2743961 (wei01-b6kc-6). The responsible engineer should reenable
141	# it once it has been fixed.
142	};
143
144	## The permanent exclude list.
145	# @note Stripped of extensions!
146	kdTestCasesBlackList = {
147	'testcase/tstClipboardX11Smoke': '', # (Old naming, deprecated) Needs X, not available on all test boxes.
148	'testcase/tstClipboardGH-X11Smoke': '', # (New name) Ditto.
149	'testcase/tstClipboardMockHGCM': '', # Ditto.
150	'tstClipboardQt': '', # Is interactive and needs Qt, needed for Qt clipboard bugfixing.
151	'testcase/tstClipboardQt': '', # In case it moves here.
152	'tstDragAndDropQt': '', # Is interactive and needs Qt, needed for Qt drag'n drop bugfixing.
153	'testcase/tstDragAndDropQt': '', # In case it moves here.
154	'testcase/tstFileLock': '',
155	'testcase/tstDisasm-2': '', # without parameters it will disassembler 1GB starting from 0
156	'testcase/tstFileAppendWin-1': '',
157	'testcase/tstDir': '', # useless without parameters
158	'testcase/tstDir-2': '', # useless without parameters
159	'testcase/tstGlobalConfig': '',
160	'testcase/tstHostHardwareLinux': '', # must be killed with CTRL-C
161	'testcase/tstHttp': '', # Talks to outside servers.
162	'testcase/tstRTHttp': '', # parameters required
163	'testcase/tstLdr-2': '', # parameters required
164	'testcase/tstLdr-3': '', # parameters required
165	'testcase/tstLdr': '', # parameters required
166	'testcase/tstLdrLoad': '', # parameters required
167	'testcase/tstMove': '', # parameters required
168	'testcase/tstRTR0Timer': '', # loads 'tstRTR0Timer.r0'
169	'testcase/tstRTR0ThreadDriver': '', # loads 'tstRTR0Thread.r0'
170	'testcase/tstRunTestcases': '', # that's a script like this one
171	'testcase/tstRTReqPool': '', # fails sometimes, testcase buggy
172	'testcase/tstRTS3': '', # parameters required
173	'testcase/tstSDL': '', # graphics test
174	'testcase/tstSupLoadModule': '', # Needs parameters and vboxdrv access. Covered elsewhere.
175	'testcase/tstSeamlessX11': '', # graphics test
176	'testcase/tstTime-3': '', # parameters required
177	'testcase/tstVBoxControl': '', # works only inside a guest
178	'testcase/tstVDCopy': '', # parameters required
179	'testcase/tstVDFill': '', # parameters required
180	'tstAnimate': '', # parameters required
181	'testcase/tstAPI': '', # user interaction required
182	'tstCollector': '', # takes forever
183	'testcase/tstHeadless': '', # parameters required
184	'tstHeadless': '', # parameters required
185	'tstMicroRC': '', # required by tstMicro
186	'tstVBoxDbg': '', # interactive test
187	'testcase/tstTestServMgr': '', # some strange xpcom18a4 test, does not work
188	'tstTestServMgr': '', # some strange xpcom18a4 test, does not work
189	'tstPDMAsyncCompletion': '', # parameters required
190	'testcase/tstXptDump': '', # parameters required
191	'tstXptDump': '', # parameters required
192	'testcase/tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
193	'tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
194	'testcase/tstSimpleTypeLib': '', # parameters required
195	'tstSimpleTypeLib': '', # parameters required
196	'testcase/tstTestAtoms': '', # additional test file (words.txt) required
197	'tstTestAtoms': '', # additional test file (words.txt) required
198	'testcase/tstXptLink': '', # parameters required
199	'tstXptLink': '', # parameters required
200	'tstXPCOMCGlue': '', # user interaction required
201	'testcase/tstXPCOMCGlue': '', # user interaction required
202	'testcase/tstCAPIGlue': '', # user interaction required
203	'testcase/tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
204	'tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
205	'testcase/tstRTFilesystem': '', # parameters required
206	'testcase/tstRTDvm': '', # parameters required
207	'tstSSLCertDownloads': '', # Obsolete.
208	# later
209	'testcase/tstIntNetR0': '', # RTSPINLOCK_FLAGS_INTERRUPT_SAFE == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE
210	# slow stuff
211	'testcase/tstAvl': '', # SLOW!
212	'testcase/tstRTAvl': '', # SLOW! (new name)
213	'testcase/tstVD': '', # 8GB fixed-sized vmdk
214	# failed or hang
215	'testcase/tstCryptoPkcs7Verify': '', # hang
216	'tstOVF': '', # hang (only ancient version, now in new place)
217	'testcase/tstRTLockValidator': '', # Lock validation is not enabled for critical sections
218	'testcase/tstGuestControlSvc': '', # failed: line 288: testHost(&svcTable):
219	# expected VINF_SUCCESS, got VERR_NOT_FOUND
220	'testcase/tstRTMemEf': '', # failed w/o error message
221	'testcase/tstSupSem': '', # failed: SRE Timeout Accuracy (ms) : FAILED (1 errors)
222	'testcase/tstCryptoPkcs7Sign': '', # failed: 29330:
223	# error:02001002:lib(2):func(1):reason(2):NA:0:fopen('server.pem': '','r')
224	'testcase/tstCompressionBenchmark': '', # failed: error: RTZipBlockCompress failed
225	# for 'RTZipBlock/LZJB' (#4): VERR_NOT_SUPPORTED
226	'tstPDMAsyncCompletionStress': '', # VERR_INVALID_PARAMETER (cbSize = 0)
227	'tstMicro': '', # doesn't work on solaris, fix later if we care.
228	'tstVMM-HwAccm': '', # failed: Only checked AMD-V on linux
229	'tstVMM-HM': '', # failed: Only checked AMD-V on linux
230	'tstVMMFork': '', # failed: xtracker 6171
231	'tstTestFactory': '', # some strange xpcom18a4 test, does not work
232	'testcase/tstRTSemXRoads': '', # sporadically failed: Traffic - 8 threads per direction, 10 sec :
233	# FAILED (8 errors)
234	'tstVBoxAPILinux': '', # creates VirtualBox directories for root user because of sudo
235	# (should be in vbox)
236	'testcase/tstVMStructDTrace': '', # This is a D-script generator.
237	'tstVMStructRC': '', # This is a C-code generator.
238	'tstDeviceStructSizeRC': '', # This is a C-code generator.
239	'testcase/tstTSC': '', # Doesn't test anything and might fail with HT or/and too many cores.
240	'testcase/tstOpenUSBDev': '', # Not a useful testcase.
241	'testcase/tstX86-1': '', # Really more guest side.
242	'testcase/tstX86-FpuSaveRestore': '', # Experiments, could be useful for the guest not the host.
243	'tstAsmStructsRC': '', # Testcase run during build time (fails to find libstdc++.so.6 on some
244	# Solaris testboxes).
245	};
246
247	# Suffix exclude list.
248	kasSuffixBlackList = [
249	'.r0',
250	'.gc',
251	'.debug',
252	'.rel',
253	'.sys',
254	'.ko',
255	'.o',
256	'.obj',
257	'.lib',
258	'.a',
259	'.so',
260	'.dll',
261	'.dylib',
262	'.tmp',
263	'.log',
264	'.py',
265	'.pyc',
266	'.pyo',
267	'.pdb',
268	'.dSYM',
269	'.sym',
270	'.template',
271	'.expected',
272	'.expect',
273	];
274
275	# White list, which contains tests considered to be safe to execute,
276	# even on remote targets (guests).
277	#
278	# When --only-whitelist is specified, this is the only list being checked for.
279	kdTestCasesWhiteList = {
280	'testcase/tstFile': '',
281	'testcase/tstFileLock': '',
282	'testcase/tstClipboardMockHGCM': '', # Requires X on Linux OSes. Execute on remote targets only (guests).
283	'testcase/tstRTLocalIpc': '',
284	'testcase/tstRTPathQueryInfo': '',
285	'testcase/tstRTPipe': '',
286	'testcase/tstRTProcCreateEx': '',
287	'testcase/tstRTProcCreatePrf': '',
288	'testcase/tstRTProcIsRunningByName': '',
289	'testcase/tstRTProcQueryUsername': '',
290	'testcase/tstRTProcWait': '',
291	'testcase/tstTime-2': '',
292	'testcase/tstTime-3': '',
293	'testcase/tstTime-4': '',
294	'testcase/tstTimer': '',
295	'testcase/tstThread-1': '',
296	'testcase/tstUtf8': ''
297	};
298
299	# Test dependency list -- libraries.
300	# Needed in order to execute testcases on remote targets which don't have a VBox installation present.
301	kdTestCaseDepsLibs = [
302	"VBoxRT"
303	];
304
305	## The exclude list.
306	# @note Stripped extensions!
307	kasHardened = [
308	"testcase/tstIntNet-1",
309	"testcase/tstR0ThreadPreemptionDriver", # VBox 4.3
310	"testcase/tstRTR0ThreadPreemptionDriver",
311	"testcase/tstRTR0MemUserKernelDriver",
312	"testcase/tstRTR0SemMutexDriver",
313	"testcase/tstRTR0TimerDriver",
314	"testcase/tstRTR0ThreadDriver",
315	'testcase/tstRTR0DbgKrnlInfoDriver',
316	"tstInt",
317	"tstPDMQueue", # Comment in testcase says its driverless, but it needs driver access.
318	"tstVMM",
319	"tstVMMFork",
320	"tstVMREQ",
321	'testcase/tstCFGM',
322	'testcase/tstContiguous',
323	'testcase/tstGetPagingMode',
324	'testcase/tstGIP-2',
325	'testcase/tstInit',
326	'testcase/tstLow',
327	'testcase/tstMMHyperHeap',
328	'testcase/tstPage',
329	'testcase/tstPin',
330	'testcase/tstRTTime', 'testcase/tstTime', # GIP test case.
331	'testcase/tstRTTime-2', 'testcase/tstTime-2', # GIP test case.
332	'testcase/tstRTTime-4', 'testcase/tstTime-4', # GIP test case.
333	'testcase/tstSSM',
334	'testcase/tstSupSem-Zombie',
335	]
336
337	## Argument lists
338	kdArguments = {
339	'testcase/tstbntest': [ '-out', os.devnull, ], # Very noisy.
340	};
341
342
343	## Status code translations.
344	## @{
345	kdExitCodeNames = {
346	0: 'RTEXITCODE_SUCCESS',
347	1: 'RTEXITCODE_FAILURE',
348	2: 'RTEXITCODE_SYNTAX',
349	3: 'RTEXITCODE_INIT',
350	4: 'RTEXITCODE_SKIPPED',
351	};
352	kdExitCodeNamesWin = {
353	-1073741515: 'STATUS_DLL_NOT_FOUND',
354	-1073741512: 'STATUS_ORDINAL_NOT_FOUND',
355	-1073741511: 'STATUS_ENTRYPOINT_NOT_FOUND',
356	-1073741502: 'STATUS_DLL_INIT_FAILED',
357	-1073741500: 'STATUS_UNHANDLED_EXCEPTION',
358	-1073741499: 'STATUS_APP_INIT_FAILURE',
359	-1073741819: 'STATUS_ACCESS_VIOLATION',
360	-1073741571: 'STATUS_STACK_OVERFLOW',
361	};
362	## @}
363
364	def __init__(self):
365	"""
366	Reinitialize child class instance.
367	"""
368	vbox.TestDriver.__init__(self);
369
370	# We need to set a default test VM set here -- otherwise the test
371	# driver base class won't let us use the "--test-vms" switch.
372	#
373	# See the "--local" switch in self.parseOption().
374	self.oTestVmSet = self.oTestVmManager.getSmokeVmSet('nat');
375
376	# Selected NIC attachment.
377	self.sNicAttachment = '';
378
379	# Session handling stuff.
380	# Only needed for remote tests executed by TxS.
381	self.oSession = None;
382	self.oTxsSession = None;
383
384	self.sVBoxInstallRoot = None;
385
386	## Testing mode being used:
387	# "local": Execute unit tests locally (same host, default).
388	# "remote-copy": Copies unit tests from host to the remote, then executing it.
389	# "remote-exec": Executes unit tests right on the remote from a given source.
390	## @todo r=bird: 'remote-exec' and 'remote-copy' are confusing. We're presumably executing the test remotely in both
391	## cases, the different being that in the latter case we copy from the valkit iso rather than uploading the test files.
392	## That's hardly clear from the names or the explanation.
393	self.sMode = 'local';
394
395	self.cSkipped = 0;
396	self.cPassed = 0;
397	self.cFailed = 0;
398
399	## The source directory where our unit tests live.
400	# This most likely is our out/ or some staging directory and
401	# also acts the source for copying over the testcases to a remote target.
402	self.sUnitTestsPathSrc = None;
403
404	# The destination directory our unit tests live when being
405	# copied over to a remote target (via TxS).
406	self.sUnitTestsPathDst = None;
407
408	# The executable suffix to use for the executing the actual testcases.
409	# Will be re-set when executing the testcases on a remote (VM) once we know
410	# what type of suffix to use then (based on guest OS).
411	self.sExeSuff = base.exeSuff();
412
413	self.aiVBoxVer = (4, 3, 0, 0);
414
415	# For testing testcase logic.
416	self.fDryRun = False;
417	self.fOnlyWhiteList = False;
418
419	@staticmethod
420	def _sanitizePath(sPath):
421	"""
422	Does a little bit of sanitizing a given path by removing quoting, if any.
423
424	This is needed because handed-in paths via command line arguments can contain variables like "${CDROM}"
425	which might need to get processed by TXS on the guest side first.
426
427	Returns the sanitized path.
428	"""
429	if sPath is None: # Keep uninitialized strings as-is.
430	return None;
431	return sPath.strip('\"').strip('\'');
432
433	def _detectPaths(self):
434	"""
435	Internal worker for actionVerify and actionExecute that detects paths.
436
437	This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
438	"""
439
440	reporter.log2('Detecting paths ...');
441
442	#
443	# We need a VBox install (/ build) to test.
444	#
445	if False is True: ## @todo r=andy ?? # pylint: disable=comparison-of-constants
446	if not self.importVBoxApi():
447	reporter.error('Unabled to import the VBox Python API.');
448	return False;
449	else:
450	self._detectBuild();
451	if self.oBuild is None:
452	reporter.error('Unabled to detect the VBox build.');
453	return False;
454
455	#
456	# Where are the files installed?
457	# Solaris requires special handling because of it's multi arch subdirs.
458	#
459	if not self.sVBoxInstallRoot and self.sMode == 'remote-exec':
460	self.sVBoxInstallRoot = '${CDROM}/${OS}/${ARCH}';
461
462	elif not self.sVBoxInstallRoot:
463	self.sVBoxInstallRoot = self.oBuild.sInstallPath;
464	if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
465	sArchDir = utils.getHostArch();
466	if sArchDir == 'x86': sArchDir = 'i386';
467	self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
468
469	## @todo r=andy Make sure the install root really exists and is accessible.
470
471	# Add the installation root to the PATH on windows so we can get DLLs from it.
472	if utils.getHostOs() == 'win':
473	sPathName = 'PATH';
474	if not sPathName in os.environ:
475	sPathName = 'Path';
476	sPath = os.environ.get(sPathName, '.');
477	if sPath and sPath[-1] != ';':
478	sPath += ';';
479	os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
480	else:
481	reporter.log2('VBox installation root already set to "%s"' % (self.sVBoxInstallRoot));
482
483	self.sVBoxInstallRoot = self._sanitizePath(self.sVBoxInstallRoot);
484
485	#
486	# The unittests are generally not installed, so look for them.
487	#
488	if not self.sUnitTestsPathSrc and self.sMode == 'remote-exec':
489	self.sUnitTestsPathSrc = '${CDROM}/testcase/${OS}/${ARCH}';
490
491	elif not self.sUnitTestsPathSrc:
492	sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
493	asCandidates = [
494	self.oBuild.sInstallPath,
495	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), self.oBuild.sType, sBinOrDist),
496	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'release', sBinOrDist),
497	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug', sBinOrDist),
498	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict', sBinOrDist),
499	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt', sBinOrDist),
500	os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'profile', sBinOrDist),
501	os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
502	os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
503	os.path.join(self.sScratchPath, sBinOrDist),
504	];
505	if utils.getHostOs() == 'darwin':
506	for i in range(1, len(asCandidates)):
507	asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
508
509	for sCandidat in asCandidates:
510	# The path of tstVMStructSize acts as a beacon to know where all other testcases are.
511	sFileBeacon = os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff);
512	reporter.log2('Searching for "%s" ...' % sFileBeacon);
513	if os.path.exists(sFileBeacon):
514	self.sUnitTestsPathSrc = sCandidat;
515	break
516
517	if self.sUnitTestsPathSrc:
518	reporter.log('Unit test source dir path: ', self.sUnitTestsPathSrc)
519	else:
520	reporter.error('Unable to find unit test source dir. Candidates: %s' % (asCandidates,));
521	if reporter.getVerbosity() >= 2:
522	reporter.log('Contents of "%s"' % self.sScratchPath);
523	for paths, dirs, files in os.walk(self.sScratchPath):
524	reporter.log('{} {} {}'.format(repr(paths), repr(dirs), repr(files)));
525	return False
526
527	else:
528	reporter.log2('Unit test source dir already set to "%s"' % (self.sUnitTestsPathSrc))
529	reporter.log('Unit test source dir path: ', self.sUnitTestsPathSrc)
530
531	self.sUnitTestsPathSrc = self._sanitizePath(self.sUnitTestsPathSrc);
532
533	return True;
534
535	#
536	# Overridden methods.
537	#
538
539	def showUsage(self):
540	"""
541	Shows the testdriver usage.
542	"""
543	fRc = vbox.TestDriver.showUsage(self);
544	reporter.log('');
545	reporter.log('Unit Test #1 options:');
546	reporter.log(' --dryrun');
547	reporter.log(' Performs a dryrun (no tests being executed).');
548	reporter.log(' --mode <local\|remote-copy\|remote-exec>');
549	reporter.log(' Specifies the test execution mode:');
550	reporter.log(' local: Locally on the same machine.');
551	reporter.log(' remote-copy: On remote (guest) by copying them from the local source. (BORKED!)');
552	reporter.log(' remote-exec: On remote (guest) directly (needs unit test source).');
553	reporter.log(' --only-whitelist');
554	reporter.log(' Only processes the white list.');
555	reporter.log(' --quick');
556	reporter.log(' Very selective testing.');
557	reporter.log(' --unittest-source <dir>');
558	reporter.log(' Sets the unit test source to <dir>.');
559	reporter.log(' Also used for remote execution.');
560	reporter.log(' --vbox-install-root <dir>');
561	reporter.log(' Sets the VBox install root to <dir>.');
562	reporter.log(' Also used for remote execution.');
563	return fRc;
564
565	def parseOption(self, asArgs, iArg):
566	"""
567	Parses the testdriver arguments from the command line.
568	"""
569	if asArgs[iArg] == '--dryrun':
570	self.fDryRun = True;
571	elif asArgs[iArg] == '--mode':
572	iArg += 1;
573	if iArg >= len(asArgs):
574	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
575	if asArgs[iArg] in ('local', 'remote-copy', 'remote-exec',):
576	self.sMode = asArgs[iArg];
577	else:
578	raise base.InvalidOption('Argument "%s" invalid' % (asArgs[iArg]));
579	elif asArgs[iArg] == '--unittest-source':
580	iArg += 1;
581	if iArg >= len(asArgs):
582	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
583	self.sUnitTestsPathSrc = asArgs[iArg];
584	elif asArgs[iArg] == '--only-whitelist':
585	self.fOnlyWhiteList = True;
586	elif asArgs[iArg] == '--quick':
587	self.fOnlyWhiteList = True;
588	elif asArgs[iArg] == '--vbox-install-root':
589	iArg += 1;
590	if iArg >= len(asArgs):
591	raise base.InvalidOption('Option "%s" needs a value' % (asArgs[iArg - 1]));
592	self.sVBoxInstallRoot = asArgs[iArg];
593	else:
594	return vbox.TestDriver.parseOption(self, asArgs, iArg);
595	return iArg + 1;
596
597	def actionVerify(self):
598	if not self._detectPaths():
599	return False;
600
601	if self.oTestVmSet:
602	return vbox.TestDriver.actionVerify(self);
603
604	return True;
605
606	def actionConfig(self):
607	# Make sure vboxapi has been imported so we can use the constants.
608	if not self.importVBoxApi():
609	return False;
610
611	# Do the configuring.
612	if self.isRemoteMode():
613	if self.sNicAttachment == 'nat': eNic0AttachType = vboxcon.NetworkAttachmentType_NAT;
614	elif self.sNicAttachment == 'bridged': eNic0AttachType = vboxcon.NetworkAttachmentType_Bridged;
615	else: eNic0AttachType = None;
616
617	# Make sure to mount the Validation Kit .ISO so that TxS has the chance
618	# to update itself.
619	#
620	# This is necessary as a lot of our test VMs nowadays have a very old TxS
621	# installed which don't understand commands like uploading files to the guest.
622	# Uploading files is needed for this test driver, however.
623	#
624	## @todo Get rid of this as soon as we create test VMs in a descriptive (automated) manner.
625	return self.oTestVmSet.actionConfig(self, eNic0AttachType = eNic0AttachType,
626	sDvdImage = self.sVBoxValidationKitIso);
627
628	return True;
629
630	def actionExecute(self):
631	# Make sure vboxapi has been imported so we can execute the driver without going thru
632	# a former configuring step.
633	if not self.importVBoxApi():
634	return False;
635	if not self._detectPaths():
636	return False;
637	reporter.log2('Unit test source path is "%s"\n' % self.sUnitTestsPathSrc);
638
639	if not self.sUnitTestsPathDst:
640	self.sUnitTestsPathDst = self.sScratchPath;
641	reporter.log2('Unit test destination path is "%s"\n' % self.sUnitTestsPathDst);
642
643	if self.isRemoteMode(): # Run on a test VM (guest).
644	if self.fpApiVer < 7.0: ## @todo Needs Validation Kit .ISO tweaking (including the unit tests) first.
645	reporter.log('Remote unit tests for non-trunk builds skipped.');
646	fRc = True;
647	else:
648	assert self.oTestVmSet is not None;
649	fRc = self.oTestVmSet.actionExecute(self, self.testOneVmConfig);
650	else: # Run locally (host).
651	self._figureVersion();
652	self._makeEnvironmentChanges();
653
654	# If this is an ASAN build and we're on linux, make sure we've got
655	# libasan.so.N in the LD_LIBRARY_PATH or stuff w/o a RPATH entry
656	# pointing to /opt/VirtualBox will fail (like tstAsmStructs).
657	if self.getBuildType() == 'asan' and utils.getHostOs() in ('linux',):
658	sLdLibraryPath = '';
659	if 'LD_LIBRARY_PATH' in os.environ:
660	sLdLibraryPath = os.environ['LD_LIBRARY_PATH'] + ':';
661	sLdLibraryPath += self.oBuild.sInstallPath;
662	os.environ['LD_LIBRARY_PATH'] = sLdLibraryPath;
663
664	fRc = self._testRunUnitTests(None);
665
666	return fRc;
667
668	#
669	# Misc.
670	#
671	def isRemoteMode(self):
672	""" Predicate method for checking if in any remote mode. """
673	return self.sMode.startswith('remote');
674
675	#
676	# Test execution helpers.
677	#
678
679	def _testRunUnitTests(self, oTestVm):
680	"""
681	Main function to execute all unit tests.
682	"""
683
684	# Determine executable suffix based on selected execution mode.
685	if self.isRemoteMode(): # Run on a test VM (guest).
686	if oTestVm.isWindows():
687	self.sExeSuff = '.exe';
688	else:
689	self.sExeSuff = '';
690	else:
691	# For local tests this already is set in __init__
692	pass;
693
694	self._testRunUnitTestsSet(oTestVm, r'^tst*', 'testcase');
695	self._testRunUnitTestsSet(oTestVm, r'^tst*', '.');
696
697	fRc = self.cFailed == 0;
698
699	reporter.log('');
700	if self.fDryRun:
701	reporter.log('*********************************************************');
702	reporter.log('DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN - DRY RUN');
703	reporter.log('*********************************************************');
704	reporter.log('*********************************************************');
705	reporter.log(' Target: %s' % (oTestVm.sVmName if oTestVm else 'local',));
706	reporter.log(' Mode: %s' % (self.sMode,));
707	reporter.log(' Exe suffix: %s' % (self.sExeSuff,));
708	reporter.log('Unit tests source: %s %s'
709	% (self.sUnitTestsPathSrc, '(on remote)' if self.sMode == 'remote-exec' else '',));
710	reporter.log('VBox install root: %s %s'
711	% (self.sVBoxInstallRoot, '(on remote)' if self.sMode == 'remote-exec' else '',));
712	reporter.log('*********************************************************');
713	reporter.log('*** PASSED: %d' % (self.cPassed,));
714	reporter.log('*** FAILED: %d' % (self.cFailed,));
715	reporter.log('*** SKIPPED: %d' % (self.cSkipped,));
716	reporter.log('*** TOTAL: %d' % (self.cPassed + self.cFailed + self.cSkipped,));
717
718	return fRc;
719
720
721	def testOneVmConfig(self, oVM, oTestVm):
722	"""
723	Runs the specified VM thru test #1.
724	"""
725
726	# Simple test.
727	self.logVmInfo(oVM);
728
729	if not self.fDryRun:
730	# Try waiting for a bit longer (5 minutes) until the CD is available to avoid running into timeouts.
731	self.oSession, self.oTxsSession = self.startVmAndConnectToTxsViaTcp(oTestVm.sVmName,
732	fCdWait = not self.fDryRun,
733	cMsCdWait = 5 * 60 * 1000);
734	if self.oSession is None:
735	return False;
736
737	self.addTask(self.oTxsSession);
738
739	# Determine the unit tests destination path.
740	self.sUnitTestsPathDst = oTestVm.pathJoin(self.getGuestTempDir(oTestVm), 'testUnitTests');
741
742	# Run the unit tests.
743	self._testRunUnitTests(oTestVm);
744
745	# Cleanup.
746	if self.oSession is not None:
747	self.removeTask(self.oTxsSession);
748	self.terminateVmBySession(self.oSession);
749	return True;
750
751	#
752	# Test execution helpers.
753	#
754
755	def _figureVersion(self):
756	""" Tries to figure which VBox version this is, setting self.aiVBoxVer. """
757	try:
758	sVer = utils.processOutputChecked(['VBoxManage', '--version'])
759
760	sVer = sVer.strip();
761	sVer = re.sub(r'_BETA.*r', '.', sVer);
762	sVer = re.sub(r'_ALPHA.*r', '.', sVer);
763	sVer = re.sub(r'_RC.*r', '.', sVer);
764	sVer = re.sub('_SPB', '', sVer)
765	sVer = sVer.replace('r', '.');
766
767	self.aiVBoxVer = [int(sComp) for sComp in sVer.split('.')];
768
769	reporter.log('VBox version: %s' % (self.aiVBoxVer,));
770	except:
771	reporter.logXcpt();
772	return False;
773	return True;
774
775	def _compareVersion(self, aiVer):
776	"""
777	Compares the give version string with the vbox version string,
778	returning a result similar to C strcmp(). aiVer is on the right side.
779	"""
780	cComponents = min(len(self.aiVBoxVer), len(aiVer));
781	for i in range(cComponents):
782	if self.aiVBoxVer[i] < aiVer[i]:
783	return -1;
784	if self.aiVBoxVer[i] > aiVer[i]:
785	return 1;
786	return len(self.aiVBoxVer) - len(aiVer);
787
788	def _isExcluded(self, sTest, dExclList):
789	""" Checks if the testcase is excluded or not. """
790	if sTest in dExclList:
791	sFullExpr = dExclList[sTest].replace(' ', '').strip();
792	if sFullExpr == '':
793	return True;
794
795	# Consider each exclusion expression. These are generally ranges,
796	# either open ended or closed: "<4.3.51r12345", ">=4.3.0 && <=4.3.4".
797	asExprs = sFullExpr.split(';');
798	for sExpr in asExprs:
799
800	# Split it on the and operator and process each sub expression.
801	fResult = True;
802	for sSubExpr in sExpr.split('&&'):
803	# Split out the comparison operator and the version value.
804	if sSubExpr.startswith('<=') or sSubExpr.startswith('>='):
805	sOp = sSubExpr[:2];
806	sValue = sSubExpr[2:];
807	elif sSubExpr.startswith('<') or sSubExpr.startswith('>') or sSubExpr.startswith('='):
808	sOp = sSubExpr[:1];
809	sValue = sSubExpr[1:];
810	else:
811	sOp = sValue = '';
812
813	# Convert the version value, making sure we've got a valid one.
814	try: aiValue = [int(sComp) for sComp in sValue.replace('r', '.').split('.')];
815	except: aiValue = ();
816	if not aiValue or len(aiValue) > 4:
817	reporter.error('Invalid exclusion expression for %s: "%s" [%s]' % (sTest, sSubExpr, dExclList[sTest]));
818	return True;
819
820	# Do the compare.
821	iCmp = self._compareVersion(aiValue);
822	if sOp == '>=' and iCmp < 0:
823	fResult = False;
824	elif sOp == '>' and iCmp <= 0:
825	fResult = False;
826	elif sOp == '<' and iCmp >= 0:
827	fResult = False;
828	elif sOp == '>=' and iCmp < 0:
829	fResult = False;
830	reporter.log2('iCmp=%s; %s %s %s -> %s' % (iCmp, self.aiVBoxVer, sOp, aiValue, fResult));
831
832	# Did the expression match?
833	if fResult:
834	return True;
835
836	return False;
837
838	def _sudoExecuteSync(self, asArgs):
839	"""
840	Executes a sudo child process synchronously.
841	Returns True if the process executed successfully and returned 0,
842	otherwise False is returned.
843	"""
844	reporter.log2('Executing [sudo]: %s' % (asArgs, ));
845	if self.isRemoteMode():
846	iRc = -1; ## @todo Not used remotely yet.
847	else:
848	try:
849	iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False);
850	except:
851	reporter.errorXcpt();
852	return False;
853	reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs));
854	return iRc == 0;
855
856
857	def _logExpandString(self, sString, cVerbosity = 2):
858	"""
859	Expands a given string by asking TxS on the guest side and logs it.
860	Uses log level 2 by default.
861
862	No-op if no TxS involved.
863	"""
864	if reporter.getVerbosity() < cVerbosity or self.oTxsSession is None:
865	return;
866	sStringExp = self.oTxsSession.syncExpandString(sString);
867	if not sStringExp:
868	return;
869	reporter.log2('_logExpandString: "%s" -> "%s"' % (sString, sStringExp));
870
871	def _wrapPathExists(self, sPath):
872	"""
873	Creates the directory specified sPath (including parents).
874	"""
875	reporter.log2('_wrapPathExists: %s' % (sPath,));
876	if self.fDryRun:
877	return True;
878	fRc = False;
879	if self.isRemoteMode():
880	self._logExpandString(sPath);
881	fRc = self.oTxsSession.syncIsDir(sPath, fIgnoreErrors = True);
882	if not fRc:
883	fRc = self.oTxsSession.syncIsFile(sPath, fIgnoreErrors = True);
884	else:
885	fRc = os.path.exists(sPath);
886	return fRc;
887
888	def _wrapMkDir(self, sPath):
889	"""
890	Creates the directory specified sPath (including parents).
891	"""
892	reporter.log2('_wrapMkDir: %s' % (sPath,));
893	if self.fDryRun:
894	return True;
895	fRc = True;
896	if self.isRemoteMode():
897	fRc = self.oTxsSession.syncMkDirPath(sPath, fMode = 0o755);
898	else:
899	if utils.getHostOs() in [ 'win', 'os2' ]:
900	os.makedirs(sPath, 0o755);
901	else:
902	fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
903	if not fRc:
904	reporter.log('Failed to create dir "%s".' % (sPath,));
905	return fRc;
906
907	def _wrapCopyFile(self, sSrc, sDst, iMode):
908	"""
909	Copies a file.
910	"""
911	reporter.log2('_wrapCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
912	if self.fDryRun:
913	return True;
914	fRc = True;
915	if self.isRemoteMode():
916	self._logExpandString(sSrc);
917	self._logExpandString(sDst);
918	if self.sMode == 'remote-exec':
919	self.oTxsSession.syncCopyFile(sSrc, sDst, iMode);
920	else:
921	fRc = self.oTxsSession.syncUploadFile(sSrc, sDst);
922	if fRc:
923	fRc = self.oTxsSession.syncChMod(sDst, iMode);
924	else:
925	if utils.getHostOs() in [ 'win', 'os2' ]:
926	utils.copyFileSimple(sSrc, sDst);
927	os.chmod(sDst, iMode);
928	else:
929	fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
930	if fRc:
931	fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
932	if fRc is not True:
933	raise Exception('Failed to chmod "%s".' % (sDst,));
934	if not fRc:
935	reporter.log('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
936	return fRc;
937
938	def _wrapDeleteFile(self, sPath):
939	"""
940	Deletes a file.
941	"""
942	reporter.log2('_wrapDeleteFile: %s' % (sPath,));
943	if self.fDryRun:
944	return True;
945	fRc = True;
946	if self.isRemoteMode():
947	if self.oTxsSession.syncIsFile(sPath):
948	fRc = self.oTxsSession.syncRmFile(sPath, fIgnoreErrors = True);
949	else:
950	if os.path.exists(sPath):
951	if utils.getHostOs() in [ 'win', 'os2' ]:
952	os.remove(sPath);
953	else:
954	fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
955	if not fRc:
956	reporter.log('Failed to remove "%s".' % (sPath,));
957	return fRc;
958
959	def _wrapRemoveDir(self, sPath):
960	"""
961	Removes a directory.
962	"""
963	reporter.log2('_wrapRemoveDir: %s' % (sPath,));
964	if self.fDryRun:
965	return True;
966	fRc = True;
967	if self.isRemoteMode():
968	if self.oTxsSession.syncIsDir(sPath):
969	fRc = self.oTxsSession.syncRmDir(sPath, fIgnoreErrors = True);
970	else:
971	if os.path.exists(sPath):
972	if utils.getHostOs() in [ 'win', 'os2' ]:
973	os.rmdir(sPath);
974	else:
975	fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
976	if not fRc:
977	reporter.log('Failed to remove "%s".' % (sPath,));
978	return fRc;
979
980	def _executeTestCase(self, oTestVm, sName, sFilePathAbs, sTestCaseSubDir, oDevNull): # pylint: disable=too-many-locals,too-many-statements
981	"""
982	Executes a test case.
983
984	sFilePathAbs contains the absolute path (including OS-dependent executable suffix) of the testcase.
985
986	Returns @c true if testcase was skipped, or @c if not.
987	"""
988
989	fSkipped = False;
990
991	#
992	# If hardening is enabled, some test cases and their dependencies needs
993	# to be copied to and execute from the source directory in order to
994	# work. They also have to be executed as root, i.e. via sudo.
995	#
996	# When executing test remotely we must also copy stuff over to the
997	# remote location. Currently there is no diferent between remote-copy
998	# and remote-exec here, which it would be nice if Andy could explain...
999	#
1000	## @todo r=bird: Please explain + fix ^^.
1001	fHardened = sName in self.kasHardened and self.sUnitTestsPathSrc != self.sVBoxInstallRoot;
1002	asFilesToRemove = []; # Stuff to clean up.
1003	asDirsToRemove = []; # Ditto.
1004
1005	if fHardened or self.isRemoteMode():
1006	if self.isRemoteMode():
1007	sDstDir = os.path.join(self.sUnitTestsPathDst, sTestCaseSubDir);
1008	else:
1009	sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
1010	if not self._wrapPathExists(sDstDir):
1011	self._wrapMkDir(sDstDir);
1012	asDirsToRemove.append(sDstDir);
1013
1014	sSrc = sFilePathAbs;
1015	# If the testcase source does not exist for whatever reason, just mark it as skipped
1016	# instead of reporting an error.
1017	if not self._wrapPathExists(sSrc): ## @todo r=bird: This doesn't add up for me with the two remote modes.
1018	self.cSkipped += 1; ## It seems to presuppose that we're in remote-exec mode as _wrapPathExists
1019	fSkipped = True; ## does not differentiate between the two remote modes and will always check
1020	return fSkipped; ## the path on the remote side.
1021
1022	sDst = os.path.join(sDstDir, os.path.basename(sFilePathAbs));
1023	fModeExe = 0;
1024	fModeDeps = 0;
1025	if not oTestVm or (oTestVm and not oTestVm.isWindows()): ## @todo NT4 does not like the chmod. Investigate this!
1026	fModeExe = 0o755;
1027	fModeDeps = 0o644;
1028	self._wrapCopyFile(sSrc, sDst, fModeExe);
1029	asFilesToRemove.append(sDst);
1030
1031	# Copy required dependencies to destination .
1032	# Note! The testcases are statically linked, so there are no VBoxRT.dll/so/dylib
1033	# to copy here. This code can be currently be ignored.
1034	if self.isRemoteMode():
1035	for sLib in self.kdTestCaseDepsLibs:
1036	for sSuff in [ '.dll', '.so', '.dylib' ]:
1037	assert self.sVBoxInstallRoot is not None;
1038	sSrc = os.path.join(self.sVBoxInstallRoot, sLib + sSuff);
1039	if self._wrapPathExists(sSrc):
1040	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1041	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1042	asFilesToRemove.append(sDst);
1043
1044	## @todo r=bird: The next two are check for _local_ files matching the remote path when in remote-mode.
1045	## It makes for very confusing reading and is a potential for trouble.
1046
1047	# Copy any associated .dll/.so/.dylib.
1048	for sSuff in [ '.dll', '.so', '.dylib' ]:
1049	sSrc = os.path.splitext(sFilePathAbs)[0] + sSuff;
1050	if os.path.exists(sSrc):
1051	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1052	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1053	asFilesToRemove.append(sDst);
1054
1055	# Copy any associated .r0, .rc and .gc modules.
1056	offDriver = sFilePathAbs.rfind('Driver')
1057	if offDriver > 0:
1058	for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
1059	sSrc = sFilePathAbs[:offDriver] + sSuff;
1060	if os.path.exists(sSrc):
1061	sDst = os.path.join(sDstDir, os.path.basename(sSrc));
1062	self._wrapCopyFile(sSrc, sDst, fModeDeps);
1063	asFilesToRemove.append(sDst);
1064
1065	sFilePathAbs = os.path.join(sDstDir, os.path.basename(sFilePathAbs));
1066
1067	#
1068	# Set up arguments.
1069	#
1070	asArgs = [sFilePathAbs,]
1071	if sName in self.kdArguments:
1072	asArgs.extend(self.kdArguments[sName]);
1073
1074	#
1075	# Set up the environment.
1076	#
1077	# - We set IPRT_TEST_OMIT_TOP_TEST to avoid the unnecessary top-test
1078	# entry when running the inner tests, as it'll just add an unnecessary
1079	# result nesting.
1080	#
1081	# - IPRT_TEST_FILE is set to a result.xml file when running locally.
1082	# This is not necessary when executing via TxS as it sets IPRT_TEST_PIPE,
1083	# which overrides IPRT_TEST_FILE, to collect the XML output.
1084	#
1085	dEnvChanges = {
1086	'IPRT_TEST_OMIT_TOP_TEST': '1',
1087	};
1088
1089	sXmlFile = os.path.join(self.sUnitTestsPathDst, 'result.xml') if not self.isRemoteMode() else None;
1090	if sXmlFile:
1091	dEnvChanges['IPRT_TEST_FILE'] = sXmlFile;
1092	if self._wrapPathExists(sXmlFile):
1093	try: os.unlink(sXmlFile);
1094	except: self._wrapDeleteFile(sXmlFile);
1095
1096	#
1097	# Execute the test case.
1098	#
1099	# Windows is confusing output. Trying a few things to get rid of this.
1100	# First, flush both stderr and stdout before running the child. Second,
1101	# assign the child stderr to stdout. If this doesn't help, we'll have
1102	# to capture the child output.
1103	#
1104	reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
1105	try: sys.stdout.flush();
1106	except: pass;
1107	try: sys.stderr.flush();
1108	except: pass;
1109
1110	iRc = 0;
1111
1112	if not self.fDryRun:
1113	if self.isRemoteMode():
1114	asRemoteEnvChg = ['%s=%s' % (sKey, sValue) for sKey, sValue in utils.iteritems(dEnvChanges)];
1115
1116	fRc = self.txsRunTest(self.oTxsSession, sName, cMsTimeout = 30 * 60 * 1000, sExecName = asArgs[0],
1117	asArgs = asArgs, asAddEnv = asRemoteEnvChg, fCheckSessionStatus = True);
1118	if fRc:
1119	iRc = 0;
1120	else:
1121	(_, sOpcode, abPayload) = self.oTxsSession.getLastReply();
1122	if sOpcode.startswith('PROC NOK '): # Extract process rc.
1123	iRc = abPayload[0]; # ASSUMES 8-bit rc for now.
1124	if iRc == 0: # Might happen if the testcase misses some dependencies. Set it to -42 then.
1125	iRc = -42;
1126	else:
1127	iRc = -1; ## @todo
1128	else:
1129	for sKey, sValue in utils.iteritems(dEnvChanges):
1130	os.environ[sKey] = sValue;
1131
1132	oChild = None;
1133	try:
1134	if fHardened:
1135	oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1136	else:
1137	oChild = utils.processPopenSafe(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
1138	except:
1139	if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
1140	]:
1141	reporter.logXcpt();
1142	fSkipped = True;
1143	else:
1144	reporter.errorXcpt();
1145	iRc = 1023;
1146	oChild = None;
1147
1148	if oChild is not None:
1149	self.pidFileAdd(oChild.pid, sName, fSudo = fHardened);
1150	iRc = oChild.wait();
1151	self.pidFileRemove(oChild.pid);
1152	#
1153	# Clean up
1154	#
1155	for sPath in asFilesToRemove:
1156	self._wrapDeleteFile(sPath);
1157	for sPath in asDirsToRemove:
1158	self._wrapRemoveDir(sPath);
1159
1160	#
1161	# Report (sXmlFile is None when in remote mode).
1162	#
1163	if sXmlFile and os.path.exists(sXmlFile):
1164	reporter.addSubXmlFile(sXmlFile);
1165	if fHardened:
1166	self._wrapDeleteFile(sXmlFile);
1167	else:
1168	os.unlink(sXmlFile);
1169
1170	if iRc == 0:
1171	reporter.log('*** %s: exit code %d' % (sFilePathAbs, iRc));
1172	self.cPassed += 1;
1173
1174	elif iRc == 4: # RTEXITCODE_SKIPPED
1175	reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFilePathAbs, iRc));
1176	fSkipped = True;
1177	self.cSkipped += 1;
1178
1179	elif fSkipped:
1180	reporter.log('*** %s: exit code %d (Skipped)' % (sFilePathAbs, iRc));
1181	self.cSkipped += 1;
1182
1183	else:
1184	sName = self.kdExitCodeNames.get(iRc, '');
1185	if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
1186	sName = self.kdExitCodeNamesWin[iRc];
1187	if sName != '':
1188	sName = ' (%s)' % (sName);
1189
1190	if iRc != 1:
1191	reporter.testFailure('Exit status: %d%s' % (iRc, sName));
1192	reporter.log( '!*! %s: exit code %d%s' % (sFilePathAbs, iRc, sName));
1193	else:
1194	reporter.error('!*! %s: exit code %d%s' % (sFilePathAbs, iRc, sName));
1195	self.cFailed += 1;
1196
1197	return fSkipped;
1198
1199	def _testRunUnitTestsSet(self, oTestVm, sTestCasePattern, sTestCaseSubDir):
1200	"""
1201	Run subset of the unit tests set.
1202	"""
1203
1204	# Open /dev/null for use as stdin further down.
1205	try:
1206	oDevNull = open(os.path.devnull, 'w+'); # pylint: disable=consider-using-with,unspecified-encoding
1207	except:
1208	oDevNull = None;
1209
1210	# Determin the host OS specific exclusion lists.
1211	dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
1212	dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
1213
1214	## @todo Add filtering for more specific OSes (like OL server, doesn't have X installed) by adding a separate
1215	# black list + using utils.getHostOsVersion().
1216
1217	#
1218	# Process the file list and run everything looking like a testcase.
1219	#
1220	if not self.fOnlyWhiteList:
1221	if self.sMode in ('local', 'remote-copy'):
1222	asFiles = sorted(os.listdir(os.path.join(self.sUnitTestsPathSrc, sTestCaseSubDir)));
1223	else: # 'remote-exec'
1224	## @todo Implement remote file enumeration / directory listing.
1225	reporter.error('Sorry, no remote file enumeration implemented yet!\nUse --only-whitelist instead.');
1226	return;
1227	else:
1228	# Transform our dict into a list, where the keys are the list elements.
1229	asFiles = list(self.kdTestCasesWhiteList.keys());
1230	# Make sure to only keep the list item's base name so that the iteration down below works
1231	# with our white list without any additional modification.
1232	asFiles = [os.path.basename(s) for s in asFiles];
1233
1234	for sFilename in asFiles:
1235	# When executing in remote execution mode, make sure to append the executable suffix here, as
1236	# the (white / black) lists do not contain any OS-specific executable suffixes.
1237	if self.sMode == 'remote-exec':
1238	sFilename = sFilename + self.sExeSuff;
1239	# Separate base and suffix and morph the base into something we
1240	# can use for reporting and array lookups.
1241	sBaseName = os.path.basename(sFilename);
1242	sName, sSuffix = os.path.splitext(sBaseName);
1243	if sTestCaseSubDir != '.':
1244	sName = sTestCaseSubDir + '/' + sName;
1245
1246	reporter.log2('sTestCasePattern=%s, sBaseName=%s, sName=%s, sSuffix=%s, sFileName=%s'
1247	% (sTestCasePattern, sBaseName, sName, sSuffix, sFilename,));
1248
1249	# Process white list first, if set.
1250	if self.fOnlyWhiteList \
1251	and not self._isExcluded(sName, self.kdTestCasesWhiteList):
1252	# (No testStart/Done or accounting here!)
1253	reporter.log('%s: SKIPPED (not in white list)' % (sName,));
1254	continue;
1255
1256	# Basic exclusion.
1257	if not re.match(sTestCasePattern, sBaseName) \
1258	or sSuffix in self.kasSuffixBlackList:
1259	reporter.log2('"%s" is not a test case.' % (sName,));
1260	continue;
1261
1262	# When not only processing the white list, do some more checking first.
1263	if not self.fOnlyWhiteList:
1264	# Check if the testcase is black listed or buggy before executing it.
1265	if self._isExcluded(sName, self.kdTestCasesBlackList):
1266	# (No testStart/Done or accounting here!)
1267	reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
1268	continue;
1269
1270	if self._isExcluded(sName, self.kdTestCasesBuggy):
1271	reporter.testStart(sName);
1272	reporter.log('%s: Skipping, buggy in general.' % (sName,));
1273	reporter.testDone(fSkipped = True);
1274	self.cSkipped += 1;
1275	continue;
1276
1277	if self._isExcluded(sName, dTestCasesBuggyForHostOs):
1278	reporter.testStart(sName);
1279	reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
1280	reporter.testDone(fSkipped = True);
1281	self.cSkipped += 1;
1282	continue;
1283	else:
1284	# Passed the white list check already above.
1285	pass;
1286
1287	sFilePathAbs = os.path.normpath(os.path.join(self.sUnitTestsPathSrc, os.path.join(sTestCaseSubDir, sFilename)));
1288	reporter.log2('sFilePathAbs=%s\n' % (sFilePathAbs,));
1289	reporter.testStart(sName);
1290	try:
1291	fSkipped = self._executeTestCase(oTestVm, sName, sFilePathAbs, sTestCaseSubDir, oDevNull);
1292	except:
1293	reporter.errorXcpt('!*!');
1294	self.cFailed += 1;
1295	fSkipped = False;
1296	reporter.testDone(fSkipped);
1297
1298
1299	if __name__ == '__main__':
1300	sys.exit(tdUnitTest1().main(sys.argv))

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/tests/unittests/tdUnitTest1.py@ 99675

Download in other formats: