VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/tests/unittests/tdUnitTest1.py@ 93842

Last change on this file since 93842 was 93842, checked in by vboxsync, 3 years ago

Validation Kit: Extended unittest test driver so that it also can install + execute unittests on remote targets (i.e. older / unsupported guests). To continue running testcases locally, the "--local" switch has to specified. Currently using the smoke test set (NAT) by default. bugref:10195
  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 43.8 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: tdUnitTest1.py 93842 2022-02-18 14:25:05Z vboxsync $
4
5"""
6VirtualBox Validation Kit - Unit Tests.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2010-2022 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 93842 $"
31
32
33# Standard Python imports.
34import os
35import sys
36import re
37
38
39# Only the main script needs to modify the path.
40try: __file__
41except: __file__ = sys.argv[0];
42g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
43sys.path.append(g_ksValidationKitDir)
44
45# Validation Kit imports.
46from common import utils;
47from testdriver import vbox
48from testdriver import reporter
49
50
51class tdUnitTest1(vbox.TestDriver):
52 """
53 Unit Tests.
54 """
55
56 ## The temporary exclude list.
57 ## @note This shall be empty before we release 4.3!
58 kdTestCasesBuggyPerOs = {
59 'darwin': {
60 'testcase/tstX86-1': '', # 'FSTP M32R, ST0' fails; no idea why.
61 },
62 'linux': {
63 'testcase/tstRTFileAio': '', # See xTracker #8035.
64 },
65 'linux.amd64': {
66 'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
67 # rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
68 },
69 'solaris': {
70 'testcase/tstIntNet-1': '', # Fails opening rge0, probably a generic issue figuring which nic to use.
71 'testcase/tstIprtList': '', # Crashes in the multithreaded test, I think.
72 'testcase/tstRTCritSect': '', # Fairness/whatever issue here.
73 'testcase/tstRTR0MemUserKernelDriver': '', # Failes when kernel to kernel buffers.
74 'testcase/tstRTSemRW': '', # line 338: RTSemRWReleaseRead(hSemRW): got VERR_ACCESS_DENIED
75 'testcase/tstRTStrAlloc': '', # VERR_NO_STR_MEMORY!
76 'testcase/tstRTFileQuerySize-1': '', # VERR_DEV_IO_ERROR on /dev/null!
77 },
78 'solaris.amd64': {
79 'testcase/tstLdr-4': '', # failed: Failed to get bits for '/home/vbox/test/tmp/bin/testcase/tstLdrObjR0.r0'/0,
80 # rc=VERR_SYMBOL_VALUE_TOO_BIG. aborting test
81 },
82 'win': {
83 'testcase/tstFile': '', # ??
84 'testcase/tstIntNet-1': '', # possibly same issue as solaris.
85 'testcase/tstMouseImpl': '', # STATUS_ACCESS_VIOLATION
86 'testcase/tstRTR0ThreadPreemptionDriver': '', # ??
87 'testcase/tstRTPath': '<4.3.51r89894',
88 'testcase/tstRTPipe': '', # ??
89 'testcase/tstRTR0MemUserKernelDriver': '', # ??
90 'testcase/tstRTR0SemMutexDriver': '', # ??
91 'testcase/tstRTStrAlloc': '', # ??
92 'testcase/tstRTStrFormat': '', # ??
93 'testcase/tstRTSystemQueryOsInfo': '', # ??
94 'testcase/tstRTTemp': '', # ??
95 'testcase/tstRTTime': '', # ??
96 'testcase/tstTime-2': '', # Total time differs too much! ... delta=-10859859
97 'testcase/tstTime-4': '', # Needs to be converted to DLL; ditto for tstTime-2.
98 'testcase/tstUtf8': '', # ??
99 'testcase/tstVMMR0CallHost-2': '', # STATUS_STACK_OVERFLOW
100 'testcase/tstX86-1': '', # Fails on win.x86.
101 'tscpasswd': '', # ??
102 'tstVMREQ': '', # ?? Same as darwin.x86?
103 },
104 'win.x86': {
105 'testcase/tstRTR0TimerDriver': '', # See xTracker #8041.
106 }
107 };
108
109 kdTestCasesBuggy = {
110 'testcase/tstGuestPropSvc': '', # GET_NOTIFICATION fails on testboxlin5.de.oracle.com and others.
111 'testcase/tstRTProcCreateEx': '', # Seen failing on wei01-b6ka-9.de.oracle.com.
112 'testcase/tstTimer': '', # Sometimes fails on linux, not important atm.
113 'testcase/tstGIP-2': '', # 2015-09-10: Fails regularly. E.g. TestSetID 2744205 (testboxsh2),
114 # 2743961 (wei01-b6kc-6). The responsible engineer should reenable
115 # it once it has been fixed.
116 };
117
118 ## The permanent exclude list.
119 # @note Stripped of extensions!
120 kdTestCasesBlackList = {
121 'testcase/tstClipboardX11Smoke': '', # (Old naming, deprecated) Needs X, not available on all test boxes.
122 'testcase/tstClipboardGH-X11Smoke': '', # (New name) Ditto.
123 'tstClipboardQt': '', # Is interactive and needs Qt, needed for Qt clipboard bugfixing.
124 'testcase/tstClipboardQt': '', # In case it moves here.
125 'testcase/tstFileLock': '',
126 'testcase/tstDisasm-2': '', # without parameters it will disassembler 1GB starting from 0
127 'testcase/tstFileAppendWin-1': '',
128 'testcase/tstDir': '', # useless without parameters
129 'testcase/tstDir-2': '', # useless without parameters
130 'testcase/tstGlobalConfig': '',
131 'testcase/tstHostHardwareLinux': '', # must be killed with CTRL-C
132 'testcase/tstHttp': '', # Talks to outside servers.
133 'testcase/tstRTHttp': '', # parameters required
134 'testcase/tstLdr-2': '', # parameters required
135 'testcase/tstLdr-3': '', # parameters required
136 'testcase/tstLdr': '', # parameters required
137 'testcase/tstLdrLoad': '', # parameters required
138 'testcase/tstMove': '', # parameters required
139 'testcase/tstRTR0Timer': '', # loads 'tstRTR0Timer.r0'
140 'testcase/tstRTR0ThreadDriver': '', # loads 'tstRTR0Thread.r0'
141 'testcase/tstRunTestcases': '', # that's a script like this one
142 'testcase/tstRTReqPool': '', # fails sometimes, testcase buggy
143 'testcase/tstRTS3': '', # parameters required
144 'testcase/tstSDL': '', # graphics test
145 'testcase/tstSupLoadModule': '', # Needs parameters and vboxdrv access. Covered elsewhere.
146 'testcase/tstSeamlessX11': '', # graphics test
147 'testcase/tstTime-3': '', # parameters required
148 'testcase/tstVBoxControl': '', # works only inside a guest
149 'testcase/tstVDCopy': '', # parameters required
150 'testcase/tstVDFill': '', # parameters required
151 'tstAnimate': '', # parameters required
152 'testcase/tstAPI': '', # user interaction required
153 'tstCollector': '', # takes forever
154 'testcase/tstHeadless': '', # parameters required
155 'tstHeadless': '', # parameters required
156 'tstMicroRC': '', # required by tstMicro
157 'tstVBoxDbg': '', # interactive test
158 'testcase/tstTestServMgr': '', # some strange xpcom18a4 test, does not work
159 'tstTestServMgr': '', # some strange xpcom18a4 test, does not work
160 'tstPDMAsyncCompletion': '', # parameters required
161 'testcase/tstXptDump': '', # parameters required
162 'tstXptDump': '', # parameters required
163 'testcase/tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
164 'tstnsIFileEnumerator': '', # some strange xpcom18a4 test, does not work
165 'testcase/tstSimpleTypeLib': '', # parameters required
166 'tstSimpleTypeLib': '', # parameters required
167 'testcase/tstTestAtoms': '', # additional test file (words.txt) required
168 'tstTestAtoms': '', # additional test file (words.txt) required
169 'testcase/tstXptLink': '', # parameters required
170 'tstXptLink': '', # parameters required
171 'tstXPCOMCGlue': '', # user interaction required
172 'testcase/tstXPCOMCGlue': '', # user interaction required
173 'testcase/tstCAPIGlue': '', # user interaction required
174 'testcase/tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
175 'tstTestCallTemplates': '', # some strange xpcom18a4 test, segfaults
176 'testcase/tstRTFilesystem': '', # parameters required
177 'testcase/tstRTDvm': '', # parameters required
178 'tstSSLCertDownloads': '', # Obsolete.
179 # later
180 'testcase/tstIntNetR0': '', # RTSPINLOCK_FLAGS_INTERRUPT_SAFE == RTSPINLOCK_FLAGS_INTERRUPT_UNSAFE
181 # slow stuff
182 'testcase/tstAvl': '', # SLOW!
183 'testcase/tstRTAvl': '', # SLOW! (new name)
184 'testcase/tstVD': '', # 8GB fixed-sized vmdk
185 # failed or hang
186 'testcase/tstCryptoPkcs7Verify': '', # hang
187 'tstOVF': '', # hang (only ancient version, now in new place)
188 'testcase/tstOVF': '', # Creates mess when fails, needs to be run in a separate test.
189 'testcase/tstRTLockValidator': '', # Lock validation is not enabled for critical sections
190 'testcase/tstGuestControlSvc': '', # failed: line 288: testHost(&svcTable):
191 # expected VINF_SUCCESS, got VERR_NOT_FOUND
192 'testcase/tstRTMemEf': '', # failed w/o error message
193 'testcase/tstSupSem': '', # failed: SRE Timeout Accuracy (ms) : FAILED (1 errors)
194 'testcase/tstCryptoPkcs7Sign': '', # failed: 29330:
195 # error:02001002:lib(2):func(1):reason(2):NA:0:fopen('server.pem': '','r')
196 'testcase/tstCompressionBenchmark': '', # failed: error: RTZipBlockCompress failed
197 # for 'RTZipBlock/LZJB' (#4): VERR_NOT_SUPPORTED
198 'tstPDMAsyncCompletionStress': '', # VERR_INVALID_PARAMETER (cbSize = 0)
199 'tstMicro': '', # doesn't work on solaris, fix later if we care.
200 'tstVMM-HwAccm': '', # failed: Only checked AMD-V on linux
201 'tstVMM-HM': '', # failed: Only checked AMD-V on linux
202 'tstVMMFork': '', # failed: xtracker 6171
203 'tstTestFactory': '', # some strange xpcom18a4 test, does not work
204 'testcase/tstRTSemXRoads': '', # sporadically failed: Traffic - 8 threads per direction, 10 sec :
205 # FAILED (8 errors)
206 'tstVBoxAPILinux': '', # creates VirtualBox directories for root user because of sudo
207 # (should be in vbox)
208 'testcase/tstVMStructDTrace': '', # This is a D-script generator.
209 'tstVMStructRC': '', # This is a C-code generator.
210 'tstDeviceStructSizeRC': '', # This is a C-code generator.
211 'testcase/tstTSC': '', # Doesn't test anything and might fail with HT or/and too many cores.
212 'testcase/tstOpenUSBDev': '', # Not a useful testcase.
213 'testcase/tstX86-1': '', # Really more guest side.
214 'testcase/tstX86-FpuSaveRestore': '', # Experiments, could be useful for the guest not the host.
215 'tstAsmStructsRC': '', # Testcase run during build time (fails to find libstdc++.so.6 on some
216 # Solaris testboxes).
217 };
218
219 # Suffix exclude list.
220 kasSuffixBlackList = [
221 '.r0',
222 '.gc',
223 '.debug',
224 '.rel',
225 '.sys',
226 '.ko',
227 '.o',
228 '.obj',
229 '.lib',
230 '.a',
231 '.so',
232 '.dll',
233 '.dylib',
234 '.tmp',
235 '.log',
236 '.py',
237 '.pyc',
238 '.pyo',
239 '.pdb',
240 '.dSYM',
241 '.sym',
242 '.template',
243 '.expected',
244 '.expect',
245 ];
246
247 # White list, which contains tests considered to be safe to execute,
248 # even on remote targets (guests).
249 kdTestCasesWhiteList = {
250 'testcase/tstFile': '',
251 'testcase/tstFileLock': '',
252 'testcase/tstRTLocalIpc': '',
253 'testcase/tstRTPathQueryInfo': '',
254 'testcase/tstRTPipe': '',
255 'testcase/tstRTProcCreateEx': '',
256 'testcase/tstRTProcCreatePrf': '',
257 'testcase/tstRTProcIsRunningByName': '',
258 'testcase/tstRTProcQueryUsername': '',
259 'testcase/tstRTProcWait': '',
260 'testcase/tstTime-2': '',
261 'testcase/tstTime-3': '',
262 'testcase/tstTime-4': '',
263 'testcase/tstTimer': '',
264 'testcase/tstThread-1': '',
265 'testcase/tstUtf8': '',
266 }
267
268 # Test dependency list -- libraries.
269 # Needed in order to execute testcases on remote targets which don't have a VBox installation present.
270 kdTestCaseDepsLibs = [
271 "VBoxRT"
272 ];
273
274 ## The exclude list.
275 # @note Stripped extensions!
276 kasHardened = [
277 "testcase/tstIntNet-1",
278 "testcase/tstR0ThreadPreemptionDriver", # VBox 4.3
279 "testcase/tstRTR0ThreadPreemptionDriver",
280 "testcase/tstRTR0MemUserKernelDriver",
281 "testcase/tstRTR0SemMutexDriver",
282 "testcase/tstRTR0TimerDriver",
283 "testcase/tstRTR0ThreadDriver",
284 'testcase/tstRTR0DbgKrnlInfoDriver',
285 "tstInt",
286 "tstVMM",
287 "tstVMMFork",
288 "tstVMREQ",
289 'testcase/tstCFGM',
290 'testcase/tstContiguous',
291 'testcase/tstGetPagingMode',
292 'testcase/tstGIP-2',
293 'testcase/tstInit',
294 'testcase/tstLow',
295 'testcase/tstMMHyperHeap',
296 'testcase/tstPage',
297 'testcase/tstPin',
298 'testcase/tstRTTime', 'testcase/tstTime', # GIP test case.
299 'testcase/tstRTTime-2', 'testcase/tstTime-2', # GIP test case.
300 'testcase/tstRTTime-4', 'testcase/tstTime-4', # GIP test case.
301 'testcase/tstSSM',
302 'testcase/tstSupSem-Zombie',
303 ]
304
305 ## Argument lists
306 kdArguments = {
307 'testcase/tstbntest': [ '-out', os.devnull, ], # Very noisy.
308 };
309
310
311 ## Status code translations.
312 ## @{
313 kdExitCodeNames = {
314 0: 'RTEXITCODE_SUCCESS',
315 1: 'RTEXITCODE_FAILURE',
316 2: 'RTEXITCODE_SYNTAX',
317 3: 'RTEXITCODE_INIT',
318 4: 'RTEXITCODE_SKIPPED',
319 };
320 kdExitCodeNamesWin = {
321 -1073741515: 'STATUS_DLL_NOT_FOUND',
322 -1073741512: 'STATUS_ORDINAL_NOT_FOUND',
323 -1073741511: 'STATUS_ENTRYPOINT_NOT_FOUND',
324 -1073741502: 'STATUS_DLL_INIT_FAILED',
325 -1073741500: 'STATUS_UNHANDLED_EXCEPTION',
326 -1073741499: 'STATUS_APP_INIT_FAILURE',
327 -1073741819: 'STATUS_ACCESS_VIOLATION',
328 -1073741571: 'STATUS_STACK_OVERFLOW',
329 };
330 ## @}
331
332 def __init__(self):
333 """
334 Reinitialize child class instance.
335 """
336 vbox.TestDriver.__init__(self);
337
338 # We need to set a default test VM set here -- otherwise the test
339 # driver base class won't let us use the "--test-vms" switch.
340 #
341 # See the "--local" switch in self.parseOption().
342 self.oTestVmSet = self.oTestVmManager.getSmokeVmSet('nat');
343
344 # Session handling stuff.
345 # Only needed for remote tests executed by TxS.
346 self.oSession = None;
347 self.oTxsSession = None;
348
349 self.sVBoxInstallRoot = None;
350
351 self.cSkipped = 0;
352 self.cPassed = 0;
353 self.cFailed = 0;
354
355 # The source directory where our unit tests live.
356 # This most likely is our out/ or some staging directory and
357 # also acts the source for copying over the testcases to a remote target.
358 self.sUnitTestsPathSrc = None;
359
360 # Array of environment variables with NAME=VAL entries
361 # to be applied for testcases.
362 #
363 # This is also needed for testcases which are being executed remotely.
364 self.asEnv = [];
365
366 # The destination directory our unit tests live when being
367 # copied over to a remote target (via TxS).
368 self.sUnitTestsPathDst = None;
369
370 self.sExeSuff = '.exe' if utils.getHostOs() in [ 'win', 'dos', 'os2' ] else '';
371
372 self.aiVBoxVer = (4, 3, 0, 0);
373
374 # For testing testcase logic.
375 self.fDryRun = False;
376 self.fOnlyWhiteList = False;
377
378
379 def _detectPaths(self):
380 """
381 Internal worker for actionVerify and actionExecute that detects paths.
382
383 This sets sVBoxInstallRoot and sUnitTestsPathBase and returns True/False.
384 """
385
386 #
387 # We need a VBox install (/ build) to test.
388 #
389 if False is True: ## @todo r=andy WTF?
390 if not self.importVBoxApi():
391 reporter.error('Unabled to import the VBox Python API.')
392 return False
393 else:
394 self._detectBuild();
395 if self.oBuild is None:
396 reporter.error('Unabled to detect the VBox build.');
397 return False;
398
399 #
400 # Where are the files installed?
401 # Solaris requires special handling because of it's multi arch subdirs.
402 #
403 self.sVBoxInstallRoot = self.oBuild.sInstallPath
404 if not self.oBuild.isDevBuild() and utils.getHostOs() == 'solaris':
405 sArchDir = utils.getHostArch();
406 if sArchDir == 'x86': sArchDir = 'i386';
407 self.sVBoxInstallRoot = os.path.join(self.sVBoxInstallRoot, sArchDir);
408
409 # Add the installation root to the PATH on windows so we can get DLLs from it.
410 if utils.getHostOs() == 'win':
411 sPathName = 'PATH';
412 if not sPathName in os.environ:
413 sPathName = 'Path';
414 sPath = os.environ.get(sPathName, '.');
415 if sPath and sPath[-1] != ';':
416 sPath += ';';
417 os.environ[sPathName] = sPath + self.sVBoxInstallRoot + ';';
418
419 #
420 # The unittests are generally not installed, so look for them.
421 #
422 sBinOrDist = 'dist' if utils.getHostOs() in [ 'darwin', ] else 'bin';
423 asCandidates = [
424 self.oBuild.sInstallPath,
425 os.path.join(self.sScratchPath, utils.getHostOsDotArch(), self.oBuild.sType, sBinOrDist),
426 os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'release', sBinOrDist),
427 os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'debug', sBinOrDist),
428 os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'strict', sBinOrDist),
429 os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'dbgopt', sBinOrDist),
430 os.path.join(self.sScratchPath, utils.getHostOsDotArch(), 'profile', sBinOrDist),
431 os.path.join(self.sScratchPath, sBinOrDist + '.' + utils.getHostArch()),
432 os.path.join(self.sScratchPath, sBinOrDist, utils.getHostArch()),
433 os.path.join(self.sScratchPath, sBinOrDist),
434 ];
435 if utils.getHostOs() == 'darwin':
436 for i in range(1, len(asCandidates)):
437 asCandidates[i] = os.path.join(asCandidates[i], 'VirtualBox.app', 'Contents', 'MacOS');
438
439 for sCandidat in asCandidates:
440 if os.path.exists(os.path.join(sCandidat, 'testcase', 'tstVMStructSize' + self.sExeSuff)):
441 self.sUnitTestsPathSrc = sCandidat;
442 return True;
443
444 reporter.error('Unable to find unit test dir. Candidates: %s' % (asCandidates,))
445 return False;
446
447 #
448 # Overridden methods.
449 #
450
451 def showUsage(self):
452 """
453 Shows the testdriver usage.
454 """
455 fRc = vbox.TestDriver.showUsage(self);
456 reporter.log('');
457 reporter.log('Unit Test #1 options:');
458 reporter.log(' --dryrun');
459 reporter.log(' Performs a dryrun (no tests being executed).');
460 reporter.log(' --local');
461 reporter.log(' Runs unit tests locally (this host).');
462 reporter.log(' --only-whitelist');
463 reporter.log(' Only processes the white list.');
464 reporter.log(' --quick');
465 reporter.log(' Very selective testing.');
466 return fRc;
467
468 def parseOption(self, asArgs, iArg):
469 """
470 Parses the testdriver arguments from the command line.
471 """
472 if asArgs[iArg] == '--dryrun':
473 self.fDryRun = True;
474 elif asArgs[iArg] == '--local':
475 # Disable the test VM set and only run stuff locally.
476 self.oTestVmSet = None;
477 elif asArgs[iArg] == '--only-whitelist':
478 self.fOnlyWhiteList = True;
479 elif asArgs[iArg] == '--quick':
480 self.fOnlyWhiteList = True;
481 else:
482 return vbox.TestDriver.parseOption(self, asArgs, iArg);
483 return iArg + 1;
484
485 def actionVerify(self):
486 if not self._detectPaths():
487 return False;
488
489 if self.oTestVmSet:
490 return vbox.TestDriver.actionVerify(self);
491
492 return True;
493
494 def actionConfig(self):
495 # Make sure vboxapi has been imported so we can use the constants.
496 if not self.importVBoxApi():
497 return False;
498
499 # Do the configuring.
500 if self.oTestVmSet:
501 if self.sNicAttachment == 'nat': eNic0AttachType = vboxcon.NetworkAttachmentType_NAT;
502 elif self.sNicAttachment == 'bridged': eNic0AttachType = vboxcon.NetworkAttachmentType_Bridged;
503 else: eNic0AttachType = None;
504 return self.oTestVmSet.actionConfig(self, eNic0AttachType = eNic0AttachType);
505
506 return True;
507
508 def actionExecute(self):
509 if self.sUnitTestsPathSrc is None and not self._detectPaths():
510 return False;
511
512 if not self.sUnitTestsPathDst:
513 self.sUnitTestsPathDst = self.sScratchPath;
514 reporter.log('Unit test destination path is "%s"\n' % self.sUnitTestsPathDst);
515
516 if self.oTestVmSet: # Run on a test VM (guest).
517 fRc = self.oTestVmSet.actionExecute(self, self.testOneVmConfig);
518 else: # Run locally (host).
519 self._figureVersion();
520 self._makeEnvironmentChanges();
521 fRc = self._testRunUnitTests(None);
522
523 return fRc;
524
525 #
526 # Test execution helpers.
527 #
528
529 def _testRunUnitTests(self, oTestVm):
530 """
531 Main function to execute all unit tests.
532 """
533 self._testRunUnitTestsSet(r'^tst*', 'testcase');
534 self._testRunUnitTestsSet(r'^tst*', '.');
535
536 fRc = self.cFailed == 0;
537
538 reporter.log('');
539 reporter.log('********************');
540 reporter.log('Target: %s' % (oTestVm.sVmName if oTestVm else 'local'));
541 reporter.log('********************');
542 reporter.log('*** PASSED: %d' % self.cPassed);
543 reporter.log('*** FAILED: %d' % self.cFailed);
544 reporter.log('*** SKIPPED: %d' % self.cSkipped);
545 reporter.log('*** TOTAL: %d' % (self.cPassed + self.cFailed + self.cSkipped));
546
547 return fRc;
548
549
550 def testOneVmConfig(self, oVM, oTestVm):
551 """
552 Runs the specified VM thru test #1.
553 """
554
555 # Simple test.
556 self.logVmInfo(oVM);
557 # Try waiting for a bit longer (5 minutes) until the CD is available to avoid running into timeouts.
558 self.oSession, self.oTxsSession = self.startVmAndConnectToTxsViaTcp(oTestVm.sVmName, fCdWait = False);
559 if self.oSession is not None:
560 self.addTask(self.oTxsSession);
561
562 # Determine the unit tests destination path.
563 self.sUnitTestsPathDst = oTestVm.pathJoin(self.getGuestTempDir(oTestVm), 'testUnitTests');
564
565 # Run the unit tests.
566 self._testRunUnitTests(oTestVm);
567
568 # Cleanup.
569 self.removeTask(self.oTxsSession);
570 self.terminateVmBySession(self.oSession);
571 return True;
572
573 return False;
574
575 #
576 # Test execution helpers.
577 #
578
579 def _figureVersion(self):
580 """ Tries to figure which VBox version this is, setting self.aiVBoxVer. """
581 try:
582 sVer = utils.processOutputChecked(['VBoxManage', '--version'])
583
584 sVer = sVer.strip();
585 sVer = re.sub(r'_BETA.*r', '.', sVer);
586 sVer = re.sub(r'_ALPHA.*r', '.', sVer);
587 sVer = re.sub(r'_RC.*r', '.', sVer);
588 sVer = re.sub('_SPB', '', sVer)
589 sVer = sVer.replace('r', '.');
590
591 self.aiVBoxVer = [int(sComp) for sComp in sVer.split('.')];
592
593 reporter.log('VBox version: %s' % (self.aiVBoxVer,));
594 except:
595 reporter.logXcpt();
596 return False;
597 return True;
598
599 def _compareVersion(self, aiVer):
600 """
601 Compares the give version string with the vbox version string,
602 returning a result similar to C strcmp(). aiVer is on the right side.
603 """
604 cComponents = min(len(self.aiVBoxVer), len(aiVer));
605 for i in range(cComponents):
606 if self.aiVBoxVer[i] < aiVer[i]:
607 return -1;
608 if self.aiVBoxVer[i] > aiVer[i]:
609 return 1;
610 return len(self.aiVBoxVer) - len(aiVer);
611
612 def _isExcluded(self, sTest, dExclList):
613 """ Checks if the testcase is excluded or not. """
614 if sTest in dExclList:
615 sFullExpr = dExclList[sTest].replace(' ', '').strip();
616 if sFullExpr == '':
617 return True;
618
619 # Consider each exclusion expression. These are generally ranges,
620 # either open ended or closed: "<4.3.51r12345", ">=4.3.0 && <=4.3.4".
621 asExprs = sFullExpr.split(';');
622 for sExpr in asExprs:
623
624 # Split it on the and operator and process each sub expression.
625 fResult = True;
626 for sSubExpr in sExpr.split('&&'):
627 # Split out the comparison operator and the version value.
628 if sSubExpr.startswith('<=') or sSubExpr.startswith('>='):
629 sOp = sSubExpr[:2];
630 sValue = sSubExpr[2:];
631 elif sSubExpr.startswith('<') or sSubExpr.startswith('>') or sSubExpr.startswith('='):
632 sOp = sSubExpr[:1];
633 sValue = sSubExpr[1:];
634 else:
635 sOp = sValue = '';
636
637 # Convert the version value, making sure we've got a valid one.
638 try: aiValue = [int(sComp) for sComp in sValue.replace('r', '.').split('.')];
639 except: aiValue = ();
640 if not aiValue or len(aiValue) > 4:
641 reporter.error('Invalid exclusion expression for %s: "%s" [%s]' % (sTest, sSubExpr, dExclList[sTest]));
642 return True;
643
644 # Do the compare.
645 iCmp = self._compareVersion(aiValue);
646 if sOp == '>=' and iCmp < 0:
647 fResult = False;
648 elif sOp == '>' and iCmp <= 0:
649 fResult = False;
650 elif sOp == '<' and iCmp >= 0:
651 fResult = False;
652 elif sOp == '>=' and iCmp < 0:
653 fResult = False;
654 reporter.log2('iCmp=%s; %s %s %s -> %s' % (iCmp, self.aiVBoxVer, sOp, aiValue, fResult));
655
656 # Did the expression match?
657 if fResult:
658 return True;
659
660 return False;
661
662 def _sudoExecuteSync(self, asArgs):
663 """
664 Executes a sudo child process synchronously.
665 Returns True if the process executed successfully and returned 0,
666 otherwise False is returned.
667 """
668 reporter.log('Executing [sudo]: %s' % (asArgs, ));
669 if self.oTestVmSet:
670 iRc = -1; ## @todo Not used remotely yet.
671 else:
672 try:
673 iRc = utils.sudoProcessCall(asArgs, shell = False, close_fds = False);
674 except:
675 reporter.errorXcpt();
676 return False;
677 reporter.log('Exit code [sudo]: %s (%s)' % (iRc, asArgs));
678 return iRc == 0;
679
680 def _hardenedPathExists(self, sPath):
681 """
682 Creates the directory specified sPath (including parents).
683 """
684 reporter.log('_hardenedPathExists: %s' % (sPath,));
685 fRc = False;
686 if self.oTestVmSet:
687 fRc = self.txsIsDir(self.oSession, self.oTxsSession, sPath, fIgnoreErrors = True);
688 if not fRc:
689 fRc = self.txsIsFile(self.oSession, self.oTxsSession, sPath, fIgnoreErrors = True);
690 else:
691 fRc = os.path.exists(sPath);
692 return fRc;
693
694 def _hardenedMkDir(self, sPath):
695 """
696 Creates the directory specified sPath (including parents).
697 """
698 reporter.log('_hardenedMkDir: %s' % (sPath,));
699 fRc = True;
700 if self.oTestVmSet:
701 fRc = self.txsMkDirPath(self.oSession, self.oTxsSession, sPath, fMode = 0o755);
702 else:
703 if utils.getHostOs() in [ 'win', 'os2' ]:
704 os.makedirs(sPath, 0o755);
705 else:
706 fRc = self._sudoExecuteSync(['/bin/mkdir', '-p', '-m', '0755', sPath]);
707 if fRc is not True:
708 raise Exception('Failed to create dir "%s".' % (sPath,));
709 return True;
710
711 def _hardenedCopyFile(self, sSrc, sDst, iMode):
712 """
713 Copies a file.
714 """
715 reporter.log('_hardenedCopyFile: %s -> %s (mode: %o)' % (sSrc, sDst, iMode,));
716 fRc = True;
717 if self.oTestVmSet:
718 fRc = self.txsUploadFile(self.oSession, self.oTxsSession, sSrc, sDst);
719 if fRc:
720 self.oTxsSession.syncChMod(sDst, 0o755);
721 else:
722 if utils.getHostOs() in [ 'win', 'os2' ]:
723 utils.copyFileSimple(sSrc, sDst);
724 os.chmod(sDst, iMode);
725 else:
726 fRc = self._sudoExecuteSync(['/bin/cp', sSrc, sDst]);
727 if fRc:
728 fRc = self._sudoExecuteSync(['/bin/chmod', '%o' % (iMode,), sDst]);
729 if fRc is not True:
730 raise Exception('Failed to chmod "%s".' % (sDst,));
731 if fRc is not True:
732 raise Exception('Failed to copy "%s" to "%s".' % (sSrc, sDst,));
733 return True;
734
735 def _hardenedDeleteFile(self, sPath):
736 """
737 Deletes a file.
738 """
739 reporter.log('_hardenedDeleteFile: %s' % (sPath,));
740 fRc = True;
741 if self.oTestVmSet:
742 if self.txsIsFile(self.oSession, self.oTxsSession, sPath):
743 fRc = self.txsRmFile(self.oSession, self.oTxsSession, sPath);
744 else:
745 if os.path.exists(sPath):
746 if utils.getHostOs() in [ 'win', 'os2' ]:
747 os.remove(sPath);
748 else:
749 fRc = self._sudoExecuteSync(['/bin/rm', sPath]);
750 if fRc is not True:
751 raise Exception('Failed to remove "%s".' % (sPath,));
752 return True;
753
754 def _hardenedRemoveDir(self, sPath):
755 """
756 Removes a directory.
757 """
758 reporter.log('_hardenedRemoveDir: %s' % (sPath,));
759 fRc = True;
760 if self.oTestVmSet:
761 if self.txsIsDir(self.oSession, self.oTxsSession, sPath):
762 fRc = self.txsRmDir(self.oSession, self.oTxsSession, sPath);
763 else:
764 if os.path.exists(sPath):
765 if utils.getHostOs() in [ 'win', 'os2' ]:
766 os.rmdir(sPath);
767 else:
768 fRc = self._sudoExecuteSync(['/bin/rmdir', sPath]);
769 if fRc is not True:
770 raise Exception('Failed to remove "%s".' % (sPath,));
771 return True;
772
773 def _envSet(self, sName, sValue):
774 if self.oTestVmSet:
775 # For remote execution we cache the environment block and pass it
776 # right when the process execution happens.
777 self.asEnv.append([ sName, sValue ]);
778 else:
779 os.environ[sName] = sValue;
780 return True;
781
782 def _executeTestCase(self, sName, sFullPath, sTestCaseSubDir, oDevNull): # pylint: disable=too-many-locals
783 """
784 Executes a test case.
785 """
786
787 fSkipped = False;
788
789 #
790 # If hardening is enabled, some test cases and their dependencies
791 # needs to be copied to and execute from the sVBoxInstallRoot
792 # directory in order to work. They also have to be executed as
793 # root, i.e. via sudo.
794 #
795 fHardened = False;
796 fToRemote = False;
797 fCopyDeps = False;
798 asFilesToRemove = []; # Stuff to clean up.
799 asDirsToRemove = []; # Ditto.
800
801 if sName in self.kasHardened \
802 and self.sUnitTestsPathSrc != self.sVBoxInstallRoot:
803 fHardened = True;
804
805 if self.oTestVmSet:
806 fToRemote = True;
807 fCopyDeps = True;
808
809 if fHardened \
810 or fToRemote:
811 if fToRemote:
812 sDstDir = os.path.join(self.sUnitTestsPathDst, sTestCaseSubDir);
813 else:
814 sDstDir = os.path.join(self.sVBoxInstallRoot, sTestCaseSubDir);
815 if not self._hardenedPathExists(sDstDir):
816 self._hardenedMkDir(sDstDir);
817 asDirsToRemove.append(sDstDir);
818
819 sDst = os.path.join(sDstDir, os.path.basename(sFullPath));
820 self._hardenedCopyFile(sFullPath, sDst, 0o755);
821 asFilesToRemove.append(sDst);
822
823 # Copy required dependencies to destination.
824 if fCopyDeps:
825 for sLib in self.kdTestCaseDepsLibs:
826 for sSuff in [ '.dll', '.so', '.dylib' ]:
827 sSrc = os.path.join(self.sVBoxInstallRoot, sLib + sSuff);
828 if os.path.exists(sSrc):
829 sDst = os.path.join(sDstDir, os.path.basename(sSrc));
830 self._hardenedCopyFile(sSrc, sDst, 0o644);
831 asFilesToRemove.append(sDst);
832
833 # Copy any associated .dll/.so/.dylib.
834 for sSuff in [ '.dll', '.so', '.dylib' ]:
835 sSrc = os.path.splitext(sFullPath)[0] + sSuff;
836 if os.path.exists(sSrc):
837 sDst = os.path.join(sDstDir, os.path.basename(sSrc));
838 self._hardenedCopyFile(sSrc, sDst, 0o644);
839 asFilesToRemove.append(sDst);
840
841 # Copy any associated .r0, .rc and .gc modules.
842 offDriver = sFullPath.rfind('Driver')
843 if offDriver > 0:
844 for sSuff in [ '.r0', 'RC.rc', 'RC.gc' ]:
845 sSrc = sFullPath[:offDriver] + sSuff;
846 if os.path.exists(sSrc):
847 sDst = os.path.join(sDstDir, os.path.basename(sSrc));
848 self._hardenedCopyFile(sSrc, sDst, 0o644);
849 asFilesToRemove.append(sDst);
850
851 sFullPath = os.path.join(sDstDir, os.path.basename(sFullPath));
852
853 #
854 # Set up arguments and environment.
855 #
856 asArgs = [sFullPath,]
857 if sName in self.kdArguments:
858 asArgs.extend(self.kdArguments[sName]);
859
860 sXmlFile = os.path.join(self.sUnitTestsPathDst, 'result.xml');
861
862 self._envSet('IPRT_TEST_OMIT_TOP_TEST', '1');
863 self._envSet('IPRT_TEST_FILE', sXmlFile);
864
865 if self._hardenedPathExists(sXmlFile):
866 try: os.unlink(sXmlFile);
867 except: self._hardenedDeleteFile(sXmlFile);
868
869 #
870 # Execute the test case.
871 #
872 # Windows is confusing output. Trying a few things to get rid of this.
873 # First, flush both stderr and stdout before running the child. Second,
874 # assign the child stderr to stdout. If this doesn't help, we'll have
875 # to capture the child output.
876 #
877 reporter.log('*** Executing %s%s...' % (asArgs, ' [hardened]' if fHardened else ''));
878 try: sys.stdout.flush();
879 except: pass;
880 try: sys.stderr.flush();
881 except: pass;
882 if not self.fDryRun:
883 if fToRemote:
884 fRc = self.txsRunTest(self.oTxsSession, sName, 30 * 60 * 1000, asArgs[0], asArgs, self.asEnv, \
885 fCheckSessionStatus = True);
886 if fRc:
887 iRc = 0;
888 else:
889 (_, sOpcode, abPayload) = self.oTxsSession.getLastReply();
890 if sOpcode.startswith('PROC NOK '): # Extract process rc.
891 iRc = abPayload[0]; # ASSUMES 8-bit rc for now.
892 if iRc == 0: # Might happen if the testcase misses some dependencies. Set it to -42 then.
893 iRc = -42;
894 else:
895 iRc = -1; ## @todo
896 else:
897 try:
898 if fHardened:
899 oChild = utils.sudoProcessPopen(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
900 else:
901 oChild = utils.processPopenSafe(asArgs, stdin = oDevNull, stdout = sys.stdout, stderr = sys.stdout);
902 except:
903 if sName in [ 'tstAsmStructsRC', # 32-bit, may fail to start on 64-bit linux. Just ignore.
904 ]:
905 reporter.logXcpt();
906 fSkipped = True;
907 else:
908 reporter.errorXcpt();
909 iRc = 1023;
910 oChild = None;
911
912 if oChild is not None:
913 self.pidFileAdd(oChild.pid, sName, fSudo = fHardened);
914 iRc = oChild.wait();
915 self.pidFileRemove(oChild.pid);
916 else:
917 iRc = 0;
918
919 #
920 # Clean up
921 #
922 for sPath in asFilesToRemove:
923 self._hardenedDeleteFile(sPath);
924 for sPath in asDirsToRemove:
925 self._hardenedRemoveDir(sPath);
926
927 #
928 # Report.
929 #
930 if os.path.exists(sXmlFile):
931 reporter.addSubXmlFile(sXmlFile);
932 if fHardened:
933 self._hardenedDeleteFile(sXmlFile);
934 else:
935 os.unlink(sXmlFile);
936
937 if iRc == 0:
938 reporter.log('*** %s: exit code %d' % (sFullPath, iRc));
939 self.cPassed += 1;
940
941 elif iRc == 4: # RTEXITCODE_SKIPPED
942 reporter.log('*** %s: exit code %d (RTEXITCODE_SKIPPED)' % (sFullPath, iRc));
943 fSkipped = True;
944 self.cSkipped += 1;
945
946 elif fSkipped:
947 reporter.log('*** %s: exit code %d (Skipped)' % (sFullPath, iRc));
948 self.cSkipped += 1;
949
950 else:
951 sName = self.kdExitCodeNames.get(iRc, '');
952 if iRc in self.kdExitCodeNamesWin and utils.getHostOs() == 'win':
953 sName = self.kdExitCodeNamesWin[iRc];
954 if sName != '':
955 sName = ' (%s)' % (sName);
956
957 if iRc != 1:
958 reporter.testFailure('Exit status: %d%s' % (iRc, sName));
959 reporter.log( '!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
960 else:
961 reporter.error('!*! %s: exit code %d%s' % (sFullPath, iRc, sName));
962 self.cFailed += 1;
963
964 return fSkipped;
965
966 def _testRunUnitTestsSet(self, sTestCasePattern, sTestCaseSubDir):
967 """
968 Run subset of the unit tests set.
969 """
970
971 # Open /dev/null for use as stdin further down.
972 try:
973 oDevNull = open(os.path.devnull, 'w+');
974 except:
975 oDevNull = None;
976
977 # Determin the host OS specific exclusion lists.
978 dTestCasesBuggyForHostOs = self.kdTestCasesBuggyPerOs.get(utils.getHostOs(), []);
979 dTestCasesBuggyForHostOs.update(self.kdTestCasesBuggyPerOs.get(utils.getHostOsDotArch(), []));
980
981 ## @todo Add filtering for more specific OSes (like OL server, doesn't have X installed) by adding a separate
982 # black list + using utils.getHostOsVersion().
983
984 #
985 # Process the file list and run everything looking like a testcase.
986 #
987 for sFilename in sorted(os.listdir(os.path.join(self.sUnitTestsPathSrc, sTestCaseSubDir))):
988 # Separate base and suffix and morph the base into something we
989 # can use for reporting and array lookups.
990 sName, sSuffix = os.path.splitext(sFilename);
991 if sTestCaseSubDir != '.':
992 sName = sTestCaseSubDir + '/' + sName;
993
994 # Process white list first, if set.
995 if self.fOnlyWhiteList \
996 and sName not in self.kdTestCasesWhiteList:
997 reporter.log2('"%s" is not in white list, skipping.' % (sFilename,));
998 continue;
999
1000 # Basic exclusion.
1001 if not re.match(sTestCasePattern, sFilename) \
1002 or sSuffix in self.kasSuffixBlackList:
1003 reporter.log2('"%s" is not a test case.' % (sFilename,));
1004 continue;
1005
1006 # Check if the testcase is black listed or buggy before executing it.
1007 if self._isExcluded(sName, self.kdTestCasesBlackList):
1008 # (No testStart/Done or accounting here!)
1009 reporter.log('%s: SKIPPED (blacklisted)' % (sName,));
1010
1011 elif self._isExcluded(sName, self.kdTestCasesBuggy):
1012 reporter.testStart(sName);
1013 reporter.log('%s: Skipping, buggy in general.' % (sName,));
1014 reporter.testDone(fSkipped = True);
1015 self.cSkipped += 1;
1016
1017 elif self._isExcluded(sName, dTestCasesBuggyForHostOs):
1018 reporter.testStart(sName);
1019 reporter.log('%s: Skipping, buggy on %s.' % (sName, utils.getHostOs(),));
1020 reporter.testDone(fSkipped = True);
1021 self.cSkipped += 1;
1022
1023 else:
1024 sFullPath = os.path.normpath(os.path.join(self.sUnitTestsPathSrc, os.path.join(sTestCaseSubDir, sFilename)));
1025 reporter.testStart(sName);
1026 try:
1027 fSkipped = self._executeTestCase(sName, sFullPath, sTestCaseSubDir, oDevNull);
1028 except:
1029 reporter.errorXcpt('!*!');
1030 self.cFailed += 1;
1031 fSkipped = False;
1032 reporter.testDone(fSkipped);
1033
1034
1035
1036if __name__ == '__main__':
1037 sys.exit(tdUnitTest1().main(sys.argv))
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette