VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 86715

Last change on this file since 86715 was 84946, checked in by vboxsync, 5 years ago

ValKit: Don't check for obsolete kBuild environment variables.

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 47.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: testboxscript_real.py 84946 2020-06-25 10:25:18Z vboxsync $
4
5"""
6TestBox Script - main().
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2020 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 84946 $"
31
32
33# Standard python imports.
34import math
35import os
36from optparse import OptionParser # pylint: disable=deprecated-module
37import platform
38import random
39import shutil
40import sys
41import tempfile
42import time
43import uuid
44
45# Only the main script needs to modify the path.
46try: __file__
47except: __file__ = sys.argv[0];
48g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
49g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
50sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
51
52# Validation Kit imports.
53from common import constants;
54from common import utils;
55import testboxcommons;
56from testboxcommons import TestBoxException;
57from testboxcommand import TestBoxCommand;
58from testboxconnection import TestBoxConnection;
59from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
60
61# Python 3 hacks:
62if sys.version_info[0] >= 3:
63 long = int; # pylint: disable=redefined-builtin,invalid-name
64
65
66class TestBoxScriptException(Exception):
67 """ For raising exceptions during TestBoxScript.__init__. """
68 pass; # pylint: disable=unnecessary-pass
69
70
71class TestBoxScript(object):
72 """
73 Implementation of the test box script.
74 Communicate with test manager and perform offered actions.
75 """
76
77 ## @name Class Constants.
78 # @{
79
80 # Scratch space round value (MB).
81 kcMbScratchSpaceRounding = 64
82 # Memory size round value (MB).
83 kcMbMemoryRounding = 4
84 # A NULL UUID in string form.
85 ksNullUuid = '00000000-0000-0000-0000-000000000000';
86 # The minimum dispatch loop delay.
87 kcSecMinDelay = 12;
88 # The maximum dispatch loop delay (inclusive).
89 kcSecMaxDelay = 24;
90 # The minimum sign-on delay.
91 kcSecMinSignOnDelay = 30;
92 # The maximum sign-on delay (inclusive).
93 kcSecMaxSignOnDelay = 60;
94
95 # Keys for config params
96 VALUE = 'value'
97 FN = 'fn' # pylint: disable=invalid-name
98
99 ## @}
100
101
102 def __init__(self, oOptions):
103 """
104 Initialize internals
105 """
106 self._oOptions = oOptions;
107 self._sTestBoxHelper = None;
108
109 # Signed-on state
110 self._cSignOnAttempts = 0;
111 self._fSignedOn = False;
112 self._fNeedReSignOn = False;
113 self._fFirstSignOn = True;
114 self._idTestBox = None;
115 self._sTestBoxName = '';
116 self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
117
118 # Command processor.
119 self._oCommand = TestBoxCommand(self);
120
121 #
122 # Scratch dir setup. Use /var/tmp instead of /tmp because we may need
123 # many many GBs for some test scenarios and /tmp can be backed by swap
124 # or be a fast+small disk of some kind, while /var/tmp is normally
125 # larger, if slower. /var/tmp is generally not cleaned up on reboot,
126 # /tmp often is, this would break host panic / triple-fault detection.
127 #
128 if self._oOptions.sScratchRoot is None:
129 if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
130 # We need *lots* of space, so avoid /tmp as it may be a memory
131 # file system backed by the swap file, or worse.
132 self._oOptions.sScratchRoot = tempfile.gettempdir();
133 else:
134 self._oOptions.sScratchRoot = '/var/tmp';
135 sSubDir = 'testbox';
136 try:
137 sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=no-member
138 except:
139 pass;
140 self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
141
142 self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
143 self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
144 self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
145
146 for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
147 if not os.path.isdir(sDir):
148 os.makedirs(sDir, 0o700);
149
150 # We count consecutive reinitScratch failures and will reboot the
151 # testbox after a while in the hope that it will correct the issue.
152 self._cReinitScratchErrors = 0;
153
154 #
155 # Mount builds and test resources if requested.
156 #
157 self.mountShares();
158
159 #
160 # Sign-on parameters: Packed into list of records of format:
161 # { <Parameter ID>: { <Current value>, <Check function> } }
162 #
163 self._ddSignOnParams = \
164 {
165 constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
166 constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
167 constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
168 constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
169 constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
170 constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
171 constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
172 constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
173 constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
174 constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
175 constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
176 #constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
177 constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
178 constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
179 constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
180 constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: utils.getPresentCpuCount },
181 constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
182 constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
183 }
184 for sItem in self._ddSignOnParams:
185 if self._ddSignOnParams[sItem][self.FN] is not None:
186 self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
187
188 testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
189 testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
190 testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
191 for sItem in self._ddSignOnParams:
192 testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
193
194 #
195 # The System UUID is the primary identification of the machine, so
196 # refuse to cooperate if it's NULL.
197 #
198 self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
199 if self._sTestBoxUuid == self.ksNullUuid:
200 raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
201
202 #
203 # Export environment variables, clearing any we don't know yet.
204 #
205 for sEnvVar in self._oOptions.asEnvVars:
206 iEqual = sEnvVar.find('=');
207 if iEqual == -1: # No '=', remove it.
208 if sEnvVar in os.environ:
209 del os.environ[sEnvVar];
210 elif iEqual > 0: # Set it.
211 os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
212 else: # Starts with '=', bad user.
213 raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
214
215 os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
216 os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
217 os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
218 os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
219 os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
220 os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
221 os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
222 os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
223 os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
224 os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
225 os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
226 os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
227 #TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
228 os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
229 os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
230 os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
231 os.environ['TESTBOX_REPORTER'] = 'remote';
232 os.environ['TESTBOX_NAME'] = '';
233 os.environ['TESTBOX_ID'] = '';
234 os.environ['TESTBOX_TEST_SET_ID'] = '';
235 os.environ['TESTBOX_TIMEOUT'] = '0';
236 os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
237
238 if utils.getHostOs() == 'win':
239 os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
240 # Currently omitting any kBuild tools.
241
242 def mountShares(self):
243 """
244 Mounts the shares.
245 Raises exception on failure.
246 """
247 self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
248 self._oOptions.sBuildsServerShare,
249 self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd,
250 self._oOptions.sBuildsServerMountOpt, 'builds');
251 self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
252 self._oOptions.sTestRsrcServerShare,
253 self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd,
254 self._oOptions.sTestRsrcServerMountOpt, 'testrsrc');
255 return True;
256
257 def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sMountOpt, sWhat):
258 """
259 Mounts the specified share if needed.
260 Raises exception on failure.
261 """
262 # Only mount if the type is specified.
263 if sType is None:
264 return True;
265
266 # Test if already mounted.
267 sTestFile = os.path.join(sMountPoint + os.path.sep, os.path.basename(sShare) + '-new.txt');
268 if os.path.isfile(sTestFile):
269 return True;
270
271 #
272 # Platform specific mount code.
273 #
274 sHostOs = utils.getHostOs()
275 if sHostOs in ('darwin', 'freebsd'):
276 if sMountOpt != '':
277 sMountOpt = ',' + sMountOpt
278 utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
279 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
280 utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=no-member
281 if sType == 'cifs':
282 # Note! no smb://server/share stuff here, 10.6.8 didn't like it.
283 utils.processOutputChecked(['/sbin/mount_smbfs',
284 '-o',
285 'automounted,nostreams,soft,noowners,noatime,rdonly' + sMountOpt,
286 '-f', '0555', '-d', '0555',
287 '//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
288 sMountPoint]);
289 else:
290 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
291
292 elif sHostOs == 'linux':
293 if sMountOpt != '':
294 sMountOpt = ',' + sMountOpt
295 utils.sudoProcessCall(['/bin/umount', sMountPoint]);
296 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
297 if sType == 'cifs':
298 utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
299 '-o',
300 'user=' + sUser
301 + ',password=' + sPassword
302 + ',sec=ntlmv2'
303 + ',uid=' + str(os.getuid()) # pylint: disable=no-member
304 + ',gid=' + str(os.getgid()) # pylint: disable=no-member
305 + ',nounix,file_mode=0555,dir_mode=0555,soft,ro'
306 + sMountOpt,
307 '//%s/%s' % (sServer, sShare),
308 sMountPoint]);
309 elif sType == 'nfs':
310 utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
311 '-o', 'soft,ro' + sMountOpt,
312 '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
313 sMountPoint]);
314
315 else:
316 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
317
318 elif sHostOs == 'solaris':
319 if sMountOpt != '':
320 sMountOpt = ',' + sMountOpt
321 utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
322 utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
323 if sType == 'cifs':
324 ## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
325 oPasswdFile = tempfile.TemporaryFile();
326 oPasswdFile.write(sPassword + '\n');
327 oPasswdFile.flush();
328 utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
329 '-o',
330 'user=' + sUser
331 + ',uid=' + str(os.getuid()) # pylint: disable=no-member
332 + ',gid=' + str(os.getgid()) # pylint: disable=no-member
333 + ',fileperms=0555,dirperms=0555,noxattr,ro'
334 + sMountOpt,
335 '//%s/%s' % (sServer, sShare),
336 sMountPoint],
337 stdin = oPasswdFile);
338 oPasswdFile.close();
339 elif sType == 'nfs':
340 utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
341 '-o', 'noxattr,ro' + sMountOpt,
342 '%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
343 sMountPoint]);
344
345 else:
346 raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
347
348
349 elif sHostOs == 'win':
350 if sType != 'cifs':
351 raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
352 utils.processCall(['net', 'use', sMountPoint, '/d']);
353 utils.processOutputChecked(['net', 'use', sMountPoint,
354 '\\\\' + sServer + '\\' + sShare,
355 sPassword,
356 '/USER:' + sUser,]);
357 else:
358 raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
359
360 #
361 # Re-test.
362 #
363 if not os.path.isfile(sTestFile):
364 raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
365 % (sWhat, sServer, sShare, sMountPoint, sTestFile));
366
367 return True;
368
369 ## @name Signon property releated methods.
370 # @{
371
372 def _getHelperOutput(self, sCmd):
373 """
374 Invokes TestBoxHelper to obtain information hard to access from python.
375 """
376 if self._sTestBoxHelper is None:
377 if not utils.isRunningFromCheckout():
378 # See VBoxTestBoxScript.zip for layout.
379 self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
380 'TestBoxHelper');
381 else: # Only for in-tree testing, so don't bother be too accurate right now.
382 sType = os.environ.get('KBUILD_TYPE', 'debug');
383 self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
384 utils.getHostOsDotArch(), sType, 'testboxscript', \
385 utils.getHostOs(), utils.getHostArch(), \
386 'TestBoxHelper');
387 if utils.getHostOs() in ['win', 'os2']:
388 self._sTestBoxHelper += '.exe';
389
390 return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
391
392 def _getHelperOutputTristate(self, sCmd, fDunnoValue):
393 """
394 Invokes TestBoxHelper to obtain information hard to access from python.
395 """
396 sValue = self._getHelperOutput(sCmd);
397 sValue = sValue.lower();
398 if sValue == 'true':
399 return True;
400 if sValue == 'false':
401 return False;
402 if sValue not in ('dunno', 'none',):
403 raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
404 return fDunnoValue;
405
406
407 @staticmethod
408 def _isUuidGood(sUuid):
409 """
410 Checks if the UUID looks good.
411
412 There are systems with really bad UUIDs, for instance
413 "03000200-0400-0500-0006-000700080009".
414 """
415 if sUuid == TestBoxScript.ksNullUuid:
416 return False;
417 sUuid = sUuid.lower();
418 for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
419 if sUuid.count(sDigit) > 16:
420 return False;
421 return True;
422
423 def _getHostSystemUuid(self):
424 """
425 Get the system UUID string from the System, return null-uuid if
426 unable to get retrieve it.
427 """
428 if self._oOptions.sSystemUuid is not None:
429 return self._oOptions.sSystemUuid;
430
431 sUuid = self.ksNullUuid;
432
433 #
434 # Try get at the firmware UUID.
435 #
436 if utils.getHostOs() == 'linux':
437 # NOTE: This requires to have kernel option enabled:
438 # Firmware Drivers -> Export DMI identification via sysfs to userspace
439 if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
440 try:
441 sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
442 sUuid = str(uuid.UUID(sVar.strip()));
443 except:
444 pass;
445 ## @todo consider dmidecoder? What about EFI systems?
446
447 elif utils.getHostOs() == 'win':
448 # Windows: WMI
449 try:
450 import win32com.client; # pylint: disable=import-error
451 oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
452 oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
453 for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
454 if oItem.UUID is not None:
455 sUuid = str(uuid.UUID(oItem.UUID));
456 except:
457 pass;
458
459 elif utils.getHostOs() == 'darwin':
460 try:
461 sVar = utils.processOutputChecked(['/bin/sh', '-c',
462 '/usr/sbin/ioreg -k IOPlatformUUID' \
463 + '| /usr/bin/grep IOPlatformUUID' \
464 + '| /usr/bin/head -1']);
465 sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
466 sUuid = str(uuid.UUID(sVar));
467 except:
468 pass;
469
470 elif utils.getHostOs() == 'solaris':
471 # Solaris: The smbios util.
472 try:
473 sVar = utils.processOutputChecked(['/bin/sh', '-c',
474 '/usr/sbin/smbios ' \
475 + '| /usr/xpg4/bin/sed -ne \'s/^.*UUID: *//p\'' \
476 + '| /usr/bin/head -1']);
477 sUuid = str(uuid.UUID(sVar.strip()));
478 except:
479 pass;
480
481 if self._isUuidGood(sUuid):
482 return sUuid;
483
484 #
485 # Try add the MAC address.
486 # uuid.getnode may provide it, or it may return a random number...
487 #
488 lMacAddr = uuid.getnode();
489 sNode = '%012x' % (lMacAddr,)
490 if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
491 return sUuid[:-12] + sNode;
492
493 return sUuid;
494
495 def _getHostCpuVendor(self):
496 """
497 Get the CPUID vendor string on intel HW.
498 """
499 return self._getHelperOutput('cpuvendor');
500
501 def _getHostCpuName(self):
502 """
503 Get the CPU name/description string.
504 """
505 return self._getHelperOutput('cpuname');
506
507 def _getHostCpuRevision(self):
508 """
509 Get the CPU revision (family/model/stepping) value.
510 """
511 return self._getHelperOutput('cpurevision');
512
513 def _hasHostHwVirt(self):
514 """
515 Check if the host supports AMD-V or VT-x
516 """
517 if self._oOptions.fHasHwVirt is None:
518 self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
519 return self._oOptions.fHasHwVirt;
520
521 def _hasHostNestedPaging(self):
522 """
523 Check if the host supports nested paging.
524 """
525 if not self._hasHostHwVirt():
526 return False;
527 if self._oOptions.fHasNestedPaging is None:
528 self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
529 return self._oOptions.fHasNestedPaging;
530
531 def _can64BitGuest(self):
532 """
533 Check if the we (VBox) can run 64-bit guests.
534 """
535 if not self._hasHostHwVirt():
536 return False;
537 if self._oOptions.fCan64BitGuest is None:
538 self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
539 return self._oOptions.fCan64BitGuest;
540
541 def _hasHostIoMmu(self):
542 """
543 Check if the host has an I/O MMU of the VT-d kind.
544 """
545 if not self._hasHostHwVirt():
546 return False;
547 if self._oOptions.fHasIoMmu is None:
548 ## @todo Any way to figure this one out on any host OS?
549 self._oOptions.fHasIoMmu = False;
550 return self._oOptions.fHasIoMmu;
551
552 def _withRawModeSupport(self):
553 """
554 Check if the testbox is configured with raw-mode support or not.
555 """
556 if self._oOptions.fWithRawMode is None:
557 self._oOptions.fWithRawMode = True;
558 return self._oOptions.fWithRawMode;
559
560 def _getHostReport(self):
561 """
562 Generate a report about the host hardware and software.
563 """
564 return self._getHelperOutput('report');
565
566
567 def _getHostMemSize(self):
568 """
569 Gets the amount of physical memory on the host (and accessible to the
570 OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
571 Unit: MiB.
572 """
573 cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
574
575 # Round it.
576 cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
577 return cMbMemory;
578
579 def _getFreeScratchSpace(self):
580 """
581 Get free space on the volume where scratch directory is located and
582 return it in bytes rounded down to nearest 64MB
583 (currently works on Linux only)
584 Unit: MiB.
585 """
586 if platform.system() == 'Windows':
587 import ctypes
588 cTypeMbFreeSpace = ctypes.c_ulonglong(0)
589 ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
590 ctypes.pointer(cTypeMbFreeSpace))
591 cMbFreeSpace = cTypeMbFreeSpace.value
592 else:
593 stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=no-member
594 cMbFreeSpace = stats.f_frsize * stats.f_bfree
595
596 # Convert to MB
597 cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
598
599 # Round free space size
600 cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
601 return cMbFreeSpace;
602
603 def _getScriptRev(self):
604 """
605 The script (subversion) revision number.
606 """
607 sRev = '@VBOX_SVN_REV@';
608 sRev = sRev.strip(); # just in case...
609 try:
610 _ = int(sRev);
611 except:
612 return __version__[11:-1].strip();
613 return sRev;
614
615 def _getPythonHexVersion(self):
616 """
617 The python hex version number.
618 """
619 uHexVersion = getattr(sys, 'hexversion', None);
620 if uHexVersion is None:
621 uHexVersion = (sys.version_info[0] << 24) | (sys.version_info[1] << 16) | (sys.version_info[2] << 8);
622 if sys.version_info[3] == 'final':
623 uHexVersion |= 0xf0;
624 return uHexVersion;
625
626 # @}
627
628 def openTestManagerConnection(self):
629 """
630 Opens up a connection to the test manager.
631
632 Raises exception on failure.
633 """
634 return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
635
636 def getSignOnParam(self, sName):
637 """
638 Returns a sign-on parameter value as string.
639 Raises exception if the name is incorrect.
640 """
641 return str(self._ddSignOnParams[sName][self.VALUE]);
642
643 def getPathState(self):
644 """
645 Get the path to the state dir in the scratch area.
646 """
647 return self._sScratchState;
648
649 def getPathScripts(self):
650 """
651 Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
652 """
653 return self._sScratchScripts;
654
655 def getPathSpill(self):
656 """
657 Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
658 """
659 return self._sScratchSpill;
660
661 def getPathBuilds(self):
662 """
663 Get the path to the builds.
664 """
665 return self._oOptions.sBuildsPath;
666
667 def getTestBoxId(self):
668 """
669 Get the TestBox ID for state saving purposes.
670 """
671 return self._idTestBox;
672
673 def getTestBoxName(self):
674 """
675 Get the TestBox name for state saving purposes.
676 """
677 return self._sTestBoxName;
678
679 def _reinitScratch(self, fnLog, fUseTheForce):
680 """
681 Wipes the scratch directories and re-initializes them.
682
683 No exceptions raise, returns success indicator instead.
684 """
685 if fUseTheForce is None:
686 fUseTheForce = self._fFirstSignOn;
687
688 class ErrorCallback(object): # pylint: disable=too-few-public-methods
689 """
690 Callbacks + state for the cleanup.
691 """
692 def __init__(self):
693 self.fRc = True;
694 def onErrorCallback(self, sFnName, sPath, aXcptInfo):
695 """ Logs error during shutil.rmtree operation. """
696 fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
697 self.fRc = False;
698 oRc = ErrorCallback();
699
700 #
701 # Cleanup.
702 #
703 for sName in os.listdir(self._oOptions.sScratchRoot):
704 sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
705 try:
706 if os.path.isdir(sFullName):
707 shutil.rmtree(sFullName, False, oRc.onErrorCallback);
708 else:
709 os.remove(sFullName);
710 if os.path.exists(sFullName):
711 raise Exception('Still exists after deletion, weird.');
712 except Exception as oXcpt:
713 if fUseTheForce is True \
714 and utils.getHostOs() not in ['win', 'os2'] \
715 and len(sFullName) >= 8 \
716 and sFullName[0] == '/' \
717 and sFullName[1] != '/' \
718 and sFullName.find('/../') < 0:
719 fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
720 try:
721 if os.path.isdir(sFullName):
722 iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
723 else:
724 iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
725 if iRc != 0:
726 raise Exception('exit code %s' % iRc);
727 if os.path.exists(sFullName):
728 raise Exception('Still exists after forced deletion, weird^2.');
729 except:
730 fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
731 oRc.fRc = False;
732 else:
733 fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
734 oRc.fRc = False;
735
736 # Display files left behind.
737 def dirEnumCallback(sName, oStat):
738 """ callback for dirEnumerateTree """
739 fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
740 utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
741
742 #
743 # Re-create the directories.
744 #
745 for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
746 if not os.path.isdir(sDir):
747 try:
748 os.makedirs(sDir, 0o700);
749 except Exception as oXcpt:
750 fnLog('Error creating "%s": %s' % (sDir, oXcpt));
751 oRc.fRc = False;
752
753 if oRc.fRc is True:
754 self._cReinitScratchErrors = 0;
755 else:
756 self._cReinitScratchErrors += 1;
757 return oRc.fRc;
758
759 def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
760 """
761 Wipes the scratch directories and re-initializes them.
762
763 Will retry according to the cRetries and cMsDelay parameters. Windows
764 forces us to apply this hack as it ships with services asynchronously
765 scanning files after they execute, thus racing us cleaning up after a
766 test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
767 vts_rm.exe kind of open, somehow preventing us from removing the
768 directory containing it, despite not issuing any errors deleting the
769 file itself. The service is called "Application Experience", which
770 feels like a weird joke here.
771
772 No exceptions raise, returns success indicator instead.
773 """
774 fRc = self._reinitScratch(fnLog, fUseTheForce)
775 while fRc is False and cRetries > 0:
776 time.sleep(cMsDelay / 1000.0);
777 fnLog('reinitScratch: Retrying...');
778 fRc = self._reinitScratch(fnLog, fUseTheForce)
779 cRetries -= 1;
780 return fRc;
781
782
783 def _doSignOn(self):
784 """
785 Worker for _maybeSignOn that does the actual signing on.
786 """
787 assert not self._oCommand.isRunning();
788
789 # Reset the siged-on state.
790 testboxcommons.log('Signing-on...')
791 self._fSignedOn = False
792 self._idTestBox = None
793 self._cSignOnAttempts += 1;
794
795 # Assemble SIGN-ON request parameters and send the request.
796 dParams = {};
797 for sParam in self._ddSignOnParams:
798 dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
799 oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
800
801 # Check response.
802 try:
803 sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
804 if sResult != constants.tbresp.STATUS_ACK:
805 raise TestBoxException('Result is %s' % (sResult,));
806 oResponse.checkParameterCount(3);
807 idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
808 sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
809 except TestBoxException as err:
810 testboxcommons.log('Failed to sign-on: %s' % (str(err),))
811 testboxcommons.log('Server response: %s' % (oResponse.toString(),));
812 return False;
813
814 # Successfully signed on, update the state.
815 self._fSignedOn = True;
816 self._fNeedReSignOn = False;
817 self._cSignOnAttempts = 0;
818 self._idTestBox = idTestBox;
819 self._sTestBoxName = sTestBoxName;
820
821 # Update the environment.
822 os.environ['TESTBOX_ID'] = str(self._idTestBox);
823 os.environ['TESTBOX_NAME'] = sTestBoxName;
824 os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
825 os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
826 os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
827
828 testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
829 % (self._idTestBox, self._sTestBoxName));
830
831 # Set up the scratch area.
832 self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
833
834 self._fFirstSignOn = False;
835 return True;
836
837 def _maybeSignOn(self):
838 """
839 Check if Test Box parameters were changed
840 and do sign-in in case of positive result
841 """
842
843 # Skip sign-on check if background command is currently in
844 # running state (avoid infinite signing on).
845 if self._oCommand.isRunning():
846 return None;
847
848 # Refresh sign-on parameters, changes triggers sign-on.
849 fNeedSignOn = not self._fSignedOn or self._fNeedReSignOn;
850 for item in self._ddSignOnParams:
851 if self._ddSignOnParams[item][self.FN] is None:
852 continue
853
854 sOldValue = self._ddSignOnParams[item][self.VALUE]
855 self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
856 if sOldValue != self._ddSignOnParams[item][self.VALUE]:
857 fNeedSignOn = True
858 testboxcommons.log('Detected %s parameter change: %s -> %s' %
859 (item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
860
861 if fNeedSignOn:
862 self._doSignOn();
863 return None;
864
865 def dispatch(self):
866 """
867 Receive orders from Test Manager and execute them
868 """
869
870 (self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
871 self._fNeedReSignOn = self._fSignedOn;
872 if self._fSignedOn:
873 os.environ['TESTBOX_ID'] = str(self._idTestBox);
874 os.environ['TESTBOX_NAME'] = self._sTestBoxName;
875
876 while True:
877 # Make sure we're signed on before trying to do anything.
878 self._maybeSignOn();
879 while not self._fSignedOn:
880 iFactor = 1 if self._cSignOnAttempts < 100 else 4;
881 time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
882 self._maybeSignOn();
883
884 # Retrieve and handle command from the TM.
885 (oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
886 self._idTestBox,
887 self._sTestBoxUuid,
888 self._oCommand.isRunning());
889 if oResponse is not None:
890 self._oCommand.handleCommand(oResponse, oConnection);
891 if oConnection is not None:
892 if oConnection.isConnected():
893 self._oCommand.flushLogOnConnection(oConnection);
894 oConnection.close();
895
896 # Automatically reboot if scratch init fails.
897 #if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
898 # testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
899 # % ( self._cReinitScratchErrors, ));
900 # self._oCommand.doReboot();
901
902 # delay a wee bit before looping.
903 ## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
904 # with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
905 iFactor = 1;
906 if self._cReinitScratchErrors > 0:
907 iFactor = 4;
908 time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
909
910 # Not reached.
911
912
913 @staticmethod
914 def main():
915 """
916 Main function a la C/C++. Returns exit code.
917 """
918
919 #
920 # Parse arguments.
921 #
922 sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
923 if utils.getHostOs() in ('win', 'os2'):
924 sDefTestRsrc = 'T:';
925 sDefBuilds = 'U:';
926 elif utils.getHostOs() == 'darwin':
927 sDefTestRsrc = '/Volumes/testrsrc';
928 sDefBuilds = '/Volumes/builds';
929 else:
930 sDefTestRsrc = '/mnt/testrsrc';
931 sDefBuilds = '/mnt/builds';
932
933 class MyOptionParser(OptionParser):
934 """ We need to override the exit code on --help, error and so on. """
935 def __init__(self, *args, **kwargs):
936 OptionParser.__init__(self, *args, **kwargs);
937 def exit(self, status = 0, msg = None):
938 OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
939
940 parser = MyOptionParser(version=__version__[11:-1].strip());
941 for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
942 sLower = sMixed.lower();
943 sPrefix = 's' + sMixed;
944 parser.add_option('--' + sLower + '-path',
945 dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
946 help='Where ' + sDesc + ' can be found');
947 parser.add_option('--' + sLower + '-server-type',
948 dest=sPrefix + 'ServerType', metavar='<nfs|cifs>', default=sDefShareType,
949 help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
950 parser.add_option('--' + sLower + '-server-name',
951 dest=sPrefix + 'ServerName', metavar='<server>',
952 default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
953 help='The name of the server with the builds.');
954 parser.add_option('--' + sLower + '-server-share',
955 dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
956 help='The name of the builds share.');
957 parser.add_option('--' + sLower + '-server-user',
958 dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
959 help='The user name to use when accessing the ' + sDesc + ' share.');
960 parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
961 dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
962 help='The password to use when accessing the ' + sDesc + ' share.');
963 parser.add_option('--' + sLower + '-server-mountopt',
964 dest=sPrefix + 'ServerMountOpt', metavar='<mountopt>', default='',
965 help='The mount options to use when accessing the ' + sDesc + ' share.');
966
967 parser.add_option("--test-manager", metavar="<url>",
968 dest="sTestManagerUrl",
969 help="Test Manager URL",
970 default="http://tindertux.de.oracle.com/testmanager")
971 parser.add_option("--scratch-root", metavar="<abs-path>",
972 dest="sScratchRoot",
973 help="Path to the scratch directory",
974 default=None)
975 parser.add_option("--system-uuid", metavar="<uuid>",
976 dest="sSystemUuid",
977 help="The system UUID of the testbox, used for uniquely identifiying the machine",
978 default=None)
979 parser.add_option("--hwvirt",
980 dest="fHasHwVirt", action="store_true", default=None,
981 help="Hardware virtualization available in the CPU");
982 parser.add_option("--no-hwvirt",
983 dest="fHasHwVirt", action="store_false", default=None,
984 help="Hardware virtualization not available in the CPU");
985 parser.add_option("--nested-paging",
986 dest="fHasNestedPaging", action="store_true", default=None,
987 help="Nested paging is available");
988 parser.add_option("--no-nested-paging",
989 dest="fHasNestedPaging", action="store_false", default=None,
990 help="Nested paging is not available");
991 parser.add_option("--64-bit-guest",
992 dest="fCan64BitGuest", action="store_true", default=None,
993 help="Host can execute 64-bit guests");
994 parser.add_option("--no-64-bit-guest",
995 dest="fCan64BitGuest", action="store_false", default=None,
996 help="Host cannot execute 64-bit guests");
997 parser.add_option("--io-mmu",
998 dest="fHasIoMmu", action="store_true", default=None,
999 help="I/O MMU available");
1000 parser.add_option("--no-io-mmu",
1001 dest="fHasIoMmu", action="store_false", default=None,
1002 help="No I/O MMU available");
1003 parser.add_option("--raw-mode",
1004 dest="fWithRawMode", action="store_true", default=None,
1005 help="Use raw-mode on this host.");
1006 parser.add_option("--no-raw-mode",
1007 dest="fWithRawMode", action="store_false", default=None,
1008 help="Disables raw-mode tests on this host.");
1009 parser.add_option("--pidfile",
1010 dest="sPidFile", default=None,
1011 help="For the parent script, ignored.");
1012 parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
1013 dest = "asEnvVars", default = [],
1014 help = "Sets an environment variable. Can be repeated.");
1015 def sbp_callback(option, opt_str, value, parser):
1016 _, _, _ = opt_str, value, option
1017 parser.values.sTestManagerUrl = 'http://10.162.100.8/testmanager/'
1018 parser.values.sBuildsServerName = 'vbox-st02.ru.oracle.com'
1019 parser.values.sTestRsrcServerName = 'vbox-st02.ru.oracle.com'
1020 parser.values.sTestRsrcServerShare = 'scratch/data/testrsrc'
1021 parser.add_option("--spb", "--load-sbp-defaults", action="callback", callback=sbp_callback,
1022 help="Load defaults for the sbp setup.")
1023
1024 (oOptions, args) = parser.parse_args()
1025 # Check command line
1026 if args != []:
1027 parser.print_help();
1028 return TBS_EXITCODE_SYNTAX;
1029
1030 if oOptions.sSystemUuid is not None:
1031 uuid.UUID(oOptions.sSystemUuid);
1032 if not oOptions.sTestManagerUrl.startswith('http://') \
1033 and not oOptions.sTestManagerUrl.startswith('https://'):
1034 print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1035 return TBS_EXITCODE_SYNTAX;
1036
1037 for sPrefix in ['sBuilds', 'sTestRsrc']:
1038 sType = getattr(oOptions, sPrefix + 'ServerType');
1039 if sType is None or not sType.strip():
1040 setattr(oOptions, sPrefix + 'ServerType', None);
1041 elif sType not in ['cifs', 'nfs']:
1042 print('Syntax error: Invalid server type "%s"' % (sType,));
1043 return TBS_EXITCODE_SYNTAX;
1044
1045
1046 #
1047 # Instantiate the testbox script and start dispatching work.
1048 #
1049 try:
1050 oTestBoxScript = TestBoxScript(oOptions);
1051 except TestBoxScriptException as oXcpt:
1052 print('Error: %s' % (oXcpt,));
1053 return TBS_EXITCODE_SYNTAX;
1054 oTestBoxScript.dispatch();
1055
1056 # Not supposed to get here...
1057 return TBS_EXITCODE_FAILURE;
1058
1059
1060
1061if __name__ == '__main__':
1062 sys.exit(TestBoxScript.main());
1063
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette