testboxscript_real.py@ 77994

Last change on this file since 77994 was 76553, checked in by vboxsync, 6 years ago
scm --update-copyright-year
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 46.5 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 76553 2019-01-01 01:45:53Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2019 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 76553 $"
31
32
33	# Standard python imports.
34	import math
35	import multiprocessing
36	import os
37	from optparse import OptionParser # pylint: disable=deprecated-module
38	import platform
39	import random
40	import shutil
41	import sys
42	import tempfile
43	import time
44	import uuid
45
46	# Only the main script needs to modify the path.
47	try: __file__
48	except: __file__ = sys.argv[0];
49	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
50	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
51	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
52
53	# Validation Kit imports.
54	from common import constants;
55	from common import utils;
56	import testboxcommons;
57	from testboxcommons import TestBoxException;
58	from testboxcommand import TestBoxCommand;
59	from testboxconnection import TestBoxConnection;
60	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
61
62	# Python 3 hacks:
63	if sys.version_info[0] >= 3:
64	long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67	class TestBoxScriptException(Exception):
68	""" For raising exceptions during TestBoxScript.__init__. """
69	pass;
70
71
72	class TestBoxScript(object):
73	"""
74	Implementation of the test box script.
75	Communicate with test manager and perform offered actions.
76	"""
77
78	## @name Class Constants.
79	# @{
80
81	# Scratch space round value (MB).
82	kcMbScratchSpaceRounding = 64
83	# Memory size round value (MB).
84	kcMbMemoryRounding = 4
85	# A NULL UUID in string form.
86	ksNullUuid = '00000000-0000-0000-0000-000000000000';
87	# The minimum dispatch loop delay.
88	kcSecMinDelay = 12;
89	# The maximum dispatch loop delay (inclusive).
90	kcSecMaxDelay = 24;
91	# The minimum sign-on delay.
92	kcSecMinSignOnDelay = 30;
93	# The maximum sign-on delay (inclusive).
94	kcSecMaxSignOnDelay = 60;
95
96	# Keys for config params
97	VALUE = 'value'
98	FN = 'fn' # pylint: disable=C0103
99
100	## @}
101
102
103	def __init__(self, oOptions):
104	"""
105	Initialize internals
106	"""
107	self._oOptions = oOptions;
108	self._sTestBoxHelper = None;
109
110	# Signed-on state
111	self._cSignOnAttempts = 0;
112	self._fSignedOn = False;
113	self._fNeedReSignOn = False;
114	self._fFirstSignOn = True;
115	self._idTestBox = None;
116	self._sTestBoxName = '';
117	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
118
119	# Command processor.
120	self._oCommand = TestBoxCommand(self);
121
122	#
123	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
124	# many many GBs for some test scenarios and /tmp can be backed by swap
125	# or be a fast+small disk of some kind, while /var/tmp is normally
126	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
127	# /tmp often is, this would break host panic / triple-fault detection.
128	#
129	if self._oOptions.sScratchRoot is None:
130	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
131	# We need lots of space, so avoid /tmp as it may be a memory
132	# file system backed by the swap file, or worse.
133	self._oOptions.sScratchRoot = tempfile.gettempdir();
134	else:
135	self._oOptions.sScratchRoot = '/var/tmp';
136	sSubDir = 'testbox';
137	try:
138	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
139	except:
140	pass;
141	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
142
143	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
144	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
145	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
146
147	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
148	if not os.path.isdir(sDir):
149	os.makedirs(sDir, 0o700);
150
151	# We count consecutive reinitScratch failures and will reboot the
152	# testbox after a while in the hope that it will correct the issue.
153	self._cReinitScratchErrors = 0;
154
155	#
156	# Mount builds and test resources if requested.
157	#
158	self.mountShares();
159
160	#
161	# Sign-on parameters: Packed into list of records of format:
162	# { <Parameter ID>: { <Current value>, <Check function> } }
163	#
164	self._ddSignOnParams = \
165	{
166	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
167	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
168	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
169	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
170	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
171	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
172	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
173	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
174	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
175	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
176	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
177	#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
178	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
179	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
180	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
181	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
182	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
183	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
184	}
185	for sItem in self._ddSignOnParams:
186	if self._ddSignOnParams[sItem][self.FN] is not None:
187	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
188
189	testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
190	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
191	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
192	for sItem in self._ddSignOnParams:
193	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
194
195	#
196	# The System UUID is the primary identification of the machine, so
197	# refuse to cooperate if it's NULL.
198	#
199	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
200	if self._sTestBoxUuid == self.ksNullUuid:
201	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
202
203	#
204	# Export environment variables, clearing any we don't know yet.
205	#
206	for sEnvVar in self._oOptions.asEnvVars:
207	iEqual = sEnvVar.find('=');
208	if iEqual == -1: # No '=', remove it.
209	if sEnvVar in os.environ:
210	del os.environ[sEnvVar];
211	elif iEqual > 0: # Set it.
212	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
213	else: # Starts with '=', bad user.
214	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
215
216	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
217	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
218	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
219	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
220	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
221	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
222	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
223	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
224	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
225	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
226	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
227	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
228	#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
229	os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
230	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
231	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
232	os.environ['TESTBOX_REPORTER'] = 'remote';
233	os.environ['TESTBOX_NAME'] = '';
234	os.environ['TESTBOX_ID'] = '';
235	os.environ['TESTBOX_TEST_SET_ID'] = '';
236	os.environ['TESTBOX_TIMEOUT'] = '0';
237	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
238
239	if utils.getHostOs() == 'win':
240	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
241	# Currently omitting any kBuild tools.
242
243	def mountShares(self):
244	"""
245	Mounts the shares.
246	Raises exception on failure.
247	"""
248	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
249	self._oOptions.sBuildsServerShare,
250	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd,
251	self._oOptions.sBuildsServerMountOpt, 'builds');
252	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
253	self._oOptions.sTestRsrcServerShare,
254	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd,
255	self._oOptions.sTestRsrcServerMountOpt, 'testrsrc');
256	return True;
257
258	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sMountOpt, sWhat):
259	"""
260	Mounts the specified share if needed.
261	Raises exception on failure.
262	"""
263	# Only mount if the type is specified.
264	if sType is None:
265	return True;
266
267	# Test if already mounted.
268	sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '-new.txt');
269	if os.path.isfile(sTestFile):
270	return True;
271
272	#
273	# Platform specific mount code.
274	#
275	sHostOs = utils.getHostOs()
276	if sHostOs in ('darwin', 'freebsd'):
277	if sMountOpt != '':
278	sMountOpt = ',' + sMountOpt
279	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
280	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
281	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
282	if sType == 'cifs':
283	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
284	utils.processOutputChecked(['/sbin/mount_smbfs',
285	'-o',
286	'automounted,nostreams,soft,noowners,noatime,rdonly' + sMountOpt,
287	'-f', '0555', '-d', '0555',
288	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
289	sMountPoint]);
290	else:
291	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
292
293	elif sHostOs == 'linux':
294	if sMountOpt != '':
295	sMountOpt = ',' + sMountOpt
296	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
297	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
298	if sType == 'cifs':
299	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
300	'-o',
301	'user=' + sUser
302	+ ',password=' + sPassword
303	+ ',sec=ntlmv2'
304	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
305	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
306	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro'
307	+ sMountOpt,
308	'//%s/%s' % (sServer, sShare),
309	sMountPoint]);
310	elif sType == 'nfs':
311	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
312	'-o', 'soft,ro' + sMountOpt,
313	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
314	sMountPoint]);
315
316	else:
317	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
318
319	elif sHostOs == 'solaris':
320	if sMountOpt != '':
321	sMountOpt = ',' + sMountOpt
322	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
323	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
324	if sType == 'cifs':
325	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
326	oPasswdFile = tempfile.TemporaryFile();
327	oPasswdFile.write(sPassword + '\n');
328	oPasswdFile.flush();
329	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
330	'-o',
331	'user=' + sUser
332	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
333	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
334	+ ',fileperms=0555,dirperms=0555,noxattr,ro'
335	+ sMountOpt,
336	'//%s/%s' % (sServer, sShare),
337	sMountPoint],
338	stdin = oPasswdFile);
339	oPasswdFile.close();
340	elif sType == 'nfs':
341	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
342	'-o', 'noxattr,ro' + sMountOpt,
343	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
344	sMountPoint]);
345
346	else:
347	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
348
349
350	elif sHostOs == 'win':
351	if sType != 'cifs':
352	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
353	utils.processCall(['net', 'use', sMountPoint, '/d']);
354	utils.processOutputChecked(['net', 'use', sMountPoint,
355	'\\\\' + sServer + '\\' + sShare,
356	sPassword,
357	'/USER:' + sUser,]);
358	else:
359	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
360
361	#
362	# Re-test.
363	#
364	if not os.path.isfile(sTestFile):
365	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
366	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
367
368	return True;
369
370	## @name Signon property releated methods.
371	# @{
372
373	def _getHelperOutput(self, sCmd):
374	"""
375	Invokes TestBoxHelper to obtain information hard to access from python.
376	"""
377	if self._sTestBoxHelper is None:
378	if not utils.isRunningFromCheckout():
379	# See VBoxTestBoxScript.zip for layout.
380	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
381	'TestBoxHelper');
382	else: # Only for in-tree testing, so don't bother be too accurate right now.
383	sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
384	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
385	utils.getHostOsDotArch(), sType, 'testboxscript', \
386	utils.getHostOs(), utils.getHostArch(), \
387	'TestBoxHelper');
388	if utils.getHostOs() in ['win', 'os2']:
389	self._sTestBoxHelper += '.exe';
390
391	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
392
393	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
394	"""
395	Invokes TestBoxHelper to obtain information hard to access from python.
396	"""
397	sValue = self._getHelperOutput(sCmd);
398	sValue = sValue.lower();
399	if sValue == 'true':
400	return True;
401	if sValue == 'false':
402	return False;
403	if sValue != 'dunno' and sValue != 'none':
404	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
405	return fDunnoValue;
406
407
408	@staticmethod
409	def _isUuidGood(sUuid):
410	"""
411	Checks if the UUID looks good.
412
413	There are systems with really bad UUIDs, for instance
414	"03000200-0400-0500-0006-000700080009".
415	"""
416	if sUuid == TestBoxScript.ksNullUuid:
417	return False;
418	sUuid = sUuid.lower();
419	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
420	if sUuid.count(sDigit) > 16:
421	return False;
422	return True;
423
424	def _getHostSystemUuid(self):
425	"""
426	Get the system UUID string from the System, return null-uuid if
427	unable to get retrieve it.
428	"""
429	if self._oOptions.sSystemUuid is not None:
430	return self._oOptions.sSystemUuid;
431
432	sUuid = self.ksNullUuid;
433
434	#
435	# Try get at the firmware UUID.
436	#
437	if utils.getHostOs() == 'linux':
438	# NOTE: This requires to have kernel option enabled:
439	# Firmware Drivers -> Export DMI identification via sysfs to userspace
440	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
441	try:
442	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
443	sUuid = str(uuid.UUID(sVar.strip()));
444	except:
445	pass;
446	## @todo consider dmidecoder? What about EFI systems?
447
448	elif utils.getHostOs() == 'win':
449	# Windows: WMI
450	try:
451	import win32com.client; # pylint: disable=F0401
452	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
453	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
454	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
455	if oItem.UUID != None:
456	sUuid = str(uuid.UUID(oItem.UUID));
457	except:
458	pass;
459
460	elif utils.getHostOs() == 'darwin':
461	try:
462	sVar = utils.processOutputChecked(['/bin/sh', '-c',
463	'/usr/sbin/ioreg -k IOPlatformUUID' \
464	+ '\| /usr/bin/grep IOPlatformUUID' \
465	+ '\| /usr/bin/head -1']);
466	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
467	sUuid = str(uuid.UUID(sVar));
468	except:
469	pass;
470
471	elif utils.getHostOs() == 'solaris':
472	# Solaris: The smbios util.
473	try:
474	sVar = utils.processOutputChecked(['/bin/sh', '-c',
475	'/usr/sbin/smbios ' \
476	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
477	+ '\| /usr/bin/head -1']);
478	sUuid = str(uuid.UUID(sVar.strip()));
479	except:
480	pass;
481
482	if self._isUuidGood(sUuid):
483	return sUuid;
484
485	#
486	# Try add the MAC address.
487	# uuid.getnode may provide it, or it may return a random number...
488	#
489	lMacAddr = uuid.getnode();
490	sNode = '%012x' % (lMacAddr,)
491	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
492	return sUuid[:-12] + sNode;
493
494	return sUuid;
495
496	def _getHostCpuVendor(self):
497	"""
498	Get the CPUID vendor string on intel HW.
499	"""
500	return self._getHelperOutput('cpuvendor');
501
502	def _getHostCpuName(self):
503	"""
504	Get the CPU name/description string.
505	"""
506	return self._getHelperOutput('cpuname');
507
508	def _getHostCpuRevision(self):
509	"""
510	Get the CPU revision (family/model/stepping) value.
511	"""
512	return self._getHelperOutput('cpurevision');
513
514	def _hasHostHwVirt(self):
515	"""
516	Check if the host supports AMD-V or VT-x
517	"""
518	if self._oOptions.fHasHwVirt is None:
519	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
520	return self._oOptions.fHasHwVirt;
521
522	def _hasHostNestedPaging(self):
523	"""
524	Check if the host supports nested paging.
525	"""
526	if not self._hasHostHwVirt():
527	return False;
528	if self._oOptions.fHasNestedPaging is None:
529	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
530	return self._oOptions.fHasNestedPaging;
531
532	def _can64BitGuest(self):
533	"""
534	Check if the we (VBox) can run 64-bit guests.
535	"""
536	if not self._hasHostHwVirt():
537	return False;
538	if self._oOptions.fCan64BitGuest is None:
539	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
540	return self._oOptions.fCan64BitGuest;
541
542	def _hasHostIoMmu(self):
543	"""
544	Check if the host has an I/O MMU of the VT-d kind.
545	"""
546	if not self._hasHostHwVirt():
547	return False;
548	if self._oOptions.fHasIoMmu is None:
549	## @todo Any way to figure this one out on any host OS?
550	self._oOptions.fHasIoMmu = False;
551	return self._oOptions.fHasIoMmu;
552
553	def _withRawModeSupport(self):
554	"""
555	Check if the testbox is configured with raw-mode support or not.
556	"""
557	if self._oOptions.fWithRawMode is None:
558	self._oOptions.fWithRawMode = True;
559	return self._oOptions.fWithRawMode;
560
561	def _getHostReport(self):
562	"""
563	Generate a report about the host hardware and software.
564	"""
565	return self._getHelperOutput('report');
566
567
568	def _getHostMemSize(self):
569	"""
570	Gets the amount of physical memory on the host (and accessible to the
571	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
572	Unit: MiB.
573	"""
574	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
575
576	# Round it.
577	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
578	return cMbMemory;
579
580	def _getFreeScratchSpace(self):
581	"""
582	Get free space on the volume where scratch directory is located and
583	return it in bytes rounded down to nearest 64MB
584	(currently works on Linux only)
585	Unit: MiB.
586	"""
587	if platform.system() == 'Windows':
588	import ctypes
589	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
590	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
591	ctypes.pointer(cTypeMbFreeSpace))
592	cMbFreeSpace = cTypeMbFreeSpace.value
593	else:
594	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=E1101
595	cMbFreeSpace = stats.f_frsize * stats.f_bfree
596
597	# Convert to MB
598	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
599
600	# Round free space size
601	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
602	return cMbFreeSpace;
603
604	def _getScriptRev(self):
605	"""
606	The script (subversion) revision number.
607	"""
608	sRev = '@VBOX_SVN_REV@';
609	sRev = sRev.strip(); # just in case...
610	try:
611	_ = int(sRev);
612	except:
613	return __version__[11:-1].strip();
614	return sRev;
615
616	def _getPythonHexVersion(self):
617	"""
618	The python hex version number.
619	"""
620	uHexVersion = getattr(sys, 'hexversion', None);
621	if uHexVersion is None:
622	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
623	if sys.version_info[3] == 'final':
624	uHexVersion \|= 0xf0;
625	return uHexVersion;
626
627	# @}
628
629	def openTestManagerConnection(self):
630	"""
631	Opens up a connection to the test manager.
632
633	Raises exception on failure.
634	"""
635	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
636
637	def getSignOnParam(self, sName):
638	"""
639	Returns a sign-on parameter value as string.
640	Raises exception if the name is incorrect.
641	"""
642	return str(self._ddSignOnParams[sName][self.VALUE]);
643
644	def getPathState(self):
645	"""
646	Get the path to the state dir in the scratch area.
647	"""
648	return self._sScratchState;
649
650	def getPathScripts(self):
651	"""
652	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
653	"""
654	return self._sScratchScripts;
655
656	def getPathSpill(self):
657	"""
658	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
659	"""
660	return self._sScratchSpill;
661
662	def getPathBuilds(self):
663	"""
664	Get the path to the builds.
665	"""
666	return self._oOptions.sBuildsPath;
667
668	def getTestBoxId(self):
669	"""
670	Get the TestBox ID for state saving purposes.
671	"""
672	return self._idTestBox;
673
674	def getTestBoxName(self):
675	"""
676	Get the TestBox name for state saving purposes.
677	"""
678	return self._sTestBoxName;
679
680	def _reinitScratch(self, fnLog, fUseTheForce):
681	"""
682	Wipes the scratch directories and re-initializes them.
683
684	No exceptions raise, returns success indicator instead.
685	"""
686	if fUseTheForce is None:
687	fUseTheForce = self._fFirstSignOn;
688
689	class ErrorCallback(object): # pylint: disable=R0903
690	"""
691	Callbacks + state for the cleanup.
692	"""
693	def __init__(self):
694	self.fRc = True;
695	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
696	""" Logs error during shutil.rmtree operation. """
697	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
698	self.fRc = False;
699	oRc = ErrorCallback();
700
701	#
702	# Cleanup.
703	#
704	for sName in os.listdir(self._oOptions.sScratchRoot):
705	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
706	try:
707	if os.path.isdir(sFullName):
708	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
709	else:
710	os.remove(sFullName);
711	if os.path.exists(sFullName):
712	raise Exception('Still exists after deletion, weird.');
713	except Exception as oXcpt:
714	if fUseTheForce is True \
715	and utils.getHostOs() not in ['win', 'os2'] \
716	and len(sFullName) >= 8 \
717	and sFullName[0] == '/' \
718	and sFullName[1] != '/' \
719	and sFullName.find('/../') < 0:
720	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
721	try:
722	if os.path.isdir(sFullName):
723	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
724	else:
725	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
726	if iRc != 0:
727	raise Exception('exit code %s' % iRc);
728	if os.path.exists(sFullName):
729	raise Exception('Still exists after forced deletion, weird^2.');
730	except:
731	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
732	oRc.fRc = False;
733	else:
734	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
735	oRc.fRc = False;
736
737	# Display files left behind.
738	def dirEnumCallback(sName, oStat):
739	""" callback for dirEnumerateTree """
740	fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
741	utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
742
743	#
744	# Re-create the directories.
745	#
746	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
747	if not os.path.isdir(sDir):
748	try:
749	os.makedirs(sDir, 0o700);
750	except Exception as oXcpt:
751	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
752	oRc.fRc = False;
753
754	if oRc.fRc is True:
755	self._cReinitScratchErrors = 0;
756	else:
757	self._cReinitScratchErrors += 1;
758	return oRc.fRc;
759
760	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
761	"""
762	Wipes the scratch directories and re-initializes them.
763
764	Will retry according to the cRetries and cMsDelay parameters. Windows
765	forces us to apply this hack as it ships with services asynchronously
766	scanning files after they execute, thus racing us cleaning up after a
767	test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
768	vts_rm.exe kind of open, somehow preventing us from removing the
769	directory containing it, despite not issuing any errors deleting the
770	file itself. The service is called "Application Experience", which
771	feels like a weird joke here.
772
773	No exceptions raise, returns success indicator instead.
774	"""
775	fRc = self._reinitScratch(fnLog, fUseTheForce)
776	while fRc is False and cRetries > 0:
777	time.sleep(cMsDelay / 1000.0);
778	fnLog('reinitScratch: Retrying...');
779	fRc = self._reinitScratch(fnLog, fUseTheForce)
780	cRetries -= 1;
781	return fRc;
782
783
784	def _doSignOn(self):
785	"""
786	Worker for _maybeSignOn that does the actual signing on.
787	"""
788	assert not self._oCommand.isRunning();
789
790	# Reset the siged-on state.
791	testboxcommons.log('Signing-on...')
792	self._fSignedOn = False
793	self._idTestBox = None
794	self._cSignOnAttempts += 1;
795
796	# Assemble SIGN-ON request parameters and send the request.
797	dParams = {};
798	for sParam in self._ddSignOnParams:
799	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
800	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
801
802	# Check response.
803	try:
804	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
805	if sResult != constants.tbresp.STATUS_ACK:
806	raise TestBoxException('Result is %s' % (sResult,));
807	oResponse.checkParameterCount(3);
808	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
809	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
810	except TestBoxException as err:
811	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
812	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
813	return False;
814
815	# Successfully signed on, update the state.
816	self._fSignedOn = True;
817	self._fNeedReSignOn = False;
818	self._cSignOnAttempts = 0;
819	self._idTestBox = idTestBox;
820	self._sTestBoxName = sTestBoxName;
821
822	# Update the environment.
823	os.environ['TESTBOX_ID'] = str(self._idTestBox);
824	os.environ['TESTBOX_NAME'] = sTestBoxName;
825	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
826	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
827	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
828
829	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
830	% (self._idTestBox, self._sTestBoxName));
831
832	# Set up the scratch area.
833	self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
834
835	self._fFirstSignOn = False;
836	return True;
837
838	def _maybeSignOn(self):
839	"""
840	Check if Test Box parameters were changed
841	and do sign-in in case of positive result
842	"""
843
844	# Skip sign-on check if background command is currently in
845	# running state (avoid infinite signing on).
846	if self._oCommand.isRunning():
847	return None;
848
849	# Refresh sign-on parameters, changes triggers sign-on.
850	fNeedSignOn = (True if not self._fSignedOn or self._fNeedReSignOn else False)
851	for item in self._ddSignOnParams:
852	if self._ddSignOnParams[item][self.FN] is None:
853	continue
854
855	sOldValue = self._ddSignOnParams[item][self.VALUE]
856	self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
857	if sOldValue != self._ddSignOnParams[item][self.VALUE]:
858	fNeedSignOn = True
859	testboxcommons.log('Detected %s parameter change: %s -> %s' %
860	(item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
861
862	if fNeedSignOn:
863	self._doSignOn();
864	return None;
865
866	def dispatch(self):
867	"""
868	Receive orders from Test Manager and execute them
869	"""
870
871	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
872	self._fNeedReSignOn = self._fSignedOn;
873	if self._fSignedOn:
874	os.environ['TESTBOX_ID'] = str(self._idTestBox);
875	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
876
877	while True:
878	# Make sure we're signed on before trying to do anything.
879	self._maybeSignOn();
880	while not self._fSignedOn:
881	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
882	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
883	self._maybeSignOn();
884
885	# Retrieve and handle command from the TM.
886	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
887	self._idTestBox,
888	self._sTestBoxUuid,
889	self._oCommand.isRunning());
890	if oResponse is not None:
891	self._oCommand.handleCommand(oResponse, oConnection);
892	if oConnection is not None:
893	if oConnection.isConnected():
894	self._oCommand.flushLogOnConnection(oConnection);
895	oConnection.close();
896
897	# Automatically reboot if scratch init fails.
898	#if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
899	# testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
900	# % ( self._cReinitScratchErrors, ));
901	# self._oCommand.doReboot();
902
903	# delay a wee bit before looping.
904	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
905	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
906	iFactor = 1;
907	if self._cReinitScratchErrors > 0:
908	iFactor = 4;
909	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
910
911	# Not reached.
912
913
914	@staticmethod
915	def main():
916	"""
917	Main function a la C/C++. Returns exit code.
918	"""
919
920	#
921	# Parse arguments.
922	#
923	sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
924	if utils.getHostOs() in ('win', 'os2'):
925	sDefTestRsrc = 'T:';
926	sDefBuilds = 'U:';
927	elif utils.getHostOs() == 'darwin':
928	sDefTestRsrc = '/Volumes/testrsrc';
929	sDefBuilds = '/Volumes/builds';
930	else:
931	sDefTestRsrc = '/mnt/testrsrc';
932	sDefBuilds = '/mnt/builds';
933
934	class MyOptionParser(OptionParser):
935	""" We need to override the exit code on --help, error and so on. """
936	def __init__(self, args, *kwargs):
937	OptionParser.__init__(self, args, *kwargs);
938	def exit(self, status = 0, msg = None):
939	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
940
941	parser = MyOptionParser(version=__version__[11:-1].strip());
942	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
943	sLower = sMixed.lower();
944	sPrefix = 's' + sMixed;
945	parser.add_option('--' + sLower + '-path',
946	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
947	help='Where ' + sDesc + ' can be found');
948	parser.add_option('--' + sLower + '-server-type',
949	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=sDefShareType,
950	help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
951	parser.add_option('--' + sLower + '-server-name',
952	dest=sPrefix + 'ServerName', metavar='<server>',
953	default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
954	help='The name of the server with the builds.');
955	parser.add_option('--' + sLower + '-server-share',
956	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
957	help='The name of the builds share.');
958	parser.add_option('--' + sLower + '-server-user',
959	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
960	help='The user name to use when accessing the ' + sDesc + ' share.');
961	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
962	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
963	help='The password to use when accessing the ' + sDesc + ' share.');
964	parser.add_option('--' + sLower + '-server-mountopt',
965	dest=sPrefix + 'ServerMountOpt', metavar='<mountopt>', default='',
966	help='The mount options to use when accessing the ' + sDesc + ' share.');
967
968	parser.add_option("--test-manager", metavar="<url>",
969	dest="sTestManagerUrl",
970	help="Test Manager URL",
971	default="http://tindertux.de.oracle.com/testmanager")
972	parser.add_option("--scratch-root", metavar="<abs-path>",
973	dest="sScratchRoot",
974	help="Path to the scratch directory",
975	default=None)
976	parser.add_option("--system-uuid", metavar="<uuid>",
977	dest="sSystemUuid",
978	help="The system UUID of the testbox, used for uniquely identifiying the machine",
979	default=None)
980	parser.add_option("--hwvirt",
981	dest="fHasHwVirt", action="store_true", default=None,
982	help="Hardware virtualization available in the CPU");
983	parser.add_option("--no-hwvirt",
984	dest="fHasHwVirt", action="store_false", default=None,
985	help="Hardware virtualization not available in the CPU");
986	parser.add_option("--nested-paging",
987	dest="fHasNestedPaging", action="store_true", default=None,
988	help="Nested paging is available");
989	parser.add_option("--no-nested-paging",
990	dest="fHasNestedPaging", action="store_false", default=None,
991	help="Nested paging is not available");
992	parser.add_option("--64-bit-guest",
993	dest="fCan64BitGuest", action="store_true", default=None,
994	help="Host can execute 64-bit guests");
995	parser.add_option("--no-64-bit-guest",
996	dest="fCan64BitGuest", action="store_false", default=None,
997	help="Host cannot execute 64-bit guests");
998	parser.add_option("--io-mmu",
999	dest="fHasIoMmu", action="store_true", default=None,
1000	help="I/O MMU available");
1001	parser.add_option("--no-io-mmu",
1002	dest="fHasIoMmu", action="store_false", default=None,
1003	help="No I/O MMU available");
1004	parser.add_option("--raw-mode",
1005	dest="fWithRawMode", action="store_true", default=None,
1006	help="Use raw-mode on this host.");
1007	parser.add_option("--no-raw-mode",
1008	dest="fWithRawMode", action="store_false", default=None,
1009	help="Disables raw-mode tests on this host.");
1010	parser.add_option("--pidfile",
1011	dest="sPidFile", default=None,
1012	help="For the parent script, ignored.");
1013	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
1014	dest = "asEnvVars", default = [],
1015	help = "Sets an environment variable. Can be repeated.");
1016
1017	(oOptions, args) = parser.parse_args()
1018	# Check command line
1019	if args != []:
1020	parser.print_help();
1021	return TBS_EXITCODE_SYNTAX;
1022
1023	if oOptions.sSystemUuid is not None:
1024	uuid.UUID(oOptions.sSystemUuid);
1025	if not oOptions.sTestManagerUrl.startswith('http://') \
1026	and not oOptions.sTestManagerUrl.startswith('https://'):
1027	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1028	return TBS_EXITCODE_SYNTAX;
1029
1030	for sPrefix in ['sBuilds', 'sTestRsrc']:
1031	sType = getattr(oOptions, sPrefix + 'ServerType');
1032	if sType is None or not sType.strip():
1033	setattr(oOptions, sPrefix + 'ServerType', None);
1034	elif sType not in ['cifs', 'nfs']:
1035	print('Syntax error: Invalid server type "%s"' % (sType,));
1036	return TBS_EXITCODE_SYNTAX;
1037
1038
1039	#
1040	# Instantiate the testbox script and start dispatching work.
1041	#
1042	try:
1043	oTestBoxScript = TestBoxScript(oOptions);
1044	except TestBoxScriptException as oXcpt:
1045	print('Error: %s' % (oXcpt,));
1046	return TBS_EXITCODE_SYNTAX;
1047	oTestBoxScript.dispatch();
1048
1049	# Not supposed to get here...
1050	return TBS_EXITCODE_FAILURE;
1051
1052
1053
1054	if __name__ == '__main__':
1055	sys.exit(TestBoxScript.main());
1056

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 77994

Download in other formats: