testboxscript_real.py@ 70548

Last change on this file since 70548 was 70548, checked in by vboxsync, 7 years ago
testboxscript: Python 3 adjustments.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 45.6 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 70548 2018-01-11 20:46:02Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2017 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 70548 $"
31
32
33	# Standard python imports.
34	import math
35	import multiprocessing
36	import os
37	from optparse import OptionParser
38	import platform
39	import random
40	import shutil
41	import sys
42	import tempfile
43	import time
44	import uuid
45
46	# Only the main script needs to modify the path.
47	try: __file__
48	except: __file__ = sys.argv[0];
49	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
50	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
51	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
52
53	# Validation Kit imports.
54	from common import constants;
55	from common import utils;
56	import testboxcommons;
57	from testboxcommons import TestBoxException;
58	from testboxcommand import TestBoxCommand;
59	from testboxconnection import TestBoxConnection;
60	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
61
62	# Python 3 hacks:
63	if sys.version_info[0] >= 3:
64	long = int; # pylint: disable=redefined-builtin,invalid-name
65
66
67	class TestBoxScriptException(Exception):
68	""" For raising exceptions during TestBoxScript.__init__. """
69	pass;
70
71
72	class TestBoxScript(object):
73	"""
74	Implementation of the test box script.
75	Communicate with test manager and perform offered actions.
76	"""
77
78	## @name Class Constants.
79	# @{
80
81	# Scratch space round value (MB).
82	kcMbScratchSpaceRounding = 64
83	# Memory size round value (MB).
84	kcMbMemoryRounding = 4
85	# A NULL UUID in string form.
86	ksNullUuid = '00000000-0000-0000-0000-000000000000';
87	# The minimum dispatch loop delay.
88	kcSecMinDelay = 12;
89	# The maximum dispatch loop delay (inclusive).
90	kcSecMaxDelay = 24;
91	# The minimum sign-on delay.
92	kcSecMinSignOnDelay = 30;
93	# The maximum sign-on delay (inclusive).
94	kcSecMaxSignOnDelay = 60;
95
96	# Keys for config params
97	VALUE = 'value'
98	FN = 'fn' # pylint: disable=C0103
99
100	## @}
101
102
103	def __init__(self, oOptions):
104	"""
105	Initialize internals
106	"""
107	self._oOptions = oOptions;
108	self._sTestBoxHelper = None;
109
110	# Signed-on state
111	self._cSignOnAttempts = 0;
112	self._fSignedOn = False;
113	self._fNeedReSignOn = False;
114	self._fFirstSignOn = True;
115	self._idTestBox = None;
116	self._sTestBoxName = '';
117	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
118
119	# Command processor.
120	self._oCommand = TestBoxCommand(self);
121
122	#
123	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
124	# many many GBs for some test scenarios and /tmp can be backed by swap
125	# or be a fast+small disk of some kind, while /var/tmp is normally
126	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
127	# /tmp often is, this would break host panic / triple-fault detection.
128	#
129	if self._oOptions.sScratchRoot is None:
130	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
131	# We need lots of space, so avoid /tmp as it may be a memory
132	# file system backed by the swap file, or worse.
133	self._oOptions.sScratchRoot = tempfile.gettempdir();
134	else:
135	self._oOptions.sScratchRoot = '/var/tmp';
136	sSubDir = 'testbox';
137	try:
138	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
139	except:
140	pass;
141	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
142
143	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
144	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
145	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
146
147	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
148	if not os.path.isdir(sDir):
149	os.makedirs(sDir, 0o700);
150
151	# We count consecutive reinitScratch failures and will reboot the
152	# testbox after a while in the hope that it will correct the issue.
153	self._cReinitScratchErrors = 0;
154
155	#
156	# Mount builds and test resources if requested.
157	#
158	self.mountShares();
159
160	#
161	# Sign-on parameters: Packed into list of records of format:
162	# { <Parameter ID>: { <Current value>, <Check function> } }
163	#
164	self._ddSignOnParams = \
165	{
166	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
167	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
168	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
169	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
170	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
171	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
172	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
173	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
174	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
175	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
176	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
177	#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
178	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
179	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
180	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
181	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
182	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
183	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
184	}
185	for sItem in self._ddSignOnParams:
186	if self._ddSignOnParams[sItem][self.FN] is not None:
187	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
188
189	testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
190	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
191	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
192	for sItem in self._ddSignOnParams:
193	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
194
195	#
196	# The System UUID is the primary identification of the machine, so
197	# refuse to cooperate if it's NULL.
198	#
199	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
200	if self._sTestBoxUuid == self.ksNullUuid:
201	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
202
203	#
204	# Export environment variables, clearing any we don't know yet.
205	#
206	for sEnvVar in self._oOptions.asEnvVars:
207	iEqual = sEnvVar.find('=');
208	if iEqual == -1: # No '=', remove it.
209	if sEnvVar in os.environ:
210	del os.environ[sEnvVar];
211	elif iEqual > 0: # Set it.
212	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
213	else: # Starts with '=', bad user.
214	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
215
216	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
217	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
218	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
219	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
220	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
221	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
222	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
223	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
224	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
225	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
226	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
227	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
228	#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
229	os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
230	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
231	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
232	os.environ['TESTBOX_REPORTER'] = 'remote';
233	os.environ['TESTBOX_NAME'] = '';
234	os.environ['TESTBOX_ID'] = '';
235	os.environ['TESTBOX_TEST_SET_ID'] = '';
236	os.environ['TESTBOX_TIMEOUT'] = '0';
237	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
238
239	if utils.getHostOs() == 'win':
240	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
241	# Currently omitting any kBuild tools.
242
243	def mountShares(self):
244	"""
245	Mounts the shares.
246	Raises exception on failure.
247	"""
248	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
249	self._oOptions.sBuildsServerShare,
250	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd, 'builds');
251	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
252	self._oOptions.sTestRsrcServerShare,
253	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd, 'testrsrc');
254	return True;
255
256	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
257	"""
258	Mounts the specified share if needed.
259	Raises exception on failure.
260	"""
261	# Only mount if the type is specified.
262	if sType is None:
263	return True;
264
265	# Test if already mounted.
266	sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '-new.txt');
267	if os.path.isfile(sTestFile):
268	return True;
269
270	#
271	# Platform specific mount code.
272	#
273	sHostOs = utils.getHostOs()
274	if sHostOs in ('darwin', 'freebsd'):
275	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
276	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
277	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
278	if sType == 'cifs':
279	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
280	utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
281	'-f', '0555', '-d', '0555',
282	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
283	sMountPoint]);
284	else:
285	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
286
287	elif sHostOs == 'linux':
288	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
289	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
290	if sType == 'cifs':
291	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
292	'-o',
293	'user=' + sUser
294	+ ',password=' + sPassword
295	+ ',sec=ntlmv2'
296	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
297	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
298	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
299	'//%s/%s' % (sServer, sShare),
300	sMountPoint]);
301	elif sType == 'nfs':
302	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
303	'-o', 'soft,ro',
304	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
305	sMountPoint]);
306
307	else:
308	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
309
310	elif sHostOs == 'solaris':
311	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
312	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
313	if sType == 'cifs':
314	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
315	oPasswdFile = tempfile.TemporaryFile();
316	oPasswdFile.write(sPassword + '\n');
317	oPasswdFile.flush();
318	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
319	'-o',
320	'user=' + sUser
321	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
322	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
323	+ ',fileperms=0555,dirperms=0555,noxattr,ro',
324	'//%s/%s' % (sServer, sShare),
325	sMountPoint],
326	stdin = oPasswdFile);
327	oPasswdFile.close();
328	elif sType == 'nfs':
329	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
330	'-o', 'noxattr,ro',
331	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
332	sMountPoint]);
333
334	else:
335	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
336
337
338	elif sHostOs == 'win':
339	if sType != 'cifs':
340	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
341	utils.processCall(['net', 'use', sMountPoint, '/d']);
342	utils.processOutputChecked(['net', 'use', sMountPoint,
343	'\\\\' + sServer + '\\' + sShare,
344	sPassword,
345	'/USER:' + sUser,]);
346	else:
347	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
348
349	#
350	# Re-test.
351	#
352	if not os.path.isfile(sTestFile):
353	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
354	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
355
356	return True;
357
358	## @name Signon property releated methods.
359	# @{
360
361	def _getHelperOutput(self, sCmd):
362	"""
363	Invokes TestBoxHelper to obtain information hard to access from python.
364	"""
365	if self._sTestBoxHelper is None:
366	if not utils.isRunningFromCheckout():
367	# See VBoxTestBoxScript.zip for layout.
368	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
369	'TestBoxHelper');
370	else: # Only for in-tree testing, so don't bother be too accurate right now.
371	sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
372	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
373	utils.getHostOsDotArch(), sType, 'testboxscript', \
374	utils.getHostOs(), utils.getHostArch(), \
375	'TestBoxHelper');
376	if utils.getHostOs() in ['win', 'os2']:
377	self._sTestBoxHelper += '.exe';
378
379	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
380
381	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
382	"""
383	Invokes TestBoxHelper to obtain information hard to access from python.
384	"""
385	sValue = self._getHelperOutput(sCmd);
386	sValue = sValue.lower();
387	if sValue == 'true':
388	return True;
389	if sValue == 'false':
390	return False;
391	if sValue != 'dunno' and sValue != 'none':
392	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
393	return fDunnoValue;
394
395
396	@staticmethod
397	def _isUuidGood(sUuid):
398	"""
399	Checks if the UUID looks good.
400
401	There are systems with really bad UUIDs, for instance
402	"03000200-0400-0500-0006-000700080009".
403	"""
404	if sUuid == TestBoxScript.ksNullUuid:
405	return False;
406	sUuid = sUuid.lower();
407	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
408	if sUuid.count(sDigit) > 16:
409	return False;
410	return True;
411
412	def _getHostSystemUuid(self):
413	"""
414	Get the system UUID string from the System, return null-uuid if
415	unable to get retrieve it.
416	"""
417	if self._oOptions.sSystemUuid is not None:
418	return self._oOptions.sSystemUuid;
419
420	sUuid = self.ksNullUuid;
421
422	#
423	# Try get at the firmware UUID.
424	#
425	if utils.getHostOs() == 'linux':
426	# NOTE: This requires to have kernel option enabled:
427	# Firmware Drivers -> Export DMI identification via sysfs to userspace
428	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
429	try:
430	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
431	sUuid = str(uuid.UUID(sVar.strip()));
432	except:
433	pass;
434	## @todo consider dmidecoder? What about EFI systems?
435
436	elif utils.getHostOs() == 'win':
437	# Windows: WMI
438	try:
439	import win32com.client; # pylint: disable=F0401
440	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
441	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
442	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
443	if oItem.UUID != None:
444	sUuid = str(uuid.UUID(oItem.UUID));
445	except:
446	pass;
447
448	elif utils.getHostOs() == 'darwin':
449	try:
450	sVar = utils.processOutputChecked(['/bin/sh', '-c',
451	'/usr/sbin/ioreg -k IOPlatformUUID' \
452	+ '\| /usr/bin/grep IOPlatformUUID' \
453	+ '\| /usr/bin/head -1']);
454	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
455	sUuid = str(uuid.UUID(sVar));
456	except:
457	pass;
458
459	elif utils.getHostOs() == 'solaris':
460	# Solaris: The smbios util.
461	try:
462	sVar = utils.processOutputChecked(['/bin/sh', '-c',
463	'/usr/sbin/smbios ' \
464	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
465	+ '\| /usr/bin/head -1']);
466	sUuid = str(uuid.UUID(sVar.strip()));
467	except:
468	pass;
469
470	if self._isUuidGood(sUuid):
471	return sUuid;
472
473	#
474	# Try add the MAC address.
475	# uuid.getnode may provide it, or it may return a random number...
476	#
477	lMacAddr = uuid.getnode();
478	sNode = '%012x' % (lMacAddr,)
479	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
480	return sUuid[:-12] + sNode;
481
482	return sUuid;
483
484	def _getHostCpuVendor(self):
485	"""
486	Get the CPUID vendor string on intel HW.
487	"""
488	return self._getHelperOutput('cpuvendor');
489
490	def _getHostCpuName(self):
491	"""
492	Get the CPU name/description string.
493	"""
494	return self._getHelperOutput('cpuname');
495
496	def _getHostCpuRevision(self):
497	"""
498	Get the CPU revision (family/model/stepping) value.
499	"""
500	return self._getHelperOutput('cpurevision');
501
502	def _hasHostHwVirt(self):
503	"""
504	Check if the host supports AMD-V or VT-x
505	"""
506	if self._oOptions.fHasHwVirt is None:
507	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
508	return self._oOptions.fHasHwVirt;
509
510	def _hasHostNestedPaging(self):
511	"""
512	Check if the host supports nested paging.
513	"""
514	if not self._hasHostHwVirt():
515	return False;
516	if self._oOptions.fHasNestedPaging is None:
517	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
518	return self._oOptions.fHasNestedPaging;
519
520	def _can64BitGuest(self):
521	"""
522	Check if the we (VBox) can run 64-bit guests.
523	"""
524	if not self._hasHostHwVirt():
525	return False;
526	if self._oOptions.fCan64BitGuest is None:
527	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
528	return self._oOptions.fCan64BitGuest;
529
530	def _hasHostIoMmu(self):
531	"""
532	Check if the host has an I/O MMU of the VT-d kind.
533	"""
534	if not self._hasHostHwVirt():
535	return False;
536	if self._oOptions.fHasIoMmu is None:
537	## @todo Any way to figure this one out on any host OS?
538	self._oOptions.fHasIoMmu = False;
539	return self._oOptions.fHasIoMmu;
540
541	def _withRawModeSupport(self):
542	"""
543	Check if the testbox is configured with raw-mode support or not.
544	"""
545	if self._oOptions.fWithRawMode is None:
546	self._oOptions.fWithRawMode = True;
547	return self._oOptions.fWithRawMode;
548
549	def _getHostReport(self):
550	"""
551	Generate a report about the host hardware and software.
552	"""
553	return self._getHelperOutput('report');
554
555
556	def _getHostMemSize(self):
557	"""
558	Gets the amount of physical memory on the host (and accessible to the
559	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
560	Unit: MiB.
561	"""
562	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
563
564	# Round it.
565	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
566	return cMbMemory;
567
568	def _getFreeScratchSpace(self):
569	"""
570	Get free space on the volume where scratch directory is located and
571	return it in bytes rounded down to nearest 64MB
572	(currently works on Linux only)
573	Unit: MiB.
574	"""
575	if platform.system() == 'Windows':
576	import ctypes
577	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
578	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
579	ctypes.pointer(cTypeMbFreeSpace))
580	cMbFreeSpace = cTypeMbFreeSpace.value
581	else:
582	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=E1101
583	cMbFreeSpace = stats.f_frsize * stats.f_bfree
584
585	# Convert to MB
586	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
587
588	# Round free space size
589	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
590	return cMbFreeSpace;
591
592	def _getScriptRev(self):
593	"""
594	The script (subversion) revision number.
595	"""
596	sRev = '@VBOX_SVN_REV@';
597	sRev = sRev.strip(); # just in case...
598	try:
599	_ = int(sRev);
600	except:
601	return __version__[11:-1].strip();
602	return sRev;
603
604	def _getPythonHexVersion(self):
605	"""
606	The python hex version number.
607	"""
608	uHexVersion = getattr(sys, 'hexversion', None);
609	if uHexVersion is None:
610	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
611	if sys.version_info[3] == 'final':
612	uHexVersion \|= 0xf0;
613	return uHexVersion;
614
615	# @}
616
617	def openTestManagerConnection(self):
618	"""
619	Opens up a connection to the test manager.
620
621	Raises exception on failure.
622	"""
623	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
624
625	def getSignOnParam(self, sName):
626	"""
627	Returns a sign-on parameter value as string.
628	Raises exception if the name is incorrect.
629	"""
630	return str(self._ddSignOnParams[sName][self.VALUE]);
631
632	def getPathState(self):
633	"""
634	Get the path to the state dir in the scratch area.
635	"""
636	return self._sScratchState;
637
638	def getPathScripts(self):
639	"""
640	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
641	"""
642	return self._sScratchScripts;
643
644	def getPathSpill(self):
645	"""
646	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
647	"""
648	return self._sScratchSpill;
649
650	def getPathBuilds(self):
651	"""
652	Get the path to the builds.
653	"""
654	return self._oOptions.sBuildsPath;
655
656	def getTestBoxId(self):
657	"""
658	Get the TestBox ID for state saving purposes.
659	"""
660	return self._idTestBox;
661
662	def getTestBoxName(self):
663	"""
664	Get the TestBox name for state saving purposes.
665	"""
666	return self._sTestBoxName;
667
668	def _reinitScratch(self, fnLog, fUseTheForce):
669	"""
670	Wipes the scratch directories and re-initializes them.
671
672	No exceptions raise, returns success indicator instead.
673	"""
674	if fUseTheForce is None:
675	fUseTheForce = self._fFirstSignOn;
676
677	class ErrorCallback(object): # pylint: disable=R0903
678	"""
679	Callbacks + state for the cleanup.
680	"""
681	def __init__(self):
682	self.fRc = True;
683	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
684	""" Logs error during shutil.rmtree operation. """
685	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
686	self.fRc = False;
687	oRc = ErrorCallback();
688
689	#
690	# Cleanup.
691	#
692	for sName in os.listdir(self._oOptions.sScratchRoot):
693	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
694	try:
695	if os.path.isdir(sFullName):
696	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
697	else:
698	os.remove(sFullName);
699	if os.path.exists(sFullName):
700	raise Exception('Still exists after deletion, weird.');
701	except Exception as oXcpt:
702	if fUseTheForce is True \
703	and utils.getHostOs() not in ['win', 'os2'] \
704	and len(sFullName) >= 8 \
705	and sFullName[0] == '/' \
706	and sFullName[1] != '/' \
707	and sFullName.find('/../') < 0:
708	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
709	try:
710	if os.path.isdir(sFullName):
711	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
712	else:
713	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
714	if iRc != 0:
715	raise Exception('exit code %s' % iRc);
716	if os.path.exists(sFullName):
717	raise Exception('Still exists after forced deletion, weird^2.');
718	except:
719	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
720	oRc.fRc = False;
721	else:
722	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
723	oRc.fRc = False;
724
725	# Display files left behind.
726	def dirEnumCallback(sName, oStat):
727	""" callback for dirEnumerateTree """
728	fnLog(u'%s %s' % (utils.formatFileStat(oStat) if oStat is not None else '????????????', sName));
729	utils.dirEnumerateTree(self._oOptions.sScratchRoot, dirEnumCallback);
730
731	#
732	# Re-create the directories.
733	#
734	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
735	if not os.path.isdir(sDir):
736	try:
737	os.makedirs(sDir, 0o700);
738	except Exception as oXcpt:
739	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
740	oRc.fRc = False;
741
742	if oRc.fRc is True:
743	self._cReinitScratchErrors = 0;
744	else:
745	self._cReinitScratchErrors += 1;
746	return oRc.fRc;
747
748	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None, cRetries = 0, cMsDelay = 5000):
749	"""
750	Wipes the scratch directories and re-initializes them.
751
752	Will retry according to the cRetries and cMsDelay parameters. Windows
753	forces us to apply this hack as it ships with services asynchronously
754	scanning files after they execute, thus racing us cleaning up after a
755	test. On testboxwin3 we had frequent trouble with aelupsvc.dll keeping
756	vts_rm.exe kind of open, somehow preventing us from removing the
757	directory containing it, despite not issuing any errors deleting the
758	file itself. The service is called "Application Experience", which
759	feels like a weird joke here.
760
761	No exceptions raise, returns success indicator instead.
762	"""
763	fRc = self._reinitScratch(fnLog, fUseTheForce)
764	while fRc is False and cRetries > 0:
765	time.sleep(cMsDelay / 1000.0);
766	fnLog('reinitScratch: Retrying...');
767	fRc = self._reinitScratch(fnLog, fUseTheForce)
768	cRetries -= 1;
769	return fRc;
770
771
772	def _doSignOn(self):
773	"""
774	Worker for _maybeSignOn that does the actual signing on.
775	"""
776	assert not self._oCommand.isRunning();
777
778	# Reset the siged-on state.
779	testboxcommons.log('Signing-on...')
780	self._fSignedOn = False
781	self._idTestBox = None
782	self._cSignOnAttempts += 1;
783
784	# Assemble SIGN-ON request parameters and send the request.
785	dParams = {};
786	for sParam in self._ddSignOnParams:
787	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
788	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
789
790	# Check response.
791	try:
792	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
793	if sResult != constants.tbresp.STATUS_ACK:
794	raise TestBoxException('Result is %s' % (sResult,));
795	oResponse.checkParameterCount(3);
796	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
797	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
798	except TestBoxException as err:
799	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
800	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
801	return False;
802
803	# Successfully signed on, update the state.
804	self._fSignedOn = True;
805	self._fNeedReSignOn = False;
806	self._cSignOnAttempts = 0;
807	self._idTestBox = idTestBox;
808	self._sTestBoxName = sTestBoxName;
809
810	# Update the environment.
811	os.environ['TESTBOX_ID'] = str(self._idTestBox);
812	os.environ['TESTBOX_NAME'] = sTestBoxName;
813	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
814	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
815	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
816
817	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
818	% (self._idTestBox, self._sTestBoxName));
819
820	# Set up the scratch area.
821	self.reinitScratch(fUseTheForce = self._fFirstSignOn, cRetries = 2);
822
823	self._fFirstSignOn = False;
824	return True;
825
826	def _maybeSignOn(self):
827	"""
828	Check if Test Box parameters were changed
829	and do sign-in in case of positive result
830	"""
831
832	# Skip sign-on check if background command is currently in
833	# running state (avoid infinite signing on).
834	if self._oCommand.isRunning():
835	return None;
836
837	# Refresh sign-on parameters, changes triggers sign-on.
838	fNeedSignOn = (True if not self._fSignedOn or self._fNeedReSignOn else False)
839	for item in self._ddSignOnParams:
840	if self._ddSignOnParams[item][self.FN] is None:
841	continue
842
843	sOldValue = self._ddSignOnParams[item][self.VALUE]
844	self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
845	if sOldValue != self._ddSignOnParams[item][self.VALUE]:
846	fNeedSignOn = True
847	testboxcommons.log('Detected %s parameter change: %s -> %s' %
848	(item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
849
850	if fNeedSignOn:
851	self._doSignOn();
852	return None;
853
854	def dispatch(self):
855	"""
856	Receive orders from Test Manager and execute them
857	"""
858
859	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
860	self._fNeedReSignOn = self._fSignedOn;
861	if self._fSignedOn:
862	os.environ['TESTBOX_ID'] = str(self._idTestBox);
863	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
864
865	while True:
866	# Make sure we're signed on before trying to do anything.
867	self._maybeSignOn();
868	while not self._fSignedOn:
869	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
870	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
871	self._maybeSignOn();
872
873	# Retrieve and handle command from the TM.
874	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
875	self._idTestBox,
876	self._sTestBoxUuid,
877	self._oCommand.isRunning());
878	if oResponse is not None:
879	self._oCommand.handleCommand(oResponse, oConnection);
880	if oConnection is not None:
881	if oConnection.isConnected():
882	self._oCommand.flushLogOnConnection(oConnection);
883	oConnection.close();
884
885	# Automatically reboot if scratch init fails.
886	#if self._cReinitScratchErrors > 8 and self.reinitScratch(cRetries = 3) is False:
887	# testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
888	# % ( self._cReinitScratchErrors, ));
889	# self._oCommand.doReboot();
890
891	# delay a wee bit before looping.
892	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
893	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
894	iFactor = 1;
895	if self._cReinitScratchErrors > 0:
896	iFactor = 4;
897	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
898
899	# Not reached.
900
901
902	@staticmethod
903	def main():
904	"""
905	Main function a la C/C++. Returns exit code.
906	"""
907
908	#
909	# Parse arguments.
910	#
911	sDefShareType = 'nfs' if utils.getHostOs() == 'solaris' else 'cifs';
912	if utils.getHostOs() in ('win', 'os2'):
913	sDefTestRsrc = 'T:';
914	sDefBuilds = 'U:';
915	elif utils.getHostOs() == 'darwin':
916	sDefTestRsrc = '/Volumes/testrsrc';
917	sDefBuilds = '/Volumes/builds';
918	else:
919	sDefTestRsrc = '/mnt/testrsrc';
920	sDefBuilds = '/mnt/builds';
921
922	class MyOptionParser(OptionParser):
923	""" We need to override the exit code on --help, error and so on. """
924	def __init__(self, args, *kwargs):
925	OptionParser.__init__(self, args, *kwargs);
926	def exit(self, status = 0, msg = None):
927	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
928
929	parser = MyOptionParser(version=__version__[11:-1].strip());
930	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
931	sLower = sMixed.lower();
932	sPrefix = 's' + sMixed;
933	parser.add_option('--' + sLower + '-path',
934	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
935	help='Where ' + sDesc + ' can be found');
936	parser.add_option('--' + sLower + '-server-type',
937	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=sDefShareType,
938	help='The type of server, cifs (default) or nfs. If empty, we won\'t try mount anything.');
939	parser.add_option('--' + sLower + '-server-name',
940	dest=sPrefix + 'ServerName', metavar='<server>',
941	default='vboxstor.de.oracle.com' if sLower == 'builds' else 'teststor.de.oracle.com',
942	help='The name of the server with the builds.');
943	parser.add_option('--' + sLower + '-server-share',
944	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
945	help='The name of the builds share.');
946	parser.add_option('--' + sLower + '-server-user',
947	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
948	help='The user name to use when accessing the ' + sDesc + ' share.');
949	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
950	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
951	help='The password to use when accessing the ' + sDesc + ' share.');
952
953	parser.add_option("--test-manager", metavar="<url>",
954	dest="sTestManagerUrl",
955	help="Test Manager URL",
956	default="http://tindertux.de.oracle.com/testmanager")
957	parser.add_option("--scratch-root", metavar="<abs-path>",
958	dest="sScratchRoot",
959	help="Path to the scratch directory",
960	default=None)
961	parser.add_option("--system-uuid", metavar="<uuid>",
962	dest="sSystemUuid",
963	help="The system UUID of the testbox, used for uniquely identifiying the machine",
964	default=None)
965	parser.add_option("--hwvirt",
966	dest="fHasHwVirt", action="store_true", default=None,
967	help="Hardware virtualization available in the CPU");
968	parser.add_option("--no-hwvirt",
969	dest="fHasHwVirt", action="store_false", default=None,
970	help="Hardware virtualization not available in the CPU");
971	parser.add_option("--nested-paging",
972	dest="fHasNestedPaging", action="store_true", default=None,
973	help="Nested paging is available");
974	parser.add_option("--no-nested-paging",
975	dest="fHasNestedPaging", action="store_false", default=None,
976	help="Nested paging is not available");
977	parser.add_option("--64-bit-guest",
978	dest="fCan64BitGuest", action="store_true", default=None,
979	help="Host can execute 64-bit guests");
980	parser.add_option("--no-64-bit-guest",
981	dest="fCan64BitGuest", action="store_false", default=None,
982	help="Host cannot execute 64-bit guests");
983	parser.add_option("--io-mmu",
984	dest="fHasIoMmu", action="store_true", default=None,
985	help="I/O MMU available");
986	parser.add_option("--no-io-mmu",
987	dest="fHasIoMmu", action="store_false", default=None,
988	help="No I/O MMU available");
989	parser.add_option("--raw-mode",
990	dest="fWithRawMode", action="store_true", default=None,
991	help="Use raw-mode on this host.");
992	parser.add_option("--no-raw-mode",
993	dest="fWithRawMode", action="store_false", default=None,
994	help="Disables raw-mode tests on this host.");
995	parser.add_option("--pidfile",
996	dest="sPidFile", default=None,
997	help="For the parent script, ignored.");
998	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
999	dest = "asEnvVars", default = [],
1000	help = "Sets an environment variable. Can be repeated.");
1001
1002	(oOptions, args) = parser.parse_args()
1003	# Check command line
1004	if args != []:
1005	parser.print_help();
1006	return TBS_EXITCODE_SYNTAX;
1007
1008	if oOptions.sSystemUuid is not None:
1009	uuid.UUID(oOptions.sSystemUuid);
1010	if not oOptions.sTestManagerUrl.startswith('http://') \
1011	and not oOptions.sTestManagerUrl.startswith('https://'):
1012	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
1013	return TBS_EXITCODE_SYNTAX;
1014
1015	for sPrefix in ['sBuilds', 'sTestRsrc']:
1016	sType = getattr(oOptions, sPrefix + 'ServerType');
1017	if sType is None or not sType.strip():
1018	setattr(oOptions, sPrefix + 'ServerType', None);
1019	elif sType not in ['cifs', 'nfs']:
1020	print('Syntax error: Invalid server type "%s"' % (sType,));
1021	return TBS_EXITCODE_SYNTAX;
1022
1023
1024	#
1025	# Instantiate the testbox script and start dispatching work.
1026	#
1027	try:
1028	oTestBoxScript = TestBoxScript(oOptions);
1029	except TestBoxScriptException as oXcpt:
1030	print('Error: %s' % (oXcpt,));
1031	return TBS_EXITCODE_SYNTAX;
1032	oTestBoxScript.dispatch();
1033
1034	# Not supposed to get here...
1035	return TBS_EXITCODE_FAILURE;
1036
1037
1038
1039	if __name__ == '__main__':
1040	sys.exit(TestBoxScript.main());
1041

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 70548

Download in other formats: