testboxscript_real.py@ 61616

Last change on this file since 61616 was 61595, checked in by vboxsync, 9 years ago
testboxscript_real.py: Before I forget, make sure we report the tree revision to the testmanager and not the last time we changed the main source file.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 43.9 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 61595 2016-06-08 20:31:49Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2016 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 61595 $"
31
32
33	# Standard python imports.
34	import math
35	import multiprocessing
36	import os
37	from optparse import OptionParser
38	import platform
39	import random
40	import shutil
41	import sys
42	import tempfile
43	import time
44	import uuid
45
46	# Only the main script needs to modify the path.
47	try: __file__
48	except: __file__ = sys.argv[0];
49	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
50	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
51	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
52
53	# Validation Kit imports.
54	from common import constants;
55	from common import utils;
56	import testboxcommons;
57	from testboxcommons import TestBoxException;
58	from testboxcommand import TestBoxCommand;
59	from testboxconnection import TestBoxConnection;
60	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
61
62
63	class TestBoxScriptException(Exception):
64	""" For raising exceptions during TestBoxScript.__init__. """
65	pass;
66
67
68	class TestBoxScript(object):
69	"""
70	Implementation of the test box script.
71	Communicate with test manager and perform offered actions.
72	"""
73
74	## @name Class Constants.
75	# @{
76
77	# Scratch space round value (MB).
78	kcMbScratchSpaceRounding = 64
79	# Memory size round value (MB).
80	kcMbMemoryRounding = 4
81	# A NULL UUID in string form.
82	ksNullUuid = '00000000-0000-0000-0000-000000000000';
83	# The minimum dispatch loop delay.
84	kcSecMinDelay = 12;
85	# The maximum dispatch loop delay (inclusive).
86	kcSecMaxDelay = 24;
87	# The minimum sign-on delay.
88	kcSecMinSignOnDelay = 30;
89	# The maximum sign-on delay (inclusive).
90	kcSecMaxSignOnDelay = 60;
91
92	# Keys for config params
93	VALUE = 'value'
94	FN = 'fn' # pylint: disable=C0103
95
96	## @}
97
98
99	def __init__(self, oOptions):
100	"""
101	Initialize internals
102	"""
103	self._oOptions = oOptions;
104	self._sTestBoxHelper = None;
105
106	# Signed-on state
107	self._cSignOnAttempts = 0;
108	self._fSignedOn = False;
109	self._fNeedReSignOn = False;
110	self._fFirstSignOn = True;
111	self._idTestBox = None;
112	self._sTestBoxName = '';
113	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
114
115	# Command processor.
116	self._oCommand = TestBoxCommand(self);
117
118	#
119	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
120	# many many GBs for some test scenarios and /tmp can be backed by swap
121	# or be a fast+small disk of some kind, while /var/tmp is normally
122	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
123	# /tmp often is, this would break host panic / triple-fault detection.
124	#
125	if self._oOptions.sScratchRoot is None:
126	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
127	# We need lots of space, so avoid /tmp as it may be a memory
128	# file system backed by the swap file, or worse.
129	self._oOptions.sScratchRoot = tempfile.gettempdir();
130	else:
131	self._oOptions.sScratchRoot = '/var/tmp';
132	sSubDir = 'testbox';
133	try:
134	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
135	except:
136	pass;
137	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
138
139	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
140	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
141	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
142
143	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
144	if not os.path.isdir(sDir):
145	os.makedirs(sDir, 0700);
146
147	# We count consecutive reinitScratch failures and will reboot the
148	# testbox after a while in the hope that it will correct the issue.
149	self._cReinitScratchErrors = 0;
150
151	#
152	# Mount builds and test resources if requested.
153	#
154	self.mountShares();
155
156	#
157	# Sign-on parameters: Packed into list of records of format:
158	# { <Parameter ID>: { <Current value>, <Check function> } }
159	#
160	self._ddSignOnParams = \
161	{
162	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
163	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
164	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
165	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
166	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
167	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
168	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
169	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
170	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
171	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
172	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
173	#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
174	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
175	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
176	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
177	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
178	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
179	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
180	}
181	for sItem in self._ddSignOnParams:
182	if self._ddSignOnParams[sItem][self.FN] is not None:
183	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
184
185	testboxcommons.log('Starting Test Box script (%s)' % (self._getScriptRev(),));
186	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
187	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
188	for sItem in self._ddSignOnParams:
189	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
190
191	#
192	# The System UUID is the primary identification of the machine, so
193	# refuse to cooperate if it's NULL.
194	#
195	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
196	if self._sTestBoxUuid == self.ksNullUuid:
197	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
198
199	#
200	# Export environment variables, clearing any we don't know yet.
201	#
202	for sEnvVar in self._oOptions.asEnvVars:
203	iEqual = sEnvVar.find('=');
204	if iEqual == -1: # No '=', remove it.
205	if sEnvVar in os.environ:
206	del os.environ[sEnvVar];
207	elif iEqual > 0: # Set it.
208	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
209	else: # Starts with '=', bad user.
210	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
211
212	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
213	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
214	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
215	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
216	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
217	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
218	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
219	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
220	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
221	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
222	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
223	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
224	#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
225	os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
226	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
227	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
228	os.environ['TESTBOX_REPORTER'] = 'remote';
229	os.environ['TESTBOX_NAME'] = '';
230	os.environ['TESTBOX_ID'] = '';
231	os.environ['TESTBOX_TEST_SET_ID'] = '';
232	os.environ['TESTBOX_TIMEOUT'] = '0';
233	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
234
235	if utils.getHostOs() is 'win':
236	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
237	# Currently omitting any kBuild tools.
238
239	def mountShares(self):
240	"""
241	Mounts the shares.
242	Raises exception on failure.
243	"""
244	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
245	self._oOptions.sBuildsServerShare,
246	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd, 'builds');
247	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
248	self._oOptions.sTestRsrcServerShare,
249	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd, 'testrsrc');
250	return True;
251
252	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
253	"""
254	Mounts the specified share if needed.
255	Raises exception on failure.
256	"""
257	# Only mount if the type is specified.
258	if sType is None:
259	return True;
260
261	# Test if already mounted.
262	sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '.txt');
263	if os.path.isfile(sTestFile):
264	return True;
265
266	#
267	# Platform specific mount code.
268	#
269	sHostOs = utils.getHostOs()
270	if sHostOs in ('darwin', 'freebsd'):
271	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
272	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
273	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
274	if sType == 'cifs':
275	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
276	utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
277	'-f', '0555', '-d', '0555',
278	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
279	sMountPoint]);
280	else:
281	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
282
283	elif sHostOs == 'linux':
284	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
285	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
286	if sType == 'cifs':
287	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
288	'-o',
289	'user=' + sUser
290	+ ',password=' + sPassword
291	+ ',sec=ntlmv2'
292	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
293	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
294	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
295	'//%s/%s' % (sServer, sShare),
296	sMountPoint]);
297	elif sType == 'nfs':
298	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
299	'-o', 'soft,ro',
300	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
301	sMountPoint]);
302
303	else:
304	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
305
306	elif sHostOs == 'solaris':
307	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
308	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
309	if sType == 'cifs':
310	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
311	oPasswdFile = tempfile.TemporaryFile();
312	oPasswdFile.write(sPassword + '\n');
313	oPasswdFile.flush();
314	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
315	'-o',
316	'user=' + sUser
317	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
318	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
319	+ ',fileperms=0555,dirperms=0555,noxattr,ro',
320	'//%s/%s' % (sServer, sShare),
321	sMountPoint],
322	stdin = oPasswdFile);
323	oPasswdFile.close();
324	elif sType == 'nfs':
325	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
326	'-o', 'noxattr,ro',
327	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
328	sMountPoint]);
329
330	else:
331	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
332
333
334	elif sHostOs == 'win':
335	if sType != 'cifs':
336	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
337	utils.processCall(['net', 'use', sMountPoint, '/d']);
338	utils.processOutputChecked(['net', 'use', sMountPoint,
339	'\\\\' + sServer + '\\' + sShare,
340	sPassword,
341	'/USER:' + sUser,]);
342	else:
343	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
344
345	#
346	# Re-test.
347	#
348	if not os.path.isfile(sTestFile):
349	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
350	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
351
352	return True;
353
354	## @name Signon property releated methods.
355	# @{
356
357	def _getHelperOutput(self, sCmd):
358	"""
359	Invokes TestBoxHelper to obtain information hard to access from python.
360	"""
361	if self._sTestBoxHelper is None:
362	if not utils.isRunningFromCheckout():
363	# See VBoxTestBoxScript.zip for layout.
364	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
365	'TestBoxHelper');
366	else: # Only for in-tree testing, so don't bother be too accurate right now.
367	sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
368	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
369	utils.getHostOsDotArch(), sType, 'testboxscript', \
370	utils.getHostOs(), utils.getHostArch(), \
371	'TestBoxHelper');
372	if utils.getHostOs() in ['win', 'os2']:
373	self._sTestBoxHelper += '.exe';
374
375	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
376
377	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
378	"""
379	Invokes TestBoxHelper to obtain information hard to access from python.
380	"""
381	sValue = self._getHelperOutput(sCmd);
382	sValue = sValue.lower();
383	if sValue == 'true':
384	return True;
385	if sValue == 'false':
386	return False;
387	if sValue != 'dunno' and sValue != 'none':
388	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
389	return fDunnoValue;
390
391
392	@staticmethod
393	def _isUuidGood(sUuid):
394	"""
395	Checks if the UUID looks good.
396
397	There are systems with really bad UUIDs, for instance
398	"03000200-0400-0500-0006-000700080009".
399	"""
400	if sUuid == TestBoxScript.ksNullUuid:
401	return False;
402	sUuid = sUuid.lower();
403	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
404	if sUuid.count(sDigit) > 16:
405	return False;
406	return True;
407
408	def _getHostSystemUuid(self):
409	"""
410	Get the system UUID string from the System, return null-uuid if
411	unable to get retrieve it.
412	"""
413	if self._oOptions.sSystemUuid is not None:
414	return self._oOptions.sSystemUuid;
415
416	sUuid = self.ksNullUuid;
417
418	#
419	# Try get at the firmware UUID.
420	#
421	if utils.getHostOs() == 'linux':
422	# NOTE: This requires to have kernel option enabled:
423	# Firmware Drivers -> Export DMI identification via sysfs to userspace
424	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
425	try:
426	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
427	sUuid = str(uuid.UUID(sVar.strip()));
428	except:
429	pass;
430	## @todo consider dmidecoder? What about EFI systems?
431
432	elif utils.getHostOs() == 'win':
433	# Windows: WMI
434	try:
435	import win32com.client; # pylint: disable=F0401
436	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
437	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
438	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
439	if oItem.UUID != None:
440	sUuid = str(uuid.UUID(oItem.UUID));
441	except:
442	pass;
443
444	elif utils.getHostOs() == 'darwin':
445	try:
446	sVar = utils.processOutputChecked(['/bin/sh', '-c',
447	'/usr/sbin/ioreg -k IOPlatformUUID' \
448	+ '\| /usr/bin/grep IOPlatformUUID' \
449	+ '\| /usr/bin/head -1']);
450	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
451	sUuid = str(uuid.UUID(sVar));
452	except:
453	pass;
454
455	elif utils.getHostOs() == 'solaris':
456	# Solaris: The smbios util.
457	try:
458	sVar = utils.processOutputChecked(['/bin/sh', '-c',
459	'/usr/sbin/smbios ' \
460	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
461	+ '\| /usr/bin/head -1']);
462	sUuid = str(uuid.UUID(sVar.strip()));
463	except:
464	pass;
465
466	if self._isUuidGood(sUuid):
467	return sUuid;
468
469	#
470	# Try add the MAC address.
471	# uuid.getnode may provide it, or it may return a random number...
472	#
473	lMacAddr = uuid.getnode();
474	sNode = '%012x' % (lMacAddr,)
475	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
476	return sUuid[:-12] + sNode;
477
478	return sUuid;
479
480	def _getHostCpuVendor(self):
481	"""
482	Get the CPUID vendor string on intel HW.
483	"""
484	return self._getHelperOutput('cpuvendor');
485
486	def _getHostCpuName(self):
487	"""
488	Get the CPU name/description string.
489	"""
490	return self._getHelperOutput('cpuname');
491
492	def _getHostCpuRevision(self):
493	"""
494	Get the CPU revision (family/model/stepping) value.
495	"""
496	return self._getHelperOutput('cpurevision');
497
498	def _hasHostHwVirt(self):
499	"""
500	Check if the host supports AMD-V or VT-x
501	"""
502	if self._oOptions.fHasHwVirt is None:
503	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
504	return self._oOptions.fHasHwVirt;
505
506	def _hasHostNestedPaging(self):
507	"""
508	Check if the host supports nested paging.
509	"""
510	if not self._hasHostHwVirt():
511	return False;
512	if self._oOptions.fHasNestedPaging is None:
513	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
514	return self._oOptions.fHasNestedPaging;
515
516	def _can64BitGuest(self):
517	"""
518	Check if the we (VBox) can run 64-bit guests.
519	"""
520	if not self._hasHostHwVirt():
521	return False;
522	if self._oOptions.fCan64BitGuest is None:
523	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
524	return self._oOptions.fCan64BitGuest;
525
526	def _hasHostIoMmu(self):
527	"""
528	Check if the host has an I/O MMU of the VT-d kind.
529	"""
530	if not self._hasHostHwVirt():
531	return False;
532	if self._oOptions.fHasIoMmu is None:
533	## @todo Any way to figure this one out on any host OS?
534	self._oOptions.fHasIoMmu = False;
535	return self._oOptions.fHasIoMmu;
536
537	def _withRawModeSupport(self):
538	"""
539	Check if the testbox is configured with raw-mode support or not.
540	"""
541	if self._oOptions.fWithRawMode is None:
542	self._oOptions.fWithRawMode = True;
543	return self._oOptions.fWithRawMode;
544
545	def _getHostReport(self):
546	"""
547	Generate a report about the host hardware and software.
548	"""
549	return self._getHelperOutput('report');
550
551
552	def _getHostMemSize(self):
553	"""
554	Gets the amount of physical memory on the host (and accessible to the
555	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
556	Unit: MiB.
557	"""
558	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
559
560	# Round it.
561	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
562	return cMbMemory;
563
564	def _getFreeScratchSpace(self):
565	"""
566	Get free space on the volume where scratch directory is located and
567	return it in bytes rounded down to nearest 64MB
568	(currently works on Linux only)
569	Unit: MiB.
570	"""
571	if platform.system() == 'Windows':
572	import ctypes
573	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
574	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
575	ctypes.pointer(cTypeMbFreeSpace))
576	cMbFreeSpace = cTypeMbFreeSpace.value
577	else:
578	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=E1101
579	cMbFreeSpace = stats.f_frsize * stats.f_bfree
580
581	# Convert to MB
582	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
583
584	# Round free space size
585	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
586	return cMbFreeSpace;
587
588	def _getScriptRev(self):
589	"""
590	The script (subversion) revision number.
591	"""
592	sRev = '@VBOX_SVN_REV@';
593	sRev = sRev.strip(); # just in case...
594	try:
595	_ = int(sRev);
596	except:
597	return __version__[11:-1].strip();
598	return sRev;
599
600	def _getPythonHexVersion(self):
601	"""
602	The python hex version number.
603	"""
604	uHexVersion = getattr(sys, 'hexversion', None);
605	if uHexVersion is None:
606	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
607	if sys.version_info[3] == 'final':
608	uHexVersion \|= 0xf0;
609	return uHexVersion;
610
611	# @}
612
613	def openTestManagerConnection(self):
614	"""
615	Opens up a connection to the test manager.
616
617	Raises exception on failure.
618	"""
619	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
620
621	def getSignOnParam(self, sName):
622	"""
623	Returns a sign-on parameter value as string.
624	Raises exception if the name is incorrect.
625	"""
626	return str(self._ddSignOnParams[sName][self.VALUE]);
627
628	def getPathState(self):
629	"""
630	Get the path to the state dir in the scratch area.
631	"""
632	return self._sScratchState;
633
634	def getPathScripts(self):
635	"""
636	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
637	"""
638	return self._sScratchScripts;
639
640	def getPathSpill(self):
641	"""
642	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
643	"""
644	return self._sScratchSpill;
645
646	def getPathBuilds(self):
647	"""
648	Get the path to the builds.
649	"""
650	return self._oOptions.sBuildsPath;
651
652	def getTestBoxId(self):
653	"""
654	Get the TestBox ID for state saving purposes.
655	"""
656	return self._idTestBox;
657
658	def getTestBoxName(self):
659	"""
660	Get the TestBox name for state saving purposes.
661	"""
662	return self._sTestBoxName;
663
664	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None):
665	"""
666	Wipes the scratch directories and re-initializes them.
667
668	No exceptions raise, returns success indicator instead.
669	"""
670	if fUseTheForce is None:
671	fUseTheForce = self._fFirstSignOn;
672
673	class ErrorCallback(object): # pylint: disable=R0903
674	"""
675	Callbacks + state for the cleanup.
676	"""
677	def __init__(self):
678	self.fRc = True;
679	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
680	""" Logs error during shutil.rmtree operation. """
681	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
682	self.fRc = False;
683	oRc = ErrorCallback();
684
685	#
686	# Cleanup.
687	#
688	for sName in os.listdir(self._oOptions.sScratchRoot):
689	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
690	try:
691	if os.path.isdir(sFullName):
692	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
693	else:
694	os.remove(sFullName);
695	if os.path.exists(sFullName):
696	raise Exception('Still exists after deletion, weird.');
697	except Exception, oXcpt:
698	# pylint: disable=too-many-boolean-expressions
699	if fUseTheForce is True \
700	and utils.getHostOs() not in ['win', 'os2'] \
701	and len(sFullName) >= 8 \
702	and sFullName[0] == '/' \
703	and sFullName[1] != '/' \
704	and sFullName.find('/../') < 0:
705	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
706	try:
707	if os.path.isdir(sFullName):
708	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
709	else:
710	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
711	if iRc != 0:
712	raise Exception('exit code %s' % iRc);
713	if os.path.exists(sFullName):
714	raise Exception('Still exists after forced deletion, weird^2.');
715	except:
716	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
717	oRc.fRc = False;
718	else:
719	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
720	oRc.fRc = False;
721
722	#
723	# Re-create the directories.
724	#
725	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
726	if not os.path.isdir(sDir):
727	try:
728	os.makedirs(sDir, 0700);
729	except Exception, oXcpt:
730	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
731	oRc.fRc = False;
732
733	if oRc.fRc is True:
734	self._cReinitScratchErrors = 0;
735	else:
736	self._cReinitScratchErrors += 1;
737	return oRc.fRc;
738
739	def _doSignOn(self):
740	"""
741	Worker for _maybeSignOn that does the actual signing on.
742	"""
743	assert not self._oCommand.isRunning();
744
745	# Reset the siged-on state.
746	testboxcommons.log('Signing-on...')
747	self._fSignedOn = False
748	self._idTestBox = None
749	self._cSignOnAttempts += 1;
750
751	# Assemble SIGN-ON request parameters and send the request.
752	dParams = {};
753	for sParam in self._ddSignOnParams:
754	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
755	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
756
757	# Check response.
758	try:
759	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
760	if sResult != constants.tbresp.STATUS_ACK:
761	raise TestBoxException('Result is %s' % (sResult,));
762	oResponse.checkParameterCount(3);
763	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
764	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
765	except TestBoxException, err:
766	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
767	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
768	return False;
769
770	# Successfully signed on, update the state.
771	self._fSignedOn = True;
772	self._fNeedReSignOn = False;
773	self._cSignOnAttempts = 0;
774	self._idTestBox = idTestBox;
775	self._sTestBoxName = sTestBoxName;
776
777	# Update the environment.
778	os.environ['TESTBOX_ID'] = str(self._idTestBox);
779	os.environ['TESTBOX_NAME'] = sTestBoxName;
780	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
781	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
782	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
783
784	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
785	% (self._idTestBox, self._sTestBoxName));
786
787	# Set up the scratch area.
788	self.reinitScratch(fUseTheForce = self._fFirstSignOn);
789
790	self._fFirstSignOn = False;
791	return True;
792
793	def _maybeSignOn(self):
794	"""
795	Check if Test Box parameters were changed
796	and do sign-in in case of positive result
797	"""
798
799	# Skip sign-on check if background command is currently in
800	# running state (avoid infinite signing on).
801	if self._oCommand.isRunning():
802	return None;
803
804	# Refresh sign-on parameters, changes triggers sign-on.
805	fNeedSignOn = (True if not self._fSignedOn or self._fNeedReSignOn else False)
806	for item in self._ddSignOnParams:
807	if self._ddSignOnParams[item][self.FN] is None:
808	continue
809
810	sOldValue = self._ddSignOnParams[item][self.VALUE]
811	self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
812	if sOldValue != self._ddSignOnParams[item][self.VALUE]:
813	fNeedSignOn = True
814	testboxcommons.log('Detected %s parameter change: %s -> %s' %
815	(item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
816
817	if fNeedSignOn:
818	self._doSignOn();
819	return None;
820
821	def dispatch(self):
822	"""
823	Receive orders from Test Manager and execute them
824	"""
825
826	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
827	self._fNeedReSignOn = self._fSignedOn;
828	if self._fSignedOn:
829	os.environ['TESTBOX_ID'] = str(self._idTestBox);
830	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
831
832	while True:
833	# Make sure we're signed on before trying to do anything.
834	self._maybeSignOn();
835	while not self._fSignedOn:
836	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
837	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
838	self._maybeSignOn();
839
840	# Retrieve and handle command from the TM.
841	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
842	self._idTestBox,
843	self._sTestBoxUuid,
844	self._oCommand.isRunning());
845	if oResponse is not None:
846	self._oCommand.handleCommand(oResponse, oConnection);
847	if oConnection is not None:
848	if oConnection.isConnected():
849	self._oCommand.flushLogOnConnection(oConnection);
850	oConnection.close();
851
852	# Automatically reboot if scratch init fails.
853	if self._cReinitScratchErrors > 8 and self.reinitScratch() is False:
854	testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
855	% ( self._cReinitScratchErrors, ));
856	self._oCommand.doReboot();
857
858	# delay a wee bit before looping.
859	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
860	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
861	iFactor = 1;
862	if self._cReinitScratchErrors > 0:
863	iFactor = 4;
864	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
865
866	# Not reached.
867
868
869	@staticmethod
870	def main():
871	"""
872	Main function a la C/C++. Returns exit code.
873	"""
874
875	#
876	# Parse arguments.
877	#
878	if utils.getHostOs() in ('win', 'os2'):
879	sDefTestRsrc = 'T:';
880	sDefBuilds = 'U:';
881	elif utils.getHostOs() == 'darwin':
882	sDefTestRsrc = '/Volumes/testrsrc';
883	sDefBuilds = '/Volumes/builds';
884	else:
885	sDefTestRsrc = '/mnt/testrsrc';
886	sDefBuilds = '/mnt/builds';
887
888	class MyOptionParser(OptionParser):
889	""" We need to override the exit code on --help, error and so on. """
890	def __init__(self, args, *kwargs):
891	OptionParser.__init__(self, args, *kwargs);
892	def exit(self, status = 0, msg = None):
893	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
894
895	parser = MyOptionParser(version=__version__[11:-1].strip());
896	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
897	sLower = sMixed.lower();
898	sPrefix = 's' + sMixed;
899	parser.add_option('--' + sLower + '-path',
900	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
901	help='Where ' + sDesc + ' can be found');
902	parser.add_option('--' + sLower + '-server-type',
903	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=None,
904	help='The type of server, cifs or nfs. If empty (default), we won\'t try mount anything.');
905	parser.add_option('--' + sLower + '-server-name',
906	dest=sPrefix + 'ServerName', metavar='<server>', default='solserv.de.oracle.com',
907	help='The name of the server with the builds.');
908	parser.add_option('--' + sLower + '-server-share',
909	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
910	help='The name of the builds share.');
911	parser.add_option('--' + sLower + '-server-user',
912	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
913	help='The user name to use when accessing the ' + sDesc + ' share.');
914	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
915	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
916	help='The password to use when accessing the ' + sDesc + ' share.');
917
918	parser.add_option("--test-manager", metavar="<url>",
919	dest="sTestManagerUrl",
920	help="Test Manager URL",
921	default="http://tindertux.de.oracle.com/testmanager")
922	parser.add_option("--scratch-root", metavar="<abs-path>",
923	dest="sScratchRoot",
924	help="Path to the scratch directory",
925	default=None)
926	parser.add_option("--system-uuid", metavar="<uuid>",
927	dest="sSystemUuid",
928	help="The system UUID of the testbox, used for uniquely identifiying the machine",
929	default=None)
930	parser.add_option("--hwvirt",
931	dest="fHasHwVirt", action="store_true", default=None,
932	help="Hardware virtualization available in the CPU");
933	parser.add_option("--no-hwvirt",
934	dest="fHasHwVirt", action="store_false", default=None,
935	help="Hardware virtualization not available in the CPU");
936	parser.add_option("--nested-paging",
937	dest="fHasNestedPaging", action="store_true", default=None,
938	help="Nested paging is available");
939	parser.add_option("--no-nested-paging",
940	dest="fHasNestedPaging", action="store_false", default=None,
941	help="Nested paging is not available");
942	parser.add_option("--64-bit-guest",
943	dest="fCan64BitGuest", action="store_true", default=None,
944	help="Host can execute 64-bit guests");
945	parser.add_option("--no-64-bit-guest",
946	dest="fCan64BitGuest", action="store_false", default=None,
947	help="Host cannot execute 64-bit guests");
948	parser.add_option("--io-mmu",
949	dest="fHasIoMmu", action="store_true", default=None,
950	help="I/O MMU available");
951	parser.add_option("--no-io-mmu",
952	dest="fHasIoMmu", action="store_false", default=None,
953	help="No I/O MMU available");
954	parser.add_option("--raw-mode",
955	dest="fWithRawMode", action="store_true", default=None,
956	help="Use raw-mode on this host.");
957	parser.add_option("--no-raw-mode",
958	dest="fWithRawMode", action="store_false", default=None,
959	help="Disables raw-mode tests on this host.");
960	parser.add_option("--pidfile",
961	dest="sPidFile", default=None,
962	help="For the parent script, ignored.");
963	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
964	dest = "asEnvVars", default = [],
965	help = "Sets an environment variable. Can be repeated.");
966
967	(oOptions, args) = parser.parse_args()
968	# Check command line
969	if args != []:
970	parser.print_help();
971	return TBS_EXITCODE_SYNTAX;
972
973	if oOptions.sSystemUuid is not None:
974	uuid.UUID(oOptions.sSystemUuid);
975	if not oOptions.sTestManagerUrl.startswith('http://') \
976	and not oOptions.sTestManagerUrl.startswith('https://'):
977	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
978	return TBS_EXITCODE_SYNTAX;
979
980	for sPrefix in ['sBuilds', 'sTestRsrc']:
981	sType = getattr(oOptions, sPrefix + 'ServerType');
982	if sType is None or len(sType.strip()) == 0:
983	setattr(oOptions, sPrefix + 'ServerType', None);
984	elif sType not in ['cifs', 'nfs']:
985	print('Syntax error: Invalid server type "%s"' % (sType,));
986	return TBS_EXITCODE_SYNTAX;
987
988
989	#
990	# Instantiate the testbox script and start dispatching work.
991	#
992	try:
993	oTestBoxScript = TestBoxScript(oOptions);
994	except TestBoxScriptException, oXcpt:
995	print('Error: %s' % (oXcpt,));
996	return TBS_EXITCODE_SYNTAX;
997	oTestBoxScript.dispatch();
998
999	# Not supposed to get here...
1000	return TBS_EXITCODE_FAILURE;
1001
1002
1003
1004	if __name__ == '__main__':
1005	sys.exit(TestBoxScript.main());
1006
1007

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 61616

Download in other formats: