testboxscript_real.py@ 58799

Last change on this file since 58799 was 57290, checked in by vboxsync, 9 years ago
ValidationKit: More cleanup.
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 43.7 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: testboxscript_real.py 57290 2015-08-12 12:15:57Z vboxsync $
4
5	"""
6	TestBox Script - main().
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2015 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 57290 $"
31
32
33	# Standard python imports.
34	import math
35	import multiprocessing
36	import os
37	from optparse import OptionParser
38	import platform
39	import random
40	import shutil
41	import sys
42	import tempfile
43	import time
44	import uuid
45
46	# Only the main script needs to modify the path.
47	try: __file__
48	except: __file__ = sys.argv[0];
49	g_ksTestScriptDir = os.path.dirname(os.path.abspath(__file__));
50	g_ksValidationKitDir = os.path.dirname(g_ksTestScriptDir);
51	sys.path.extend([g_ksTestScriptDir, g_ksValidationKitDir]);
52
53	# Validation Kit imports.
54	from common import constants;
55	from common import utils;
56	import testboxcommons;
57	from testboxcommons import TestBoxException;
58	from testboxcommand import TestBoxCommand;
59	from testboxconnection import TestBoxConnection;
60	from testboxscript import TBS_EXITCODE_SYNTAX, TBS_EXITCODE_FAILURE;
61
62
63	class TestBoxScriptException(Exception):
64	""" For raising exceptions during TestBoxScript.__init__. """
65	pass;
66
67
68	class TestBoxScript(object):
69	"""
70	Implementation of the test box script.
71	Communicate with test manager and perform offered actions.
72	"""
73
74	## @name Class Constants.
75	# @{
76
77	# Scratch space round value (MB).
78	kcMbScratchSpaceRounding = 64
79	# Memory size round value (MB).
80	kcMbMemoryRounding = 4
81	# A NULL UUID in string form.
82	ksNullUuid = '00000000-0000-0000-0000-000000000000';
83	# The minimum dispatch loop delay.
84	kcSecMinDelay = 12;
85	# The maximum dispatch loop delay (inclusive).
86	kcSecMaxDelay = 24;
87	# The minimum sign-on delay.
88	kcSecMinSignOnDelay = 30;
89	# The maximum sign-on delay (inclusive).
90	kcSecMaxSignOnDelay = 60;
91
92	# Keys for config params
93	VALUE = 'value'
94	FN = 'fn' # pylint: disable=C0103
95
96	## @}
97
98
99	def __init__(self, oOptions):
100	"""
101	Initialize internals
102	"""
103	self._oOptions = oOptions;
104	self._sTestBoxHelper = None;
105
106	# Signed-on state
107	self._cSignOnAttempts = 0;
108	self._fSignedOn = False;
109	self._fNeedReSignOn = False;
110	self._fFirstSignOn = True;
111	self._idTestBox = None;
112	self._sTestBoxName = '';
113	self._sTestBoxUuid = self.ksNullUuid; # convenience, assigned below.
114
115	# Command processor.
116	self._oCommand = TestBoxCommand(self);
117
118	#
119	# Scratch dir setup. Use /var/tmp instead of /tmp because we may need
120	# many many GBs for some test scenarios and /tmp can be backed by swap
121	# or be a fast+small disk of some kind, while /var/tmp is normally
122	# larger, if slower. /var/tmp is generally not cleaned up on reboot,
123	# /tmp often is, this would break host panic / triple-fault detection.
124	#
125	if self._oOptions.sScratchRoot is None:
126	if utils.getHostOs() in ('win', 'os2', 'haiku', 'dos'):
127	# We need lots of space, so avoid /tmp as it may be a memory
128	# file system backed by the swap file, or worse.
129	self._oOptions.sScratchRoot = tempfile.gettempdir();
130	else:
131	self._oOptions.sScratchRoot = '/var/tmp';
132	sSubDir = 'testbox';
133	try:
134	sSubDir = '%s-%u' % (sSubDir, os.getuid()); # pylint: disable=E1101
135	except:
136	pass;
137	self._oOptions.sScratchRoot = os.path.join(self._oOptions.sScratchRoot, sSubDir);
138
139	self._sScratchSpill = os.path.join(self._oOptions.sScratchRoot, 'scratch');
140	self._sScratchScripts = os.path.join(self._oOptions.sScratchRoot, 'scripts');
141	self._sScratchState = os.path.join(self._oOptions.sScratchRoot, 'state'); # persistant storage.
142
143	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
144	if not os.path.isdir(sDir):
145	os.makedirs(sDir, 0700);
146
147	# We count consecutive reinitScratch failures and will reboot the
148	# testbox after a while in the hope that it will correct the issue.
149	self._cReinitScratchErrors = 0;
150
151	#
152	# Mount builds and test resources if requested.
153	#
154	self.mountShares();
155
156	#
157	# Sign-on parameters: Packed into list of records of format:
158	# { <Parameter ID>: { <Current value>, <Check function> } }
159	#
160	self._ddSignOnParams = \
161	{
162	constants.tbreq.ALL_PARAM_TESTBOX_UUID: { self.VALUE: self._getHostSystemUuid(), self.FN: None },
163	constants.tbreq.SIGNON_PARAM_OS: { self.VALUE: utils.getHostOs(), self.FN: None },
164	constants.tbreq.SIGNON_PARAM_OS_VERSION: { self.VALUE: utils.getHostOsVersion(), self.FN: None },
165	constants.tbreq.SIGNON_PARAM_CPU_ARCH: { self.VALUE: utils.getHostArch(), self.FN: None },
166	constants.tbreq.SIGNON_PARAM_CPU_VENDOR: { self.VALUE: self._getHostCpuVendor(), self.FN: None },
167	constants.tbreq.SIGNON_PARAM_CPU_NAME: { self.VALUE: self._getHostCpuName(), self.FN: None },
168	constants.tbreq.SIGNON_PARAM_CPU_REVISION: { self.VALUE: self._getHostCpuRevision(), self.FN: None },
169	constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT: { self.VALUE: self._hasHostHwVirt(), self.FN: None },
170	constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING:{ self.VALUE: self._hasHostNestedPaging(), self.FN: None },
171	constants.tbreq.SIGNON_PARAM_HAS_64_BIT_GUEST: { self.VALUE: self._can64BitGuest(), self.FN: None },
172	constants.tbreq.SIGNON_PARAM_HAS_IOMMU: { self.VALUE: self._hasHostIoMmu(), self.FN: None },
173	#constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE: { self.VALUE: self._withRawModeSupport(), self.FN: None },
174	constants.tbreq.SIGNON_PARAM_SCRIPT_REV: { self.VALUE: self._getScriptRev(), self.FN: None },
175	constants.tbreq.SIGNON_PARAM_REPORT: { self.VALUE: self._getHostReport(), self.FN: None },
176	constants.tbreq.SIGNON_PARAM_PYTHON_VERSION: { self.VALUE: self._getPythonHexVersion(), self.FN: None },
177	constants.tbreq.SIGNON_PARAM_CPU_COUNT: { self.VALUE: None, self.FN: multiprocessing.cpu_count },
178	constants.tbreq.SIGNON_PARAM_MEM_SIZE: { self.VALUE: None, self.FN: self._getHostMemSize },
179	constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE: { self.VALUE: None, self.FN: self._getFreeScratchSpace },
180	}
181	for sItem in self._ddSignOnParams:
182	if self._ddSignOnParams[sItem][self.FN] is not None:
183	self._ddSignOnParams[sItem][self.VALUE] = self._ddSignOnParams[sItem][self.FN]()
184
185	testboxcommons.log('Starting Test Box script (%s)' % __version__)
186	testboxcommons.log('Test Manager URL: %s' % self._oOptions.sTestManagerUrl,)
187	testboxcommons.log('Scratch root path: %s' % self._oOptions.sScratchRoot,)
188	for sItem in self._ddSignOnParams:
189	testboxcommons.log('Sign-On value %18s: %s' % (sItem, self._ddSignOnParams[sItem][self.VALUE]));
190
191	#
192	# The System UUID is the primary identification of the machine, so
193	# refuse to cooperate if it's NULL.
194	#
195	self._sTestBoxUuid = self.getSignOnParam(constants.tbreq.ALL_PARAM_TESTBOX_UUID);
196	if self._sTestBoxUuid == self.ksNullUuid:
197	raise TestBoxScriptException('Couldn\'t determine the System UUID, please use --system-uuid to specify it.');
198
199	#
200	# Export environment variables, clearing any we don't know yet.
201	#
202	for sEnvVar in self._oOptions.asEnvVars:
203	iEqual = sEnvVar.find('=');
204	if iEqual == -1: # No '=', remove it.
205	if sEnvVar in os.environ:
206	del os.environ[sEnvVar];
207	elif iEqual > 0: # Set it.
208	os.environ[sEnvVar[:iEqual]] = sEnvVar[iEqual+1:];
209	else: # Starts with '=', bad user.
210	raise TestBoxScriptException('Invalid -E argument: "%s"' % (sEnvVar,));
211
212	os.environ['TESTBOX_PATH_BUILDS'] = self._oOptions.sBuildsPath;
213	os.environ['TESTBOX_PATH_RESOURCES'] = self._oOptions.sTestRsrcPath;
214	os.environ['TESTBOX_PATH_SCRATCH'] = self._sScratchSpill;
215	os.environ['TESTBOX_PATH_SCRIPTS'] = self._sScratchScripts;
216	os.environ['TESTBOX_PATH_UPLOAD'] = self._sScratchSpill; ## @todo drop the UPLOAD dir?
217	os.environ['TESTBOX_HAS_HW_VIRT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_HW_VIRT);
218	os.environ['TESTBOX_HAS_NESTED_PAGING'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_NESTED_PAGING);
219	os.environ['TESTBOX_HAS_IOMMU'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_HAS_IOMMU);
220	os.environ['TESTBOX_SCRIPT_REV'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRIPT_REV);
221	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
222	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
223	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
224	#TODO: os.environ['TESTBOX_WITH_RAW_MODE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_WITH_RAW_MODE);
225	os.environ['TESTBOX_WITH_RAW_MODE'] = str(self._withRawModeSupport());
226	os.environ['TESTBOX_MANAGER_URL'] = self._oOptions.sTestManagerUrl;
227	os.environ['TESTBOX_UUID'] = self._sTestBoxUuid;
228	os.environ['TESTBOX_REPORTER'] = 'remote';
229	os.environ['TESTBOX_NAME'] = '';
230	os.environ['TESTBOX_ID'] = '';
231	os.environ['TESTBOX_TEST_SET_ID'] = '';
232	os.environ['TESTBOX_TIMEOUT'] = '0';
233	os.environ['TESTBOX_TIMEOUT_ABS'] = '0';
234
235	if utils.getHostOs() is 'win':
236	os.environ['COMSPEC'] = os.path.join(os.environ['SystemRoot'], 'System32', 'cmd.exe');
237	# Currently omitting any kBuild tools.
238
239	def mountShares(self):
240	"""
241	Mounts the shares.
242	Raises exception on failure.
243	"""
244	self._mountShare(self._oOptions.sBuildsPath, self._oOptions.sBuildsServerType, self._oOptions.sBuildsServerName,
245	self._oOptions.sBuildsServerShare,
246	self._oOptions.sBuildsServerUser, self._oOptions.sBuildsServerPasswd, 'builds');
247	self._mountShare(self._oOptions.sTestRsrcPath, self._oOptions.sTestRsrcServerType, self._oOptions.sTestRsrcServerName,
248	self._oOptions.sTestRsrcServerShare,
249	self._oOptions.sTestRsrcServerUser, self._oOptions.sTestRsrcServerPasswd, 'testrsrc');
250	return True;
251
252	def _mountShare(self, sMountPoint, sType, sServer, sShare, sUser, sPassword, sWhat):
253	"""
254	Mounts the specified share if needed.
255	Raises exception on failure.
256	"""
257	# Only mount if the type is specified.
258	if sType is None:
259	return True;
260
261	# Test if already mounted.
262	sTestFile = os.path.join(sMountPoint + os.path.sep, sShare + '.txt');
263	if os.path.isfile(sTestFile):
264	return True;
265
266	#
267	# Platform specific mount code.
268	#
269	sHostOs = utils.getHostOs()
270	if sHostOs in ('darwin', 'freebsd'):
271	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
272	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
273	utils.sudoProcessCall(['/usr/sbin/chown', str(os.getuid()), sMountPoint]); # pylint: disable=E1101
274	if sType == 'cifs':
275	# Note! no smb://server/share stuff here, 10.6.8 didn't like it.
276	utils.processOutputChecked(['/sbin/mount_smbfs', '-o', 'automounted,nostreams,soft,noowners,noatime,rdonly',
277	'-f', '0555', '-d', '0555',
278	'//%s:%s@%s/%s' % (sUser, sPassword, sServer, sShare),
279	sMountPoint]);
280	else:
281	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
282
283	elif sHostOs == 'linux':
284	utils.sudoProcessCall(['/bin/umount', sMountPoint]);
285	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
286	if sType == 'cifs':
287	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'cifs',
288	'-o',
289	'user=' + sUser
290	+ ',password=' + sPassword
291	+ ',sec=ntlmv2'
292	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
293	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
294	+ ',nounix,file_mode=0555,dir_mode=0555,soft,ro',
295	'//%s/%s' % (sServer, sShare),
296	sMountPoint]);
297	elif sType == 'nfs':
298	utils.sudoProcessOutputChecked(['/bin/mount', '-t', 'nfs',
299	'-o', 'soft,ro',
300	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
301	sMountPoint]);
302
303	else:
304	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
305
306	elif sHostOs == 'solaris':
307	utils.sudoProcessCall(['/sbin/umount', sMountPoint]);
308	utils.sudoProcessCall(['/bin/mkdir', '-p', sMountPoint]);
309	if sType == 'cifs':
310	## @todo This stuff doesn't work on wei01-x4600b.de.oracle.com running 11.1. FIXME!
311	oPasswdFile = tempfile.TemporaryFile();
312	oPasswdFile.write(sPassword + '\n');
313	oPasswdFile.flush();
314	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'smbfs',
315	'-o',
316	'user=' + sUser
317	+ ',uid=' + str(os.getuid()) # pylint: disable=E1101
318	+ ',gid=' + str(os.getgid()) # pylint: disable=E1101
319	+ ',fileperms=0555,dirperms=0555,noxattr,ro',
320	'//%s/%s' % (sServer, sShare),
321	sMountPoint],
322	stdin = oPasswdFile);
323	oPasswdFile.close();
324	elif sType == 'nfs':
325	utils.sudoProcessOutputChecked(['/sbin/mount', '-F', 'nfs',
326	'-o', 'noxattr,ro',
327	'%s:%s' % (sServer, sShare if sShare.find('/') >= 0 else ('/export/' + sShare)),
328	sMountPoint]);
329
330	else:
331	raise TestBoxScriptException('Unsupported server type %s.' % (sType,));
332
333
334	elif sHostOs == 'win':
335	if sType != 'cifs':
336	raise TestBoxScriptException('Only CIFS mounts are supported on Windows.');
337	utils.processCall(['net', 'use', sMountPoint, '/d']);
338	utils.processOutputChecked(['net', 'use', sMountPoint,
339	'\\\\' + sServer + '\\' + sShare,
340	sPassword,
341	'/USER:' + sUser,]);
342	else:
343	raise TestBoxScriptException('Unsupported host %s' % (sHostOs,));
344
345	#
346	# Re-test.
347	#
348	if not os.path.isfile(sTestFile):
349	raise TestBoxException('Failed to mount %s (%s[%s]) at %s: %s not found'
350	% (sWhat, sServer, sShare, sMountPoint, sTestFile));
351
352	return True;
353
354	## @name Signon property releated methods.
355	# @{
356
357	def _getHelperOutput(self, sCmd):
358	"""
359	Invokes TestBoxHelper to obtain information hard to access from python.
360	"""
361	if self._sTestBoxHelper is None:
362	if not utils.isRunningFromCheckout():
363	# See VBoxTestBoxScript.zip for layout.
364	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, utils.getHostOs(), utils.getHostArch(), \
365	'TestBoxHelper');
366	else: # Only for in-tree testing, so don't bother be too accurate right now.
367	sType = os.environ.get('KBUILD_TYPE', os.environ.get('BUILD_TYPE', 'debug'));
368	self._sTestBoxHelper = os.path.join(g_ksValidationKitDir, os.pardir, os.pardir, os.pardir, 'out', \
369	utils.getHostOsDotArch(), sType, 'testboxscript', \
370	utils.getHostOs(), utils.getHostArch(), \
371	'TestBoxHelper');
372	if utils.getHostOs() in ['win', 'os2']:
373	self._sTestBoxHelper += '.exe';
374
375	return utils.processOutputChecked([self._sTestBoxHelper, sCmd]).strip();
376
377	def _getHelperOutputTristate(self, sCmd, fDunnoValue):
378	"""
379	Invokes TestBoxHelper to obtain information hard to access from python.
380	"""
381	sValue = self._getHelperOutput(sCmd);
382	sValue = sValue.lower();
383	if sValue == 'true':
384	return True;
385	if sValue == 'false':
386	return False;
387	if sValue != 'dunno' and sValue != 'none':
388	raise TestBoxException('Unexpected response "%s" to helper command "%s"' % (sValue, sCmd));
389	return fDunnoValue;
390
391
392	@staticmethod
393	def _isUuidGood(sUuid):
394	"""
395	Checks if the UUID looks good.
396
397	There are systems with really bad UUIDs, for instance
398	"03000200-0400-0500-0006-000700080009".
399	"""
400	if sUuid == TestBoxScript.ksNullUuid:
401	return False;
402	sUuid = sUuid.lower();
403	for sDigit in ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f']:
404	if sUuid.count(sDigit) > 16:
405	return False;
406	return True;
407
408	def _getHostSystemUuid(self):
409	"""
410	Get the system UUID string from the System, return null-uuid if
411	unable to get retrieve it.
412	"""
413	if self._oOptions.sSystemUuid is not None:
414	return self._oOptions.sSystemUuid;
415
416	sUuid = self.ksNullUuid;
417
418	#
419	# Try get at the firmware UUID.
420	#
421	if utils.getHostOs() == 'linux':
422	# NOTE: This requires to have kernel option enabled:
423	# Firmware Drivers -> Export DMI identification via sysfs to userspace
424	if os.path.exists('/sys/devices/virtual/dmi/id/product_uuid'):
425	try:
426	sVar = utils.sudoProcessOutputChecked(['cat', '/sys/devices/virtual/dmi/id/product_uuid']);
427	sUuid = str(uuid.UUID(sVar.strip()));
428	except:
429	pass;
430	## @todo consider dmidecoder? What about EFI systems?
431
432	elif utils.getHostOs() == 'win':
433	# Windows: WMI
434	try:
435	import win32com.client; # pylint: disable=F0401
436	oWmi = win32com.client.Dispatch('WbemScripting.SWbemLocator');
437	oWebm = oWmi.ConnectServer('.', 'root\\cimv2');
438	for oItem in oWebm.ExecQuery('SELECT * FROM Win32_ComputerSystemProduct'):
439	if oItem.UUID != None:
440	sUuid = str(uuid.UUID(oItem.UUID));
441	except:
442	pass;
443
444	elif utils.getHostOs() == 'darwin':
445	try:
446	sVar = utils.processOutputChecked(['/bin/sh', '-c',
447	'/usr/sbin/ioreg -k IOPlatformUUID' \
448	+ '\| /usr/bin/grep IOPlatformUUID' \
449	+ '\| /usr/bin/head -1']);
450	sVar = sVar.strip()[-(len(self.ksNullUuid) + 1):-1];
451	sUuid = str(uuid.UUID(sVar));
452	except:
453	pass;
454
455	elif utils.getHostOs() == 'solaris':
456	# Solaris: The smbios util.
457	try:
458	sVar = utils.processOutputChecked(['/bin/sh', '-c',
459	'/usr/sbin/smbios ' \
460	+ '\| /usr/xpg4/bin/sed -ne \'s/^.UUID: //p\'' \
461	+ '\| /usr/bin/head -1']);
462	sUuid = str(uuid.UUID(sVar.strip()));
463	except:
464	pass;
465
466	if self._isUuidGood(sUuid):
467	return sUuid;
468
469	#
470	# Try add the MAC address.
471	# uuid.getnode may provide it, or it may return a random number...
472	#
473	lMacAddr = uuid.getnode();
474	sNode = '%012x' % (lMacAddr,)
475	if lMacAddr == uuid.getnode() and lMacAddr != 0 and len(sNode) == 12:
476	return sUuid[:-12] + sNode;
477
478	return sUuid;
479
480	def _getHostCpuVendor(self):
481	"""
482	Get the CPUID vendor string on intel HW.
483	"""
484	return self._getHelperOutput('cpuvendor');
485
486	def _getHostCpuName(self):
487	"""
488	Get the CPU name/description string.
489	"""
490	return self._getHelperOutput('cpuname');
491
492	def _getHostCpuRevision(self):
493	"""
494	Get the CPU revision (family/model/stepping) value.
495	"""
496	return self._getHelperOutput('cpurevision');
497
498	def _hasHostHwVirt(self):
499	"""
500	Check if the host supports AMD-V or VT-x
501	"""
502	if self._oOptions.fHasHwVirt is None:
503	self._oOptions.fHasHwVirt = self._getHelperOutput('cpuhwvirt');
504	return self._oOptions.fHasHwVirt;
505
506	def _hasHostNestedPaging(self):
507	"""
508	Check if the host supports nested paging.
509	"""
510	if not self._hasHostHwVirt():
511	return False;
512	if self._oOptions.fHasNestedPaging is None:
513	self._oOptions.fHasNestedPaging = self._getHelperOutputTristate('nestedpaging', False);
514	return self._oOptions.fHasNestedPaging;
515
516	def _can64BitGuest(self):
517	"""
518	Check if the we (VBox) can run 64-bit guests.
519	"""
520	if not self._hasHostHwVirt():
521	return False;
522	if self._oOptions.fCan64BitGuest is None:
523	self._oOptions.fCan64BitGuest = self._getHelperOutputTristate('longmode', True);
524	return self._oOptions.fCan64BitGuest;
525
526	def _hasHostIoMmu(self):
527	"""
528	Check if the host has an I/O MMU of the VT-d kind.
529	"""
530	if not self._hasHostHwVirt():
531	return False;
532	if self._oOptions.fHasIoMmu is None:
533	## @todo Any way to figure this one out on any host OS?
534	self._oOptions.fHasIoMmu = False;
535	return self._oOptions.fHasIoMmu;
536
537	def _withRawModeSupport(self):
538	"""
539	Check if the testbox is configured with raw-mode support or not.
540	"""
541	if self._oOptions.fWithRawMode is None:
542	self._oOptions.fWithRawMode = True;
543	return self._oOptions.fWithRawMode;
544
545	def _getHostReport(self):
546	"""
547	Generate a report about the host hardware and software.
548	"""
549	return self._getHelperOutput('report');
550
551
552	def _getHostMemSize(self):
553	"""
554	Gets the amount of physical memory on the host (and accessible to the
555	OS, i.e. don't report stuff over 4GB if Windows doesn't wanna use it).
556	Unit: MiB.
557	"""
558	cMbMemory = long(self._getHelperOutput('memsize').strip()) / (1024 * 1024);
559
560	# Round it.
561	cMbMemory = long(math.floor(cMbMemory / self.kcMbMemoryRounding)) * self.kcMbMemoryRounding;
562	return cMbMemory;
563
564	def _getFreeScratchSpace(self):
565	"""
566	Get free space on the volume where scratch directory is located and
567	return it in bytes rounded down to nearest 64MB
568	(currently works on Linux only)
569	Unit: MiB.
570	"""
571	if platform.system() == 'Windows':
572	import ctypes
573	cTypeMbFreeSpace = ctypes.c_ulonglong(0)
574	ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self._oOptions.sScratchRoot), None, None,
575	ctypes.pointer(cTypeMbFreeSpace))
576	cMbFreeSpace = cTypeMbFreeSpace.value
577	else:
578	stats = os.statvfs(self._oOptions.sScratchRoot); # pylint: disable=E1101
579	cMbFreeSpace = stats.f_frsize * stats.f_bfree
580
581	# Convert to MB
582	cMbFreeSpace = long(cMbFreeSpace) /(1024 * 1024)
583
584	# Round free space size
585	cMbFreeSpace = long(math.floor(cMbFreeSpace / self.kcMbScratchSpaceRounding)) * self.kcMbScratchSpaceRounding;
586	return cMbFreeSpace;
587
588	def _getScriptRev(self):
589	"""
590	The script (subversion) revision number.
591	"""
592	return __version__[11:-1].strip();
593
594	def _getPythonHexVersion(self):
595	"""
596	The python hex version number.
597	"""
598	uHexVersion = getattr(sys, 'hexversion', None);
599	if uHexVersion is None:
600	uHexVersion = (sys.version_info[0] << 24) \| (sys.version_info[1] << 16) \| (sys.version_info[2] << 8);
601	if sys.version_info[3] == 'final':
602	uHexVersion \|= 0xf0;
603	return uHexVersion;
604
605	# @}
606
607	def openTestManagerConnection(self):
608	"""
609	Opens up a connection to the test manager.
610
611	Raises exception on failure.
612	"""
613	return TestBoxConnection(self._oOptions.sTestManagerUrl, self._idTestBox, self._sTestBoxUuid);
614
615	def getSignOnParam(self, sName):
616	"""
617	Returns a sign-on parameter value as string.
618	Raises exception if the name is incorrect.
619	"""
620	return str(self._ddSignOnParams[sName][self.VALUE]);
621
622	def getPathState(self):
623	"""
624	Get the path to the state dir in the scratch area.
625	"""
626	return self._sScratchState;
627
628	def getPathScripts(self):
629	"""
630	Get the path to the scripts dir (TESTBOX_PATH_SCRIPTS) in the scratch area.
631	"""
632	return self._sScratchScripts;
633
634	def getPathSpill(self):
635	"""
636	Get the path to the spill dir (TESTBOX_PATH_SCRATCH) in the scratch area.
637	"""
638	return self._sScratchSpill;
639
640	def getPathBuilds(self):
641	"""
642	Get the path to the builds.
643	"""
644	return self._oOptions.sBuildsPath;
645
646	def getTestBoxId(self):
647	"""
648	Get the TestBox ID for state saving purposes.
649	"""
650	return self._idTestBox;
651
652	def getTestBoxName(self):
653	"""
654	Get the TestBox name for state saving purposes.
655	"""
656	return self._sTestBoxName;
657
658	def reinitScratch(self, fnLog = testboxcommons.log, fUseTheForce = None):
659	"""
660	Wipes the scratch directories and re-initializes them.
661
662	No exceptions raise, returns success indicator instead.
663	"""
664	if fUseTheForce is None:
665	fUseTheForce = self._fFirstSignOn;
666
667	class ErrorCallback(object): # pylint: disable=R0903
668	"""
669	Callbacks + state for the cleanup.
670	"""
671	def __init__(self):
672	self.fRc = True;
673	def onErrorCallback(self, sFnName, sPath, aXcptInfo):
674	""" Logs error during shutil.rmtree operation. """
675	fnLog('Error removing "%s": fn=%s %s' % (sPath, sFnName, aXcptInfo[1]));
676	self.fRc = False;
677	oRc = ErrorCallback();
678
679	#
680	# Cleanup.
681	#
682	for sName in os.listdir(self._oOptions.sScratchRoot):
683	sFullName = os.path.join(self._oOptions.sScratchRoot, sName);
684	try:
685	if os.path.isdir(sFullName):
686	shutil.rmtree(sFullName, False, oRc.onErrorCallback);
687	else:
688	os.remove(sFullName);
689	if os.path.exists(sFullName):
690	raise Exception('Still exists after deletion, weird.');
691	except Exception, oXcpt:
692	if fUseTheForce is True \
693	and utils.getHostOs() not in ['win', 'os2'] \
694	and len(sFullName) >= 8 \
695	and sFullName[0] == '/' \
696	and sFullName[1] != '/' \
697	and sFullName.find('/../') < 0:
698	fnLog('Problems deleting "%s" (%s) using the force...' % (sFullName, oXcpt));
699	try:
700	if os.path.isdir(sFullName):
701	iRc = utils.sudoProcessCall(['/bin/rm', '-Rf', sFullName])
702	else:
703	iRc = utils.sudoProcessCall(['/bin/rm', '-f', sFullName])
704	if iRc != 0:
705	raise Exception('exit code %s' % iRc);
706	if os.path.exists(sFullName):
707	raise Exception('Still exists after forced deletion, weird^2.');
708	except:
709	fnLog('Error sudo deleting "%s": %s' % (sFullName, oXcpt));
710	oRc.fRc = False;
711	else:
712	fnLog('Error deleting "%s": %s' % (sFullName, oXcpt));
713	oRc.fRc = False;
714
715	#
716	# Re-create the directories.
717	#
718	for sDir in [self._oOptions.sScratchRoot, self._sScratchSpill, self._sScratchScripts, self._sScratchState]:
719	if not os.path.isdir(sDir):
720	try:
721	os.makedirs(sDir, 0700);
722	except Exception, oXcpt:
723	fnLog('Error creating "%s": %s' % (sDir, oXcpt));
724	oRc.fRc = False;
725
726	if oRc.fRc is True:
727	self._cReinitScratchErrors = 0;
728	else:
729	self._cReinitScratchErrors += 1;
730	return oRc.fRc;
731
732	def _doSignOn(self):
733	"""
734	Worker for _maybeSignOn that does the actual signing on.
735	"""
736	assert not self._oCommand.isRunning();
737
738	# Reset the siged-on state.
739	testboxcommons.log('Signing-on...')
740	self._fSignedOn = False
741	self._idTestBox = None
742	self._cSignOnAttempts += 1;
743
744	# Assemble SIGN-ON request parameters and send the request.
745	dParams = {};
746	for sParam in self._ddSignOnParams:
747	dParams[sParam] = self._ddSignOnParams[sParam][self.VALUE];
748	oResponse = TestBoxConnection.sendSignOn(self._oOptions.sTestManagerUrl, dParams);
749
750	# Check response.
751	try:
752	sResult = oResponse.getStringChecked(constants.tbresp.ALL_PARAM_RESULT);
753	if sResult != constants.tbresp.STATUS_ACK:
754	raise TestBoxException('Result is %s' % (sResult,));
755	oResponse.checkParameterCount(3);
756	idTestBox = oResponse.getIntChecked(constants.tbresp.SIGNON_PARAM_ID, 1, 0x7ffffffe);
757	sTestBoxName = oResponse.getStringChecked(constants.tbresp.SIGNON_PARAM_NAME);
758	except TestBoxException, err:
759	testboxcommons.log('Failed to sign-on: %s' % (str(err),))
760	testboxcommons.log('Server response: %s' % (oResponse.toString(),));
761	return False;
762
763	# Successfully signed on, update the state.
764	self._fSignedOn = True;
765	self._fNeedReSignOn = False;
766	self._cSignOnAttempts = 0;
767	self._idTestBox = idTestBox;
768	self._sTestBoxName = sTestBoxName;
769
770	# Update the environment.
771	os.environ['TESTBOX_ID'] = str(self._idTestBox);
772	os.environ['TESTBOX_NAME'] = sTestBoxName;
773	os.environ['TESTBOX_CPU_COUNT'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_CPU_COUNT);
774	os.environ['TESTBOX_MEM_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_MEM_SIZE);
775	os.environ['TESTBOX_SCRATCH_SIZE'] = self.getSignOnParam(constants.tbreq.SIGNON_PARAM_SCRATCH_SIZE);
776
777	testboxcommons.log('Successfully signed-on with Test Box ID #%s and given the name "%s"' \
778	% (self._idTestBox, self._sTestBoxName));
779
780	# Set up the scratch area.
781	self.reinitScratch(fUseTheForce = self._fFirstSignOn);
782
783	self._fFirstSignOn = False;
784	return True;
785
786	def _maybeSignOn(self):
787	"""
788	Check if Test Box parameters were changed
789	and do sign-in in case of positive result
790	"""
791
792	# Skip sign-on check if background command is currently in
793	# running state (avoid infinite signing on).
794	if self._oCommand.isRunning():
795	return None;
796
797	# Refresh sign-on parameters, changes triggers sign-on.
798	fNeedSignOn = (True if not self._fSignedOn or self._fNeedReSignOn else False)
799	for item in self._ddSignOnParams:
800	if self._ddSignOnParams[item][self.FN] is None:
801	continue
802
803	sOldValue = self._ddSignOnParams[item][self.VALUE]
804	self._ddSignOnParams[item][self.VALUE] = self._ddSignOnParams[item][self.FN]()
805	if sOldValue != self._ddSignOnParams[item][self.VALUE]:
806	fNeedSignOn = True
807	testboxcommons.log('Detected %s parameter change: %s -> %s' %
808	(item, sOldValue, self._ddSignOnParams[item][self.VALUE]))
809
810	if fNeedSignOn:
811	self._doSignOn();
812	return None;
813
814	def dispatch(self):
815	"""
816	Receive orders from Test Manager and execute them
817	"""
818
819	(self._idTestBox, self._sTestBoxName, self._fSignedOn) = self._oCommand.resumeIncompleteCommand();
820	self._fNeedReSignOn = self._fSignedOn;
821	if self._fSignedOn:
822	os.environ['TESTBOX_ID'] = str(self._idTestBox);
823	os.environ['TESTBOX_NAME'] = self._sTestBoxName;
824
825	while True:
826	# Make sure we're signed on before trying to do anything.
827	self._maybeSignOn();
828	while not self._fSignedOn:
829	iFactor = 1 if self._cSignOnAttempts < 100 else 4;
830	time.sleep(random.randint(self.kcSecMinSignOnDelay * iFactor, self.kcSecMaxSignOnDelay * iFactor));
831	self._maybeSignOn();
832
833	# Retrieve and handle command from the TM.
834	(oResponse, oConnection) = TestBoxConnection.requestCommandWithConnection(self._oOptions.sTestManagerUrl,
835	self._idTestBox,
836	self._sTestBoxUuid,
837	self._oCommand.isRunning());
838	if oResponse is not None:
839	self._oCommand.handleCommand(oResponse, oConnection);
840	if oConnection is not None:
841	if oConnection.isConnected():
842	self._oCommand.flushLogOnConnection(oConnection);
843	oConnection.close();
844
845	# Automatically reboot if scratch init fails.
846	if self._cReinitScratchErrors > 8 and self.reinitScratch() is False:
847	testboxcommons.log('Scratch does not initialize cleanly after %d attempts, rebooting...'
848	% ( self._cReinitScratchErrors, ));
849	self._oCommand.doReboot();
850
851	# delay a wee bit before looping.
852	## @todo We shouldn't bother the server too frequently. We should try combine the test reporting done elsewhere
853	# with the command retrieval done here. I believe tinderclient.pl is capable of doing that.
854	iFactor = 1;
855	if self._cReinitScratchErrors > 0:
856	iFactor = 4;
857	time.sleep(random.randint(self.kcSecMinDelay * iFactor, self.kcSecMaxDelay * iFactor));
858
859	# Not reached.
860
861
862	@staticmethod
863	def main():
864	"""
865	Main function a la C/C++. Returns exit code.
866	"""
867
868	#
869	# Parse arguments.
870	#
871	if utils.getHostOs() in ('win', 'os2'):
872	sDefTestRsrc = 'T:';
873	sDefBuilds = 'U:';
874	elif utils.getHostOs() == 'darwin':
875	sDefTestRsrc = '/Volumes/testrsrc';
876	sDefBuilds = '/Volumes/builds';
877	else:
878	sDefTestRsrc = '/mnt/testrsrc';
879	sDefBuilds = '/mnt/builds';
880
881	class MyOptionParser(OptionParser):
882	""" We need to override the exit code on --help, error and so on. """
883	def __init__(self, args, *kwargs):
884	OptionParser.__init__(self, args, *kwargs);
885	def exit(self, status = 0, msg = None):
886	OptionParser.exit(self, TBS_EXITCODE_SYNTAX, msg);
887
888	parser = MyOptionParser(version=__version__[11:-1].strip());
889	for sMixed, sDefault, sDesc in [('Builds', sDefBuilds, 'builds'), ('TestRsrc', sDefTestRsrc, 'test resources') ]:
890	sLower = sMixed.lower();
891	sPrefix = 's' + sMixed;
892	parser.add_option('--' + sLower + '-path',
893	dest=sPrefix + 'Path', metavar='<abs-path>', default=sDefault,
894	help='Where ' + sDesc + ' can be found');
895	parser.add_option('--' + sLower + '-server-type',
896	dest=sPrefix + 'ServerType', metavar='<nfs\|cifs>', default=None,
897	help='The type of server, cifs or nfs. If empty (default), we won\'t try mount anything.');
898	parser.add_option('--' + sLower + '-server-name',
899	dest=sPrefix + 'ServerName', metavar='<server>', default='solserv.de.oracle.com',
900	help='The name of the server with the builds.');
901	parser.add_option('--' + sLower + '-server-share',
902	dest=sPrefix + 'ServerShare', metavar='<share>', default=sLower,
903	help='The name of the builds share.');
904	parser.add_option('--' + sLower + '-server-user',
905	dest=sPrefix + 'ServerUser', metavar='<user>', default='guestr',
906	help='The user name to use when accessing the ' + sDesc + ' share.');
907	parser.add_option('--' + sLower + '-server-passwd', '--' + sLower + '-server-password',
908	dest=sPrefix + 'ServerPasswd', metavar='<password>', default='guestr',
909	help='The password to use when accessing the ' + sDesc + ' share.');
910
911	parser.add_option("--test-manager", metavar="<url>",
912	dest="sTestManagerUrl",
913	help="Test Manager URL",
914	default="http://tindertux.de.oracle.com/testmanager")
915	parser.add_option("--scratch-root", metavar="<abs-path>",
916	dest="sScratchRoot",
917	help="Path to the scratch directory",
918	default=None)
919	parser.add_option("--system-uuid", metavar="<uuid>",
920	dest="sSystemUuid",
921	help="The system UUID of the testbox, used for uniquely identifiying the machine",
922	default=None)
923	parser.add_option("--hwvirt",
924	dest="fHasHwVirt", action="store_true", default=None,
925	help="Hardware virtualization available in the CPU");
926	parser.add_option("--no-hwvirt",
927	dest="fHasHwVirt", action="store_false", default=None,
928	help="Hardware virtualization not available in the CPU");
929	parser.add_option("--nested-paging",
930	dest="fHasNestedPaging", action="store_true", default=None,
931	help="Nested paging is available");
932	parser.add_option("--no-nested-paging",
933	dest="fHasNestedPaging", action="store_false", default=None,
934	help="Nested paging is not available");
935	parser.add_option("--64-bit-guest",
936	dest="fCan64BitGuest", action="store_true", default=None,
937	help="Host can execute 64-bit guests");
938	parser.add_option("--no-64-bit-guest",
939	dest="fCan64BitGuest", action="store_false", default=None,
940	help="Host cannot execute 64-bit guests");
941	parser.add_option("--io-mmu",
942	dest="fHasIoMmu", action="store_true", default=None,
943	help="I/O MMU available");
944	parser.add_option("--no-io-mmu",
945	dest="fHasIoMmu", action="store_false", default=None,
946	help="No I/O MMU available");
947	parser.add_option("--raw-mode",
948	dest="fWithRawMode", action="store_true", default=None,
949	help="Use raw-mode on this host.");
950	parser.add_option("--no-raw-mode",
951	dest="fWithRawMode", action="store_false", default=None,
952	help="Disables raw-mode tests on this host.");
953	parser.add_option("--pidfile",
954	dest="sPidFile", default=None,
955	help="For the parent script, ignored.");
956	parser.add_option("-E", "--putenv", metavar = "<variable>=<value>", action = "append",
957	dest = "asEnvVars", default = [],
958	help = "Sets an environment variable. Can be repeated.");
959
960	(oOptions, args) = parser.parse_args()
961	# Check command line
962	if args != []:
963	parser.print_help();
964	return TBS_EXITCODE_SYNTAX;
965
966	if oOptions.sSystemUuid is not None:
967	uuid.UUID(oOptions.sSystemUuid);
968	if not oOptions.sTestManagerUrl.startswith('http://') \
969	and not oOptions.sTestManagerUrl.startswith('https://'):
970	print('Syntax error: Invalid test manager URL "%s"' % (oOptions.sTestManagerUrl,));
971	return TBS_EXITCODE_SYNTAX;
972
973	for sPrefix in ['sBuilds', 'sTestRsrc']:
974	sType = getattr(oOptions, sPrefix + 'ServerType');
975	if sType is None or len(sType.strip()) == 0:
976	setattr(oOptions, sPrefix + 'ServerType', None);
977	elif sType not in ['cifs', 'nfs']:
978	print('Syntax error: Invalid server type "%s"' % (sType,));
979	return TBS_EXITCODE_SYNTAX;
980
981
982	#
983	# Instantiate the testbox script and start dispatching work.
984	#
985	try:
986	oTestBoxScript = TestBoxScript(oOptions);
987	except TestBoxScriptException, oXcpt:
988	print('Error: %s' % (oXcpt,));
989	return TBS_EXITCODE_SYNTAX;
990	oTestBoxScript.dispatch();
991
992	# Not supposed to get here...
993	return TBS_EXITCODE_FAILURE;
994
995
996
997	if __name__ == '__main__':
998	sys.exit(TestBoxScript.main());
999

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testboxscript/testboxscript_real.py@ 58799

Download in other formats: