status.py@ 91692

Last change on this file since 91692 was 86956, checked in by vboxsync, 4 years ago
testmanager/status.py: Unused functions. bugref:9788
Property svn:eol-style set to `LF` Property svn:executable set to ``* Property svn:keywords set to `Author Date Id Revision`
File size: 19.3 KB

Line
1	#!/usr/bin/env python
2	# -- coding: utf-8 --
3	# $Id: status.py 86956 2020-11-23 10:41:20Z vboxsync $
4
5	"""
6	CGI - Administrator Web-UI.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2012-2020 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 86956 $"
31
32
33	# Standard python imports.
34	import os
35	import sys
36
37	# Only the main script needs to modify the path.
38	g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
39	sys.path.append(g_ksValidationKitDir);
40
41	# Validation Kit imports.
42	from testmanager import config;
43	from testmanager.core.webservergluecgi import WebServerGlueCgi;
44
45	from common import constants;
46	from testmanager.core.base import TMExceptionBase;
47	from testmanager.core.db import TMDatabaseConnection;
48
49
50
51	def timeDeltaToHours(oTimeDelta):
52	return oTimeDelta.days * 24 + oTimeDelta.seconds // 3600
53
54
55	def testbox_data_processing(oDb):
56	testboxes_dict = {}
57	while True:
58	line = oDb.fetchOne();
59	if line is None:
60	break;
61	testbox_name = line[0]
62	test_result = line[1]
63	oTimeDeltaSinceStarted = line[2]
64	test_box_os = line[3]
65	test_sched_group = line[4]
66	testboxes_dict = dict_update(testboxes_dict, testbox_name, test_result)
67
68	if "testbox_os" not in testboxes_dict[testbox_name]:
69	testboxes_dict[testbox_name].update({"testbox_os": test_box_os})
70
71	if "sched_group" not in testboxes_dict[testbox_name]:
72	testboxes_dict[testbox_name].update({"sched_group": test_sched_group})
73	elif test_sched_group not in testboxes_dict[testbox_name]["sched_group"]:
74	testboxes_dict[testbox_name]["sched_group"] += "," + test_sched_group
75
76	if test_result == "running":
77	testboxes_dict[testbox_name].update({"hours_running": timeDeltaToHours(oTimeDeltaSinceStarted)})
78
79	return testboxes_dict;
80
81
82	def os_results_separating(vb_dict, test_name, testbox_os, test_result):
83	if testbox_os == "linux":
84	dict_update(vb_dict, test_name + " / linux", test_result)
85	elif testbox_os == "win":
86	dict_update(vb_dict, test_name + " / windows", test_result)
87	elif testbox_os == "darwin":
88	dict_update(vb_dict, test_name + " / darwin", test_result)
89	elif testbox_os == "solaris":
90	dict_update(vb_dict, test_name + " / solaris", test_result)
91	else:
92	dict_update(vb_dict, test_name + " / other", test_result)
93
94
95	# const/immutable.
96	g_kdTestStatuses = {
97	'running': 0,
98	'success': 0,
99	'skipped': 0,
100	'bad-testbox': 0,
101	'aborted': 0,
102	'failure': 0,
103	'timed-out': 0,
104	'rebooted': 0,
105	}
106
107	def dict_update(target_dict, key_name, test_result):
108	if key_name not in target_dict:
109	target_dict.update({key_name: g_kdTestStatuses.copy()})
110	if test_result in g_kdTestStatuses:
111	target_dict[key_name][test_result] += 1
112	return target_dict
113
114
115	def formatDataEntry(sKey, dEntry):
116	# There are variations in the first and second "columns".
117	if "hours_running" in dEntry:
118	sRet = "%s;%s;%s \| running: %s;%s" \
119	% (sKey, dEntry["testbox_os"], dEntry["sched_group"], dEntry["running"], dEntry["hours_running"]);
120	else:
121	if "testbox_os" in dEntry:
122	sRet = "%s;%s;%s" % (sKey, dEntry["testbox_os"], dEntry["sched_group"],);
123	else:
124	sRet = sKey;
125	sRet += " \| running: %s" % (dEntry["running"],)
126
127	# The rest is currently identical:
128	sRet += " \| success: %s \| skipped: %s \| bad-testbox: %s \| aborted: %s \| failure: %s \| timed-out: %s \| rebooted: %s \| \n" \
129	% (dEntry["success"], dEntry["skipped"], dEntry["bad-testbox"], dEntry["aborted"],
130	dEntry["failure"], dEntry["timed-out"], dEntry["rebooted"],);
131	return sRet;
132
133
134	def format_data(dData, fSorted):
135	sRet = "";
136	if not fSorted:
137	for sKey in dData:
138	sRet += formatDataEntry(sKey, dData[sKey]);
139	else:
140	for sKey in sorted(dData.keys()):
141	sRet += formatDataEntry(sKey, dData[sKey]);
142	return sRet;
143
144	######
145
146	class StatusDispatcherException(TMExceptionBase):
147	"""
148	Exception class for TestBoxController.
149	"""
150	pass; # pylint: disable=unnecessary-pass
151
152
153	class StatusDispatcher(object): # pylint: disable=too-few-public-methods
154	"""
155	Status dispatcher class.
156	"""
157
158
159	def __init__(self, oSrvGlue):
160	"""
161	Won't raise exceptions.
162	"""
163	self._oSrvGlue = oSrvGlue;
164	self._sAction = None; # _getStandardParams / dispatchRequest sets this later on.
165	self._dParams = None; # _getStandardParams / dispatchRequest sets this later on.
166	self._asCheckedParams = [];
167	self._dActions = \
168	{
169	'MagicMirrorTestResults': self._actionMagicMirrorTestResults,
170	'MagicMirrorTestBoxes': self._actionMagicMirrorTestBoxes,
171	};
172
173	def _getStringParam(self, sName, asValidValues = None, fStrip = False, sDefValue = None):
174	"""
175	Gets a string parameter (stripped).
176
177	Raises exception if not found and no default is provided, or if the
178	value isn't found in asValidValues.
179	"""
180	if sName not in self._dParams:
181	if sDefValue is None:
182	raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
183	return sDefValue;
184	sValue = self._dParams[sName];
185	if fStrip:
186	sValue = sValue.strip();
187
188	if sName not in self._asCheckedParams:
189	self._asCheckedParams.append(sName);
190
191	if asValidValues is not None and sValue not in asValidValues:
192	raise StatusDispatcherException('%s parameter %s value "%s" not in %s '
193	% (self._sAction, sName, sValue, asValidValues));
194	return sValue;
195
196	def _getIntParam(self, sName, iMin = None, iMax = None, iDefValue = None):
197	"""
198	Gets a string parameter.
199	Raises exception if not found, not a valid integer, or if the value
200	isn't in the range defined by iMin and iMax.
201	"""
202	if sName not in self._dParams:
203	if iDefValue is None:
204	raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
205	return iDefValue;
206	sValue = self._dParams[sName];
207	try:
208	iValue = int(sValue, 0);
209	except:
210	raise StatusDispatcherException('%s parameter %s value "%s" cannot be convert to an integer'
211	% (self._sAction, sName, sValue));
212	if sName not in self._asCheckedParams:
213	self._asCheckedParams.append(sName);
214
215	if (iMin is not None and iValue < iMin) \
216	or (iMax is not None and iValue > iMax):
217	raise StatusDispatcherException('%s parameter %s value %d is out of range [%s..%s]'
218	% (self._sAction, sName, iValue, iMin, iMax));
219	return iValue;
220
221	def _getBoolParam(self, sName, fDefValue = None):
222	"""
223	Gets a boolean parameter.
224
225	Raises exception if not found and no default is provided, or if not a
226	valid boolean.
227	"""
228	sValue = self._getStringParam(sName, [ 'True', 'true', '1', 'False', 'false', '0'], sDefValue = str(fDefValue));
229	return sValue in ('True', 'true', '1',);
230
231	def _checkForUnknownParameters(self):
232	"""
233	Check if we've handled all parameters, raises exception if anything
234	unknown was found.
235	"""
236
237	if len(self._asCheckedParams) != len(self._dParams):
238	sUnknownParams = '';
239	for sKey in self._dParams:
240	if sKey not in self._asCheckedParams:
241	sUnknownParams += ' ' + sKey + '=' + self._dParams[sKey];
242	raise StatusDispatcherException('Unknown parameters: ' + sUnknownParams);
243
244	return True;
245
246	def _connectToDb(self):
247	"""
248	Connects to the database.
249
250	Returns (TMDatabaseConnection, (more later perhaps) ) on success.
251	Returns (None, ) on failure after sending the box an appropriate response.
252	May raise exception on DB error.
253	"""
254	return (TMDatabaseConnection(self._oSrvGlue.dprint),);
255
256	def _actionMagicMirrorTestBoxes(self):
257	"""
258	Produces test result status for the magic mirror dashboard
259	"""
260
261	#
262	# Parse arguments and connect to the database.
263	#
264	cHoursBack = self._getIntParam('cHours', 1, 24*14, 12);
265	fSorted = self._getBoolParam('fSorted', False);
266	self._checkForUnknownParameters();
267
268	#
269	# Get the data.
270	#
271	# Note! We're not joining on TestBoxesWithStrings.idTestBox =
272	# TestSets.idGenTestBox here because of indexes. This is
273	# also more consistent with the rest of the query.
274	# Note! The original SQL is slow because of the 'OR TestSets.tsDone'
275	# part, using AND and UNION is significatly faster because
276	# it matches the TestSetsGraphBoxIdx (index).
277	#
278	(oDb,) = self._connectToDb();
279	if oDb is None:
280	return False;
281
282	# oDb.execute('''
283	#SELECT TestBoxesWithStrings.sName,
284	# TestSets.enmStatus,
285	# CURRENT_TIMESTAMP - TestSets.tsCreated,
286	# TestBoxesWithStrings.sOS,
287	# SchedGroupNames.sSchedGroupNames
288	#FROM (SELECT TestBoxesInSchedGroups.idTestBox AS idTestBox,
289	# STRING_AGG(SchedGroups.sName, ',') AS sSchedGroupNames
290	# FROM TestBoxesInSchedGroups
291	# INNER JOIN SchedGroups
292	# ON SchedGroups.idSchedGroup = TestBoxesInSchedGroups.idSchedGroup
293	# WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
294	# AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
295	# GROUP BY TestBoxesInSchedGroups.idTestBox)
296	# AS SchedGroupNames,
297	# TestBoxesWithStrings
298	#LEFT OUTER JOIN TestSets
299	# ON TestSets.idTestBox = TestBoxesWithStrings.idTestBox
300	# AND ( TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
301	# OR TestSets.tsDone IS NULL)
302	#WHERE TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
303	# AND SchedGroupNames.idTestBox = TestBoxesWithStrings.idTestBox
304	#''', (cHoursBack,));
305	oDb.execute('''
306	( SELECT TestBoxesWithStrings.sName,
307	TestSets.enmStatus,
308	CURRENT_TIMESTAMP - TestSets.tsCreated,
309	TestBoxesWithStrings.sOS,
310	SchedGroupNames.sSchedGroupNames
311	FROM (
312	SELECT TestBoxesInSchedGroups.idTestBox AS idTestBox,
313	STRING_AGG(SchedGroups.sName, ',') AS sSchedGroupNames
314	FROM TestBoxesInSchedGroups
315	INNER JOIN SchedGroups
316	ON SchedGroups.idSchedGroup = TestBoxesInSchedGroups.idSchedGroup
317	WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
318	AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
319	GROUP BY TestBoxesInSchedGroups.idTestBox
320	) AS SchedGroupNames,
321	TestBoxesWithStrings
322	LEFT OUTER JOIN TestSets
323	ON TestSets.idTestBox = TestBoxesWithStrings.idTestBox
324	AND TestSets.tsCreated >= (CURRENT_TIMESTAMP - '%s hours'::interval)
325	AND TestSets.tsDone IS NOT NULL
326	WHERE TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
327	AND SchedGroupNames.idTestBox = TestBoxesWithStrings.idTestBox
328	) UNION (
329	SELECT TestBoxesWithStrings.sName,
330	TestSets.enmStatus,
331	CURRENT_TIMESTAMP - TestSets.tsCreated,
332	TestBoxesWithStrings.sOS,
333	SchedGroupNames.sSchedGroupNames
334	FROM (
335	SELECT TestBoxesInSchedGroups.idTestBox AS idTestBox,
336	STRING_AGG(SchedGroups.sName, ',') AS sSchedGroupNames
337	FROM TestBoxesInSchedGroups
338	INNER JOIN SchedGroups
339	ON SchedGroups.idSchedGroup = TestBoxesInSchedGroups.idSchedGroup
340	WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
341	AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
342	GROUP BY TestBoxesInSchedGroups.idTestBox
343	) AS SchedGroupNames,
344	TestBoxesWithStrings
345	LEFT OUTER JOIN TestSets
346	ON TestSets.idTestBox = TestBoxesWithStrings.idTestBox
347	AND TestSets.tsCreated < (CURRENT_TIMESTAMP - '%s hours'::interval)
348	AND TestSets.tsDone IS NULL
349	WHERE TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
350	AND SchedGroupNames.idTestBox = TestBoxesWithStrings.idTestBox
351	)
352	''', (cHoursBack, cHoursBack,));
353
354
355	#
356	# Process, format and output data.
357	#
358	dResult = testbox_data_processing(oDb);
359	self._oSrvGlue.setContentType('text/plain');
360	self._oSrvGlue.write(format_data(dResult, fSorted));
361
362	return True;
363
364	def _actionMagicMirrorTestResults(self):
365	"""
366	Produces test result status for the magic mirror dashboard
367	"""
368
369	#
370	# Parse arguments and connect to the database.
371	#
372	sBranch = self._getStringParam('sBranch');
373	cHoursBack = self._getIntParam('cHours', 1, 24*14, 6); ## @todo why 6 hours here and 12 for test boxes?
374	fSorted = self._getBoolParam('fSorted', False);
375	self._checkForUnknownParameters();
376
377	#
378	# Get the data.
379	#
380	# Note! These queries should be joining TestBoxesWithStrings and TestSets
381	# on idGenTestBox rather than on idTestBox and tsExpire=inf, but
382	# we don't have any index matching those. So, we'll ignore tests
383	# performed by deleted testboxes for the present as that doesn't
384	# happen often and we want the ~1000x speedup.
385	#
386	(oDb,) = self._connectToDb();
387	if oDb is None:
388	return False;
389
390	if sBranch == 'all':
391	oDb.execute('''
392	SELECT TestSets.enmStatus,
393	TestCases.sName,
394	TestBoxesWithStrings.sOS
395	FROM TestSets
396	INNER JOIN TestCases
397	ON TestCases.idGenTestCase = TestSets.idGenTestCase
398	INNER JOIN TestBoxesWithStrings
399	ON TestBoxesWithStrings.idTestBox = TestSets.idTestBox
400	AND TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
401	WHERE TestSets.tsCreated >= (CURRENT_TIMESTAMP - '%s hours'::interval)
402	''', (cHoursBack,));
403	else:
404	oDb.execute('''
405	SELECT TestSets.enmStatus,
406	TestCases.sName,
407	TestBoxesWithStrings.sOS
408	FROM TestSets
409	INNER JOIN BuildCategories
410	ON BuildCategories.idBuildCategory = TestSets.idBuildCategory
411	AND BuildCategories.sBranch = %s
412	INNER JOIN TestCases
413	ON TestCases.idGenTestCase = TestSets.idGenTestCase
414	INNER JOIN TestBoxesWithStrings
415	ON TestBoxesWithStrings.idTestBox = TestSets.idTestBox
416	AND TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
417	WHERE TestSets.tsCreated >= (CURRENT_TIMESTAMP - '%s hours'::interval)
418	''', (sBranch, cHoursBack,));
419
420	# Process the data
421	dResult = {};
422	while True:
423	aoRow = oDb.fetchOne();
424	if aoRow is None:
425	break;
426	os_results_separating(dResult, aoRow[1], aoRow[2], aoRow[0]) # save all test results
427
428	# Format and output it.
429	self._oSrvGlue.setContentType('text/plain');
430	self._oSrvGlue.write(format_data(dResult, fSorted));
431
432	return True;
433
434	def _getStandardParams(self, dParams):
435	"""
436	Gets the standard parameters and validates them.
437
438	The parameters are returned as a tuple: sAction, (more later, maybe)
439	Note! the sTextBoxId can be None if it's a SIGNON request.
440
441	Raises StatusDispatcherException on invalid input.
442	"""
443	#
444	# Get the action parameter and validate it.
445	#
446	if constants.tbreq.ALL_PARAM_ACTION not in dParams:
447	raise StatusDispatcherException('No "%s" parameter in request (params: %s)'
448	% (constants.tbreq.ALL_PARAM_ACTION, dParams,));
449	sAction = dParams[constants.tbreq.ALL_PARAM_ACTION];
450
451	if sAction not in self._dActions:
452	raise StatusDispatcherException('Unknown action "%s" in request (params: %s; action: %s)'
453	% (sAction, dParams, self._dActions));
454	#
455	# Update the list of checked parameters.
456	#
457	self._asCheckedParams.extend([constants.tbreq.ALL_PARAM_ACTION,]);
458
459	return (sAction,);
460
461	def dispatchRequest(self):
462	"""
463	Dispatches the incoming request.
464
465	Will raise StatusDispatcherException on failure.
466	"""
467
468	#
469	# Must be a GET request.
470	#
471	try:
472	sMethod = self._oSrvGlue.getMethod();
473	except Exception as oXcpt:
474	raise StatusDispatcherException('Error retriving request method: %s' % (oXcpt,));
475	if sMethod != 'GET':
476	raise StatusDispatcherException('Error expected POST request not "%s"' % (sMethod,));
477
478	#
479	# Get the parameters and checks for duplicates.
480	#
481	try:
482	dParams = self._oSrvGlue.getParameters();
483	except Exception as oXcpt:
484	raise StatusDispatcherException('Error retriving parameters: %s' % (oXcpt,));
485	for sKey in dParams.keys():
486	if len(dParams[sKey]) > 1:
487	raise StatusDispatcherException('Parameter "%s" is given multiple times: %s' % (sKey, dParams[sKey]));
488	dParams[sKey] = dParams[sKey][0];
489	self._dParams = dParams;
490
491	#
492	# Get+validate the standard action parameters and dispatch the request.
493	#
494	(self._sAction, ) = self._getStandardParams(dParams);
495	return self._dActions[self._sAction]();
496
497
498	def main():
499	"""
500	Main function a la C/C++. Returns exit code.
501	"""
502
503	oSrvGlue = WebServerGlueCgi(g_ksValidationKitDir, fHtmlOutput = False);
504	try:
505	oDisp = StatusDispatcher(oSrvGlue);
506	oDisp.dispatchRequest();
507	oSrvGlue.flush();
508	except Exception as oXcpt:
509	return oSrvGlue.errorPage('Internal error: %s' % (str(oXcpt),), sys.exc_info());
510
511	return 0;
512
513	if __name__ == '__main__':
514	if config.g_kfProfileAdmin:
515	from testmanager.debug import cgiprofiling;
516	sys.exit(cgiprofiling.profileIt(main));
517	else:
518	sys.exit(main());
519

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/cgi/status.py@ 91692

Download in other formats: