VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testmanager/cgi/status.py@ 96807

Last change on this file since 96807 was 96407, checked in by vboxsync, 2 years ago

scm copyright and license note update

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 19.5 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: status.py 96407 2022-08-22 17:43:14Z vboxsync $
4
5"""
6CGI - Administrator Web-UI.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2022 Oracle and/or its affiliates.
12
13This file is part of VirtualBox base platform packages, as
14available from https://www.virtualbox.org.
15
16This program is free software; you can redistribute it and/or
17modify it under the terms of the GNU General Public License
18as published by the Free Software Foundation, in version 3 of the
19License.
20
21This program is distributed in the hope that it will be useful, but
22WITHOUT ANY WARRANTY; without even the implied warranty of
23MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
24General Public License for more details.
25
26You should have received a copy of the GNU General Public License
27along with this program; if not, see <https://www.gnu.org/licenses>.
28
29The contents of this file may alternatively be used under the terms
30of the Common Development and Distribution License Version 1.0
31(CDDL), a copy of it is provided in the "COPYING.CDDL" file included
32in the VirtualBox distribution, in which case the provisions of the
33CDDL are applicable instead of those of the GPL.
34
35You may elect to license modified versions of this file under the
36terms and conditions of either the GPL or the CDDL or both.
37
38SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
39"""
40__version__ = "$Revision: 96407 $"
41
42
43# Standard python imports.
44import os
45import sys
46
47# Only the main script needs to modify the path.
48g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
49sys.path.append(g_ksValidationKitDir);
50
51# Validation Kit imports.
52from testmanager import config;
53from testmanager.core.webservergluecgi import WebServerGlueCgi;
54
55from common import constants;
56from testmanager.core.base import TMExceptionBase;
57from testmanager.core.db import TMDatabaseConnection;
58
59
60
61def timeDeltaToHours(oTimeDelta):
62 return oTimeDelta.days * 24 + oTimeDelta.seconds // 3600
63
64
65def testbox_data_processing(oDb):
66 testboxes_dict = {}
67 while True:
68 line = oDb.fetchOne();
69 if line is None:
70 break;
71 testbox_name = line[0]
72 test_result = line[1]
73 oTimeDeltaSinceStarted = line[2]
74 test_box_os = line[3]
75 test_sched_group = line[4]
76 testboxes_dict = dict_update(testboxes_dict, testbox_name, test_result)
77
78 if "testbox_os" not in testboxes_dict[testbox_name]:
79 testboxes_dict[testbox_name].update({"testbox_os": test_box_os})
80
81 if "sched_group" not in testboxes_dict[testbox_name]:
82 testboxes_dict[testbox_name].update({"sched_group": test_sched_group})
83 elif test_sched_group not in testboxes_dict[testbox_name]["sched_group"]:
84 testboxes_dict[testbox_name]["sched_group"] += "," + test_sched_group
85
86 if test_result == "running":
87 testboxes_dict[testbox_name].update({"hours_running": timeDeltaToHours(oTimeDeltaSinceStarted)})
88
89 return testboxes_dict;
90
91
92def os_results_separating(vb_dict, test_name, testbox_os, test_result):
93 if testbox_os == "linux":
94 dict_update(vb_dict, test_name + " / linux", test_result)
95 elif testbox_os == "win":
96 dict_update(vb_dict, test_name + " / windows", test_result)
97 elif testbox_os == "darwin":
98 dict_update(vb_dict, test_name + " / darwin", test_result)
99 elif testbox_os == "solaris":
100 dict_update(vb_dict, test_name + " / solaris", test_result)
101 else:
102 dict_update(vb_dict, test_name + " / other", test_result)
103
104
105# const/immutable.
106g_kdTestStatuses = {
107 'running': 0,
108 'success': 0,
109 'skipped': 0,
110 'bad-testbox': 0,
111 'aborted': 0,
112 'failure': 0,
113 'timed-out': 0,
114 'rebooted': 0,
115}
116
117def dict_update(target_dict, key_name, test_result):
118 if key_name not in target_dict:
119 target_dict.update({key_name: g_kdTestStatuses.copy()})
120 if test_result in g_kdTestStatuses:
121 target_dict[key_name][test_result] += 1
122 return target_dict
123
124
125def formatDataEntry(sKey, dEntry):
126 # There are variations in the first and second "columns".
127 if "hours_running" in dEntry:
128 sRet = "%s;%s;%s | running: %s;%s" \
129 % (sKey, dEntry["testbox_os"], dEntry["sched_group"], dEntry["running"], dEntry["hours_running"]);
130 else:
131 if "testbox_os" in dEntry:
132 sRet = "%s;%s;%s" % (sKey, dEntry["testbox_os"], dEntry["sched_group"],);
133 else:
134 sRet = sKey;
135 sRet += " | running: %s" % (dEntry["running"],)
136
137 # The rest is currently identical:
138 sRet += " | success: %s | skipped: %s | bad-testbox: %s | aborted: %s | failure: %s | timed-out: %s | rebooted: %s | \n" \
139 % (dEntry["success"], dEntry["skipped"], dEntry["bad-testbox"], dEntry["aborted"],
140 dEntry["failure"], dEntry["timed-out"], dEntry["rebooted"],);
141 return sRet;
142
143
144def format_data(dData, fSorted):
145 sRet = "";
146 if not fSorted:
147 for sKey in dData:
148 sRet += formatDataEntry(sKey, dData[sKey]);
149 else:
150 for sKey in sorted(dData.keys()):
151 sRet += formatDataEntry(sKey, dData[sKey]);
152 return sRet;
153
154######
155
156class StatusDispatcherException(TMExceptionBase):
157 """
158 Exception class for TestBoxController.
159 """
160 pass; # pylint: disable=unnecessary-pass
161
162
163class StatusDispatcher(object): # pylint: disable=too-few-public-methods
164 """
165 Status dispatcher class.
166 """
167
168
169 def __init__(self, oSrvGlue):
170 """
171 Won't raise exceptions.
172 """
173 self._oSrvGlue = oSrvGlue;
174 self._sAction = None; # _getStandardParams / dispatchRequest sets this later on.
175 self._dParams = None; # _getStandardParams / dispatchRequest sets this later on.
176 self._asCheckedParams = [];
177 self._dActions = \
178 {
179 'MagicMirrorTestResults': self._actionMagicMirrorTestResults,
180 'MagicMirrorTestBoxes': self._actionMagicMirrorTestBoxes,
181 };
182
183 def _getStringParam(self, sName, asValidValues = None, fStrip = False, sDefValue = None):
184 """
185 Gets a string parameter (stripped).
186
187 Raises exception if not found and no default is provided, or if the
188 value isn't found in asValidValues.
189 """
190 if sName not in self._dParams:
191 if sDefValue is None:
192 raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
193 return sDefValue;
194 sValue = self._dParams[sName];
195 if fStrip:
196 sValue = sValue.strip();
197
198 if sName not in self._asCheckedParams:
199 self._asCheckedParams.append(sName);
200
201 if asValidValues is not None and sValue not in asValidValues:
202 raise StatusDispatcherException('%s parameter %s value "%s" not in %s '
203 % (self._sAction, sName, sValue, asValidValues));
204 return sValue;
205
206 def _getIntParam(self, sName, iMin = None, iMax = None, iDefValue = None):
207 """
208 Gets a string parameter.
209 Raises exception if not found, not a valid integer, or if the value
210 isn't in the range defined by iMin and iMax.
211 """
212 if sName not in self._dParams:
213 if iDefValue is None:
214 raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
215 return iDefValue;
216 sValue = self._dParams[sName];
217 try:
218 iValue = int(sValue, 0);
219 except:
220 raise StatusDispatcherException('%s parameter %s value "%s" cannot be convert to an integer'
221 % (self._sAction, sName, sValue));
222 if sName not in self._asCheckedParams:
223 self._asCheckedParams.append(sName);
224
225 if (iMin is not None and iValue < iMin) \
226 or (iMax is not None and iValue > iMax):
227 raise StatusDispatcherException('%s parameter %s value %d is out of range [%s..%s]'
228 % (self._sAction, sName, iValue, iMin, iMax));
229 return iValue;
230
231 def _getBoolParam(self, sName, fDefValue = None):
232 """
233 Gets a boolean parameter.
234
235 Raises exception if not found and no default is provided, or if not a
236 valid boolean.
237 """
238 sValue = self._getStringParam(sName, [ 'True', 'true', '1', 'False', 'false', '0'], sDefValue = str(fDefValue));
239 return sValue in ('True', 'true', '1',);
240
241 def _checkForUnknownParameters(self):
242 """
243 Check if we've handled all parameters, raises exception if anything
244 unknown was found.
245 """
246
247 if len(self._asCheckedParams) != len(self._dParams):
248 sUnknownParams = '';
249 for sKey in self._dParams:
250 if sKey not in self._asCheckedParams:
251 sUnknownParams += ' ' + sKey + '=' + self._dParams[sKey];
252 raise StatusDispatcherException('Unknown parameters: ' + sUnknownParams);
253
254 return True;
255
256 def _connectToDb(self):
257 """
258 Connects to the database.
259
260 Returns (TMDatabaseConnection, (more later perhaps) ) on success.
261 Returns (None, ) on failure after sending the box an appropriate response.
262 May raise exception on DB error.
263 """
264 return (TMDatabaseConnection(self._oSrvGlue.dprint),);
265
266 def _actionMagicMirrorTestBoxes(self):
267 """
268 Produces test result status for the magic mirror dashboard
269 """
270
271 #
272 # Parse arguments and connect to the database.
273 #
274 cHoursBack = self._getIntParam('cHours', 1, 24*14, 12);
275 fSorted = self._getBoolParam('fSorted', False);
276 self._checkForUnknownParameters();
277
278 #
279 # Get the data.
280 #
281 # Note! We're not joining on TestBoxesWithStrings.idTestBox =
282 # TestSets.idGenTestBox here because of indexes. This is
283 # also more consistent with the rest of the query.
284 # Note! The original SQL is slow because of the 'OR TestSets.tsDone'
285 # part, using AND and UNION is significatly faster because
286 # it matches the TestSetsGraphBoxIdx (index).
287 #
288 (oDb,) = self._connectToDb();
289 if oDb is None:
290 return False;
291
292 # oDb.execute('''
293 #SELECT TestBoxesWithStrings.sName,
294 # TestSets.enmStatus,
295 # CURRENT_TIMESTAMP - TestSets.tsCreated,
296 # TestBoxesWithStrings.sOS,
297 # SchedGroupNames.sSchedGroupNames
298 #FROM (SELECT TestBoxesInSchedGroups.idTestBox AS idTestBox,
299 # STRING_AGG(SchedGroups.sName, ',') AS sSchedGroupNames
300 # FROM TestBoxesInSchedGroups
301 # INNER JOIN SchedGroups
302 # ON SchedGroups.idSchedGroup = TestBoxesInSchedGroups.idSchedGroup
303 # WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
304 # AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
305 # GROUP BY TestBoxesInSchedGroups.idTestBox)
306 # AS SchedGroupNames,
307 # TestBoxesWithStrings
308 #LEFT OUTER JOIN TestSets
309 # ON TestSets.idTestBox = TestBoxesWithStrings.idTestBox
310 # AND ( TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
311 # OR TestSets.tsDone IS NULL)
312 #WHERE TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
313 # AND SchedGroupNames.idTestBox = TestBoxesWithStrings.idTestBox
314 #''', (cHoursBack,));
315 oDb.execute('''
316( SELECT TestBoxesWithStrings.sName,
317 TestSets.enmStatus,
318 CURRENT_TIMESTAMP - TestSets.tsCreated,
319 TestBoxesWithStrings.sOS,
320 SchedGroupNames.sSchedGroupNames
321 FROM (
322 SELECT TestBoxesInSchedGroups.idTestBox AS idTestBox,
323 STRING_AGG(SchedGroups.sName, ',') AS sSchedGroupNames
324 FROM TestBoxesInSchedGroups
325 INNER JOIN SchedGroups
326 ON SchedGroups.idSchedGroup = TestBoxesInSchedGroups.idSchedGroup
327 WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
328 AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
329 GROUP BY TestBoxesInSchedGroups.idTestBox
330 ) AS SchedGroupNames,
331 TestBoxesWithStrings
332 LEFT OUTER JOIN TestSets
333 ON TestSets.idTestBox = TestBoxesWithStrings.idTestBox
334 AND TestSets.tsCreated >= (CURRENT_TIMESTAMP - '%s hours'::interval)
335 AND TestSets.tsDone IS NOT NULL
336 WHERE TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
337 AND SchedGroupNames.idTestBox = TestBoxesWithStrings.idTestBox
338) UNION (
339 SELECT TestBoxesWithStrings.sName,
340 TestSets.enmStatus,
341 CURRENT_TIMESTAMP - TestSets.tsCreated,
342 TestBoxesWithStrings.sOS,
343 SchedGroupNames.sSchedGroupNames
344 FROM (
345 SELECT TestBoxesInSchedGroups.idTestBox AS idTestBox,
346 STRING_AGG(SchedGroups.sName, ',') AS sSchedGroupNames
347 FROM TestBoxesInSchedGroups
348 INNER JOIN SchedGroups
349 ON SchedGroups.idSchedGroup = TestBoxesInSchedGroups.idSchedGroup
350 WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
351 AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
352 GROUP BY TestBoxesInSchedGroups.idTestBox
353 ) AS SchedGroupNames,
354 TestBoxesWithStrings
355 LEFT OUTER JOIN TestSets
356 ON TestSets.idTestBox = TestBoxesWithStrings.idTestBox
357 AND TestSets.tsCreated < (CURRENT_TIMESTAMP - '%s hours'::interval)
358 AND TestSets.tsDone IS NULL
359 WHERE TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
360 AND SchedGroupNames.idTestBox = TestBoxesWithStrings.idTestBox
361)
362''', (cHoursBack, cHoursBack,));
363
364
365 #
366 # Process, format and output data.
367 #
368 dResult = testbox_data_processing(oDb);
369 self._oSrvGlue.setContentType('text/plain');
370 self._oSrvGlue.write(format_data(dResult, fSorted));
371
372 return True;
373
374 def _actionMagicMirrorTestResults(self):
375 """
376 Produces test result status for the magic mirror dashboard
377 """
378
379 #
380 # Parse arguments and connect to the database.
381 #
382 sBranch = self._getStringParam('sBranch');
383 cHoursBack = self._getIntParam('cHours', 1, 24*14, 6); ## @todo why 6 hours here and 12 for test boxes?
384 fSorted = self._getBoolParam('fSorted', False);
385 self._checkForUnknownParameters();
386
387 #
388 # Get the data.
389 #
390 # Note! These queries should be joining TestBoxesWithStrings and TestSets
391 # on idGenTestBox rather than on idTestBox and tsExpire=inf, but
392 # we don't have any index matching those. So, we'll ignore tests
393 # performed by deleted testboxes for the present as that doesn't
394 # happen often and we want the ~1000x speedup.
395 #
396 (oDb,) = self._connectToDb();
397 if oDb is None:
398 return False;
399
400 if sBranch == 'all':
401 oDb.execute('''
402SELECT TestSets.enmStatus,
403 TestCases.sName,
404 TestBoxesWithStrings.sOS
405FROM TestSets
406INNER JOIN TestCases
407 ON TestCases.idGenTestCase = TestSets.idGenTestCase
408INNER JOIN TestBoxesWithStrings
409 ON TestBoxesWithStrings.idTestBox = TestSets.idTestBox
410 AND TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
411WHERE TestSets.tsCreated >= (CURRENT_TIMESTAMP - '%s hours'::interval)
412''', (cHoursBack,));
413 else:
414 oDb.execute('''
415SELECT TestSets.enmStatus,
416 TestCases.sName,
417 TestBoxesWithStrings.sOS
418FROM TestSets
419INNER JOIN BuildCategories
420 ON BuildCategories.idBuildCategory = TestSets.idBuildCategory
421 AND BuildCategories.sBranch = %s
422INNER JOIN TestCases
423 ON TestCases.idGenTestCase = TestSets.idGenTestCase
424INNER JOIN TestBoxesWithStrings
425 ON TestBoxesWithStrings.idTestBox = TestSets.idTestBox
426 AND TestBoxesWithStrings.tsExpire = 'infinity'::TIMESTAMP
427WHERE TestSets.tsCreated >= (CURRENT_TIMESTAMP - '%s hours'::interval)
428''', (sBranch, cHoursBack,));
429
430 # Process the data
431 dResult = {};
432 while True:
433 aoRow = oDb.fetchOne();
434 if aoRow is None:
435 break;
436 os_results_separating(dResult, aoRow[1], aoRow[2], aoRow[0]) # save all test results
437
438 # Format and output it.
439 self._oSrvGlue.setContentType('text/plain');
440 self._oSrvGlue.write(format_data(dResult, fSorted));
441
442 return True;
443
444 def _getStandardParams(self, dParams):
445 """
446 Gets the standard parameters and validates them.
447
448 The parameters are returned as a tuple: sAction, (more later, maybe)
449 Note! the sTextBoxId can be None if it's a SIGNON request.
450
451 Raises StatusDispatcherException on invalid input.
452 """
453 #
454 # Get the action parameter and validate it.
455 #
456 if constants.tbreq.ALL_PARAM_ACTION not in dParams:
457 raise StatusDispatcherException('No "%s" parameter in request (params: %s)'
458 % (constants.tbreq.ALL_PARAM_ACTION, dParams,));
459 sAction = dParams[constants.tbreq.ALL_PARAM_ACTION];
460
461 if sAction not in self._dActions:
462 raise StatusDispatcherException('Unknown action "%s" in request (params: %s; action: %s)'
463 % (sAction, dParams, self._dActions));
464 #
465 # Update the list of checked parameters.
466 #
467 self._asCheckedParams.extend([constants.tbreq.ALL_PARAM_ACTION,]);
468
469 return (sAction,);
470
471 def dispatchRequest(self):
472 """
473 Dispatches the incoming request.
474
475 Will raise StatusDispatcherException on failure.
476 """
477
478 #
479 # Must be a GET request.
480 #
481 try:
482 sMethod = self._oSrvGlue.getMethod();
483 except Exception as oXcpt:
484 raise StatusDispatcherException('Error retriving request method: %s' % (oXcpt,));
485 if sMethod != 'GET':
486 raise StatusDispatcherException('Error expected POST request not "%s"' % (sMethod,));
487
488 #
489 # Get the parameters and checks for duplicates.
490 #
491 try:
492 dParams = self._oSrvGlue.getParameters();
493 except Exception as oXcpt:
494 raise StatusDispatcherException('Error retriving parameters: %s' % (oXcpt,));
495 for sKey in dParams.keys():
496 if len(dParams[sKey]) > 1:
497 raise StatusDispatcherException('Parameter "%s" is given multiple times: %s' % (sKey, dParams[sKey]));
498 dParams[sKey] = dParams[sKey][0];
499 self._dParams = dParams;
500
501 #
502 # Get+validate the standard action parameters and dispatch the request.
503 #
504 (self._sAction, ) = self._getStandardParams(dParams);
505 return self._dActions[self._sAction]();
506
507
508def main():
509 """
510 Main function a la C/C++. Returns exit code.
511 """
512
513 oSrvGlue = WebServerGlueCgi(g_ksValidationKitDir, fHtmlOutput = False);
514 try:
515 oDisp = StatusDispatcher(oSrvGlue);
516 oDisp.dispatchRequest();
517 oSrvGlue.flush();
518 except Exception as oXcpt:
519 return oSrvGlue.errorPage('Internal error: %s' % (str(oXcpt),), sys.exc_info());
520
521 return 0;
522
523if __name__ == '__main__':
524 if config.g_kfProfileAdmin:
525 from testmanager.debug import cgiprofiling;
526 sys.exit(cgiprofiling.profileIt(main));
527 else:
528 sys.exit(main());
529
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette