VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testmanager/cgi/status.py@ 86933

Last change on this file since 86933 was 86933, checked in by vboxsync, 4 years ago

testmanager/status.py: Fix.

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 19.1 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: status.py 86933 2020-11-20 14:55:05Z vboxsync $
4
5"""
6CGI - Administrator Web-UI.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2020 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 86933 $"
31
32
33# Standard python imports.
34import os
35import sys
36import datetime
37
38# Only the main script needs to modify the path.
39g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
40sys.path.append(g_ksValidationKitDir);
41
42# Validation Kit imports.
43from testmanager import config;
44from testmanager.core.webservergluecgi import WebServerGlueCgi;
45
46from common import constants;
47from testmanager.core.base import TMExceptionBase;
48from testmanager.core.db import TMDatabaseConnection;
49
50
51
52def how_many_days_in_month(year, month):
53 def leap_year_check(year):
54 if year % 4 == 0 and year % 100 != 0:
55 return True
56 if year % 100 == 0 and year % 400 == 0:
57 return True
58 return False
59
60 month31 = (1, 3, 5, 7, 8, 10, 12)
61 month30 = (4, 6, 9, 11)
62 if month in month31:
63 days = 31
64 elif month in month30:
65 days = 30
66 else:
67 if leap_year_check(year):
68 days = 29
69 else:
70 days = 28
71 return days
72
73
74def target_date_from_time_span(cur_date, time_span_hours):
75 cur_year = cur_date.year
76 cur_month = cur_date.month
77 cur_day = cur_date.day
78 cur_hour = cur_date.hour
79 if cur_hour >= time_span_hours:
80 return cur_date.replace(hour=cur_hour-time_span_hours)
81 if cur_day > 1:
82 return cur_date.replace(day=cur_day-1,
83 hour=24+cur_hour-time_span_hours)
84 if cur_month > 1:
85 return cur_date.replace(month=cur_month-1,
86 day=how_many_days_in_month(cur_year, cur_month-1),
87 hour=24+cur_hour-time_span_hours)
88 return cur_date.replace(year=cur_year-1,
89 month=12,
90 day=31,
91 hour=24+cur_hour-time_span_hours)
92
93
94def find_test_duration(created):
95 now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
96 diff = now - created
97 days, seconds = diff.days, diff.seconds
98 hours = days * 24 + seconds // 3600
99 return hours
100
101
102def testbox_data_processing(oDb):
103 testboxes_dict = {}
104 for line in oDb.fetchOne():
105 testbox_name = line[0]
106 test_result = line[1]
107 test_created = line[2]
108 test_box_os = line[3]
109 test_sched_group = line[4]
110 testboxes_dict = dict_update(testboxes_dict, testbox_name, test_result)
111
112 if "testbox_os" not in testboxes_dict[testbox_name].keys(): ## bird: you don't need to use keys() here.
113 testboxes_dict[testbox_name].update({"testbox_os": test_box_os})
114
115 if "sched_group" not in testboxes_dict[testbox_name].keys():
116 testboxes_dict[testbox_name].update({"sched_group": test_sched_group})
117 elif test_sched_group not in testboxes_dict[testbox_name]["sched_group"]:
118 testboxes_dict[testbox_name]["sched_group"] = testboxes_dict[testbox_name]["sched_group"] + ","+ test_sched_group
119
120 if test_result == "running":
121 testboxes_dict[testbox_name].update({"hours_running": find_test_duration(test_created)})
122
123 #earlier: clean_file_and_insert_data(testboxes_dict, target_file)
124 return testboxes_dict;
125
126
127def os_results_separating(vb_dict, test_name, testbox_os, test_result):
128 if testbox_os == "linux":
129 dict_update(vb_dict, test_name + " / linux", test_result)
130 elif testbox_os == "win":
131 dict_update(vb_dict, test_name + " / windows", test_result)
132 elif testbox_os == "darwin":
133 dict_update(vb_dict, test_name + " / darwin", test_result)
134 elif testbox_os == "solaris":
135 dict_update(vb_dict, test_name + " / solaris", test_result)
136 else:
137 dict_update(vb_dict, test_name + " / other", test_result)
138
139
140def dict_update(target_dict, key_name, test_result):
141 def test_statuses_dict_default():
142 test_statuses_dict = {
143 'running': 0,
144 'success': 0,
145 'skipped': 0,
146 'bad-testbox': 0,
147 'aborted': 0,
148 'failure': 0,
149 'timed-out': 0,
150 'rebooted': 0,
151 }
152 return test_statuses_dict
153 if key_name not in target_dict.keys():
154 target_dict.update({key_name: test_statuses_dict_default().copy()})
155 if test_result in test_statuses_dict_default().keys():
156 target_dict[key_name][test_result] += 1
157 return target_dict
158
159
160def format_data(target_dict):
161 content = ""
162 for key in target_dict:
163 if "hours_running" in target_dict[key].keys():
164 content += "{};{};{} | running: {};{} | success: {} | skipped: {} | ".format(key,
165 target_dict[key]["testbox_os"],
166 target_dict[key]["sched_group"],
167 target_dict[key]["running"],
168 target_dict[key]["hours_running"],
169 target_dict[key]["success"],
170 target_dict[key]["skipped"],
171 )
172 elif "testbox_os" in target_dict[key].keys():
173 content += "{};{};{} | running: {} | success: {} | skipped: {} | ".format(key,
174 target_dict[key]["testbox_os"],
175 target_dict[key]["sched_group"],
176 target_dict[key]["running"],
177 target_dict[key]["success"],
178 target_dict[key]["skipped"],
179 )
180 else:
181 content += "{} | running: {} | success: {} | skipped: {} | ".format(key,
182 target_dict[key]["running"],
183 target_dict[key]["success"],
184 target_dict[key]["skipped"],
185 )
186 content += "bad-testbox: {} | aborted: {} | failure: {} | ".format(
187 target_dict[key]["bad-testbox"],
188 target_dict[key]["aborted"],
189 target_dict[key]["failure"],
190 )
191 content += "timed-out: {} | rebooted: {} | \n".format(
192 target_dict[key]["timed-out"],
193 target_dict[key]["rebooted"],
194 )
195 return content
196
197######
198
199class StatusDispatcherException(TMExceptionBase):
200 """
201 Exception class for TestBoxController.
202 """
203 pass; # pylint: disable=unnecessary-pass
204
205
206class StatusDispatcher(object): # pylint: disable=too-few-public-methods
207 """
208 Status dispatcher class.
209 """
210
211
212 def __init__(self, oSrvGlue):
213 """
214 Won't raise exceptions.
215 """
216 self._oSrvGlue = oSrvGlue;
217 self._sAction = None; # _getStandardParams / dispatchRequest sets this later on.
218 self._dParams = None; # _getStandardParams / dispatchRequest sets this later on.
219 self._asCheckedParams = [];
220 self._dActions = \
221 {
222 'MagicMirrorTestResults': self._actionMagicMirrorTestResults,
223 'MagicMirrorTestBoxes': self._actionMagicMirrorTestBoxes,
224 };
225
226 def _getStringParam(self, sName, asValidValues = None, fStrip = False, sDefValue = None):
227 """
228 Gets a string parameter (stripped).
229
230 Raises exception if not found and no default is provided, or if the
231 value isn't found in asValidValues.
232 """
233 if sName not in self._dParams:
234 if sDefValue is None:
235 raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
236 return sDefValue;
237 sValue = self._dParams[sName];
238 if fStrip:
239 sValue = sValue.strip();
240
241 if sName not in self._asCheckedParams:
242 self._asCheckedParams.append(sName);
243
244 if asValidValues is not None and sValue not in asValidValues:
245 raise StatusDispatcherException('%s parameter %s value "%s" not in %s '
246 % (self._sAction, sName, sValue, asValidValues));
247 return sValue;
248
249 def _getIntParam(self, sName, iMin = None, iMax = None, iDefValue = None):
250 """
251 Gets a string parameter.
252 Raises exception if not found, not a valid integer, or if the value
253 isn't in the range defined by iMin and iMax.
254 """
255 if sName not in self._dParams:
256 if iDefValue is None:
257 raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
258 return iDefValue;
259 sValue = self._dParams[sName];
260 try:
261 iValue = int(sValue, 0);
262 except:
263 raise StatusDispatcherException('%s parameter %s value "%s" cannot be convert to an integer'
264 % (self._sAction, sName, sValue));
265 if sName not in self._asCheckedParams:
266 self._asCheckedParams.append(sName);
267
268 if (iMin is not None and iValue < iMin) \
269 or (iMax is not None and iValue > iMax):
270 raise StatusDispatcherException('%s parameter %s value %d is out of range [%s..%s]'
271 % (self._sAction, sName, iValue, iMin, iMax));
272 return iValue;
273
274 def _checkForUnknownParameters(self):
275 """
276 Check if we've handled all parameters, raises exception if anything
277 unknown was found.
278 """
279
280 if len(self._asCheckedParams) != len(self._dParams):
281 sUnknownParams = '';
282 for sKey in self._dParams:
283 if sKey not in self._asCheckedParams:
284 sUnknownParams += ' ' + sKey + '=' + self._dParams[sKey];
285 raise StatusDispatcherException('Unknown parameters: ' + sUnknownParams);
286
287 return True;
288
289 def _connectToDb(self):
290 """
291 Connects to the database.
292
293 Returns (TMDatabaseConnection, (more later perhaps) ) on success.
294 Returns (None, ) on failure after sending the box an appropriate response.
295 May raise exception on DB error.
296 """
297 return (TMDatabaseConnection(self._oSrvGlue.dprint),);
298
299 def _actionMagicMirrorTestBoxes(self):
300 """
301 Produces test result status for the magic mirror dashboard
302 """
303
304 #
305 # Parse arguments and connect to the database.
306 #
307 cHoursBack = self._getIntParam('cHours', 1, 24*14, 12);
308 self._checkForUnknownParameters();
309
310 #
311 # Get the data.
312 # bird: I changed these to join on idGenTestBox and skipping the tsExpire condition.
313 # @todo The query isn't very efficient as postgresql probably will repeat the
314 # subselect in column 5 for each result row.
315 #
316 (oDb,) = self._connectToDb();
317 if oDb is None:
318 return False;
319
320 oDb.execute('''
321SELECT TestBoxesWithStrings.sName,
322 TestSets.enmStatus,
323 TestSets.tsCreated,
324 TestBoxesWithStrings.sOS,
325 ( SELECT STRING_AGG(SchedGroups.sName, ',')
326 FROM SchedGroups
327 INNER JOIN TestBoxesInSchedGroups
328 ON TestBoxesInSchedGroups.idSchedGroup = SchedGroups.idSchedGroup
329 AND TestBoxesInSchedGroups.idTestBox = TestBoxesWithStrings.idTestBox
330 WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
331 AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
332 ) AS SchedGroupName
333FROM TestBoxesWithStrings
334LEFT OUTER JOIN TestSets
335 ON TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox
336 AND ( TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
337 OR TestSets.tsDone IS NULL)
338''', (cHoursBack,));
339
340 # Process the data
341 dResult = testbox_data_processing(oDb);
342
343 # Format and output it.
344 self._oSrvGlue.setContentType('text/plain');
345 self._oSrvGlue.write(format_data(dResult));
346
347 return True;
348
349 def _actionMagicMirrorTestResults(self):
350 """
351 Produces test result status for the magic mirror dashboard
352 """
353
354 #
355 # Parse arguments and connect to the database.
356 #
357 sBranch = self._getStringParam('sBranch');
358 cHoursBack = self._getIntParam('cHours', 1, 24*14, 6); ## @todo why 6 hours here and 12 for test boxes?
359 self._checkForUnknownParameters();
360
361 #
362 # Get the data.
363 # bird: I changed these to join on idGenTestBox and idGenTestCase instead of
364 # also needing to check the tsExpire columns.
365 #
366 (oDb,) = self._connectToDb();
367 if oDb is None:
368 return False;
369
370 if sBranch == 'all':
371 oDb.execute('''
372SELECT TestSets.enmStatus,
373 TestCases.sName,
374 TestBoxesWithStrings.sOS
375FROM TestSets
376INNER JOIN TestCases
377 ON TestCases.idGenTestCase = TestCases.idGenTestCase
378INNER JOIN TestBoxesWithStrings
379 ON TestBoxesWithStrings.idGenTestBox = TestSets.idGetTestBox
380WHERE TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
381''', (cHoursBack,));
382 else:
383 oDb.execute('''
384SELECT TestSets.enmStatus,
385 TestCases.sName,
386 TestBoxesWithStrings.sOS
387FROM TestSets
388INNER JOIN BuildCategories
389 ON BuildCategories.idBuildCategory = TestSets.idBuildCategory
390 AND BuildCategories.sBuildCategories = '%s'
391INNER JOIN TestCases
392 ON TestCases.idGenTestCase = TestSets.idGetTestCase
393INNER JOIN TestBoxesWithStrings
394 ON TestBoxesWithStrings.idGenTestBox = TestSets.idGetTestBox
395WHERE TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
396''', (sBranch, cHoursBack,));
397
398 # Process the data
399 dResult = {};
400 for aoRow in oDb.fetchOne():
401 os_results_separating(dResult, aoRow[1], aoRow[2], aoRow[0]) # save all test results
402
403 # Format and output it.
404 self._oSrvGlue.setContentType('text/plain');
405 self._oSrvGlue.write(format_data(dResult));
406
407 return True;
408
409 def _getStandardParams(self, dParams):
410 """
411 Gets the standard parameters and validates them.
412
413 The parameters are returned as a tuple: sAction, (more later, maybe)
414 Note! the sTextBoxId can be None if it's a SIGNON request.
415
416 Raises StatusDispatcherException on invalid input.
417 """
418 #
419 # Get the action parameter and validate it.
420 #
421 if constants.tbreq.ALL_PARAM_ACTION not in dParams:
422 raise StatusDispatcherException('No "%s" parameter in request (params: %s)'
423 % (constants.tbreq.ALL_PARAM_ACTION, dParams,));
424 sAction = dParams[constants.tbreq.ALL_PARAM_ACTION];
425
426 if sAction not in self._dActions:
427 raise StatusDispatcherException('Unknown action "%s" in request (params: %s; action: %s)'
428 % (sAction, dParams, self._dActions));
429 #
430 # Update the list of checked parameters.
431 #
432 self._asCheckedParams.extend([constants.tbreq.ALL_PARAM_ACTION,]);
433
434 return (sAction,);
435
436 def dispatchRequest(self):
437 """
438 Dispatches the incoming request.
439
440 Will raise StatusDispatcherException on failure.
441 """
442
443 #
444 # Must be a GET request.
445 #
446 try:
447 sMethod = self._oSrvGlue.getMethod();
448 except Exception as oXcpt:
449 raise StatusDispatcherException('Error retriving request method: %s' % (oXcpt,));
450 if sMethod != 'GET':
451 raise StatusDispatcherException('Error expected POST request not "%s"' % (sMethod,));
452
453 #
454 # Get the parameters and checks for duplicates.
455 #
456 try:
457 dParams = self._oSrvGlue.getParameters();
458 except Exception as oXcpt:
459 raise StatusDispatcherException('Error retriving parameters: %s' % (oXcpt,));
460 for sKey in dParams.keys():
461 if len(dParams[sKey]) > 1:
462 raise StatusDispatcherException('Parameter "%s" is given multiple times: %s' % (sKey, dParams[sKey]));
463 dParams[sKey] = dParams[sKey][0];
464 self._dParams = dParams;
465
466 #
467 # Get+validate the standard action parameters and dispatch the request.
468 #
469 (self._sAction, ) = self._getStandardParams(dParams);
470 return self._dActions[self._sAction]();
471
472
473def main():
474 """
475 Main function a la C/C++. Returns exit code.
476 """
477
478 oSrvGlue = WebServerGlueCgi(g_ksValidationKitDir, fHtmlOutput = False);
479 try:
480 oDisp = StatusDispatcher(oSrvGlue);
481 oDisp.dispatchRequest();
482 oSrvGlue.flush();
483 except Exception as oXcpt:
484 return oSrvGlue.errorPage('Internal error: %s' % (str(oXcpt),), sys.exc_info());
485
486 return 0;
487
488if __name__ == '__main__':
489 if config.g_kfProfileAdmin:
490 from testmanager.debug import cgiprofiling;
491 sys.exit(cgiprofiling.profileIt(main));
492 else:
493 sys.exit(main());
494
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette