VirtualBox

source: vbox/trunk/src/VBox/ValidationKit/testmanager/cgi/status.py@ 86935

Last change on this file since 86935 was 86935, checked in by vboxsync, 4 years ago

testmanager/status.py: Fixes.

  • Property svn:eol-style set to LF
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Revision
File size: 19.2 KB
Line 
1#!/usr/bin/env python
2# -*- coding: utf-8 -*-
3# $Id: status.py 86935 2020-11-20 15:05:08Z vboxsync $
4
5"""
6CGI - Administrator Web-UI.
7"""
8
9__copyright__ = \
10"""
11Copyright (C) 2012-2020 Oracle Corporation
12
13This file is part of VirtualBox Open Source Edition (OSE), as
14available from http://www.virtualbox.org. This file is free software;
15you can redistribute it and/or modify it under the terms of the GNU
16General Public License (GPL) as published by the Free Software
17Foundation, in version 2 as it comes in the "COPYING" file of the
18VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21The contents of this file may alternatively be used under the terms
22of the Common Development and Distribution License Version 1.0
23(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24VirtualBox OSE distribution, in which case the provisions of the
25CDDL are applicable instead of those of the GPL.
26
27You may elect to license modified versions of this file under the
28terms and conditions of either the GPL or the CDDL or both.
29"""
30__version__ = "$Revision: 86935 $"
31
32
33# Standard python imports.
34import os
35import sys
36import datetime
37
38# Only the main script needs to modify the path.
39g_ksValidationKitDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))));
40sys.path.append(g_ksValidationKitDir);
41
42# Validation Kit imports.
43from testmanager import config;
44from testmanager.core.webservergluecgi import WebServerGlueCgi;
45
46from common import constants;
47from testmanager.core.base import TMExceptionBase;
48from testmanager.core.db import TMDatabaseConnection;
49
50
51
52def how_many_days_in_month(year, month):
53 def leap_year_check(year):
54 if year % 4 == 0 and year % 100 != 0:
55 return True
56 if year % 100 == 0 and year % 400 == 0:
57 return True
58 return False
59
60 month31 = (1, 3, 5, 7, 8, 10, 12)
61 month30 = (4, 6, 9, 11)
62 if month in month31:
63 days = 31
64 elif month in month30:
65 days = 30
66 else:
67 if leap_year_check(year):
68 days = 29
69 else:
70 days = 28
71 return days
72
73
74def target_date_from_time_span(cur_date, time_span_hours):
75 cur_year = cur_date.year
76 cur_month = cur_date.month
77 cur_day = cur_date.day
78 cur_hour = cur_date.hour
79 if cur_hour >= time_span_hours:
80 return cur_date.replace(hour=cur_hour-time_span_hours)
81 if cur_day > 1:
82 return cur_date.replace(day=cur_day-1,
83 hour=24+cur_hour-time_span_hours)
84 if cur_month > 1:
85 return cur_date.replace(month=cur_month-1,
86 day=how_many_days_in_month(cur_year, cur_month-1),
87 hour=24+cur_hour-time_span_hours)
88 return cur_date.replace(year=cur_year-1,
89 month=12,
90 day=31,
91 hour=24+cur_hour-time_span_hours)
92
93
94def find_test_duration(created):
95 now = datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc)
96 diff = now - created
97 days, seconds = diff.days, diff.seconds
98 hours = days * 24 + seconds // 3600
99 return hours
100
101
102def testbox_data_processing(oDb):
103 testboxes_dict = {}
104 while True:
105 line = oDb.fetchOne();
106 if line is None:
107 break;
108 testbox_name = line[0]
109 test_result = line[1]
110 test_created = line[2]
111 test_box_os = line[3]
112 test_sched_group = line[4]
113 testboxes_dict = dict_update(testboxes_dict, testbox_name, test_result)
114
115 if "testbox_os" not in testboxes_dict[testbox_name].keys(): ## bird: you don't need to use keys() here.
116 testboxes_dict[testbox_name].update({"testbox_os": test_box_os})
117
118 if "sched_group" not in testboxes_dict[testbox_name].keys():
119 testboxes_dict[testbox_name].update({"sched_group": test_sched_group})
120 elif test_sched_group not in testboxes_dict[testbox_name]["sched_group"]:
121 testboxes_dict[testbox_name]["sched_group"] = testboxes_dict[testbox_name]["sched_group"] + ","+ test_sched_group
122
123 if test_result == "running":
124 testboxes_dict[testbox_name].update({"hours_running": find_test_duration(test_created)})
125
126 #earlier: clean_file_and_insert_data(testboxes_dict, target_file)
127 return testboxes_dict;
128
129
130def os_results_separating(vb_dict, test_name, testbox_os, test_result):
131 if testbox_os == "linux":
132 dict_update(vb_dict, test_name + " / linux", test_result)
133 elif testbox_os == "win":
134 dict_update(vb_dict, test_name + " / windows", test_result)
135 elif testbox_os == "darwin":
136 dict_update(vb_dict, test_name + " / darwin", test_result)
137 elif testbox_os == "solaris":
138 dict_update(vb_dict, test_name + " / solaris", test_result)
139 else:
140 dict_update(vb_dict, test_name + " / other", test_result)
141
142
143def dict_update(target_dict, key_name, test_result):
144 def test_statuses_dict_default():
145 test_statuses_dict = {
146 'running': 0,
147 'success': 0,
148 'skipped': 0,
149 'bad-testbox': 0,
150 'aborted': 0,
151 'failure': 0,
152 'timed-out': 0,
153 'rebooted': 0,
154 }
155 return test_statuses_dict
156 if key_name not in target_dict.keys():
157 target_dict.update({key_name: test_statuses_dict_default().copy()})
158 if test_result in test_statuses_dict_default().keys():
159 target_dict[key_name][test_result] += 1
160 return target_dict
161
162
163def format_data(target_dict):
164 content = ""
165 for key in target_dict:
166 if "hours_running" in target_dict[key].keys():
167 content += "{};{};{} | running: {};{} | success: {} | skipped: {} | ".format(key,
168 target_dict[key]["testbox_os"],
169 target_dict[key]["sched_group"],
170 target_dict[key]["running"],
171 target_dict[key]["hours_running"],
172 target_dict[key]["success"],
173 target_dict[key]["skipped"],
174 )
175 elif "testbox_os" in target_dict[key].keys():
176 content += "{};{};{} | running: {} | success: {} | skipped: {} | ".format(key,
177 target_dict[key]["testbox_os"],
178 target_dict[key]["sched_group"],
179 target_dict[key]["running"],
180 target_dict[key]["success"],
181 target_dict[key]["skipped"],
182 )
183 else:
184 content += "{} | running: {} | success: {} | skipped: {} | ".format(key,
185 target_dict[key]["running"],
186 target_dict[key]["success"],
187 target_dict[key]["skipped"],
188 )
189 content += "bad-testbox: {} | aborted: {} | failure: {} | ".format(
190 target_dict[key]["bad-testbox"],
191 target_dict[key]["aborted"],
192 target_dict[key]["failure"],
193 )
194 content += "timed-out: {} | rebooted: {} | \n".format(
195 target_dict[key]["timed-out"],
196 target_dict[key]["rebooted"],
197 )
198 return content
199
200######
201
202class StatusDispatcherException(TMExceptionBase):
203 """
204 Exception class for TestBoxController.
205 """
206 pass; # pylint: disable=unnecessary-pass
207
208
209class StatusDispatcher(object): # pylint: disable=too-few-public-methods
210 """
211 Status dispatcher class.
212 """
213
214
215 def __init__(self, oSrvGlue):
216 """
217 Won't raise exceptions.
218 """
219 self._oSrvGlue = oSrvGlue;
220 self._sAction = None; # _getStandardParams / dispatchRequest sets this later on.
221 self._dParams = None; # _getStandardParams / dispatchRequest sets this later on.
222 self._asCheckedParams = [];
223 self._dActions = \
224 {
225 'MagicMirrorTestResults': self._actionMagicMirrorTestResults,
226 'MagicMirrorTestBoxes': self._actionMagicMirrorTestBoxes,
227 };
228
229 def _getStringParam(self, sName, asValidValues = None, fStrip = False, sDefValue = None):
230 """
231 Gets a string parameter (stripped).
232
233 Raises exception if not found and no default is provided, or if the
234 value isn't found in asValidValues.
235 """
236 if sName not in self._dParams:
237 if sDefValue is None:
238 raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
239 return sDefValue;
240 sValue = self._dParams[sName];
241 if fStrip:
242 sValue = sValue.strip();
243
244 if sName not in self._asCheckedParams:
245 self._asCheckedParams.append(sName);
246
247 if asValidValues is not None and sValue not in asValidValues:
248 raise StatusDispatcherException('%s parameter %s value "%s" not in %s '
249 % (self._sAction, sName, sValue, asValidValues));
250 return sValue;
251
252 def _getIntParam(self, sName, iMin = None, iMax = None, iDefValue = None):
253 """
254 Gets a string parameter.
255 Raises exception if not found, not a valid integer, or if the value
256 isn't in the range defined by iMin and iMax.
257 """
258 if sName not in self._dParams:
259 if iDefValue is None:
260 raise StatusDispatcherException('%s parameter %s is missing' % (self._sAction, sName));
261 return iDefValue;
262 sValue = self._dParams[sName];
263 try:
264 iValue = int(sValue, 0);
265 except:
266 raise StatusDispatcherException('%s parameter %s value "%s" cannot be convert to an integer'
267 % (self._sAction, sName, sValue));
268 if sName not in self._asCheckedParams:
269 self._asCheckedParams.append(sName);
270
271 if (iMin is not None and iValue < iMin) \
272 or (iMax is not None and iValue > iMax):
273 raise StatusDispatcherException('%s parameter %s value %d is out of range [%s..%s]'
274 % (self._sAction, sName, iValue, iMin, iMax));
275 return iValue;
276
277 def _checkForUnknownParameters(self):
278 """
279 Check if we've handled all parameters, raises exception if anything
280 unknown was found.
281 """
282
283 if len(self._asCheckedParams) != len(self._dParams):
284 sUnknownParams = '';
285 for sKey in self._dParams:
286 if sKey not in self._asCheckedParams:
287 sUnknownParams += ' ' + sKey + '=' + self._dParams[sKey];
288 raise StatusDispatcherException('Unknown parameters: ' + sUnknownParams);
289
290 return True;
291
292 def _connectToDb(self):
293 """
294 Connects to the database.
295
296 Returns (TMDatabaseConnection, (more later perhaps) ) on success.
297 Returns (None, ) on failure after sending the box an appropriate response.
298 May raise exception on DB error.
299 """
300 return (TMDatabaseConnection(self._oSrvGlue.dprint),);
301
302 def _actionMagicMirrorTestBoxes(self):
303 """
304 Produces test result status for the magic mirror dashboard
305 """
306
307 #
308 # Parse arguments and connect to the database.
309 #
310 cHoursBack = self._getIntParam('cHours', 1, 24*14, 12);
311 self._checkForUnknownParameters();
312
313 #
314 # Get the data.
315 # bird: I changed these to join on idGenTestBox and skipping the tsExpire condition.
316 # @todo The query isn't very efficient as postgresql probably will repeat the
317 # subselect in column 5 for each result row.
318 #
319 (oDb,) = self._connectToDb();
320 if oDb is None:
321 return False;
322
323 oDb.execute('''
324SELECT TestBoxesWithStrings.sName,
325 TestSets.enmStatus,
326 TestSets.tsCreated,
327 TestBoxesWithStrings.sOS,
328 ( SELECT STRING_AGG(SchedGroups.sName, ',')
329 FROM SchedGroups
330 INNER JOIN TestBoxesInSchedGroups
331 ON TestBoxesInSchedGroups.idSchedGroup = SchedGroups.idSchedGroup
332 AND TestBoxesInSchedGroups.idTestBox = TestBoxesWithStrings.idTestBox
333 WHERE TestBoxesInSchedGroups.tsExpire = 'infinity'::TIMESTAMP
334 AND SchedGroups.tsExpire = 'infinity'::TIMESTAMP
335 ) AS SchedGroupName
336FROM TestBoxesWithStrings
337LEFT OUTER JOIN TestSets
338 ON TestSets.idGenTestBox = TestBoxesWithStrings.idGenTestBox
339 AND ( TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
340 OR TestSets.tsDone IS NULL)
341''', (cHoursBack,));
342
343 # Process the data
344 dResult = testbox_data_processing(oDb);
345
346 # Format and output it.
347 self._oSrvGlue.setContentType('text/plain');
348 self._oSrvGlue.write(format_data(dResult));
349
350 return True;
351
352 def _actionMagicMirrorTestResults(self):
353 """
354 Produces test result status for the magic mirror dashboard
355 """
356
357 #
358 # Parse arguments and connect to the database.
359 #
360 sBranch = self._getStringParam('sBranch');
361 cHoursBack = self._getIntParam('cHours', 1, 24*14, 6); ## @todo why 6 hours here and 12 for test boxes?
362 self._checkForUnknownParameters();
363
364 #
365 # Get the data.
366 # bird: I changed these to join on idGenTestBox and idGenTestCase instead of
367 # also needing to check the tsExpire columns.
368 #
369 (oDb,) = self._connectToDb();
370 if oDb is None:
371 return False;
372
373 if sBranch == 'all':
374 oDb.execute('''
375SELECT TestSets.enmStatus,
376 TestCases.sName,
377 TestBoxesWithStrings.sOS
378FROM TestSets
379INNER JOIN TestCases
380 ON TestCases.idGenTestCase = TestCases.idGenTestCase
381INNER JOIN TestBoxesWithStrings
382 ON TestBoxesWithStrings.idGenTestBox = TestSets.idGenTestBox
383WHERE TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
384''', (cHoursBack,));
385 else:
386 oDb.execute('''
387SELECT TestSets.enmStatus,
388 TestCases.sName,
389 TestBoxesWithStrings.sOS
390FROM TestSets
391INNER JOIN BuildCategories
392 ON BuildCategories.idBuildCategory = TestSets.idBuildCategory
393 AND BuildCategories.sBuildCategories = '%s'
394INNER JOIN TestCases
395 ON TestCases.idGenTestCase = TestSets.idGenTestCase
396INNER JOIN TestBoxesWithStrings
397 ON TestBoxesWithStrings.idGenTestBox = TestSets.idGenTestBox
398WHERE TestSets.tsCreated > (CURRENT_TIMESTAMP - '%s hours'::interval)
399''', (sBranch, cHoursBack,));
400
401 # Process the data
402 dResult = {};
403 while True:
404 aoRow = oDb.fetchOne();
405 if aoRow is None:
406 break;
407 os_results_separating(dResult, aoRow[1], aoRow[2], aoRow[0]) # save all test results
408
409 # Format and output it.
410 self._oSrvGlue.setContentType('text/plain');
411 self._oSrvGlue.write(format_data(dResult));
412
413 return True;
414
415 def _getStandardParams(self, dParams):
416 """
417 Gets the standard parameters and validates them.
418
419 The parameters are returned as a tuple: sAction, (more later, maybe)
420 Note! the sTextBoxId can be None if it's a SIGNON request.
421
422 Raises StatusDispatcherException on invalid input.
423 """
424 #
425 # Get the action parameter and validate it.
426 #
427 if constants.tbreq.ALL_PARAM_ACTION not in dParams:
428 raise StatusDispatcherException('No "%s" parameter in request (params: %s)'
429 % (constants.tbreq.ALL_PARAM_ACTION, dParams,));
430 sAction = dParams[constants.tbreq.ALL_PARAM_ACTION];
431
432 if sAction not in self._dActions:
433 raise StatusDispatcherException('Unknown action "%s" in request (params: %s; action: %s)'
434 % (sAction, dParams, self._dActions));
435 #
436 # Update the list of checked parameters.
437 #
438 self._asCheckedParams.extend([constants.tbreq.ALL_PARAM_ACTION,]);
439
440 return (sAction,);
441
442 def dispatchRequest(self):
443 """
444 Dispatches the incoming request.
445
446 Will raise StatusDispatcherException on failure.
447 """
448
449 #
450 # Must be a GET request.
451 #
452 try:
453 sMethod = self._oSrvGlue.getMethod();
454 except Exception as oXcpt:
455 raise StatusDispatcherException('Error retriving request method: %s' % (oXcpt,));
456 if sMethod != 'GET':
457 raise StatusDispatcherException('Error expected POST request not "%s"' % (sMethod,));
458
459 #
460 # Get the parameters and checks for duplicates.
461 #
462 try:
463 dParams = self._oSrvGlue.getParameters();
464 except Exception as oXcpt:
465 raise StatusDispatcherException('Error retriving parameters: %s' % (oXcpt,));
466 for sKey in dParams.keys():
467 if len(dParams[sKey]) > 1:
468 raise StatusDispatcherException('Parameter "%s" is given multiple times: %s' % (sKey, dParams[sKey]));
469 dParams[sKey] = dParams[sKey][0];
470 self._dParams = dParams;
471
472 #
473 # Get+validate the standard action parameters and dispatch the request.
474 #
475 (self._sAction, ) = self._getStandardParams(dParams);
476 return self._dActions[self._sAction]();
477
478
479def main():
480 """
481 Main function a la C/C++. Returns exit code.
482 """
483
484 oSrvGlue = WebServerGlueCgi(g_ksValidationKitDir, fHtmlOutput = False);
485 try:
486 oDisp = StatusDispatcher(oSrvGlue);
487 oDisp.dispatchRequest();
488 oSrvGlue.flush();
489 except Exception as oXcpt:
490 return oSrvGlue.errorPage('Internal error: %s' % (str(oXcpt),), sys.exc_info());
491
492 return 0;
493
494if __name__ == '__main__':
495 if config.g_kfProfileAdmin:
496 from testmanager.debug import cgiprofiling;
497 sys.exit(cgiprofiling.profileIt(main));
498 else:
499 sys.exit(main());
500
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette