buildsource.py@ 79092

Last change on this file since 79092 was 79092, checked in by vboxsync, 6 years ago
ValKit,++: Pylint 2.3.1 adjustments.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 21.5 KB

Line
1	# -- coding: utf-8 --
2	# $Id: buildsource.py 79092 2019-06-11 15:26:40Z vboxsync $
3
4	"""
5	Test Manager - Build Sources.
6	"""
7
8	__copyright__ = \
9	"""
10	Copyright (C) 2012-2019 Oracle Corporation
11
12	This file is part of VirtualBox Open Source Edition (OSE), as
13	available from http://www.virtualbox.org. This file is free software;
14	you can redistribute it and/or modify it under the terms of the GNU
15	General Public License (GPL) as published by the Free Software
16	Foundation, in version 2 as it comes in the "COPYING" file of the
17	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
18	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
19
20	The contents of this file may alternatively be used under the terms
21	of the Common Development and Distribution License Version 1.0
22	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
23	VirtualBox OSE distribution, in which case the provisions of the
24	CDDL are applicable instead of those of the GPL.
25
26	You may elect to license modified versions of this file under the
27	terms and conditions of either the GPL or the CDDL or both.
28	"""
29	__version__ = "$Revision: 79092 $"
30
31
32	# Standard python imports.
33	import unittest;
34
35	# Validation Kit imports.
36	from common import utils;
37	from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMRowAlreadyExists, \
38	TMRowInUse, TMInvalidData, TMRowNotFound;
39	from testmanager.core import coreconsts;
40
41
42	class BuildSourceData(ModelDataBase):
43	"""
44	A build source.
45	"""
46
47	ksIdAttr = 'idBuildSrc';
48
49	ksParam_idBuildSrc = 'BuildSource_idBuildSrc';
50	ksParam_tsEffective = 'BuildSource_tsEffective';
51	ksParam_tsExpire = 'BuildSource_tsExpire';
52	ksParam_uidAuthor = 'BuildSource_uidAuthor';
53	ksParam_sName = 'BuildSource_sName';
54	ksParam_sDescription = 'BuildSource_sDescription';
55	ksParam_sProduct = 'BuildSource_sProduct';
56	ksParam_sBranch = 'BuildSource_sBranch';
57	ksParam_asTypes = 'BuildSource_asTypes';
58	ksParam_asOsArches = 'BuildSource_asOsArches';
59	ksParam_iFirstRevision = 'BuildSource_iFirstRevision';
60	ksParam_iLastRevision = 'BuildSource_iLastRevision';
61	ksParam_cSecMaxAge = 'BuildSource_cSecMaxAge';
62
63	kasAllowNullAttributes = [ 'idBuildSrc', 'tsEffective', 'tsExpire', 'uidAuthor', 'sDescription', 'asTypes',
64	'asOsArches', 'iFirstRevision', 'iLastRevision', 'cSecMaxAge' ];
65
66	def __init__(self):
67	ModelDataBase.__init__(self);
68
69	#
70	# Initialize with defaults.
71	# See the database for explanations of each of these fields.
72	#
73	self.idBuildSrc = None;
74	self.tsEffective = None;
75	self.tsExpire = None;
76	self.uidAuthor = None;
77	self.sName = None;
78	self.sDescription = None;
79	self.sProduct = None;
80	self.sBranch = None;
81	self.asTypes = None;
82	self.asOsArches = None;
83	self.iFirstRevision = None;
84	self.iLastRevision = None;
85	self.cSecMaxAge = None;
86
87	def initFromDbRow(self, aoRow):
88	"""
89	Re-initializes the object from a SELECT * FROM BuildSources row.
90	Returns self. Raises exception if aoRow is None.
91	"""
92	if aoRow is None:
93	raise TMRowNotFound('Build source not found.');
94
95	self.idBuildSrc = aoRow[0];
96	self.tsEffective = aoRow[1];
97	self.tsExpire = aoRow[2];
98	self.uidAuthor = aoRow[3];
99	self.sName = aoRow[4];
100	self.sDescription = aoRow[5];
101	self.sProduct = aoRow[6];
102	self.sBranch = aoRow[7];
103	self.asTypes = aoRow[8];
104	self.asOsArches = aoRow[9];
105	self.iFirstRevision = aoRow[10];
106	self.iLastRevision = aoRow[11];
107	self.cSecMaxAge = aoRow[12];
108	return self;
109
110	def initFromDbWithId(self, oDb, idBuildSrc, tsNow = None, sPeriodBack = None):
111	"""
112	Initialize from the database, given the ID of a row.
113	"""
114	oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb,
115	'SELECT *\n'
116	'FROM BuildSources\n'
117	'WHERE idBuildSrc = %s\n'
118	, ( idBuildSrc,), tsNow, sPeriodBack));
119	aoRow = oDb.fetchOne()
120	if aoRow is None:
121	raise TMRowNotFound('idBuildSrc=%s not found (tsNow=%s sPeriodBack=%s)' % (idBuildSrc, tsNow, sPeriodBack,));
122	return self.initFromDbRow(aoRow);
123
124	def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb):
125	# Handle asType and asOsArches specially.
126	if sAttr == 'sType':
127	(oNewValue, sError) = ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue,
128	aoNilValues, fAllowNull, oDb);
129	if sError is None:
130	if not self.asTypes:
131	oNewValue = None;
132	else:
133	for sType in oNewValue:
134	if len(sType) < 2 or sType.lower() != sType:
135	if sError is None: sError = '';
136	else: sError += ', ';
137	sError += 'invalid value "%s"' % (sType,);
138
139	elif sAttr == 'asOsArches':
140	(oNewValue, sError) = self.validateListOfStr(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull,
141	asValidValues = coreconsts.g_kasOsDotCpusAll);
142	if sError is not None and oNewValue is not None:
143	oNewValue = sorted(oNewValue); # Must be sorted!
144
145	elif sAttr == 'cSecMaxAge' and oValue not in aoNilValues: # Allow human readable interval formats.
146	(oNewValue, sError) = utils.parseIntervalSeconds(oValue);
147	else:
148	return ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb);
149
150	return (oNewValue, sError);
151
152	class BuildSourceLogic(ModelLogicBase): # pylint: disable=too-few-public-methods
153	"""
154	Build source database logic.
155	"""
156
157	def __init__(self, oDb):
158	ModelLogicBase.__init__(self, oDb)
159	self.dCache = None;
160
161	#
162	# Standard methods.
163	#
164
165	def fetchForListing(self, iStart, cMaxRows, tsNow, aiSortColumns = None):
166	"""
167	Fetches build sources.
168
169	Returns an array (list) of BuildSourceData items, empty list if none.
170	Raises exception on error.
171	"""
172	_ = aiSortColumns;
173
174	if tsNow is None:
175	self._oDb.execute('SELECT *\n'
176	'FROM BuildSources\n'
177	'WHERE tsExpire = \'infinity\'::TIMESTAMP\n'
178	'ORDER BY idBuildSrc DESC\n'
179	'LIMIT %s OFFSET %s\n'
180	, (cMaxRows, iStart,));
181	else:
182	self._oDb.execute('SELECT *\n'
183	'FROM BuildSources\n'
184	'WHERE tsExpire > %s\n'
185	' AND tsEffective <= %s\n'
186	'ORDER BY idBuildSrc DESC\n'
187	'LIMIT %s OFFSET %s\n'
188	, (tsNow, tsNow, cMaxRows, iStart,));
189
190	aoRows = []
191	for aoRow in self._oDb.fetchAll():
192	aoRows.append(BuildSourceData().initFromDbRow(aoRow))
193	return aoRows
194
195	def fetchForCombo(self):
196	"""Fetch data which is aimed to be passed to HTML form"""
197	self._oDb.execute('SELECT idBuildSrc, sName, sProduct\n'
198	'FROM BuildSources\n'
199	'WHERE tsExpire = \'infinity\'::TIMESTAMP\n'
200	'ORDER BY idBuildSrc DESC\n')
201	asRet = self._oDb.fetchAll();
202	asRet.insert(0, (-1, 'None', 'None'));
203	return asRet;
204
205
206	def addEntry(self, oData, uidAuthor, fCommit = False):
207	"""
208	Add a new build source to the database.
209	"""
210
211	#
212	# Validate the input.
213	#
214	dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add);
215	if dErrors:
216	raise TMInvalidData('addEntry invalid input: %s' % (dErrors,));
217	self._assertUnique(oData, None);
218
219	#
220	# Add it.
221	#
222	self._oDb.execute('INSERT INTO BuildSources (\n'
223	' uidAuthor,\n'
224	' sName,\n'
225	' sDescription,\n'
226	' sProduct,\n'
227	' sBranch,\n'
228	' asTypes,\n'
229	' asOsArches,\n'
230	' iFirstRevision,\n'
231	' iLastRevision,\n'
232	' cSecMaxAge)\n'
233	'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n'
234	, ( uidAuthor,
235	oData.sName,
236	oData.sDescription,
237	oData.sProduct,
238	oData.sBranch,
239	oData.asTypes,
240	oData.asOsArches,
241	oData.iFirstRevision,
242	oData.iLastRevision,
243	oData.cSecMaxAge, ));
244
245	self._oDb.maybeCommit(fCommit);
246	return True;
247
248	def editEntry(self, oData, uidAuthor, fCommit = False):
249	"""
250	Modifies a build source.
251	"""
252
253	#
254	# Validate the input and read the old entry.
255	#
256	dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit);
257	if dErrors:
258	raise TMInvalidData('addEntry invalid input: %s' % (dErrors,));
259	self._assertUnique(oData, oData.idBuildSrc);
260	oOldData = BuildSourceData().initFromDbWithId(self._oDb, oData.idBuildSrc);
261
262	#
263	# Make the changes (if something actually changed).
264	#
265	if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]):
266	self._historizeBuildSource(oData.idBuildSrc);
267	self._oDb.execute('INSERT INTO BuildSources (\n'
268	' uidAuthor,\n'
269	' idBuildSrc,\n'
270	' sName,\n'
271	' sDescription,\n'
272	' sProduct,\n'
273	' sBranch,\n'
274	' asTypes,\n'
275	' asOsArches,\n'
276	' iFirstRevision,\n'
277	' iLastRevision,\n'
278	' cSecMaxAge)\n'
279	'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n'
280	, ( uidAuthor,
281	oData.idBuildSrc,
282	oData.sName,
283	oData.sDescription,
284	oData.sProduct,
285	oData.sBranch,
286	oData.asTypes,
287	oData.asOsArches,
288	oData.iFirstRevision,
289	oData.iLastRevision,
290	oData.cSecMaxAge, ));
291	self._oDb.maybeCommit(fCommit);
292	return True;
293
294	def removeEntry(self, uidAuthor, idBuildSrc, fCascade = False, fCommit = False):
295	"""
296	Deletes a build sources.
297	"""
298
299	#
300	# Check cascading.
301	#
302	if fCascade is not True:
303	self._oDb.execute('SELECT idSchedGroup, sName\n'
304	'FROM SchedGroups\n'
305	'WHERE idBuildSrc = %s\n'
306	' OR idBuildSrcTestSuite = %s\n'
307	, (idBuildSrc, idBuildSrc,));
308	if self._oDb.getRowCount() > 0:
309	asGroups = [];
310	for aoRow in self._oDb.fetchAll():
311	asGroups.append('%s (#%d)' % (aoRow[1], aoRow[0]));
312	raise TMRowInUse('Build source #%d is used by one or more scheduling groups: %s'
313	% (idBuildSrc, ', '.join(asGroups),));
314	else:
315	self._oDb.execute('UPDATE SchedGroups\n'
316	'SET idBuildSrc = NULL\n'
317	'WHERE idBuildSrc = %s'
318	, ( idBuildSrc,));
319	self._oDb.execute('UPDATE SchedGroups\n'
320	'SET idBuildSrcTestSuite = NULL\n'
321	'WHERE idBuildSrcTestSuite = %s'
322	, ( idBuildSrc,));
323
324	#
325	# Do the job.
326	#
327	self._historizeBuildSource(idBuildSrc, None);
328	_ = uidAuthor; ## @todo record deleter.
329
330	self._oDb.maybeCommit(fCommit);
331	return True;
332
333	def cachedLookup(self, idBuildSrc):
334	"""
335	Looks up the most recent BuildSourceData object for idBuildSrc
336	via an object cache.
337
338	Returns a shared BuildSourceData object. None if not found.
339	Raises exception on DB error.
340	"""
341	if self.dCache is None:
342	self.dCache = self._oDb.getCache('BuildSourceData');
343	oEntry = self.dCache.get(idBuildSrc, None);
344	if oEntry is None:
345	self._oDb.execute('SELECT *\n'
346	'FROM BuildSources\n'
347	'WHERE idBuildSrc = %s\n'
348	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
349	, (idBuildSrc, ));
350	if self._oDb.getRowCount() == 0:
351	# Maybe it was deleted, try get the last entry.
352	self._oDb.execute('SELECT *\n'
353	'FROM BuildSources\n'
354	'WHERE idBuildSrc = %s\n'
355	'ORDER BY tsExpire DESC\n'
356	'LIMIT 1\n'
357	, (idBuildSrc, ));
358	elif self._oDb.getRowCount() > 1:
359	raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idBuildSrc));
360
361	if self._oDb.getRowCount() == 1:
362	aaoRow = self._oDb.fetchOne();
363	oEntry = BuildSourceData();
364	oEntry.initFromDbRow(aaoRow);
365	self.dCache[idBuildSrc] = oEntry;
366	return oEntry;
367
368	#
369	# Other methods.
370	#
371
372	def openBuildCursor(self, oBuildSource, sOs, sCpuArch, tsNow):
373	"""
374	Opens a cursor (SELECT) using the criteria found in the build source
375	and the given OS.CPUARCH.
376
377	Returns database cursor. May raise exception on bad input or logic error.
378
379	Used by SchedulerBase.
380	"""
381
382	oCursor = self._oDb.openCursor();
383
384	#
385	# Construct the extra conditionals.
386	#
387	sExtraConditions = '';
388
389	# Types
390	if oBuildSource.asTypes is not None and oBuildSource.asTypes:
391	if len(oBuildSource.asTypes) == 1:
392	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.sType = %s', (oBuildSource.asTypes[0],));
393	else:
394	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.sType IN (%s', (oBuildSource.asTypes[0],))
395	for i in range(1, len(oBuildSource.asTypes) - 1):
396	sExtraConditions += oCursor.formatBindArgs(', %s', (oBuildSource.asTypes[i],));
397	sExtraConditions += oCursor.formatBindArgs(', %s)\n', (oBuildSource.asTypes[-1],));
398
399	# BuildSource OSes.ARCHes. (Paranoia: use a dictionary to avoid duplicate values.)
400	if oBuildSource.asOsArches is not None and oBuildSource.asOsArches:
401	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.asOsArches && %s', (oBuildSource.asOsArches,));
402
403	# TestBox OSes.ARCHes. (Paranoia: use a dictionary to avoid duplicate values.)
404	dOsDotArches = {};
405	dOsDotArches[sOs + '.' + sCpuArch] = 1;
406	dOsDotArches[sOs + '.' + coreconsts.g_ksCpuArchAgnostic] = 1;
407	dOsDotArches[coreconsts.g_ksOsAgnostic + '.' + sCpuArch] = 1;
408	dOsDotArches[coreconsts.g_ksOsDotArchAgnostic] = 1;
409	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.asOsArches && %s', (list(dOsDotArches.keys()),));
410
411	# Revision range.
412	if oBuildSource.iFirstRevision is not None:
413	sExtraConditions += oCursor.formatBindArgs(' AND Builds.iRevision >= %s\n', (oBuildSource.iFirstRevision,));
414	if oBuildSource.iLastRevision is not None:
415	sExtraConditions += oCursor.formatBindArgs(' AND Builds.iRevision <= %s\n', (oBuildSource.iLastRevision,));
416
417	# Max age.
418	if oBuildSource.cSecMaxAge is not None:
419	sExtraConditions += oCursor.formatBindArgs(' AND Builds.tsCreated >= (%s - \'%s seconds\'::INTERVAL)\n',
420	(tsNow, oBuildSource.cSecMaxAge,));
421
422	#
423	# Execute the query.
424	#
425	oCursor.execute('SELECT Builds., BuildCategories.,\n'
426	' EXISTS( SELECT tsExpire\n'
427	' FROM BuildBlacklist\n'
428	' WHERE BuildBlacklist.tsExpire = \'infinity\'::TIMESTAMP\n'
429	' AND BuildBlacklist.sProduct = %s\n'
430	' AND BuildBlacklist.sBranch = %s\n'
431	' AND BuildBlacklist.iFirstRevision <= Builds.iRevision\n'
432	' AND BuildBlacklist.iLastRevision >= Builds.iRevision ) AS fMaybeBlacklisted\n'
433	'FROM Builds, BuildCategories\n'
434	'WHERE Builds.idBuildCategory = BuildCategories.idBuildCategory\n'
435	' AND Builds.tsExpire = \'infinity\'::TIMESTAMP\n'
436	' AND Builds.tsEffective <= %s\n'
437	' AND Builds.fBinariesDeleted is FALSE\n'
438	' AND BuildCategories.sProduct = %s\n'
439	' AND BuildCategories.sBranch = %s\n'
440	+ sExtraConditions +
441	'ORDER BY Builds.idBuild DESC\n'
442	'LIMIT 256\n'
443	, ( oBuildSource.sProduct, oBuildSource.sBranch,
444	tsNow, oBuildSource.sProduct, oBuildSource.sBranch,));
445
446	return oCursor;
447
448
449	def getById(self, idBuildSrc):
450	"""Get Build Source data by idBuildSrc"""
451
452	self._oDb.execute('SELECT *\n'
453	'FROM BuildSources\n'
454	'WHERE tsExpire = \'infinity\'::timestamp\n'
455	' AND idBuildSrc = %s;', (idBuildSrc,))
456	aRows = self._oDb.fetchAll()
457	if len(aRows) not in (0, 1):
458	raise self._oDb.integrityException(
459	'Found more than one build sources with the same credentials. Database structure is corrupted.')
460	try:
461	return BuildSourceData().initFromDbRow(aRows[0])
462	except IndexError:
463	return None
464
465	#
466	# Internal helpers.
467	#
468
469	def _assertUnique(self, oData, idBuildSrcIgnore):
470	""" Checks that the build source name is unique, raises exception if it isn't. """
471	self._oDb.execute('SELECT idBuildSrc\n'
472	'FROM BuildSources\n'
473	'WHERE sName = %s\n'
474	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
475	+ ('' if idBuildSrcIgnore is None else ' AND idBuildSrc <> %d\n' % (idBuildSrcIgnore,))
476	, ( oData.sName, ))
477	if self._oDb.getRowCount() > 0:
478	raise TMRowAlreadyExists('A build source with name "%s" already exist.' % (oData.sName,));
479	return True;
480
481
482	def _historizeBuildSource(self, idBuildSrc, tsExpire = None):
483	""" Historizes the current build source entry. """
484	if tsExpire is None:
485	self._oDb.execute('UPDATE BuildSources\n'
486	'SET tsExpire = CURRENT_TIMESTAMP\n'
487	'WHERE idBuildSrc = %s\n'
488	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
489	, ( idBuildSrc, ));
490	else:
491	self._oDb.execute('UPDATE BuildSources\n'
492	'SET tsExpire = %s\n'
493	'WHERE idBuildSrc = %s\n'
494	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
495	, ( tsExpire, idBuildSrc, ));
496	return True;
497
498
499
500
501
502	#
503	# Unit testing.
504	#
505
506	# pylint: disable=missing-docstring
507	class BuildSourceDataTestCase(ModelDataBaseTestCase):
508	def setUp(self):
509	self.aoSamples = [BuildSourceData(),];
510
511	if __name__ == '__main__':
512	unittest.main();
513	# not reached.
514

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/core/buildsource.py@ 79092

Download in other formats: