buildsource.py@ 65040

Last change on this file since 65040 was 65040, checked in by vboxsync, 8 years ago
testmanager: More details in the system wide changelog.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 21.4 KB

Line
1	# -- coding: utf-8 --
2	# $Id: buildsource.py 65040 2016-12-31 02:29:50Z vboxsync $
3
4	"""
5	Test Manager - Build Sources.
6	"""
7
8	__copyright__ = \
9	"""
10	Copyright (C) 2012-2016 Oracle Corporation
11
12	This file is part of VirtualBox Open Source Edition (OSE), as
13	available from http://www.virtualbox.org. This file is free software;
14	you can redistribute it and/or modify it under the terms of the GNU
15	General Public License (GPL) as published by the Free Software
16	Foundation, in version 2 as it comes in the "COPYING" file of the
17	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
18	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
19
20	The contents of this file may alternatively be used under the terms
21	of the Common Development and Distribution License Version 1.0
22	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
23	VirtualBox OSE distribution, in which case the provisions of the
24	CDDL are applicable instead of those of the GPL.
25
26	You may elect to license modified versions of this file under the
27	terms and conditions of either the GPL or the CDDL or both.
28	"""
29	__version__ = "$Revision: 65040 $"
30
31
32	# Standard python imports.
33	import unittest;
34
35	# Validation Kit imports.
36	from common import utils;
37	from testmanager.core.base import ModelDataBase, ModelDataBaseTestCase, ModelLogicBase, TMRowAlreadyExists, \
38	TMRowInUse, TMInvalidData, TMRowNotFound;
39	from testmanager.core import coreconsts;
40
41
42	class BuildSourceData(ModelDataBase):
43	"""
44	A build source.
45	"""
46
47	ksIdAttr = 'idBuildSrc';
48
49	ksParam_idBuildSrc = 'BuildSource_idBuildSrc';
50	ksParam_tsEffective = 'BuildSource_tsEffective';
51	ksParam_tsExpire = 'BuildSource_tsExpire';
52	ksParam_uidAuthor = 'BuildSource_uidAuthor';
53	ksParam_sName = 'BuildSource_sName';
54	ksParam_sDescription = 'BuildSource_sDescription';
55	ksParam_sProduct = 'BuildSource_sProduct';
56	ksParam_sBranch = 'BuildSource_sBranch';
57	ksParam_asTypes = 'BuildSource_asTypes';
58	ksParam_asOsArches = 'BuildSource_asOsArches';
59	ksParam_iFirstRevision = 'BuildSource_iFirstRevision';
60	ksParam_iLastRevision = 'BuildSource_iLastRevision';
61	ksParam_cSecMaxAge = 'BuildSource_cSecMaxAge';
62
63	kasAllowNullAttributes = [ 'idBuildSrc', 'tsEffective', 'tsExpire', 'uidAuthor', 'sDescription', 'asTypes',
64	'asOsArches', 'iFirstRevision', 'iLastRevision', 'cSecMaxAge' ];
65
66	def __init__(self):
67	ModelDataBase.__init__(self);
68
69	#
70	# Initialize with defaults.
71	# See the database for explanations of each of these fields.
72	#
73	self.idBuildSrc = None;
74	self.tsEffective = None;
75	self.tsExpire = None;
76	self.uidAuthor = None;
77	self.sName = None;
78	self.sDescription = None;
79	self.sProduct = None;
80	self.sBranch = None;
81	self.asTypes = None;
82	self.asOsArches = None;
83	self.iFirstRevision = None;
84	self.iLastRevision = None;
85	self.cSecMaxAge = None;
86
87	def initFromDbRow(self, aoRow):
88	"""
89	Re-initializes the object from a SELECT * FROM BuildSources row.
90	Returns self. Raises exception if aoRow is None.
91	"""
92	if aoRow is None:
93	raise TMRowNotFound('Build source not found.');
94
95	self.idBuildSrc = aoRow[0];
96	self.tsEffective = aoRow[1];
97	self.tsExpire = aoRow[2];
98	self.uidAuthor = aoRow[3];
99	self.sName = aoRow[4];
100	self.sDescription = aoRow[5];
101	self.sProduct = aoRow[6];
102	self.sBranch = aoRow[7];
103	self.asTypes = aoRow[8];
104	self.asOsArches = aoRow[9];
105	self.iFirstRevision = aoRow[10];
106	self.iLastRevision = aoRow[11];
107	self.cSecMaxAge = aoRow[12];
108	return self;
109
110	def initFromDbWithId(self, oDb, idBuildSrc, tsNow = None, sPeriodBack = None):
111	"""
112	Initialize from the database, given the ID of a row.
113	"""
114	oDb.execute(self.formatSimpleNowAndPeriodQuery(oDb,
115	'SELECT *\n'
116	'FROM BuildSources\n'
117	'WHERE idBuildSrc = %s\n'
118	, ( idBuildSrc,), tsNow, sPeriodBack));
119	aoRow = oDb.fetchOne()
120	if aoRow is None:
121	raise TMRowNotFound('idBuildSrc=%s not found (tsNow=%s sPeriodBack=%s)' % (idBuildSrc, tsNow, sPeriodBack,));
122	return self.initFromDbRow(aoRow);
123
124	def _validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb):
125	# Handle asType and asOsArches specially.
126	if sAttr == 'sType':
127	(oNewValue, sError) = ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue,
128	aoNilValues, fAllowNull, oDb);
129	if sError is None:
130	if len(self.asTypes) <= 0:
131	oNewValue = None;
132	else:
133	for sType in oNewValue:
134	if len(sType) < 2 or sType.lower() != sType:
135	if sError is None: sError = '';
136	else: sError += ', ';
137	sError += 'invalid value "%s"' % (sType,);
138
139	elif sAttr == 'asOsArches':
140	(oNewValue, sError) = self.validateListOfStr(oValue, aoNilValues = aoNilValues, fAllowNull = fAllowNull,
141	asValidValues = coreconsts.g_kasOsDotCpusAll);
142	if sError is not None and oNewValue is not None:
143	oNewValue = sorted(oNewValue); # Must be sorted!
144
145	elif sAttr == 'cSecMaxAge' and oValue not in aoNilValues: # Allow human readable interval formats.
146	(oNewValue, sError) = utils.parseIntervalSeconds(oValue);
147	else:
148	return ModelDataBase._validateAndConvertAttribute(self, sAttr, sParam, oValue, aoNilValues, fAllowNull, oDb);
149
150	return (oNewValue, sError);
151
152	class BuildSourceLogic(ModelLogicBase): # pylint: disable=R0903
153	"""
154	Build source database logic.
155	"""
156
157	def __init__(self, oDb):
158	ModelLogicBase.__init__(self, oDb)
159	self.dCache = None;
160
161	#
162	# Standard methods.
163	#
164
165	def fetchForListing(self, iStart, cMaxRows, tsNow):
166	"""
167	Fetches build sources.
168
169	Returns an array (list) of BuildSourceData items, empty list if none.
170	Raises exception on error.
171	"""
172	if tsNow is None:
173	self._oDb.execute('SELECT *\n'
174	'FROM BuildSources\n'
175	'WHERE tsExpire = \'infinity\'::TIMESTAMP\n'
176	'ORDER BY idBuildSrc DESC\n'
177	'LIMIT %s OFFSET %s\n'
178	, (cMaxRows, iStart,));
179	else:
180	self._oDb.execute('SELECT *\n'
181	'FROM BuildSources\n'
182	'WHERE tsExpire > %s\n'
183	' AND tsEffective <= %s\n'
184	'ORDER BY idBuildSrc DESC\n'
185	'LIMIT %s OFFSET %s\n'
186	, (tsNow, tsNow, cMaxRows, iStart,));
187
188	aoRows = []
189	for aoRow in self._oDb.fetchAll():
190	aoRows.append(BuildSourceData().initFromDbRow(aoRow))
191	return aoRows
192
193	def fetchForCombo(self):
194	"""Fetch data which is aimed to be passed to HTML form"""
195	self._oDb.execute('SELECT idBuildSrc, sName, sProduct\n'
196	'FROM BuildSources\n'
197	'WHERE tsExpire = \'infinity\'::TIMESTAMP\n'
198	'ORDER BY idBuildSrc DESC\n')
199	asRet = self._oDb.fetchAll();
200	asRet.insert(0, (-1, 'None', 'None'));
201	return asRet;
202
203
204	def addEntry(self, oData, uidAuthor, fCommit = False):
205	"""
206	Add a new build source to the database.
207	"""
208
209	#
210	# Validate the input.
211	#
212	dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Add);
213	if len(dErrors) > 0:
214	raise TMInvalidData('addEntry invalid input: %s' % (dErrors,));
215	self._assertUnique(oData, None);
216
217	#
218	# Add it.
219	#
220	self._oDb.execute('INSERT INTO BuildSources (\n'
221	' uidAuthor,\n'
222	' sName,\n'
223	' sDescription,\n'
224	' sProduct,\n'
225	' sBranch,\n'
226	' asTypes,\n'
227	' asOsArches,\n'
228	' iFirstRevision,\n'
229	' iLastRevision,\n'
230	' cSecMaxAge)\n'
231	'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n'
232	, ( uidAuthor,
233	oData.sName,
234	oData.sDescription,
235	oData.sProduct,
236	oData.sBranch,
237	oData.asTypes,
238	oData.asOsArches,
239	oData.iFirstRevision,
240	oData.iLastRevision,
241	oData.cSecMaxAge, ));
242
243	self._oDb.maybeCommit(fCommit);
244	return True;
245
246	def editEntry(self, oData, uidAuthor, fCommit = False):
247	"""
248	Modifies a build source.
249	"""
250
251	#
252	# Validate the input and read the old entry.
253	#
254	dErrors = oData.validateAndConvert(self._oDb, oData.ksValidateFor_Edit);
255	if len(dErrors) > 0:
256	raise TMInvalidData('addEntry invalid input: %s' % (dErrors,));
257	self._assertUnique(oData, oData.idBuildSrc);
258	oOldData = BuildSourceData().initFromDbWithId(self._oDb, oData.idBuildSrc);
259
260	#
261	# Make the changes (if something actually changed).
262	#
263	if not oData.isEqualEx(oOldData, [ 'tsEffective', 'tsExpire', 'uidAuthor', ]):
264	self._historizeBuildSource(oData.idBuildSrc);
265	self._oDb.execute('INSERT INTO BuildSources (\n'
266	' uidAuthor,\n'
267	' idBuildSrc,\n'
268	' sName,\n'
269	' sDescription,\n'
270	' sProduct,\n'
271	' sBranch,\n'
272	' asTypes,\n'
273	' asOsArches,\n'
274	' iFirstRevision,\n'
275	' iLastRevision,\n'
276	' cSecMaxAge)\n'
277	'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)\n'
278	, ( uidAuthor,
279	oData.idBuildSrc,
280	oData.sName,
281	oData.sDescription,
282	oData.sProduct,
283	oData.sBranch,
284	oData.asTypes,
285	oData.asOsArches,
286	oData.iFirstRevision,
287	oData.iLastRevision,
288	oData.cSecMaxAge, ));
289	self._oDb.maybeCommit(fCommit);
290	return True;
291
292	def removeEntry(self, uidAuthor, idBuildSrc, fCascade = False, fCommit = False):
293	"""
294	Deletes a build sources.
295	"""
296
297	#
298	# Check cascading.
299	#
300	if fCascade is not True:
301	self._oDb.execute('SELECT idSchedGroup, sName\n'
302	'FROM SchedGroups\n'
303	'WHERE idBuildSrc = %s\n'
304	' OR idBuildSrcTestSuite = %s\n'
305	, (idBuildSrc, idBuildSrc,));
306	if self._oDb.getRowCount() > 0:
307	asGroups = [];
308	for aoRow in self._oDb.fetchAll():
309	asGroups.append('%s (#%d)' % (aoRow[1], aoRow[0]));
310	raise TMRowInUse('Build source #%d is used by one or more scheduling groups: %s'
311	% (idBuildSrc, ', '.join(asGroups),));
312	else:
313	self._oDb.execute('UPDATE SchedGroups\n'
314	'SET idBuildSrc = NULL\n'
315	'WHERE idBuildSrc = %s'
316	, ( idBuildSrc,));
317	self._oDb.execute('UPDATE SchedGroups\n'
318	'SET idBuildSrcTestSuite = NULL\n'
319	'WHERE idBuildSrcTestSuite = %s'
320	, ( idBuildSrc,));
321
322	#
323	# Do the job.
324	#
325	self._historizeBuildSource(idBuildSrc, None);
326	_ = uidAuthor; ## @todo record deleter.
327
328	self._oDb.maybeCommit(fCommit);
329	return True;
330
331	def cachedLookup(self, idBuildSrc):
332	"""
333	Looks up the most recent BuildSourceData object for idBuildSrc
334	via an object cache.
335
336	Returns a shared BuildSourceData object. None if not found.
337	Raises exception on DB error.
338	"""
339	if self.dCache is None:
340	self.dCache = self._oDb.getCache('BuildSourceData');
341	oEntry = self.dCache.get(idBuildSrc, None);
342	if oEntry is None:
343	self._oDb.execute('SELECT *\n'
344	'FROM BuildSources\n'
345	'WHERE idBuildSrc = %s\n'
346	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
347	, (idBuildSrc, ));
348	if self._oDb.getRowCount() == 0:
349	# Maybe it was deleted, try get the last entry.
350	self._oDb.execute('SELECT *\n'
351	'FROM BuildSources\n'
352	'WHERE idBuildSrc = %s\n'
353	'ORDER BY tsExpire DESC\n'
354	'LIMIT 1\n'
355	, (idBuildSrc, ));
356	elif self._oDb.getRowCount() > 1:
357	raise self._oDb.integrityException('%s infinity rows for %s' % (self._oDb.getRowCount(), idBuildSrc));
358
359	if self._oDb.getRowCount() == 1:
360	aaoRow = self._oDb.fetchOne();
361	oEntry = BuildSourceData();
362	oEntry.initFromDbRow(aaoRow);
363	self.dCache[idBuildSrc] = oEntry;
364	return oEntry;
365
366	#
367	# Other methods.
368	#
369
370	def openBuildCursor(self, oBuildSource, sOs, sCpuArch, tsNow):
371	"""
372	Opens a cursor (SELECT) using the criteria found in the build source
373	and the given OS.CPUARCH.
374
375	Returns database cursor. May raise exception on bad input or logic error.
376
377	Used by SchedulerBase.
378	"""
379
380	oCursor = self._oDb.openCursor();
381
382	#
383	# Construct the extra conditionals.
384	#
385	sExtraConditions = '';
386
387	# Types
388	if oBuildSource.asTypes is not None and len(oBuildSource.asTypes) > 0:
389	if len(oBuildSource.asTypes) == 1:
390	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.sType = %s', (oBuildSource.asTypes[0],));
391	else:
392	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.sType IN (%s', (oBuildSource.asTypes[0],))
393	for i in range(1, len(oBuildSource.asTypes) - 1):
394	sExtraConditions += oCursor.formatBindArgs(', %s', (oBuildSource.asTypes[i],));
395	sExtraConditions += oCursor.formatBindArgs(', %s)\n', (oBuildSource.asTypes[-1],));
396
397	# BuildSource OSes.ARCHes. (Paranoia: use a dictionary to avoid duplicate values.)
398	if oBuildSource.asOsArches is not None and len(oBuildSource.asOsArches) > 0:
399	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.asOsArches && %s', (oBuildSource.asOsArches,));
400
401	# TestBox OSes.ARCHes. (Paranoia: use a dictionary to avoid duplicate values.)
402	dOsDotArches = {};
403	dOsDotArches[sOs + '.' + sCpuArch] = 1;
404	dOsDotArches[sOs + '.' + coreconsts.g_ksCpuArchAgnostic] = 1;
405	dOsDotArches[coreconsts.g_ksOsAgnostic + '.' + sCpuArch] = 1;
406	dOsDotArches[coreconsts.g_ksOsDotArchAgnostic] = 1;
407	sExtraConditions += oCursor.formatBindArgs(' AND BuildCategories.asOsArches && %s', (list(dOsDotArches.keys()),));
408
409	# Revision range.
410	if oBuildSource.iFirstRevision is not None:
411	sExtraConditions += oCursor.formatBindArgs(' AND Builds.iRevision >= %s\n', (oBuildSource.iFirstRevision,));
412	if oBuildSource.iLastRevision is not None:
413	sExtraConditions += oCursor.formatBindArgs(' AND Builds.iRevision <= %s\n', (oBuildSource.iLastRevision,));
414
415	# Max age.
416	if oBuildSource.cSecMaxAge is not None:
417	sExtraConditions += oCursor.formatBindArgs(' AND Builds.tsCreated >= (%s - \'%s seconds\'::INTERVAL)\n',
418	(tsNow, oBuildSource.cSecMaxAge,));
419
420	#
421	# Execute the query.
422	#
423	oCursor.execute('SELECT Builds., BuildCategories.,\n'
424	' EXISTS( SELECT tsExpire\n'
425	' FROM BuildBlacklist\n'
426	' WHERE BuildBlacklist.tsExpire = \'infinity\'::TIMESTAMP\n'
427	' AND BuildBlacklist.sProduct = %s\n'
428	' AND BuildBlacklist.sBranch = %s\n'
429	' AND BuildBlacklist.iFirstRevision <= Builds.iRevision\n'
430	' AND BuildBlacklist.iLastRevision >= Builds.iRevision ) AS fMaybeBlacklisted\n'
431	'FROM Builds, BuildCategories\n'
432	'WHERE Builds.idBuildCategory = BuildCategories.idBuildCategory\n'
433	' AND Builds.tsExpire = \'infinity\'::TIMESTAMP\n'
434	' AND Builds.tsEffective <= %s\n'
435	' AND Builds.fBinariesDeleted is FALSE\n'
436	' AND BuildCategories.sProduct = %s\n'
437	' AND BuildCategories.sBranch = %s\n'
438	+ sExtraConditions +
439	'ORDER BY Builds.idBuild DESC\n'
440	'LIMIT 256\n'
441	, ( oBuildSource.sProduct, oBuildSource.sBranch,
442	tsNow, oBuildSource.sProduct, oBuildSource.sBranch,));
443
444	return oCursor;
445
446
447	def getById(self, idBuildSrc):
448	"""Get Build Source data by idBuildSrc"""
449
450	self._oDb.execute('SELECT *\n'
451	'FROM BuildSources\n'
452	'WHERE tsExpire = \'infinity\'::timestamp\n'
453	' AND idBuildSrc = %s;', (idBuildSrc,))
454	aRows = self._oDb.fetchAll()
455	if len(aRows) not in (0, 1):
456	raise self._oDb.integrityException(
457	'Found more than one build sources with the same credentials. Database structure is corrupted.')
458	try:
459	return BuildSourceData().initFromDbRow(aRows[0])
460	except IndexError:
461	return None
462
463	#
464	# Internal helpers.
465	#
466
467	def _assertUnique(self, oData, idBuildSrcIgnore):
468	""" Checks that the build source name is unique, raises exception if it isn't. """
469	self._oDb.execute('SELECT idBuildSrc\n'
470	'FROM BuildSources\n'
471	'WHERE sName = %s\n'
472	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
473	+ ('' if idBuildSrcIgnore is None else ' AND idBuildSrc <> %d\n' % (idBuildSrcIgnore,))
474	, ( oData.sName, ))
475	if self._oDb.getRowCount() > 0:
476	raise TMRowAlreadyExists('A build source with name "%s" already exist.' % (oData.sName,));
477	return True;
478
479
480	def _historizeBuildSource(self, idBuildSrc, tsExpire = None):
481	""" Historizes the current build source entry. """
482	if tsExpire is None:
483	self._oDb.execute('UPDATE BuildSources\n'
484	'SET tsExpire = CURRENT_TIMESTAMP\n'
485	'WHERE idBuildSrc = %s\n'
486	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
487	, ( idBuildSrc, ));
488	else:
489	self._oDb.execute('UPDATE BuildSources\n'
490	'SET tsExpire = %s\n'
491	'WHERE idBuildSrc = %s\n'
492	' AND tsExpire = \'infinity\'::TIMESTAMP\n'
493	, ( tsExpire, idBuildSrc, ));
494	return True;
495
496
497
498
499
500	#
501	# Unit testing.
502	#
503
504	# pylint: disable=C0111
505	class BuildSourceDataTestCase(ModelDataBaseTestCase):
506	def setUp(self):
507	self.aoSamples = [BuildSourceData(),];
508
509	if __name__ == '__main__':
510	unittest.main();
511	# not reached.
512

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testmanager/core/buildsource.py@ 65040

Download in other formats: