1 | # -*- coding: utf-8 -*-
|
---|
2 | # $Id: db.py 76553 2019-01-01 01:45:53Z vboxsync $
|
---|
3 |
|
---|
4 | """
|
---|
5 | Test Manager - Database Interface.
|
---|
6 | """
|
---|
7 |
|
---|
8 | __copyright__ = \
|
---|
9 | """
|
---|
10 | Copyright (C) 2012-2019 Oracle Corporation
|
---|
11 |
|
---|
12 | This file is part of VirtualBox Open Source Edition (OSE), as
|
---|
13 | available from http://www.virtualbox.org. This file is free software;
|
---|
14 | you can redistribute it and/or modify it under the terms of the GNU
|
---|
15 | General Public License (GPL) as published by the Free Software
|
---|
16 | Foundation, in version 2 as it comes in the "COPYING" file of the
|
---|
17 | VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
---|
18 | hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
---|
19 |
|
---|
20 | The contents of this file may alternatively be used under the terms
|
---|
21 | of the Common Development and Distribution License Version 1.0
|
---|
22 | (CDDL) only, as it comes in the "COPYING.CDDL" file of the
|
---|
23 | VirtualBox OSE distribution, in which case the provisions of the
|
---|
24 | CDDL are applicable instead of those of the GPL.
|
---|
25 |
|
---|
26 | You may elect to license modified versions of this file under the
|
---|
27 | terms and conditions of either the GPL or the CDDL or both.
|
---|
28 | """
|
---|
29 | __version__ = "$Revision: 76553 $"
|
---|
30 |
|
---|
31 |
|
---|
32 | # Standard python imports.
|
---|
33 | import datetime;
|
---|
34 | import os;
|
---|
35 | import sys;
|
---|
36 | import psycopg2; # pylint: disable=import-error
|
---|
37 | import psycopg2.extensions; # pylint: disable=import-error
|
---|
38 |
|
---|
39 | # Validation Kit imports.
|
---|
40 | from common import utils, webutils;
|
---|
41 | from testmanager import config;
|
---|
42 |
|
---|
43 | # Fix psycho unicode handling in psycopg2 with python 2.x.
|
---|
44 | if sys.version_info[0] < 3:
|
---|
45 | psycopg2.extensions.register_type(psycopg2.extensions.UNICODE);
|
---|
46 | psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY);
|
---|
47 | else:
|
---|
48 | unicode = str; # pylint: disable=redefined-builtin,invalid-name
|
---|
49 |
|
---|
50 |
|
---|
51 |
|
---|
52 | def isDbTimestampInfinity(tsValue):
|
---|
53 | """
|
---|
54 | Checks if tsValue is an infinity timestamp.
|
---|
55 | """
|
---|
56 | ## @todo improve this test...
|
---|
57 | return tsValue.year >= 9999;
|
---|
58 |
|
---|
59 | def isDbTimestamp(oValue):
|
---|
60 | """
|
---|
61 | Checks if oValue is a DB timestamp object.
|
---|
62 | """
|
---|
63 | if isinstance(oValue, datetime.datetime):
|
---|
64 | return True;
|
---|
65 | if utils.isString(oValue):
|
---|
66 | ## @todo detect strings as well.
|
---|
67 | return False;
|
---|
68 | return getattr(oValue, 'pydatetime', None) != None;
|
---|
69 |
|
---|
70 | def dbTimestampToDatetime(oValue):
|
---|
71 | """
|
---|
72 | Converts a database timestamp to a datetime instance.
|
---|
73 | """
|
---|
74 | if isinstance(oValue, datetime.datetime):
|
---|
75 | return oValue;
|
---|
76 | if utils.isString(oValue):
|
---|
77 | raise Exception('TODO');
|
---|
78 | return oValue.pydatetime();
|
---|
79 |
|
---|
80 | def dbTimestampToZuluDatetime(oValue):
|
---|
81 | """
|
---|
82 | Converts a database timestamp to a zulu datetime instance.
|
---|
83 | """
|
---|
84 | tsValue = dbTimestampToDatetime(oValue);
|
---|
85 |
|
---|
86 | class UTC(datetime.tzinfo):
|
---|
87 | """UTC TZ Info Class"""
|
---|
88 | def utcoffset(self, _):
|
---|
89 | return datetime.timedelta(0);
|
---|
90 | def tzname(self, _):
|
---|
91 | return "UTC";
|
---|
92 | def dst(self, _):
|
---|
93 | return datetime.timedelta(0);
|
---|
94 | if tsValue.tzinfo is not None:
|
---|
95 | tsValue = tsValue.astimezone(UTC());
|
---|
96 | else:
|
---|
97 | tsValue = tsValue.replace(tzinfo=UTC());
|
---|
98 | return tsValue;
|
---|
99 |
|
---|
100 | def dbTimestampPythonNow():
|
---|
101 | """
|
---|
102 | Gets the current python timestamp in a database compatible way.
|
---|
103 | """
|
---|
104 | return dbTimestampToZuluDatetime(datetime.datetime.utcnow());
|
---|
105 |
|
---|
106 | def dbTimestampMinusOneTick(oValue):
|
---|
107 | """
|
---|
108 | Returns a new timestamp that's one tick before the given one.
|
---|
109 | """
|
---|
110 | oValue = dbTimestampToZuluDatetime(oValue);
|
---|
111 | return oValue - datetime.timedelta(microseconds = 1);
|
---|
112 |
|
---|
113 | def isDbInterval(oValue):
|
---|
114 | """
|
---|
115 | Checks if oValue is a DB interval object.
|
---|
116 | """
|
---|
117 | if isinstance(oValue, datetime.timedelta):
|
---|
118 | return True;
|
---|
119 | return False;
|
---|
120 |
|
---|
121 |
|
---|
122 | class TMDatabaseIntegrityException(Exception):
|
---|
123 | """
|
---|
124 | Herolds a database integrity error up the callstack.
|
---|
125 |
|
---|
126 | Do NOT use directly, only thru TMDatabaseConnection.integrityException.
|
---|
127 | Otherwise, we won't be able to log the issue.
|
---|
128 | """
|
---|
129 | pass;
|
---|
130 |
|
---|
131 |
|
---|
132 | class TMDatabaseCursor(object):
|
---|
133 | """ Cursor wrapper class. """
|
---|
134 |
|
---|
135 | def __init__(self, oDb, oCursor):
|
---|
136 | self._oDb = oDb;
|
---|
137 | self._oCursor = oCursor;
|
---|
138 |
|
---|
139 | def execute(self, sOperation, aoArgs = None):
|
---|
140 | """ See TMDatabaseConnection.execute()"""
|
---|
141 | return self._oDb.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName());
|
---|
142 |
|
---|
143 | def callProc(self, sProcedure, aoArgs = None):
|
---|
144 | """ See TMDatabaseConnection.callProc()"""
|
---|
145 | return self._oDb.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName());
|
---|
146 |
|
---|
147 | def insertList(self, sInsertSql, aoList, fnEntryFmt):
|
---|
148 | """ See TMDatabaseConnection.insertList. """
|
---|
149 | return self._oDb.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName());
|
---|
150 |
|
---|
151 | def fetchOne(self):
|
---|
152 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
153 | return self._oCursor.fetchone();
|
---|
154 |
|
---|
155 | def fetchMany(self, cRows = None):
|
---|
156 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
157 | return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize);
|
---|
158 |
|
---|
159 | def fetchAll(self):
|
---|
160 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
161 | return self._oCursor.fetchall();
|
---|
162 |
|
---|
163 | def getRowCount(self):
|
---|
164 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
165 | return self._oCursor.rowcount;
|
---|
166 |
|
---|
167 | def formatBindArgs(self, sStatement, aoArgs):
|
---|
168 | """Wrapper around Psycopg2.cursor.mogrify."""
|
---|
169 | oRet = self._oCursor.mogrify(sStatement, aoArgs);
|
---|
170 | if sys.version_info[0] >= 3 and not isinstance(oRet, str):
|
---|
171 | oRet = oRet.decode('utf-8');
|
---|
172 | return oRet;
|
---|
173 |
|
---|
174 | def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192):
|
---|
175 | """ See TMDatabaseConnection.copyExpert()"""
|
---|
176 | return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf);
|
---|
177 |
|
---|
178 | @staticmethod
|
---|
179 | def isTsInfinity(tsValue):
|
---|
180 | """ Checks if tsValue is an infinity timestamp. """
|
---|
181 | return isDbTimestampInfinity(tsValue);
|
---|
182 |
|
---|
183 |
|
---|
184 | class TMDatabaseConnection(object):
|
---|
185 | """
|
---|
186 | Test Manager Database Access class.
|
---|
187 |
|
---|
188 | This class contains no logic, just raw access abstraction and utilities,
|
---|
189 | as well as some debug help and some statistics.
|
---|
190 | """
|
---|
191 |
|
---|
192 | def __init__(self, fnDPrint = None, oSrvGlue = None):
|
---|
193 | """
|
---|
194 | Database connection wrapper.
|
---|
195 | The fnDPrint is for debug logging of all database activity.
|
---|
196 |
|
---|
197 | Raises an exception on failure.
|
---|
198 | """
|
---|
199 |
|
---|
200 | sAppName = '%s-%s' % (os.getpid(), os.path.basename(sys.argv[0]),)
|
---|
201 | if len(sAppName) >= 64:
|
---|
202 | sAppName = sAppName[:64];
|
---|
203 | os.environ['PGAPPNAME'] = sAppName;
|
---|
204 |
|
---|
205 | dArgs = \
|
---|
206 | { \
|
---|
207 | 'database': config.g_ksDatabaseName,
|
---|
208 | 'user': config.g_ksDatabaseUser,
|
---|
209 | 'password': config.g_ksDatabasePassword,
|
---|
210 | # 'application_name': sAppName, - Darn stale debian! :/
|
---|
211 | };
|
---|
212 | if config.g_ksDatabaseAddress is not None:
|
---|
213 | dArgs['host'] = config.g_ksDatabaseAddress;
|
---|
214 | if config.g_ksDatabasePort is not None:
|
---|
215 | dArgs['port'] = config.g_ksDatabasePort;
|
---|
216 | self._oConn = psycopg2.connect(**dArgs); # pylint: disable=W0142
|
---|
217 | self._oConn.set_client_encoding('UTF-8');
|
---|
218 | self._oCursor = self._oConn.cursor();
|
---|
219 | self._oExplainConn = None;
|
---|
220 | self._oExplainCursor = None;
|
---|
221 | if config.g_kfWebUiSqlTraceExplain and config.g_kfWebUiSqlTrace:
|
---|
222 | self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=W0142
|
---|
223 | self._oExplainConn.set_client_encoding('UTF-8');
|
---|
224 | self._oExplainCursor = self._oExplainConn.cursor();
|
---|
225 | self._fTransaction = False;
|
---|
226 | self._tsCurrent = None;
|
---|
227 | self._tsCurrentMinusOne = None;
|
---|
228 |
|
---|
229 | assert self.isAutoCommitting() is False;
|
---|
230 |
|
---|
231 | # Debug and introspection.
|
---|
232 | self._fnDPrint = fnDPrint;
|
---|
233 | self._aoTraceBack = [];
|
---|
234 |
|
---|
235 | # Exception class handles.
|
---|
236 | self.oXcptError = psycopg2.Error;
|
---|
237 |
|
---|
238 | if oSrvGlue is not None:
|
---|
239 | oSrvGlue.registerDebugInfoCallback(self.debugInfoCallback);
|
---|
240 |
|
---|
241 | # Object caches (used by database logic classes).
|
---|
242 | self.ddCaches = dict();
|
---|
243 |
|
---|
244 | def isAutoCommitting(self):
|
---|
245 | """ Work around missing autocommit attribute in older versions."""
|
---|
246 | return getattr(self._oConn, 'autocommit', False);
|
---|
247 |
|
---|
248 | def close(self):
|
---|
249 | """
|
---|
250 | Closes the connection and renders all cursors useless.
|
---|
251 | """
|
---|
252 | if self._oCursor is not None:
|
---|
253 | self._oCursor.close();
|
---|
254 | self._oCursor = None;
|
---|
255 |
|
---|
256 | if self._oConn is not None:
|
---|
257 | self._oConn.close();
|
---|
258 | self._oConn = None;
|
---|
259 |
|
---|
260 | if self._oExplainCursor is not None:
|
---|
261 | self._oExplainCursor.close();
|
---|
262 | self._oExplainCursor = None;
|
---|
263 |
|
---|
264 | if self._oExplainConn is not None:
|
---|
265 | self._oExplainConn.close();
|
---|
266 | self._oExplainConn = None;
|
---|
267 |
|
---|
268 |
|
---|
269 | def _startedTransaction(self):
|
---|
270 | """
|
---|
271 | Called to work the _fTransaction and related variables when starting
|
---|
272 | a transaction.
|
---|
273 | """
|
---|
274 | self._fTransaction = True;
|
---|
275 | self._tsCurrent = None;
|
---|
276 | self._tsCurrentMinusOne = None;
|
---|
277 | return None;
|
---|
278 |
|
---|
279 | def _endedTransaction(self):
|
---|
280 | """
|
---|
281 | Called to work the _fTransaction and related variables when ending
|
---|
282 | a transaction.
|
---|
283 | """
|
---|
284 | self._fTransaction = False;
|
---|
285 | self._tsCurrent = None;
|
---|
286 | self._tsCurrentMinusOne = None;
|
---|
287 | return None;
|
---|
288 |
|
---|
289 | def begin(self):
|
---|
290 | """
|
---|
291 | Currently just for marking where a transaction starts in the code.
|
---|
292 | """
|
---|
293 | assert self._oConn is not None;
|
---|
294 | assert self.isAutoCommitting() is False;
|
---|
295 | self._aoTraceBack.append([utils.timestampNano(), 'START TRANSACTION', 0, 0, utils.getCallerName(), None]);
|
---|
296 | self._startedTransaction();
|
---|
297 | return True;
|
---|
298 |
|
---|
299 | def commit(self, sCallerName = None):
|
---|
300 | """ Wrapper around Psycopg2.connection.commit."""
|
---|
301 | assert self._fTransaction is True;
|
---|
302 |
|
---|
303 | nsStart = utils.timestampNano();
|
---|
304 | oRc = self._oConn.commit();
|
---|
305 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
306 |
|
---|
307 | if sCallerName is None:
|
---|
308 | sCallerName = utils.getCallerName();
|
---|
309 | self._aoTraceBack.append([nsStart, 'COMMIT', cNsElapsed, 0, sCallerName, None]);
|
---|
310 | self._endedTransaction();
|
---|
311 | return oRc;
|
---|
312 |
|
---|
313 | def maybeCommit(self, fCommit):
|
---|
314 | """
|
---|
315 | Commits if fCommit is True.
|
---|
316 | Returns True if committed, False if not.
|
---|
317 | """
|
---|
318 | if fCommit is True:
|
---|
319 | self.commit(utils.getCallerName());
|
---|
320 | return True;
|
---|
321 | return False;
|
---|
322 |
|
---|
323 | def rollback(self):
|
---|
324 | """ Wrapper around Psycopg2.connection.rollback."""
|
---|
325 | nsStart = utils.timestampNano();
|
---|
326 | oRc = self._oConn.rollback();
|
---|
327 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
328 |
|
---|
329 | self._aoTraceBack.append([nsStart, 'ROLLBACK', cNsElapsed, 0, utils.getCallerName(), None]);
|
---|
330 | self._endedTransaction();
|
---|
331 | return oRc;
|
---|
332 |
|
---|
333 | #
|
---|
334 | # Internal cursor workers.
|
---|
335 | #
|
---|
336 |
|
---|
337 | def executeInternal(self, oCursor, sOperation, aoArgs, sCallerName):
|
---|
338 | """
|
---|
339 | Execute a query or command.
|
---|
340 |
|
---|
341 | Mostly a wrapper around the psycopg2 cursor method with the same name,
|
---|
342 | but collect data for traceback.
|
---|
343 | """
|
---|
344 | if aoArgs is not None:
|
---|
345 | sBound = oCursor.mogrify(unicode(sOperation), aoArgs);
|
---|
346 | elif sOperation.find('%') < 0:
|
---|
347 | sBound = oCursor.mogrify(unicode(sOperation), list());
|
---|
348 | else:
|
---|
349 | sBound = unicode(sOperation);
|
---|
350 |
|
---|
351 | if sys.version_info[0] >= 3 and not isinstance(sBound, str):
|
---|
352 | sBound = sBound.decode('utf-8'); # pylint: disable=redefined-variable-type
|
---|
353 |
|
---|
354 | aasExplain = None;
|
---|
355 | if self._oExplainCursor is not None and not sBound.startswith('DROP'):
|
---|
356 | try:
|
---|
357 | if config.g_kfWebUiSqlTraceExplainTiming:
|
---|
358 | self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE, TIMING) ' + sBound);
|
---|
359 | else:
|
---|
360 | self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE) ' + sBound);
|
---|
361 | except Exception as oXcpt:
|
---|
362 | aasExplain = [ ['Explain exception: '], [str(oXcpt)]];
|
---|
363 | try: self._oExplainConn.rollback();
|
---|
364 | except: pass;
|
---|
365 | else:
|
---|
366 | aasExplain = self._oExplainCursor.fetchall();
|
---|
367 |
|
---|
368 | nsStart = utils.timestampNano();
|
---|
369 | try:
|
---|
370 | oRc = oCursor.execute(sBound);
|
---|
371 | except Exception as oXcpt:
|
---|
372 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
373 | self._aoTraceBack.append([nsStart, 'oXcpt=%s; Statement: %s' % (oXcpt, sBound), cNsElapsed, 0, sCallerName, None]);
|
---|
374 | if self._fnDPrint is not None:
|
---|
375 | self._fnDPrint('db::execute %u ns, caller %s: oXcpt=%s; Statement: %s'
|
---|
376 | % (cNsElapsed, sCallerName, oXcpt, sBound));
|
---|
377 | raise;
|
---|
378 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
379 |
|
---|
380 | if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ.
|
---|
381 | self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]);
|
---|
382 | self._startedTransaction();
|
---|
383 | self._aoTraceBack.append([nsStart, sBound, cNsElapsed, oCursor.rowcount, sCallerName, aasExplain]);
|
---|
384 | if self._fnDPrint is not None:
|
---|
385 | self._fnDPrint('db::execute %u ns, caller %s: "\n%s"' % (cNsElapsed, sCallerName, sBound));
|
---|
386 | if self.isAutoCommitting():
|
---|
387 | self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, None]);
|
---|
388 |
|
---|
389 | return oRc;
|
---|
390 |
|
---|
391 | def callProcInternal(self, oCursor, sProcedure, aoArgs, sCallerName):
|
---|
392 | """
|
---|
393 | Call a stored procedure.
|
---|
394 |
|
---|
395 | Mostly a wrapper around the psycopg2 cursor method 'callproc', but
|
---|
396 | collect data for traceback.
|
---|
397 | """
|
---|
398 | if aoArgs is None:
|
---|
399 | aoArgs = list();
|
---|
400 |
|
---|
401 | nsStart = utils.timestampNano();
|
---|
402 | try:
|
---|
403 | oRc = oCursor.callproc(sProcedure, aoArgs);
|
---|
404 | except Exception as oXcpt:
|
---|
405 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
406 | self._aoTraceBack.append([nsStart, 'oXcpt=%s; Calling: %s(%s)' % (oXcpt, sProcedure, aoArgs),
|
---|
407 | cNsElapsed, 0, sCallerName, None]);
|
---|
408 | if self._fnDPrint is not None:
|
---|
409 | self._fnDPrint('db::callproc %u ns, caller %s: oXcpt=%s; Calling: %s(%s)'
|
---|
410 | % (cNsElapsed, sCallerName, oXcpt, sProcedure, aoArgs));
|
---|
411 | raise;
|
---|
412 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
413 |
|
---|
414 | if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ.
|
---|
415 | self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]);
|
---|
416 | self._startedTransaction();
|
---|
417 | self._aoTraceBack.append([nsStart, '%s(%s)' % (sProcedure, aoArgs), cNsElapsed, oCursor.rowcount, sCallerName, None]);
|
---|
418 | if self._fnDPrint is not None:
|
---|
419 | self._fnDPrint('db::callproc %u ns, caller %s: "%s(%s)"' % (cNsElapsed, sCallerName, sProcedure, aoArgs));
|
---|
420 | if self.isAutoCommitting():
|
---|
421 | self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, sCallerName, None]);
|
---|
422 |
|
---|
423 | return oRc;
|
---|
424 |
|
---|
425 | def insertListInternal(self, oCursor, sInsertSql, aoList, fnEntryFmt, sCallerName):
|
---|
426 | """
|
---|
427 | Optimizes the insertion of a list of values.
|
---|
428 | """
|
---|
429 | oRc = None;
|
---|
430 | asValues = [];
|
---|
431 | for aoEntry in aoList:
|
---|
432 | asValues.append(fnEntryFmt(aoEntry));
|
---|
433 | if len(asValues) > 256:
|
---|
434 | oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName);
|
---|
435 | asValues = [];
|
---|
436 | if asValues:
|
---|
437 | oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName);
|
---|
438 | return oRc
|
---|
439 |
|
---|
440 | def _fetchOne(self, oCursor):
|
---|
441 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
442 | oRow = oCursor.fetchone()
|
---|
443 | if self._fnDPrint is not None:
|
---|
444 | self._fnDPrint('db:fetchOne returns: %s' % (oRow,));
|
---|
445 | return oRow;
|
---|
446 |
|
---|
447 | def _fetchMany(self, oCursor, cRows):
|
---|
448 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
449 | return oCursor.fetchmany(cRows if cRows is not None else oCursor.arraysize);
|
---|
450 |
|
---|
451 | def _fetchAll(self, oCursor):
|
---|
452 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
453 | return oCursor.fetchall()
|
---|
454 |
|
---|
455 | def _getRowCountWorker(self, oCursor):
|
---|
456 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
457 | return oCursor.rowcount;
|
---|
458 |
|
---|
459 |
|
---|
460 | #
|
---|
461 | # Default cursor access.
|
---|
462 | #
|
---|
463 |
|
---|
464 | def execute(self, sOperation, aoArgs = None):
|
---|
465 | """
|
---|
466 | Execute a query or command.
|
---|
467 |
|
---|
468 | Mostly a wrapper around the psycopg2 cursor method with the same name,
|
---|
469 | but collect data for traceback.
|
---|
470 | """
|
---|
471 | return self.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName());
|
---|
472 |
|
---|
473 | def callProc(self, sProcedure, aoArgs = None):
|
---|
474 | """
|
---|
475 | Call a stored procedure.
|
---|
476 |
|
---|
477 | Mostly a wrapper around the psycopg2 cursor method 'callproc', but
|
---|
478 | collect data for traceback.
|
---|
479 | """
|
---|
480 | return self.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName());
|
---|
481 |
|
---|
482 | def insertList(self, sInsertSql, aoList, fnEntryFmt):
|
---|
483 | """
|
---|
484 | Optimizes the insertion of a list of values.
|
---|
485 | """
|
---|
486 | return self.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName());
|
---|
487 |
|
---|
488 | def fetchOne(self):
|
---|
489 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
490 | return self._oCursor.fetchone();
|
---|
491 |
|
---|
492 | def fetchMany(self, cRows = None):
|
---|
493 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
494 | return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize);
|
---|
495 |
|
---|
496 | def fetchAll(self):
|
---|
497 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
498 | return self._oCursor.fetchall();
|
---|
499 |
|
---|
500 | def getRowCount(self):
|
---|
501 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
502 | return self._oCursor.rowcount;
|
---|
503 |
|
---|
504 | def formatBindArgs(self, sStatement, aoArgs):
|
---|
505 | """Wrapper around Psycopg2.cursor.mogrify."""
|
---|
506 | oRet = self._oCursor.mogrify(sStatement, aoArgs);
|
---|
507 | if sys.version_info[0] >= 3 and not isinstance(oRet, str):
|
---|
508 | oRet = oRet.decode('utf-8');
|
---|
509 | return oRet;
|
---|
510 |
|
---|
511 | def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192):
|
---|
512 | """ Wrapper around Psycopg2.cursor.copy_expert. """
|
---|
513 | return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf);
|
---|
514 |
|
---|
515 | def getCurrentTimestamps(self):
|
---|
516 | """
|
---|
517 | Returns the current timestamp and the current timestamp minus one tick.
|
---|
518 | This will start a transaction if necessary.
|
---|
519 | """
|
---|
520 | if self._tsCurrent is None:
|
---|
521 | self.execute('SELECT CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - INTERVAL \'1 microsecond\'');
|
---|
522 | (self._tsCurrent, self._tsCurrentMinusOne) = self.fetchOne();
|
---|
523 | return (self._tsCurrent, self._tsCurrentMinusOne);
|
---|
524 |
|
---|
525 | def getCurrentTimestamp(self):
|
---|
526 | """
|
---|
527 | Returns the current timestamp.
|
---|
528 | This will start a transaction if necessary.
|
---|
529 | """
|
---|
530 | if self._tsCurrent is None:
|
---|
531 | self.getCurrentTimestamps();
|
---|
532 | return self._tsCurrent;
|
---|
533 |
|
---|
534 | def getCurrentTimestampMinusOne(self):
|
---|
535 | """
|
---|
536 | Returns the current timestamp minus one tick.
|
---|
537 | This will start a transaction if necessary.
|
---|
538 | """
|
---|
539 | if self._tsCurrentMinusOne is None:
|
---|
540 | self.getCurrentTimestamps();
|
---|
541 | return self._tsCurrentMinusOne;
|
---|
542 |
|
---|
543 |
|
---|
544 | #
|
---|
545 | # Additional cursors.
|
---|
546 | #
|
---|
547 | def openCursor(self):
|
---|
548 | """
|
---|
549 | Opens a new cursor (TMDatabaseCursor).
|
---|
550 | """
|
---|
551 | oCursor = self._oConn.cursor();
|
---|
552 | return TMDatabaseCursor(self, oCursor);
|
---|
553 |
|
---|
554 | #
|
---|
555 | # Cache support.
|
---|
556 | #
|
---|
557 | def getCache(self, sType):
|
---|
558 | """ Returns the cache dictionary for this data type. """
|
---|
559 | dRet = self.ddCaches.get(sType, None);
|
---|
560 | if dRet is None:
|
---|
561 | dRet = dict();
|
---|
562 | self.ddCaches[sType] = dRet;
|
---|
563 | return dRet;
|
---|
564 |
|
---|
565 |
|
---|
566 | #
|
---|
567 | # Utilities.
|
---|
568 | #
|
---|
569 |
|
---|
570 | @staticmethod
|
---|
571 | def isTsInfinity(tsValue):
|
---|
572 | """ Checks if tsValue is an infinity timestamp. """
|
---|
573 | return isDbTimestampInfinity(tsValue);
|
---|
574 |
|
---|
575 | #
|
---|
576 | # Error stuff.
|
---|
577 | #
|
---|
578 | def integrityException(self, sMessage):
|
---|
579 | """
|
---|
580 | Database integrity reporter and exception factory.
|
---|
581 | Returns an TMDatabaseIntegrityException which the caller can raise.
|
---|
582 | """
|
---|
583 | ## @todo Create a new database connection and log the issue in the SystemLog table.
|
---|
584 | ## Alternatively, rollback whatever is going on and do it using the current one.
|
---|
585 | return TMDatabaseIntegrityException(sMessage);
|
---|
586 |
|
---|
587 |
|
---|
588 | #
|
---|
589 | # Debugging.
|
---|
590 | #
|
---|
591 |
|
---|
592 | def dprint(self, sText):
|
---|
593 | """
|
---|
594 | Debug output.
|
---|
595 | """
|
---|
596 | if not self._fnDPrint:
|
---|
597 | return False;
|
---|
598 | self._fnDPrint(sText);
|
---|
599 | return True;
|
---|
600 |
|
---|
601 | def debugHtmlReport(self, tsStart = 0):
|
---|
602 | """
|
---|
603 | Used to get a SQL activity dump as HTML, usually for WuiBase._sDebug.
|
---|
604 | """
|
---|
605 | cNsElapsed = 0;
|
---|
606 | for aEntry in self._aoTraceBack:
|
---|
607 | cNsElapsed += aEntry[2];
|
---|
608 |
|
---|
609 | sDebug = '<h3>SQL Debug Log (total time %s ns):</h3>\n' \
|
---|
610 | '<table class="tmsqltable">\n' \
|
---|
611 | ' <tr>\n' \
|
---|
612 | ' <th>No.</th>\n' \
|
---|
613 | ' <th>Timestamp (ns)</th>\n' \
|
---|
614 | ' <th>Elapsed (ns)</th>\n' \
|
---|
615 | ' <th>Rows Returned</th>\n' \
|
---|
616 | ' <th>Command</th>\n' \
|
---|
617 | ' <th>Caller</th>\n' \
|
---|
618 | ' </tr>\n' \
|
---|
619 | % (utils.formatNumber(cNsElapsed, ' '),);
|
---|
620 |
|
---|
621 | iEntry = 0;
|
---|
622 | for aEntry in self._aoTraceBack:
|
---|
623 | iEntry += 1;
|
---|
624 | sDebug += ' <tr>\n' \
|
---|
625 | ' <td align="right">%s</td>\n' \
|
---|
626 | ' <td align="right">%s</td>\n' \
|
---|
627 | ' <td align="right">%s</td>\n' \
|
---|
628 | ' <td align="right">%s</td>\n' \
|
---|
629 | ' <td><pre>%s</pre></td>\n' \
|
---|
630 | ' <td>%s</td>\n' \
|
---|
631 | ' </tr>\n' \
|
---|
632 | % (iEntry,
|
---|
633 | utils.formatNumber(aEntry[0] - tsStart, ' '),
|
---|
634 | utils.formatNumber(aEntry[2], ' '),
|
---|
635 | utils.formatNumber(aEntry[3], ' '),
|
---|
636 | webutils.escapeElem(aEntry[1]),
|
---|
637 | webutils.escapeElem(aEntry[4]),
|
---|
638 | );
|
---|
639 | if aEntry[5] is not None:
|
---|
640 | sDebug += ' <tr>\n' \
|
---|
641 | ' <td colspan="6"><pre style="white-space: pre-wrap;">%s</pre></td>\n' \
|
---|
642 | ' </tr>\n' \
|
---|
643 | % (webutils.escapeElem('\n'.join([aoRow[0] for aoRow in aEntry[5]])),);
|
---|
644 |
|
---|
645 | sDebug += '</table>';
|
---|
646 | return sDebug;
|
---|
647 |
|
---|
648 | def debugTextReport(self, tsStart = 0):
|
---|
649 | """
|
---|
650 | Used to get a SQL activity dump as text.
|
---|
651 | """
|
---|
652 | cNsElapsed = 0;
|
---|
653 | for aEntry in self._aoTraceBack:
|
---|
654 | cNsElapsed += aEntry[2];
|
---|
655 |
|
---|
656 | sHdr = 'SQL Debug Log (total time %s ns)' % (utils.formatNumber(cNsElapsed),);
|
---|
657 | sDebug = sHdr + '\n' + '-' * len(sHdr) + '\n';
|
---|
658 |
|
---|
659 | iEntry = 0;
|
---|
660 | for aEntry in self._aoTraceBack:
|
---|
661 | iEntry += 1;
|
---|
662 | sHdr = 'Query #%s Timestamp: %s ns Elapsed: %s ns Rows: %s Caller: %s' \
|
---|
663 | % ( iEntry,
|
---|
664 | utils.formatNumber(aEntry[0] - tsStart),
|
---|
665 | utils.formatNumber(aEntry[2]),
|
---|
666 | utils.formatNumber(aEntry[3]),
|
---|
667 | aEntry[4], );
|
---|
668 | sDebug += '\n' + sHdr + '\n' + '-' * len(sHdr) + '\n';
|
---|
669 |
|
---|
670 | sDebug += aEntry[1];
|
---|
671 | if sDebug[-1] != '\n':
|
---|
672 | sDebug += '\n';
|
---|
673 |
|
---|
674 | if aEntry[5] is not None:
|
---|
675 | sDebug += 'Explain:\n' \
|
---|
676 | ' %s\n' \
|
---|
677 | % ( '\n'.join([aoRow[0] for aoRow in aEntry[5]]),);
|
---|
678 |
|
---|
679 | return sDebug;
|
---|
680 |
|
---|
681 | def debugInfoCallback(self, oGlue, fHtml):
|
---|
682 | """ Called back by the glue code on error. """
|
---|
683 | oGlue.write('\n');
|
---|
684 | if not fHtml: oGlue.write(self.debugTextReport());
|
---|
685 | else: oGlue.write(self.debugHtmlReport());
|
---|
686 | oGlue.write('\n');
|
---|
687 | return True;
|
---|
688 |
|
---|
689 | def debugEnableExplain(self):
|
---|
690 | """ Enabled explain. """
|
---|
691 | if self._oExplainConn is None:
|
---|
692 | dArgs = \
|
---|
693 | { \
|
---|
694 | 'database': config.g_ksDatabaseName,
|
---|
695 | 'user': config.g_ksDatabaseUser,
|
---|
696 | 'password': config.g_ksDatabasePassword,
|
---|
697 | # 'application_name': sAppName, - Darn stale debian! :/
|
---|
698 | };
|
---|
699 | if config.g_ksDatabaseAddress is not None:
|
---|
700 | dArgs['host'] = config.g_ksDatabaseAddress;
|
---|
701 | if config.g_ksDatabasePort is not None:
|
---|
702 | dArgs['port'] = config.g_ksDatabasePort;
|
---|
703 | self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=W0142
|
---|
704 | self._oExplainCursor = self._oExplainConn.cursor();
|
---|
705 | return True;
|
---|
706 |
|
---|
707 | def debugDisableExplain(self):
|
---|
708 | """ Disables explain. """
|
---|
709 | self._oExplainCursor = None;
|
---|
710 | self._oExplainConn = None
|
---|
711 | return True;
|
---|
712 |
|
---|
713 | def debugIsExplainEnabled(self):
|
---|
714 | """ Check if explaining of SQL statements is enabled. """
|
---|
715 | return self._oExplainConn is not None;
|
---|
716 |
|
---|