1 | # -*- coding: utf-8 -*-
|
---|
2 | # $Id: db.py 83396 2020-03-24 20:08:41Z vboxsync $
|
---|
3 |
|
---|
4 | """
|
---|
5 | Test Manager - Database Interface.
|
---|
6 | """
|
---|
7 |
|
---|
8 | __copyright__ = \
|
---|
9 | """
|
---|
10 | Copyright (C) 2012-2020 Oracle Corporation
|
---|
11 |
|
---|
12 | This file is part of VirtualBox Open Source Edition (OSE), as
|
---|
13 | available from http://www.virtualbox.org. This file is free software;
|
---|
14 | you can redistribute it and/or modify it under the terms of the GNU
|
---|
15 | General Public License (GPL) as published by the Free Software
|
---|
16 | Foundation, in version 2 as it comes in the "COPYING" file of the
|
---|
17 | VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
---|
18 | hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
---|
19 |
|
---|
20 | The contents of this file may alternatively be used under the terms
|
---|
21 | of the Common Development and Distribution License Version 1.0
|
---|
22 | (CDDL) only, as it comes in the "COPYING.CDDL" file of the
|
---|
23 | VirtualBox OSE distribution, in which case the provisions of the
|
---|
24 | CDDL are applicable instead of those of the GPL.
|
---|
25 |
|
---|
26 | You may elect to license modified versions of this file under the
|
---|
27 | terms and conditions of either the GPL or the CDDL or both.
|
---|
28 | """
|
---|
29 | __version__ = "$Revision: 83396 $"
|
---|
30 |
|
---|
31 |
|
---|
32 | # Standard python imports.
|
---|
33 | import datetime;
|
---|
34 | import os;
|
---|
35 | import sys;
|
---|
36 | import psycopg2; # pylint: disable=import-error
|
---|
37 | import psycopg2.extensions; # pylint: disable=import-error
|
---|
38 |
|
---|
39 | # Validation Kit imports.
|
---|
40 | from common import utils, webutils;
|
---|
41 | from testmanager import config;
|
---|
42 |
|
---|
43 | # Fix psycho unicode handling in psycopg2 with python 2.x.
|
---|
44 | if sys.version_info[0] < 3:
|
---|
45 | psycopg2.extensions.register_type(psycopg2.extensions.UNICODE);
|
---|
46 | psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY);
|
---|
47 | else:
|
---|
48 | unicode = str; # pylint: disable=redefined-builtin,invalid-name
|
---|
49 |
|
---|
50 |
|
---|
51 |
|
---|
52 | def isDbTimestampInfinity(tsValue):
|
---|
53 | """
|
---|
54 | Checks if tsValue is an infinity timestamp.
|
---|
55 | """
|
---|
56 | ## @todo improve this test...
|
---|
57 | return tsValue.year >= 9999;
|
---|
58 |
|
---|
59 | def isDbTimestamp(oValue):
|
---|
60 | """
|
---|
61 | Checks if oValue is a DB timestamp object.
|
---|
62 | """
|
---|
63 | if isinstance(oValue, datetime.datetime):
|
---|
64 | return True;
|
---|
65 | if utils.isString(oValue):
|
---|
66 | ## @todo detect strings as well.
|
---|
67 | return False;
|
---|
68 | return getattr(oValue, 'pydatetime', None) is not None;
|
---|
69 |
|
---|
70 | def dbTimestampToDatetime(oValue):
|
---|
71 | """
|
---|
72 | Converts a database timestamp to a datetime instance.
|
---|
73 | """
|
---|
74 | if isinstance(oValue, datetime.datetime):
|
---|
75 | return oValue;
|
---|
76 | if utils.isString(oValue):
|
---|
77 | return utils.parseIsoTimestamp(oValue);
|
---|
78 | return oValue.pydatetime();
|
---|
79 |
|
---|
80 | def dbTimestampToZuluDatetime(oValue):
|
---|
81 | """
|
---|
82 | Converts a database timestamp to a zulu datetime instance.
|
---|
83 | """
|
---|
84 | tsValue = dbTimestampToDatetime(oValue);
|
---|
85 |
|
---|
86 | class UTC(datetime.tzinfo):
|
---|
87 | """UTC TZ Info Class"""
|
---|
88 | def utcoffset(self, _):
|
---|
89 | return datetime.timedelta(0);
|
---|
90 | def tzname(self, _):
|
---|
91 | return "UTC";
|
---|
92 | def dst(self, _):
|
---|
93 | return datetime.timedelta(0);
|
---|
94 | if tsValue.tzinfo is not None:
|
---|
95 | tsValue = tsValue.astimezone(UTC());
|
---|
96 | else:
|
---|
97 | tsValue = tsValue.replace(tzinfo=UTC());
|
---|
98 | return tsValue;
|
---|
99 |
|
---|
100 | def dbTimestampPythonNow():
|
---|
101 | """
|
---|
102 | Gets the current python timestamp in a database compatible way.
|
---|
103 | """
|
---|
104 | return dbTimestampToZuluDatetime(datetime.datetime.utcnow());
|
---|
105 |
|
---|
106 | def dbOneTickIntervalString():
|
---|
107 | """
|
---|
108 | Returns the interval string for one tick.
|
---|
109 |
|
---|
110 | Mogrify the return value into the SQL:
|
---|
111 | "... %s::INTERVAL ..."
|
---|
112 | or
|
---|
113 | "INTERVAL %s"
|
---|
114 | The completed SQL will contain the necessary ticks.
|
---|
115 | """
|
---|
116 | return '1 microsecond';
|
---|
117 |
|
---|
118 | def dbTimestampMinusOneTick(oValue):
|
---|
119 | """
|
---|
120 | Returns a new timestamp that's one tick before the given one.
|
---|
121 | """
|
---|
122 | oValue = dbTimestampToZuluDatetime(oValue);
|
---|
123 | return oValue - datetime.timedelta(microseconds = 1);
|
---|
124 |
|
---|
125 | def dbTimestampPlusOneTick(oValue):
|
---|
126 | """
|
---|
127 | Returns a new timestamp that's one tick after the given one.
|
---|
128 | """
|
---|
129 | oValue = dbTimestampToZuluDatetime(oValue);
|
---|
130 | return oValue + datetime.timedelta(microseconds = 1);
|
---|
131 |
|
---|
132 | def isDbInterval(oValue):
|
---|
133 | """
|
---|
134 | Checks if oValue is a DB interval object.
|
---|
135 | """
|
---|
136 | if isinstance(oValue, datetime.timedelta):
|
---|
137 | return True;
|
---|
138 | return False;
|
---|
139 |
|
---|
140 |
|
---|
141 | class TMDatabaseIntegrityException(Exception):
|
---|
142 | """
|
---|
143 | Herolds a database integrity error up the callstack.
|
---|
144 |
|
---|
145 | Do NOT use directly, only thru TMDatabaseConnection.integrityException.
|
---|
146 | Otherwise, we won't be able to log the issue.
|
---|
147 | """
|
---|
148 | pass; # pylint: disable=unnecessary-pass
|
---|
149 |
|
---|
150 |
|
---|
151 | class TMDatabaseCursor(object):
|
---|
152 | """ Cursor wrapper class. """
|
---|
153 |
|
---|
154 | def __init__(self, oDb, oCursor):
|
---|
155 | self._oDb = oDb;
|
---|
156 | self._oCursor = oCursor;
|
---|
157 |
|
---|
158 | def execute(self, sOperation, aoArgs = None):
|
---|
159 | """ See TMDatabaseConnection.execute()"""
|
---|
160 | return self._oDb.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName());
|
---|
161 |
|
---|
162 | def callProc(self, sProcedure, aoArgs = None):
|
---|
163 | """ See TMDatabaseConnection.callProc()"""
|
---|
164 | return self._oDb.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName());
|
---|
165 |
|
---|
166 | def insertList(self, sInsertSql, aoList, fnEntryFmt):
|
---|
167 | """ See TMDatabaseConnection.insertList. """
|
---|
168 | return self._oDb.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName());
|
---|
169 |
|
---|
170 | def fetchOne(self):
|
---|
171 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
172 | return self._oCursor.fetchone();
|
---|
173 |
|
---|
174 | def fetchMany(self, cRows = None):
|
---|
175 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
176 | return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize);
|
---|
177 |
|
---|
178 | def fetchAll(self):
|
---|
179 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
180 | return self._oCursor.fetchall();
|
---|
181 |
|
---|
182 | def getRowCount(self):
|
---|
183 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
184 | return self._oCursor.rowcount;
|
---|
185 |
|
---|
186 | def formatBindArgs(self, sStatement, aoArgs):
|
---|
187 | """Wrapper around Psycopg2.cursor.mogrify."""
|
---|
188 | oRet = self._oCursor.mogrify(sStatement, aoArgs);
|
---|
189 | if sys.version_info[0] >= 3 and not isinstance(oRet, str):
|
---|
190 | oRet = oRet.decode('utf-8');
|
---|
191 | return oRet;
|
---|
192 |
|
---|
193 | def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192):
|
---|
194 | """ See TMDatabaseConnection.copyExpert()"""
|
---|
195 | return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf);
|
---|
196 |
|
---|
197 | @staticmethod
|
---|
198 | def isTsInfinity(tsValue):
|
---|
199 | """ Checks if tsValue is an infinity timestamp. """
|
---|
200 | return isDbTimestampInfinity(tsValue);
|
---|
201 |
|
---|
202 |
|
---|
203 | class TMDatabaseConnection(object):
|
---|
204 | """
|
---|
205 | Test Manager Database Access class.
|
---|
206 |
|
---|
207 | This class contains no logic, just raw access abstraction and utilities,
|
---|
208 | as well as some debug help and some statistics.
|
---|
209 | """
|
---|
210 |
|
---|
211 | def __init__(self, fnDPrint = None, oSrvGlue = None):
|
---|
212 | """
|
---|
213 | Database connection wrapper.
|
---|
214 | The fnDPrint is for debug logging of all database activity.
|
---|
215 |
|
---|
216 | Raises an exception on failure.
|
---|
217 | """
|
---|
218 |
|
---|
219 | sAppName = '%s-%s' % (os.getpid(), os.path.basename(sys.argv[0]),)
|
---|
220 | if len(sAppName) >= 64:
|
---|
221 | sAppName = sAppName[:64];
|
---|
222 | os.environ['PGAPPNAME'] = sAppName;
|
---|
223 |
|
---|
224 | dArgs = \
|
---|
225 | { \
|
---|
226 | 'database': config.g_ksDatabaseName,
|
---|
227 | 'user': config.g_ksDatabaseUser,
|
---|
228 | 'password': config.g_ksDatabasePassword,
|
---|
229 | # 'application_name': sAppName, - Darn stale debian! :/
|
---|
230 | };
|
---|
231 | if config.g_ksDatabaseAddress is not None:
|
---|
232 | dArgs['host'] = config.g_ksDatabaseAddress;
|
---|
233 | if config.g_ksDatabasePort is not None:
|
---|
234 | dArgs['port'] = config.g_ksDatabasePort;
|
---|
235 | self._oConn = psycopg2.connect(**dArgs); # pylint: disable=star-args
|
---|
236 | self._oConn.set_client_encoding('UTF-8');
|
---|
237 | self._oCursor = self._oConn.cursor();
|
---|
238 | self._oExplainConn = None;
|
---|
239 | self._oExplainCursor = None;
|
---|
240 | if config.g_kfWebUiSqlTraceExplain and config.g_kfWebUiSqlTrace:
|
---|
241 | self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=star-args
|
---|
242 | self._oExplainConn.set_client_encoding('UTF-8');
|
---|
243 | self._oExplainCursor = self._oExplainConn.cursor();
|
---|
244 | self._fTransaction = False;
|
---|
245 | self._tsCurrent = None;
|
---|
246 | self._tsCurrentMinusOne = None;
|
---|
247 |
|
---|
248 | assert self.isAutoCommitting() is False;
|
---|
249 |
|
---|
250 | # Debug and introspection.
|
---|
251 | self._fnDPrint = fnDPrint;
|
---|
252 | self._aoTraceBack = [];
|
---|
253 |
|
---|
254 | # Exception class handles.
|
---|
255 | self.oXcptError = psycopg2.Error;
|
---|
256 |
|
---|
257 | if oSrvGlue is not None:
|
---|
258 | oSrvGlue.registerDebugInfoCallback(self.debugInfoCallback);
|
---|
259 |
|
---|
260 | # Object caches (used by database logic classes).
|
---|
261 | self.ddCaches = dict();
|
---|
262 |
|
---|
263 | def isAutoCommitting(self):
|
---|
264 | """ Work around missing autocommit attribute in older versions."""
|
---|
265 | return getattr(self._oConn, 'autocommit', False);
|
---|
266 |
|
---|
267 | def close(self):
|
---|
268 | """
|
---|
269 | Closes the connection and renders all cursors useless.
|
---|
270 | """
|
---|
271 | if self._oCursor is not None:
|
---|
272 | self._oCursor.close();
|
---|
273 | self._oCursor = None;
|
---|
274 |
|
---|
275 | if self._oConn is not None:
|
---|
276 | self._oConn.close();
|
---|
277 | self._oConn = None;
|
---|
278 |
|
---|
279 | if self._oExplainCursor is not None:
|
---|
280 | self._oExplainCursor.close();
|
---|
281 | self._oExplainCursor = None;
|
---|
282 |
|
---|
283 | if self._oExplainConn is not None:
|
---|
284 | self._oExplainConn.close();
|
---|
285 | self._oExplainConn = None;
|
---|
286 |
|
---|
287 |
|
---|
288 | def _startedTransaction(self):
|
---|
289 | """
|
---|
290 | Called to work the _fTransaction and related variables when starting
|
---|
291 | a transaction.
|
---|
292 | """
|
---|
293 | self._fTransaction = True;
|
---|
294 | self._tsCurrent = None;
|
---|
295 | self._tsCurrentMinusOne = None;
|
---|
296 | return None;
|
---|
297 |
|
---|
298 | def _endedTransaction(self):
|
---|
299 | """
|
---|
300 | Called to work the _fTransaction and related variables when ending
|
---|
301 | a transaction.
|
---|
302 | """
|
---|
303 | self._fTransaction = False;
|
---|
304 | self._tsCurrent = None;
|
---|
305 | self._tsCurrentMinusOne = None;
|
---|
306 | return None;
|
---|
307 |
|
---|
308 | def begin(self):
|
---|
309 | """
|
---|
310 | Currently just for marking where a transaction starts in the code.
|
---|
311 | """
|
---|
312 | assert self._oConn is not None;
|
---|
313 | assert self.isAutoCommitting() is False;
|
---|
314 | self._aoTraceBack.append([utils.timestampNano(), 'START TRANSACTION', 0, 0, utils.getCallerName(), None]);
|
---|
315 | self._startedTransaction();
|
---|
316 | return True;
|
---|
317 |
|
---|
318 | def commit(self, sCallerName = None):
|
---|
319 | """ Wrapper around Psycopg2.connection.commit."""
|
---|
320 | assert self._fTransaction is True;
|
---|
321 |
|
---|
322 | nsStart = utils.timestampNano();
|
---|
323 | oRc = self._oConn.commit();
|
---|
324 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
325 |
|
---|
326 | if sCallerName is None:
|
---|
327 | sCallerName = utils.getCallerName();
|
---|
328 | self._aoTraceBack.append([nsStart, 'COMMIT', cNsElapsed, 0, sCallerName, None]);
|
---|
329 | self._endedTransaction();
|
---|
330 | return oRc;
|
---|
331 |
|
---|
332 | def maybeCommit(self, fCommit):
|
---|
333 | """
|
---|
334 | Commits if fCommit is True.
|
---|
335 | Returns True if committed, False if not.
|
---|
336 | """
|
---|
337 | if fCommit is True:
|
---|
338 | self.commit(utils.getCallerName());
|
---|
339 | return True;
|
---|
340 | return False;
|
---|
341 |
|
---|
342 | def rollback(self):
|
---|
343 | """ Wrapper around Psycopg2.connection.rollback."""
|
---|
344 | nsStart = utils.timestampNano();
|
---|
345 | oRc = self._oConn.rollback();
|
---|
346 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
347 |
|
---|
348 | self._aoTraceBack.append([nsStart, 'ROLLBACK', cNsElapsed, 0, utils.getCallerName(), None]);
|
---|
349 | self._endedTransaction();
|
---|
350 | return oRc;
|
---|
351 |
|
---|
352 | #
|
---|
353 | # Internal cursor workers.
|
---|
354 | #
|
---|
355 |
|
---|
356 | def executeInternal(self, oCursor, sOperation, aoArgs, sCallerName):
|
---|
357 | """
|
---|
358 | Execute a query or command.
|
---|
359 |
|
---|
360 | Mostly a wrapper around the psycopg2 cursor method with the same name,
|
---|
361 | but collect data for traceback.
|
---|
362 | """
|
---|
363 | if aoArgs is not None:
|
---|
364 | sBound = oCursor.mogrify(unicode(sOperation), aoArgs);
|
---|
365 | elif sOperation.find('%') < 0:
|
---|
366 | sBound = oCursor.mogrify(unicode(sOperation), list());
|
---|
367 | else:
|
---|
368 | sBound = unicode(sOperation);
|
---|
369 |
|
---|
370 | if sys.version_info[0] >= 3 and not isinstance(sBound, str):
|
---|
371 | sBound = sBound.decode('utf-8'); # pylint: disable=redefined-variable-type
|
---|
372 |
|
---|
373 | aasExplain = None;
|
---|
374 | if self._oExplainCursor is not None and not sBound.startswith('DROP'):
|
---|
375 | try:
|
---|
376 | if config.g_kfWebUiSqlTraceExplainTiming:
|
---|
377 | self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE, TIMING) ' + sBound);
|
---|
378 | else:
|
---|
379 | self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE) ' + sBound);
|
---|
380 | except Exception as oXcpt:
|
---|
381 | aasExplain = [ ['Explain exception: '], [str(oXcpt)]];
|
---|
382 | try: self._oExplainConn.rollback();
|
---|
383 | except: pass;
|
---|
384 | else:
|
---|
385 | aasExplain = self._oExplainCursor.fetchall();
|
---|
386 |
|
---|
387 | nsStart = utils.timestampNano();
|
---|
388 | try:
|
---|
389 | oRc = oCursor.execute(sBound);
|
---|
390 | except Exception as oXcpt:
|
---|
391 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
392 | self._aoTraceBack.append([nsStart, 'oXcpt=%s; Statement: %s' % (oXcpt, sBound), cNsElapsed, 0, sCallerName, None]);
|
---|
393 | if self._fnDPrint is not None:
|
---|
394 | self._fnDPrint('db::execute %u ns, caller %s: oXcpt=%s; Statement: %s'
|
---|
395 | % (cNsElapsed, sCallerName, oXcpt, sBound));
|
---|
396 | raise;
|
---|
397 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
398 |
|
---|
399 | if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ.
|
---|
400 | self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]);
|
---|
401 | self._startedTransaction();
|
---|
402 | self._aoTraceBack.append([nsStart, sBound, cNsElapsed, oCursor.rowcount, sCallerName, aasExplain]);
|
---|
403 | if self._fnDPrint is not None:
|
---|
404 | self._fnDPrint('db::execute %u ns, caller %s: "\n%s"' % (cNsElapsed, sCallerName, sBound));
|
---|
405 | if self.isAutoCommitting():
|
---|
406 | self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, None]);
|
---|
407 |
|
---|
408 | return oRc;
|
---|
409 |
|
---|
410 | def callProcInternal(self, oCursor, sProcedure, aoArgs, sCallerName):
|
---|
411 | """
|
---|
412 | Call a stored procedure.
|
---|
413 |
|
---|
414 | Mostly a wrapper around the psycopg2 cursor method 'callproc', but
|
---|
415 | collect data for traceback.
|
---|
416 | """
|
---|
417 | if aoArgs is None:
|
---|
418 | aoArgs = list();
|
---|
419 |
|
---|
420 | nsStart = utils.timestampNano();
|
---|
421 | try:
|
---|
422 | oRc = oCursor.callproc(sProcedure, aoArgs);
|
---|
423 | except Exception as oXcpt:
|
---|
424 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
425 | self._aoTraceBack.append([nsStart, 'oXcpt=%s; Calling: %s(%s)' % (oXcpt, sProcedure, aoArgs),
|
---|
426 | cNsElapsed, 0, sCallerName, None]);
|
---|
427 | if self._fnDPrint is not None:
|
---|
428 | self._fnDPrint('db::callproc %u ns, caller %s: oXcpt=%s; Calling: %s(%s)'
|
---|
429 | % (cNsElapsed, sCallerName, oXcpt, sProcedure, aoArgs));
|
---|
430 | raise;
|
---|
431 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
432 |
|
---|
433 | if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ.
|
---|
434 | self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]);
|
---|
435 | self._startedTransaction();
|
---|
436 | self._aoTraceBack.append([nsStart, '%s(%s)' % (sProcedure, aoArgs), cNsElapsed, oCursor.rowcount, sCallerName, None]);
|
---|
437 | if self._fnDPrint is not None:
|
---|
438 | self._fnDPrint('db::callproc %u ns, caller %s: "%s(%s)"' % (cNsElapsed, sCallerName, sProcedure, aoArgs));
|
---|
439 | if self.isAutoCommitting():
|
---|
440 | self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, sCallerName, None]);
|
---|
441 |
|
---|
442 | return oRc;
|
---|
443 |
|
---|
444 | def insertListInternal(self, oCursor, sInsertSql, aoList, fnEntryFmt, sCallerName):
|
---|
445 | """
|
---|
446 | Optimizes the insertion of a list of values.
|
---|
447 | """
|
---|
448 | oRc = None;
|
---|
449 | asValues = [];
|
---|
450 | for aoEntry in aoList:
|
---|
451 | asValues.append(fnEntryFmt(aoEntry));
|
---|
452 | if len(asValues) > 256:
|
---|
453 | oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName);
|
---|
454 | asValues = [];
|
---|
455 | if asValues:
|
---|
456 | oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName);
|
---|
457 | return oRc
|
---|
458 |
|
---|
459 | def _fetchOne(self, oCursor):
|
---|
460 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
461 | oRow = oCursor.fetchone()
|
---|
462 | if self._fnDPrint is not None:
|
---|
463 | self._fnDPrint('db:fetchOne returns: %s' % (oRow,));
|
---|
464 | return oRow;
|
---|
465 |
|
---|
466 | def _fetchMany(self, oCursor, cRows):
|
---|
467 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
468 | return oCursor.fetchmany(cRows if cRows is not None else oCursor.arraysize);
|
---|
469 |
|
---|
470 | def _fetchAll(self, oCursor):
|
---|
471 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
472 | return oCursor.fetchall()
|
---|
473 |
|
---|
474 | def _getRowCountWorker(self, oCursor):
|
---|
475 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
476 | return oCursor.rowcount;
|
---|
477 |
|
---|
478 |
|
---|
479 | #
|
---|
480 | # Default cursor access.
|
---|
481 | #
|
---|
482 |
|
---|
483 | def execute(self, sOperation, aoArgs = None):
|
---|
484 | """
|
---|
485 | Execute a query or command.
|
---|
486 |
|
---|
487 | Mostly a wrapper around the psycopg2 cursor method with the same name,
|
---|
488 | but collect data for traceback.
|
---|
489 | """
|
---|
490 | return self.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName());
|
---|
491 |
|
---|
492 | def callProc(self, sProcedure, aoArgs = None):
|
---|
493 | """
|
---|
494 | Call a stored procedure.
|
---|
495 |
|
---|
496 | Mostly a wrapper around the psycopg2 cursor method 'callproc', but
|
---|
497 | collect data for traceback.
|
---|
498 | """
|
---|
499 | return self.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName());
|
---|
500 |
|
---|
501 | def insertList(self, sInsertSql, aoList, fnEntryFmt):
|
---|
502 | """
|
---|
503 | Optimizes the insertion of a list of values.
|
---|
504 | """
|
---|
505 | return self.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName());
|
---|
506 |
|
---|
507 | def fetchOne(self):
|
---|
508 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
509 | return self._oCursor.fetchone();
|
---|
510 |
|
---|
511 | def fetchMany(self, cRows = None):
|
---|
512 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
513 | return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize);
|
---|
514 |
|
---|
515 | def fetchAll(self):
|
---|
516 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
517 | return self._oCursor.fetchall();
|
---|
518 |
|
---|
519 | def getRowCount(self):
|
---|
520 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
521 | return self._oCursor.rowcount;
|
---|
522 |
|
---|
523 | def formatBindArgs(self, sStatement, aoArgs):
|
---|
524 | """Wrapper around Psycopg2.cursor.mogrify."""
|
---|
525 | oRet = self._oCursor.mogrify(sStatement, aoArgs);
|
---|
526 | if sys.version_info[0] >= 3 and not isinstance(oRet, str):
|
---|
527 | oRet = oRet.decode('utf-8');
|
---|
528 | return oRet;
|
---|
529 |
|
---|
530 | def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192):
|
---|
531 | """ Wrapper around Psycopg2.cursor.copy_expert. """
|
---|
532 | return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf);
|
---|
533 |
|
---|
534 | def getCurrentTimestamps(self):
|
---|
535 | """
|
---|
536 | Returns the current timestamp and the current timestamp minus one tick.
|
---|
537 | This will start a transaction if necessary.
|
---|
538 | """
|
---|
539 | if self._tsCurrent is None:
|
---|
540 | self.execute('SELECT CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - INTERVAL \'1 microsecond\'');
|
---|
541 | (self._tsCurrent, self._tsCurrentMinusOne) = self.fetchOne();
|
---|
542 | return (self._tsCurrent, self._tsCurrentMinusOne);
|
---|
543 |
|
---|
544 | def getCurrentTimestamp(self):
|
---|
545 | """
|
---|
546 | Returns the current timestamp.
|
---|
547 | This will start a transaction if necessary.
|
---|
548 | """
|
---|
549 | if self._tsCurrent is None:
|
---|
550 | self.getCurrentTimestamps();
|
---|
551 | return self._tsCurrent;
|
---|
552 |
|
---|
553 | def getCurrentTimestampMinusOne(self):
|
---|
554 | """
|
---|
555 | Returns the current timestamp minus one tick.
|
---|
556 | This will start a transaction if necessary.
|
---|
557 | """
|
---|
558 | if self._tsCurrentMinusOne is None:
|
---|
559 | self.getCurrentTimestamps();
|
---|
560 | return self._tsCurrentMinusOne;
|
---|
561 |
|
---|
562 |
|
---|
563 | #
|
---|
564 | # Additional cursors.
|
---|
565 | #
|
---|
566 | def openCursor(self):
|
---|
567 | """
|
---|
568 | Opens a new cursor (TMDatabaseCursor).
|
---|
569 | """
|
---|
570 | oCursor = self._oConn.cursor();
|
---|
571 | return TMDatabaseCursor(self, oCursor);
|
---|
572 |
|
---|
573 | #
|
---|
574 | # Cache support.
|
---|
575 | #
|
---|
576 | def getCache(self, sType):
|
---|
577 | """ Returns the cache dictionary for this data type. """
|
---|
578 | dRet = self.ddCaches.get(sType, None);
|
---|
579 | if dRet is None:
|
---|
580 | dRet = dict();
|
---|
581 | self.ddCaches[sType] = dRet;
|
---|
582 | return dRet;
|
---|
583 |
|
---|
584 |
|
---|
585 | #
|
---|
586 | # Utilities.
|
---|
587 | #
|
---|
588 |
|
---|
589 | @staticmethod
|
---|
590 | def isTsInfinity(tsValue):
|
---|
591 | """ Checks if tsValue is an infinity timestamp. """
|
---|
592 | return isDbTimestampInfinity(tsValue);
|
---|
593 |
|
---|
594 | #
|
---|
595 | # Error stuff.
|
---|
596 | #
|
---|
597 | def integrityException(self, sMessage):
|
---|
598 | """
|
---|
599 | Database integrity reporter and exception factory.
|
---|
600 | Returns an TMDatabaseIntegrityException which the caller can raise.
|
---|
601 | """
|
---|
602 | ## @todo Create a new database connection and log the issue in the SystemLog table.
|
---|
603 | ## Alternatively, rollback whatever is going on and do it using the current one.
|
---|
604 | return TMDatabaseIntegrityException(sMessage);
|
---|
605 |
|
---|
606 |
|
---|
607 | #
|
---|
608 | # Debugging.
|
---|
609 | #
|
---|
610 |
|
---|
611 | def dprint(self, sText):
|
---|
612 | """
|
---|
613 | Debug output.
|
---|
614 | """
|
---|
615 | if not self._fnDPrint:
|
---|
616 | return False;
|
---|
617 | self._fnDPrint(sText);
|
---|
618 | return True;
|
---|
619 |
|
---|
620 | def debugHtmlReport(self, tsStart = 0):
|
---|
621 | """
|
---|
622 | Used to get a SQL activity dump as HTML, usually for WuiBase._sDebug.
|
---|
623 | """
|
---|
624 | cNsElapsed = 0;
|
---|
625 | for aEntry in self._aoTraceBack:
|
---|
626 | cNsElapsed += aEntry[2];
|
---|
627 |
|
---|
628 | sDebug = '<h3>SQL Debug Log (total time %s ns):</h3>\n' \
|
---|
629 | '<table class="tmsqltable">\n' \
|
---|
630 | ' <tr>\n' \
|
---|
631 | ' <th>No.</th>\n' \
|
---|
632 | ' <th>Timestamp (ns)</th>\n' \
|
---|
633 | ' <th>Elapsed (ns)</th>\n' \
|
---|
634 | ' <th>Rows Returned</th>\n' \
|
---|
635 | ' <th>Command</th>\n' \
|
---|
636 | ' <th>Caller</th>\n' \
|
---|
637 | ' </tr>\n' \
|
---|
638 | % (utils.formatNumber(cNsElapsed, ' '),);
|
---|
639 |
|
---|
640 | iEntry = 0;
|
---|
641 | for aEntry in self._aoTraceBack:
|
---|
642 | iEntry += 1;
|
---|
643 | sDebug += ' <tr>\n' \
|
---|
644 | ' <td>%s</td>\n' \
|
---|
645 | ' <td>%s</td>\n' \
|
---|
646 | ' <td>%s</td>\n' \
|
---|
647 | ' <td>%s</td>\n' \
|
---|
648 | ' <td><pre>%s</pre></td>\n' \
|
---|
649 | ' <td>%s</td>\n' \
|
---|
650 | ' </tr>\n' \
|
---|
651 | % (iEntry,
|
---|
652 | utils.formatNumber(aEntry[0] - tsStart, ' '),
|
---|
653 | utils.formatNumber(aEntry[2], ' '),
|
---|
654 | utils.formatNumber(aEntry[3], ' '),
|
---|
655 | webutils.escapeElem(aEntry[1]),
|
---|
656 | webutils.escapeElem(aEntry[4]),
|
---|
657 | );
|
---|
658 | if aEntry[5] is not None:
|
---|
659 | sDebug += ' <tr>\n' \
|
---|
660 | ' <td colspan="6"><pre style="white-space: pre-wrap;">%s</pre></td>\n' \
|
---|
661 | ' </tr>\n' \
|
---|
662 | % (webutils.escapeElem('\n'.join([aoRow[0] for aoRow in aEntry[5]])),);
|
---|
663 |
|
---|
664 | sDebug += '</table>';
|
---|
665 | return sDebug;
|
---|
666 |
|
---|
667 | def debugTextReport(self, tsStart = 0):
|
---|
668 | """
|
---|
669 | Used to get a SQL activity dump as text.
|
---|
670 | """
|
---|
671 | cNsElapsed = 0;
|
---|
672 | for aEntry in self._aoTraceBack:
|
---|
673 | cNsElapsed += aEntry[2];
|
---|
674 |
|
---|
675 | sHdr = 'SQL Debug Log (total time %s ns)' % (utils.formatNumber(cNsElapsed),);
|
---|
676 | sDebug = sHdr + '\n' + '-' * len(sHdr) + '\n';
|
---|
677 |
|
---|
678 | iEntry = 0;
|
---|
679 | for aEntry in self._aoTraceBack:
|
---|
680 | iEntry += 1;
|
---|
681 | sHdr = 'Query #%s Timestamp: %s ns Elapsed: %s ns Rows: %s Caller: %s' \
|
---|
682 | % ( iEntry,
|
---|
683 | utils.formatNumber(aEntry[0] - tsStart),
|
---|
684 | utils.formatNumber(aEntry[2]),
|
---|
685 | utils.formatNumber(aEntry[3]),
|
---|
686 | aEntry[4], );
|
---|
687 | sDebug += '\n' + sHdr + '\n' + '-' * len(sHdr) + '\n';
|
---|
688 |
|
---|
689 | sDebug += aEntry[1];
|
---|
690 | if sDebug[-1] != '\n':
|
---|
691 | sDebug += '\n';
|
---|
692 |
|
---|
693 | if aEntry[5] is not None:
|
---|
694 | sDebug += 'Explain:\n' \
|
---|
695 | ' %s\n' \
|
---|
696 | % ( '\n'.join([aoRow[0] for aoRow in aEntry[5]]),);
|
---|
697 |
|
---|
698 | return sDebug;
|
---|
699 |
|
---|
700 | def debugInfoCallback(self, oGlue, fHtml):
|
---|
701 | """ Called back by the glue code on error. """
|
---|
702 | oGlue.write('\n');
|
---|
703 | if not fHtml: oGlue.write(self.debugTextReport());
|
---|
704 | else: oGlue.write(self.debugHtmlReport());
|
---|
705 | oGlue.write('\n');
|
---|
706 | return True;
|
---|
707 |
|
---|
708 | def debugEnableExplain(self):
|
---|
709 | """ Enabled explain. """
|
---|
710 | if self._oExplainConn is None:
|
---|
711 | dArgs = \
|
---|
712 | { \
|
---|
713 | 'database': config.g_ksDatabaseName,
|
---|
714 | 'user': config.g_ksDatabaseUser,
|
---|
715 | 'password': config.g_ksDatabasePassword,
|
---|
716 | # 'application_name': sAppName, - Darn stale debian! :/
|
---|
717 | };
|
---|
718 | if config.g_ksDatabaseAddress is not None:
|
---|
719 | dArgs['host'] = config.g_ksDatabaseAddress;
|
---|
720 | if config.g_ksDatabasePort is not None:
|
---|
721 | dArgs['port'] = config.g_ksDatabasePort;
|
---|
722 | self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=star-args
|
---|
723 | self._oExplainCursor = self._oExplainConn.cursor();
|
---|
724 | return True;
|
---|
725 |
|
---|
726 | def debugDisableExplain(self):
|
---|
727 | """ Disables explain. """
|
---|
728 | self._oExplainCursor = None;
|
---|
729 | self._oExplainConn = None
|
---|
730 | return True;
|
---|
731 |
|
---|
732 | def debugIsExplainEnabled(self):
|
---|
733 | """ Check if explaining of SQL statements is enabled. """
|
---|
734 | return self._oExplainConn is not None;
|
---|
735 |
|
---|