1 | # -*- coding: utf-8 -*-
|
---|
2 | # $Id: db.py 65980 2017-03-07 13:00:36Z vboxsync $
|
---|
3 |
|
---|
4 | """
|
---|
5 | Test Manager - Database Interface.
|
---|
6 | """
|
---|
7 |
|
---|
8 | __copyright__ = \
|
---|
9 | """
|
---|
10 | Copyright (C) 2012-2016 Oracle Corporation
|
---|
11 |
|
---|
12 | This file is part of VirtualBox Open Source Edition (OSE), as
|
---|
13 | available from http://www.virtualbox.org. This file is free software;
|
---|
14 | you can redistribute it and/or modify it under the terms of the GNU
|
---|
15 | General Public License (GPL) as published by the Free Software
|
---|
16 | Foundation, in version 2 as it comes in the "COPYING" file of the
|
---|
17 | VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
---|
18 | hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
---|
19 |
|
---|
20 | The contents of this file may alternatively be used under the terms
|
---|
21 | of the Common Development and Distribution License Version 1.0
|
---|
22 | (CDDL) only, as it comes in the "COPYING.CDDL" file of the
|
---|
23 | VirtualBox OSE distribution, in which case the provisions of the
|
---|
24 | CDDL are applicable instead of those of the GPL.
|
---|
25 |
|
---|
26 | You may elect to license modified versions of this file under the
|
---|
27 | terms and conditions of either the GPL or the CDDL or both.
|
---|
28 | """
|
---|
29 | __version__ = "$Revision: 65980 $"
|
---|
30 |
|
---|
31 |
|
---|
32 | # Standard python imports.
|
---|
33 | import datetime;
|
---|
34 | import os;
|
---|
35 | import sys;
|
---|
36 | import psycopg2; # pylint: disable=import-error
|
---|
37 | import psycopg2.extensions; # pylint: disable=import-error
|
---|
38 |
|
---|
39 | # Validation Kit imports.
|
---|
40 | from common import utils, webutils;
|
---|
41 | from testmanager import config;
|
---|
42 |
|
---|
43 | # Fix psycho unicode handling in psycopg2 with python 2.x.
|
---|
44 | if sys.version_info[0] < 3:
|
---|
45 | psycopg2.extensions.register_type(psycopg2.extensions.UNICODE);
|
---|
46 | psycopg2.extensions.register_type(psycopg2.extensions.UNICODEARRAY);
|
---|
47 |
|
---|
48 |
|
---|
49 | def isDbTimestampInfinity(tsValue):
|
---|
50 | """
|
---|
51 | Checks if tsValue is an infinity timestamp.
|
---|
52 | """
|
---|
53 | ## @todo improve this test...
|
---|
54 | return tsValue.year >= 9999;
|
---|
55 |
|
---|
56 | def isDbTimestamp(oValue):
|
---|
57 | """
|
---|
58 | Checks if oValue is a DB timestamp object.
|
---|
59 | """
|
---|
60 | if isinstance(oValue, datetime.datetime):
|
---|
61 | return True;
|
---|
62 | if utils.isString(oValue):
|
---|
63 | ## @todo detect strings as well.
|
---|
64 | return False;
|
---|
65 | return getattr(oValue, 'pydatetime', None) != None;
|
---|
66 |
|
---|
67 | def dbTimestampToDatetime(oValue):
|
---|
68 | """
|
---|
69 | Converts a database timestamp to a datetime instance.
|
---|
70 | """
|
---|
71 | if isinstance(oValue, datetime.datetime):
|
---|
72 | return oValue;
|
---|
73 | if utils.isString(oValue):
|
---|
74 | raise Exception('TODO');
|
---|
75 | return oValue.pydatetime();
|
---|
76 |
|
---|
77 | def dbTimestampToZuluDatetime(oValue):
|
---|
78 | """
|
---|
79 | Converts a database timestamp to a zulu datetime instance.
|
---|
80 | """
|
---|
81 | tsValue = dbTimestampToDatetime(oValue);
|
---|
82 |
|
---|
83 | class UTC(datetime.tzinfo):
|
---|
84 | """UTC TZ Info Class"""
|
---|
85 | def utcoffset(self, _):
|
---|
86 | return datetime.timedelta(0);
|
---|
87 | def tzname(self, _):
|
---|
88 | return "UTC";
|
---|
89 | def dst(self, _):
|
---|
90 | return datetime.timedelta(0);
|
---|
91 | if tsValue.tzinfo is not None:
|
---|
92 | tsValue = tsValue.astimezone(UTC());
|
---|
93 | else:
|
---|
94 | tsValue = tsValue.replace(tzinfo=UTC());
|
---|
95 | return tsValue;
|
---|
96 |
|
---|
97 | def dbTimestampPythonNow():
|
---|
98 | """
|
---|
99 | Gets the current python timestamp in a database compatible way.
|
---|
100 | """
|
---|
101 | return dbTimestampToZuluDatetime(datetime.datetime.utcnow());
|
---|
102 |
|
---|
103 | def dbTimestampMinusOneTick(oValue):
|
---|
104 | """
|
---|
105 | Returns a new timestamp that's one tick before the given one.
|
---|
106 | """
|
---|
107 | oValue = dbTimestampToZuluDatetime(oValue);
|
---|
108 | return oValue - datetime.timedelta(microseconds = 1);
|
---|
109 |
|
---|
110 | def isDbInterval(oValue):
|
---|
111 | """
|
---|
112 | Checks if oValue is a DB interval object.
|
---|
113 | """
|
---|
114 | if isinstance(oValue, datetime.timedelta):
|
---|
115 | return True;
|
---|
116 | return False;
|
---|
117 |
|
---|
118 |
|
---|
119 | class TMDatabaseIntegrityException(Exception):
|
---|
120 | """
|
---|
121 | Herolds a database integrity error up the callstack.
|
---|
122 |
|
---|
123 | Do NOT use directly, only thru TMDatabaseConnection.integrityException.
|
---|
124 | Otherwise, we won't be able to log the issue.
|
---|
125 | """
|
---|
126 | pass;
|
---|
127 |
|
---|
128 |
|
---|
129 | class TMDatabaseCursor(object):
|
---|
130 | """ Cursor wrapper class. """
|
---|
131 |
|
---|
132 | def __init__(self, oDb, oCursor):
|
---|
133 | self._oDb = oDb;
|
---|
134 | self._oCursor = oCursor;
|
---|
135 |
|
---|
136 | def execute(self, sOperation, aoArgs = None):
|
---|
137 | """ See TMDatabaseConnection.execute()"""
|
---|
138 | return self._oDb.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName());
|
---|
139 |
|
---|
140 | def callProc(self, sProcedure, aoArgs = None):
|
---|
141 | """ See TMDatabaseConnection.callProc()"""
|
---|
142 | return self._oDb.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName());
|
---|
143 |
|
---|
144 | def insertList(self, sInsertSql, aoList, fnEntryFmt):
|
---|
145 | """ See TMDatabaseConnection.insertList. """
|
---|
146 | return self._oDb.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName());
|
---|
147 |
|
---|
148 | def fetchOne(self):
|
---|
149 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
150 | return self._oCursor.fetchone();
|
---|
151 |
|
---|
152 | def fetchMany(self, cRows = None):
|
---|
153 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
154 | return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize);
|
---|
155 |
|
---|
156 | def fetchAll(self):
|
---|
157 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
158 | return self._oCursor.fetchall();
|
---|
159 |
|
---|
160 | def getRowCount(self):
|
---|
161 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
162 | return self._oCursor.rowcount;
|
---|
163 |
|
---|
164 | def formatBindArgs(self, sStatement, aoArgs):
|
---|
165 | """Wrapper around Psycopg2.cursor.mogrify."""
|
---|
166 | oRet = self._oCursor.mogrify(sStatement, aoArgs);
|
---|
167 | if sys.version_info[0] >= 3 and not isinstance(oRet, str):
|
---|
168 | oRet = oRet.decode('utf-8');
|
---|
169 | return oRet;
|
---|
170 |
|
---|
171 | def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192):
|
---|
172 | """ See TMDatabaseConnection.copyExpert()"""
|
---|
173 | return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf);
|
---|
174 |
|
---|
175 | @staticmethod
|
---|
176 | def isTsInfinity(tsValue):
|
---|
177 | """ Checks if tsValue is an infinity timestamp. """
|
---|
178 | return isDbTimestampInfinity(tsValue);
|
---|
179 |
|
---|
180 |
|
---|
181 | class TMDatabaseConnection(object):
|
---|
182 | """
|
---|
183 | Test Manager Database Access class.
|
---|
184 |
|
---|
185 | This class contains no logic, just raw access abstraction and utilities,
|
---|
186 | as well as some debug help and some statistics.
|
---|
187 | """
|
---|
188 |
|
---|
189 | def __init__(self, fnDPrint = None, oSrvGlue = None):
|
---|
190 | """
|
---|
191 | Database connection wrapper.
|
---|
192 | The fnDPrint is for debug logging of all database activity.
|
---|
193 |
|
---|
194 | Raises an exception on failure.
|
---|
195 | """
|
---|
196 |
|
---|
197 | sAppName = '%s-%s' % (os.getpid(), os.path.basename(sys.argv[0]),)
|
---|
198 | if len(sAppName) >= 64:
|
---|
199 | sAppName = sAppName[:64];
|
---|
200 | os.environ['PGAPPNAME'] = sAppName;
|
---|
201 |
|
---|
202 | dArgs = \
|
---|
203 | { \
|
---|
204 | 'database': config.g_ksDatabaseName,
|
---|
205 | 'user': config.g_ksDatabaseUser,
|
---|
206 | 'password': config.g_ksDatabasePassword,
|
---|
207 | # 'application_name': sAppName, - Darn stale debian! :/
|
---|
208 | };
|
---|
209 | if config.g_ksDatabaseAddress is not None:
|
---|
210 | dArgs['host'] = config.g_ksDatabaseAddress;
|
---|
211 | if config.g_ksDatabasePort is not None:
|
---|
212 | dArgs['port'] = config.g_ksDatabasePort;
|
---|
213 | self._oConn = psycopg2.connect(**dArgs); # pylint: disable=W0142
|
---|
214 | self._oConn.set_client_encoding('UTF-8');
|
---|
215 | self._oCursor = self._oConn.cursor();
|
---|
216 | self._oExplainConn = None;
|
---|
217 | self._oExplainCursor = None;
|
---|
218 | if config.g_kfWebUiSqlTraceExplain and config.g_kfWebUiSqlTrace:
|
---|
219 | self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=W0142
|
---|
220 | self._oExplainConn.set_client_encoding('UTF-8');
|
---|
221 | self._oExplainCursor = self._oExplainConn.cursor();
|
---|
222 | self._fTransaction = False;
|
---|
223 | self._tsCurrent = None;
|
---|
224 | self._tsCurrentMinusOne = None;
|
---|
225 |
|
---|
226 | assert self.isAutoCommitting() is False;
|
---|
227 |
|
---|
228 | # Debug and introspection.
|
---|
229 | self._fnDPrint = fnDPrint;
|
---|
230 | self._aoTraceBack = [];
|
---|
231 |
|
---|
232 | # Exception class handles.
|
---|
233 | self.oXcptError = psycopg2.Error;
|
---|
234 |
|
---|
235 | if oSrvGlue is not None:
|
---|
236 | oSrvGlue.registerDebugInfoCallback(self.debugInfoCallback);
|
---|
237 |
|
---|
238 | # Object caches (used by database logic classes).
|
---|
239 | self.ddCaches = dict();
|
---|
240 |
|
---|
241 | def isAutoCommitting(self):
|
---|
242 | """ Work around missing autocommit attribute in older versions."""
|
---|
243 | return getattr(self._oConn, 'autocommit', False);
|
---|
244 |
|
---|
245 | def close(self):
|
---|
246 | """
|
---|
247 | Closes the connection and renders all cursors useless.
|
---|
248 | """
|
---|
249 | if self._oCursor is not None:
|
---|
250 | self._oCursor.close();
|
---|
251 | self._oCursor = None;
|
---|
252 |
|
---|
253 | if self._oConn is not None:
|
---|
254 | self._oConn.close();
|
---|
255 | self._oConn = None;
|
---|
256 |
|
---|
257 | if self._oExplainCursor is not None:
|
---|
258 | self._oExplainCursor.close();
|
---|
259 | self._oExplainCursor = None;
|
---|
260 |
|
---|
261 | if self._oExplainConn is not None:
|
---|
262 | self._oExplainConn.close();
|
---|
263 | self._oExplainConn = None;
|
---|
264 |
|
---|
265 |
|
---|
266 | def _startedTransaction(self):
|
---|
267 | """
|
---|
268 | Called to work the _fTransaction and related variables when starting
|
---|
269 | a transaction.
|
---|
270 | """
|
---|
271 | self._fTransaction = True;
|
---|
272 | self._tsCurrent = None;
|
---|
273 | self._tsCurrentMinusOne = None;
|
---|
274 | return None;
|
---|
275 |
|
---|
276 | def _endedTransaction(self):
|
---|
277 | """
|
---|
278 | Called to work the _fTransaction and related variables when ending
|
---|
279 | a transaction.
|
---|
280 | """
|
---|
281 | self._fTransaction = False;
|
---|
282 | self._tsCurrent = None;
|
---|
283 | self._tsCurrentMinusOne = None;
|
---|
284 | return None;
|
---|
285 |
|
---|
286 | def begin(self):
|
---|
287 | """
|
---|
288 | Currently just for marking where a transaction starts in the code.
|
---|
289 | """
|
---|
290 | assert self._oConn is not None;
|
---|
291 | assert self.isAutoCommitting() is False;
|
---|
292 | self._aoTraceBack.append([utils.timestampNano(), 'START TRANSACTION', 0, 0, utils.getCallerName(), None]);
|
---|
293 | self._startedTransaction();
|
---|
294 | return True;
|
---|
295 |
|
---|
296 | def commit(self, sCallerName = None):
|
---|
297 | """ Wrapper around Psycopg2.connection.commit."""
|
---|
298 | assert self._fTransaction is True;
|
---|
299 |
|
---|
300 | nsStart = utils.timestampNano();
|
---|
301 | oRc = self._oConn.commit();
|
---|
302 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
303 |
|
---|
304 | if sCallerName is None:
|
---|
305 | sCallerName = utils.getCallerName();
|
---|
306 | self._aoTraceBack.append([nsStart, 'COMMIT', cNsElapsed, 0, sCallerName, None]);
|
---|
307 | self._endedTransaction();
|
---|
308 | return oRc;
|
---|
309 |
|
---|
310 | def maybeCommit(self, fCommit):
|
---|
311 | """
|
---|
312 | Commits if fCommit is True.
|
---|
313 | Returns True if committed, False if not.
|
---|
314 | """
|
---|
315 | if fCommit is True:
|
---|
316 | self.commit(utils.getCallerName());
|
---|
317 | return True;
|
---|
318 | return False;
|
---|
319 |
|
---|
320 | def rollback(self):
|
---|
321 | """ Wrapper around Psycopg2.connection.rollback."""
|
---|
322 | nsStart = utils.timestampNano();
|
---|
323 | oRc = self._oConn.rollback();
|
---|
324 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
325 |
|
---|
326 | self._aoTraceBack.append([nsStart, 'ROLLBACK', cNsElapsed, 0, utils.getCallerName(), None]);
|
---|
327 | self._endedTransaction();
|
---|
328 | return oRc;
|
---|
329 |
|
---|
330 | #
|
---|
331 | # Internal cursor workers.
|
---|
332 | #
|
---|
333 |
|
---|
334 | def executeInternal(self, oCursor, sOperation, aoArgs, sCallerName):
|
---|
335 | """
|
---|
336 | Execute a query or command.
|
---|
337 |
|
---|
338 | Mostly a wrapper around the psycopg2 cursor method with the same name,
|
---|
339 | but collect data for traceback.
|
---|
340 | """
|
---|
341 | if aoArgs is not None:
|
---|
342 | sBound = oCursor.mogrify(unicode(sOperation), aoArgs);
|
---|
343 | elif sOperation.find('%') < 0:
|
---|
344 | sBound = oCursor.mogrify(unicode(sOperation), list());
|
---|
345 | else:
|
---|
346 | sBound = unicode(sOperation);
|
---|
347 |
|
---|
348 | if sys.version_info[0] >= 3 and not isinstance(sBound, str):
|
---|
349 | sBound = sBound.decode('utf-8');
|
---|
350 |
|
---|
351 | aasExplain = None;
|
---|
352 | if self._oExplainCursor is not None and not sBound.startswith('DROP'):
|
---|
353 | try:
|
---|
354 | if config.g_kfWebUiSqlTraceExplainTiming:
|
---|
355 | self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE, TIMING) ' + sBound);
|
---|
356 | else:
|
---|
357 | self._oExplainCursor.execute('EXPLAIN (ANALYZE, BUFFERS, COSTS, VERBOSE) ' + sBound);
|
---|
358 | except Exception as oXcpt:
|
---|
359 | aasExplain = [ ['Explain exception: '], [str(oXcpt)]];
|
---|
360 | try: self._oExplainConn.rollback();
|
---|
361 | except: pass;
|
---|
362 | else:
|
---|
363 | aasExplain = self._oExplainCursor.fetchall();
|
---|
364 |
|
---|
365 | nsStart = utils.timestampNano();
|
---|
366 | try:
|
---|
367 | oRc = oCursor.execute(sBound);
|
---|
368 | except Exception as oXcpt:
|
---|
369 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
370 | self._aoTraceBack.append([nsStart, 'oXcpt=%s; Statement: %s' % (oXcpt, sBound), cNsElapsed, 0, sCallerName, None]);
|
---|
371 | if self._fnDPrint is not None:
|
---|
372 | self._fnDPrint('db::execute %u ns, caller %s: oXcpt=%s; Statement: %s'
|
---|
373 | % (cNsElapsed, sCallerName, oXcpt, sBound));
|
---|
374 | raise;
|
---|
375 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
376 |
|
---|
377 | if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ.
|
---|
378 | self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]);
|
---|
379 | self._startedTransaction();
|
---|
380 | self._aoTraceBack.append([nsStart, sBound, cNsElapsed, oCursor.rowcount, sCallerName, aasExplain]);
|
---|
381 | if self._fnDPrint is not None:
|
---|
382 | self._fnDPrint('db::execute %u ns, caller %s: "\n%s"' % (cNsElapsed, sCallerName, sBound));
|
---|
383 | if self.isAutoCommitting():
|
---|
384 | self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, None]);
|
---|
385 |
|
---|
386 | return oRc;
|
---|
387 |
|
---|
388 | def callProcInternal(self, oCursor, sProcedure, aoArgs, sCallerName):
|
---|
389 | """
|
---|
390 | Call a stored procedure.
|
---|
391 |
|
---|
392 | Mostly a wrapper around the psycopg2 cursor method 'callproc', but
|
---|
393 | collect data for traceback.
|
---|
394 | """
|
---|
395 | if aoArgs is None:
|
---|
396 | aoArgs = list();
|
---|
397 |
|
---|
398 | nsStart = utils.timestampNano();
|
---|
399 | try:
|
---|
400 | oRc = oCursor.callproc(sProcedure, aoArgs);
|
---|
401 | except Exception as oXcpt:
|
---|
402 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
403 | self._aoTraceBack.append([nsStart, 'oXcpt=%s; Calling: %s(%s)' % (oXcpt, sProcedure, aoArgs),
|
---|
404 | cNsElapsed, 0, sCallerName, None]);
|
---|
405 | if self._fnDPrint is not None:
|
---|
406 | self._fnDPrint('db::callproc %u ns, caller %s: oXcpt=%s; Calling: %s(%s)'
|
---|
407 | % (cNsElapsed, sCallerName, oXcpt, sProcedure, aoArgs));
|
---|
408 | raise;
|
---|
409 | cNsElapsed = utils.timestampNano() - nsStart;
|
---|
410 |
|
---|
411 | if self._fTransaction is False and not self.isAutoCommitting(): # Even SELECTs starts transactions with psycopg2, see FAQ.
|
---|
412 | self._aoTraceBack.append([nsStart, '[START TRANSACTION]', 0, 0, sCallerName, None]);
|
---|
413 | self._startedTransaction();
|
---|
414 | self._aoTraceBack.append([nsStart, '%s(%s)' % (sProcedure, aoArgs), cNsElapsed, oCursor.rowcount, sCallerName, None]);
|
---|
415 | if self._fnDPrint is not None:
|
---|
416 | self._fnDPrint('db::callproc %u ns, caller %s: "%s(%s)"' % (cNsElapsed, sCallerName, sProcedure, aoArgs));
|
---|
417 | if self.isAutoCommitting():
|
---|
418 | self._aoTraceBack.append([nsStart, '[AUTO COMMIT]', 0, 0, sCallerName, sCallerName, None]);
|
---|
419 |
|
---|
420 | return oRc;
|
---|
421 |
|
---|
422 | def insertListInternal(self, oCursor, sInsertSql, aoList, fnEntryFmt, sCallerName):
|
---|
423 | """
|
---|
424 | Optimizes the insertion of a list of values.
|
---|
425 | """
|
---|
426 | oRc = None;
|
---|
427 | asValues = [];
|
---|
428 | for aoEntry in aoList:
|
---|
429 | asValues.append(fnEntryFmt(aoEntry));
|
---|
430 | if len(asValues) > 256:
|
---|
431 | oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName);
|
---|
432 | asValues = [];
|
---|
433 | if asValues:
|
---|
434 | oRc = self.executeInternal(oCursor, sInsertSql + 'VALUES' + ', '.join(asValues), None, sCallerName);
|
---|
435 | return oRc
|
---|
436 |
|
---|
437 | def _fetchOne(self, oCursor):
|
---|
438 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
439 | oRow = oCursor.fetchone()
|
---|
440 | if self._fnDPrint is not None:
|
---|
441 | self._fnDPrint('db:fetchOne returns: %s' % (oRow,));
|
---|
442 | return oRow;
|
---|
443 |
|
---|
444 | def _fetchMany(self, oCursor, cRows):
|
---|
445 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
446 | return oCursor.fetchmany(cRows if cRows is not None else oCursor.arraysize);
|
---|
447 |
|
---|
448 | def _fetchAll(self, oCursor):
|
---|
449 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
450 | return oCursor.fetchall()
|
---|
451 |
|
---|
452 | def _getRowCountWorker(self, oCursor):
|
---|
453 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
454 | return oCursor.rowcount;
|
---|
455 |
|
---|
456 |
|
---|
457 | #
|
---|
458 | # Default cursor access.
|
---|
459 | #
|
---|
460 |
|
---|
461 | def execute(self, sOperation, aoArgs = None):
|
---|
462 | """
|
---|
463 | Execute a query or command.
|
---|
464 |
|
---|
465 | Mostly a wrapper around the psycopg2 cursor method with the same name,
|
---|
466 | but collect data for traceback.
|
---|
467 | """
|
---|
468 | return self.executeInternal(self._oCursor, sOperation, aoArgs, utils.getCallerName());
|
---|
469 |
|
---|
470 | def callProc(self, sProcedure, aoArgs = None):
|
---|
471 | """
|
---|
472 | Call a stored procedure.
|
---|
473 |
|
---|
474 | Mostly a wrapper around the psycopg2 cursor method 'callproc', but
|
---|
475 | collect data for traceback.
|
---|
476 | """
|
---|
477 | return self.callProcInternal(self._oCursor, sProcedure, aoArgs, utils.getCallerName());
|
---|
478 |
|
---|
479 | def insertList(self, sInsertSql, aoList, fnEntryFmt):
|
---|
480 | """
|
---|
481 | Optimizes the insertion of a list of values.
|
---|
482 | """
|
---|
483 | return self.insertListInternal(self._oCursor, sInsertSql, aoList, fnEntryFmt, utils.getCallerName());
|
---|
484 |
|
---|
485 | def fetchOne(self):
|
---|
486 | """Wrapper around Psycopg2.cursor.fetchone."""
|
---|
487 | return self._oCursor.fetchone();
|
---|
488 |
|
---|
489 | def fetchMany(self, cRows = None):
|
---|
490 | """Wrapper around Psycopg2.cursor.fetchmany."""
|
---|
491 | return self._oCursor.fetchmany(cRows if cRows is not None else self._oCursor.arraysize);
|
---|
492 |
|
---|
493 | def fetchAll(self):
|
---|
494 | """Wrapper around Psycopg2.cursor.fetchall."""
|
---|
495 | return self._oCursor.fetchall();
|
---|
496 |
|
---|
497 | def getRowCount(self):
|
---|
498 | """Wrapper around Psycopg2.cursor.rowcount."""
|
---|
499 | return self._oCursor.rowcount;
|
---|
500 |
|
---|
501 | def formatBindArgs(self, sStatement, aoArgs):
|
---|
502 | """Wrapper around Psycopg2.cursor.mogrify."""
|
---|
503 | oRet = self._oCursor.mogrify(sStatement, aoArgs);
|
---|
504 | if sys.version_info[0] >= 3 and not isinstance(oRet, str):
|
---|
505 | oRet = oRet.decode('utf-8');
|
---|
506 | return oRet;
|
---|
507 |
|
---|
508 | def copyExpert(self, sSqlCopyStmt, oFile, cbBuf = 8192):
|
---|
509 | """ Wrapper around Psycopg2.cursor.copy_expert. """
|
---|
510 | return self._oCursor.copy_expert(sSqlCopyStmt, oFile, cbBuf);
|
---|
511 |
|
---|
512 | def getCurrentTimestamps(self):
|
---|
513 | """
|
---|
514 | Returns the current timestamp and the current timestamp minus one tick.
|
---|
515 | This will start a transaction if necessary.
|
---|
516 | """
|
---|
517 | if self._tsCurrent is None:
|
---|
518 | self.execute('SELECT CURRENT_TIMESTAMP, CURRENT_TIMESTAMP - INTERVAL \'1 microsecond\'');
|
---|
519 | (self._tsCurrent, self._tsCurrentMinusOne) = self.fetchOne();
|
---|
520 | return (self._tsCurrent, self._tsCurrentMinusOne);
|
---|
521 |
|
---|
522 | def getCurrentTimestamp(self):
|
---|
523 | """
|
---|
524 | Returns the current timestamp.
|
---|
525 | This will start a transaction if necessary.
|
---|
526 | """
|
---|
527 | if self._tsCurrent is None:
|
---|
528 | self.getCurrentTimestamps();
|
---|
529 | return self._tsCurrent;
|
---|
530 |
|
---|
531 | def getCurrentTimestampMinusOne(self):
|
---|
532 | """
|
---|
533 | Returns the current timestamp minus one tick.
|
---|
534 | This will start a transaction if necessary.
|
---|
535 | """
|
---|
536 | if self._tsCurrentMinusOne is None:
|
---|
537 | self.getCurrentTimestamps();
|
---|
538 | return self._tsCurrentMinusOne;
|
---|
539 |
|
---|
540 |
|
---|
541 | #
|
---|
542 | # Additional cursors.
|
---|
543 | #
|
---|
544 | def openCursor(self):
|
---|
545 | """
|
---|
546 | Opens a new cursor (TMDatabaseCursor).
|
---|
547 | """
|
---|
548 | oCursor = self._oConn.cursor();
|
---|
549 | return TMDatabaseCursor(self, oCursor);
|
---|
550 |
|
---|
551 | #
|
---|
552 | # Cache support.
|
---|
553 | #
|
---|
554 | def getCache(self, sType):
|
---|
555 | """ Returns the cache dictionary for this data type. """
|
---|
556 | dRet = self.ddCaches.get(sType, None);
|
---|
557 | if dRet is None:
|
---|
558 | dRet = dict();
|
---|
559 | self.ddCaches[sType] = dRet;
|
---|
560 | return dRet;
|
---|
561 |
|
---|
562 |
|
---|
563 | #
|
---|
564 | # Utilities.
|
---|
565 | #
|
---|
566 |
|
---|
567 | @staticmethod
|
---|
568 | def isTsInfinity(tsValue):
|
---|
569 | """ Checks if tsValue is an infinity timestamp. """
|
---|
570 | return isDbTimestampInfinity(tsValue);
|
---|
571 |
|
---|
572 | #
|
---|
573 | # Error stuff.
|
---|
574 | #
|
---|
575 | def integrityException(self, sMessage):
|
---|
576 | """
|
---|
577 | Database integrity reporter and exception factory.
|
---|
578 | Returns an TMDatabaseIntegrityException which the caller can raise.
|
---|
579 | """
|
---|
580 | ## @todo Create a new database connection and log the issue in the SystemLog table.
|
---|
581 | ## Alternatively, rollback whatever is going on and do it using the current one.
|
---|
582 | return TMDatabaseIntegrityException(sMessage);
|
---|
583 |
|
---|
584 |
|
---|
585 | #
|
---|
586 | # Debugging.
|
---|
587 | #
|
---|
588 |
|
---|
589 | def dprint(self, sText):
|
---|
590 | """
|
---|
591 | Debug output.
|
---|
592 | """
|
---|
593 | if not self._fnDPrint:
|
---|
594 | return False;
|
---|
595 | self._fnDPrint(sText);
|
---|
596 | return True;
|
---|
597 |
|
---|
598 | def debugHtmlReport(self, tsStart = 0):
|
---|
599 | """
|
---|
600 | Used to get a SQL activity dump as HTML, usually for WuiBase._sDebug.
|
---|
601 | """
|
---|
602 | cNsElapsed = 0;
|
---|
603 | for aEntry in self._aoTraceBack:
|
---|
604 | cNsElapsed += aEntry[2];
|
---|
605 |
|
---|
606 | sDebug = '<h3>SQL Debug Log (total time %s ns):</h3>\n' \
|
---|
607 | '<table class="tmsqltable">\n' \
|
---|
608 | ' <tr>\n' \
|
---|
609 | ' <th>No.</th>\n' \
|
---|
610 | ' <th>Timestamp (ns)</th>\n' \
|
---|
611 | ' <th>Elapsed (ns)</th>\n' \
|
---|
612 | ' <th>Rows Returned</th>\n' \
|
---|
613 | ' <th>Command</th>\n' \
|
---|
614 | ' <th>Caller</th>\n' \
|
---|
615 | ' </tr>\n' \
|
---|
616 | % (utils.formatNumber(cNsElapsed, ' '),);
|
---|
617 |
|
---|
618 | iEntry = 0;
|
---|
619 | for aEntry in self._aoTraceBack:
|
---|
620 | iEntry += 1;
|
---|
621 | sDebug += ' <tr>\n' \
|
---|
622 | ' <td align="right">%s</td>\n' \
|
---|
623 | ' <td align="right">%s</td>\n' \
|
---|
624 | ' <td align="right">%s</td>\n' \
|
---|
625 | ' <td align="right">%s</td>\n' \
|
---|
626 | ' <td><pre>%s</pre></td>\n' \
|
---|
627 | ' <td>%s</td>\n' \
|
---|
628 | ' </tr>\n' \
|
---|
629 | % (iEntry,
|
---|
630 | utils.formatNumber(aEntry[0] - tsStart, ' '),
|
---|
631 | utils.formatNumber(aEntry[2], ' '),
|
---|
632 | utils.formatNumber(aEntry[3], ' '),
|
---|
633 | webutils.escapeElem(aEntry[1]),
|
---|
634 | webutils.escapeElem(aEntry[4]),
|
---|
635 | );
|
---|
636 | if aEntry[5] is not None:
|
---|
637 | sDebug += ' <tr>\n' \
|
---|
638 | ' <td colspan="6"><pre style="white-space: pre-wrap;">%s</pre></td>\n' \
|
---|
639 | ' </tr>\n' \
|
---|
640 | % (webutils.escapeElem('\n'.join([aoRow[0] for aoRow in aEntry[5]])),);
|
---|
641 |
|
---|
642 | sDebug += '</table>';
|
---|
643 | return sDebug;
|
---|
644 |
|
---|
645 | def debugTextReport(self, tsStart = 0):
|
---|
646 | """
|
---|
647 | Used to get a SQL activity dump as text.
|
---|
648 | """
|
---|
649 | cNsElapsed = 0;
|
---|
650 | for aEntry in self._aoTraceBack:
|
---|
651 | cNsElapsed += aEntry[2];
|
---|
652 |
|
---|
653 | sHdr = 'SQL Debug Log (total time %s ns)' % (utils.formatNumber(cNsElapsed),);
|
---|
654 | sDebug = sHdr + '\n' + '-' * len(sHdr) + '\n';
|
---|
655 |
|
---|
656 | iEntry = 0;
|
---|
657 | for aEntry in self._aoTraceBack:
|
---|
658 | iEntry += 1;
|
---|
659 | sHdr = 'Query #%s Timestamp: %s ns Elapsed: %s ns Rows: %s Caller: %s' \
|
---|
660 | % ( iEntry,
|
---|
661 | utils.formatNumber(aEntry[0] - tsStart),
|
---|
662 | utils.formatNumber(aEntry[2]),
|
---|
663 | utils.formatNumber(aEntry[3]),
|
---|
664 | aEntry[4], );
|
---|
665 | sDebug += '\n' + sHdr + '\n' + '-' * len(sHdr) + '\n';
|
---|
666 |
|
---|
667 | sDebug += aEntry[1];
|
---|
668 | if sDebug[-1] != '\n':
|
---|
669 | sDebug += '\n';
|
---|
670 |
|
---|
671 | if aEntry[5] is not None:
|
---|
672 | sDebug += 'Explain:\n' \
|
---|
673 | ' %s\n' \
|
---|
674 | % ( '\n'.join([aoRow[0] for aoRow in aEntry[5]]),);
|
---|
675 |
|
---|
676 | return sDebug;
|
---|
677 |
|
---|
678 | def debugInfoCallback(self, oGlue, fHtml):
|
---|
679 | """ Called back by the glue code on error. """
|
---|
680 | oGlue.write('\n');
|
---|
681 | if not fHtml: oGlue.write(self.debugTextReport());
|
---|
682 | else: oGlue.write(self.debugHtmlReport());
|
---|
683 | oGlue.write('\n');
|
---|
684 | return True;
|
---|
685 |
|
---|
686 | def debugEnableExplain(self):
|
---|
687 | """ Enabled explain. """
|
---|
688 | if self._oExplainConn is None:
|
---|
689 | dArgs = \
|
---|
690 | { \
|
---|
691 | 'database': config.g_ksDatabaseName,
|
---|
692 | 'user': config.g_ksDatabaseUser,
|
---|
693 | 'password': config.g_ksDatabasePassword,
|
---|
694 | # 'application_name': sAppName, - Darn stale debian! :/
|
---|
695 | };
|
---|
696 | if config.g_ksDatabaseAddress is not None:
|
---|
697 | dArgs['host'] = config.g_ksDatabaseAddress;
|
---|
698 | if config.g_ksDatabasePort is not None:
|
---|
699 | dArgs['port'] = config.g_ksDatabasePort;
|
---|
700 | self._oExplainConn = psycopg2.connect(**dArgs); # pylint: disable=W0142
|
---|
701 | self._oExplainCursor = self._oExplainConn.cursor();
|
---|
702 | return True;
|
---|
703 |
|
---|
704 | def debugDisableExplain(self):
|
---|
705 | """ Disables explain. """
|
---|
706 | self._oExplainCursor = None;
|
---|
707 | self._oExplainConn = None
|
---|
708 | return True;
|
---|
709 |
|
---|
710 | def debugIsExplainEnabled(self):
|
---|
711 | """ Check if explaining of SQL statements is enabled. """
|
---|
712 | return self._oExplainConn is not None;
|
---|
713 |
|
---|