btresolver.py@ 65370

Last change on this file since 65370 was 65369, checked in by vboxsync, 8 years ago
ValidationKit: Implement method to symbolize stack traces from VM process reports using the provided debugging symbols and RTLdrFlt
Property svn:executable set to ``*
File size: 20.3 KB

Line
1	# -- coding: utf-8 --
2	# $Id$
3	# pylint: disable=C0302
4
5	"""
6	Backtrace resolver using external debugging symbols and RTLdrFlt.
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2016 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision$"
31
32
33	# Standard Python imports.
34	import os;
35	import re;
36	import shutil;
37	import subprocess;
38
39	# Validation Kit imports.
40	from common import utils;
41
42	def getRTLdrFltPath(asPaths):
43	"""
44	Returns the path to the RTLdrFlt tool looking in the provided paths
45	or None if not found.
46	"""
47
48	for sPath in asPaths:
49	for sDirPath, _, asFiles in os.walk(sPath):
50	if 'RTLdrFlt' in asFiles:
51	return os.path.join(sDirPath, 'RTLdrFlt');
52
53	return None;
54
55
56
57	class BacktraceResolverOs(object):
58	"""
59	Base class for all OS specific resolvers.
60	"""
61
62	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
63	self.sScratchPath = sScratchPath;
64	self.sBuildRoot = sBuildRoot;
65	self.fnLog = fnLog;
66
67	def log(self, sText):
68	"""
69	Internal logger callback.
70	"""
71	if self.fnLog is not None:
72	self.fnLog(sText);
73
74
75
76	class BacktraceResolverOsLinux(BacktraceResolverOs):
77	"""
78	Linux specific backtrace resolver.
79	"""
80
81	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
82	"""
83	Constructs a Linux host specific backtrace resolver.
84	"""
85	BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog);
86
87	self.asDbgFiles = {};
88
89	def prepareEnv(self):
90	"""
91	Prepares the environment for annotating Linux reports.
92	"""
93	fRc = False;
94	try:
95	sDbgArchive = os.path.join(self.sBuildRoot, 'bin', 'VirtualBox-dbg.tar.bz2');
96
97	# Extract debug symbol archive if it was found.
98	if os.path.exists(sDbgArchive):
99	asMembers = utils.unpackFile(sDbgArchive, self.sScratchPath, self.fnLog,
100	self.fnLog);
101	if asMembers is not None and len(asMembers) > 0:
102	# Populate the list of debug files.
103	for sMember in asMembers:
104	if os.path.isfile(sMember):
105	self.asDbgFiles[os.path.basename(sMember)] = sMember;
106	fRc = True;
107	except:
108	self.log('Failed to setup debug symbols');
109
110	return fRc;
111
112	def cleanupEnv(self):
113	"""
114	Cleans up the environment.
115	"""
116	fRc = False;
117	try:
118	shutil.rmtree(self.sScratchPath, True);
119	fRc = True;
120	except:
121	pass;
122
123	return fRc;
124
125	def getDbgSymPathFromBinary(self, sBinary, sArch):
126	"""
127	Returns the path to file containing the debug symbols for the specified binary.
128	"""
129	_ = sArch;
130	sDbgFilePath = None;
131	try:
132	sDbgFilePath = self.asDbgFiles[sBinary];
133	except:
134	pass;
135
136	return sDbgFilePath;
137
138	def getBinaryListWithLoadAddrFromReport(self, asReport):
139	"""
140	Parses the given VM state report and returns a list of binaries and their
141	load address.
142
143	Returns a list if tuples containing the binary and load addres or an empty
144	list on failure.
145	"""
146	asListBinaries = [];
147
148	# Look for the line "Mapped address spaces:"
149	iLine = 0;
150	while iLine < len(asReport):
151	if asReport[iLine].startswith('Mapped address spaces:'):
152	break;
153	iLine += 1;
154
155	for sLine in asReport[iLine:]:
156	asCandidate = sLine.split();
157	if len(asCandidate) is 5 \
158	and asCandidate[0].startswith('0x') \
159	and asCandidate[1].startswith('0x') \
160	and asCandidate[2].startswith('0x') \
161	and asCandidate[3] == '0x0' \
162	and 'VirtualBox' in asCandidate[4]:
163	asListBinaries.append((asCandidate[0], os.path.basename(asCandidate[4])));
164
165	return asListBinaries;
166
167
168
169	class BacktraceResolverOsDarwin(BacktraceResolverOs):
170	"""
171	Darwin specific backtrace resolver.
172	"""
173
174	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
175	"""
176	Constructs a Linux host specific backtrace resolver.
177	"""
178	BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog);
179
180	self.asDbgFiles = {};
181
182	def prepareEnv(self):
183	"""
184	Prepares the environment for annotating Darwin reports.
185	"""
186	fRc = False;
187	try:
188	#
189	# Walk the build root directory and look for .dSYM directories, building a
190	# list of them.
191	#
192	asDSymPaths = [];
193
194	for sDirPath, asDirs, _ in os.walk(self.sBuildRoot):
195	for sDir in asDirs:
196	if sDir.endswith('.dSYM'):
197	asDSymPaths.append(os.path.join(sDirPath, sDir));
198
199	# Expand the dSYM paths to full DWARF debug files in the next step
200	# and add them to the debug files dictionary.
201	for sDSymPath in asDSymPaths:
202	sBinary = os.path.basename(sDSymPath).strip('.dSYM');
203	self.asDbgFiles[sBinary] = os.path.join(sDSymPath, 'Contents', 'Resources',
204	'DWARF', sBinary);
205
206	fRc = True;
207	except:
208	self.log('Failed to setup debug symbols');
209
210	return fRc;
211
212	def cleanupEnv(self):
213	"""
214	Cleans up the environment.
215	"""
216	fRc = False;
217	try:
218	shutil.rmtree(self.sScratchPath, True);
219	fRc = True;
220	except:
221	pass;
222
223	return fRc;
224
225	def getDbgSymPathFromBinary(self, sBinary, sArch):
226	"""
227	Returns the path to file containing the debug symbols for the specified binary.
228	"""
229	# Hack to exclude executables as RTLdrFlt has some problems with it currently.
230	_ = sArch;
231	sDbgSym = None;
232	try:
233	sDbgSym = self.asDbgFiles[sBinary];
234	except:
235	pass;
236
237	if sDbgSym is not None and sDbgSym.endswith('.dylib'):
238	return sDbgSym;
239
240	return None;
241
242	def _getReportVersion(self, asReport):
243	"""
244	Returns the version of the darwin report.
245	"""
246	# Find the line starting with "Report Version:"
247	iLine = 0;
248	iVersion = 0;
249	while iLine < len(asReport):
250	if asReport[iLine].startswith('Report Version:'):
251	break;
252	iLine += 1;
253
254	if iLine < len(asReport):
255	# Look for the start of the number
256	sVersion = asReport[iLine];
257	iStartVersion = len('Report Version:');
258	iEndVersion = len(sVersion);
259
260	while iStartVersion < len(sVersion) \
261	and not sVersion[iStartVersion:iStartVersion+1].isdigit():
262	iStartVersion += 1;
263
264	while iEndVersion > 0 \
265	and not sVersion[iEndVersion-1:iEndVersion].isdigit():
266	iEndVersion -= 1;
267
268	iVersion = int(sVersion[iStartVersion:iEndVersion]);
269	else:
270	self.log('Couldn\'t find the report version');
271
272	return iVersion;
273
274	def _getListOfBinariesFromReportPreSierra(self, asReport):
275	"""
276	Returns a list of loaded binaries with their load address obtained from
277	a pre Sierra report.
278	"""
279	asListBinaries = [];
280
281	# Find the line starting with "Binary Images:"
282	iLine = 0;
283	while iLine < len(asReport):
284	if asReport[iLine].startswith('Binary Images:'):
285	break;
286	iLine += 1;
287
288	if iLine < len(asReport):
289	# List starts after that
290	iLine += 1;
291
292	# A line for a loaded binary looks like the following:
293	# 0x100042000 - 0x100095fff +VBoxDDU.dylib (4.3.15) <EB19C44D-F882-0803-DBDD-9995723111B7> /Application...
294	# We need the start address and the library name.
295	# To distinguish between our own libraries and ones from Apple we check whether the path at the end starts with
296	# /Applications/VirtualBox.app/Contents/MacOS
297	oRegExpPath = re.compile(r'/VirtualBox.app/Contents/MacOS');
298	oRegExpAddr = re.compile(r'0x\w+');
299	oRegExpBinPath = re.compile(r'VirtualBox.app/Contents/MacOS/\S*');
300	while iLine < len(asReport):
301	asMatches = oRegExpPath.findall(asReport[iLine]);
302	if len(asMatches) > 0:
303	# Line contains the path, extract start address and path to binary
304	sAddr = oRegExpAddr.findall(asReport[iLine]);
305	sPath = oRegExpBinPath.findall(asReport[iLine]);
306
307	if len(sAddr) > 0 and len(sPath) > 0:
308	# Construct the path in into the build cache containing the debug symbols
309	oRegExp = re.compile(r'\w+\.{0,1}\w*$');
310	sFilename = oRegExp.findall(sPath[0]);
311
312	asListBinaries.append((sAddr[0], sFilename[0]));
313	else:
314	break; # End of image list
315	iLine += 1;
316	else:
317	self.log('Couldn\'t find the list of loaded binaries in the given report');
318
319	return asListBinaries;
320
321	def _getListOfBinariesFromReportSierra(self, asReport):
322	"""
323	Returns a list of loaded binaries with their load address obtained from
324	a Sierra+ report.
325	"""
326	asListBinaries = [];
327
328	# A line for a loaded binary looks like the following:
329	# 4 VBoxXPCOMIPCC.dylib 0x00000001139f17ea 0x1139e4000 + 55274
330	# We need the start address and the library name.
331	# To distinguish between our own libraries and ones from Apple we check whether the library
332	# name contains VBox or VirtualBox
333	iLine = 0;
334	while iLine < len(asReport):
335	asStackTrace = asReport[iLine].split();
336
337	# Check whether the line is made up of 6 elements separated by whitespace
338	# and the first one is a number.
339	if len(asStackTrace) == 6 and asStackTrace[0].isdigit() \
340	and (asStackTrace[1].find('VBox') is not -1 or asStackTrace[1].find('VirtualBox') is not -1) \
341	and asStackTrace[3].startswith('0x'):
342
343	# Check whether the library is already in our list an only add new ones
344	fFound = False;
345	for _, sLibrary in asListBinaries:
346	if asStackTrace[1] == sLibrary:
347	fFound = True;
348	break;
349
350	if not fFound:
351	asListBinaries.append((asStackTrace[3], asStackTrace[1]));
352	iLine += 1;
353
354	return asListBinaries;
355
356	def getBinaryListWithLoadAddrFromReport(self, asReport):
357	"""
358	Parses the given VM state report and returns a list of binaries and their
359	load address.
360
361	Returns a list if tuples containing the binary and load addres or an empty
362	list on failure.
363	"""
364	asListBinaries = [];
365
366	iVersion = self._getReportVersion(asReport);
367	if iVersion > 0:
368	if iVersion <= 11:
369	self.log('Pre Sierra Report');
370	asListBinaries = self._getListOfBinariesFromReportPreSierra(asReport);
371	elif iVersion == 12:
372	self.log('Sierra report');
373	asListBinaries = self._getListOfBinariesFromReportSierra(asReport);
374	else:
375	self.log('Unsupported report version %s' % (iVersion, ));
376
377	return asListBinaries;
378
379
380
381	class BacktraceResolverOsSolaris(BacktraceResolverOs):
382	"""
383	Solaris specific backtrace resolver.
384	"""
385
386	def __init__(self, sScratchPath, sBuildRoot, fnLog = None):
387	"""
388	Constructs a Linux host specific backtrace resolver.
389	"""
390	BacktraceResolverOs.__init__(self, sScratchPath, sBuildRoot, fnLog);
391
392	self.asDbgFiles = {};
393
394	def prepareEnv(self):
395	"""
396	Prepares the environment for annotating Linux reports.
397	"""
398	fRc = False;
399	try:
400	sDbgArchive = os.path.join(self.sBuildRoot, 'bin', 'VirtualBoxDebug.tar.bz2');
401
402	# Extract debug symbol archive if it was found.
403	if os.path.exists(sDbgArchive):
404	asMembers = utils.unpackFile(sDbgArchive, self.sScratchPath, self.fnLog,
405	self.fnLog);
406	if asMembers is not None and len(asMembers) > 0:
407	# Populate the list of debug files.
408	for sMember in asMembers:
409	if os.path.isfile(sMember):
410	sArch = '';
411	if 'amd64' in sMember:
412	sArch = 'amd64';
413	else:
414	sArch = 'x86';
415	self.asDbgFiles[os.path.basename(sMember) + '/' + sArch] = sMember;
416	fRc = True;
417	else:
418	self.log('Unpacking the debug archive failed');
419	except:
420	self.log('Failed to setup debug symbols');
421
422	return fRc;
423
424	def cleanupEnv(self):
425	"""
426	Cleans up the environment.
427	"""
428	fRc = False;
429	try:
430	shutil.rmtree(self.sScratchPath, True);
431	fRc = True;
432	except:
433	pass;
434
435	return fRc;
436
437	def getDbgSymPathFromBinary(self, sBinary, sArch):
438	"""
439	Returns the path to file containing the debug symbols for the specified binary.
440	"""
441	sDbgFilePath = None;
442	try:
443	sDbgFilePath = self.asDbgFiles[sBinary + '/' + sArch];
444	except:
445	pass;
446
447	return sDbgFilePath;
448
449	def getBinaryListWithLoadAddrFromReport(self, asReport):
450	"""
451	Parses the given VM state report and returns a list of binaries and their
452	load address.
453
454	Returns a list if tuples containing the binary and load addres or an empty
455	list on failure.
456	"""
457	asListBinaries = [];
458
459	# Look for the beginning of the process address space mappings"
460	for sLine in asReport:
461	asItems = sLine.split();
462	if len(asItems) == 4 \
463	and asItems[3].startswith('/opt/VirtualBox') \
464	and asItems[2] == 'r-x--':
465	fFound = False;
466	sBinaryFile = os.path.basename(asItems[3]);
467	for _, sBinary in asListBinaries:
468	if sBinary == sBinaryFile:
469	fFound = True;
470	break;
471	if not fFound:
472	asListBinaries.append(('0x' + asItems[0], sBinaryFile));
473
474	return asListBinaries;
475
476
477
478	class BacktraceResolver(object):
479	"""
480	A backtrace resolving class.
481	"""
482
483	def __init__(self, sScratchPath, sBuildRoot, sTargetOs, sArch, sRTLdrFltPath = None, fnLog = None):
484	"""
485	Constructs a backtrace resolver object for the given target OS,
486	architecture and path to the directory containing the debug symbols and tools
487	we need.
488	"""
489	# Initialize all members first.
490	self.sScratchPath = sScratchPath;
491	self.sBuildRoot = sBuildRoot;
492	self.sTargetOs = sTargetOs;
493	self.sArch = sArch;
494	self.sRTLdrFltPath = sRTLdrFltPath;
495	self.fnLog = fnLog;
496	self.sDbgSymPath = None;
497	self.oResolverOs = None;
498	self.sScratchDbgPath = os.path.join(self.sScratchPath, 'dbgsymbols');
499
500	if self.sRTLdrFltPath is None:
501	self.sRTLdrFltPath = getRTLdrFltPath([self.sScratchPath, self.sBuildRoot]);
502
503	if self.fnLog is None:
504	self.fnLog = self.logStub;
505
506	def log(self, sText):
507	"""
508	Internal logger callback.
509	"""
510	if self.fnLog is not None:
511	self.fnLog(sText);
512
513	def logStub(self, sText):
514	"""
515	Logging stub doing nothing.
516	"""
517	_ = sText;
518
519	def prepareEnv(self):
520	"""
521	Prepares the environment to annotate backtraces, finding the required tools
522	and retrieving the debug symbols depending on the host OS.
523
524	Returns True on success and False on error or if not supported.
525	"""
526
527	# No access to the RTLdrFlt tool means no symbols so no point in trying
528	# to set something up.
529	if self.sRTLdrFltPath is None:
530	return False;
531
532	# Create a directory containing the scratch space for the OS resolver backends.
533	fRc = True;
534	if not os.path.exists(self.sScratchDbgPath):
535	try:
536	os.makedirs(self.sScratchDbgPath, 0750);
537	except:
538	fRc = False;
539	self.log('Failed to create scratch directory for debug symbols');
540
541	if fRc:
542	if self.sTargetOs == 'linux':
543	self.oResolverOs = BacktraceResolverOsLinux(self.sScratchDbgPath, self.sBuildRoot, self.fnLog);
544	elif self.sTargetOs == 'darwin':
545	self.oResolverOs = BacktraceResolverOsDarwin(self.sScratchDbgPath, self.sBuildRoot, self.fnLog); # pylint: disable=R0204
546	elif self.sTargetOs == 'solaris':
547	self.oResolverOs = BacktraceResolverOsSolaris(self.sScratchDbgPath, self.sBuildRoot, self.fnLog); # pylint: disable=R0204
548	else:
549	self.log('The backtrace resolver is not supported on %s' % (self.sTargetOs,));
550	fRc = False;
551
552	if fRc:
553	fRc = self.oResolverOs.prepareEnv();
554	if not fRc:
555	self.oResolverOs = None;
556
557	if not fRc:
558	shutil.rmtree(self.sScratchDbgPath, True)
559
560	return fRc;
561
562	def cleanupEnv(self):
563	"""
564	Prepares the environment to annotate backtraces, finding the required tools
565	and retrieving the debug symbols depending on the host OS.
566
567	Returns True on success and False on error or if not supported.
568	"""
569	fRc = False;
570	if self.oResolverOs is not None:
571	fRc = self.oResolverOs.cleanupEnv();
572
573	shutil.rmtree(self.sScratchDbgPath, True);
574	return fRc;
575
576	def annotateReport(self, sReport):
577	"""
578	Annotates the given report with the previously prepared environment.
579
580	Returns the annotated report on success or None on failure.
581	"""
582	sReportAn = None;
583
584	if self.oResolverOs is not None:
585	asListBinaries = self.oResolverOs.getBinaryListWithLoadAddrFromReport(sReport.split('\n'));
586
587	if len(asListBinaries) > 0:
588	asArgs = [self.sRTLdrFltPath, ];
589
590	for sLoadAddr, sBinary in asListBinaries:
591	sDbgSymPath = self.oResolverOs.getDbgSymPathFromBinary(sBinary, self.sArch);
592	if sDbgSymPath is not None:
593	asArgs.append(sDbgSymPath);
594	asArgs.append(sLoadAddr);
595
596	oRTLdrFltProc = subprocess.Popen(asArgs, stdin=subprocess.PIPE, stdout=subprocess.PIPE, bufsize=0);
597	if oRTLdrFltProc is not None:
598	sReportAn, _ = oRTLdrFltProc.communicate(sReport);
599	else:
600	self.log('Error spawning RTLdrFlt process');
601	else:
602	self.log('Getting list of loaded binaries failed');
603	else:
604	self.log('Backtrace resolver not fully initialized, not possible to annotate');
605
606	return sReportAn;
607

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testdriver/btresolver.py@ 65370

Download in other formats: