testfileset.py@ 79255

Last change on this file since 79255 was 79255, checked in by vboxsync, 5 years ago
tdAddGuestCtrl.py: Rewrote the file_read test (testGuestCtrlFileRead) to also cover readAt, offset and two seek variants. readAt is still a little buggy wrt file offset afterwards (initially didn't work at all due to VBoxService offset type mixup fixed in r131435). bugref:9151 bugref:9320
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 22.1 KB

Line
1	# -- coding: utf-8 --
2	# $Id: testfileset.py 79255 2019-06-20 03:14:07Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Test File Set
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2019 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 79255 $"
31
32
33	# Standard Python imports.
34	import os;
35	import random;
36	import string;
37	import sys;
38	import tarfile;
39	import unittest;
40
41	# Validation Kit imports.
42	from common import utils;
43	from common import pathutils;
44	from testdriver import reporter;
45
46	# Python 3 hacks:
47	if sys.version_info[0] >= 3:
48	xrange = range; # pylint: disable=redefined-builtin,invalid-name
49
50
51
52	class TestFsObj(object):
53	""" A file system object we created in for test purposes. """
54	def __init__(self, oParent, sPath, sName = None):
55	self.oParent = oParent # type: TestDir
56	self.sPath = sPath # type: str
57	self.sName = sName # type: str
58	if oParent:
59	assert sPath.startswith(oParent.sPath);
60	assert sName is None;
61	self.sName = sPath[len(oParent.sPath) + 1:];
62	# Add to parent.
63	oParent.aoChildren.append(self);
64	oParent.dChildrenUpper[self.sName.upper()] = self;
65
66	def buildPath(self, sRoot, sSep):
67	"""
68	Build the path from sRoot using sSep.
69
70	This is handy for getting the path to an object in a different context
71	(OS, path) than what it was generated for.
72	"""
73	if self.oParent:
74	return self.oParent.buildPath(sRoot, sSep) + sSep + self.sName;
75	return sRoot + sSep + self.sName;
76
77
78	class TestFile(TestFsObj):
79	""" A file object in the guest. """
80	def __init__(self, oParent, sPath, abContent):
81	TestFsObj.__init__(self, oParent, sPath);
82	self.abContent = abContent # type: bytearray
83	self.cbContent = len(abContent);
84	self.off = 0;
85
86	def read(self, cbToRead):
87	""" read() emulation. """
88	assert self.off <= self.cbContent;
89	cbLeft = self.cbContent - self.off;
90	if cbLeft < cbToRead:
91	cbToRead = cbLeft;
92	abRet = self.abContent[self.off:(self.off + cbToRead)];
93	assert len(abRet) == cbToRead;
94	self.off += cbToRead;
95	if sys.version_info[0] < 3:
96	return bytes(abRet);
97	return abRet;
98
99	def equalFile(self, oFile):
100	""" Compares the content of oFile with self.abContent. """
101
102	# Check the size first.
103	try:
104	cbFile = os.fstat(oFile.fileno()).st_size;
105	except:
106	return reporter.errorXcpt();
107	if cbFile != self.cbContent:
108	return reporter.error('file size differs: %s, cbContent=%s' % (cbFile, self.cbContent));
109
110	# Compare the bytes next.
111	offFile = 0;
112	try:
113	oFile.seek(offFile);
114	except:
115	return reporter.error('seek error');
116	while offFile < self.cbContent:
117	cbToRead = self.cbContent - offFile;
118	if cbToRead > 256*1024:
119	cbToRead = 256*1024;
120	try:
121	abRead = oFile.read(cbToRead);
122	except:
123	return reporter.error('read error at offset %s' % (offFile,));
124	cbRead = len(abRead);
125	if cbRead == 0:
126	return reporter.error('premature end of file at offset %s' % (offFile,));
127	if not utils.areBytesEqual(abRead, self.abContent[offFile:(offFile + cbRead)]):
128	return reporter.error('%s byte block at offset %s differs' % (cbRead, offFile,));
129	# Advance:
130	offFile += cbRead;
131
132	return True;
133
134	@staticmethod
135	def hexFormatBytes(abBuf):
136	""" Formats a buffer/string/whatever as a string of hex bytes """
137	if sys.version_info[0] >= 3:
138	if utils.isString(abBuf):
139	try: abBuf = bytes(abBuf, 'utf-8');
140	except: pass;
141	else:
142	if utils.isString(abBuf):
143	try: abBuf = bytearray(abBuf, 'utf-8'); # pylint: disable=redefined-variable-type
144	except: pass;
145	sRet = '';
146	off = 0;
147	for off, bByte in enumerate(abBuf):
148	if off > 0:
149	sRet += ' ' if off & 7 else '-';
150	if isinstance(bByte, int):
151	sRet += '%02x' % (bByte,);
152	else:
153	sRet += '%02x' % (ord(bByte),);
154	return sRet;
155
156	def equalMemory(self, abBuf, offFile = 0):
157	"""
158	Compares the content of the given buffer with the file content at that
159	file offset.
160
161	Returns True if it matches, False + error logging if it does not match.
162	"""
163	if not abBuf:
164	return True;
165	if offFile >= self.cbContent:
166	return reporter.error('buffer @ %s LB %s is beyond the end of the file (%s bytes)!'
167	% (offFile, len(abBuf), self.cbContent,));
168	if offFile + len(abBuf) > self.cbContent:
169	return reporter.error('buffer @ %s LB %s is partially beyond the end of the file (%s bytes)!'
170	% (offFile, len(abBuf), self.cbContent,));
171	if utils.areBytesEqual(abBuf, self.abContent[offFile:(offFile + len(abBuf))]):
172	return True;
173
174	reporter.error('mismatch with buffer @ %s LB %s (cbContent=%s)!' % (offFile, len(abBuf), self.cbContent,));
175	reporter.error(' type(abBuf): %s' % (type(abBuf),));
176	#if isinstance(abBuf, memoryview):
177	# reporter.error(' nbytes=%s len=%s itemsize=%s type(obj)=%s'
178	# % (abBuf.nbytes, len(abBuf), abBuf.itemsize, type(abBuf.obj),));
179	reporter.error('type(abContent): %s' % (type(self.abContent),));
180
181	offBuf = 0;
182	cbLeft = len(abBuf);
183	while cbLeft > 0:
184	cbLine = min(16, cbLeft);
185	abBuf1 = abBuf[offBuf:(offBuf + cbLine)];
186	abBuf2 = self.abContent[offFile:(offFile + cbLine)];
187	if not utils.areBytesEqual(abBuf1, abBuf2):
188	try: sStr1 = self.hexFormatBytes(abBuf1);
189	except: sStr1 = 'oops';
190	try: sStr2 = self.hexFormatBytes(abBuf2);
191	except: sStr2 = 'oops';
192	reporter.log('%#10x: %s' % (offBuf, sStr1,));
193	reporter.log('%#10x: %s' % (offFile, sStr2,));
194
195	# Advance.
196	offBuf += 16;
197	offFile += 16;
198	cbLeft -= 16;
199
200	return False;
201
202
203
204
205	class TestDir(TestFsObj):
206	""" A file object in the guest. """
207	def __init__(self, oParent, sPath, sName = None):
208	TestFsObj.__init__(self, oParent, sPath, sName);
209	self.aoChildren = [] # type: list(TestFsObj)
210	self.dChildrenUpper = {} # type: dict(str, TestFsObj)
211
212	def contains(self, sName):
213	""" Checks if the directory contains the given name. """
214	return sName.upper() in self.dChildrenUpper
215
216
217	class TestFileSet(object):
218	"""
219	A generated set of files and directories for use in a test.
220
221	Can be wrapped up into a tarball or written directly to the file system.
222	"""
223
224	ksReservedWinOS2 = '/\\"*:<>?\|\t\v\n\r\f\a\b';
225	ksReservedUnix = '/';
226	ksReservedTrailingWinOS2 = ' .';
227	ksReservedTrailingUnix = '';
228
229	## @name Path style.
230	## @{
231
232	## @}
233
234	def __init__(self, fDosStyle, sBasePath, sSubDir, # pylint: disable=too-many-arguments
235	asCompatibleWith = None, # List of getHostOs values to the names must be compatible with.
236	oRngFileSizes = xrange(0, 16384),
237	oRngManyFiles = xrange(128, 512),
238	oRngTreeFiles = xrange(128, 384),
239	oRngTreeDepth = xrange(92, 256),
240	oRngTreeDirs = xrange(2, 16),
241	cchMaxPath = 230,
242	cchMaxName = 230,
243	uSeed = None):
244	## @name Parameters
245	## @{
246	self.fDosStyle = fDosStyle;
247	self.sMinStyle = 'win' if fDosStyle else 'linux';
248	if asCompatibleWith is not None:
249	for sOs in asCompatibleWith:
250	assert sOs in ('win', 'os2', 'darwin', 'linux', 'solaris',), sOs;
251	if 'os2' in asCompatibleWith:
252	self.sMinStyle = 'os2';
253	elif 'win' in asCompatibleWith:
254	self.sMinStyle = 'win';
255	self.sBasePath = sBasePath;
256	self.sSubDir = sSubDir;
257	self.oRngFileSizes = oRngFileSizes;
258	self.oRngManyFiles = oRngManyFiles;
259	self.oRngTreeFiles = oRngTreeFiles;
260	self.oRngTreeDepth = oRngTreeDepth;
261	self.oRngTreeDirs = oRngTreeDirs;
262	self.cchMaxPath = cchMaxPath;
263	self.cchMaxName = cchMaxName
264	## @}
265
266	## @name Charset stuff
267	## @todo allow more chars for unix hosts + guests.
268	## @todo include unicode stuff, except on OS/2 and DOS.
269	## @{
270	## The filename charset.
271	self.sFileCharset = string.printable;
272	## Set of characters that should not trail a guest filename.
273	self.sReservedTrailing = self.ksReservedTrailingWinOS2;
274	if self.sMinStyle in ('win', 'os2'):
275	for ch in self.ksReservedWinOS2:
276	self.sFileCharset = self.sFileCharset.replace(ch, '');
277	else:
278	self.sReservedTrailing = self.ksReservedTrailingUnix;
279	for ch in self.ksReservedUnix:
280	self.sFileCharset = self.sFileCharset.replace(ch, '');
281	# More spaces and dot:
282	self.sFileCharset += ' ...';
283	## @}
284
285	## The root directory.
286	self.oRoot = None # type: TestDir;
287	## An empty directory (under root).
288	self.oEmptyDir = None # type: TestDir;
289
290	## A directory with a lot of files in it.
291	self.oManyDir = None # type: TestDir;
292
293	## A directory with a mixed tree structure under it.
294	self.oTreeDir = None # type: TestDir;
295	## Number of files in oTreeDir.
296	self.cTreeFiles = 0;
297	## Number of directories under oTreeDir.
298	self.cTreeDirs = 0;
299	## Number of other file types under oTreeDir.
300	self.cTreeOthers = 0;
301
302	## All directories in creation order.
303	self.aoDirs = [] # type: list(TestDir);
304	## All files in creation order.
305	self.aoFiles = [] # type: list(TestFile);
306	## Path to object lookup.
307	self.dPaths = {} # type: dict(str, TestFsObj);
308
309	#
310	# Do the creating.
311	#
312	self.uSeed = uSeed if uSeed is not None else utils.timestampMilli();
313	self.oRandom = random.Random();
314	self.oRandom.seed(self.uSeed);
315	reporter.log('prepareGuestForTesting: random seed %s' % (self.uSeed,));
316
317	self.__createTestStuff();
318
319	def __createFilename(self, oParent, sCharset, sReservedTrailing):
320	"""
321	Creates a filename contains random characters from sCharset and together
322	with oParent.sPath doesn't exceed the given max chars in length.
323	"""
324	## @todo Consider extending this to take UTF-8 and UTF-16 encoding so we
325	## can safely use the full unicode range. Need to check how
326	## RTZipTarCmd handles file name encoding in general...
327
328	if oParent:
329	cchMaxName = self.cchMaxPath - len(oParent.sPath) - 1;
330	else:
331	cchMaxName = self.cchMaxPath - 4;
332	if cchMaxName > self.cchMaxName:
333	cchMaxName = self.cchMaxName;
334	if cchMaxName <= 1:
335	cchMaxName = 2;
336
337	while True:
338	cchName = self.oRandom.randrange(1, cchMaxName);
339	sName = ''.join(self.oRandom.choice(sCharset) for _ in xrange(cchName));
340	if oParent is None or not oParent.contains(sName):
341	if sName[-1] not in sReservedTrailing:
342	if sName not in ('.', '..',):
343	return sName;
344	return ''; # never reached, but makes pylint happy.
345
346	def generateFilenameEx(self, cchMax = -1, cchMin = -1):
347	"""
348	Generates a filename according to the given specs.
349
350	This is for external use, whereas __createFilename is for internal.
351
352	Returns generated filename.
353	"""
354	assert cchMax == -1 or (cchMax >= 1 and cchMax > cchMin);
355	if cchMin <= 0:
356	cchMin = 1;
357	if cchMax < cchMin:
358	cchMax = self.cchMaxName;
359
360	while True:
361	cchName = self.oRandom.randrange(cchMin, cchMax + 1);
362	sName = ''.join(self.oRandom.choice(self.sFileCharset) for _ in xrange(cchName));
363	if sName[-1] not in self.sReservedTrailing:
364	if sName not in ('.', '..',):
365	return sName;
366	return ''; # never reached, but makes pylint happy.
367
368	def __createTestDir(self, oParent, sDir, sName = None):
369	"""
370	Creates a test directory.
371	"""
372	oDir = TestDir(oParent, sDir, sName);
373	self.aoDirs.append(oDir);
374	self.dPaths[sDir] = oDir;
375	return oDir;
376
377	def __createTestFile(self, oParent, sFile):
378	"""
379	Creates a test file with random size up to cbMaxContent and random content.
380	"""
381	cbFile = self.oRandom.choice(self.oRngFileSizes);
382	abContent = bytearray(self.oRandom.getrandbits(8) for _ in xrange(cbFile));
383
384	oFile = TestFile(oParent, sFile, abContent);
385	self.aoFiles.append(oFile);
386	self.dPaths[sFile] = oFile;
387	return oFile;
388
389	def __createTestStuff(self):
390	"""
391	Create a random file set that we can work on in the tests.
392	Returns True/False.
393	"""
394
395	#
396	# Create the root test dir.
397	#
398	sRoot = pathutils.joinEx(self.fDosStyle, self.sBasePath, self.sSubDir);
399	self.oRoot = self.__createTestDir(None, sRoot, self.sSubDir);
400	self.oEmptyDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'empty'));
401
402	#
403	# Create a directory with lots of files in it:
404	#
405	oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'many'));
406	self.oManyDir = oDir;
407	cManyFiles = self.oRandom.choice(self.oRngManyFiles);
408	for _ in xrange(cManyFiles):
409	sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
410	self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
411
412	#
413	# Generate a tree of files and dirs.
414	#
415	oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'tree'));
416	uMaxDepth = self.oRandom.choice(self.oRngTreeDepth);
417	cMaxFiles = self.oRandom.choice(self.oRngTreeFiles);
418	cMaxDirs = self.oRandom.choice(self.oRngTreeDirs);
419	self.oTreeDir = oDir;
420	self.cTreeFiles = 0;
421	self.cTreeDirs = 0;
422	uDepth = 0;
423	while self.cTreeFiles < cMaxFiles and self.cTreeDirs < cMaxDirs:
424	iAction = self.oRandom.randrange(0, 2+1);
425	# 0: Add a file:
426	if iAction == 0 and self.cTreeFiles < cMaxFiles and len(oDir.sPath) < 230 - 2:
427	sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
428	self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
429	self.cTreeFiles += 1;
430	# 1: Add a subdirector and descend into it:
431	elif iAction == 1 and self.cTreeDirs < cMaxDirs and uDepth < uMaxDepth and len(oDir.sPath) < 220:
432	sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
433	oDir = self.__createTestDir(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
434	self.cTreeDirs += 1;
435	uDepth += 1;
436	# 2: Ascend to parent dir:
437	elif iAction == 2 and uDepth > 0:
438	oDir = oDir.oParent;
439	uDepth -= 1;
440
441	return True;
442
443	def createTarball(self, sTarFileHst):
444	"""
445	Creates a tarball on the host.
446	Returns success indicator.
447	"""
448	reporter.log('Creating tarball "%s" with test files for the guest...' % (sTarFileHst,));
449
450	cchSkip = len(self.sBasePath) + 1;
451
452	# Open the tarball:
453	try:
454	oTarFile = tarfile.open(sTarFileHst, 'w:gz');
455	except:
456	return reporter.errorXcpt('Failed to open new tar file: %s' % (sTarFileHst,));
457
458	# Directories:
459	for oDir in self.aoDirs:
460	sPath = oDir.sPath[cchSkip:];
461	if self.fDosStyle:
462	sPath = sPath.replace('\\', '/');
463	oTarInfo = tarfile.TarInfo(sPath + '/');
464	oTarInfo.mode = 0o777;
465	oTarInfo.type = tarfile.DIRTYPE;
466	try:
467	oTarFile.addfile(oTarInfo);
468	except:
469	return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oDir.sPath,));
470
471	# Files:
472	for oFile in self.aoFiles:
473	sPath = oFile.sPath[cchSkip:];
474	if self.fDosStyle:
475	sPath = sPath.replace('\\', '/');
476	oTarInfo = tarfile.TarInfo(sPath);
477	oTarInfo.size = len(oFile.abContent);
478	oFile.off = 0;
479	try:
480	oTarFile.addfile(oTarInfo, oFile);
481	except:
482	return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oFile.sPath,));
483
484	# Complete the tarball.
485	try:
486	oTarFile.close();
487	except:
488	return reporter.errorXcpt('Error closing new tar file: %s' % (sTarFileHst,));
489	return True;
490
491	def writeToDisk(self, sAltBase = None):
492	"""
493	Writes out the files to disk.
494	Returns True on success, False + error logging on failure.
495	"""
496
497	# We only need to flip DOS slashes to unix ones, since windows & OS/2 can handle unix slashes.
498	fDosToUnix = self.fDosStyle and os.path.sep != '\\';
499
500	# The directories:
501	for oDir in self.aoDirs:
502	sPath = oDir.sPath;
503	if sAltBase:
504	if fDosToUnix:
505	sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep);
506	else:
507	sPath = sAltBase + sPath[len(self.sBasePath):];
508	elif fDosToUnix:
509	sPath = sPath.replace('\\', os.path.sep);
510
511	try:
512	os.mkdir(sPath, 0o770);
513	except:
514	return reporter.errorXcpt('mkdir(%s) failed' % (sPath,));
515
516	# The files:
517	for oFile in self.aoFiles:
518	sPath = oFile.sPath;
519	if sAltBase:
520	if fDosToUnix:
521	sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep);
522	else:
523	sPath = sAltBase + sPath[len(self.sBasePath):];
524	elif fDosToUnix:
525	sPath = sPath.replace('\\', os.path.sep);
526
527	try:
528	oOutFile = open(sPath, 'wb');
529	except:
530	return reporter.errorXcpt('open(%s, "wb") failed' % (sPath,));
531	try:
532	if sys.version_info[0] < 3:
533	oOutFile.write(bytes(oFile.abContent));
534	else:
535	oOutFile.write(oFile.abContent);
536	except:
537	try: oOutFile.close();
538	except: pass;
539	return reporter.errorXcpt('%s: write(%s bytes) failed' % (sPath, oFile.cbContent,));
540	try:
541	oOutFile.close();
542	except:
543	return reporter.errorXcpt('%s: close() failed' % (sPath,));
544
545	return True;
546
547
548	def chooseRandomFile(self):
549	"""
550	Returns a random file.
551	"""
552	return self.aoFiles[self.oRandom.choice(xrange(len(self.aoFiles)))];
553
554	def chooseRandomDirFromTree(self, fLeaf = False, fNonEmpty = False):
555	"""
556	Returns a random directory from the tree (self.oTreeDir).
557	"""
558	while True:
559	oDir = self.aoDirs[self.oRandom.choice(xrange(len(self.aoDirs)))];
560	# Check fNonEmpty requirement:
561	if not fNonEmpty or oDir.aoChildren:
562	# Check leaf requirement:
563	if not fLeaf:
564	for oChild in oDir.aoChildren:
565	if isinstance(oChild, TestDir):
566	continue; # skip it.
567
568	# Return if in the tree:
569	oParent = oDir.oParent;
570	while oParent is not None:
571	if oParent is self.oTreeDir:
572	return oDir;
573	oParent = oParent.oParent;
574	return None; # make pylint happy
575
576	#
577	# Unit testing.
578	#
579
580	# pylint: disable=missing-docstring
581	# pylint: disable=undefined-variable
582	class TestFileSetUnitTests(unittest.TestCase):
583	def testGeneral(self):
584	oSet = TestFileSet(False, '/tmp', 'unittest');
585	self.assertTrue(isinstance(oSet.chooseRandomDirFromTree(), TestDir));
586	self.assertTrue(isinstance(oSet.chooseRandomFile(), TestFile));
587
588	def testHexFormatBytes(self):
589	self.assertEqual(TestFile.hexFormatBytes(bytearray([0,1,2,3,4,5,6,7,8,9])),
590	'00 01 02 03 04 05 06 07-08 09');
591	self.assertEqual(TestFile.hexFormatBytes(memoryview(bytearray([0,1,2,3,4,5,6,7,8,9,10, 16]))),
592	'00 01 02 03 04 05 06 07-08 09 0a 10');
593
594
595	if __name__ == '__main__':
596	unittest.main();
597	# not reached.
598

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testdriver/testfileset.py@ 79255

Download in other formats: