testfileset.py@ 79198

Last change on this file since 79198 was 79198, checked in by vboxsync, 5 years ago
testfileset.py: Preps for using the TestFileSet for the copy-to-guest tests too. bugref:9151 bugref:9320
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 18.4 KB

Line
1	# -- coding: utf-8 --
2	# $Id: testfileset.py 79198 2019-06-18 08:44:46Z vboxsync $
3	# pylint: disable=too-many-lines
4
5	"""
6	Test File Set
7	"""
8
9	__copyright__ = \
10	"""
11	Copyright (C) 2010-2019 Oracle Corporation
12
13	This file is part of VirtualBox Open Source Edition (OSE), as
14	available from http://www.virtualbox.org. This file is free software;
15	you can redistribute it and/or modify it under the terms of the GNU
16	General Public License (GPL) as published by the Free Software
17	Foundation, in version 2 as it comes in the "COPYING" file of the
18	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
19	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
20
21	The contents of this file may alternatively be used under the terms
22	of the Common Development and Distribution License Version 1.0
23	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
24	VirtualBox OSE distribution, in which case the provisions of the
25	CDDL are applicable instead of those of the GPL.
26
27	You may elect to license modified versions of this file under the
28	terms and conditions of either the GPL or the CDDL or both.
29	"""
30	__version__ = "$Revision: 79198 $"
31
32
33	# Standard Python imports.
34	import os;
35	import random;
36	import string;
37	import sys;
38	import tarfile;
39	import unittest;
40
41	# Validation Kit imports.
42	from common import utils;
43	from common import pathutils;
44	from testdriver import reporter;
45
46	# Python 3 hacks:
47	if sys.version_info[0] >= 3:
48	xrange = range; # pylint: disable=redefined-builtin,invalid-name
49
50
51
52	class TestFsObj(object):
53	""" A file system object we created in for test purposes. """
54	def __init__(self, oParent, sPath):
55	self.oParent = oParent # type: TestDir
56	self.sPath = sPath # type: str
57	self.sName = sPath # type: str
58	if oParent:
59	assert sPath.startswith(oParent.sPath);
60	self.sName = sPath[len(oParent.sPath) + 1:];
61	# Add to parent.
62	oParent.aoChildren.append(self);
63	oParent.dChildrenUpper[self.sName.upper()] = self;
64
65	class TestFile(TestFsObj):
66	""" A file object in the guest. """
67	def __init__(self, oParent, sPath, abContent):
68	TestFsObj.__init__(self, oParent, sPath);
69	self.abContent = abContent # type: bytearray
70	self.cbContent = len(abContent);
71	self.off = 0;
72
73	def read(self, cbToRead):
74	""" read() emulation. """
75	assert self.off <= self.cbContent;
76	cbLeft = self.cbContent - self.off;
77	if cbLeft < cbToRead:
78	cbToRead = cbLeft;
79	abRet = self.abContent[self.off:(self.off + cbToRead)];
80	assert len(abRet) == cbToRead;
81	self.off += cbToRead;
82	if sys.version_info[0] < 3:
83	return bytes(abRet);
84	return abRet;
85
86	def equalFile(self, oFile):
87	""" Compares the content of oFile with self.abContent. """
88
89	# Check the size first.
90	try:
91	cbFile = os.fstat(oFile.fileno()).st_size;
92	except:
93	return reporter.errorXcpt();
94	if cbFile != self.cbContent:
95	return reporter.error('file size differs: %s, cbContent=%s' % (cbFile, self.cbContent));
96
97	# Compare the bytes next.
98	offFile = 0;
99	try:
100	oFile.seek(offFile);
101	except:
102	return reporter.error('seek error');
103	while offFile < self.cbContent:
104	cbToRead = self.cbContent - offFile;
105	if cbToRead > 256*1024:
106	cbToRead = 256*1024;
107	try:
108	abRead = oFile.read(cbToRead);
109	except:
110	return reporter.error('read error at offset %s' % (offFile,));
111	cbRead = len(abRead);
112	if cbRead == 0:
113	return reporter.error('premature end of file at offset %s' % (offFile,));
114	if not utils.areBytesEqual(abRead, self.abContent[offFile:(offFile + cbRead)]):
115	return reporter.error('%s byte block at offset %s differs' % (cbRead, offFile,));
116	# Advance:
117	offFile += cbRead;
118
119	return True;
120
121
122	class TestDir(TestFsObj):
123	""" A file object in the guest. """
124	def __init__(self, oParent, sPath):
125	TestFsObj.__init__(self, oParent, sPath);
126	self.aoChildren = [] # type: list(TestFsObj)
127	self.dChildrenUpper = {} # type: dict(str, TestFsObj)
128
129	def contains(self, sName):
130	""" Checks if the directory contains the given name. """
131	return sName.upper() in self.dChildrenUpper
132
133
134	class TestFileSet(object):
135	"""
136	A generated set of files and directories for use in a test.
137
138	Can be wrapped up into a tarball or written directly to the file system.
139	"""
140
141	ksReservedWinOS2 = '/\\"*:<>?\|\t\v\n\r\f\a\b';
142	ksReservedUnix = '/';
143	ksReservedTrailingWinOS2 = ' .';
144	ksReservedTrailingUnix = '';
145
146	## @name Path style.
147	## @{
148
149	## @}
150
151	def __init__(self, fDosStyle, sBasePath, sSubDir, # pylint: disable=too-many-arguments
152	asCompatibleWith = None, # List of getHostOs values to the names must be compatible with.
153	oRngFileSizes = xrange(0, 16384),
154	oRngManyFiles = xrange(128, 512),
155	oRngTreeFiles = xrange(128, 384),
156	oRngTreeDepth = xrange(92, 256),
157	oRngTreeDirs = xrange(2, 16),
158	cchMaxPath = 230,
159	cchMaxName = 230,
160	uSeed = None):
161	## @name Parameters
162	## @{
163	self.fDosStyle = fDosStyle;
164	self.sMinStyle = 'win' if fDosStyle else 'linux';
165	if asCompatibleWith is not None:
166	for sOs in asCompatibleWith:
167	assert sOs in ('win', 'os2', 'darwin', 'linux', 'solaris',), sOs;
168	if 'os2' in asCompatibleWith:
169	self.sMinStyle = 'os2';
170	elif 'win' in asCompatibleWith:
171	self.sMinStyle = 'win';
172	self.sBasePath = sBasePath;
173	self.sSubDir = sSubDir;
174	self.oRngFileSizes = oRngFileSizes;
175	self.oRngManyFiles = oRngManyFiles;
176	self.oRngTreeFiles = oRngTreeFiles;
177	self.oRngTreeDepth = oRngTreeDepth;
178	self.oRngTreeDirs = oRngTreeDirs;
179	self.cchMaxPath = cchMaxPath;
180	self.cchMaxName = cchMaxName
181	## @}
182
183	## @name Charset stuff
184	## @todo allow more chars for unix hosts + guests.
185	## @todo include unicode stuff, except on OS/2 and DOS.
186	## @{
187	## The filename charset.
188	self.sFileCharset = string.printable;
189	## Set of characters that should not trail a guest filename.
190	self.sReservedTrailing = self.ksReservedTrailingWinOS2;
191	if self.sMinStyle in ('win', 'os2'):
192	for ch in self.ksReservedWinOS2:
193	self.sFileCharset = self.sFileCharset.replace(ch, '');
194	else:
195	self.sReservedTrailing = self.ksReservedTrailingUnix;
196	for ch in self.ksReservedUnix:
197	self.sFileCharset = self.sFileCharset.replace(ch, '');
198	# More spaces and dot:
199	self.sFileCharset += ' ...';
200	## @}
201
202	## The root directory.
203	self.oRoot = None # type: TestDir;
204	## An empty directory (under root).
205	self.oEmptyDir = None # type: TestDir;
206
207	## A directory with a lot of files in it.
208	self.oManyDir = None # type: TestDir;
209
210	## A directory with a mixed tree structure under it.
211	self.oTreeDir = None # type: TestDir;
212	## Number of files in oTreeDir.
213	self.cTreeFiles = 0;
214	## Number of directories under oTreeDir.
215	self.cTreeDirs = 0;
216	## Number of other file types under oTreeDir.
217	self.cTreeOthers = 0;
218
219	## All directories in creation order.
220	self.aoDirs = [] # type: list(TestDir);
221	## All files in creation order.
222	self.aoFiles = [] # type: list(TestFile);
223	## Path to object lookup.
224	self.dPaths = {} # type: dict(str, TestFsObj);
225
226	#
227	# Do the creating.
228	#
229	self.uSeed = uSeed if uSeed is not None else utils.timestampMilli();
230	self.oRandom = random.Random();
231	self.oRandom.seed(self.uSeed);
232	reporter.log('prepareGuestForTesting: random seed %s' % (self.uSeed,));
233
234	self.__createTestStuff();
235
236	def __createFilename(self, oParent, sCharset, sReservedTrailing):
237	"""
238	Creates a filename contains random characters from sCharset and together
239	with oParent.sPath doesn't exceed the given max chars in length.
240	"""
241	## @todo Consider extending this to take UTF-8 and UTF-16 encoding so we
242	## can safely use the full unicode range. Need to check how
243	## RTZipTarCmd handles file name encoding in general...
244
245	if oParent:
246	cchMaxName = self.cchMaxPath - len(oParent.sPath) - 1;
247	else:
248	cchMaxName = self.cchMaxPath - 4;
249	if cchMaxName > self.cchMaxName:
250	cchMaxName = self.cchMaxName;
251	if cchMaxName <= 1:
252	cchMaxName = 2;
253
254	while True:
255	cchName = self.oRandom.randrange(1, cchMaxName);
256	sName = ''.join(self.oRandom.choice(sCharset) for _ in xrange(cchName));
257	if oParent is None or not oParent.contains(sName):
258	if sName[-1] not in sReservedTrailing:
259	if sName not in ('.', '..',):
260	return sName;
261	return ''; # never reached, but makes pylint happy.
262
263	def generateFilenameEx(self, cchMax = -1, cchMin = -1):
264	"""
265	Generates a filename according to the given specs.
266
267	This is for external use, whereas __createFilename is for internal.
268
269	Returns generated filename.
270	"""
271	assert cchMax == -1 or (cchMax >= 1 and cchMax > cchMin);
272	if cchMin <= 0:
273	cchMin = 1;
274	if cchMax < cchMin:
275	cchMax = self.cchMaxName;
276
277	while True:
278	cchName = self.oRandom.randrange(cchMin, cchMax + 1);
279	sName = ''.join(self.oRandom.choice(self.sFileCharset) for _ in xrange(cchName));
280	if sName[-1] not in self.sReservedTrailing:
281	if sName not in ('.', '..',):
282	return sName;
283	return ''; # never reached, but makes pylint happy.
284
285	def __createTestDir(self, oParent, sDir):
286	"""
287	Creates a test directory.
288	"""
289	oDir = TestDir(oParent, sDir);
290	self.aoDirs.append(oDir);
291	self.dPaths[sDir] = oDir;
292	return oDir;
293
294	def __createTestFile(self, oParent, sFile):
295	"""
296	Creates a test file with random size up to cbMaxContent and random content.
297	"""
298	cbFile = self.oRandom.choice(self.oRngFileSizes);
299	abContent = bytearray(self.oRandom.getrandbits(8) for _ in xrange(cbFile));
300
301	oFile = TestFile(oParent, sFile, abContent);
302	self.aoFiles.append(oFile);
303	self.dPaths[sFile] = oFile;
304	return oFile;
305
306	def __createTestStuff(self):
307	"""
308	Create a random file set that we can work on in the tests.
309	Returns True/False.
310	"""
311
312	#
313	# Create the root test dir.
314	#
315	sRoot = pathutils.joinEx(self.fDosStyle, self.sBasePath, self.sSubDir);
316	self.oRoot = self.__createTestDir(None, sRoot);
317	self.oEmptyDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'empty'));
318
319	#
320	# Create a directory with lots of files in it:
321	#
322	oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'many'));
323	self.oManyDir = oDir;
324	cManyFiles = self.oRandom.choice(self.oRngManyFiles);
325	for _ in xrange(cManyFiles):
326	sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
327	self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
328
329	#
330	# Generate a tree of files and dirs.
331	#
332	oDir = self.__createTestDir(self.oRoot, pathutils.joinEx(self.fDosStyle, sRoot, 'tree'));
333	uMaxDepth = self.oRandom.choice(self.oRngTreeDepth);
334	cMaxFiles = self.oRandom.choice(self.oRngTreeFiles);
335	cMaxDirs = self.oRandom.choice(self.oRngTreeDirs);
336	self.oTreeDir = oDir;
337	self.cTreeFiles = 0;
338	self.cTreeDirs = 0;
339	uDepth = 0;
340	while self.cTreeFiles < cMaxFiles and self.cTreeDirs < cMaxDirs:
341	iAction = self.oRandom.randrange(0, 2+1);
342	# 0: Add a file:
343	if iAction == 0 and self.cTreeFiles < cMaxFiles and len(oDir.sPath) < 230 - 2:
344	sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
345	self.__createTestFile(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
346	self.cTreeFiles += 1;
347	# 1: Add a subdirector and descend into it:
348	elif iAction == 1 and self.cTreeDirs < cMaxDirs and uDepth < uMaxDepth and len(oDir.sPath) < 220:
349	sName = self.__createFilename(oDir, self.sFileCharset, self.sReservedTrailing);
350	oDir = self.__createTestDir(oDir, pathutils.joinEx(self.fDosStyle, oDir.sPath, sName));
351	self.cTreeDirs += 1;
352	uDepth += 1;
353	# 2: Ascend to parent dir:
354	elif iAction == 2 and uDepth > 0:
355	oDir = oDir.oParent;
356	uDepth -= 1;
357
358	return True;
359
360	def createTarball(self, sTarFileHst):
361	"""
362	Creates a tarball on the host.
363	Returns success indicator.
364	"""
365	reporter.log('Creating tarball "%s" with test files for the guest...' % (sTarFileHst,));
366
367	cchSkip = len(self.sBasePath) + 1;
368
369	# Open the tarball:
370	try:
371	oTarFile = tarfile.open(sTarFileHst, 'w:gz');
372	except:
373	return reporter.errorXcpt('Failed to open new tar file: %s' % (sTarFileHst,));
374
375	# Directories:
376	for oDir in self.aoDirs:
377	sPath = oDir.sPath[cchSkip:];
378	if self.fDosStyle:
379	sPath = sPath.replace('\\', '/');
380	oTarInfo = tarfile.TarInfo(sPath + '/');
381	oTarInfo.mode = 0o777;
382	oTarInfo.type = tarfile.DIRTYPE;
383	try:
384	oTarFile.addfile(oTarInfo);
385	except:
386	return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oDir.sPath,));
387
388	# Files:
389	for oFile in self.aoFiles:
390	sPath = oFile.sPath[cchSkip:];
391	if self.fDosStyle:
392	sPath = sPath.replace('\\', '/');
393	oTarInfo = tarfile.TarInfo(sPath);
394	oTarInfo.size = len(oFile.abContent);
395	oFile.off = 0;
396	try:
397	oTarFile.addfile(oTarInfo, oFile);
398	except:
399	return reporter.errorXcpt('Failed adding directory tarfile: %s' % (oFile.sPath,));
400
401	# Complete the tarball.
402	try:
403	oTarFile.close();
404	except:
405	return reporter.errorXcpt('Error closing new tar file: %s' % (sTarFileHst,));
406	return True;
407
408	def writeToDisk(self, sAltBase = None):
409	"""
410	Writes out the files to disk.
411	Returns True on success, False + error logging on failure.
412	"""
413
414	# We only need to flip DOS slashes to unix ones, since windows & OS/2 can handle unix slashes.
415	fDosToUnix = self.fDosStyle and os.path.sep != '\\';
416
417	# The directories:
418	for oDir in self.aoDirs:
419	sPath = oDir.sPath;
420	if sAltBase:
421	if fDosToUnix:
422	sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep);
423	else:
424	sPath = sAltBase + sPath[len(self.sBasePath):];
425	elif fDosToUnix:
426	sPath = sPath.replace('\\', os.path.sep);
427
428	try:
429	os.mkdir(sPath, 0o770);
430	except:
431	return reporter.errorXcpt('mkdir(%s) failed' % (sPath,));
432
433	# The files:
434	for oFile in self.aoFiles:
435	sPath = oFile.sPath;
436	if sAltBase:
437	if fDosToUnix:
438	sPath = sAltBase + sPath[len(self.sBasePath):].replace('\\', os.path.sep);
439	else:
440	sPath = sAltBase + sPath[len(self.sBasePath):];
441	elif fDosToUnix:
442	sPath = sPath.replace('\\', os.path.sep);
443
444	try:
445	oFile = open(sPath, 'wb');
446	except:
447	return reporter.errorXcpt('open(%s, "wb") failed' % (sPath,));
448	try:
449	if sys.version_info[0] < 3:
450	oFile.write(bytes(oFile.abContent));
451	else:
452	oFile.write(oFile.abContent);
453	except:
454	try: oFile.close();
455	except: pass;
456	return reporter.errorXcpt('%s: write(%s bytes) failed' % (sPath, oFile.cbContent,));
457	try:
458	oFile.close();
459	except:
460	return reporter.errorXcpt('%s: close() failed' % (sPath,));
461
462	return True;
463
464
465	def chooseRandomFile(self):
466	"""
467	Returns a random file.
468	"""
469	return self.aoFiles[self.oRandom.choice(xrange(len(self.aoFiles)))];
470
471	def chooseRandomDirFromTree(self, fLeaf = False, fNonEmpty = False):
472	"""
473	Returns a random directory from the tree (self.oTreeDir).
474	"""
475	while True:
476	oDir = self.aoDirs[self.oRandom.choice(xrange(len(self.aoDirs)))];
477	# Check fNonEmpty requirement:
478	if not fNonEmpty or oDir.aoChildren:
479	# Check leaf requirement:
480	if not fLeaf:
481	for oChild in oDir.aoChildren:
482	if isinstance(oChild, TestDir):
483	continue; # skip it.
484
485	# Return if in the tree:
486	oParent = oDir.oParent;
487	while oParent is not None:
488	if oParent is self.oTreeDir:
489	return oDir;
490	oParent = oParent.oParent;
491	return None; # make pylint happy
492
493	#
494	# Unit testing.
495	#
496
497	# pylint: disable=missing-docstring
498	# pylint: disable=undefined-variable
499	class TestFileSetUnitTests(unittest.TestCase):
500	def testGeneral(self):
501	oSet = TestFileSet(False, '/tmp', 'unittest');
502	self.assertTrue(isinstance(oSet.chooseRandomDirFromTree(), TestDir));
503	self.assertTrue(isinstance(oSet.chooseRandomFile(), TestFile));
504
505
506	if __name__ == '__main__':
507	unittest.main();
508	# not reached.
509

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/testdriver/testfileset.py@ 79198

Download in other formats: