storagecfg.py@ 79593

Last change on this file since 79593 was 79593, checked in by vboxsync, 5 years ago
ValidationKit: Build fix
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 21.8 KB

Line
1	# -- coding: utf-8 --
2	# $Id: storagecfg.py 79593 2019-07-08 12:41:20Z vboxsync $
3
4	"""
5	VirtualBox Validation Kit - Storage test configuration API.
6	"""
7
8	__copyright__ = \
9	"""
10	Copyright (C) 2016-2019 Oracle Corporation
11
12	This file is part of VirtualBox Open Source Edition (OSE), as
13	available from http://www.virtualbox.org. This file is free software;
14	you can redistribute it and/or modify it under the terms of the GNU
15	General Public License (GPL) as published by the Free Software
16	Foundation, in version 2 as it comes in the "COPYING" file of the
17	VirtualBox OSE distribution. VirtualBox OSE is distributed in the
18	hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
19
20	The contents of this file may alternatively be used under the terms
21	of the Common Development and Distribution License Version 1.0
22	(CDDL) only, as it comes in the "COPYING.CDDL" file of the
23	VirtualBox OSE distribution, in which case the provisions of the
24	CDDL are applicable instead of those of the GPL.
25
26	You may elect to license modified versions of this file under the
27	terms and conditions of either the GPL or the CDDL or both.
28	"""
29	__version__ = "$Revision: 79593 $"
30
31	# Standard Python imports.
32	import os;
33	import re;
34
35
36	class StorageDisk(object):
37	"""
38	Class representing a disk for testing.
39	"""
40
41	def __init__(self, sPath, fRamDisk = False):
42	self.sPath = sPath;
43	self.fUsed = False;
44	self.fRamDisk = fRamDisk;
45
46	def getPath(self):
47	"""
48	Return the disk path.
49	"""
50	return self.sPath;
51
52	def isUsed(self):
53	"""
54	Returns whether the disk is currently in use.
55	"""
56	return self.fUsed;
57
58	def isRamDisk(self):
59	"""
60	Returns whether the disk objecthas a RAM backing.
61	"""
62	return self.fRamDisk;
63
64	def setUsed(self, fUsed):
65	"""
66	Sets the used flag for the disk.
67	"""
68	if fUsed:
69	if self.fUsed:
70	return False;
71
72	self.fUsed = True;
73	else:
74	self.fUsed = fUsed;
75
76	return True;
77
78	class StorageConfigOs(object):
79	"""
80	Base class for a single hosts OS storage configuration.
81	"""
82
83	def _getDisksMatchingRegExpWithPath(self, sPath, sRegExp):
84	"""
85	Adds new disks to the config matching the given regular expression.
86	"""
87
88	lstDisks = [];
89	oRegExp = re.compile(sRegExp);
90	asFiles = os.listdir(sPath);
91	for sFile in asFiles:
92	if oRegExp.match(os.path.basename(sFile)) and os.path.exists(sPath + '/' + sFile):
93	lstDisks.append(StorageDisk(sPath + '/' + sFile));
94
95	return lstDisks;
96
97	class StorageConfigOsSolaris(StorageConfigOs):
98	"""
99	Class implementing the Solaris specifics for a storage configuration.
100	"""
101
102	def __init__(self):
103	StorageConfigOs.__init__(self);
104	self.idxRamDisk = 0;
105
106	def _getActivePoolsStartingWith(self, oExec, sPoolIdStart):
107	"""
108	Returns a list of pools starting with the given ID or None on failure.
109	"""
110	lstPools = None;
111	fRc, sOutput, _ = oExec.execBinary('zpool', ('list', '-H'));
112	if fRc:
113	lstPools = [];
114	asPools = sOutput.splitlines();
115	for sPool in asPools:
116	if sPool.startswith(sPoolIdStart):
117	# Extract the whole name and add it to the list.
118	asItems = sPool.split('\t');
119	lstPools.append(asItems[0]);
120	return lstPools;
121
122	def _getActiveVolumesInPoolStartingWith(self, oExec, sPool, sVolumeIdStart):
123	"""
124	Returns a list of active volumes for the given pool starting with the given
125	identifier or None on failure.
126	"""
127	lstVolumes = None;
128	fRc, sOutput, _ = oExec.execBinary('zfs', ('list', '-H'));
129	if fRc:
130	lstVolumes = [];
131	asVolumes = sOutput.splitlines();
132	for sVolume in asVolumes:
133	if sVolume.startswith(sPool + '/' + sVolumeIdStart):
134	# Extract the whole name and add it to the list.
135	asItems = sVolume.split('\t');
136	lstVolumes.append(asItems[0]);
137	return lstVolumes;
138
139	def getDisksMatchingRegExp(self, sRegExp):
140	"""
141	Returns a list of disks matching the regular expression.
142	"""
143	return self._getDisksMatchingRegExpWithPath('/dev/dsk', sRegExp);
144
145	def getMntBase(self):
146	"""
147	Returns the mountpoint base for the host.
148	"""
149	return '/pools';
150
151	def createStoragePool(self, oExec, sPool, asDisks, sRaidLvl):
152	"""
153	Creates a new storage pool with the given disks and the given RAID level.
154	"""
155	sZPoolRaid = None;
156	if len(asDisks) > 1 and (sRaidLvl == 'raid5' or sRaidLvl is None):
157	sZPoolRaid = 'raidz';
158
159	fRc = True;
160	if sZPoolRaid is not None:
161	fRc = oExec.execBinaryNoStdOut('zpool', ('create', '-f', sPool, sZPoolRaid,) + tuple(asDisks));
162	else:
163	fRc = oExec.execBinaryNoStdOut('zpool', ('create', '-f', sPool,) + tuple(asDisks));
164
165	return fRc;
166
167	def createVolume(self, oExec, sPool, sVol, sMountPoint, cbVol = None):
168	"""
169	Creates and mounts a filesystem at the given mountpoint using the
170	given pool and volume IDs.
171	"""
172	fRc = True;
173	if cbVol is not None:
174	fRc = oExec.execBinaryNoStdOut('zfs', ('create', '-o', 'mountpoint='+sMountPoint, '-V', cbVol, sPool + '/' + sVol));
175	else:
176	fRc = oExec.execBinaryNoStdOut('zfs', ('create', '-o', 'mountpoint='+sMountPoint, sPool + '/' + sVol));
177
178	return fRc;
179
180	def destroyVolume(self, oExec, sPool, sVol):
181	"""
182	Destroys the given volume.
183	"""
184	fRc = oExec.execBinaryNoStdOut('zfs', ('destroy', sPool + '/' + sVol));
185	return fRc;
186
187	def destroyPool(self, oExec, sPool):
188	"""
189	Destroys the given storage pool.
190	"""
191	fRc = oExec.execBinaryNoStdOut('zpool', ('destroy', sPool));
192	return fRc;
193
194	def cleanupPoolsAndVolumes(self, oExec, sPoolIdStart, sVolIdStart):
195	"""
196	Cleans up any pools and volumes starting with the name in the given
197	parameters.
198	"""
199	fRc = True;
200	lstPools = self._getActivePoolsStartingWith(oExec, sPoolIdStart);
201	if lstPools is not None:
202	for sPool in lstPools:
203	lstVolumes = self._getActiveVolumesInPoolStartingWith(oExec, sPool, sVolIdStart);
204	if lstVolumes is not None:
205	# Destroy all the volumes first
206	for sVolume in lstVolumes:
207	fRc2 = oExec.execBinaryNoStdOut('zfs', ('destroy', sVolume));
208	if not fRc2:
209	fRc = fRc2;
210
211	# Destroy the pool
212	fRc2 = self.destroyPool(oExec, sPool);
213	if not fRc2:
214	fRc = fRc2;
215	else:
216	fRc = False;
217	else:
218	fRc = False;
219
220	return fRc;
221
222	def createRamDisk(self, oExec, cbRamDisk):
223	"""
224	Creates a RAM backed disk with the given size.
225	"""
226	oDisk = None;
227	sRamDiskName = 'ramdisk%u' % (self.idxRamDisk,);
228	fRc, _ , _ = oExec.execBinary('ramdiskadm', ('-a', sRamDiskName, str(cbRamDisk)));
229	if fRc:
230	self.idxRamDisk += 1;
231	oDisk = StorageDisk('/dev/ramdisk/%s' % (sRamDiskName, ), True);
232
233	return oDisk;
234
235	def destroyRamDisk(self, oExec, oDisk):
236	"""
237	Destroys the given ramdisk object.
238	"""
239	sRamDiskName = os.path.basename(oDisk.getPath());
240	return oExec.execBinaryNoStdOut('ramdiskadm', ('-d', sRamDiskName));
241
242	class StorageConfigOsLinux(StorageConfigOs):
243	"""
244	Class implementing the Linux specifics for a storage configuration.
245	"""
246
247	def __init__(self):
248	StorageConfigOs.__init__(self);
249	self.dSimplePools = { }; # Simple storage pools which don't use lvm (just one partition)
250	self.dMounts = { }; # Pool/Volume to mountpoint mapping.
251
252	def _getDmRaidLevelFromLvl(self, sRaidLvl):
253	"""
254	Converts our raid level indicators to something mdadm can understand.
255	"""
256	if sRaidLvl is None or sRaidLvl == 'raid0':
257	return 'stripe';
258	if sRaidLvl == 'raid5':
259	return '5';
260	if sRaidLvl == 'raid1':
261	return 'mirror';
262	return 'stripe';
263
264	def getDisksMatchingRegExp(self, sRegExp):
265	"""
266	Returns a list of disks matching the regular expression.
267	"""
268	return self._getDisksMatchingRegExpWithPath('/dev/', sRegExp);
269
270	def getMntBase(self):
271	"""
272	Returns the mountpoint base for the host.
273	"""
274	return '/mnt';
275
276	def createStoragePool(self, oExec, sPool, asDisks, sRaidLvl):
277	"""
278	Creates a new storage pool with the given disks and the given RAID level.
279	"""
280	fRc = True;
281	if len(asDisks) == 1 and sRaidLvl is None:
282	# Doesn't require LVM, put into the simple pools dictionary so we can
283	# use it when creating a volume later.
284	self.dSimplePools[sPool] = asDisks[0];
285	else:
286	# If a RAID is required use dm-raid first to create one.
287	asLvmPvDisks = asDisks;
288	fRc = oExec.execBinaryNoStdOut('mdadm', ('--create', '/dev/md0', '--assume-clean',
289	'--level=' + self._getDmRaidLevelFromLvl(sRaidLvl),
290	'--raid-devices=' + str(len(asDisks))) + tuple(asDisks));
291	if fRc:
292	# /dev/md0 is the only block device to use for our volume group.
293	asLvmPvDisks = [ '/dev/md0' ];
294
295	# Create a physical volume on every disk first.
296	for sLvmPvDisk in asLvmPvDisks:
297	fRc = oExec.execBinaryNoStdOut('pvcreate', (sLvmPvDisk, ));
298	if not fRc:
299	break;
300
301	if fRc:
302	# Create volume group with all physical volumes included
303	fRc = oExec.execBinaryNoStdOut('vgcreate', (sPool, ) + tuple(asLvmPvDisks));
304	return fRc;
305
306	def createVolume(self, oExec, sPool, sVol, sMountPoint, cbVol = None):
307	"""
308	Creates and mounts a filesystem at the given mountpoint using the
309	given pool and volume IDs.
310	"""
311	fRc = True;
312	sBlkDev = None;
313	if sPool in self.dSimplePools:
314	sDiskPath = self.dSimplePools.get(sPool);
315	if sDiskPath.find('zram') != -1:
316	sBlkDev = sDiskPath;
317	else:
318	# Create a partition with the requested size
319	sFdiskScript = ';\n'; # Single partition filling everything
320	if cbVol is not None:
321	sFdiskScript = ',' + str(cbVol // 512) + '\n'; # Get number of sectors
322	fRc = oExec.execBinaryNoStdOut('sfdisk', ('--no-reread', '--wipe', 'always', '-q', '-f', sDiskPath), \
323	sFdiskScript);
324	if fRc:
325	if sDiskPath.find('nvme') != -1:
326	sBlkDev = sDiskPath + 'p1';
327	else:
328	sBlkDev = sDiskPath + '1';
329	else:
330	if cbVol is None:
331	fRc = oExec.execBinaryNoStdOut('lvcreate', ('-l', '100%FREE', '-n', sVol, sPool));
332	else:
333	fRc = oExec.execBinaryNoStdOut('lvcreate', ('-L', str(cbVol), '-n', sVol, sPool));
334	if fRc:
335	sBlkDev = '/dev/mapper' + sPool + '-' + sVol;
336
337	if fRc is True and sBlkDev is not None:
338	# Create a filesystem and mount it
339	fRc = oExec.execBinaryNoStdOut('mkfs.ext4', ('-F', '-F', sBlkDev,));
340	fRc = fRc and oExec.mkDir(sMountPoint);
341	fRc = fRc and oExec.execBinaryNoStdOut('mount', (sBlkDev, sMountPoint));
342	if fRc:
343	self.dMounts[sPool + '/' + sVol] = sMountPoint;
344	return fRc;
345
346	def destroyVolume(self, oExec, sPool, sVol):
347	"""
348	Destroys the given volume.
349	"""
350	# Unmount first
351	sMountPoint = self.dMounts[sPool + '/' + sVol];
352	fRc = oExec.execBinaryNoStdOut('umount', (sMountPoint,));
353	self.dMounts.pop(sPool + '/' + sVol);
354	oExec.rmDir(sMountPoint);
355	if sPool in self.dSimplePools:
356	# Wipe partition table
357	sDiskPath = self.dSimplePools.get(sPool);
358	if sDiskPath.find('zram') == -1:
359	fRc = oExec.execBinaryNoStdOut('sfdisk', ('--no-reread', '--wipe', 'always', '-q', '-f', '--delete', \
360	sDiskPath));
361	else:
362	fRc = oExec.execBinaryNoStdOut('lvremove', (sPool + '/' + sVol,));
363	return fRc;
364
365	def destroyPool(self, oExec, sPool):
366	"""
367	Destroys the given storage pool.
368	"""
369	fRc = True;
370	if sPool in self.dSimplePools:
371	self.dSimplePools.pop(sPool);
372	else:
373	fRc = oExec.execBinaryNoStdOut('vgremove', (sPool,));
374	return fRc;
375
376	def cleanupPoolsAndVolumes(self, oExec, sPoolIdStart, sVolIdStart):
377	"""
378	Cleans up any pools and volumes starting with the name in the given
379	parameters.
380	"""
381	# @todo: Needs implementation, for LVM based configs a similar approach can be used
382	# as for Solaris.
383	_ = oExec;
384	_ = sPoolIdStart;
385	_ = sVolIdStart;
386	return True;
387
388	def createRamDisk(self, oExec, cbRamDisk):
389	"""
390	Creates a RAM backed disk with the given size.
391	"""
392	# Make sure the ZRAM module is loaded.
393	oDisk = None;
394	fRc = oExec.execBinaryNoStdOut('modprobe', ('zram',));
395	if fRc:
396	fRc, sOut, _ = oExec.execBinary('zramctl', ('--raw', '-f', '-s', str(cbRamDisk)));
397	if fRc:
398	oDisk = StorageDisk(sOut.rstrip(), True);
399
400	return oDisk;
401
402	def destroyRamDisk(self, oExec, oDisk):
403	"""
404	Destroys the given ramdisk object.
405	"""
406	return oExec.execBinaryNoStdOut('zramctl', ('-r', oDisk.getPath()));
407
408	## @name Host disk config types.
409	## @{
410	g_ksDiskCfgStatic = 'StaticDir';
411	g_ksDiskCfgRegExp = 'RegExp';
412	g_ksDiskCfgList = 'DiskList';
413	## @}
414
415	class DiskCfg(object):
416	"""
417	Host disk configuration.
418	"""
419
420	def __init__(self, sTargetOs, sCfgType, oDisks):
421	self.sTargetOs = sTargetOs;
422	self.sCfgType = sCfgType;
423	self.oDisks = oDisks;
424
425	def getTargetOs(self):
426	return self.sTargetOs;
427
428	def getCfgType(self):
429	return self.sCfgType;
430
431	def isCfgStaticDir(self):
432	return self.sCfgType == g_ksDiskCfgStatic;
433
434	def isCfgRegExp(self):
435	return self.sCfgType == g_ksDiskCfgRegExp;
436
437	def isCfgList(self):
438	return self.sCfgType == g_ksDiskCfgList;
439
440	def getDisks(self):
441	return self.oDisks;
442
443	class StorageCfg(object):
444	"""
445	Storage configuration helper class taking care of the different host OS.
446	"""
447
448	def __init__(self, oExec, oDiskCfg):
449	self.oExec = oExec;
450	self.lstDisks = [ ]; # List of disks present in the system.
451	self.dPools = { }; # Dictionary of storage pools.
452	self.dVols = { }; # Dictionary of volumes.
453	self.iPoolId = 0;
454	self.iVolId = 0;
455	self.oDiskCfg = oDiskCfg;
456
457	fRc = True;
458	oStorOs = None;
459	if oDiskCfg.getTargetOs() == 'solaris':
460	oStorOs = StorageConfigOsSolaris();
461	elif oDiskCfg.getTargetOs() == 'linux':
462	oStorOs = StorageConfigOsLinux(); # pylint: disable=redefined-variable-type
463	elif not oDiskCfg.isCfgStaticDir(): # For unknown hosts we only a static testing directory we don't care about setting up.
464	fRc = False;
465
466	if fRc:
467	self.oStorOs = oStorOs;
468	if oDiskCfg.isCfgRegExp():
469	self.lstDisks = oStorOs.getDisksMatchingRegExp(oDiskCfg.getDisks());
470	elif oDiskCfg.isCfgList():
471	# Assume a list of of disks and add.
472	for sDisk in oDiskCfg.getDisks():
473	self.lstDisks.append(StorageDisk(sDisk));
474
475	def __del__(self):
476	self.cleanup();
477
478	def cleanup(self):
479	"""
480	Cleans up any created storage configs.
481	"""
482
483	if not self.oDiskCfg.isCfgStaticDir():
484	# Destroy all volumes first.
485	for sMountPoint in self.dVols.keys(): # pylint: disable=consider-iterating-dictionary
486	self.destroyVolume(sMountPoint);
487
488	# Destroy all pools.
489	for sPool in self.dPools.keys(): # pylint: disable=consider-iterating-dictionary
490	self.destroyStoragePool(sPool);
491
492	self.dVols.clear();
493	self.dPools.clear();
494	self.oDiskCfg = None;
495	self.iPoolId = 0;
496	self.iVolId = 0;
497
498	def getRawDisk(self):
499	"""
500	Returns a raw disk device from the list of free devices for use.
501	"""
502
503	for oDisk in self.lstDisks:
504	if oDisk.isUsed() is False:
505	oDisk.setUsed(True);
506	return oDisk.getPath();
507
508	return None;
509
510	def getUnusedDiskCount(self):
511	"""
512	Returns the number of unused disks.
513	"""
514
515	cDisksUnused = 0;
516	for oDisk in self.lstDisks:
517	if not oDisk.isUsed():
518	cDisksUnused += 1;
519
520	return cDisksUnused;
521
522	def createStoragePool(self, cDisks = 0, sRaidLvl = None,
523	cbPool = None, fRamDisk = False):
524	"""
525	Create a new storage pool
526	"""
527	lstDisks = [ ];
528	fRc = True;
529	sPool = None;
530
531	if not self.oDiskCfg.isCfgStaticDir():
532	if fRamDisk:
533	oDisk = self.oStorOs.createRamDisk(self.oExec, cbPool);
534	if oDisk is not None:
535	lstDisks.append(oDisk);
536	cDisks = 1;
537	else:
538	if cDisks == 0:
539	cDisks = self.getUnusedDiskCount();
540
541	for oDisk in self.lstDisks:
542	if not oDisk.isUsed():
543	oDisk.setUsed(True);
544	lstDisks.append(oDisk);
545	if len(lstDisks) == cDisks:
546	break;
547
548	# Enough drives to satisfy the request?
549	if len(lstDisks) == cDisks:
550	# Create a list of all device paths
551	lstDiskPaths = [ ];
552	for oDisk in lstDisks:
553	lstDiskPaths.append(oDisk.getPath());
554
555	# Find a name for the pool
556	sPool = 'pool' + str(self.iPoolId);
557	self.iPoolId += 1;
558
559	fRc = self.oStorOs.createStoragePool(self.oExec, sPool, lstDiskPaths, sRaidLvl);
560	if fRc:
561	self.dPools[sPool] = lstDisks;
562	else:
563	self.iPoolId -= 1;
564	else:
565	fRc = False;
566
567	# Cleanup in case of error.
568	if not fRc:
569	for oDisk in lstDisks:
570	oDisk.setUsed(False);
571	if oDisk.isRamDisk():
572	self.oStorOs.destroyRamDisk(self.oExec, oDisk);
573	else:
574	sPool = 'StaticDummy';
575
576	return fRc, sPool;
577
578	def destroyStoragePool(self, sPool):
579	"""
580	Destroys the storage pool with the given ID.
581	"""
582
583	fRc = True;
584
585	if not self.oDiskCfg.isCfgStaticDir():
586	lstDisks = self.dPools.get(sPool);
587	if lstDisks is not None:
588	fRc = self.oStorOs.destroyPool(self.oExec, sPool);
589	if fRc:
590	# Mark disks as unused
591	self.dPools.pop(sPool);
592	for oDisk in lstDisks:
593	oDisk.setUsed(False);
594	if oDisk.isRamDisk():
595	self.oStorOs.destroyRamDisk(self.oExec, oDisk);
596	else:
597	fRc = False;
598
599	return fRc;
600
601	def createVolume(self, sPool, cbVol = None):
602	"""
603	Creates a new volume from the given pool returning the mountpoint.
604	"""
605
606	fRc = True;
607	sMountPoint = None;
608	if not self.oDiskCfg.isCfgStaticDir():
609	if sPool in self.dPools:
610	sVol = 'vol' + str(self.iVolId);
611	sMountPoint = self.oStorOs.getMntBase() + '/' + sVol;
612	self.iVolId += 1;
613	fRc = self.oStorOs.createVolume(self.oExec, sPool, sVol, sMountPoint, cbVol);
614	if fRc:
615	self.dVols[sMountPoint] = (sVol, sPool);
616	else:
617	self.iVolId -= 1;
618	else:
619	fRc = False;
620	else:
621	sMountPoint = self.oDiskCfg.getDisks();
622
623	return fRc, sMountPoint;
624
625	def destroyVolume(self, sMountPoint):
626	"""
627	Destroy the volume at the given mount point.
628	"""
629
630	fRc = True;
631	if not self.oDiskCfg.isCfgStaticDir():
632	sVol, sPool = self.dVols.get(sMountPoint);
633	if sVol is not None:
634	fRc = self.oStorOs.destroyVolume(self.oExec, sPool, sVol);
635	if fRc:
636	self.dVols.pop(sMountPoint);
637	else:
638	fRc = False;
639
640	return fRc;
641
642	def mkDirOnVolume(self, sMountPoint, sDir, fMode = 0o700):
643	"""
644	Creates a new directory on the volume pointed to by the given mount point.
645	"""
646	return self.oExec.mkDir(sMountPoint + '/' + sDir, fMode);
647
648	def cleanupLeftovers(self):
649	"""
650	Tries to cleanup any leftover pools and volumes from a failed previous run.
651	"""
652	if not self.oDiskCfg.isCfgStaticDir():
653	return self.oStorOs.cleanupPoolsAndVolumes(self.oExec, 'pool', 'vol');
654
655	fRc = True;
656	for sEntry in os.listdir(self.oDiskCfg.getDisks()):
657	fRc = fRc and self.oExec.rmTree(os.path.join(self.oDiskCfg.getDisks(), sEntry));
658
659	return fRc;

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/ValidationKit/tests/storage/storagecfg.py@ 79593

Download in other formats: