| 1 | #!/usr/bin/env python
|
|---|
| 2 | # -*- coding: utf-8 -*-
|
|---|
| 3 | # $Id: quota.py 98103 2023-01-17 14:15:46Z vboxsync $
|
|---|
| 4 | # pylint: disable=line-too-long
|
|---|
| 5 |
|
|---|
| 6 | """
|
|---|
| 7 | A cronjob that applies quotas to large files in testsets.
|
|---|
| 8 | """
|
|---|
| 9 |
|
|---|
| 10 | from __future__ import print_function;
|
|---|
| 11 |
|
|---|
| 12 | __copyright__ = \
|
|---|
| 13 | """
|
|---|
| 14 | Copyright (C) 2012-2023 Oracle and/or its affiliates.
|
|---|
| 15 |
|
|---|
| 16 | This file is part of VirtualBox base platform packages, as
|
|---|
| 17 | available from https://www.virtualbox.org.
|
|---|
| 18 |
|
|---|
| 19 | This program is free software; you can redistribute it and/or
|
|---|
| 20 | modify it under the terms of the GNU General Public License
|
|---|
| 21 | as published by the Free Software Foundation, in version 3 of the
|
|---|
| 22 | License.
|
|---|
| 23 |
|
|---|
| 24 | This program is distributed in the hope that it will be useful, but
|
|---|
| 25 | WITHOUT ANY WARRANTY; without even the implied warranty of
|
|---|
| 26 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|---|
| 27 | General Public License for more details.
|
|---|
| 28 |
|
|---|
| 29 | You should have received a copy of the GNU General Public License
|
|---|
| 30 | along with this program; if not, see <https://www.gnu.org/licenses>.
|
|---|
| 31 |
|
|---|
| 32 | The contents of this file may alternatively be used under the terms
|
|---|
| 33 | of the Common Development and Distribution License Version 1.0
|
|---|
| 34 | (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
|
|---|
| 35 | in the VirtualBox distribution, in which case the provisions of the
|
|---|
| 36 | CDDL are applicable instead of those of the GPL.
|
|---|
| 37 |
|
|---|
| 38 | You may elect to license modified versions of this file under the
|
|---|
| 39 | terms and conditions of either the GPL or the CDDL or both.
|
|---|
| 40 |
|
|---|
| 41 | SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
|
|---|
| 42 | """
|
|---|
| 43 | __version__ = "$Revision: 98103 $"
|
|---|
| 44 |
|
|---|
| 45 | # Standard python imports
|
|---|
| 46 | import sys
|
|---|
| 47 | import os
|
|---|
| 48 | from optparse import OptionParser; # pylint: disable=deprecated-module
|
|---|
| 49 | import shutil
|
|---|
| 50 | import tempfile;
|
|---|
| 51 | import zipfile;
|
|---|
| 52 |
|
|---|
| 53 | # Add Test Manager's modules path
|
|---|
| 54 | g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|---|
| 55 | sys.path.append(g_ksTestManagerDir)
|
|---|
| 56 |
|
|---|
| 57 | # Test Manager imports
|
|---|
| 58 | from testmanager import config;
|
|---|
| 59 | from testmanager.core.db import TMDatabaseConnection;
|
|---|
| 60 | from testmanager.core.testset import TestSetLogic;
|
|---|
| 61 |
|
|---|
| 62 |
|
|---|
| 63 | class ArchiveDelFilesBatchJob(object): # pylint: disable=too-few-public-methods
|
|---|
| 64 | """
|
|---|
| 65 | Log+files comp
|
|---|
| 66 | """
|
|---|
| 67 |
|
|---|
| 68 | def __init__(self, oOptions):
|
|---|
| 69 | """
|
|---|
| 70 | Parse command line
|
|---|
| 71 | """
|
|---|
| 72 | self.fDryRun = oOptions.fDryRun;
|
|---|
| 73 | self.fVerbose = oOptions.fVerbose;
|
|---|
| 74 | self.sTempDir = tempfile.gettempdir();
|
|---|
| 75 |
|
|---|
| 76 | self.dprint('Connecting to DB ...');
|
|---|
| 77 | self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
|
|---|
| 78 |
|
|---|
| 79 | ## Fetches (and handles) all testsets up to this age (in hours).
|
|---|
| 80 | self.uHoursAgeToHandle = 24;
|
|---|
| 81 | ## Always remove files with these extensions.
|
|---|
| 82 | self.asRemoveFileExt = [ 'webm' ];
|
|---|
| 83 | ## Always remove files which are bigger than this limit.
|
|---|
| 84 | # Set to 0 to disable.
|
|---|
| 85 | self.cbRemoveBiggerThan = 128 * 1024 * 1024;
|
|---|
| 86 |
|
|---|
| 87 | def dprint(self, sText):
|
|---|
| 88 | """ Verbose output. """
|
|---|
| 89 | if self.fVerbose:
|
|---|
| 90 | print(sText);
|
|---|
| 91 | return True;
|
|---|
| 92 |
|
|---|
| 93 | def warning(self, sText):
|
|---|
| 94 | """Prints a warning."""
|
|---|
| 95 | print(sText);
|
|---|
| 96 | return True;
|
|---|
| 97 |
|
|---|
| 98 | def _replaceFile(self, sDstFile, sSrcFile, fDryRun = False, fForce = False):
|
|---|
| 99 | """
|
|---|
| 100 | Replaces / moves a file safely by backing up the existing destination file (if any).
|
|---|
| 101 |
|
|---|
| 102 | Returns success indicator.
|
|---|
| 103 | """
|
|---|
| 104 |
|
|---|
| 105 | fRc = True;
|
|---|
| 106 |
|
|---|
| 107 | # Rename the destination file first (if any).
|
|---|
| 108 | sDstFileTmp = None;
|
|---|
| 109 | if os.path.exists(sDstFile):
|
|---|
| 110 | sDstFileTmp = sDstFile + ".bak";
|
|---|
| 111 | if os.path.exists(sDstFileTmp):
|
|---|
| 112 | if not fForce:
|
|---|
| 113 | print('Replace file: Warning: Temporary destination file "%s" already exists, skipping' % (sDstFileTmp,));
|
|---|
| 114 | fRc = False;
|
|---|
| 115 | else:
|
|---|
| 116 | try:
|
|---|
| 117 | os.remove(sDstFileTmp);
|
|---|
| 118 | except Exception as e:
|
|---|
| 119 | print('Replace file: Error deleting old temporary destination file "%s": %s' % (sDstFileTmp, e));
|
|---|
| 120 | fRc = False;
|
|---|
| 121 | try:
|
|---|
| 122 | if not fDryRun:
|
|---|
| 123 | shutil.move(sDstFile, sDstFileTmp);
|
|---|
| 124 | except Exception as e:
|
|---|
| 125 | print('Replace file: Error moving old destination file "%s" to temporary file "%s": %s' \
|
|---|
| 126 | % (sDstFile, sDstFileTmp, e));
|
|---|
| 127 | fRc = False;
|
|---|
| 128 |
|
|---|
| 129 | if not fRc:
|
|---|
| 130 | return False;
|
|---|
| 131 |
|
|---|
| 132 | try:
|
|---|
| 133 | if not fDryRun:
|
|---|
| 134 | shutil.move(sSrcFile, sDstFile);
|
|---|
| 135 | except Exception as e:
|
|---|
| 136 | print('Replace file: Error moving source file "%s" to destination "%s": %s' % (sSrcFile, sDstFile, e,));
|
|---|
| 137 | fRc = False;
|
|---|
| 138 |
|
|---|
| 139 | if sDstFileTmp:
|
|---|
| 140 | if fRc: # Move succeeded, remove backup.
|
|---|
| 141 | try:
|
|---|
| 142 | if not fDryRun:
|
|---|
| 143 | os.remove(sDstFileTmp);
|
|---|
| 144 | except Exception as e:
|
|---|
| 145 | print('Replace file: Error deleting temporary destination file "%s": %s' % (sDstFileTmp, e));
|
|---|
| 146 | fRc = False;
|
|---|
| 147 | else: # Final move failed, roll back.
|
|---|
| 148 | try:
|
|---|
| 149 | if not fDryRun:
|
|---|
| 150 | shutil.move(sDstFileTmp, sDstFile);
|
|---|
| 151 | except Exception as e:
|
|---|
| 152 | print('Replace file: Error restoring old destination file "%s": %s' % (sDstFile, e));
|
|---|
| 153 | fRc = False;
|
|---|
| 154 | return fRc;
|
|---|
| 155 |
|
|---|
| 156 | def _processTestSetZip(self, idTestSet, sSrcZipFileAbs):
|
|---|
| 157 | """
|
|---|
| 158 | Worker for processOneTestSet, which processes the testset's ZIP file.
|
|---|
| 159 |
|
|---|
| 160 | Returns success indicator.
|
|---|
| 161 | """
|
|---|
| 162 | _ = idTestSet
|
|---|
| 163 |
|
|---|
| 164 | with tempfile.NamedTemporaryFile(dir=self.sTempDir, delete=False) as tmpfile:
|
|---|
| 165 | sDstZipFileAbs = tmpfile.name;
|
|---|
| 166 |
|
|---|
| 167 | fRc = True;
|
|---|
| 168 |
|
|---|
| 169 | try:
|
|---|
| 170 | oSrcZipFile = zipfile.ZipFile(sSrcZipFileAbs, 'r'); # pylint: disable=consider-using-with
|
|---|
| 171 | self.dprint('Processing ZIP archive "%s" ...' % (sSrcZipFileAbs));
|
|---|
| 172 | try:
|
|---|
| 173 | if not self.fDryRun:
|
|---|
| 174 | oDstZipFile = zipfile.ZipFile(sDstZipFileAbs, 'w'); # pylint: disable=consider-using-with
|
|---|
| 175 | self.dprint('Using temporary ZIP archive "%s"' % (sDstZipFileAbs));
|
|---|
| 176 | try:
|
|---|
| 177 | #
|
|---|
| 178 | # First pass: Gather information if we need to do some re-packing.
|
|---|
| 179 | #
|
|---|
| 180 | fDoRepack = False;
|
|---|
| 181 | aoFilesToRepack = [];
|
|---|
| 182 | for oCurFile in oSrcZipFile.infolist():
|
|---|
| 183 | self.dprint('Handling File "%s" ...' % (oCurFile.filename))
|
|---|
| 184 | sFileExt = os.path.splitext(oCurFile.filename)[1];
|
|---|
| 185 |
|
|---|
| 186 | if sFileExt \
|
|---|
| 187 | and sFileExt[1:] in self.asRemoveFileExt:
|
|---|
| 188 | self.dprint('\tMatches excluded extensions')
|
|---|
| 189 | fDoRepack = True;
|
|---|
| 190 | elif self.cbRemoveBiggerThan \
|
|---|
| 191 | and oCurFile.file_size > self.cbRemoveBiggerThan:
|
|---|
| 192 | self.dprint('\tIs bigger than %d bytes (%d bytes)' % (self.cbRemoveBiggerThan, oCurFile.file_size))
|
|---|
| 193 | fDoRepack = True;
|
|---|
| 194 | else:
|
|---|
| 195 | aoFilesToRepack.append(oCurFile);
|
|---|
| 196 |
|
|---|
| 197 | if not fDoRepack:
|
|---|
| 198 | oSrcZipFile.close();
|
|---|
| 199 | self.dprint('No re-packing necessary, skipping ZIP archive');
|
|---|
| 200 | return True;
|
|---|
| 201 |
|
|---|
| 202 | #
|
|---|
| 203 | # Second pass: Re-pack all needed files into our temporary ZIP archive.
|
|---|
| 204 | #
|
|---|
| 205 | for oCurFile in aoFilesToRepack:
|
|---|
| 206 | self.dprint('Re-packing file "%s"' % (oCurFile.filename,))
|
|---|
| 207 | if not self.fDryRun:
|
|---|
| 208 | oBuf = oSrcZipFile.read(oCurFile);
|
|---|
| 209 | oDstZipFile.writestr(oCurFile, oBuf);
|
|---|
| 210 |
|
|---|
| 211 | if not self.fDryRun:
|
|---|
| 212 | oDstZipFile.close();
|
|---|
| 213 |
|
|---|
| 214 | except Exception as oXcpt4:
|
|---|
| 215 | print('Error handling file "%s" of archive "%s": %s' % (oCurFile.filename, sSrcZipFileAbs, oXcpt4,));
|
|---|
| 216 | return False;
|
|---|
| 217 |
|
|---|
| 218 | oSrcZipFile.close();
|
|---|
| 219 |
|
|---|
| 220 | if fRc:
|
|---|
| 221 | self.dprint('Moving file "%s" to "%s"' % (sDstZipFileAbs, sSrcZipFileAbs));
|
|---|
| 222 | fRc = self._replaceFile(sSrcZipFileAbs, sDstZipFileAbs, self.fDryRun);
|
|---|
| 223 |
|
|---|
| 224 | except Exception as oXcpt3:
|
|---|
| 225 | print('Error creating temporary ZIP archive "%s": %s' % (sDstZipFileAbs, oXcpt3,));
|
|---|
| 226 | return False;
|
|---|
| 227 |
|
|---|
| 228 | except Exception as oXcpt1:
|
|---|
| 229 | # Construct a meaningful error message.
|
|---|
| 230 | if os.path.exists(sSrcZipFileAbs):
|
|---|
| 231 | print('Error: Opening file "%s" failed: %s' % (sSrcZipFileAbs, oXcpt1));
|
|---|
| 232 | else:
|
|---|
| 233 | print('Error: File "%s" not found.' % (sSrcZipFileAbs,));
|
|---|
| 234 | return False;
|
|---|
| 235 |
|
|---|
| 236 | return fRc;
|
|---|
| 237 |
|
|---|
| 238 |
|
|---|
| 239 | def processOneTestSet(self, idTestSet, sBasename):
|
|---|
| 240 | """
|
|---|
| 241 | Processes one single testset.
|
|---|
| 242 |
|
|---|
| 243 | Returns success indicator.
|
|---|
| 244 | """
|
|---|
| 245 |
|
|---|
| 246 | fRc = True;
|
|---|
| 247 | self.dprint('Processing testset %d' % (idTestSet,));
|
|---|
| 248 |
|
|---|
| 249 | # Construct absolute ZIP file path.
|
|---|
| 250 | # ZIP is hardcoded in config, so do here.
|
|---|
| 251 | sSrcZipFileAbs = os.path.join(config.g_ksZipFileAreaRootDir, sBasename + '.zip');
|
|---|
| 252 |
|
|---|
| 253 | if self._processTestSetZip(idTestSet, sSrcZipFileAbs) is not True:
|
|---|
| 254 | fRc = False;
|
|---|
| 255 |
|
|---|
| 256 | return fRc;
|
|---|
| 257 |
|
|---|
| 258 | def processTestSets(self):
|
|---|
| 259 | """
|
|---|
| 260 | Processes all testsets according to the set configuration.
|
|---|
| 261 |
|
|---|
| 262 | Returns success indicator.
|
|---|
| 263 | """
|
|---|
| 264 |
|
|---|
| 265 | aoTestSets = self.oTestSetLogic.fetchByAge(cHoursBack = self.uHoursAgeToHandle);
|
|---|
| 266 | cTestSets = len(aoTestSets);
|
|---|
| 267 | print('Found %d entries in DB' % cTestSets);
|
|---|
| 268 | if not cTestSets:
|
|---|
| 269 | return True; # Nothing to do (yet).
|
|---|
| 270 |
|
|---|
| 271 | fRc = True;
|
|---|
| 272 | for oTestSet in aoTestSets:
|
|---|
| 273 | fRc = self.processOneTestSet(oTestSet.idTestSet, oTestSet.sBaseFilename) and fRc;
|
|---|
| 274 | # Keep going.
|
|---|
| 275 |
|
|---|
| 276 | return fRc;
|
|---|
| 277 |
|
|---|
| 278 | @staticmethod
|
|---|
| 279 | def main():
|
|---|
| 280 | """ C-style main(). """
|
|---|
| 281 | #
|
|---|
| 282 | # Parse options.
|
|---|
| 283 | #
|
|---|
| 284 |
|
|---|
| 285 | oParser = OptionParser();
|
|---|
| 286 |
|
|---|
| 287 | # Generic options.
|
|---|
| 288 | oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False,
|
|---|
| 289 | help = 'Verbose output.');
|
|---|
| 290 | oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False,
|
|---|
| 291 | help = 'Quiet operation.');
|
|---|
| 292 | oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False,
|
|---|
| 293 | help = 'Dry run, do not make any changes.');
|
|---|
| 294 |
|
|---|
| 295 | (oOptions, asArgs) = oParser.parse_args(sys.argv[1:]);
|
|---|
| 296 | if asArgs != []:
|
|---|
| 297 | oParser.print_help();
|
|---|
| 298 | return 1;
|
|---|
| 299 |
|
|---|
| 300 | if oOptions.fDryRun:
|
|---|
| 301 | print('***********************************');
|
|---|
| 302 | print('*** DRY RUN - NO FILES MODIFIED ***');
|
|---|
| 303 | print('***********************************');
|
|---|
| 304 |
|
|---|
| 305 | #
|
|---|
| 306 | # Do the work.
|
|---|
| 307 | #
|
|---|
| 308 | fRc = False;
|
|---|
| 309 |
|
|---|
| 310 | oBatchJob = ArchiveDelFilesBatchJob(oOptions);
|
|---|
| 311 | fRc = oBatchJob.processTestSets();
|
|---|
| 312 |
|
|---|
| 313 | if oOptions.fVerbose:
|
|---|
| 314 | print('SUCCESS' if fRc else 'FAILURE');
|
|---|
| 315 |
|
|---|
| 316 | return 0 if fRc is True else 1;
|
|---|
| 317 |
|
|---|
| 318 | if __name__ == '__main__':
|
|---|
| 319 | sys.exit(ArchiveDelFilesBatchJob.main());
|
|---|
