| 1 | #!/usr/bin/env python
|
|---|
| 2 | # -*- coding: utf-8 -*-
|
|---|
| 3 | # $Id: maintain_archived_testsets.py 96565 2022-09-01 15:18:10Z vboxsync $
|
|---|
| 4 | # pylint: disable=line-too-long
|
|---|
| 5 |
|
|---|
| 6 | """
|
|---|
| 7 | A cronjob that maintains archived testsets.
|
|---|
| 8 | """
|
|---|
| 9 |
|
|---|
| 10 | from __future__ import print_function;
|
|---|
| 11 |
|
|---|
| 12 | __copyright__ = \
|
|---|
| 13 | """
|
|---|
| 14 | Copyright (C) 2022 Oracle Corporation
|
|---|
| 15 |
|
|---|
| 16 | This file is part of VirtualBox Open Source Edition (OSE), as
|
|---|
| 17 | available from http://www.virtualbox.org. This file is free software;
|
|---|
| 18 | you can redistribute it and/or modify it under the terms of the GNU
|
|---|
| 19 | General Public License (GPL) as published by the Free Software
|
|---|
| 20 | Foundation, in version 2 as it comes in the "COPYING" file of the
|
|---|
| 21 | VirtualBox OSE distribution. VirtualBox OSE is distributed in the
|
|---|
| 22 | hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
|
|---|
| 23 |
|
|---|
| 24 | The contents of this file may alternatively be used under the terms
|
|---|
| 25 | of the Common Development and Distribution License Version 1.0
|
|---|
| 26 | (CDDL) only, as it comes in the "COPYING.CDDL" file of the
|
|---|
| 27 | VirtualBox OSE distribution, in which case the provisions of the
|
|---|
| 28 | CDDL are applicable instead of those of the GPL.
|
|---|
| 29 |
|
|---|
| 30 | You may elect to license modified versions of this file under the
|
|---|
| 31 | terms and conditions of either the GPL or the CDDL or both.
|
|---|
| 32 | """
|
|---|
| 33 | __version__ = "$Revision: 96565 $"
|
|---|
| 34 |
|
|---|
| 35 | # Standard python imports
|
|---|
| 36 | from datetime import datetime, timedelta
|
|---|
| 37 | import sys
|
|---|
| 38 | import os
|
|---|
| 39 | from optparse import OptionParser, OptionGroup; # pylint: disable=deprecated-module
|
|---|
| 40 | import shutil
|
|---|
| 41 | import tempfile;
|
|---|
| 42 | import time;
|
|---|
| 43 | import zipfile;
|
|---|
| 44 |
|
|---|
| 45 | # Add Test Manager's modules path
|
|---|
| 46 | g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|---|
| 47 | sys.path.append(g_ksTestManagerDir)
|
|---|
| 48 |
|
|---|
| 49 | # Test Manager imports
|
|---|
| 50 | from common import utils;
|
|---|
| 51 | from testmanager import config;
|
|---|
| 52 | from testmanager.core.db import TMDatabaseConnection;
|
|---|
| 53 | from testmanager.core.testset import TestSetData, TestSetLogic;
|
|---|
| 54 |
|
|---|
| 55 |
|
|---|
| 56 |
|
|---|
| 57 | class ArchiveDelFilesBatchJob(object): # pylint: disable=too-few-public-methods
|
|---|
| 58 | """
|
|---|
| 59 | Log+files comp
|
|---|
| 60 | """
|
|---|
| 61 |
|
|---|
| 62 | def __init__(self, sCmd, oOptions):
|
|---|
| 63 | """
|
|---|
| 64 | Parse command line
|
|---|
| 65 | """
|
|---|
| 66 | self.fVerbose = oOptions.fVerbose;
|
|---|
| 67 | self.sCmd = sCmd;
|
|---|
| 68 | self.sSrcDir = oOptions.sSrcDir;
|
|---|
| 69 | if not self.sSrcDir :
|
|---|
| 70 | self.sSrcDir = config.g_ksFileAreaRootDir;
|
|---|
| 71 | self.sDstDir = config.g_ksZipFileAreaRootDir;
|
|---|
| 72 | self.sTempDir = oOptions.sTempDir;
|
|---|
| 73 | if not self.sTempDir:
|
|---|
| 74 | self.sTempDir = '/tmp'; ## @todo Make this more flexible.
|
|---|
| 75 | #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None));
|
|---|
| 76 | #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None));
|
|---|
| 77 | self.fDryRun = oOptions.fDryRun;
|
|---|
| 78 | self.asFileExt = [];
|
|---|
| 79 | self.asFileExt = oOptions.asFileExt and oOptions.asFileExt.split(',');
|
|---|
| 80 | self.cOlderThanDays = oOptions.cOlderThanDays;
|
|---|
| 81 | self.cbBiggerThan = oOptions.uBiggerThanKb * 1024; # Kilobyte (kB) to bytes.
|
|---|
| 82 | self.fForce = oOptions.fForce;
|
|---|
| 83 |
|
|---|
| 84 | def dprint(self, sText):
|
|---|
| 85 | """ Verbose output. """
|
|---|
| 86 | if self.fVerbose:
|
|---|
| 87 | print(sText);
|
|---|
| 88 | return True;
|
|---|
| 89 |
|
|---|
| 90 | def warning(self, sText):
|
|---|
| 91 | """Prints a warning."""
|
|---|
| 92 | print(sText);
|
|---|
| 93 | return True;
|
|---|
| 94 |
|
|---|
| 95 | def _processTestSetZip(self, idTestSet, sFile, sCurDir):
|
|---|
| 96 | """
|
|---|
| 97 | Worker for processDir.
|
|---|
| 98 | Same return codes as processDir.
|
|---|
| 99 | """
|
|---|
| 100 |
|
|---|
| 101 | sSrcZipFileAbs = os.path.join(sCurDir, sFile);
|
|---|
| 102 | print('Processing ZIP archive "%s" ...' % (sSrcZipFileAbs));
|
|---|
| 103 |
|
|---|
| 104 | sDstZipFileAbs = os.path.join(self.sTempDir, next(tempfile._get_candidate_names()) + ".zip");
|
|---|
| 105 | self.dprint('Using temporary ZIP archive "%s"' % (sDstZipFileAbs));
|
|---|
| 106 |
|
|---|
| 107 | fRc = True;
|
|---|
| 108 |
|
|---|
| 109 | try:
|
|---|
| 110 | oSrcZipFile = zipfile.ZipFile(sSrcZipFileAbs, 'r'); # pylint: disable=consider-using-with
|
|---|
| 111 | try:
|
|---|
| 112 | if not self.fDryRun:
|
|---|
| 113 | oDstZipFile = zipfile.ZipFile(sDstZipFileAbs, 'w');
|
|---|
| 114 | try:
|
|---|
| 115 | for oCurFile in oSrcZipFile.infolist():
|
|---|
| 116 |
|
|---|
| 117 | self.dprint('Handling File "%s" ...' % (oCurFile.filename))
|
|---|
| 118 | sFileExt = os.path.splitext(oCurFile.filename)[1];
|
|---|
| 119 |
|
|---|
| 120 | fDoRepack = True; # Re-pack all unless told otherwise.
|
|---|
| 121 |
|
|---|
| 122 | if sFileExt \
|
|---|
| 123 | and sFileExt[1:] in self.asFileExt:
|
|---|
| 124 | self.dprint('\tMatches excluded extensions')
|
|---|
| 125 | fDoRepack = False;
|
|---|
| 126 |
|
|---|
| 127 | if self.cbBiggerThan \
|
|---|
| 128 | and oCurFile.file_size > self.cbBiggerThan:
|
|---|
| 129 | self.dprint('\tIs bigger than %d bytes (%d bytes)' % (self.cbBiggerThan, oCurFile.file_size))
|
|---|
| 130 | fDoRepack = False;
|
|---|
| 131 |
|
|---|
| 132 | if fDoRepack \
|
|---|
| 133 | and self.cOlderThanDays:
|
|---|
| 134 | tsMaxAge = datetime.now() - timedelta(days = self.cOlderThanDays);
|
|---|
| 135 | tsFile = datetime(year = oCurFile.date_time[0], \
|
|---|
| 136 | month = oCurFile.date_time[1], \
|
|---|
| 137 | day = oCurFile.date_time[2],
|
|---|
| 138 | hour = oCurFile.date_time[3],
|
|---|
| 139 | minute = oCurFile.date_time[4],
|
|---|
| 140 | second = oCurFile.date_time[5],);
|
|---|
| 141 | if tsFile < tsMaxAge:
|
|---|
| 142 | self.dprint('\tIs older than %d days (%s)' % (self.cOlderThanDays, tsFile))
|
|---|
| 143 | fDoRepack = False;
|
|---|
| 144 |
|
|---|
| 145 | if fDoRepack:
|
|---|
| 146 | self.dprint('Re-packing file "%s"' % (oCurFile.filename,))
|
|---|
| 147 | if not self.fDryRun:
|
|---|
| 148 | oBuf = oSrcZipFile.read(oCurFile);
|
|---|
| 149 | oDstZipFile.writestr(oCurFile, oBuf);
|
|---|
| 150 | else:
|
|---|
| 151 | print('Deleting file "%s"' % (oCurFile.filename,))
|
|---|
| 152 | if not self.fDryRun:
|
|---|
| 153 | oDstZipFile.close();
|
|---|
| 154 | except Exception as oXcpt4:
|
|---|
| 155 | print(oXcpt4);
|
|---|
| 156 | return (None, 'Error handling file "%s" of archive "%s": %s' \
|
|---|
| 157 | % (oCurFile.filename, sSrcZipFileAbs, oXcpt4,), None);
|
|---|
| 158 |
|
|---|
| 159 | oSrcZipFile.close();
|
|---|
| 160 |
|
|---|
| 161 | try:
|
|---|
| 162 | self.dprint('Deleting ZIP file "%s"' % (sSrcZipFileAbs));
|
|---|
| 163 | if not self.fDryRun:
|
|---|
| 164 | os.remove(sSrcZipFileAbs);
|
|---|
| 165 | except Exception as oXcpt:
|
|---|
| 166 | return (None, 'Error deleting ZIP file "%s": %s' % (sSrcZipFileAbs, oXcpt,), None);
|
|---|
| 167 | try:
|
|---|
| 168 | self.dprint('Moving ZIP "%s" to "%s"' % (sDstZipFileAbs, sSrcZipFileAbs));
|
|---|
| 169 | if not self.fDryRun:
|
|---|
| 170 | shutil.move(sDstZipFileAbs, sSrcZipFileAbs);
|
|---|
| 171 | except Exception as oXcpt5:
|
|---|
| 172 | return (None, 'Error moving temporary ZIP "%s" to original ZIP file "%s": %s' \
|
|---|
| 173 | % (sDstZipFileAbs, sSrcZipFileAbs, oXcpt5,), None);
|
|---|
| 174 | except Exception as oXcpt3:
|
|---|
| 175 | return (None, 'Error creating temporary ZIP archive "%s": %s' % (sDstZipFileAbs, oXcpt3,), None);
|
|---|
| 176 | except Exception as oXcpt1:
|
|---|
| 177 | # Construct a meaningful error message.
|
|---|
| 178 | try:
|
|---|
| 179 | if os.path.exists(sSrcZipFileAbs):
|
|---|
| 180 | return (None, 'Error opening "%s": %s' % (sSrcZipFileAbs, oXcpt1), None);
|
|---|
| 181 | if not os.path.exists(sFile):
|
|---|
| 182 | return (None, 'File "%s" not found. [%s, %s]' % (sSrcZipFileAbs, sFile), None);
|
|---|
| 183 | return (None, 'Error opening "%s" inside "%s": %s' % (sSrcZipFileAbs, sFile, oXcpt1), None);
|
|---|
| 184 | except Exception as oXcpt2:
|
|---|
| 185 | return (None, 'WTF? %s; %s' % (oXcpt1, oXcpt2,), None);
|
|---|
| 186 |
|
|---|
| 187 | return fRc;
|
|---|
| 188 |
|
|---|
| 189 |
|
|---|
| 190 | def processDir(self, sCurDir):
|
|---|
| 191 | """
|
|---|
| 192 | Process the given directory (relative to sSrcDir and sDstDir).
|
|---|
| 193 | Returns success indicator.
|
|---|
| 194 | """
|
|---|
| 195 |
|
|---|
| 196 | if not self.asFileExt:
|
|---|
| 197 | print('Must specify at least one file extension to delete.');
|
|---|
| 198 | return False;
|
|---|
| 199 |
|
|---|
| 200 | if self.fVerbose:
|
|---|
| 201 | self.dprint('Processing directory: %s' % (sCurDir,));
|
|---|
| 202 |
|
|---|
| 203 | #
|
|---|
| 204 | # Sift thought the directory content, collecting subdirectories and
|
|---|
| 205 | # sort relevant files by test set.
|
|---|
| 206 | # Generally there will either be subdirs or there will be files.
|
|---|
| 207 | #
|
|---|
| 208 | asSubDirs = [];
|
|---|
| 209 | dTestSets = {};
|
|---|
| 210 | sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir));
|
|---|
| 211 | for sFile in os.listdir(sCurPath):
|
|---|
| 212 | if os.path.isdir(os.path.join(sCurPath, sFile)):
|
|---|
| 213 | if sFile not in [ '.', '..' ]:
|
|---|
| 214 | asSubDirs.append(sFile);
|
|---|
| 215 | elif sFile.startswith('TestSet-') \
|
|---|
| 216 | and sFile.endswith('zip'):
|
|---|
| 217 | # Parse the file name. ASSUMES 'TestSet-%d-filename' format.
|
|---|
| 218 | iSlash1 = sFile.find('-');
|
|---|
| 219 | iSlash2 = sFile.find('-', iSlash1 + 1);
|
|---|
| 220 | if iSlash2 <= iSlash1:
|
|---|
| 221 | self.warning('Bad filename (1): "%s"' % (sFile,));
|
|---|
| 222 | continue;
|
|---|
| 223 |
|
|---|
| 224 | try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]);
|
|---|
| 225 | except:
|
|---|
| 226 | self.warning('Bad filename (2): "%s"' % (sFile,));
|
|---|
| 227 | if self.fVerbose:
|
|---|
| 228 | self.dprint('\n'.join(utils.getXcptInfo(4)));
|
|---|
| 229 | continue;
|
|---|
| 230 |
|
|---|
| 231 | if idTestSet <= 0:
|
|---|
| 232 | self.warning('Bad filename (3): "%s"' % (sFile,));
|
|---|
| 233 | continue;
|
|---|
| 234 |
|
|---|
| 235 | if iSlash2 + 2 >= len(sFile):
|
|---|
| 236 | self.warning('Bad filename (4): "%s"' % (sFile,));
|
|---|
| 237 | continue;
|
|---|
| 238 | sName = sFile;
|
|---|
| 239 |
|
|---|
| 240 | # Add it.
|
|---|
| 241 | if idTestSet not in dTestSets:
|
|---|
| 242 | dTestSets[idTestSet] = [];
|
|---|
| 243 | asTestSet = dTestSets[idTestSet];
|
|---|
| 244 | asTestSet.append(sName);
|
|---|
| 245 |
|
|---|
| 246 | #
|
|---|
| 247 | # Test sets.
|
|---|
| 248 | #
|
|---|
| 249 | fRc = True;
|
|---|
| 250 | for idTestSet, oTestSet in dTestSets.items():
|
|---|
| 251 | try:
|
|---|
| 252 | if self._processTestSetZip(idTestSet, oTestSet[0], sCurDir) is not True:
|
|---|
| 253 | fRc = False;
|
|---|
| 254 | except:
|
|---|
| 255 | self.warning('TestSet %d: Exception in _processTestSetZip:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),));
|
|---|
| 256 | fRc = False;
|
|---|
| 257 |
|
|---|
| 258 | #
|
|---|
| 259 | # Sub dirs.
|
|---|
| 260 | #
|
|---|
| 261 | self.dprint('Processing sub-directories');
|
|---|
| 262 | for sSubDir in asSubDirs:
|
|---|
| 263 | if self.processDir(os.path.join(sCurDir, sSubDir)) is not True:
|
|---|
| 264 | fRc = False;
|
|---|
| 265 |
|
|---|
| 266 | #
|
|---|
| 267 | # Try Remove the directory iff it's not '.' and it's been unmodified
|
|---|
| 268 | # for the last 24h (race protection).
|
|---|
| 269 | #
|
|---|
| 270 | if sCurDir != '.':
|
|---|
| 271 | try:
|
|---|
| 272 | fpModTime = float(os.path.getmtime(sCurPath));
|
|---|
| 273 | if fpModTime + (24*3600) <= time.time():
|
|---|
| 274 | if utils.noxcptRmDir(sCurPath) is True:
|
|---|
| 275 | self.dprint('Removed "%s".' % (sCurPath,));
|
|---|
| 276 | except:
|
|---|
| 277 | pass;
|
|---|
| 278 |
|
|---|
| 279 | return fRc;
|
|---|
| 280 |
|
|---|
| 281 | @staticmethod
|
|---|
| 282 | def main():
|
|---|
| 283 | """ C-style main(). """
|
|---|
| 284 | #
|
|---|
| 285 | # Parse options.
|
|---|
| 286 | #
|
|---|
| 287 |
|
|---|
| 288 | if len(sys.argv) < 2:
|
|---|
| 289 | print('Must specify a main command!\n');
|
|---|
| 290 | return 1;
|
|---|
| 291 |
|
|---|
| 292 | sCommand = sys.argv[1];
|
|---|
| 293 |
|
|---|
| 294 | asCmds = [ 'delete-files' ];
|
|---|
| 295 | if sCommand not in asCmds:
|
|---|
| 296 | print('Unknown main command! Must be one of: %s\n' % ', '.join(asCmds));
|
|---|
| 297 | return 1;
|
|---|
| 298 |
|
|---|
| 299 | oParser = OptionParser();
|
|---|
| 300 |
|
|---|
| 301 | # Generic options.
|
|---|
| 302 | oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False,
|
|---|
| 303 | help = 'Verbose output.');
|
|---|
| 304 | oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False,
|
|---|
| 305 | help = 'Quiet operation.');
|
|---|
| 306 | oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False,
|
|---|
| 307 | help = 'Dry run, do not make any changes.');
|
|---|
| 308 | oParser.add_option('--source-dir', type = 'string', dest = 'sSrcDir',
|
|---|
| 309 | help = 'Specifies the source directory to process.');
|
|---|
| 310 | oParser.add_option('--temp-dir', type = 'string', dest = 'sTempDir',
|
|---|
| 311 | help = 'Specifies the temp directory to use.');
|
|---|
| 312 | oParser.add_option('--force', dest = 'fForce', action = 'store_true', default = False,
|
|---|
| 313 | help = 'Forces the operation.');
|
|---|
| 314 |
|
|---|
| 315 | if sCommand == 'delete-files':
|
|---|
| 316 | oGroup = OptionGroup(oParser, "File deletion options", "Deletes files from testset archives.");
|
|---|
| 317 | oGroup.add_option('--file-ext', type = 'string', dest = 'asFileExt',
|
|---|
| 318 | help = 'Selects files with the given extensions.');
|
|---|
| 319 | oGroup.add_option('--older-than-days', type = 'int', dest = 'cOlderThanDays', default = 0,
|
|---|
| 320 | help = 'Selects all files which are older than NUM days.');
|
|---|
| 321 | oGroup.add_option('--bigger-than-kb', type = 'int', dest = 'uBiggerThanKb', default = 0,
|
|---|
| 322 | help = 'Selects all files which are bigger than (kB).\nA kilobyte is 1024 bytes.');
|
|---|
| 323 | oParser.add_option_group(oGroup);
|
|---|
| 324 |
|
|---|
| 325 | (oOptions, asArgs) = oParser.parse_args(sys.argv[2:]);
|
|---|
| 326 | if asArgs != []:
|
|---|
| 327 | oParser.print_help();
|
|---|
| 328 | return 1;
|
|---|
| 329 |
|
|---|
| 330 | if oOptions.fDryRun:
|
|---|
| 331 | print('***********************************');
|
|---|
| 332 | print('*** DRY RUN - NO FILES MODIFIED ***');
|
|---|
| 333 | print('***********************************');
|
|---|
| 334 |
|
|---|
| 335 | #
|
|---|
| 336 | # Do the work.
|
|---|
| 337 | #
|
|---|
| 338 | fRc = False;
|
|---|
| 339 |
|
|---|
| 340 | if sCommand == 'delete-files':
|
|---|
| 341 | print('Job: Deleting files from archive');
|
|---|
| 342 | oBatchJob = ArchiveDelFilesBatchJob(sCommand, oOptions);
|
|---|
| 343 | fRc = oBatchJob.processDir(oBatchJob.sSrcDir);
|
|---|
| 344 |
|
|---|
| 345 | if oOptions.fVerbose:
|
|---|
| 346 | print('SUCCESS' if fRc else 'FAILURE');
|
|---|
| 347 |
|
|---|
| 348 | return 0 if fRc is True else 1;
|
|---|
| 349 |
|
|---|
| 350 | if __name__ == '__main__':
|
|---|
| 351 | sys.exit(ArchiveDelFilesBatchJob.main());
|
|---|
