#!/usr/bin/env python # -*- coding: utf-8 -*- # $Id: filearchiver.py 76553 2019-01-01 01:45:53Z vboxsync $ # pylint: disable=C0301 """ A cronjob that compresses logs and other files, moving them to the g_ksZipFileAreaRootDir storage area. """ from __future__ import print_function; __copyright__ = \ """ Copyright (C) 2012-2019 Oracle Corporation This file is part of VirtualBox Open Source Edition (OSE), as available from http://www.virtualbox.org. This file is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License (GPL) as published by the Free Software Foundation, in version 2 as it comes in the "COPYING" file of the VirtualBox OSE distribution. VirtualBox OSE is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY of any kind. The contents of this file may alternatively be used under the terms of the Common Development and Distribution License Version 1.0 (CDDL) only, as it comes in the "COPYING.CDDL" file of the VirtualBox OSE distribution, in which case the provisions of the CDDL are applicable instead of those of the GPL. You may elect to license modified versions of this file under the terms and conditions of either the GPL or the CDDL or both. """ __version__ = "$Revision: 76553 $" # Standard python imports import sys import os from optparse import OptionParser; # pylint: disable=deprecated-module import time; import zipfile; # Add Test Manager's modules path g_ksTestManagerDir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.append(g_ksTestManagerDir) # Test Manager imports from common import utils; from testmanager import config; from testmanager.core.db import TMDatabaseConnection; from testmanager.core.testset import TestSetData, TestSetLogic; class FileArchiverBatchJob(object): # pylint: disable=R0903 """ Log+files comp """ def __init__(self, oOptions): """ Parse command line """ self.fVerbose = oOptions.fVerbose; self.sSrcDir = config.g_ksFileAreaRootDir; self.sDstDir = config.g_ksZipFileAreaRootDir; #self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(self.dprint if self.fVerbose else None)); self.oTestSetLogic = TestSetLogic(TMDatabaseConnection(None)); self.fDryRun = oOptions.fDryRun; def dprint(self, sText): """ Verbose output. """ if self.fVerbose: print(sText); return True; def warning(self, sText): """Prints a warning.""" print(sText); return True; def _processTestSet(self, idTestSet, asFiles, sCurDir): """ Worker for processDir. Same return codes as processDir. """ sBaseFilename = os.path.join(sCurDir, 'TestSet-%d' % (idTestSet,)); if sBaseFilename[0:2] == ('.' + os.path.sep): sBaseFilename = sBaseFilename[2:]; sSrcFileBase = os.path.join(self.sSrcDir, sBaseFilename + '-'); # # Skip the file if the test set is still running. # But delete them if the testset is not found. # oTestSet = self.oTestSetLogic.tryFetch(idTestSet); if oTestSet is not None and sBaseFilename != oTestSet.sBaseFilename: self.warning('TestSet %d: Deleting because sBaseFilename differs: "%s" (disk) vs "%s" (db)' \ % (idTestSet, sBaseFilename, oTestSet.sBaseFilename,)); oTestSet = None; if oTestSet is not None: if oTestSet.enmStatus == TestSetData.ksTestStatus_Running: self.dprint('Skipping test set #%d, still running' % (idTestSet,)); return True; # # If we have a zip file already, don't try recreate it as we might # have had trouble removing the source files. # sDstDirPath = os.path.join(self.sDstDir, sCurDir); sZipFileNm = os.path.join(sDstDirPath, 'TestSet-%d.zip' % (idTestSet,)); if not os.path.exists(sZipFileNm): # # Create zip file with all testset files as members. # self.dprint('TestSet %d: Creating %s...' % (idTestSet, sZipFileNm,)); if not self.fDryRun: if not os.path.exists(sDstDirPath): os.makedirs(sDstDirPath, 0o755); utils.noxcptDeleteFile(sZipFileNm + '.tmp'); oZipFile = zipfile.ZipFile(sZipFileNm + '.tmp', 'w', zipfile.ZIP_DEFLATED, allowZip64 = True); for sFile in asFiles: sSuff = os.path.splitext(sFile)[1]; if sSuff in [ '.png', '.webm', '.gz', '.bz2', '.zip', '.mov', '.avi', '.mpg', '.gif', '.jpg' ]: ## @todo Consider storing these files outside the zip if they are a little largish. self.dprint('TestSet %d: Storing %s...' % (idTestSet, sFile)); oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_STORED); else: self.dprint('TestSet %d: Deflating %s...' % (idTestSet, sFile)); oZipFile.write(sSrcFileBase + sFile, sFile, zipfile.ZIP_DEFLATED); oZipFile.close(); # # .zip.tmp -> .zip. # utils.noxcptDeleteFile(sZipFileNm); os.rename(sZipFileNm + '.tmp', sZipFileNm); #else: Dry run. else: self.dprint('TestSet %d: zip file exists already (%s)' % (idTestSet, sZipFileNm,)); # # Delete the files. # fRc = True; if self.fVerbose: self.dprint('TestSet %d: deleting file: %s' % (idTestSet, asFiles)); if not self.fDryRun: for sFile in asFiles: if utils.noxcptDeleteFile(sSrcFileBase + sFile) is False: self.warning('TestSet %d: Failed to delete "%s" (%s)' % (idTestSet, sFile, sSrcFileBase + sFile,)); fRc = False; return fRc; def processDir(self, sCurDir): """ Process the given directory (relative to sSrcDir and sDstDir). Returns success indicator. """ if self.fVerbose: self.dprint('processDir: %s' % (sCurDir,)); # # Sift thought the directory content, collecting subdirectories and # sort relevant files by test set. # Generally there will either be subdirs or there will be files. # asSubDirs = []; dTestSets = {}; sCurPath = os.path.abspath(os.path.join(self.sSrcDir, sCurDir)); for sFile in os.listdir(sCurPath): if os.path.isdir(os.path.join(sCurPath, sFile)): if sFile not in [ '.', '..' ]: asSubDirs.append(sFile); elif sFile.startswith('TestSet-'): # Parse the file name. ASSUMES 'TestSet-%d-filename' format. iSlash1 = sFile.find('-'); iSlash2 = sFile.find('-', iSlash1 + 1); if iSlash2 <= iSlash1: self.warning('Bad filename (1): "%s"' % (sFile,)); continue; try: idTestSet = int(sFile[(iSlash1 + 1):iSlash2]); except: self.warning('Bad filename (2): "%s"' % (sFile,)); if self.fVerbose: self.dprint('\n'.join(utils.getXcptInfo(4))); continue; if idTestSet <= 0: self.warning('Bad filename (3): "%s"' % (sFile,)); continue; if iSlash2 + 2 >= len(sFile): self.warning('Bad filename (4): "%s"' % (sFile,)); continue; sName = sFile[(iSlash2 + 1):]; # Add it. if idTestSet not in dTestSets: dTestSets[idTestSet] = []; asTestSet = dTestSets[idTestSet]; asTestSet.append(sName); # # Test sets. # fRc = True; for idTestSet in dTestSets: try: if self._processTestSet(idTestSet, dTestSets[idTestSet], sCurDir) is not True: fRc = False; except: self.warning('TestSet %d: Exception in _processTestSet:\n%s' % (idTestSet, '\n'.join(utils.getXcptInfo()),)); fRc = False; # # Sub dirs. # for sSubDir in asSubDirs: if self.processDir(os.path.join(sCurDir, sSubDir)) is not True: fRc = False; # # Try Remove the directory iff it's not '.' and it's been unmodified # for the last 24h (race protection). # if sCurDir != '.': try: fpModTime = float(os.path.getmtime(sCurPath)); if fpModTime + (24*3600) <= time.time(): if utils.noxcptRmDir(sCurPath) is True: self.dprint('Removed "%s".' % (sCurPath,)); except: pass; return fRc; @staticmethod def main(): """ C-style main(). """ # # Parse options. # oParser = OptionParser(); oParser.add_option('-v', '--verbose', dest = 'fVerbose', action = 'store_true', default = False, help = 'Verbose output.'); oParser.add_option('-q', '--quiet', dest = 'fVerbose', action = 'store_false', default = False, help = 'Quiet operation.'); oParser.add_option('-d', '--dry-run', dest = 'fDryRun', action = 'store_true', default = False, help = 'Dry run, do not make any changes.'); (oOptions, asArgs) = oParser.parse_args() if asArgs != []: oParser.print_help(); return 1; # # Do the work. # oBatchJob = FileArchiverBatchJob(oOptions); fRc = oBatchJob.processDir('.'); return 0 if fRc is True else 1; if __name__ == '__main__': sys.exit(FileArchiverBatchJob.main());