Changeset 91119 in vbox

Timestamp:

Sep 6, 2021 9:35:26 AM (3 years ago)

Author:

vboxsync

Message:

Audio/Validation Kit: Separated the audio testing and the actual audio verification into two steps to have a more fine-grained result in the test manager. Also, now using more meaningful test tags which contain a timestamp and the VM name for easier retrieval / identification. ​bugref:10008

Location:

trunk/src/VBox/ValidationKit

Files:

• tests/audio/tdAudioTest.py (modified) (10 diffs)
• utils/audio/vkat.cpp (modified) (4 diffs)

trunk/src/VBox/ValidationKit/tests/audio/tdAudioTest.py

@@ -34 +34 @@
 
 # Standard Python imports.
+from datetime import datetime
 import os
 import sys
 import signal
 import subprocess
-import uuid
 
 # Only the main script needs to modify the path.
@@ -397 +397 @@
         return iRc;
 
-    def startVkatOnGuest(self, oTestVm, oSession, oTxsSession):
+    def startVkatOnGuest(self, oTestVm, oSession, oTxsSession, sTag):
         """
         Starts VKAT on the guest (running in background).
@@ -407 +407 @@
         reporter.log('Guest audio test temp path is \"%s\"' % (sPathAudioOut));
         reporter.log('Guest audio test output path is \"%s\"' % (sPathAudioTemp));
+        reporter.log('Guest audio test tag is \"%s\"' % (sTag));
 
         fRc, sVkatExe = self.locateGstVkat(oSession, oTxsSession);
@@ -415 +416 @@
 
             asArgsVkat = [ sVkatExe, 'test', '--mode', 'guest', '--probe-backends', \
-                           '--tempdir', sPathAudioTemp, '--outdir', sPathAudioOut ];
+                           '--tempdir', sPathAudioTemp, '--outdir', sPathAudioOut, \
+                           '--tag', sTag ];
 
             asArgs.extend(asArgsVkat);
@@ -467 +469 @@
         return fRc;
 
-    def runTests(self, oTestVm, oSession, oTxsSession, sDesc, asTests):
+    def runTests(self, oTestVm, oSession, oTxsSession, sDesc, sTag, asTests):
         """
         Runs one or more tests using VKAT on the host, which in turn will
@@ -474 +476 @@
         """
         _              = oSession, oTxsSession;
-        sTag           = uuid.uuid4();
 
         sPathTemp      = self.sScratchPath;
@@ -492 +493 @@
         # Build the base command line, exclude all tests by default.
         asArgs = [ sVkatExe, 'test', '--mode', 'host', '--probe-backends', \
-                             '--tempdir', sPathAudioTemp, '--outdir', sPathAudioOut, '-a' ];
+                             '--tempdir', sPathAudioTemp, '--outdir', sPathAudioOut, '-a', \
+                             '--tag', sTag, \
+                             '--no-verify' ]; # We do the verification separately in the step below.
 
         for _ in range(1, reporter.getVerbosity()): # Verbosity always is initialized at 1.
@@ -507 +510 @@
         reporter.testDone();
 
+        if fRc:
+            #
+            # When running the test(s) above were successful, do the verification step next.
+            # This gives us a bit more fine-grained test results in the test manager.
+            #
+            reporter.testStart('Verifying audio data');
+
+            sNameSetHst = '%s-host.tar.gz' % (sTag);
+            sPathSetHst = oTestVm.pathJoin(sPathAudioOut, sNameSetHst);
+            sNameSetGst = '%s-guest.tar.gz' % (sTag);
+            sPathSetGst = oTestVm.pathJoin(sPathAudioOut, sNameSetGst);
+
+            asArgs = [ sVkatExe, 'verify', sPathSetHst, sPathSetGst ];
+
+            for _ in range(1, reporter.getVerbosity()): # Verbosity always is initialized at 1.
+                asArgs.extend([ '-v' ]);
+
+            fRc = self.executeHst("VKAT Host Verify", asArgs);
+            if fRc:
+                reporter.error("Verification audio data successful");
+            else:
+                #
+                # Add the test sets to the test manager for later (manual) diagnosis.
+                #
+                reporter.addLogFile(sPathSetGst, 'misc/other', 'Guest audio test set');
+                reporter.addLogFile(sPathSetHst, 'misc/other', 'Host audio test set');
+
+                reporter.error("Verification of audio data failed");
+
+            reporter.testDone();
+
         return fRc;
 
@@ -527 +561 @@
         reporter.log("Active tests: %s" % (self.asTests,));
 
-        fRc = self.startVkatOnGuest(oTestVm, oSession, oTxsSession);
+        # Define a tag for the whole run.
+        sTag = oTestVm.sVmName + "_" + datetime.now().strftime("%Y%m%d_%H%M%S");
+
+        fRc = self.startVkatOnGuest(oTestVm, oSession, oTxsSession, sTag);
         if fRc:
             #
@@ -533 +570 @@
             #
             if "guest_tone_playback" in self.asTests:
-                fRc = self.runTests(oTestVm, oSession, oTxsSession, 'Guest audio playback', asTests = [ '-i0' ]);
+                fRc = self.runTests(oTestVm, oSession, oTxsSession, \
+                                    'Guest audio playback', sTag + "_test_playback", \
+                                    asTests = [ '-i0' ]);
             if "guest_tone_recording" in self.asTests:
-                fRc = fRc and self.runTests(oTestVm, oSession, oTxsSession, 'Guest audio recording', asTests = [ '-i1' ]);
+                fRc = fRc and self.runTests(oTestVm, oSession, oTxsSession, \
+                                            'Guest audio recording', sTag + "_test_recording", \
+                                            asTests = [ '-i1' ]);
 
             # Cancel guest VKAT execution task summoned by startVkatOnGuest().

trunk/src/VBox/ValidationKit/utils/audio/vkat.cpp

@@ -670 +670 @@
                     RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Verification skipped\n");
 
-                RTFileDelete(pTstEnv->u.Host.szPathTestSetGuest);
-                RTFileDelete(pTstEnv->u.Host.szPathTestSetValKit);
+                if (!pTstEnv->fSkipVerify)
+                {
+                    RTFileDelete(pTstEnv->u.Host.szPathTestSetGuest);
+                    RTFileDelete(pTstEnv->u.Host.szPathTestSetValKit);
+                }
+                else
+                    RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Leaving test set files behind\n");
             }
         }
@@ -750 +755 @@
     int         rc;
 
-    const char *pszTag        = NULL; /* Custom tag to use. Can be NULL if not being used. */
     PCPDMDRVREG pDrvReg       = AudioTestGetDefaultBackend();
     bool        fWithDrvAudio = false;
@@ -862 +866 @@
 
             case VKAT_TEST_OPT_TAG:
-                pszTag = ValueUnion.psz;
+                rc = RTStrCopy(TstEnv.szTag, sizeof(TstEnv.szTag), ValueUnion.psz);
+                if (RT_FAILURE(rc))
+                    return RTMsgErrorExit(RTEXITCODE_FAILURE, "Tag invalid, rc=%Rrc", rc);
                 break;
 
@@ -978 +984 @@
 
     const bool fPacked = AudioTestSetIsPacked(pszPathSet);
+
     if (fPacked)
     {
         RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Test set is an archive and needs to be unpacked\n");
 
-        char szPathTemp[RTPATH_MAX];
-        rc = RTPathTemp(szPathTemp, sizeof(szPathTemp));
+        if (!RTFileExists(pszPathSet))
+        {
+            RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Test set '%s' does not exist\n", pszPathSet);
+            rc = VERR_FILE_NOT_FOUND;
+        }
+        else
+            rc = VINF_SUCCESS;
+
         if (RT_SUCCESS(rc))
         {
-            RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Using temporary directory '%s'\n", szPathTemp);
-
-            rc = RTPathJoin(szPathExtracted, sizeof(szPathExtracted), szPathTemp, "vkat-testset-XXXX");
+            char szPathTemp[RTPATH_MAX];
+            rc = RTPathTemp(szPathTemp, sizeof(szPathTemp));
             if (RT_SUCCESS(rc))
             {
-                rc = RTDirCreateTemp(szPathExtracted, 0755);
+                RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Using temporary directory '%s'\n", szPathTemp);
+
+                rc = RTPathJoin(szPathExtracted, sizeof(szPathExtracted), szPathTemp, "vkat-testset-XXXX");
                 if (RT_SUCCESS(rc))
                 {
-                    RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Unpacking archive to '%s'\n", szPathExtracted);
-                    rc = AudioTestSetUnpack(pszPathSet, szPathExtracted);
+                    rc = RTDirCreateTemp(szPathExtracted, 0755);
                     if (RT_SUCCESS(rc))
-                        RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Archive successfully unpacked\n");
+                    {
+                        RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Unpacking archive to '%s'\n", szPathExtracted);
+                        rc = AudioTestSetUnpack(pszPathSet, szPathExtracted);
+                        if (RT_SUCCESS(rc))
+                            RTTestPrintf(g_hTest, RTTESTLVL_ALWAYS, "Archive successfully unpacked\n");
+                    }
                 }
             }