Changeset 96284 in vbox for trunk/src/VBox/Main

Timestamp:

Aug 18, 2022 7:11:55 AM (2 years ago)

Author:

vboxsync

Message:

Recording/Main: Fixed audio/video sync issues, made timestamp documentation more uniform. bugref:10275

Location:

trunk/src/VBox/Main

Files:

• include/RecordingInternals.h (modified) (3 diffs)
• src-client/DrvAudioRec.cpp (modified) (1 diff)
• src-client/Recording.cpp (modified) (5 diffs)
• src-client/RecordingCodec.cpp (modified) (8 diffs)
• src-client/RecordingStream.cpp (modified) (8 diffs)

trunk/src/VBox/Main/include/RecordingInternals.h

@@ -282 +282 @@
 typedef struct RECORDINGCODECSTATE
 {
-    /** Timestamp (in ms, absolute) of the last frame was encoded. */
+    /** Timestamp Timestamp (PTS, in ms) of the last frame was encoded. */
     uint64_t            tsLastWrittenMs;
     /** Number of encoding errors. */
@@ -389 +389 @@
 typedef struct RECORDINGFRAME
 {
+    /** Timestamp (PTS, in ms). */
     uint64_t                msTimestamp;
     union
@@ -485 +486 @@
 int recordingCodecEncode(PRECORDINGCODEC pCodec, const PRECORDINGFRAME pFrame, size_t *pcEncoded, size_t *pcbEncoded);
 int recordingCodecFinalize(PRECORDINGCODEC pCodec);
+uint32_t recordingCodecGetWritable(PRECORDINGCODEC pCodec, uint64_t msTimestamp);
 #endif /* !MAIN_INCLUDED_RecordingInternals_h */
 

trunk/src/VBox/Main/src-client/DrvAudioRec.cpp

@@ -601 +601 @@
             if (cbSrc == cbFrame) /* Only send full codec frames. */
             {
-                vrc = pRecStream->SendAudioFrame(pStreamAV->pvSrcBuf, cbSrc, 0);
+                vrc = pRecStream->SendAudioFrame(pStreamAV->pvSrcBuf, cbSrc, RTTimeProgramMilliTS());
                 if (RT_FAILURE(vrc))
                     break;

trunk/src/VBox/Main/src-client/Recording.cpp

@@ -727 +727 @@
  * @returns @c true if the specified screen is ready, @c false if not.
  * @param   uScreen             Screen ID.
- * @param   msTimestamp         Current timestamp (in ms). Currently not being used.
+ * @param   msTimestamp         Timestamp (PTS, in ms). Currently not being used.
  */
 bool RecordingContext::IsReady(uint32_t uScreen, uint64_t msTimestamp)
@@ -792 +792 @@
  * @returns true if any limit has been reached.
  * @param   uScreen             Screen ID.
- * @param   msTimestamp         Timestamp (in ms) to check for.
+ * @param   msTimestamp         Timestamp (PTS, in ms) to check for.
  */
 bool RecordingContext::IsLimitReached(uint32_t uScreen, uint64_t msTimestamp)
@@ -830 +830 @@
 
 /**
- * Sends an audio frame to the video encoding thread.
+ * Sends an audio frame to the recording thread.
  *
  * @returns VBox status code.
  * @param   pvData              Audio frame data to send.
  * @param   cbData              Size (in bytes) of (encoded) audio frame data.
- * @param   msTimestamp         Timestamp (in ms) of audio playback.
+ * @param   msTimestamp         Timestamp (PTS, in ms) of audio playback.
  */
 int RecordingContext::SendAudioFrame(const void *pvData, size_t cbData, uint64_t msTimestamp)
@@ -849 +849 @@
 
 /**
- * Copies a source video frame to the intermediate RGB buffer.
- * This function is executed only once per time.
+ * Sends a video frame to the recording thread.
  *
  * @thread  EMT
@@ -864 +863 @@
  * @param   uSrcHeight         Height of the video frame.
  * @param   puSrcData          Pointer to video frame data.
- * @param   msTimestamp        Timestamp (in ms).
+ * @param   msTimestamp        Timestamp (PTS, in ms).
  */
 int RecordingContext::SendVideoFrame(uint32_t uScreen, uint32_t x, uint32_t y,

trunk/src/VBox/Main/src-client/RecordingCodec.cpp

@@ -43 +43 @@
     AssertPtrReturn(pCodec->pvScratch, VERR_NO_MEMORY);
 
+    pCodec->Parms.csFrame  = 0;
+    pCodec->Parms.cbFrame  = pCodec->Parms.Video.uWidth * pCodec->Parms.Video.uHeight * 4 /* 32-bit */;
+    pCodec->Parms.msFrame  = 1; /* 1ms per frame. */
+
 # ifdef VBOX_WITH_LIBVPX_VP9
     vpx_codec_iface_t *pCodecIface = vpx_codec_vp9_cx();
@@ -63 +67 @@
     /* Frame height. */
     pVPX->Cfg.g_h = pCodec->Parms.Video.uHeight;
-    /* 1ms per frame. */
-    pVPX->Cfg.g_timebase.num = 1;
+    /* ms per frame. */
+    pVPX->Cfg.g_timebase.num = pCodec->Parms.msFrame;
     pVPX->Cfg.g_timebase.den = 1000;
     /* Disable multithreading. */
@@ -185 +189 @@
                                       * (uint64_t)pCodec->Video.VPX.Cfg.g_timebase.num / pCodec->Video.VPX.Cfg.g_timebase.den;
 
-                pCodec->State.tsLastWrittenMs += pCodec->Parms.msFrame;
-
                 const bool fKeyframe = RT_BOOL(pPkt->data.frame.flags & VPX_FRAME_IS_KEY);
 
@@ -333 +335 @@
                                              RECORDINGCODEC_ENC_F_BLOCK_IS_KEY /* Every Opus frame is a key frame */,
                                              pCodec->Callbacks.pvUser);
-        if (RT_SUCCESS(vrc))
-            pCodec->State.tsLastWrittenMs += pCodec->Parms.msFrame;
     }
 
@@ -509 +509 @@
         }
 
-        uint64_t const uDurationMs = pCodec->Parms.msFrame;
-
         /* Vorbis expects us to flush packets one at a time directly to the container.
          *
@@ -524 +522 @@
                                                  RECORDINGCODEC_ENC_F_BLOCK_IS_KEY /* Every Vorbis frame is a key frame */,
                                                  pCodec->Callbacks.pvUser);
-            if (RT_SUCCESS(vrc))
-                pCodec->State.tsLastWrittenMs += uDurationMs;
         }
 
@@ -954 +950 @@
     if (RT_SUCCESS(vrc))
     {
+        pCodec->State.tsLastWrittenMs = pFrame->msTimestamp;
+
 #ifdef VBOX_WITH_STATISTICS
         pCodec->STAM.cEncBlocks += cEncoded;
@@ -980 +978 @@
 }
 
+/**
+ * Returns the number of writable bytes for a given timestamp.
+ *
+ * This basically is a helper function to respect the set frames per second (FPS).
+ *
+ * @returns Number of writable bytes.
+ * @param   pCodec              Codec to return number of writable bytes for.
+ * @param   msTimestamp         Timestamp (PTS, in ms) return number of writable bytes for.
+ */
+uint32_t recordingCodecGetWritable(PRECORDINGCODEC pCodec, uint64_t msTimestamp)
+{
+    Log3Func(("%RU64 -- tsLastWrittenMs=%RU64 + uDelayMs=%RU32\n",
+              msTimestamp, pCodec->State.tsLastWrittenMs,pCodec->Parms.Video.uDelayMs));
+
+    if (msTimestamp < pCodec->State.tsLastWrittenMs + pCodec->Parms.Video.uDelayMs)
+        return 0; /* Too early for writing (respect set FPS). */
+
+    /* For now we just return the complete frame space. */
+    AssertMsg(pCodec->Parms.cbFrame, ("Codec not initialized yet\n"));
+    return pCodec->Parms.cbFrame;
+}
+

trunk/src/VBox/Main/src-client/RecordingStream.cpp

@@ -175 +175 @@
  * Checks if a specified limit for a recording stream has been reached, internal version.
  *
- * @returns true if any limit has been reached.
- * @param   msTimestamp     Timestamp (in ms) to check for.
+ * @returns \c true if any limit has been reached, \c false if not.
+ * @param   msTimestamp         Timestamp (PTS, in ms) to check for.
  */
 bool RecordingStream::isLimitReachedInternal(uint64_t msTimestamp) const
@@ -221 +221 @@
  *
  * @returns VBox status code.
- * @param   msTimestamp         Current timestamp (in ms).
+ * @param   msTimestamp         Timestamp (PTS, in ms).
  */
 int RecordingStream::iterateInternal(uint64_t msTimestamp)
@@ -261 +261 @@
  * Checks if a specified limit for a recording stream has been reached.
  *
- * @returns true if any limit has been reached.
- * @param   msTimestamp         Timestamp (in ms) to check for.
+ * @returns \c true if any limit has been reached, \c false if not.
+ * @param   msTimestamp         Timestamp (PTS, in ms) to check for.
  */
 bool RecordingStream::IsLimitReached(uint64_t msTimestamp) const
@@ -427 +427 @@
  * @param   pvData              Pointer to audio data.
  * @param   cbData              Size (in bytes) of \a pvData.
- * @param   msTimestamp         Absolute PTS timestamp (in ms).
+ * @param   msTimestamp         Timestamp (PTS, in ms).
  */
 int RecordingStream::SendAudioFrame(const void *pvData, size_t cbData, uint64_t msTimestamp)
 {
+    AssertPtrReturn(m_pCtx, VERR_WRONG_ORDER);
+
+    Log3Func(("cbData=%zu, msTimestamp=%RU64\n", cbData, msTimestamp));
+
     /* As audio data is common across all streams, re-route this to the recording context, where
      * the data is being encoded and stored in the common blocks queue. */
@@ -450 +454 @@
  * @param   uSrcHeight          Height (in pixels) of the video frame.
  * @param   puSrcData           Actual pixel data of the video frame.
- * @param   msTimestamp         Absolute PTS timestamp (in ms).
+ * @param   msTimestamp         Timestamp (PTS, in ms).
  */
 int RecordingStream::SendVideoFrame(uint32_t x, uint32_t y, uint32_t uPixelFormat, uint32_t uBPP, uint32_t uBytesPerLine,
@@ -457 +461 @@
     lock();
 
-    LogFlowFunc(("tsAbsPTSMs=%RU64\n", msTimestamp));
+    Log3Func(("[%RU32 %RU32 %RU32 %RU32] msTimestamp=%RU64\n", x , y, uSrcWidth, uSrcHeight, msTimestamp));
 
     PRECORDINGCODEC pCodec = &this->CodecVideo;
@@ -472 +476 @@
     do
     {
-        if (msTimestamp < pCodec->State.tsLastWrittenMs + pCodec->Parms.Video.uDelayMs)
-        {
-            vrc = VINF_RECORDING_THROTTLED; /* Respect maximum frames per second. */
-            break;
-        }
-
-        pCodec->State.tsLastWrittenMs = msTimestamp;
+        if (recordingCodecGetWritable(pCodec, msTimestamp) == 0)
+        {
+            vrc = VINF_RECORDING_THROTTLED; /* Respect maximum frames per second (FPS). */
+            break;
+        }
 
         int xDiff = ((int)this->ScreenSettings.Video.ulWidth - (int)uSrcWidth) / 2;
@@ -671 +673 @@
     unlock();
 
+    LogFlowFuncLeaveRC(vrc);
     return vrc;
 }