Changeset 67123 in vbox for trunk/src/VBox/Runtime/common/zip

Timestamp:

May 28, 2017 12:19:53 PM (8 years ago)

Author:

vboxsync

Message:

IPRT: Some more tar vfs writer bits.

Location:

trunk/src/VBox/Runtime/common/zip

Files:

• pkzipvfs.cpp (modified) (1 diff)
• tar.h (modified) (5 diffs)
• tarvfs.cpp (modified) (1 diff)
• tarvfswriter.cpp (modified) (15 diffs)
• xarvfs.cpp (modified) (1 diff)

trunk/src/VBox/Runtime/common/zip/pkzipvfs.cpp

@@ -1240 +1240 @@
     rtZipPkzipFss_Next,
     NULL,
+    NULL,
     RTVFSFSSTREAMOPS_VERSION
 };

trunk/src/VBox/Runtime/common/zip/tar.h

@@ -93 +93 @@
  * @remarks The terminator character is not part of the field.  */
 #define RTZIPTAR_USTAR_VERSION  "00"
+
+/** The GNU magic + version value. */
+#define RTZIPTAR_USTAR_GNU_MAGIC    "ustar  "
 
 
@@ -136 +139 @@
 AssertCompileMemberOffset(RTZIPTARHDRPOSIX, prefix,    345);
 
+/**
+ * GNU sparse data segment descriptor entry.
+ */
+typedef struct RTZIPTARGNUSPARSE
+{
+    char    offset[12];     /**< Absolute offset relative to the start of the file. */
+    char    numbytes[12];
+} RTZIPTARGNUSPARSE;
+AssertCompileSize(RTZIPTARGNUSPARSE, 24);
+AssertCompileMemberOffset(RTZIPTARGNUSPARSE, offset,    0);
+AssertCompileMemberOffset(RTZIPTARGNUSPARSE, numbytes,  12);
 
 /**
@@ -161 +175 @@
     char    longnames[4];
     char    unused[1];
-    struct
-    {
-        char offset[12];
-        char numbytes[12];
-    }       sparse[4];
-    char    isextended;
+    RTZIPTARGNUSPARSE sparse[4];
+    char    isextended; /**< More headers about sparse stuff if binary value 1. */
     char    realsize[12];
     char    unused2[17];
@@ -197 +207 @@
 
 /**
+ * GNU sparse header.
+ */
+typedef struct RTZIPTARHDRGNUSPARSE
+{
+    RTZIPTARGNUSPARSE sp[21];
+    char    isextended;
+    char    unused[7];
+} RTZIPTARHDRGNUSPARSE;
+AssertCompileSize(RTZIPTARHDRGNUSPARSE, 512);
+AssertCompileMemberOffset(RTZIPTARHDRGNUSPARSE, sp,         0);
+AssertCompileMemberOffset(RTZIPTARHDRGNUSPARSE, isextended, 504);
+AssertCompileMemberOffset(RTZIPTARHDRGNUSPARSE, unused,     505);
+
+
+/**
  * The bits common to posix and GNU.
  */
@@ -218 +243 @@
     char    not_common[155+12];
 } RTZIPTARHDRCOMMON;
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, name,      RTZIPTARHDRPOSIX, name);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, mode,      RTZIPTARHDRPOSIX, mode);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, uid,       RTZIPTARHDRPOSIX, uid);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, gid,       RTZIPTARHDRPOSIX, gid);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, size,      RTZIPTARHDRPOSIX, size);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, mtime,     RTZIPTARHDRPOSIX, mtime);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, chksum,    RTZIPTARHDRPOSIX, chksum);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, typeflag,  RTZIPTARHDRPOSIX, typeflag);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, linkname,  RTZIPTARHDRPOSIX, linkname);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, magic,     RTZIPTARHDRPOSIX, magic);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, version,   RTZIPTARHDRPOSIX, version);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, uname,     RTZIPTARHDRPOSIX, uname);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, gname,     RTZIPTARHDRPOSIX, gname);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, devmajor,  RTZIPTARHDRPOSIX, devmajor);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, devminor,  RTZIPTARHDRPOSIX, devminor);
+
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, name,      RTZIPTARHDRGNU, name);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, mode,      RTZIPTARHDRGNU, mode);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, uid,       RTZIPTARHDRGNU, uid);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, gid,       RTZIPTARHDRGNU, gid);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, size,      RTZIPTARHDRGNU, size);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, mtime,     RTZIPTARHDRGNU, mtime);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, chksum,    RTZIPTARHDRGNU, chksum);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, typeflag,  RTZIPTARHDRGNU, typeflag);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, linkname,  RTZIPTARHDRGNU, linkname);
+AssertCompileMembersAtSameOffset(     RTZIPTARHDRCOMMON, magic,     RTZIPTARHDRGNU, magic);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, uname,     RTZIPTARHDRGNU, uname);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, gname,     RTZIPTARHDRGNU, gname);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, devmajor,  RTZIPTARHDRGNU, devmajor);
+AssertCompileMembersSameSizeAndOffset(RTZIPTARHDRCOMMON, devminor,  RTZIPTARHDRGNU, devminor);
+
 
 

trunk/src/VBox/Runtime/common/zip/tarvfs.cpp

@@ -1475 +1475 @@
     rtZipTarFss_Next,
     NULL,
+    NULL,
     RTVFSFSSTREAMOPS_VERSION
 };

trunk/src/VBox/Runtime/common/zip/tarvfswriter.cpp

@@ -34 +34 @@
 #include <iprt/assert.h>
 #include <iprt/err.h>
+#include <iprt/file.h>
 #include <iprt/mem.h>
 #include <iprt/path.h>
@@ -39 +40 @@
 #include <iprt/vfs.h>
 #include <iprt/vfslowlevel.h>
+#include <iprt/zero.h>
 
 #include "tar.h"
@@ -54 +56 @@
     RTVFSIOSTREAM           hVfsIos;
 
+    /** The TAR format. */
+    RTZIPTARFORMAT          enmFormat;
     /** Set if we've encountered a fatal error. */
     int                     rcFatal;
@@ -85 +89 @@
                             8 /*uBase*/, -1 /*cchWidth*/, sizeof(pHdr->Common.chksum) - 1, RTSTR_F_ZEROPAD);
     AssertRCReturn(rc, VERR_TAR_NUM_VALUE_TOO_LARGE);
+    return VINF_SUCCESS;
+}
+
+
+
+/**
+ * Formats a 12 character wide file offset or size field.
+ *
+ * This is mainly used for RTZIPTARHDR::Common.size, but also for formatting the
+ * sparse map.
+ *
+ * @returns IPRT status code.
+ * @param   pach12Field     The 12 character wide destination field.
+ * @param   off             The offset to set.
+ */
+static int rtZipTarFssWriter_FormatOffset(char pach12Field[12], uint64_t off)
+{
+    /*
+     * Is the size small enough for the standard octal string encoding?
+     *
+     * Note! We could actually use the terminator character as well if we liked,
+     *       but let not do that as it's easier to test this way.
+     */
+    if (off < _4G * 2U)
+    {
+        int rc = RTStrFormatU64(pach12Field, 12, off, 8 /*uBase*/, -1 /*cchWidth*/, 12 - 1, RTSTR_F_ZEROPAD);
+        AssertRCReturn(rc, rc);
+    }
+    /*
+     * No, use the base 256 extension. Set the highest bit of the left most
+     * character.  We don't deal with negatives here, cause the size have to
+     * be greater than zero.
+     *
+     * Note! The base-256 extension are never used by gtar or libarchive
+     *       with the "ustar  \0" format version, only the later
+     *       "ustar\000" version.  However, this shouldn't cause much
+     *       trouble as they are not picky about what they read.
+     */
+    else
+    {
+        size_t         cchField  = 12 - 1;
+        unsigned char *puchField = (unsigned char *)pach12Field;
+        puchField[0] = 0x80;
+        do
+        {
+            puchField[cchField--] = off & 0xff;
+            off >>= 8;
+        } while (cchField);
+    }
+
     return VINF_SUCCESS;
 }
@@ -156 +210 @@
 
     /*
-     * Is the size small enough for the standard octal string encoding?
-     *
-     * Note! We could actually use the terminator character as well if we liked,
-     *       but let not do that as it's easier to test this way.
-     */
-    uint64_t cbObject = pObjInfo->cbObject;
-    if (cbObject < _4G * 2U)
-    {
-        rc = RTStrFormatU64(pThis->aHdrs[0].Common.size, sizeof(pThis->aHdrs[0].Common.size), cbObject,
-                            8 /*uBase*/, -1 /*cchWidth*/, sizeof(pThis->aHdrs[0].Common.size) - 1, RTSTR_F_ZEROPAD);
-        AssertRCReturn(rc, rc);
-    }
-    /*
-     * No, use the base 256 extension. Set the highest bit of the left most
-     * character.  We don't deal with negatives here, cause the size have to
-     * be greater than zero.
-     *
-     * Note! The base-256 extension are never used by gtar or libarchive
-     *       with the "ustar  \0" format version, only the later
-     *       "ustar\000" version.  However, this shouldn't cause much
-     *       trouble as they are not picky about what they read.
-     */
-    else
-    {
-        size_t         cchField  = sizeof(pThis->aHdrs[0].Common.size) - 1;
-        unsigned char *puchField = (unsigned char*)pThis->aHdrs[0].Common.size;
-        puchField[0] = 0x80;
-        do
-        {
-            puchField[cchField--] = cbObject & 0xff;
-            cbObject >>= 8;
-        } while (cchField);
-    }
+     * The file size.
+     */
+    rc = rtZipTarFssWriter_FormatOffset(pThis->aHdrs[0].Common.size, pObjInfo->cbObject);
+    AssertRCReturn(rc, rc);
 
     /*
@@ -271 +296 @@
                                      PCRTFSOBJINFO pObjInfo, const char *pszOwnerNm, const char *pszGroupNm)
 {
-    RT_NOREF(fFlags);
-
     /*
      * Append the header.
@@ -279 +302 @@
     if (RT_SUCCESS(rc))
     {
+        RTFOFF const offHdr = fFlags & RTVFSFSSTRM_ADD_F_STREAM ? RTVfsIoStrmTell(pThis->hVfsIos) : RTFOFF_MAX;
         rc = RTVfsIoStrmWrite(pThis->hVfsIos, pThis->aHdrs, pThis->cHdrs * sizeof(pThis->aHdrs[0]), true /*fBlocking*/, NULL);
         if (RT_SUCCESS(rc))
@@ -296 +320 @@
                 if (!pbBuf)
                 {
-                    cbBuf = sizeof(pThis->aHdrs);
-                    pbBuf = (uint8_t *)&pThis->aHdrs[0];
+                    cbBuf = sizeof(pThis->aHdrs) - sizeof(pThis->aHdrs[0]);
+                    pbBuf = (uint8_t *)&pThis->aHdrs[1];
                 }
             }
@@ -304 +328 @@
              * Copy the bytes.  Padding the last buffer to a multiple of 512.
              */
-            uint64_t cbLeft = pObjInfo->cbObject;
-            while (cbLeft > 0 && RT_SUCCESS(rc))
+            if (!(fFlags & RTVFSFSSTRM_ADD_F_STREAM))
             {
-                size_t cbRead = cbLeft > cbBuf ? cbBuf : (size_t)cbBuf;
-                rc = RTVfsIoStrmRead(hVfsIos, pbBuf, cbRead, true /*fBlocking*/, NULL);
-                if (RT_FAILURE(rc))
+                uint64_t cbLeft = pObjInfo->cbObject;
+                while (cbLeft > 0)
+                {
+                    size_t cbRead = cbLeft > cbBuf ? cbBuf : (size_t)cbBuf;
+                    rc = RTVfsIoStrmRead(hVfsIos, pbBuf, cbRead, true /*fBlocking*/, NULL);
+                    if (RT_FAILURE(rc))
+                        break;
+
+                    size_t cbToWrite = cbRead;
+                    if (cbRead & (sizeof(RTZIPTARHDR) - 1))
+                    {
+                        size_t cbToZero = sizeof(RTZIPTARHDR) - (cbRead & (sizeof(RTZIPTARHDR) - 1));
+                        memset(&pbBuf[cbRead], 0, cbToZero);
+                        cbToWrite += cbToZero;
+                    }
+
+                    rc = RTVfsIoStrmWrite(pThis->hVfsIos, pbBuf, cbToWrite, true /*fBlocking*/, NULL);
+                    if (RT_FAILURE(rc))
+                        break;
+                    pThis->cbWritten += cbToWrite;
+                    cbLeft -= cbRead;
+                }
+            }
+            /*
+             * Same as above, only here we don't know the exact input length.
+             */
+            else
+            {
+                uint64_t cbReadTotal = 0;
+                for (;;)
+                {
+                    size_t cbRead = 0;
+                    int rc2 = rc = RTVfsIoStrmRead(hVfsIos, pbBuf, cbBuf, true /*fBlocking*/, &cbRead);
+                    if (RT_SUCCESS(rc))
+                    {
+                        cbReadTotal += cbRead;
+                        rc = RTVfsIoStrmWrite(pThis->hVfsIos, pbBuf, cbRead, true /*fBlocking*/, NULL);
+                        if (RT_SUCCESS(rc))
+                        {
+                            pThis->cbWritten += cbRead;
+                            if (rc2 != VINF_EOF)
+                                continue;
+                        }
+                    }
+                    Assert(rc != VERR_EOF /* expecting VINF_EOF! */);
                     break;
-
-                size_t cbToWrite = cbRead;
-                if (cbRead & (sizeof(RTZIPTARHDR) - 1))
+                }
+
+                /* Do the zero padding. */
+                if ((cbReadTotal & (sizeof(RTZIPTARHDR) - 1)) && RT_SUCCESS(rc))
                 {
-                    size_t cbToZero = sizeof(RTZIPTARHDR) - (cbRead & (sizeof(RTZIPTARHDR) - 1));
-                    memset(&pbBuf[cbRead], 0, cbToZero);
-                    cbToWrite += cbToZero;
+                    size_t cbToZero = sizeof(RTZIPTARHDR) - (cbReadTotal & (sizeof(RTZIPTARHDR) - 1));
+                    rc = RTVfsIoStrmWrite(pThis->hVfsIos, g_abRTZero4K, cbToZero, true /*fBlocking*/, NULL);
+                    if (RT_SUCCESS(rc))
+                        pThis->cbWritten += cbToZero;
                 }
 
-                rc = RTVfsIoStrmWrite(pThis->hVfsIos, pbBuf, cbToWrite, true /*fBlocking*/, NULL);
-                if (RT_FAILURE(rc))
-                    break;
-                pThis->cbWritten += cbToWrite;
-                cbLeft -= cbRead;
+                /*
+                 * Update the header.  We ASSUME that aHdr[0] is unmodified
+                 * from before the data pumping above and just update the size.
+                 */
+                if (RT_SUCCESS(rc) && (RTFOFF)cbReadTotal != pObjInfo->cbObject)
+                {
+                    RTVFSFILE hVfsFile = RTVfsIoStrmToFile(pThis->hVfsIos);
+                    if (hVfsFile != NIL_RTVFSFILE)
+                    {
+                        RTFOFF offRestore = RTVfsFileTell(hVfsFile);
+                        if (offRestore >= 0)
+                        {
+                            rc = rtZipTarFssWriter_FormatOffset(pThis->aHdrs[0].Common.size, cbReadTotal);
+                            if (RT_SUCCESS(rc))
+                                rc = rtZipTarFssWriter_ChecksumHdr(&pThis->aHdrs[0]);
+                            if (RT_SUCCESS(rc))
+                            {
+                                rc = RTVfsFileWriteAt(hVfsFile, offHdr, &pThis->aHdrs[0], sizeof(pThis->aHdrs[0]), NULL);
+                                if (RT_SUCCESS(rc))
+                                    rc = RTVfsFileSeek(hVfsFile, offRestore, RTFILE_SEEK_BEGIN, NULL);
+                            }
+                        }
+                        else
+                            rc = (int)offRestore;
+                        RTVfsFileRelease(hVfsFile);
+                    }
+                    else
+                        AssertFailedStmt(rc = VERR_NOT_A_FILE);
+                }
             }
 
@@ -442 +533 @@
     pThis->hVfsIos = NIL_RTVFSIOSTREAM;
 
-    /** @todo investigate zero end records.  */
-
     return VINF_SUCCESS;
 }
@@ -466 +555 @@
 {
     PRTZIPTARFSSTREAMWRITER pThis = (PRTZIPTARFSSTREAMWRITER)pvThis;
-    RT_NOREF(pThis, pszPath, hVfsObj, fFlags);
 
     /*
@@ -517 +605 @@
 }
 
+
+/**
+ * @interface_method_impl{RTVFSFSSTREAMOPS,pfnEnd}
+ */
+static DECLCALLBACK(int) rtZipTarFssWriter_End(void *pvThis)
+{
+    PRTZIPTARFSSTREAMWRITER pThis = (PRTZIPTARFSSTREAMWRITER)pvThis;
+    if (RT_SUCCESS(pThis->rcFatal))
+    {
+        /** @todo investigate zero end records. */
+        return VINF_SUCCESS;
+    }
+    return pThis->rcFatal;
+}
 
 
@@ -536 +638 @@
     NULL,
     rtZipTarFssWriter_Add,
+    rtZipTarFssWriter_End,
     RTVFSFSSTREAMOPS_VERSION
 };
 
 
-RTDECL(int) RTZipTarFsStreamToIoStream(RTVFSIOSTREAM hVfsIosOut, uint32_t fFlags, PRTVFSFSSTREAM phVfsFss)
+RTDECL(int) RTZipTarFsStreamToIoStream(RTVFSIOSTREAM hVfsIosOut, RTZIPTARFORMAT enmFormat,
+                                       uint32_t fFlags, PRTVFSFSSTREAM phVfsFss)
 {
     /*
@@ -548 +652 @@
     *phVfsFss = NIL_RTVFSFSSTREAM;
     AssertPtrReturn(hVfsIosOut, VERR_INVALID_HANDLE);
-    AssertReturn(!fFlags, VERR_INVALID_PARAMETER);
+    AssertReturn(enmFormat > RTZIPTARFORMAT_INVALID && enmFormat < RTZIPTARFORMAT_END, VERR_INVALID_PARAMETER);
+    AssertReturn(!(fFlags & ~RTZIPTAR_C_VALID_MASK), VERR_INVALID_FLAGS);
+
+    if (enmFormat == RTZIPTARFORMAT_DEFAULT)
+        enmFormat = RTZIPTARFORMAT_GNU;
+    AssertReturn(enmFormat == RTZIPTARFORMAT_GNU, VERR_NOT_IMPLEMENTED); /* Only implementing GNU output at the moment. */
 
     uint32_t cRefs = RTVfsIoStrmRetain(hVfsIosOut);
@@ -562 +671 @@
     if (RT_SUCCESS(rc))
     {
-        pThis->hVfsIos = hVfsIosOut;
-        pThis->rcFatal = VINF_SUCCESS;
+        pThis->hVfsIos   = hVfsIosOut;
+        pThis->enmFormat = enmFormat;
+        pThis->rcFatal   = VINF_SUCCESS;
 
         *phVfsFss = hVfsFss;

trunk/src/VBox/Runtime/common/zip/xarvfs.cpp

@@ -1802 +1802 @@
     0,
     rtZipXarFss_Next,
+    NULL,
     NULL,
     RTVFSFSSTREAMOPS_VERSION