Index: /trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py	(revision 101721)
+++ /trunk/src/VBox/VMM/VMMAll/IEMAllInstPython.py	(revision 101722)
@@ -1973,4 +1973,7 @@
     'IEM_CIMPL_F_XCPT':                 ('IEM_CIMPL_F_BRANCH_INDIRECT', 'IEM_CIMPL_F_BRANCH_FAR',
                                          'IEM_CIMPL_F_MODE', 'IEM_CIMPL_F_RFLAGS', 'IEM_CIMPL_F_VMEXIT', ),
+    'IEM_CIMPL_F_CALLS_CIMPL':              (),
+    'IEM_CIMPL_F_CALLS_AIMPL':              (),
+    'IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE': (),
 };
 class McBlock(object):
@@ -1978,4 +1981,11 @@
     Microcode block (IEM_MC_BEGIN ... IEM_MC_END, IEM_MC_DEFER_TO_CIMPL_x_RET).
     """
+
+    ## @name Macro expansion types.
+    ## @{
+    kMacroExp_None    = 0;
+    kMacroExp_Entire  = 1; ##< Entire block (iBeginLine == iEndLine), original line may contain multiple blocks.
+    kMacroExp_Partial = 2; ##< Partial/mixed (cmpxchg16b), safe to assume single block.
+    ## @}
 
     def __init__(self, sSrcFile, iBeginLine, offBeginLine, oFunction, iInFunction, cchIndent = None):
@@ -2001,4 +2011,7 @@
         ##< The raw lines the block is made up of.
         self.asLines      = []              # type: List[str]
+        ## Indicates whether the block includes macro expansion parts (kMacroExp_None,
+        ## kMacroExp_Entrie, kMacroExp_Partial).
+        self.iMacroExp    = self.kMacroExp_None;
         ## IEM_MC_BEGIN: Argument count.
         self.cArgs        = -1;
@@ -4870,5 +4883,5 @@
 
         #
-        # HACK ALERT! For blocks orginating from macro expansion the start and
+        # HACK ALERT! For blocks originating from macro expansion the start and
         #             end line will be the same, but the line has multiple
         #             newlines inside it.  So, we have to do some extra tricks
@@ -4880,5 +4893,20 @@
             if not asLines[0].strip().startswith('IEM_MC_BEGIN'):
                 self.raiseError('IEM_MC_BEGIN is not the first word on the line');
+
+            # Hack alert! Detect mixed tail/head macros a la cmpxchg16b and split up the lines
+            #             so we can deal correctly with IEM_MC_END below and everything else.
+            for sLine in asLines:
+                cNewLines = sLine.count('\n');
+                assert cNewLines > 0;
+                if cNewLines > 1:
+                    asLines = self.extractLinesFromMacroExpansionLine(''.join(asLines),
+                                                                      self.oCurMcBlock.offBeginLine,
+                                                                        offEndStatementInLine
+                                                                      + sum(len(s) for s in asLines)
+                                                                      - len(asLines[-1]));
+                    self.oCurMcBlock.iMacroExp = McBlock.kMacroExp_Partial;
+                    break;
         else:
+            self.oCurMcBlock.iMacroExp = McBlock.kMacroExp_Entire;
             asLines = self.extractLinesFromMacroExpansionLine(self.asLines[self.iLine - 1],
                                                               self.oCurMcBlock.offBeginLine, offEndStatementInLine);
@@ -5207,4 +5235,6 @@
         # nested macro expansion, just to avoid lots of extra work.
         #
+        # There is only limited support for macros expanding to partial MC blocks.
+        #
         # Note! IEMOP_HLP_DONE_DECODING_NO_LOCK_PREFIX and other macros someone making
         #       use of IEMOP_RAISE_INVALID_LOCK_PREFIX_RET() will be ignored here and
@@ -5214,5 +5244,5 @@
         #       Also, this way we don't produce lots of unnecessary threaded functions.
         #
-        if sBody.find("IEM_MC_BEGIN") < 0:
+        if sBody.find("IEM_MC_BEGIN") < 0 and sBody.find("IEM_MC_END") < 0:
             #self.debug('workerPreProcessDefine: irrelevant (%s: %s)' % (sName, sBody));
             return True;
Index: /trunk/src/VBox/VMM/VMMAll/IEMAllInstTwoByte0f.cpp.h
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IEMAllInstTwoByte0f.cpp.h	(revision 101721)
+++ /trunk/src/VBox/VMM/VMMAll/IEMAllInstTwoByte0f.cpp.h	(revision 101722)
@@ -12428,27 +12428,43 @@
     if (IEM_GET_GUEST_CPU_FEATURES(pVCpu)->fCmpXchg16b)
     {
-        IEM_MC_BEGIN(4, 3, IEM_MC_F_64BIT, 0);
-        IEM_MC_ARG(PRTUINT128U, pu128MemDst,     0);
-        IEM_MC_ARG(PRTUINT128U, pu128RaxRdx,     1);
-        IEM_MC_ARG(PRTUINT128U, pu128RbxRcx,     2);
-        IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3);
-        IEM_MC_LOCAL(RTUINT128U, u128RaxRdx);
-        IEM_MC_LOCAL(RTUINT128U, u128RbxRcx);
-        IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst);
-
-        IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0);
-        IEMOP_HLP_DONE_DECODING();
-        IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(GCPtrEffDst, 16);
-        IEM_MC_MEM_MAP(pu128MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/);
-
-        IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Lo, X86_GREG_xAX);
-        IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Hi, X86_GREG_xDX);
-        IEM_MC_REF_LOCAL(pu128RaxRdx, u128RaxRdx);
-
-        IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Lo, X86_GREG_xBX);
-        IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Hi, X86_GREG_xCX);
-        IEM_MC_REF_LOCAL(pu128RbxRcx, u128RbxRcx);
-
-        IEM_MC_FETCH_EFLAGS(EFlags);
+        /*
+         * This is hairy, very hairy macro fun.   We're walking a fine line
+         * here to make the code parsable by IEMAllInstPython.py and fit into
+         * the patterns IEMAllThrdPython.py requires for the code morphing.
+         */
+#define BODY_CMPXCHG16B_HEAD \
+            IEM_MC_BEGIN(4, 3, IEM_MC_F_64BIT, 0); \
+            IEM_MC_ARG(PRTUINT128U, pu128MemDst,     0); \
+            IEM_MC_ARG(PRTUINT128U, pu128RaxRdx,     1); \
+            IEM_MC_ARG(PRTUINT128U, pu128RbxRcx,     2); \
+            IEM_MC_ARG_LOCAL_EFLAGS(pEFlags, EFlags, 3); \
+            IEM_MC_LOCAL(RTUINT128U, u128RaxRdx); \
+            IEM_MC_LOCAL(RTUINT128U, u128RbxRcx); \
+            IEM_MC_LOCAL(RTGCPTR, GCPtrEffDst); \
+            \
+            IEM_MC_CALC_RM_EFF_ADDR(GCPtrEffDst, bRm, 0); \
+            IEMOP_HLP_DONE_DECODING(); \
+            IEM_MC_RAISE_GP0_IF_EFF_ADDR_UNALIGNED(GCPtrEffDst, 16); \
+            IEM_MC_MEM_MAP(pu128MemDst, IEM_ACCESS_DATA_RW, pVCpu->iem.s.iEffSeg, GCPtrEffDst, 0 /*arg*/); \
+            \
+            IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Lo, X86_GREG_xAX); \
+            IEM_MC_FETCH_GREG_U64(u128RaxRdx.s.Hi, X86_GREG_xDX); \
+            IEM_MC_REF_LOCAL(pu128RaxRdx, u128RaxRdx); \
+            \
+            IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Lo, X86_GREG_xBX); \
+            IEM_MC_FETCH_GREG_U64(u128RbxRcx.s.Hi, X86_GREG_xCX); \
+            IEM_MC_REF_LOCAL(pu128RbxRcx, u128RbxRcx); \
+            \
+            IEM_MC_FETCH_EFLAGS(EFlags)
+
+#define BODY_CMPXCHG16B_TAIL \
+            IEM_MC_MEM_COMMIT_AND_UNMAP(pu128MemDst, IEM_ACCESS_DATA_RW); \
+            IEM_MC_COMMIT_EFLAGS(EFlags); \
+            IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) { \
+                IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u128RaxRdx.s.Lo); \
+                IEM_MC_STORE_GREG_U64(X86_GREG_xDX, u128RaxRdx.s.Hi); \
+            } IEM_MC_ENDIF(); \
+            IEM_MC_ADVANCE_RIP_AND_FINISH(); \
+            IEM_MC_END()
 
 #ifdef RT_ARCH_AMD64 /* some code duplication here because IEMAllInstPython.py cannot parse if/else/#if spaghetti. */
@@ -12457,15 +12473,31 @@
             if (   !(pVCpu->iem.s.fExec & IEM_F_X86_DISREGARD_LOCK)
                 && (pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+            {
+                BODY_CMPXCHG16B_HEAD;
                 IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+                BODY_CMPXCHG16B_TAIL;
+            }
             else
+            {
+                BODY_CMPXCHG16B_HEAD;
                 IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+                BODY_CMPXCHG16B_TAIL;
+            }
         }
         else
         {   /* (see comments in #else case below) */
             if (pVCpu->CTX_SUFF(pVM)->cCpus == 1)
+            {
+                BODY_CMPXCHG16B_HEAD;
                 IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+                BODY_CMPXCHG16B_TAIL;
+            }
             else
+            {
+                BODY_CMPXCHG16B_HEAD;
                 IEM_MC_CALL_CIMPL_4(IEM_CIMPL_F_STATUS_FLAGS,
                                     iemCImpl_cmpxchg16b_fallback_rendezvous, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+                IEM_MC_END();
+            }
         }
 
@@ -12473,7 +12505,15 @@
         /** @todo may require fallback for unaligned accesses... */
         if (!(pVCpu->iem.s.fPrefixes & IEM_OP_PRF_LOCK))
+        {
+            BODY_CMPXCHG16B_HEAD;
             IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            BODY_CMPXCHG16B_TAIL;
+        }
         else
+        {
+            BODY_CMPXCHG16B_HEAD;
             IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_locked, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            BODY_CMPXCHG16B_TAIL;
+        }
 
 #else
@@ -12483,22 +12523,20 @@
                  but to use a rendezvous callback here.  Sigh. */
         if (pVCpu->CTX_SUFF(pVM)->cCpus == 1)
+        {
+            BODY_CMPXCHG16B_HEAD;
             IEM_MC_CALL_VOID_AIMPL_4(iemAImpl_cmpxchg16b_fallback, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            BODY_CMPXCHG16B_TAIL;
+        }
         else
         {
-            IEM_MC_CALL_CIMPL_4(IEM_CIMPL_F_STATUS_FLAGS,
-                                iemCImpl_cmpxchg16b_fallback_rendezvous, pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            BODY_CMPXCHG16B_HEAD;
+            IEM_MC_CALL_CIMPL_4(IEM_CIMPL_F_STATUS_FLAGS, iemCImpl_cmpxchg16b_fallback_rendezvous,
+                                pu128MemDst, pu128RaxRdx, pu128RbxRcx, pEFlags);
+            IEM_MC_END();
             /* Does not get here, tail code is duplicated in iemCImpl_cmpxchg16b_fallback_rendezvous. */
         }
 #endif
 
-        IEM_MC_MEM_COMMIT_AND_UNMAP(pu128MemDst, IEM_ACCESS_DATA_RW);
-        IEM_MC_COMMIT_EFLAGS(EFlags);
-        IEM_MC_IF_EFL_BIT_NOT_SET(X86_EFL_ZF) {
-            IEM_MC_STORE_GREG_U64(X86_GREG_xAX, u128RaxRdx.s.Lo);
-            IEM_MC_STORE_GREG_U64(X86_GREG_xDX, u128RaxRdx.s.Hi);
-        } IEM_MC_ENDIF();
-        IEM_MC_ADVANCE_RIP_AND_FINISH();
-
-        IEM_MC_END();
+#undef BODY_CMPXCHG16B
     }
     Log(("cmpxchg16b -> #UD\n"));
Index: /trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py	(revision 101721)
+++ /trunk/src/VBox/VMM/VMMAll/IEMAllN8vePython.py	(revision 101722)
@@ -38,5 +38,5 @@
 
 # Standard python imports:
-#import sys;
+import copy;
 
 # Out python imports:
@@ -196,5 +196,13 @@
         Returns the native recompiler function body for this threaded variant.
         """
-        aoStmts = self.oVariation.aoStmtsForThreadedFunction # type: list(McStmt)
+        # Take the threaded function statement list and add detected
+        # IEM_CIMPL_F_XXX flags to the IEM_MC_BEGIN statement.
+        aoStmts = list(self.oVariation.aoStmtsForThreadedFunction) # type: list(McStmt)
+        for iStmt, oStmt in enumerate(aoStmts):
+            if oStmt.sName == 'IEM_MC_BEGIN' and self.oVariation.oParent.dsCImplFlags:
+                oNewStmt = copy.deepcopy(oStmt);
+                oNewStmt.asParams[3] = ' | '.join(sorted(self.oVariation.oParent.dsCImplFlags.keys()));
+                aoStmts[iStmt] = oNewStmt;
+
         return iai.McStmt.renderCodeForList(aoStmts, cchIndent);
 
@@ -272,5 +280,5 @@
                 g_dUnsupportedMcStmtLastOneStats[sStmt] = [oVariation,];
 
-    if (    len(dUnsupportedStmts) == 1
+    if (    len(dUnsupportedStmts) in (1,2)
         and iai.McStmt.findStmtByNames(aoStmts,
                                        { 'IEM_MC_LOCAL': 1, 'IEM_MC_LOCAL_CONST': 1, 'IEM_MC_ARG': 1, 'IEM_MC_ARG_CONST': 1,
Index: /trunk/src/VBox/VMM/VMMAll/IEMAllThrdPython.py
===================================================================
--- /trunk/src/VBox/VMM/VMMAll/IEMAllThrdPython.py	(revision 101721)
+++ /trunk/src/VBox/VMM/VMMAll/IEMAllThrdPython.py	(revision 101722)
@@ -419,4 +419,7 @@
         'IEM_CIMPL_F_END_TB':                       True,
         'IEM_CIMPL_F_XCPT':                         True,
+        'IEM_CIMPL_F_CALLS_CIMPL':                  False,
+        'IEM_CIMPL_F_CALLS_AIMPL':                  False,
+        'IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE':     False,
     };
 
@@ -1331,4 +1334,19 @@
                     self.dsCImplFlags['IEM_CIMPL_F_BRANCH_CONDITIONAL'] = True;
 
+            # Check for CIMPL and AIMPL calls.
+            if oStmt.sName.startswith('IEM_MC_CALL_'):
+                if oStmt.sName.startswith('IEM_MC_CALL_CIMPL_'):
+                    self.dsCImplFlags['IEM_CIMPL_F_CALLS_CIMPL'] = True;
+                elif (   oStmt.sName.startswith('IEM_MC_CALL_VOID_AIMPL_')
+                      or oStmt.sName.startswith('IEM_MC_CALL_AIMPL_')
+                      or oStmt.sName.startswith('IEM_MC_CALL_AVX_AIMPL_')):
+                    self.dsCImplFlags['IEM_CIMPL_F_CALLS_AIMPL'] = True;
+                elif (   oStmt.sName.startswith('IEM_MC_CALL_SSE_AIMPL_')
+                      or oStmt.sName.startswith('IEM_MC_CALL_MMX_AIMPL_')
+                      or oStmt.sName.startswith('IEM_MC_CALL_FPU_AIMPL_')):
+                    self.dsCImplFlags['IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE'] = True;
+                else:
+                    raise Exception('Unknown IEM_MC_CALL_* statement: %s' % (oStmt.sName,));
+
             # Process branches of conditionals recursively.
             if isinstance(oStmt, iai.McStmtCond):
@@ -1361,4 +1379,8 @@
         self.dsCImplFlags = self.oMcBlock.dsCImplFlags.copy();
         self.analyzeCodeOperation(aoStmts);
+        if (   ('IEM_CIMPL_F_CALLS_CIMPL' in self.dsCImplFlags)
+             + ('IEM_CIMPL_F_CALLS_AIMPL' in self.dsCImplFlags)
+             + ('IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE' in self.dsCImplFlags) > 1):
+            self.raiseProblem('Mixing CIMPL/AIMPL/AIMPL_WITH_FXSTATE calls');
 
         # Create variations as needed.
@@ -1824,7 +1846,7 @@
             if ian.g_dUnsupportedMcStmtLastOneVarStats:
                 asTopKeys = sorted(ian.g_dUnsupportedMcStmtLastOneVarStats, reverse = True,
-                                   key = lambda sSortKey: len(ian.g_dUnsupportedMcStmtLastOneVarStats[sSortKey]))[:10];
+                                   key = lambda sSortKey: len(ian.g_dUnsupportedMcStmtLastOneVarStats[sSortKey]))[:16];
                 print('todo:', file = sys.stderr);
-                print('todo: Top %s variations with variables and one unsupported statement dependency:' % (len(asTopKeys),),
+                print('todo: Top %s variations with variables and 1-2 unsupported statement dependency:' % (len(asTopKeys),),
                       file = sys.stderr);
                 cchMaxKey = max([len(sKey) for sKey in asTopKeys]);
@@ -1861,5 +1883,5 @@
                 cbVars = 0;
                 for oVar in oThreadedFunction.oMcBlock.aoLocals:
-                     cbVars += (getTypeBitCount(oVar.sType) + 63) // 64 * 8;
+                    cbVars += (getTypeBitCount(oVar.sType) + 63) // 64 * 8;
                 cbMaxVars        = max(cbMaxVars, cbVars);
                 cbMaxArgs        = max(cbMaxArgs, cbArgs);
@@ -2366,5 +2388,6 @@
                 #
                 elif oThreadedFunction.oMcBlock.iBeginLine != oThreadedFunction.oMcBlock.iEndLine:
-                    assert sLine.count('IEM_MC_') - sLine.count('IEM_MC_F_') == 1, 'sLine="%s"' % (sLine,);
+                    assert (   (sLine.count('IEM_MC_') - sLine.count('IEM_MC_F_') == 1)
+                            or oThreadedFunction.oMcBlock.iMacroExp == iai.McBlock.kMacroExp_Partial), 'sLine="%s"' % (sLine,);
                     oOut.write(sLine[:oThreadedFunction.oMcBlock.offBeginLine]);
                     sModified = oThreadedFunction.generateInputCode().strip();
@@ -2373,5 +2396,7 @@
                     iLine = oThreadedFunction.oMcBlock.iEndLine;
                     sLine = oParser.asLines[iLine - 1];
-                    assert sLine.count('IEM_MC_') - sLine.count('IEM_MC_F_') == 1 or len(oThreadedFunction.oMcBlock.aoStmts) == 1;
+                    assert (   sLine.count('IEM_MC_') - sLine.count('IEM_MC_F_') == 1
+                            or len(oThreadedFunction.oMcBlock.aoStmts) == 1
+                            or oThreadedFunction.oMcBlock.iMacroExp == iai.McBlock.kMacroExp_Partial);
                     oOut.write(sLine[oThreadedFunction.oMcBlock.offAfterEnd : ]);
 
Index: /trunk/src/VBox/VMM/include/IEMInternal.h
===================================================================
--- /trunk/src/VBox/VMM/include/IEMInternal.h	(revision 101721)
+++ /trunk/src/VBox/VMM/include/IEMInternal.h	(revision 101722)
@@ -622,4 +622,20 @@
 #define IEM_CIMPL_F_XCPT \
     (IEM_CIMPL_F_BRANCH_INDIRECT | IEM_CIMPL_F_BRANCH_FAR | IEM_CIMPL_F_MODE | IEM_CIMPL_F_RFLAGS | IEM_CIMPL_F_VMEXIT)
+
+/** The block calls a C-implementation instruction function with two implicit arguments.
+ * Mutually exclusive with IEM_CIMPL_F_CALLS_AIMPL and
+ * IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
+ * @note The python scripts will add this is missing.  */
+#define IEM_CIMPL_F_CALLS_CIMPL                 RT_BIT_32(16)
+/** The block calls an ASM-implementation instruction function.
+ * Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL and
+ * IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE.
+ * @note The python scripts will add this is missing.  */
+#define IEM_CIMPL_F_CALLS_AIMPL                 RT_BIT_32(17)
+/** The block calls an ASM-implementation instruction function with an implicit
+ * X86FXSTATE pointer argument.
+ * Mutually exclusive with IEM_CIMPL_F_CALLS_CIMPL and IEM_CIMPL_F_CALLS_AIMPL.
+ * @note The python scripts will add this is missing.  */
+#define IEM_CIMPL_F_CALLS_AIMPL_WITH_FXSTATE    RT_BIT_32(18)
 /** @} */
 
