Changeset 104051 in vbox for trunk/src/VBox/VMM

Timestamp:

Mar 26, 2024 2:10:26 AM (11 months ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

162442

Message:

VMM/IEM: Optimizing (hopefully) and correcting flag handling in IEMAImpl.asm. Prep for shl, shr, and friends. bugref:10376

Location:

trunk/src/VBox/VMM

Files:

• VMMAll/IEMAllAImpl.asm (modified) (119 diffs)
• VMMAll/IEMAllAImplC.cpp (modified) (17 diffs)
• testcase/tstIEMAImpl.cpp (modified) (20 diffs)

trunk/src/VBox/VMM/VMMAll/IEMAllAImpl.asm

@@ -274 +274 @@
 
 ;;
+; This is handy for generating absolutly correct EFLAGS.
+;%define IEM_AIMPL_WITH_LOAD_AND_SAVE_ALL_STATUS_FLAGS
+
+;;
 ; Load the relevant flags from [%1] if there are undefined flags (%3).
 ;
 ; @remarks      Clobbers T0, stack. Changes EFLAGS.
-; @param        A2      The register pointing to the flags.
 ; @param        1       The parameter (A0..A3) pointing to the eflags.
 ; @param        2       The set of modified flags.
 ; @param        3       The set of undefined flags.
-; @param        4       Force loading the flags.
-;
-%macro IEM_MAYBE_LOAD_FLAGS 3-4 1
- %if (%3 + %4) != 0
+; @param        4       The flags that must be loaded.
+;
+%macro IEM_MAYBE_LOAD_FLAGS 4
+ %ifdef IEM_AIMPL_WITH_LOAD_AND_SAVE_ALL_STATUS_FLAGS
+        pushf                                                   ; store current flags
+        mov     T0_32, [%1]                                     ; load the guest flags
+        and     dword [xSP], ~(%2 | %3 | X86_EFL_STATUS_BITS)   ; mask out the modified and undefined flags
+        and     T0_32, (%2 | %3 | X86_EFL_STATUS_BITS)          ; select the modified and undefined flags.
+        or      [xSP], T0                                       ; merge guest flags with host flags.
+        popf                                                    ; load the mixed flags.
+
+ %elif (%3 + %4) != 0
+  %if 1        ; This approach seems faster on intel 10980XE
+   %if (%3 | %4) == X86_EFL_CF
+        ; Use bt to load bit into CF
+        bt      dword [%1], X86_EFL_CF_BIT
+   %else
+        ; Use ADD to set OF and SHAF for the rest. ASSUMES T0_32 is eax!
+        mov     eax, [%1]
+    %if (%3 | %4) == X86_EFL_OF
+        ; Use ADD to set OF.
+        shl     eax, 31 - X86_EFL_OF_BIT
+        add     eax, 80000000h
+    %elif ((%3 | %4) & X86_EFL_OF) != 0
+        ; Use ADD to set OF.
+        xchg    al, ah
+        shl     al, 15 - X86_EFL_OF_BIT
+        add     al, 80h
+        ; Use SAHF to set the other status flags.
+        sahf
+    %else ; OF not needed; so al -> ah and load ah into eflags.
+     %if 1 ; Pretty similar on 10980XE, but shl seems faster on average.
+        shl     eax, 8
+     %else
+        xchg    al, ah
+     %endif
+        sahf
+    %endif
+   %endif
+
+  %else
         pushf                           ; store current flags
         mov     T0_32, [%1]             ; load the guest flags
@@ -291 +331 @@
         or      [xSP], T0               ; merge guest flags with host flags.
         popf                            ; load the mixed flags.
+  %endif
  %endif
 %endmacro
@@ -298 +339 @@
 ;
 ; @remarks      Clobbers T0, stack. Changes EFLAGS.
-; @param        A2      The register pointing to the flags.
 ; @param        1       The parameter (A0..A3) pointing to the eflags.
 ; @param        2       The set of flags to load.
@@ -304 +344 @@
 ;
 %macro IEM_LOAD_FLAGS 3
+ %ifdef IEM_AIMPL_WITH_LOAD_AND_SAVE_ALL_STATUS_FLAGS
+        pushf                                                   ; store current flags
+        mov     T0_32, [%1]                                     ; load the guest flags
+        and     dword [xSP], ~(%2 | %3 | X86_EFL_STATUS_BITS)   ; mask out the modified, undefined and status flags
+        and     T0_32, (%2 | %3 | X86_EFL_STATUS_BITS)          ; select the modified, undefined and status flags.
+        or      [xSP], T0                                       ; merge guest flags with host flags.
+        popf                                                    ; load the mixed flags.
+
+ %elif 1        ; This approach seems faster on intel 10980XE
+  %if (%3 | %2) == X86_EFL_CF
+        ; Use bt to load bit into CF
+        bt      dword [%1], X86_EFL_CF_BIT
+  %else
+        mov     eax, [%1]                 ; ASSUMES T0_32 is eax!!
+   %if (%3 | %2) == X86_EFL_OF
+        ; Use ADD to set OF.
+        shl     eax, 31 - X86_EFL_OF_BIT
+        add     eax, 80000000h
+   %elif ((%3 | %2) & X86_EFL_OF) != 0
+        ; Use ADD to set OF.
+        xchg    al, ah
+        shl     al, 15 - X86_EFL_OF_BIT
+        add     al, 80h
+        ; Use SAHF to set the other status flags.
+        sahf
+   %else ; OF not needed; so al -> ah and load ah into eflags.
+    %if 1 ; Pretty similar on 10980XE, but shl seems faster on average.
+        shl     eax, 8
+    %else
+        xchg    al, ah
+    %endif
+        sahf
+   %endif
+  %endif ; (%3 | %2) != X86_EFL_CF
+
+ %else
         pushf                           ; store current flags
         mov     T0_32, [%1]             ; load the guest flags
@@ -310 +386 @@
         or      [xSP], T0               ; merge guest flags with host flags.
         popf                            ; load the mixed flags.
+ %endif
 %endmacro
 
@@ -319 +396 @@
 ; @param        2       The mask of modified flags to save.
 ; @param        3       The mask of undefined flags to (maybe) save.
-;
-%macro IEM_SAVE_FLAGS 3
- %if (%2 | %3) != 0
+; @param        4       The mask of flags that are zeroed (and thus doesn't require loading, just clearing)
+;
+%macro IEM_SAVE_FLAGS 3-4 0
+ %if (%2 | %3 | %4) != 0
+        mov     T1_32, [%1]                                     ; flags
+  %ifdef IEM_AIMPL_WITH_LOAD_AND_SAVE_ALL_STATUS_FLAGS
         pushf
-        pop     T1
-        mov     T0_32, [%1]             ; flags
-        and     T0_32, ~(%2 | %3)       ; clear the modified & undefined flags.
-        and     T1_32, (%2 | %3)        ; select the modified and undefined flags.
+        pop     T0
+        and     T1_32, ~(%2 | %3 | %4 | X86_EFL_STATUS_BITS)    ; clear the modified & undefined & zeroed & status flags.
+        and     T0_32, (%2 | %3 | X86_EFL_STATUS_BITS)          ; select the modified, undefined and status flags.
+  %else
+   %if (%2 | %3 | %4) == X86_EFL_CF
+        setc    T0_8
+   %elif (%2 | %3) == X86_EFL_OF
+        seto    T0_8
+        shl     T0_32, X86_EFL_OF_BIT
+   %elif (%2 | %3) == X86_EFL_ZF
+        setz    T0_8                    ; On 10980XE this is faster than the next option 5596 vs 5936 ps/call (cmpxchg8b-positive).
+        shl     T0_32, X86_EFL_ZF_BIT
+   %elif (%2 | %3) <= 0xff
+        lahf
+        movzx   eax, ah                 ; ASSUMES T0_32 is eax!
+   %elif 1                              ; The locked functions are generally faster on 10980XE with this approach
+        lahf                            ; while there seems only to be a tiny advantage in most other test.
+        movzx   eax, ah                 ; ASSUMES T0_32 is eax!
+        jno     .of_is_clear
+        or      eax, X86_EFL_OF
+.of_is_clear:
+   %else
+        pushf                           ; this is a bit slow
+        pop     T0
+   %endif
+        and     T1_32, ~(%2 | %3 | %4)  ; clear the modified & undefined & zeroed flags.
+        and     T0_32, (%2 | %3)        ; select the modified and undefined flags.
+  %endif
         or      T0_32, T1_32            ; combine the flags.
         mov     [%1], T0_32             ; save the flags.
@@ -494 +598 @@
 ; @param        3       The modified flags.
 ; @param        4       The undefined flags.
-; @param        5       Force flag loading (ADC, SBC).
-;
-%macro IEMIMPL_BIN_OP 5
+; @param        5       The flags that must be loaded (ADC, SBC).
+; @param        6       The flags that will be zeroed by the operation.
+;
+%macro IEMIMPL_BIN_OP 6
 BEGINCODE
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8, 12
@@ -502 +607 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         %1      byte [A0], A1_8
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u8
@@ -510 +615 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         %1      word [A0], A1_16
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
@@ -518 +623 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         %1      dword [A0], A1_32
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -527 +632 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         %1      qword [A0], A1
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS_EX 8
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -538 +643 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         lock %1 byte [A0], A1_8
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u8_locked
@@ -546 +651 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         lock %1 word [A0], A1_16
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
@@ -554 +659 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         lock %1 dword [A0], A1_32
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
@@ -563 +668 @@
         IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, %5
         lock %1 qword [A0], A1
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, %6
         EPILOGUE_3_ARGS_EX 8
 ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
@@ -570 +675 @@
 %endmacro
 
-;            instr,lock, modified-flags,                                                               undefined flags, force loading flags
-IEMIMPL_BIN_OP add,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               0
-IEMIMPL_BIN_OP adc,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               1
-IEMIMPL_BIN_OP sub,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               0
-IEMIMPL_BIN_OP sbb,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               1
-IEMIMPL_BIN_OP or,   1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF,      0
-IEMIMPL_BIN_OP xor,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF,      0
-IEMIMPL_BIN_OP and,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF,      0
-IEMIMPL_BIN_OP cmp,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               0
-IEMIMPL_BIN_OP test, 0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF),              X86_EFL_AF,      0
+;            instr,lock, modified-flags,                                                               undefined flags, must be loaded, zeroed flags
+IEMIMPL_BIN_OP add,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               0,              0
+IEMIMPL_BIN_OP adc,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               X86_EFL_CF,     0
+IEMIMPL_BIN_OP sub,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               0,              0
+IEMIMPL_BIN_OP sbb,  1, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               X86_EFL_CF,     0
+IEMIMPL_BIN_OP cmp,  0, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0,               0,              0
+IEMIMPL_BIN_OP or,   1, (X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF),                                        X86_EFL_AF,      0,              X86_EFL_OF | X86_EFL_CF
+IEMIMPL_BIN_OP xor,  1, (X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF),                                        X86_EFL_AF,      0,              X86_EFL_OF | X86_EFL_CF
+IEMIMPL_BIN_OP and,  1, (X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF),                                        X86_EFL_AF,      0,              X86_EFL_OF | X86_EFL_CF
+IEMIMPL_BIN_OP test, 0, (X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF),                                        X86_EFL_AF,      0,              X86_EFL_OF | X86_EFL_CF
 
 
@@ -595 +700 @@
 ; @param        2       The modified flags.
 ; @param        3       The undefined flags.
-;
-%macro IEMIMPL_VEX_BIN_OP 3
+; @param        4       The zeroed flags.
+;
+%macro IEMIMPL_VEX_BIN_OP 4
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3, 0 ;; @todo do we need to load undefined flags for any platform?
         %1      T0_32, A1_32, A2_32
         mov     [A0], T0_32
-        IEM_SAVE_FLAGS                 A3, %2, %3
+        IEM_SAVE_FLAGS                 A3, %2, %3, %4
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -609 +715 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A3, %2, %3, 0
         %1      T0, A1, A2
         mov     [A0], T0
-        IEM_SAVE_FLAGS                 A3, %2, %3
+        IEM_SAVE_FLAGS                 A3, %2, %3, %4
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -618 +724 @@
 %endmacro
 
-;                 instr,  modified-flags,                                                                undefined-flags
-IEMIMPL_VEX_BIN_OP andn,  (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF),                           (X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_VEX_BIN_OP bextr, (X86_EFL_OF | X86_EFL_ZF | X86_EFL_CF),                                        (X86_EFL_SF | X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_VEX_BIN_OP bzhi,  (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF),                           (X86_EFL_AF | X86_EFL_PF)
+;                 instr,  modified-flags,                       undefined-flags,                        zeroed-flags
+IEMIMPL_VEX_BIN_OP andn,  X86_EFL_SF | X86_EFL_ZF,              X86_EFL_AF | X86_EFL_PF,                X86_EFL_OF | X86_EFL_CF
+IEMIMPL_VEX_BIN_OP bextr, X86_EFL_ZF,                           X86_EFL_SF | X86_EFL_AF | X86_EFL_PF,   X86_EFL_OF | X86_EFL_CF
+IEMIMPL_VEX_BIN_OP bzhi,  X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF, X86_EFL_AF | X86_EFL_PF,                X86_EFL_OF
 
 ;;
@@ -635 +741 @@
 ; @param        2       The modified flags.
 ; @param        3       The undefined flags.
-;
-%macro IEMIMPL_VEX_BIN_OP_2 3
+; @param        4       The zeroed flags.
+;
+%macro IEMIMPL_VEX_BIN_OP_2 4
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3, 0 ;; @todo check if any undefined flags are passed thru
         mov     T0_32, [A0]
         %1      T0_32, A1_32
         mov     [A0], T0_32
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, %4
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -650 +757 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 12
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3, 0
         mov     T0, [A0]
         %1      T0, A1
         mov     [A0], T0
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, %4
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -660 +767 @@
 %endmacro
 
-;                    instr,  modified-flags,                                      undefined-flags
-IEMIMPL_VEX_BIN_OP_2 blsr,   (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_VEX_BIN_OP_2 blsmsk, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_VEX_BIN_OP_2 blsi,   (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF)
+;                    instr,     modified-flags,                         undefined-flags             zeroed-flags
+IEMIMPL_VEX_BIN_OP_2 blsr,      (X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF),  X86_EFL_OF
+IEMIMPL_VEX_BIN_OP_2 blsmsk,    (X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF),  X86_EFL_OF
+IEMIMPL_VEX_BIN_OP_2 blsi,      (X86_EFL_SF | X86_EFL_ZF | X86_EFL_CF), (X86_EFL_AF | X86_EFL_PF),  X86_EFL_OF
 
 
@@ -874 +981 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, 0
         %1      word [A0], A1_16
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
@@ -882 +989 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, 0
         %1      dword [A0], A1_32
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -891 +998 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, 0
         %1      qword [A0], A1
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, 0
         EPILOGUE_3_ARGS_EX 8
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -902 +1009 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16_locked, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, 0
         lock %1 word [A0], A1_16
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
@@ -910 +1017 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32_locked, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, 0
         lock %1 dword [A0], A1_32
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
@@ -919 +1026 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64_locked, 16
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4
+        IEM_MAYBE_LOAD_FLAGS           A2, %3, %4, 0
         lock %1 qword [A0], A1
-        IEM_SAVE_FLAGS                 A2, %3, %4
+        IEM_SAVE_FLAGS                 A2, %3, %4, 0
         EPILOGUE_3_ARGS_EX 8
 ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
@@ -927 +1034 @@
  %endif ; locked
 %endmacro
+
+;  Undefined flags are passed thru here by the intel and amd CPUs we have.
 ;                      modified efl, undefined eflags
-IEMIMPL_BIT_OP bt,  0, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_BIT_OP btc, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_BIT_OP bts, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
-IEMIMPL_BIT_OP btr, 1, (X86_EFL_CF), (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP bt,  0, (X86_EFL_CF), 0 ;passed-thru (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP btc, 1, (X86_EFL_CF), 0 ;passed-thru (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP bts, 1, (X86_EFL_CF), 0 ;passed-thru (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
+IEMIMPL_BIT_OP btr, 1, (X86_EFL_CF), 0 ;passed-thru (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF)
 
 ;;
@@ -944 +1053 @@
 ; In the ZF case the destination register is 'undefined', however it seems that
 ; both AMD and Intel just leaves it as is.  The undefined EFLAGS differs between
-; AMD and Intel and accoridng to https://www.sandpile.org/x86/flags.htm between
+; AMD and Intel and according to https://www.sandpile.org/x86/flags.htm between
 ; Intel microarchitectures.  We only implement 'intel' and 'amd' variation with
-; the behaviour of more recent CPUs (Intel 10980X and AMD 3990X).
+; the behaviour of more recent CPUs (Intel 10980XE and AMD 3990X).
+;
+; Intel: Clear all and calculate PF in addition to ZF.
+; AMD:   Passthru all flags other than ZF.
 ;
 ; @param        1       The instruction mnemonic.
@@ -955 +1067 @@
 %macro IEMIMPL_BIT_OP2 4
 BEGINCODE
+;   16-bit
+
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3, %3 ; Must load undefined flags since AMD passes them thru
         %1      T0_16, A1_16
 %if %4 != 0
@@ -964 +1078 @@
         mov     [A0], T0_16
 .unchanged_dst:
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
 
-BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _intel, 12
-        PROLOGUE_3_ARGS
-        %1      T1_16, A1_16
-%if %4 != 0
-        jz      .unchanged_dst
-%endif
-        mov     [A0], T1_16
-        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
-        EPILOGUE_3_ARGS
-.unchanged_dst:
-        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
-        EPILOGUE_3_ARGS
-ENDPROC iemAImpl_ %+ %1 %+ _u16_intel
-
-BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _amd, 12
-        PROLOGUE_3_ARGS
-        %1      T0_16, A1_16
-%if %4 != 0
-        jz      .unchanged_dst
-%endif
-        mov     [A0], T0_16
-.unchanged_dst:
-        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
-        EPILOGUE_3_ARGS
-ENDPROC iemAImpl_ %+ %1 %+ _u16_amd
-
+;bad;BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _intel, 12
+;bad;        PROLOGUE_3_ARGS
+;bad;        %1      T1_16, A1_16
+;bad;        jz      .unchanged_dst
+;bad;        mov     [A0], T1_16
+;bad;        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
+;bad;        EPILOGUE_3_ARGS
+;bad;.unchanged_dst:
+;bad;%if %4 != 0
+;bad;        mov     [A0], T1_16
+;bad;%endif
+;bad;        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
+;bad;        EPILOGUE_3_ARGS
+;bad;ENDPROC iemAImpl_ %+ %1 %+ _u16_intel
+;bad;
+;bad;BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ _amd, 12
+;bad;        PROLOGUE_3_ARGS
+;bad;        %1      T0_16, A1_16
+;bad;%if %4 != 0
+;bad;        jz      .unchanged_dst
+;bad;%endif
+;bad;        mov     [A0], T0_16
+;bad;.unchanged_dst:
+;bad;        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
+;bad;        EPILOGUE_3_ARGS
+;bad;ENDPROC iemAImpl_ %+ %1 %+ _u16_amd
+
+;   32-bit
 
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3, %3 ; Must load undefined flags since AMD passes them thru
         %1      T0_32, A1_32
 %if %4 != 0
@@ -1004 +1120 @@
         mov     [A0], T0_32
 .unchanged_dst:
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
 
-BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _intel, 12
-        PROLOGUE_3_ARGS
-        %1      T1_32, A1_32
-%if %4 != 0
-        jz      .unchanged_dst
-%endif
-        mov     [A0], T1_32
-        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
-        EPILOGUE_3_ARGS
-.unchanged_dst:
-        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
-        EPILOGUE_3_ARGS
-ENDPROC iemAImpl_ %+ %1 %+ _u32_intel
-
-BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _amd, 12
-        PROLOGUE_3_ARGS
-        %1      T0_32, A1_32
-%if %4 != 0
-        jz      .unchanged_dst
-%endif
-        mov     [A0], T0_32
-.unchanged_dst:
-        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
-        EPILOGUE_3_ARGS
-ENDPROC iemAImpl_ %+ %1 %+ _u32_amd
+;bad;BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _intel, 12
+;bad;        PROLOGUE_3_ARGS
+;bad;        %1      T1_32, A1_32
+;bad;%if %4 != 0
+;bad;        jz      .unchanged_dst
+;bad;%endif
+;bad;        mov     [A0], T1_32
+;bad;        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
+;bad;        EPILOGUE_3_ARGS
+;bad;.unchanged_dst:
+;bad;        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
+;bad;        EPILOGUE_3_ARGS
+;bad;ENDPROC iemAImpl_ %+ %1 %+ _u32_intel
+;bad;
+;bad;BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ _amd, 12
+;bad;        PROLOGUE_3_ARGS
+;bad;        %1      T0_32, A1_32
+;bad;%if %4 != 0
+;bad;        jz      .unchanged_dst
+;bad;%endif
+;bad;        mov     [A0], T0_32
+;bad;.unchanged_dst:
+;bad;        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
+;bad;        EPILOGUE_3_ARGS
+;bad;ENDPROC iemAImpl_ %+ %1 %+ _u32_amd
 
 
  %ifdef RT_ARCH_AMD64
+;   64-bit
 
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 16
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3, %3 ; Must load undefined flags since AMD passes them thru
         %1      T0, A1
 %if %4 != 0
@@ -1046 +1163 @@
         mov     [A0], T0
 .unchanged_dst:
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, 0
         EPILOGUE_3_ARGS_EX 8
 ENDPROC iemAImpl_ %+ %1 %+ _u64
 
-BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _intel, 16
-        PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS           A2, %2, %3
-        %1      T1, A1
-%if %4 != 0
-        jz      .unchanged_dst
-%endif
-        mov     [A0], T1
-        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
-        EPILOGUE_3_ARGS
-.unchanged_dst:
-        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
-        EPILOGUE_3_ARGS
-ENDPROC iemAImpl_ %+ %1 %+ _u64_intel
-
-BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _amd, 16
-        PROLOGUE_3_ARGS
-        %1      T0, A1
-%if %4 != 0
-        jz      .unchanged_dst
-%endif
-        mov     [A0], T0
-.unchanged_dst:
-        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
-        EPILOGUE_3_ARGS_EX 8
-ENDPROC iemAImpl_ %+ %1 %+ _u64_amd
+;bad;BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _intel, 16
+;bad;        PROLOGUE_3_ARGS
+;bad;        %1      T1, A1
+;bad;%if %4 != 0
+;bad;        jz      .unchanged_dst
+;bad;%endif
+;bad;        mov     [A0], T1
+;bad;        IEM_ADJUST_FLAGS_WITH_PARITY    A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF | X86_EFL_ZF, 0, T1
+;bad;        EPILOGUE_3_ARGS
+;bad;.unchanged_dst:
+;bad;        IEM_ADJUST_FLAGS                A2, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_CF, X86_EFL_ZF | X86_EFL_PF
+;bad;        EPILOGUE_3_ARGS
+;bad;ENDPROC iemAImpl_ %+ %1 %+ _u64_intel
+;bad;
+;bad;BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ _amd, 16
+;bad;        PROLOGUE_3_ARGS
+;bad;        %1      T0, A1
+;bad;%if %4 != 0
+;bad;        jz      .unchanged_dst
+;bad;%endif
+;bad;        mov     [A0], T0
+;bad;.unchanged_dst:
+;bad;        IEM_SAVE_AND_ADJUST_FLAGS       A2, %2, 0, 0    ; Only the ZF flag is modified on AMD Zen 2.
+;bad;        EPILOGUE_3_ARGS_EX 8
+;bad;ENDPROC iemAImpl_ %+ %1 %+ _u64_amd
 
  %endif ; RT_ARCH_AMD64
@@ -1102 +1218 @@
 ; @param        2       The modified flags.
 ; @param        3       The undefined flags.
-;
-%macro IEMIMPL_BIT_OP3 3
+; @param        4       The zeroed flags.
+;
+%macro IEMIMPL_BIT_OP3 4
 BEGINCODE
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 12
@@ -1110 +1227 @@
         %1      T0_16, A1_16
         mov     [A0], T0_16
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, %4
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
@@ -1119 +1236 @@
         %1      T0_32, A1_32
         mov     [A0], T0_32
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, %4
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -1129 +1246 @@
         %1      T0, A1
         mov     [A0], T0
-        IEM_SAVE_FLAGS                 A2, %2, %3
+        IEM_SAVE_FLAGS                 A2, %2, %3, %4
         EPILOGUE_3_ARGS_EX 8
 ENDPROC iemAImpl_ %+ %1 %+ _u64
  %endif ; RT_ARCH_AMD64
 %endmacro
-IEMIMPL_BIT_OP3 popcnt, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF), 0
+IEMIMPL_BIT_OP3 popcnt, X86_EFL_ZF, 0, X86_EFL_CF | X86_EFL_OF | X86_EFL_SF | X86_EFL_AF | X86_EFL_PF
 
 
@@ -1145 +1262 @@
 ; @param        2       Undefined EFLAGS.
 ; @param        3       Function suffix.
-; @param        4       EFLAGS variation: 0 for native, 1 for intel (ignored),
+; @param        4       EFLAGS variation: 0 for native, 1 for intel,
 ;                       2 for AMD (set AF, clear PF, ZF and SF).
 %macro IEMIMPL_IMUL_TWO 4
 BEGINPROC_FASTCALL iemAImpl_imul_two_u16 %+ %3, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
+        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2, %2 ; Undefined flags may be passed thru (AMD)
         imul    A1_16, word [A0]
         mov     [A0], A1_16
  %if %4 != 1
-        IEM_SAVE_FLAGS                          A2, %1, %2
+        IEM_SAVE_FLAGS                          A2, %1, %2, 0
  %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_16, 16, A1
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_16, 16, A1 ; intel
  %endif
         EPILOGUE_3_ARGS
@@ -1163 +1280 @@
 BEGINPROC_FASTCALL iemAImpl_imul_two_u32 %+ %3, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
+        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2, %2 ; Undefined flags may be passed thru (AMD)
         imul    A1_32, dword [A0]
         mov     [A0], A1_32
  %if %4 != 1
-        IEM_SAVE_FLAGS                          A2, %1, %2
+        IEM_SAVE_FLAGS                          A2, %1, %2, 0
  %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_32, 32, A1
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1_32, 32, A1 ; intel
  %endif
         EPILOGUE_3_ARGS
@@ -1177 +1294 @@
 BEGINPROC_FASTCALL iemAImpl_imul_two_u64 %+ %3, 16
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2
+        IEM_MAYBE_LOAD_FLAGS                    A2, %1, %2, %2 ; Undefined flags may be passed thru (AMD)
         imul    A1, qword [A0]
         mov     [A0], A1
  %if %4 != 1
-        IEM_SAVE_FLAGS                          A2, %1, %2
+        IEM_SAVE_FLAGS                          A2, %1, %2, 0
  %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1, 64, A1
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF    A2, %1, X86_EFL_AF | X86_EFL_ZF, A1, 64, A1 ; intel
  %endif
         EPILOGUE_3_ARGS_EX 8
@@ -1189 +1306 @@
  %endif ; RT_ARCH_AMD64
 %endmacro
+; The SF, ZF, AF and PF flags are "undefined". AMD (3990x) leaves these
+; flags as is.  Whereas Intel skylake (6700K and 10980XE (Cascade Lake)) always
+; clear AF and ZF and calculates SF and PF as per the lower half of the result.
 IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF,       , 0
 IEMIMPL_IMUL_TWO X86_EFL_OF | X86_EFL_CF, 0,                                                 _intel, 1
@@ -1290 +1410 @@
         xadd    [A0], T0_8
         mov     [A1], T0_8
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u8
@@ -1300 +1420 @@
         xadd    [A0], T0_16
         mov     [A1], T0_16
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u16
@@ -1310 +1430 @@
         xadd    [A0], T0_32
         mov     [A1], T0_32
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u32
@@ -1321 +1441 @@
         xadd    [A0], T0
         mov     [A1], T0
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u64
@@ -1332 +1452 @@
         lock xadd [A0], T0_8
         mov     [A1], T0_8
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u8_locked
@@ -1342 +1462 @@
         lock xadd [A0], T0_16
         mov     [A1], T0_16
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u16_locked
@@ -1352 +1472 @@
         lock xadd [A0], T0_32
         mov     [A1], T0_32
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u32_locked
@@ -1363 +1483 @@
         lock xadd [A0], T0
         mov     [A1], T0
-        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0
+        IEM_SAVE_FLAGS       A2, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF | X86_EFL_CF), 0, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_xadd_u64_locked
@@ -1402 +1522 @@
         mov     [r11], eax
         mov     [r11 + 4], edx
-        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1422 +1542 @@
         mov     [rsi], eax
         mov     [rsi + 4], edx
-        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1449 +1569 @@
         mov     [esi], eax
         mov     [esi + 4], edx
-        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, edi)
 
         pop     ebp
@@ -1477 +1597 @@
         mov     [r11], eax
         mov     [r11 + 4], edx
-        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1497 +1617 @@
         mov     [rsi], eax
         mov     [rsi + 4], edx
-        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1524 +1644 @@
         mov     [esi], eax
         mov     [esi + 4], edx
-        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, edi)
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, edi)
 
         pop     ebp
@@ -1568 +1688 @@
         mov     [r11], rax
         mov     [r11 + 8], rdx
-        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1588 +1708 @@
         mov     [rsi], rax
         mov     [rsi + 8], rdx
-        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1613 +1733 @@
         mov     [r11], rax
         mov     [r11 + 8], rdx
-        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r9, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1633 +1753 @@
         mov     [rsi], rax
         mov     [rsi + 8], rdx
-        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0 ; clobbers T0+T1 (eax, r11)
+        IEM_SAVE_FLAGS       r10, (X86_EFL_ZF), 0, 0 ; clobbers T0+T1 (eax, r11)
 
         pop     rbx
@@ -1660 +1780 @@
         %1 cmpxchg [A0], A2_8
         mov     [A1], al
-        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0, 0 ; clobbers T0+T1 (eax, r11/edi)
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_cmpxchg_u8 %+ %2
@@ -1670 +1790 @@
         %1 cmpxchg [A0], A2_16
         mov     [A1], ax
-        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0, 0 ; clobbers T0+T1 (eax, r11/edi)
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_cmpxchg_u16 %+ %2
@@ -1680 +1800 @@
         %1 cmpxchg [A0], A2_32
         mov     [A1], eax
-        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0, 0 ; clobbers T0+T1 (eax, r11/edi)
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_cmpxchg_u32 %+ %2
@@ -1691 +1811 @@
         %1 cmpxchg [A0], A2
         mov     [A1], rax
-        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, r11/edi)
+        IEM_SAVE_FLAGS       A3, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0, 0 ; clobbers T0+T1 (eax, r11/edi)
         EPILOGUE_4_ARGS
 %else
@@ -1722 +1842 @@
         mov     [esi], eax
         mov     [esi + 4], edx
-        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0 ; clobbers T0+T1 (eax, edi)
+        IEM_SAVE_FLAGS       ebp, (X86_EFL_ZF | X86_EFL_CF | X86_EFL_PF | X86_EFL_AF | X86_EFL_SF | X86_EFL_OF), 0, 0 ; clobbers T0+T1 (eax, edi)
 
         pop     ebp
@@ -1742 +1862 @@
 IEMIMPL_CMPXCHG , ,
 IEMIMPL_CMPXCHG lock, _locked
+
+
 
 ;;
@@ -1763 +1885 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         %1      byte [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u8
@@ -1771 +1893 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         lock %1 byte [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u8_locked
@@ -1779 +1901 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         %1      word [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
@@ -1787 +1909 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         lock %1 word [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16_locked
@@ -1795 +1917 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         %1      dword [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -1803 +1925 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         lock %1 dword [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32_locked
@@ -1812 +1934 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         %1      qword [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -1820 +1942 @@
         IEM_MAYBE_LOAD_FLAGS A1, %2, %3, 0
         lock %1 qword [A0]
-        IEM_SAVE_FLAGS       A1, %2, %3
+        IEM_SAVE_FLAGS       A1, %2, %3, 0
         EPILOGUE_2_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u64_locked
@@ -1906 +2028 @@
         %1      byte [A1], cl
  %endif
-        IEM_SAVE_FLAGS       A2, %2, %3
+        IEM_SAVE_FLAGS       A2, %2, %3, 0
+.zero_shift:
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u8
@@ -1920 +2043 @@
         %1      word [A1], cl
  %endif
-        IEM_SAVE_FLAGS       A2, %2, %3
+        IEM_SAVE_FLAGS       A2, %2, %3, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
@@ -1934 +2057 @@
         %1      dword [A1], cl
  %endif
-        IEM_SAVE_FLAGS       A2, %2, %3
+        IEM_SAVE_FLAGS       A2, %2, %3, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -1949 +2072 @@
         %1      qword [A1], cl
   %endif
-        IEM_SAVE_FLAGS       A2, %2, %3
+        IEM_SAVE_FLAGS       A2, %2, %3, 0
         EPILOGUE_3_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -1956 +2079 @@
 %endmacro
 
-;; @todo some questions wrt flags when the shift count is high according to intel docs...
-IEMIMPL_SHIFT_OP rol, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF
-IEMIMPL_SHIFT_OP ror, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF
-IEMIMPL_SHIFT_OP rcl, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF
-IEMIMPL_SHIFT_OP rcr, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF
-IEMIMPL_SHIFT_OP shl, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), 0
-IEMIMPL_SHIFT_OP shr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), 0
-IEMIMPL_SHIFT_OP sar, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), 0
+; These instructions will NOT modify flags if the masked shift count is zero
+; (the mask is 0x3f for 64-bit instructions and 0x1f for the others).  Thus,
+; we have to force load all modified and undefined.
+IEMIMPL_SHIFT_OP rol, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF | X86_EFL_OF
+IEMIMPL_SHIFT_OP ror, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF | X86_EFL_OF
+IEMIMPL_SHIFT_OP rcl, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF | X86_EFL_OF
+IEMIMPL_SHIFT_OP rcr, (X86_EFL_OF | X86_EFL_CF), 0,                                                   X86_EFL_CF | X86_EFL_OF
+IEMIMPL_SHIFT_OP shl, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), X86_EFL_STATUS_BITS
+IEMIMPL_SHIFT_OP shr, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), X86_EFL_STATUS_BITS
+IEMIMPL_SHIFT_OP sar, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), X86_EFL_STATUS_BITS
 
 
@@ -1978 +2103 @@
 ; @param        2       The modified flags.
 ; @param        3       The undefined flags.
+; @param        4       The force loaded flags.
 ;
 ; Makes ASSUMPTIONS about A0, A1, A2 and A3 assignments.
@@ -1983 +2109 @@
 ; @note the _intel and _amd variants are implemented in C.
 ;
-%macro IEMIMPL_SHIFT_DBL_OP 3
+%macro IEMIMPL_SHIFT_DBL_OP 4
 BEGINCODE
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16, 16
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        ;IEM_LOAD_FLAGS      A3, %4, %3
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3, %4
  %ifdef ASM_CALL64_GCC
         xchg    A3, A2
@@ -1996 +2123 @@
         %1      [A2], A1_16, cl
  %endif
-        IEM_SAVE_FLAGS       A3, %2, %3
+        IEM_SAVE_FLAGS      A3, %2, %3, 0
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u16
@@ -2002 +2129 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32, 16
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        ;IEM_LOAD_FLAGS      A3, %4, %3
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3, %4
  %ifdef ASM_CALL64_GCC
         xchg    A3, A2
@@ -2011 +2139 @@
         %1      [A2], A1_32, cl
  %endif
-        IEM_SAVE_FLAGS       A3, %2, %3
+        IEM_SAVE_FLAGS      A3, %2, %3, 0
         EPILOGUE_4_ARGS
 ENDPROC iemAImpl_ %+ %1 %+ _u32
@@ -2018 +2146 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64, 20
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        ;IEM_LOAD_FLAGS      A3, %4, %3
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3, %4
  %ifdef ASM_CALL64_GCC
         xchg    A3, A2
@@ -2027 +2156 @@
         %1      [A2], A1, cl
  %endif
-        IEM_SAVE_FLAGS       A3, %2, %3
+        IEM_SAVE_FLAGS      A3, %2, %3, 0
         EPILOGUE_4_ARGS_EX 12
 ENDPROC iemAImpl_ %+ %1 %+ _u64
@@ -2034 +2163 @@
 %endmacro
 
-IEMIMPL_SHIFT_DBL_OP shld, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
-IEMIMPL_SHIFT_DBL_OP shrd, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF)
+; These instructions will NOT modify flags if the masked shift count is zero
+; (the mask is 0x3f for 64-bit instructions and 0x1f for the others).  Thus,
+; we have to force load all modified and undefined.
+IEMIMPL_SHIFT_DBL_OP shld, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), X86_EFL_STATUS_BITS
+IEMIMPL_SHIFT_DBL_OP shrd, (X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF), (X86_EFL_AF), X86_EFL_STATUS_BITS
 
 
@@ -2063 +2195 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u8 %+ %4, 12
         PROLOGUE_3_ARGS
-        IEM_MAYBE_LOAD_FLAGS                     A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS                     A2, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
         mov     al, [A0]
         %1      A1_8
         mov     [A0], ax
  %if %5 != 1
-        IEM_SAVE_FLAGS                           A2, %2, %3
+        IEM_SAVE_FLAGS                           A2, %2, %3, 0
  %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A2, %2, X86_EFL_AF | X86_EFL_ZF, ax, 8, xAX
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A2, %2, X86_EFL_AF | X86_EFL_ZF, ax, 8, xAX ; intel
  %endif
         xor     eax, eax
@@ -2078 +2210 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u16 %+ %4, 16
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
         mov     ax, [A0]
  %ifdef ASM_CALL64_GCC
@@ -2091 +2223 @@
  %endif
  %if %5 != 1
-        IEM_SAVE_FLAGS                           A3, %2, %3
+        IEM_SAVE_FLAGS                           A3, %2, %3, 0
  %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, ax, 16, xAX
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, ax, 16, xAX ; intel
  %endif
         xor     eax, eax
@@ -2101 +2233 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u32 %+ %4, 16
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
         mov     eax, [A0]
  %ifdef ASM_CALL64_GCC
@@ -2114 +2246 @@
  %endif
  %if %5 != 1
-        IEM_SAVE_FLAGS                           A3, %2, %3
+        IEM_SAVE_FLAGS                           A3, %2, %3, 0
  %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, eax, 32, xAX
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, eax, 32, xAX ; intel
  %endif
         xor     eax, eax
@@ -2125 +2257 @@
 BEGINPROC_FASTCALL iemAImpl_ %+ %1 %+ _u64 %+ %4, 20
         PROLOGUE_4_ARGS
-        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS                     A3, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
         mov     rax, [A0]
   %ifdef ASM_CALL64_GCC
@@ -2138 +2270 @@
   %endif
   %if %5 != 1
-        IEM_SAVE_FLAGS                           A3, %2, %3
+        IEM_SAVE_FLAGS                           A3, %2, %3, 0
   %else
-        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, rax, 64, xAX
+        IEM_SAVE_FLAGS_ADJUST_AND_CALC_SF_PF     A3, %2, X86_EFL_AF | X86_EFL_ZF, rax, 64, xAX ; intel
   %endif
         xor     eax, eax
@@ -2259 +2391 @@
  %endif
 
-        IEM_MAYBE_LOAD_FLAGS A2, %2, %3
+        IEM_MAYBE_LOAD_FLAGS A2, %2, %3, %3 ; Undefined flags may be passed thru (Intel)
         mov     ax, [A0]
         %1      A1_8
@@ -2266 +2398 @@
         IEM_ADJUST_FLAGS    A2, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
  %else
-        IEM_SAVE_FLAGS      A2, %2, %3
+        IEM_SAVE_FLAGS      A2, %2, %3, 0
  %endif
         xor     eax, eax
@@ -2325 +2457 @@
  %endif
 
-        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
  %ifdef ASM_CALL64_GCC
         mov     T1, A2
@@ -2344 +2476 @@
         IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
  %else
-        IEM_SAVE_FLAGS      A3, %2, %3
+        IEM_SAVE_FLAGS      A3, %2, %3, 0
  %endif
         xor     eax, eax
@@ -2411 +2543 @@
  %endif
 
-        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
         mov     eax, [A0]
  %ifdef ASM_CALL64_GCC
@@ -2431 +2563 @@
         IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
  %else
-        IEM_SAVE_FLAGS      A3, %2, %3
+        IEM_SAVE_FLAGS      A3, %2, %3, 0
  %endif
         xor     eax, eax
@@ -2499 +2631 @@
   %endif
 
-        IEM_MAYBE_LOAD_FLAGS A3, %2, %3
+        IEM_MAYBE_LOAD_FLAGS A3, %2, %3, %3 ; Undefined flags may be passed thru (AMD)
         mov     rax, [A0]
   %ifdef ASM_CALL64_GCC
@@ -2519 +2651 @@
         IEM_ADJUST_FLAGS    A3, X86_EFL_PF | X86_EFL_ZF | X86_EFL_SF, X86_EFL_AF
   %else
-        IEM_SAVE_FLAGS      A3, %2, %3
+        IEM_SAVE_FLAGS      A3, %2, %3, 0
   %endif
         xor     eax, eax
@@ -4593 +4725 @@
         movdqu  xmm1, [A1]
         ptest   xmm0, xmm1
-        IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A2, X86_EFL_ZF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_AF | X86_EFL_PF | X86_EFL_SF
 
         IEMIMPL_SSE_EPILOGUE
@@ -4606 +4738 @@
         vmovdqu ymm1, [A1]
         vptest  ymm0, ymm1
-        IEM_SAVE_FLAGS A2, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A2, X86_EFL_ZF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_AF | X86_EFL_PF | X86_EFL_SF
 
         IEMIMPL_SSE_EPILOGUE
@@ -5640 +5772 @@
         call    T1
 
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_CF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_OF, 0, X86_EFL_AF | X86_EFL_PF
         mov    [T2], ecx
 
@@ -5687 +5819 @@
 
         pop     xDX
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_CF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_OF, 0, X86_EFL_AF | X86_EFL_PF
         mov    [T2], ecx
 
@@ -5729 +5861 @@
         call    T1
 
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_CF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_OF, 0, X86_EFL_AF | X86_EFL_PF
         movdqu  [A0], xmm0
 
@@ -5775 +5907 @@
 
         pop     xDX
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_CF | X86_EFL_ZF | X86_EFL_SF | X86_EFL_OF, 0, X86_EFL_AF | X86_EFL_PF
         movdqu  [A0], xmm0
 
@@ -6295 +6427 @@
         movdqu  xmm1, [A3]
         ucomiss xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6310 +6442 @@
         movdqu  xmm1, [A3]
         vucomiss xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6334 +6466 @@
         movdqu  xmm1, [A3]
         ucomisd xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6349 +6481 @@
         movdqu  xmm1, [A3]
         vucomisd xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6372 +6504 @@
         movdqu  xmm1, [A3]
         comiss xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6387 +6519 @@
         movdqu  xmm1, [A3]
         vcomiss xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6411 +6543 @@
         movdqu  xmm1, [A3]
         comisd xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6426 +6558 @@
         movdqu  xmm1, [A3]
         vcomisd xmm0, xmm1
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_ZF | X86_EFL_PF | X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_AF
 
         SSE_ST_FXSTATE_MXCSR_ONLY_NO_FXSTATE A0
@@ -6696 +6828 @@
         %1      %2
         mov     [A0], %2
-        IEM_SAVE_FLAGS A1, X86_EFL_STATUS_BITS, 0
+        IEM_SAVE_FLAGS A1, X86_EFL_CF, 0, X86_EFL_OF | X86_EFL_SF | X86_EFL_ZF | X86_EFL_AF | X86_EFL_PF
 
         EPILOGUE_2_ARGS
@@ -6789 +6921 @@
         %1      A1_32, [A0]
         mov     [A0], A1_32
-        IEM_SAVE_FLAGS A2, %2, 0
+        IEM_SAVE_FLAGS A2, %2, 0, 0
 
         EPILOGUE_4_ARGS
@@ -6809 +6941 @@
         %1      A1, [A0]
         mov     [A0], A1
-        IEM_SAVE_FLAGS A2, %2, 0
+        IEM_SAVE_FLAGS A2, %2, 0, 0
 
         EPILOGUE_4_ARGS

trunk/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp

@@ -1398 +1398 @@
 # endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
 
+#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */
 
 /*
@@ -1430 +1431 @@
     } while (0)
 
-
 /*
  * BSF - first (least significant) bit set
  */
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
 {
     SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
@@ -1449 +1451 @@
 }
 
-# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
-
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
 {
     SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
@@ -1467 +1469 @@
 
 
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
 {
     SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
@@ -1482 +1486 @@
 }
 
-# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
 
 
@@ -1488 +1491 @@
  * BSR - last (most significant) bit set
  */
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
 {
     SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
@@ -1503 +1508 @@
 }
 
-# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
-
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
 {
     SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
@@ -1521 +1527 @@
 
 
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
 {
     SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
@@ -1535 +1543 @@
     SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
 }
-
-# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
 
 
@@ -1569 +1575 @@
  * LZCNT - count leading zero bits.
  */
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
 {
     iemAImpl_lzcnt_u64_intel(puDst, uSrc, pfEFlags);
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
@@ -1584 +1592 @@
 }
 
-# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
-
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
 {
     iemAImpl_lzcnt_u32_intel(puDst, uSrc, pfEFlags);
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
@@ -1602 +1611 @@
 
 
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
 {
     iemAImpl_lzcnt_u16_intel(puDst, uSrc, pfEFlags);
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_lzcnt_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
@@ -1616 +1627 @@
     SET_BIT_CNT_SEARCH_RESULT_AMD(puDst, uSrc, pfEFlags, ASMCountLeadingZerosU16(uSrc));
 }
-
-# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
 
 
@@ -1623 +1632 @@
  * TZCNT - count leading zero bits.
  */
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
 {
     iemAImpl_tzcnt_u64_intel(puDst, uSrc, pfEFlags);
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u64_intel,(uint64_t *puDst, uint64_t uSrc, uint32_t *pfEFlags))
@@ -1638 +1649 @@
 }
 
-# if !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY)
-
+
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
 {
     iemAImpl_tzcnt_u32_intel(puDst, uSrc, pfEFlags);
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u32_intel,(uint32_t *puDst, uint32_t uSrc, uint32_t *pfEFlags))
@@ -1656 +1668 @@
 
 
+#if !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY)
 IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
 {
     iemAImpl_tzcnt_u16_intel(puDst, uSrc, pfEFlags);
 }
+#endif
 
 IEM_DECL_IMPL_DEF(void, iemAImpl_tzcnt_u16_intel,(uint16_t *puDst, uint16_t uSrc, uint32_t *pfEFlags))
@@ -1671 +1685 @@
 }
 
-# endif /* !defined(RT_ARCH_X86) || defined(IEM_WITHOUT_ASSEMBLY) */
-#endif /* !defined(RT_ARCH_AMD64) || defined(IEM_WITHOUT_ASSEMBLY) */
+
 
 /*

trunk/src/VBox/VMM/testcase/tstIEMAImpl.cpp

@@ -2250 +2250 @@
 }
 
+
+typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCMPXCHG8B,(uint64_t *, PRTUINT64U, PRTUINT64U, uint32_t *));
+
+static uint64_t CmpXchg8bBench(uint32_t cIterations, FNIEMAIMPLCMPXCHG8B *pfn, uint64_t const uDstValue,
+                               uint64_t const uOldValue, uint64_t const uNewValue, uint32_t const fEflIn)
+{
+    cIterations /= 4;
+    RTThreadYield();
+    uint64_t const nsStart = RTTimeNanoTS();
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        RTUINT64U uA, uB;
+        uint32_t fEfl    = fEflIn;
+        uint64_t uDst    = uDstValue;
+        uB.u             = uNewValue;
+        uA.u             = uOldValue;
+        pfn(&uDst, &uA, &uB, &fEfl);
+
+        fEfl = fEflIn;
+        uDst = uDstValue;
+        uB.u = uNewValue;
+        uA.u = uOldValue;
+        pfn(&uDst, &uA, &uB, &fEfl);
+
+        fEfl = fEflIn;
+        uDst = uDstValue;
+        uB.u = uNewValue;
+        uA.u = uOldValue;
+        pfn(&uDst, &uA, &uB, &fEfl);
+
+        fEfl = fEflIn;
+        uDst = uDstValue;
+        uB.u = uNewValue;
+        uA.u = uOldValue;
+        pfn(&uDst, &uA, &uB, &fEfl);
+    }
+    return RTTimeNanoTS() - nsStart;
+}
+
 static void CmpXchg8bTest(void)
 {
-    typedef IEM_DECL_IMPL_TYPE(void, FNIEMAIMPLCMPXCHG8B,(uint64_t *, PRTUINT64U, PRTUINT64U, uint32_t *));
     static struct
     {
@@ -2303 +2341 @@
                              (fEflIn & ~X86_EFL_ZF), uExpect, uExpect, EFlagsDiff(fEfl, fEflIn & ~X86_EFL_ZF));
             RTTEST_CHECK(g_hTest, uB.u == uNewValue);
+
+            if (iTest == 2 && g_cPicoSecBenchmark && RTTestSubErrorCount(g_hTest) == 0)
+            {
+                uint32_t cIterations = EstimateIterations(_64K, CmpXchg8bBench(_64K, s_aFuncs[iFn].pfn,
+                                                                               uOldValue, uOldValue, uNewValue, fEflIn));
+                uint64_t cNsRealRun  = CmpXchg8bBench(cIterations, s_aFuncs[iFn].pfn, uOldValue, uOldValue, uNewValue, fEflIn);
+                RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL,
+                             "%s-positive", s_aFuncs[iFn].pszName);
+
+                cIterations = EstimateIterations(_64K, CmpXchg8bBench(_64K, s_aFuncs[iFn].pfn,
+                                                                      ~uOldValue, uOldValue, uNewValue, fEflIn));
+                cNsRealRun  = CmpXchg8bBench(cIterations, s_aFuncs[iFn].pfn, ~uOldValue, uOldValue, uNewValue, fEflIn);
+                RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL,
+                             "%s-negative", s_aFuncs[iFn].pszName);
+            }
         }
     }
@@ -2423 +2476 @@
 #endif
 
-#define TEST_SHIFT_DBL(a_cBits, a_Type, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
+#define TEST_SHIFT_DBL(a_cBits, a_uType, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
 TYPEDEF_SUBTEST_TYPE(a_SubTestType, a_TestType, PFNIEMAIMPLSHIFTDBLU ## a_cBits); \
 \
@@ -2435 +2488 @@
 \
 GEN_SHIFT_DBL(a_cBits, a_Fmt, a_TestType, a_aSubTests) \
+\
+static uint64_t ShiftDblU ## a_cBits ## Bench(uint32_t cIterations, PFNIEMAIMPLSHIFTDBLU ## a_cBits pfn, a_TestType const *pEntry) \
+{ \
+    uint32_t const fEflIn = pEntry->fEflIn; \
+    a_uType  const uDstIn = pEntry->uDstIn; \
+    a_uType  const uSrcIn = pEntry->uSrcIn; \
+    a_uType  const cShift = pEntry->uMisc; \
+    cIterations /= 4; \
+    RTThreadYield(); \
+    uint64_t const nsStart     = RTTimeNanoTS(); \
+    for (uint32_t i = 0; i < cIterations; i++) \
+    { \
+        uint32_t fBenchEfl = fEflIn; \
+        a_uType  uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, uSrcIn, cShift, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, uSrcIn, cShift, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, uSrcIn, cShift, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, uSrcIn, cShift, &fBenchEfl); \
+    } \
+    return RTTimeNanoTS() - nsStart; \
+} \
 \
 static void ShiftDblU ## a_cBits ## Test(void) \
@@ -2452 +2535 @@
             { \
                 uint32_t fEfl = paTests[iTest].fEflIn; \
-                a_Type   uDst = paTests[iTest].uDstIn; \
+                a_uType  uDst = paTests[iTest].uDstIn; \
                 pfn(&uDst, paTests[iTest].uSrcIn, paTests[iTest].uMisc, &fEfl); \
                 if (   uDst != paTests[iTest].uDstOut \
@@ -2470 +2553 @@
                 } \
             } \
+            \
+            /* Benchmark if all succeeded. */ \
+            if (g_cPicoSecBenchmark && RTTestSubErrorCount(g_hTest) == 0) \
+            { \
+                uint32_t const iTest       = cTests / 2; \
+                uint32_t const cIterations = EstimateIterations(_64K, ShiftDblU ## a_cBits ## Bench(_64K, pfn, &paTests[iTest])); \
+                uint64_t const cNsRealRun  = ShiftDblU ## a_cBits ## Bench(cIterations, pfn, &paTests[iTest]); \
+                RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL, \
+                             "%s%s", a_aSubTests[iFn].pszName, iVar ? "-native" : ""); \
+            } \
+            \
+            /* Next variation is native. */ \
             pfn = a_aSubTests[iFn].pfnNative; \
         } \
@@ -2543 +2638 @@
 #endif
 
-#define TEST_UNARY(a_cBits, a_Type, a_Fmt, a_TestType, a_SubTestType) \
+#define TEST_UNARY(a_cBits, a_uType, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
 TYPEDEF_SUBTEST_TYPE(a_SubTestType, a_TestType, PFNIEMAIMPLUNARYU ## a_cBits); \
-static a_SubTestType g_aUnaryU ## a_cBits [] = \
+static a_SubTestType a_aSubTests[] = \
 { \
     ENTRY_BIN(inc_u ## a_cBits), \
@@ -2557 +2652 @@
 }; \
 \
-GEN_UNARY(a_cBits, a_Type, a_Fmt, a_TestType, a_SubTestType) \
+GEN_UNARY(a_cBits, a_uType, a_Fmt, a_TestType, a_SubTestType) \
+\
+static uint64_t UnaryU ## a_cBits ## Bench(uint32_t cIterations, PFNIEMAIMPLUNARYU ## a_cBits pfn, a_TestType const *pEntry) \
+{ \
+    uint32_t const fEflIn  = pEntry->fEflIn; \
+    a_uType  const uDstIn  = pEntry->uDstIn; \
+    cIterations /= 4; \
+    RTThreadYield(); \
+    uint64_t const nsStart     = RTTimeNanoTS(); \
+    for (uint32_t i = 0; i < cIterations; i++) \
+    { \
+        uint32_t fBenchEfl = fEflIn; \
+        a_uType  uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, &fBenchEfl); \
+    } \
+    return RTTimeNanoTS() - nsStart; \
+} \
 \
 static void UnaryU ## a_cBits ## Test(void) \
 { \
-    for (size_t iFn = 0; iFn < RT_ELEMENTS(g_aUnaryU ## a_cBits); iFn++) \
+    for (size_t iFn = 0; iFn < RT_ELEMENTS(a_aSubTests); iFn++) \
     { \
-        if (!SUBTEST_CHECK_IF_ENABLED_AND_DECOMPRESS(g_aUnaryU ## a_cBits[iFn])) \
+        if (!SUBTEST_CHECK_IF_ENABLED_AND_DECOMPRESS(a_aSubTests[iFn])) \
             continue; \
-        a_TestType const * const paTests = g_aUnaryU ## a_cBits[iFn].paTests; \
-        uint32_t const           cTests  = g_aUnaryU ## a_cBits[iFn].cTests; \
+        PFNIEMAIMPLUNARYU ## a_cBits const pfn     = a_aSubTests[iFn].pfn; \
+        a_TestType const * const           paTests = a_aSubTests[iFn].paTests; \
+        uint32_t const                     cTests  = a_aSubTests[iFn].cTests; \
         if (!cTests) RTTestSkipped(g_hTest, "no tests"); \
         for (uint32_t iTest = 0; iTest < cTests; iTest++ ) \
         { \
             uint32_t fEfl = paTests[iTest].fEflIn; \
-            a_Type   uDst = paTests[iTest].uDstIn; \
-            g_aUnaryU ## a_cBits[iFn].pfn(&uDst, &fEfl); \
+            a_uType  uDst = paTests[iTest].uDstIn; \
+            pfn(&uDst, &fEfl); \
             if (   uDst != paTests[iTest].uDstOut \
                 || fEfl != paTests[iTest].fEflOut) \
@@ -2583 +2707 @@
                  *g_pu ## a_cBits  = paTests[iTest].uDstIn; \
                  *g_pfEfl          = paTests[iTest].fEflIn; \
-                 g_aUnaryU ## a_cBits[iFn].pfn(g_pu ## a_cBits, g_pfEfl); \
+                 pfn(g_pu ## a_cBits, g_pfEfl); \
                  RTTEST_CHECK(g_hTest, *g_pu ## a_cBits == paTests[iTest].uDstOut); \
                  RTTEST_CHECK(g_hTest, *g_pfEfl == paTests[iTest].fEflOut); \
             } \
         } \
-        FREE_DECOMPRESSED_TESTS(g_aUnaryU ## a_cBits[iFn]); \
+        \
+        if (g_cPicoSecBenchmark && RTTestSubErrorCount(g_hTest) == 0) \
+        { \
+            uint32_t const iTest       = cTests / 2; \
+            uint32_t const cIterations = EstimateIterations(_64K, UnaryU ## a_cBits ## Bench(_64K, pfn, &paTests[iTest])); \
+            uint64_t const cNsRealRun  = UnaryU ## a_cBits ## Bench(cIterations, pfn, &paTests[iTest]); \
+            RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL, "%s", a_aSubTests[iFn].pszName); \
+        } \
+        \
+        FREE_DECOMPRESSED_TESTS(a_aSubTests[iFn]); \
     } \
 }
-TEST_UNARY(8,  uint8_t,  "%#04RX8",   BINU8_TEST_T,  INT_UNARY_U8_T)
-TEST_UNARY(16, uint16_t, "%#06RX16",  BINU16_TEST_T, INT_UNARY_U16_T)
-TEST_UNARY(32, uint32_t, "%#010RX32", BINU32_TEST_T, INT_UNARY_U32_T)
-TEST_UNARY(64, uint64_t, "%#018RX64", BINU64_TEST_T, INT_UNARY_U64_T)
+TEST_UNARY(8,  uint8_t,  "%#04RX8",   BINU8_TEST_T,  INT_UNARY_U8_T,  g_aUnaryU8)
+TEST_UNARY(16, uint16_t, "%#06RX16",  BINU16_TEST_T, INT_UNARY_U16_T, g_aUnaryU16)
+TEST_UNARY(32, uint32_t, "%#010RX32", BINU32_TEST_T, INT_UNARY_U32_T, g_aUnaryU32)
+TEST_UNARY(64, uint64_t, "%#018RX64", BINU64_TEST_T, INT_UNARY_U64_T, g_aUnaryU64)
 
 #ifdef TSTIEMAIMPL_WITH_GENERATOR
@@ -2674 +2807 @@
 #endif
 
-#define TEST_SHIFT(a_cBits, a_Type, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
+#define TEST_SHIFT(a_cBits, a_uType, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
 TYPEDEF_SUBTEST_TYPE(a_SubTestType, a_TestType, PFNIEMAIMPLSHIFTU ## a_cBits); \
 static a_SubTestType a_aSubTests[] = \
@@ -2685 +2818 @@
     ENTRY_BIN_INTEL(rcl_u ## a_cBits, X86_EFL_OF), \
     ENTRY_BIN_AMD(  rcr_u ## a_cBits, X86_EFL_OF), \
-    ENTRY_BIN_INTEL(rcr_u ## a_cBits, X86_EFL_OF), \
+    ENTRY_BIN_INTEL(rcr_u ## a_cBits, X86_EFL_OF),  \
     ENTRY_BIN_AMD(  shl_u ## a_cBits, X86_EFL_OF | X86_EFL_AF), \
     ENTRY_BIN_INTEL(shl_u ## a_cBits, X86_EFL_OF | X86_EFL_AF), \
@@ -2695 +2828 @@
 \
 GEN_SHIFT(a_cBits, a_Fmt, a_TestType, a_aSubTests) \
+\
+static uint64_t ShiftU ## a_cBits ## Bench(uint32_t cIterations, PFNIEMAIMPLSHIFTU ## a_cBits pfn, a_TestType const *pEntry) \
+{ \
+    uint32_t const fEflIn = pEntry->fEflIn; \
+    a_uType  const uDstIn = pEntry->uDstIn; \
+    a_uType  const cShift = pEntry->uMisc; \
+    cIterations /= 4; \
+    RTThreadYield(); \
+    uint64_t const nsStart     = RTTimeNanoTS(); \
+    for (uint32_t i = 0; i < cIterations; i++) \
+    { \
+        uint32_t fBenchEfl = fEflIn; \
+        a_uType  uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, cShift, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, cShift, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, cShift, &fBenchEfl); \
+        \
+        fBenchEfl = fEflIn; \
+        uBenchDst = uDstIn;  \
+        pfn(&uBenchDst, cShift, &fBenchEfl); \
+    } \
+    return RTTimeNanoTS() - nsStart; \
+} \
 \
 static void ShiftU ## a_cBits ## Test(void) \
@@ -2712 +2874 @@
             { \
                 uint32_t fEfl = paTests[iTest].fEflIn; \
-                a_Type   uDst = paTests[iTest].uDstIn; \
+                a_uType  uDst = paTests[iTest].uDstIn; \
                 pfn(&uDst, paTests[iTest].uMisc, &fEfl); \
                 if (   uDst != paTests[iTest].uDstOut \
@@ -2730 +2892 @@
                 } \
             } \
+            \
+            /* Benchmark if all succeeded. */ \
+            if (g_cPicoSecBenchmark && RTTestSubErrorCount(g_hTest) == 0) \
+            { \
+                uint32_t const iTest       = cTests / 2; \
+                uint32_t const cIterations = EstimateIterations(_64K, ShiftU ## a_cBits ## Bench(_64K, pfn, &paTests[iTest])); \
+                uint64_t const cNsRealRun  = ShiftU ## a_cBits ## Bench(cIterations, pfn, &paTests[iTest]); \
+                RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL, \
+                             "%s%s", a_aSubTests[iFn].pszName, iVar ? "-native" : ""); \
+            } \
+            \
+            /* Next variation is native. */ \
             pfn = a_aSubTests[iFn].pfnNative; \
         } \
@@ -2853 +3027 @@
 #endif
 
+static uint64_t MulDivU8Bench(uint32_t cIterations, PFNIEMAIMPLMULDIVU8 pfn, MULDIVU8_TEST_T const *pEntry)
+{
+    uint32_t const fEflIn = pEntry->fEflIn;
+    uint16_t const uDstIn = pEntry->uDstIn;
+    uint8_t  const uSrcIn  = pEntry->uSrcIn;
+    cIterations /= 4;
+    RTThreadYield();
+    uint64_t const nsStart     = RTTimeNanoTS();
+    for (uint32_t i = 0; i < cIterations; i++)
+    {
+        uint32_t fBenchEfl  = fEflIn;
+        uint16_t uBenchDst = uDstIn;
+        pfn(&uBenchDst, uSrcIn, &fBenchEfl);
+
+        fBenchEfl = fEflIn;
+        uBenchDst = uDstIn;
+        pfn(&uBenchDst, uSrcIn, &fBenchEfl);
+
+        fBenchEfl = fEflIn;
+        uBenchDst = uDstIn;
+        pfn(&uBenchDst, uSrcIn, &fBenchEfl);
+
+        fBenchEfl = fEflIn;
+        uBenchDst = uDstIn;
+        pfn(&uBenchDst, uSrcIn, &fBenchEfl);
+    }
+    return RTTimeNanoTS() - nsStart;
+}
+
 static void MulDivU8Test(void)
 {
     for (size_t iFn = 0; iFn < RT_ELEMENTS(g_aMulDivU8); iFn++)
     {
-        if (!SUBTEST_CHECK_IF_ENABLED_AND_DECOMPRESS(g_aMulDivU8[iFn])) \
-            continue; \
+        if (!SUBTEST_CHECK_IF_ENABLED_AND_DECOMPRESS(g_aMulDivU8[iFn]))
+            continue;
         MULDIVU8_TEST_T const * const paTests = g_aMulDivU8[iFn].paTests;
         uint32_t const                cTests  = g_aMulDivU8[iFn].cTests;
         uint32_t const                fEflIgn = g_aMulDivU8[iFn].uExtra;
         PFNIEMAIMPLMULDIVU8           pfn     = g_aMulDivU8[iFn].pfn;
-        uint32_t const                cVars   = COUNT_VARIATIONS(g_aMulDivU8[iFn]); \
+        uint32_t const                cVars   = COUNT_VARIATIONS(g_aMulDivU8[iFn]);
         if (!cTests) RTTestSkipped(g_hTest, "no tests");
         for (uint32_t iVar = 0; iVar < cVars; iVar++)
@@ -2892 +3095 @@
                 }
             }
+
+            /* Benchmark if all succeeded. */
+            if (g_cPicoSecBenchmark && RTTestSubErrorCount(g_hTest) == 0)
+            {
+                uint32_t const iTest       = cTests / 2;
+                uint32_t const cIterations = EstimateIterations(_64K, MulDivU8Bench(_64K, pfn, &paTests[iTest]));
+                uint64_t const cNsRealRun  = MulDivU8Bench(cIterations, pfn, &paTests[iTest]);
+                RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL,
+                             "%s%s", g_aMulDivU8[iFn].pszName, iVar ? "-native" : "");
+            }
+
+            /* Next variation is native. */
             pfn = g_aMulDivU8[iFn].pfnNative;
         }
-        FREE_DECOMPRESSED_TESTS(g_aMulDivU8[iFn]); \
+        FREE_DECOMPRESSED_TESTS(g_aMulDivU8[iFn]);
     }
 }
@@ -2986 +3201 @@
 #endif
 
-#define TEST_MULDIV(a_cBits, a_Type, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
+#define TEST_MULDIV(a_cBits, a_uType, a_Fmt, a_TestType, a_SubTestType, a_aSubTests) \
 TYPEDEF_SUBTEST_TYPE(a_SubTestType, a_TestType, PFNIEMAIMPLMULDIVU ## a_cBits); \
 static a_SubTestType a_aSubTests [] = \
@@ -3001 +3216 @@
 \
 GEN_MULDIV(a_cBits, a_Fmt, a_TestType, a_aSubTests) \
+\
+static uint64_t MulDivU ## a_cBits ## Bench(uint32_t cIterations, PFNIEMAIMPLMULDIVU ## a_cBits pfn, a_TestType const *pEntry) \
+{ \
+    uint32_t const fEflIn  = pEntry->fEflIn; \
+    a_uType  const uDst1In = pEntry->uDst1In; \
+    a_uType  const uDst2In = pEntry->uDst2In; \
+    a_uType  const uSrcIn  = pEntry->uSrcIn; \
+    cIterations /= 4; \
+    RTThreadYield(); \
+    uint64_t const nsStart     = RTTimeNanoTS(); \
+    for (uint32_t i = 0; i < cIterations; i++) \
+    { \
+        uint32_t fBenchEfl  = fEflIn; \
+        a_uType  uBenchDst1 = uDst1In;  \
+        a_uType  uBenchDst2 = uDst2In;  \
+        pfn(&uBenchDst1, &uBenchDst2, uSrcIn, &fBenchEfl); \
+        \
+        fBenchEfl  = fEflIn; \
+        uBenchDst1 = uDst1In; \
+        uBenchDst2 = uDst2In; \
+        pfn(&uBenchDst1, &uBenchDst2, uSrcIn, &fBenchEfl); \
+        \
+        fBenchEfl  = fEflIn; \
+        uBenchDst1 = uDst1In; \
+        uBenchDst2 = uDst2In; \
+        pfn(&uBenchDst1, &uBenchDst2, uSrcIn, &fBenchEfl); \
+        \
+        fBenchEfl  = fEflIn; \
+        uBenchDst1 = uDst1In; \
+        uBenchDst2 = uDst2In; \
+        pfn(&uBenchDst1, &uBenchDst2, uSrcIn, &fBenchEfl); \
+    } \
+    return RTTimeNanoTS() - nsStart; \
+} \
 \
 static void MulDivU ## a_cBits ## Test(void) \
@@ -3019 +3268 @@
             { \
                 uint32_t fEfl  = paTests[iTest].fEflIn; \
-                a_Type   uDst1 = paTests[iTest].uDst1In; \
-                a_Type   uDst2 = paTests[iTest].uDst2In; \
+                a_uType  uDst1 = paTests[iTest].uDst1In; \
+                a_uType  uDst2 = paTests[iTest].uDst2In; \
                 int rc = pfn(&uDst1, &uDst2, paTests[iTest].uSrcIn, &fEfl); \
                 if (   uDst1 != paTests[iTest].uDst1Out \
@@ -3048 +3297 @@
                 } \
             } \
+            \
+            /* Benchmark if all succeeded. */ \
+            if (g_cPicoSecBenchmark && RTTestSubErrorCount(g_hTest) == 0) \
+            { \
+                uint32_t const iTest       = cTests / 2; \
+                uint32_t const cIterations = EstimateIterations(_64K, MulDivU ## a_cBits ## Bench(_64K, pfn, &paTests[iTest])); \
+                uint64_t const cNsRealRun  = MulDivU ## a_cBits ## Bench(cIterations, pfn, &paTests[iTest]); \
+                RTTestValueF(g_hTest, cNsRealRun * 1000 / cIterations, RTTESTUNIT_PS_PER_CALL, \
+                             "%s%s", a_aSubTests[iFn].pszName, iVar ? "-native" : ""); \
+            } \
+            \
+            /* Next variation is native. */ \
             pfn = a_aSubTests[iFn].pfnNative; \
         } \