Changeset 36170 in vbox for trunk/src

Timestamp:

Mar 4, 2011 12:49:02 PM (14 years ago)

Author:

vboxsync

svn:sync-xref-src-repo-rev:

70361

Message:

rem: synced up to ​svn://svn.savannah.nongnu.org/qemu/trunk@6686 (repo UUID c046a42c-6fe2-441c-8c8c-71466251a162).

Location:

trunk/src

Files:

• VBox/VMM/include/REMInternal.h (modified) (1 diff)
• recompiler/COPYING.LIB (modified) (3 diffs)
• recompiler/Sun/config-host.h (modified) (1 diff)
• recompiler/VBoxRecompiler.c (modified) (11 diffs)
• recompiler/bswap.h (modified) (3 diffs)
• recompiler/cache-utils.h (added)
• recompiler/cpu-all.h (modified) (42 diffs)
• recompiler/cpu-defs.h (modified) (8 diffs)
• recompiler/cpu-exec.c (modified) (31 diffs)
• recompiler/cutils.c (modified) (3 diffs)
• recompiler/def-helper.h (added)
• recompiler/disas.h (modified) (1 diff)
• recompiler/dyngen-exec.h (modified) (5 diffs)
• recompiler/exec-all.h (modified) (10 diffs)
• recompiler/exec.c (modified) (84 diffs)
• recompiler/fpu/softfloat-macros.h (modified) (1 diff)
• recompiler/fpu/softfloat-native.c (modified) (9 diffs)
• recompiler/fpu/softfloat-native.h (modified) (8 diffs)
• recompiler/fpu/softfloat-specialize.h (modified) (7 diffs)
• recompiler/fpu/softfloat.c (modified) (14 diffs)
• recompiler/fpu/softfloat.h (modified) (8 diffs)
• recompiler/gen-icount.h (modified) (4 diffs)
• recompiler/hostregs_helper.h (modified) (3 diffs)
• recompiler/kvm.h (added)
• recompiler/osdep.h (modified) (4 diffs)
• recompiler/qemu-common.h (modified) (12 diffs)
• recompiler/qemu-lock.h (modified) (2 diffs)
• recompiler/qemu-log.h (modified) (1 diff)
• recompiler/softmmu_exec.h (modified) (1 diff)
• recompiler/softmmu_header.h (modified) (1 diff)
• recompiler/softmmu_template.h (modified) (2 diffs)
• recompiler/sys-queue.h (added)
• recompiler/target-i386/cpu.h (modified) (19 diffs)
• recompiler/target-i386/exec.h (modified) (4 diffs)
• recompiler/target-i386/helper.c (modified) (17 diffs)
• recompiler/target-i386/helper.h (modified) (3 diffs)
• recompiler/target-i386/helper_template.h (modified) (3 diffs)
• recompiler/target-i386/op_helper.c (modified) (92 diffs)
• recompiler/target-i386/ops_sse.h (modified) (17 diffs)
• recompiler/target-i386/ops_sse_header.h (modified) (6 diffs)
• recompiler/target-i386/translate.c (modified) (198 diffs)
• recompiler/tcg/README (modified) (8 diffs)
• recompiler/tcg/i386/tcg-target.c (modified) (4 diffs)
• recompiler/tcg/i386/tcg-target.h (modified) (1 diff)
• recompiler/tcg/tcg-op.h (modified) (39 diffs)
• recompiler/tcg/tcg-opc.h (modified) (1 diff)
• recompiler/tcg/tcg-runtime.c (modified) (3 diffs)
• recompiler/tcg/tcg.c (modified) (23 diffs)
• recompiler/tcg/tcg.h (modified) (10 diffs)
• recompiler/tcg/x86_64/tcg-target.c (modified) (2 diffs)
• recompiler/tests/linux-test.c (modified) (15 diffs)
• recompiler/tests/qruncom.c (modified) (1 diff)
• recompiler/tests/sha1.c (modified) (1 diff)
• recompiler/tests/test-i386-shift.h (modified) (1 diff)
• recompiler/tests/test-i386.c (modified) (1 diff)
• recompiler/tests/test-mmap.c (modified) (17 diffs)
• recompiler/tests/test_path.c (modified) (1 diff)
• recompiler/translate-all.c (modified) (4 diffs)

trunk/src/VBox/VMM/include/REMInternal.h

@@ -246 +246 @@
 int     remR3NotifyTrap(CPUState *env, uint32_t uTrap, uint32_t uErrorCode, RTGCPTR pvNextEIP);
 void    remR3TrapStat(CPUState *env, uint32_t uTrap);
-void    remR3CpuId(CPUState *env, unsigned uOperator, void *pvEAX, void *pvEBX, void *pvECX, void *pvEDX);
 void    remR3RecordCall(CPUState *env);
 #endif /* REM_INCLUDE_CPU_H */

trunk/src/recompiler/COPYING.LIB

@@ -3 +3 @@
 
  Copyright (C) 1991, 1999 Free Software Foundation, Inc.
-     59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+        51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
@@ -149 +149 @@
 writing it).  Whether that is true depends on what the Library does
 and what the program that uses the Library does.
-  
+
   1. You may copy and distribute verbatim copies of the Library's
 complete source code as you receive it, in any medium, provided that
@@ -495 +495 @@
     You should have received a copy of the GNU Lesser General Public
     License along with this library; if not, write to the Free Software
-    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+    Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
 
 Also add information on how to contact you by electronic and paper mail.

trunk/src/recompiler/Sun/config-host.h

@@ -30 +30 @@
 #  define CONFIG_DARWIN
 # elif defined(RT_OS_FREEBSD) || defined(RT_OS_NETBSD) || defined(RT_OS_OPENBSD)
+#  define HAVE_MACHINE_BSWAP_H
 /*#  define CONFIG_BSD*/
 # elif defined(RT_OS_SOLARIS)

trunk/src/recompiler/VBoxRecompiler.c

@@ -23 +23 @@
 #include <stdio.h>      /* FILE */
 #include "osdep.h"
+#include "config.h"
+#include "cpu.h"
 #include "exec-all.h"
-#include "config.h"
-#include "cpu-all.h"
 
 #include <VBox/vmm/rem.h>
@@ -821 +821 @@
      */
     GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
-    fBp = !cpu_breakpoint_remove(&pVM->rem.s.Env, GCPtrPC);
+    fBp = !cpu_breakpoint_remove(&pVM->rem.s.Env, GCPtrPC, BP_GDB);
 
     /*
@@ -864 +864 @@
     if (fBp)
     {
-        int rc2 = cpu_breakpoint_insert(&pVM->rem.s.Env, GCPtrPC);
+        int rc2 = cpu_breakpoint_insert(&pVM->rem.s.Env, GCPtrPC, BP_GDB, NULL);
         Assert(rc2 == 0); NOREF(rc2);
     }
@@ -885 +885 @@
 {
     VM_ASSERT_EMT(pVM);
-    if (!cpu_breakpoint_insert(&pVM->rem.s.Env, Address))
+    if (!cpu_breakpoint_insert(&pVM->rem.s.Env, Address, BP_GDB, NULL))
     {
         LogFlow(("REMR3BreakpointSet: Address=%RGv\n", Address));
@@ -906 +906 @@
 {
     VM_ASSERT_EMT(pVM);
-    if (!cpu_breakpoint_remove(&pVM->rem.s.Env, Address))
+    if (!cpu_breakpoint_remove(&pVM->rem.s.Env, Address, BP_GDB))
     {
         LogFlow(("REMR3BreakpointClear: Address=%RGv\n", Address));
@@ -990 +990 @@
              */
             case EXCP_DEBUG:
-            {
-                /* breakpoint or single step? */
-                RTGCPTR     GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
-                int         iBP;
-                rc = VINF_EM_DBG_STEPPED;
-                for (iBP = 0; iBP < pVM->rem.s.Env.nb_breakpoints; iBP++)
-                    if (pVM->rem.s.Env.breakpoints[iBP] == GCPtrPC)
-                    {
-                        rc = VINF_EM_DBG_BREAKPOINT;
-                        break;
-                    }
-                Log2(("REMR3EmulateInstruction: cpu_exec -> EXCP_DEBUG rc=%Rrc iBP=%d GCPtrPC=%RGv\n", rc, iBP, GCPtrPC));
+                if (pVM->rem.s.Env.watchpoint_hit)
+                {
+                    /** @todo deal with watchpoints */
+                    Log2(("REMR3EmulateInstruction: cpu_exec -> EXCP_DEBUG rc=%Rrc !watchpoint_hit!\n", rc));
+                    rc = VINF_EM_DBG_BREAKPOINT;
+                }
+                else
+                {
+                    CPUBreakpoint  *pBP;
+                    RTGCPTR         GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
+                    TAILQ_FOREACH(pBP, &pVM->rem.s.Env.breakpoints, entry)
+                        if (pBP->pc == GCPtrPC)
+                            break;
+                    rc = pBP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_STEPPED;
+                    Log2(("REMR3EmulateInstruction: cpu_exec -> EXCP_DEBUG rc=%Rrc pBP=%p GCPtrPC=%RGv\n", rc, pBP, GCPtrPC));
+                }
                 break;
-            }
 
             /*
@@ -1170 +1173 @@
 #endif
             case EXCP_DEBUG:
-                rc = VINF_EM_DBG_STEPPED;
-                if (pVM->rem.s.Env.nb_breakpoints > 0)
+                if (pVM->rem.s.Env.watchpoint_hit)
                 {
-                    RTGCPTR     GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
-                    int         iBP;
-                    for (iBP = 0; iBP < pVM->rem.s.Env.nb_breakpoints; iBP++)
-                        if (pVM->rem.s.Env.breakpoints[iBP] == GCPtrPC)
-                        {
-                            rc = VINF_EM_DBG_BREAKPOINT;
-                            RTLogPrintf("remR3RunLoggingStep: cpu_exec -> EXCP_DEBUG rc=%Rrc iBP=%d GCPtrPC=%RGv\n", rc, iBP, GCPtrPC);
+                    /** @todo deal with watchpoints */
+                    Log2(("remR3RunLoggingStep: cpu_exec -> EXCP_DEBUG rc=%Rrc !watchpoint_hit!\n", rc));
+                    rc = VINF_EM_DBG_BREAKPOINT;
+                }
+                else
+                {
+                    CPUBreakpoint  *pBP;
+                    RTGCPTR         GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
+                    TAILQ_FOREACH(pBP, &pVM->rem.s.Env.breakpoints, entry)
+                        if (pBP->pc == GCPtrPC)
                             break;
-                        }
+                    rc = pBP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_STEPPED;
+                    Log2(("remR3RunLoggingStep: cpu_exec -> EXCP_DEBUG rc=%Rrc pBP=%p GCPtrPC=%RGv\n", rc, pBP, GCPtrPC));
                 }
 #ifdef REM_USE_QEMU_SINGLE_STEP_FOR_LOGGING
@@ -1325 +1331 @@
          */
         case EXCP_DEBUG:
-        {
-#if 0//def DEBUG_bird
-            static int iBP = 0;
-            printf("howdy, breakpoint! iBP=%d\n", iBP);
-            switch (iBP)
+            if (pVM->rem.s.Env.watchpoint_hit)
             {
-                case 0:
-                    cpu_breakpoint_remove(&pVM->rem.s.Env, pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base);
-                    pVM->rem.s.Env.state |= CPU_EMULATE_SINGLE_STEP;
-                    //pVM->rem.s.Env.interrupt_request = 0;
-                    //pVM->rem.s.Env.exception_index = -1;
-                    //g_fInterruptDisabled = 1;
-                    rc = VINF_SUCCESS;
-                    asm("int3");
-                    break;
-                default:
-                    asm("int3");
-                    break;
+                /** @todo deal with watchpoints */
+                Log2(("REMR3Run: cpu_exec -> EXCP_DEBUG rc=%Rrc !watchpoint_hit!\n", rc));
+                rc = VINF_EM_DBG_BREAKPOINT;
             }
-            iBP++;
-#else
-            /* breakpoint or single step? */
-            RTGCPTR     GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
-            int         iBP;
-            rc = VINF_EM_DBG_STEPPED;
-            for (iBP = 0; iBP < pVM->rem.s.Env.nb_breakpoints; iBP++)
-                if (pVM->rem.s.Env.breakpoints[iBP] == GCPtrPC)
-                {
-                    rc = VINF_EM_DBG_BREAKPOINT;
-                    break;
-                }
-            Log2(("REMR3Run: cpu_exec -> EXCP_DEBUG rc=%Rrc iBP=%d GCPtrPC=%RGv\n", rc, iBP, GCPtrPC));
-#endif
+            else
+            {
+                CPUBreakpoint  *pBP;
+                RTGCPTR         GCPtrPC = pVM->rem.s.Env.eip + pVM->rem.s.Env.segs[R_CS].base;
+                TAILQ_FOREACH(pBP, &pVM->rem.s.Env.breakpoints, entry)
+                    if (pBP->pc == GCPtrPC)
+                        break;
+                rc = pBP ? VINF_EM_DBG_BREAKPOINT : VINF_EM_DBG_STEPPED;
+                Log2(("REMR3Run: cpu_exec -> EXCP_DEBUG rc=%Rrc pBP=%p GCPtrPC=%RGv\n", rc, pBP, GCPtrPC));
+            }
             break;
-        }
 
         /*
@@ -1540 +1528 @@
     }
 
-    if (env->nb_breakpoints > 0)
+    if (!TAILQ_EMPTY(&env->breakpoints))
     {
         //Log2(("raw mode refused: Breakpoints\n"));
+        return false;
+    }
+
+    if (!TAILQ_EMPTY(&env->watchpoints))
+    {
+        //Log2(("raw mode refused: Watchpoints\n"));
         return false;
     }
@@ -4625 +4619 @@
  * Perform the CPUID instruction.
  *
- * ASMCpuId cannot be invoked from some source files where this is used because of global
- * register allocations.
- *
  * @param   env         Pointer to the recompiler CPU structure.
- * @param   uOperator   CPUID operation (eax).
+ * @param   idx         The CPUID leaf (eax).
+ * @param   idxSub      The CPUID sub-leaf (ecx) where applicable.
  * @param   pvEAX       Where to store eax.
  * @param   pvEBX       Where to store ebx.
@@ -4635 +4627 @@
  * @param   pvEDX       Where to store edx.
  */
-void remR3CpuId(CPUState *env, unsigned uOperator, void *pvEAX, void *pvEBX, void *pvECX, void *pvEDX)
-{
-    CPUMGetGuestCpuId(env->pVCpu, uOperator, (uint32_t *)pvEAX, (uint32_t *)pvEBX, (uint32_t *)pvECX, (uint32_t *)pvEDX);
+void cpu_x86_cpuid(CPUX86State *env, uint32_t idx, uint32_t idxSub,
+                   uint32_t *pEAX, uint32_t *pEBX, uint32_t *pECX, uint32_t *pEDX)
+{
+    NOREF(idxSub);
+    CPUMGetGuestCpuId(env->pVCpu, idx, pEAX, pEBX, pECX, pEDX);
 }
 

trunk/src/recompiler/bswap.h

@@ -5 +5 @@
 
 #include <inttypes.h>
+
+#ifdef HAVE_MACHINE_BSWAP_H
+#include <sys/endian.h>
+#include <sys/types.h>
+#include <machine/bswap.h>
+#else
 
 #ifdef HAVE_BYTESWAP_H
@@ -44 +50 @@
 #endif /* !HAVE_BYTESWAP_H */
 
-#ifndef bswap16 /* BSD endian.h clash */
 static inline uint16_t bswap16(uint16_t x)
 {
     return bswap_16(x);
 }
-#endif
-
-#ifndef bswap32 /* BSD endian.h clash */
+
 static inline uint32_t bswap32(uint32_t x)
 {
     return bswap_32(x);
 }
-#endif
-
-#ifndef bswap64 /* BSD endian.h clash. */
+
 static inline uint64_t bswap64(uint64_t x)
 {
     return bswap_64(x);
 }
-#endif
+
+#endif /* ! HAVE_MACHINE_BSWAP_H */
 
 static inline void bswap16s(uint16_t *s)
@@ -133 +135 @@
 /* unaligned versions (optimized for frequent unaligned accesses)*/
 
-#if defined(__i386__) || defined(__powerpc__)
+#if defined(__i386__) || defined(_ARCH_PPC)
 
 #define cpu_to_le16wu(p, v) cpu_to_le16w(p, v)

trunk/src/recompiler/cpu-all.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -37 +37 @@
 # include <VBox/log.h>
 # include <VBox/vmm/pgm.h> /* PGM_DYNAMIC_RAM_ALLOC */
-#endif
+#endif /* VBOX */
+#include "qemu-common.h"
 
 #if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__)
@@ -252 +253 @@
 #if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
 
-DECLINLINE(uint8_t) ldub_p(void *ptr)
+DECLINLINE(uint8_t) ldub_p(const void *ptr)
 {
     VBOX_CHECK_ADDR(ptr);
@@ -258 +259 @@
 }
 
-DECLINLINE(int8_t) ldsb_p(void *ptr)
+DECLINLINE(int8_t) ldsb_p(const void *ptr)
 {
     VBOX_CHECK_ADDR(ptr);
@@ -270 +271 @@
 }
 
-DECLINLINE(uint32_t) lduw_le_p(void *ptr)
+DECLINLINE(uint32_t) lduw_le_p(const void *ptr)
 {
     VBOX_CHECK_ADDR(ptr);
@@ -276 +277 @@
 }
 
-DECLINLINE(int32_t) ldsw_le_p(void *ptr)
+DECLINLINE(int32_t) ldsw_le_p(const void *ptr)
 {
     VBOX_CHECK_ADDR(ptr);
@@ -288 +289 @@
 }
 
-DECLINLINE(uint32_t) ldl_le_p(void *ptr)
+DECLINLINE(uint32_t) ldl_le_p(const void *ptr)
 {
     VBOX_CHECK_ADDR(ptr);
@@ -306 +307 @@
 }
 
-DECLINLINE(uint64_t) ldq_le_p(void *ptr)
+DECLINLINE(uint64_t) ldq_le_p(const void *ptr)
 {
     VBOX_CHECK_ADDR(ptr);
@@ -316 +317 @@
 /* float access */
 
-DECLINLINE(float32) ldfl_le_p(void *ptr)
+DECLINLINE(float32) ldfl_le_p(const void *ptr)
 {
     union {
@@ -336 +337 @@
 }
 
-DECLINLINE(float64) ldfq_le_p(void *ptr)
+DECLINLINE(float64) ldfq_le_p(const void *ptr)
 {
     CPU_DoubleU u;
@@ -354 +355 @@
 #else  /* !VBOX */
 
-static inline int ldub_p(void *ptr)
+static inline int ldub_p(const void *ptr)
 {
     return *(uint8_t *)ptr;
 }
 
-static inline int ldsb_p(void *ptr)
+static inline int ldsb_p(const void *ptr)
 {
     return *(int8_t *)ptr;
@@ -375 +376 @@
 
 /* conservative code for little endian unaligned accesses */
-static inline int lduw_le_p(void *ptr)
-{
-#ifdef __powerpc__
+static inline int lduw_le_p(const void *ptr)
+{
+#ifdef _ARCH_PPC
     int val;
     __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (ptr));
     return val;
 #else
-    uint8_t *p = ptr;
+    const uint8_t *p = ptr;
     return p[0] | (p[1] << 8);
 #endif
 }
 
-static inline int ldsw_le_p(void *ptr)
-{
-#ifdef __powerpc__
+static inline int ldsw_le_p(const void *ptr)
+{
+#ifdef _ARCH_PPC
     int val;
     __asm__ __volatile__ ("lhbrx %0,0,%1" : "=r" (val) : "r" (ptr));
     return (int16_t)val;
 #else
-    uint8_t *p = ptr;
+    const uint8_t *p = ptr;
     return (int16_t)(p[0] | (p[1] << 8));
 #endif
 }
 
-static inline int ldl_le_p(void *ptr)
-{
-#ifdef __powerpc__
+static inline int ldl_le_p(const void *ptr)
+{
+#ifdef _ARCH_PPC
     int val;
     __asm__ __volatile__ ("lwbrx %0,0,%1" : "=r" (val) : "r" (ptr));
     return val;
 #else
-    uint8_t *p = ptr;
+    const uint8_t *p = ptr;
     return p[0] | (p[1] << 8) | (p[2] << 16) | (p[3] << 24);
 #endif
 }
 
-static inline uint64_t ldq_le_p(void *ptr)
-{
-    uint8_t *p = ptr;
+static inline uint64_t ldq_le_p(const void *ptr)
+{
+    const uint8_t *p = ptr;
     uint32_t v1, v2;
     v1 = ldl_le_p(p);
@@ -422 +423 @@
 static inline void stw_le_p(void *ptr, int v)
 {
-#ifdef __powerpc__
+#ifdef _ARCH_PPC
     __asm__ __volatile__ ("sthbrx %1,0,%2" : "=m" (*(uint16_t *)ptr) : "r" (v), "r" (ptr));
 #else
@@ -433 +434 @@
 static inline void stl_le_p(void *ptr, int v)
 {
-#ifdef __powerpc__
+#ifdef _ARCH_PPC
     __asm__ __volatile__ ("stwbrx %1,0,%2" : "=m" (*(uint32_t *)ptr) : "r" (v), "r" (ptr));
 #else
@@ -453 +454 @@
 /* float access */
 
-static inline float32 ldfl_le_p(void *ptr)
+static inline float32 ldfl_le_p(const void *ptr)
 {
     union {
@@ -473 +474 @@
 }
 
-static inline float64 ldfq_le_p(void *ptr)
+static inline float64 ldfq_le_p(const void *ptr)
 {
     CPU_DoubleU u;
@@ -491 +492 @@
 #else
 
-static inline int lduw_le_p(void *ptr)
+static inline int lduw_le_p(const void *ptr)
 {
     return *(uint16_t *)ptr;
 }
 
-static inline int ldsw_le_p(void *ptr)
+static inline int ldsw_le_p(const void *ptr)
 {
     return *(int16_t *)ptr;
 }
 
-static inline int ldl_le_p(void *ptr)
+static inline int ldl_le_p(const void *ptr)
 {
     return *(uint32_t *)ptr;
 }
 
-static inline uint64_t ldq_le_p(void *ptr)
+static inline uint64_t ldq_le_p(const void *ptr)
 {
     return *(uint64_t *)ptr;
@@ -528 +529 @@
 /* float access */
 
-static inline float32 ldfl_le_p(void *ptr)
+static inline float32 ldfl_le_p(const void *ptr)
 {
     return *(float32 *)ptr;
 }
 
-static inline float64 ldfq_le_p(void *ptr)
+static inline float64 ldfq_le_p(const void *ptr)
 {
     return *(float64 *)ptr;
@@ -552 +553 @@
 #if !defined(WORDS_BIGENDIAN) || defined(WORDS_ALIGNED)
 
-static inline int lduw_be_p(void *ptr)
+static inline int lduw_be_p(const void *ptr)
 {
 #if defined(__i386__)
@@ -562 +563 @@
     return val;
 #else
-    uint8_t *b = (uint8_t *) ptr;
+    const uint8_t *b = ptr;
     return ((b[0] << 8) | b[1]);
 #endif
 }
 
-static inline int ldsw_be_p(void *ptr)
+static inline int ldsw_be_p(const void *ptr)
 {
 #if defined(__i386__)
@@ -577 +578 @@
     return (int16_t)val;
 #else
-    uint8_t *b = (uint8_t *) ptr;
+    const uint8_t *b = ptr;
     return (int16_t)((b[0] << 8) | b[1]);
 #endif
 }
 
-static inline int ldl_be_p(void *ptr)
+static inline int ldl_be_p(const void *ptr)
 {
 #if defined(__i386__) || defined(__x86_64__)
@@ -592 +593 @@
     return val;
 #else
-    uint8_t *b = (uint8_t *) ptr;
+    const uint8_t *b = ptr;
     return (b[0] << 24) | (b[1] << 16) | (b[2] << 8) | b[3];
 #endif
 }
 
-static inline uint64_t ldq_be_p(void *ptr)
+static inline uint64_t ldq_be_p(const void *ptr)
 {
     uint32_t a,b;
@@ -643 +644 @@
 /* float access */
 
-static inline float32 ldfl_be_p(void *ptr)
+static inline float32 ldfl_be_p(const void *ptr)
 {
     union {
@@ -663 +664 @@
 }
 
-static inline float64 ldfq_be_p(void *ptr)
+static inline float64 ldfq_be_p(const void *ptr)
 {
     CPU_DoubleU u;
@@ -681 +682 @@
 #else
 
-static inline int lduw_be_p(void *ptr)
+static inline int lduw_be_p(const void *ptr)
 {
     return *(uint16_t *)ptr;
 }
 
-static inline int ldsw_be_p(void *ptr)
+static inline int ldsw_be_p(const void *ptr)
 {
     return *(int16_t *)ptr;
 }
 
-static inline int ldl_be_p(void *ptr)
+static inline int ldl_be_p(const void *ptr)
 {
     return *(uint32_t *)ptr;
 }
 
-static inline uint64_t ldq_be_p(void *ptr)
+static inline uint64_t ldq_be_p(const void *ptr)
 {
     return *(uint64_t *)ptr;
@@ -718 +719 @@
 /* float access */
 
-static inline float32 ldfl_be_p(void *ptr)
+static inline float32 ldfl_be_p(const void *ptr)
 {
     return *(float32 *)ptr;
 }
 
-static inline float64 ldfq_be_p(void *ptr)
+static inline float64 ldfq_be_p(const void *ptr)
 {
     return *(float64 *)ptr;
@@ -770 +771 @@
 
 #if defined(CONFIG_USER_ONLY)
+#include <assert.h>
+#include "qemu-types.h"
+
 /* On some host systems the guest address space is reserved on the host.
  * This allows the guest address space to be offset to a convenient location.
@@ -778 +782 @@
 /* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
 #define g2h(x) ((void *)((unsigned long)(x) + GUEST_BASE))
-#define h2g(x) ((target_ulong)((unsigned long)(x) - GUEST_BASE))
+#define h2g(x) ({ \
+    unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
+    /* Check if given address fits target address space */ \
+    assert(__ret == (abi_ulong)__ret); \
+    (abi_ulong)__ret; \
+})
+#define h2g_valid(x) ({ \
+    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
+    (__guest == (abi_ulong)__guest); \
+})
 
 #define saddr(x) g2h(x)
@@ -877 +890 @@
 void page_set_flags(target_ulong start, target_ulong end, int flags);
 int page_check_range(target_ulong start, target_ulong len, int flags);
-void page_unprotect_range(target_ulong data, target_ulong data_size);
+
+void cpu_exec_init_all(unsigned long tb_size);
+CPUState *cpu_copy(CPUState *env);
 
 void cpu_dump_state(CPUState *env, FILE *f,
@@ -886 +901 @@
                           int flags);
 
-void cpu_abort(CPUState *env, const char *fmt, ...)
+void QEMU_NORETURN cpu_abort(CPUState *env, const char *fmt, ...)
 #ifndef VBOX
-    __attribute__ ((__format__ (__printf__, 2, 3)))
-#endif
-    __attribute__ ((__noreturn__));
+    __attribute__ ((__format__ (__printf__, 2, 3)));
+#else
+    ;
+#endif
 extern CPUState *first_cpu;
 extern CPUState *cpu_single_env;
@@ -926 +942 @@
 void cpu_reset_interrupt(CPUState *env, int mask);
 
-int cpu_watchpoint_insert(CPUState *env, target_ulong addr, int type);
-int cpu_watchpoint_remove(CPUState *env, target_ulong addr);
-void cpu_watchpoint_remove_all(CPUState *env);
-int cpu_breakpoint_insert(CPUState *env, target_ulong pc);
-int cpu_breakpoint_remove(CPUState *env, target_ulong pc);
-void cpu_breakpoint_remove_all(CPUState *env);
+/* Breakpoint/watchpoint flags */
+#define BP_MEM_READ           0x01
+#define BP_MEM_WRITE          0x02
+#define BP_MEM_ACCESS         (BP_MEM_READ | BP_MEM_WRITE)
+#define BP_STOP_BEFORE_ACCESS 0x04
+#define BP_WATCHPOINT_HIT     0x08
+#define BP_GDB                0x10
+#define BP_CPU                0x20
+
+int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
+                          CPUBreakpoint **breakpoint);
+int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags);
+void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint);
+void cpu_breakpoint_remove_all(CPUState *env, int mask);
+int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
+                          int flags, CPUWatchpoint **watchpoint);
+int cpu_watchpoint_remove(CPUState *env, target_ulong addr,
+                          target_ulong len, int flags);
+void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint);
+void cpu_watchpoint_remove_all(CPUState *env, int mask);
 
 #define SSTEP_ENABLE  0x1  /* Enable simulated HW single stepping */
@@ -954 +984 @@
 #define CPU_LOG_IOPORT     (1 << 7)
 #define CPU_LOG_TB_CPU     (1 << 8)
+#define CPU_LOG_RESET      (1 << 9)
 
 /* define log items */
@@ -1035 +1066 @@
 typedef uint32_t CPUReadMemoryFunc(void *opaque, target_phys_addr_t addr);
 
-void cpu_register_physical_memory(target_phys_addr_t start_addr,
-                                  ram_addr_t size,
-                                  ram_addr_t phys_offset);
+void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
+                                         ram_addr_t size,
+                                         ram_addr_t phys_offset,
+                                         ram_addr_t region_offset);
+static inline void cpu_register_physical_memory(target_phys_addr_t start_addr,
+                                                ram_addr_t size,
+                                                ram_addr_t phys_offset)
+{
+    cpu_register_physical_memory_offset(start_addr, size, phys_offset, 0);
+}
+
 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr);
 ram_addr_t qemu_ram_alloc(ram_addr_t);
@@ -1045 +1084 @@
                            CPUWriteMemoryFunc **mem_write,
                            void *opaque);
+void cpu_unregister_io_memory(int table_address);
 CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index);
 CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index);
@@ -1060 +1100 @@
     cpu_physical_memory_rw(addr, (uint8_t *)buf, len, 1);
 }
+void *cpu_physical_memory_map(target_phys_addr_t addr,
+                              target_phys_addr_t *plen,
+                              int is_write);
+void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
+                               int is_write, target_phys_addr_t access_len);
+void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque));
+void cpu_unregister_map_client(void *cookie);
+
 uint32_t ldub_phys(target_phys_addr_t addr);
 uint32_t lduw_phys(target_phys_addr_t addr);
@@ -1087 +1135 @@
     return phys_ram_dirty[addr >> TARGET_PAGE_BITS] == 0xff;
 }
-#else
+#else  /* VBOX */
 DECLINLINE(int) cpu_physical_memory_is_dirty(ram_addr_t addr)
 {
@@ -1098 +1146 @@
     return phys_ram_dirty[addr >> TARGET_PAGE_BITS] == 0xff;
 }
-#endif
+#endif /* VBOX */
 
 #ifndef VBOX
@@ -1106 +1154 @@
     return phys_ram_dirty[addr >> TARGET_PAGE_BITS] & dirty_flags;
 }
-#else
+#else  /* VBOX */
 DECLINLINE(int) cpu_physical_memory_get_dirty(ram_addr_t addr,
                                               int dirty_flags)
@@ -1118 +1166 @@
     return phys_ram_dirty[addr >> TARGET_PAGE_BITS] & dirty_flags;
 }
-#endif
+#endif /* VBOX */
 
 #ifndef VBOX
@@ -1125 +1173 @@
     phys_ram_dirty[addr >> TARGET_PAGE_BITS] = 0xff;
 }
-#else
+#else  /* VBOX */
 DECLINLINE(void) cpu_physical_memory_set_dirty(ram_addr_t addr)
 {
@@ -1136 +1184 @@
     phys_ram_dirty[addr >> TARGET_PAGE_BITS] = 0xff;
 }
-#endif
+#endif /* VBOX */
 
 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
@@ -1146 +1194 @@
 int cpu_physical_memory_get_dirty_tracking(void);
 
+void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr);
+
 void dump_exec_info(FILE *f,
                     int (*cpu_fprintf)(FILE *f, const char *fmt, ...));
 
+/* Coalesced MMIO regions are areas where write operations can be reordered.
+ * This usually implies that write operations are side-effect free.  This allows
+ * batching which can make a major impact on performance when using
+ * virtualization.
+ */
+void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
+
+void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
+
 /*******************************************/
 /* host CPU ticks (if available) */
 
-#if defined(__powerpc__)
-
-static inline uint32_t get_tbl(void)
-{
-    uint32_t tbl;
-    asm volatile("mftb %0" : "=r" (tbl));
-    return tbl;
-}
-
-static inline uint32_t get_tbu(void)
-{
-        uint32_t tbl;
-        asm volatile("mftbu %0" : "=r" (tbl));
-        return tbl;
-}
+#if defined(_ARCH_PPC)
 
 static inline int64_t cpu_get_real_ticks(void)
 {
-    uint32_t l, h, h1;
-    /* NOTE: we test if wrapping has occurred */
-    do {
-        h = get_tbu();
-        l = get_tbl();
-        h1 = get_tbu();
-    } while (h != h1);
-    return ((int64_t)h << 32) | l;
+    int64_t retval;
+#ifdef _ARCH_PPC64
+    /* This reads timebase in one 64bit go and includes Cell workaround from:
+       http://ozlabs.org/pipermail/linuxppc-dev/2006-October/027052.html
+     */
+    __asm__ __volatile__ (
+        "mftb    %0\n\t"
+        "cmpwi   %0,0\n\t"
+        "beq-    $-8"
+        : "=r" (retval));
+#else
+    /* http://ozlabs.org/pipermail/linuxppc-dev/1999-October/003889.html */
+    unsigned long junk;
+    __asm__ __volatile__ (
+        "mftbu   %1\n\t"
+        "mftb    %L0\n\t"
+        "mftbu   %0\n\t"
+        "cmpw    %0,%1\n\t"
+        "bne     $-16"
+        : "=r" (retval), "=r" (junk));
+#endif
+    return retval;
 }
 

trunk/src/recompiler/cpu-defs.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -39 +39 @@
 #include <inttypes.h>
 #include "osdep.h"
+#include "sys-queue.h"
 
 #ifndef TARGET_LONG_BITS
@@ -93 +94 @@
 #define EXCP_DEBUG      0x10002 /* cpu stopped after a breakpoint or singlestep */
 #define EXCP_HALTED     0x10003 /* cpu is halted (waiting for external event) */
-#if defined(VBOX)
-#define EXCP_EXECUTE_RAW    0x11024 /* execute raw mode. */
-#define EXCP_EXECUTE_HWACC  0x11025 /* execute hardware accelerated raw mode. */
-#define EXCP_SINGLE_INSTR   0x11026 /* executed single instruction. */
-#define EXCP_RC             0x11027 /* a EM rc was raised (VMR3Reset/Suspend/PowerOff). */
+#ifdef VBOX
+# define EXCP_EXECUTE_RAW   0x11024 /**< execute raw mode. */
+# define EXCP_EXECUTE_HWACC 0x11025 /**< execute hardware accelerated raw mode. */
+# define EXCP_SINGLE_INSTR  0x11026 /**< executed single instruction. */
+# define EXCP_RC            0x11027 /**< a EM rc was raised (VMR3Reset/Suspend/PowerOff). */
 #endif /* VBOX */
-#define MAX_BREAKPOINTS 32
-#define MAX_WATCHPOINTS 32
 
 #define TB_JMP_CACHE_BITS 12
@@ -133 +132 @@
     target_ulong addr_code;
     /* Addend to virtual address to get physical address.  IO accesses
-       use the correcponding iotlb value.  */
+       use the corresponding iotlb value.  */
 #if TARGET_PHYS_ADDR_BITS == 64
     /* on i386 Linux make sure it is aligned */
@@ -158 +157 @@
 } icount_decr_u16;
 #endif
+
+struct kvm_run;
+struct KVMState;
+
+typedef struct CPUBreakpoint {
+    target_ulong pc;
+    int flags; /* BP_* */
+    TAILQ_ENTRY(CPUBreakpoint) entry;
+} CPUBreakpoint;
+
+typedef struct CPUWatchpoint {
+    target_ulong vaddr;
+    target_ulong len_mask;
+    int flags; /* BP_* */
+    TAILQ_ENTRY(CPUWatchpoint) entry;
+} CPUWatchpoint;
 
 #define CPU_TEMP_BUF_NLONGS 128
@@ -193 +208 @@
     /* from this point: preserved by CPU reset */                       \
     /* ice debug support */                                             \
-    target_ulong breakpoints[MAX_BREAKPOINTS];                          \
-    int nb_breakpoints;                                                 \
+    TAILQ_HEAD(breakpoints_head, CPUBreakpoint) breakpoints;            \
     int singlestep_enabled;                                             \
                                                                         \
-    struct {                                                            \
-        target_ulong vaddr;                                             \
-        int type; /* PAGE_READ/PAGE_WRITE */                            \
-    } watchpoint[MAX_WATCHPOINTS];                                      \
-    int nb_watchpoints;                                                 \
-    int watchpoint_hit;                                                 \
+    TAILQ_HEAD(watchpoints_head, CPUWatchpoint) watchpoints;            \
+    CPUWatchpoint *watchpoint_hit;                                      \
                                                                         \
     struct GDBRegisterState *gdb_regs;                                  \
@@ -209 +219 @@
     jmp_buf jmp_env;                                                    \
     int exception_index;                                                \
-                                                                        \
-    int user_mode_only;                                                 \
                                                                         \
     void *next_cpu; /* next CPU sharing TB cache */                     \
@@ -218 +226 @@
     void *opaque;                                                       \
                                                                         \
-    const char *cpu_model_str;
-
-#endif
+    const char *cpu_model_str;                                          \
+    struct KVMState *kvm_state;                                         \
+    struct kvm_run *kvm_run;                                            \
+    int kvm_fd;
+
+#endif

trunk/src/recompiler/cpu-exec.c

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -33 +33 @@
 #include "disas.h"
 #include "tcg.h"
+#include "kvm.h"
 
 #if !defined(CONFIG_SOFTMMU)
@@ -45 +46 @@
 #undef EIP
 #include <signal.h>
+#ifdef __linux__
 #include <sys/ucontext.h>
+#endif
 #endif
 
@@ -67 +70 @@
 }
 
-#if !(defined(TARGET_SPARC) || defined(TARGET_SH4) || defined(TARGET_M68K))
-#define reg_T2
-#endif
-
 /* exit the current TB from a signal handler. The host registers are
    restored in a state compatible with the CPU emulator
@@ -77 +76 @@
 {
 #if !defined(CONFIG_SOFTMMU)
+#ifdef __linux__
     struct ucontext *uc = puc;
+#elif defined(__OpenBSD__)
+    struct sigcontext *uc = puc;
+#endif
 #endif
 
@@ -87 +90 @@
     if (puc) {
         /* XXX: use siglongjmp ? */
+#ifdef __linux__
         sigprocmask(SIG_SETMASK, &uc->uc_sigmask, NULL);
-    }
-#endif
+#elif defined(__OpenBSD__)
+        sigprocmask(SIG_SETMASK, &uc->sc_mask, NULL);
+#endif
+    }
+#endif
+    env->exception_index = -1;
     longjmp(env->jmp_env, 1);
 }
@@ -118 +126 @@
         /* Restore PC.  This may happen if async event occurs before
            the TB starts executing.  */
-        CPU_PC_FROM_TB(env, tb);
+        cpu_pc_from_tb(env, tb);
     }
     tb_phys_invalidate(tb, -1);
@@ -177 +185 @@
     TranslationBlock *tb;
     target_ulong cs_base, pc;
-    uint64_t flags;
+    int flags;
 
     /* we record a subset of the CPU state. It will
        always be the same before a given translated block
        is executed. */
-#if defined(TARGET_I386)
-    flags = env->hflags;
-    flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
-    cs_base = env->segs[R_CS].base;
-    pc = cs_base + env->eip;
-#elif defined(TARGET_ARM)
-    flags = env->thumb | (env->vfp.vec_len << 1)
-            | (env->vfp.vec_stride << 4);
-    if ((env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR)
-        flags |= (1 << 6);
-    if (env->vfp.xregs[ARM_VFP_FPEXC] & (1 << 30))
-        flags |= (1 << 7);
-    flags |= (env->condexec_bits << 8);
-    cs_base = 0;
-    pc = env->regs[15];
-#elif defined(TARGET_SPARC)
-#ifdef TARGET_SPARC64
-    // AM . Combined FPU enable bits . PRIV . DMMU enabled . IMMU enabled
-    flags = ((env->pstate & PS_AM) << 2)
-        | (((env->pstate & PS_PEF) >> 1) | ((env->fprs & FPRS_FEF) << 2))
-        | (env->pstate & PS_PRIV) | ((env->lsu & (DMMU_E | IMMU_E)) >> 2);
-#else
-    // FPU enable . Supervisor
-    flags = (env->psref << 4) | env->psrs;
-#endif
-    cs_base = env->npc;
-    pc = env->pc;
-#elif defined(TARGET_PPC)
-    flags = env->hflags;
-    cs_base = 0;
-    pc = env->nip;
-#elif defined(TARGET_MIPS)
-    flags = env->hflags & (MIPS_HFLAG_TMASK | MIPS_HFLAG_BMASK);
-    cs_base = 0;
-    pc = env->active_tc.PC;
-#elif defined(TARGET_M68K)
-    flags = (env->fpcr & M68K_FPCR_PREC)  /* Bit  6 */
-            | (env->sr & SR_S)            /* Bit  13 */
-            | ((env->macsr >> 4) & 0xf);  /* Bits 0-3 */
-    cs_base = 0;
-    pc = env->pc;
-#elif defined(TARGET_SH4)
-    flags = (env->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL
-                    | DELAY_SLOT_TRUE | DELAY_SLOT_CLEARME))   /* Bits  0- 3 */
-            | (env->fpscr & (FPSCR_FR | FPSCR_SZ | FPSCR_PR))  /* Bits 19-21 */
-            | (env->sr & (SR_MD | SR_RB));                     /* Bits 29-30 */
-    cs_base = 0;
-    pc = env->pc;
-#elif defined(TARGET_ALPHA)
-    flags = env->ps;
-    cs_base = 0;
-    pc = env->pc;
-#elif defined(TARGET_CRIS)
-    flags = env->pregs[PR_CCS] & (S_FLAG | P_FLAG | U_FLAG | X_FLAG);
-    flags |= env->dslot;
-    cs_base = 0;
-    pc = env->pc;
-#else
-#error unsupported CPU
-#endif
+    cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
     tb = env->tb_jmp_cache[tb_jmp_cache_hash_func(pc)];
     if (unlikely(!tb || tb->pc != pc || tb->cs_base != cs_base ||
@@ -248 +197 @@
     }
     return tb;
+}
+
+static CPUDebugExcpHandler *debug_excp_handler;
+
+CPUDebugExcpHandler *cpu_set_debug_excp_handler(CPUDebugExcpHandler *handler)
+{
+    CPUDebugExcpHandler *old_handler = debug_excp_handler;
+
+    debug_excp_handler = handler;
+    return old_handler;
+}
+
+static void cpu_handle_debug_exception(CPUState *env)
+{
+    CPUWatchpoint *wp;
+
+    if (!env->watchpoint_hit)
+        TAILQ_FOREACH(wp, &env->watchpoints, entry)
+            wp->flags &= ~BP_WATCHPOINT_HIT;
+
+    if (debug_excp_handler)
+        debug_excp_handler(env);
 }
 
@@ -314 +285 @@
             /* if an exception is pending, we execute it here */
             if (env->exception_index >= 0) {
-                Assert(!env->user_mode_only);
                 if (env->exception_index >= EXCP_INTERRUPT) {
                     /* exit request from the cpu execution loop */
                     ret = env->exception_index;
+                    if (ret == EXCP_DEBUG)
+                        cpu_handle_debug_exception(env);
                     break;
                 } else {
@@ -340 +312 @@
             {
                 interrupt_request = env->interrupt_request;
-#ifndef VBOX
-                if (__builtin_expect(interrupt_request, 0))
-#else
-                if (RT_UNLIKELY(interrupt_request != 0))
-#endif
-                {
+                if (unlikely(interrupt_request)) {
+                    if (unlikely(env->singlestep_enabled & SSTEP_NOIRQ)) {
+                        /* Mask out external interrupts for this step. */
+                        interrupt_request &= ~(CPU_INTERRUPT_HARD |
+                                               CPU_INTERRUPT_FIQ |
+                                               CPU_INTERRUPT_SMI |
+                                               CPU_INTERRUPT_NMI);
+                    }
+                    if (interrupt_request & CPU_INTERRUPT_DEBUG) {
+                        env->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
+                        env->exception_index = EXCP_DEBUG;
+                        cpu_loop_exit();
+                    }
                     /** @todo: reconcile with what QEMU really does */
 
@@ -467 +446 @@
 
                 env->current_tb = tb;
+
+                /* cpu_interrupt might be called while translating the
+                   TB, but before it is linked into a potentially
+                   infinite loop and becomes env->current_tb. Avoid
+                   starting execution if there is a pending interrupt. */
+                if (unlikely (env->interrupt_request & CPU_INTERRUPT_EXIT))
+                    env->current_tb = NULL;
+
                 while (env->current_tb) {
                     tc_ptr = tb->tc_ptr;
@@ -483 +470 @@
                         tb = (TranslationBlock *)(long)(next_tb & ~3);
                         /* Restore PC.  */
-                        CPU_PC_FROM_TB(env, tb);
+                        cpu_pc_from_tb(env, tb);
                         insns_left = env->icount_decr.u32;
                         if (env->icount_extra && insns_left >= 0) {
@@ -540 +527 @@
 #if defined(TARGET_I386)
     /* restore flags in standard format */
-    env->eflags = env->eflags | cc_table[CC_OP].compute_all() | (DF & DF_MASK);
+    env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
 #else
 #error unsupported target CPU
@@ -601 +588 @@
                     /* exit request from the cpu execution loop */
                     ret = env->exception_index;
+                    if (ret == EXCP_DEBUG)
+                        cpu_handle_debug_exception(env);
                     break;
-                } else if (env->user_mode_only) {
+                } else {
+#if defined(CONFIG_USER_ONLY)
                     /* if user mode only, we simulate a fake exception
                        which will be handled outside the cpu execution
@@ -616 +606 @@
                     ret = env->exception_index;
                     break;
-                } else {
+#else
 #if defined(TARGET_I386)
                     /* simulate a real cpu exception. On i386, it can
@@ -644 +634 @@
                     do_interrupt(0);
 #endif
+#endif
                 }
                 env->exception_index = -1;
@@ -650 +641 @@
             if (kqemu_is_ok(env) && env->interrupt_request == 0) {
                 int ret;
-                env->eflags = env->eflags | cc_table[CC_OP].compute_all() | (DF & DF_MASK);
+                env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
                 ret = kqemu_cpu_exec(env);
                 /* put eflags in CPU temporary format */
@@ -673 +664 @@
 #endif
 
+            if (kvm_enabled()) {
+                kvm_cpu_exec(env);
+                longjmp(env->jmp_env, 1);
+            }
+
             next_tb = 0; /* force lookup of first TB */
             for(;;) {
                 interrupt_request = env->interrupt_request;
-                if (unlikely(interrupt_request) &&
-                    likely(!(env->singlestep_enabled & SSTEP_NOIRQ))) {
+                if (unlikely(interrupt_request)) {
+                    if (unlikely(env->singlestep_enabled & SSTEP_NOIRQ)) {
+                        /* Mask out external interrupts for this step. */
+                        interrupt_request &= ~(CPU_INTERRUPT_HARD |
+                                               CPU_INTERRUPT_FIQ |
+                                               CPU_INTERRUPT_SMI |
+                                               CPU_INTERRUPT_NMI);
+                    }
                     if (interrupt_request & CPU_INTERRUPT_DEBUG) {
                         env->interrupt_request &= ~CPU_INTERRUPT_DEBUG;
@@ -716 +718 @@
                             env->interrupt_request &= ~(CPU_INTERRUPT_HARD | CPU_INTERRUPT_VIRQ);
                             intno = cpu_get_pic_interrupt(env);
-                            if (loglevel & CPU_LOG_TB_IN_ASM) {
-                                fprintf(logfile, "Servicing hardware INT=0x%02x\n", intno);
-                            }
+                            qemu_log_mask(CPU_LOG_TB_IN_ASM, "Servicing hardware INT=0x%02x\n", intno);
                             do_interrupt(intno, 0, 0, 0, 1);
                             /* ensure that no TB jump will be modified as
@@ -730 +730 @@
                             /* FIXME: this should respect TPR */
                             svm_check_intercept(SVM_EXIT_VINTR);
+                            intno = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_vector));
+                            qemu_log_mask(CPU_LOG_TB_IN_ASM, "Servicing virtual hardware INT=0x%02x\n", intno);
+                            do_interrupt(intno, 0, 0, 0, 1);
                             env->interrupt_request &= ~CPU_INTERRUPT_VIRQ;
-                            intno = ldl_phys(env->vm_vmcb + offsetof(struct vmcb, control.int_vector));
-                            if (loglevel & CPU_LOG_TB_IN_ASM)
-                                fprintf(logfile, "Servicing virtual hardware INT=0x%02x\n", intno);
-                            do_interrupt(intno, 0, 0, 0, 1);
                             next_tb = 0;
 #endif
@@ -861 +860 @@
                 }
 #ifdef DEBUG_EXEC
-                if ((loglevel & CPU_LOG_TB_CPU)) {
+                if (qemu_loglevel_mask(CPU_LOG_TB_CPU)) {
                     /* restore flags in standard format */
                     regs_to_env();
 #if defined(TARGET_I386)
-                    env->eflags = env->eflags | cc_table[CC_OP].compute_all() | (DF & DF_MASK);
-                    cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
+                    env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
+                    log_cpu_state(env, X86_DUMP_CCOP);
                     env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
 #elif defined(TARGET_ARM)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_SPARC)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_PPC)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_M68K)
                     cpu_m68k_flush_flags(env, env->cc_op);
@@ -879 +878 @@
                     env->sr = (env->sr & 0xffe0)
                               | env->cc_dest | (env->cc_x << 4);
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_MIPS)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_SH4)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_ALPHA)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #elif defined(TARGET_CRIS)
-                    cpu_dump_state(env, logfile, fprintf, 0);
+                    log_cpu_state(env, 0);
 #else
 #error unsupported target CPU
@@ -905 +904 @@
                 }
 #ifdef DEBUG_EXEC
-                if ((loglevel & CPU_LOG_EXEC)) {
-                    fprintf(logfile, "Trace 0x%08lx [" TARGET_FMT_lx "] %s\n",
-                            (long)tb->tc_ptr, tb->pc,
-                            lookup_symbol(tb->pc));
-                }
+                qemu_log_mask(CPU_LOG_EXEC, "Trace 0x%08lx [" TARGET_FMT_lx "] %s\n",
+                             (long)tb->tc_ptr, tb->pc,
+                             lookup_symbol(tb->pc));
 #endif
                 /* see if we can patch the calling TB. When the TB
@@ -925 +922 @@
                 spin_unlock(&tb_lock);
                 env->current_tb = tb;
+
+                /* cpu_interrupt might be called while translating the
+                   TB, but before it is linked into a potentially
+                   infinite loop and becomes env->current_tb. Avoid
+                   starting execution if there is a pending interrupt. */
+                if (unlikely (env->interrupt_request & CPU_INTERRUPT_EXIT))
+                    env->current_tb = NULL;
+
                 while (env->current_tb) {
                     tc_ptr = tb->tc_ptr;
@@ -940 +945 @@
                         tb = (TranslationBlock *)(long)(next_tb & ~3);
                         /* Restore PC.  */
-                        CPU_PC_FROM_TB(env, tb);
+                        cpu_pc_from_tb(env, tb);
                         insns_left = env->icount_decr.u32;
                         if (env->icount_extra && insns_left >= 0) {
@@ -981 +986 @@
 #if defined(TARGET_I386)
     /* restore flags in standard format */
-    env->eflags = env->eflags | cc_table[CC_OP].compute_all() | (DF & DF_MASK);
+    env->eflags = env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
 #elif defined(TARGET_ARM)
     /* XXX: Save/restore host fpu exception state?.  */
@@ -1241 +1246 @@
        do it (XXX: use sigsetjmp) */
         sigprocmask(SIG_SETMASK, old_set, NULL);
-        do_raise_exception_err(env->exception_index, env->error_code);
+        cpu_loop_exit();
     } else {
         /* activate soft MMU for this block */
@@ -1330 +1335 @@
        do it (XXX: use sigsetjmp) */
         sigprocmask(SIG_SETMASK, old_set, NULL);
-        do_raise_exception_err(env->exception_index, env->error_code);
+        cpu_loop_exit();
     } else {
         /* activate soft MMU for this block */
@@ -1511 +1516 @@
 #elif defined(__x86_64__)
 
+#ifdef __NetBSD__
+#define REG_ERR _REG_ERR
+#define REG_TRAPNO _REG_TRAPNO
+
+#define QEMU_UC_MCONTEXT_GREGS(uc, reg) (uc)->uc_mcontext.__gregs[(reg)]
+#define QEMU_UC_MACHINE_PC(uc)          _UC_MACHINE_PC(uc)
+#else
+#define QEMU_UC_MCONTEXT_GREGS(uc, reg) (uc)->uc_mcontext.gregs[(reg)]
+#define QEMU_UC_MACHINE_PC(uc)          QEMU_UC_MCONTEXT_GREGS(uc, REG_RIP)
+#endif
+
 int cpu_signal_handler(int host_signum, void *pinfo,
                        void *puc)
 {
     siginfo_t *info = pinfo;
+    unsigned long pc;
+#ifdef __NetBSD__
+    ucontext_t *uc = puc;
+#else
     struct ucontext *uc = puc;
-    unsigned long pc;
-
-    pc = uc->uc_mcontext.gregs[REG_RIP];
+#endif
+
+    pc = QEMU_UC_MACHINE_PC(uc);
     return handle_cpu_signal(pc, (unsigned long)info->si_addr,
-                             uc->uc_mcontext.gregs[REG_TRAPNO] == 0xe ?
-                             (uc->uc_mcontext.gregs[REG_ERR] >> 1) & 1 : 0,
+                             QEMU_UC_MCONTEXT_GREGS(uc, REG_TRAPNO) == 0xe ?
+                             (QEMU_UC_MCONTEXT_GREGS(uc, REG_ERR) >> 1) & 1 : 0,
                              &uc->uc_sigmask, puc);
 }
 
-#elif defined(__powerpc__)
+#elif defined(_ARCH_PPC)
 
 /***********************************************************************
@@ -1642 +1662 @@
     unsigned long pc = regs[1];
 #else
+#ifdef __linux__
     struct sigcontext *sc = puc;
     unsigned long pc = sc->sigc_regs.tpc;
     void *sigmask = (void *)sc->sigc_mask;
+#elif defined(__OpenBSD__)
+    struct sigcontext *uc = puc;
+    unsigned long pc = uc->sc_pc;
+    void *sigmask = (void *)(long)uc->sc_mask;
+#endif
 #endif
 

trunk/src/recompiler/cutils.c

@@ -23 +23 @@
  */
 #include "qemu-common.h"
+#include "host-utils.h"
 
 #ifdef VBOX
 #include "osdep.h"
+
 
 static inline int toupper(int ch) {
@@ -565 +567 @@
     q = val;
     while (*q != '\0') {
-        if (toupper(*p) != toupper(*q))
+        if (qemu_toupper(*p) != qemu_toupper(*q))
             return 0;
         p++;
@@ -589 +591 @@
     return t;
 }
-#endif /* VBOX */
+#endif /* !VBOX */
+
+int qemu_fls(int i)
+{
+    return 32 - clz32(i);
+}
+
+#ifndef VBOX
+/* io vectors */
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
+{
+    qiov->iov = qemu_malloc(alloc_hint * sizeof(struct iovec));
+    qiov->niov = 0;
+    qiov->nalloc = alloc_hint;
+    qiov->size = 0;
+}
+
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
+{
+    if (qiov->niov == qiov->nalloc) {
+        qiov->nalloc = 2 * qiov->nalloc + 1;
+        qiov->iov = qemu_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
+    }
+    qiov->iov[qiov->niov].iov_base = base;
+    qiov->iov[qiov->niov].iov_len = len;
+    qiov->size += len;
+    ++qiov->niov;
+}
+
+void qemu_iovec_destroy(QEMUIOVector *qiov)
+{
+    qemu_free(qiov->iov);
+}
+
+void qemu_iovec_reset(QEMUIOVector *qiov)
+{
+    qiov->niov = 0;
+    qiov->size = 0;
+}
+
+void qemu_iovec_to_buffer(QEMUIOVector *qiov, void *buf)
+{
+    uint8_t *p = (uint8_t *)buf;
+    int i;
+
+    for (i = 0; i < qiov->niov; ++i) {
+        memcpy(p, qiov->iov[i].iov_base, qiov->iov[i].iov_len);
+        p += qiov->iov[i].iov_len;
+    }
+}
+
+void qemu_iovec_from_buffer(QEMUIOVector *qiov, const void *buf, size_t count)
+{
+    const uint8_t *p = (const uint8_t *)buf;
+    size_t copy;
+    int i;
+
+    for (i = 0; i < qiov->niov && count; ++i) {
+        copy = count;
+        if (copy > qiov->iov[i].iov_len)
+            copy = qiov->iov[i].iov_len;
+        memcpy(qiov->iov[i].iov_base, p, copy);
+        p     += copy;
+        count -= copy;
+    }
+}
+#endif /* !VBOX */

trunk/src/recompiler/disas.h

@@ -11 +11 @@
 const char *lookup_symbol(target_ulong orig_addr);
 
-/* Filled in by elfload.c.  Simplistic, but will do for now. */
-extern struct syminfo {
+struct syminfo;
+struct elf32_sym;
+struct elf64_sym;
+
+typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_ulong orig_addr);
+
+struct syminfo {
+    lookup_symbol_t lookup_symbol;
     unsigned int disas_num_syms;
-    void *disas_symtab;
+    union {
+      struct elf32_sym *elf32;
+      struct elf64_sym *elf64;
+    } disas_symtab;
     const char *disas_strtab;
     struct syminfo *next;
-} *syminfos;
+};
+
+/* Filled in by elfload.c.  Simplistic, but will do for now. */
+extern struct syminfo *syminfos;
 
 #endif /* _QEMU_DISAS_H */

trunk/src/recompiler/dyngen-exec.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -56 +56 @@
 #if !(defined (__sparc_v9__) && defined(__linux__)) && !(defined(__APPLE__) && defined(__x86_64__))
 /* XXX may be done for all 64 bits targets ? */
-#if defined (__x86_64__) || defined(__ia64) || defined(__s390x__) || defined(__alpha__) || defined(__powerpc64__)
+#if defined (__x86_64__) || defined(__ia64) || defined(__s390x__) || defined(__alpha__) || defined(_ARCH_PPC64)
 typedef unsigned long uint64_t;
 #else
@@ -73 +73 @@
 // Linux/Sparc64 defines int64_t
 #if !(defined (__sparc_v9__) && defined(__linux__)) && !(defined(__APPLE__) && defined(__x86_64__))
-#if defined (__x86_64__) || defined(__ia64) || defined(__s390x__) || defined(__alpha__) || defined(__powerpc64__)
+#if defined (__x86_64__) || defined(__ia64) || defined(__s390x__) || defined(__alpha__) || defined(_ARCH_PPC64)
 typedef signed long int64_t;
 #else
@@ -135 +135 @@
 //#define AREG4 "rbp"
 //#define AREG5 "rbx"
-#elif defined(__powerpc__)
+#elif defined(_ARCH_PPC)
 #define AREG0 "r27"
 #define AREG1 "r24"
@@ -228 +228 @@
 #endif
 
-#ifndef VBOX /* WHY DO WE UNSUBSCRIBE TO THIS MACRO? */
-/* force GCC to generate only one epilog at the end of the function */
-#define FORCE_RET() __asm__ __volatile__("" : : : "memory");
-#endif
-
-#ifndef OPPROTO
-#define OPPROTO
-#endif
-
 #define xglue(x, y) x ## y
 #define glue(x, y) xglue(x, y)
 #define stringify(s)    tostring(s)
 #define tostring(s)     #s
-
-#if defined(__alpha__) || defined(__s390__)
-/* the symbols are considered non exported so a br immediate is generated */
-#define __hidden __attribute__((visibility("hidden")))
-#else
-#define __hidden
-#endif
-
-#if defined(__alpha__)
-/* Suggested by Richard Henderson. This will result in code like
-        ldah $0,__op_param1($29)        !gprelhigh
-        lda $0,__op_param1($0)          !gprellow
-   We can then conveniently change $29 to $31 and adapt the offsets to
-   emit the appropriate constant.  */
-extern int __op_param1 __hidden;
-extern int __op_param2 __hidden;
-extern int __op_param3 __hidden;
-#define PARAM1 ({ int _r; asm("" : "=r"(_r) : "0" (&__op_param1)); _r; })
-#define PARAM2 ({ int _r; asm("" : "=r"(_r) : "0" (&__op_param2)); _r; })
-#define PARAM3 ({ int _r; asm("" : "=r"(_r) : "0" (&__op_param3)); _r; })
-#elif defined(__s390__)
-extern int __op_param1 __hidden;
-extern int __op_param2 __hidden;
-extern int __op_param3 __hidden;
-#define PARAM1 ({ int _r; asm("bras %0,8; .long " ASM_NAME(__op_param1) "; l %0,0(%0)" : "=r"(_r) : ); _r; })
-#define PARAM2 ({ int _r; asm("bras %0,8; .long " ASM_NAME(__op_param2) "; l %0,0(%0)" : "=r"(_r) : ); _r; })
-#define PARAM3 ({ int _r; asm("bras %0,8; .long " ASM_NAME(__op_param3) "; l %0,0(%0)" : "=r"(_r) : ); _r; })
-#else
-#if defined(__APPLE__)
-static int __op_param1, __op_param2, __op_param3;
-#else
-extern int __op_param1, __op_param2, __op_param3;
-#endif
-#define PARAM1 ((long)(&__op_param1))
-#define PARAM2 ((long)(&__op_param2))
-#define PARAM3 ((long)(&__op_param3))
-#endif /* !defined(__alpha__) */
-
-extern int __op_jmp0, __op_jmp1, __op_jmp2, __op_jmp3;
-
-#if defined(_WIN32) || defined(__APPLE__) || defined(__OS2__)
-#define ASM_NAME(x) "_" #x
-#else
-#define ASM_NAME(x) #x
-#endif
-
-#if defined(__i386__)
-#define EXIT_TB() asm volatile ("ret")
-#define GOTO_LABEL_PARAM(n) asm volatile ("jmp " ASM_NAME(__op_gen_label) #n)
-#elif defined(__x86_64__)
-#define EXIT_TB() asm volatile ("ret")
-#define GOTO_LABEL_PARAM(n) asm volatile ("jmp " ASM_NAME(__op_gen_label) #n)
-#elif defined(__powerpc__)
-#define EXIT_TB() asm volatile ("blr")
-#define GOTO_LABEL_PARAM(n) asm volatile ("b " ASM_NAME(__op_gen_label) #n)
-#elif defined(__s390__)
-#define EXIT_TB() asm volatile ("br %r14")
-#define GOTO_LABEL_PARAM(n) asm volatile ("larl %r7,12; l %r7,0(%r7); br %r7; .long " ASM_NAME(__op_gen_label) #n)
-#elif defined(__alpha__)
-#define EXIT_TB() asm volatile ("ret")
-#elif defined(__ia64__)
-#define EXIT_TB() asm volatile ("br.ret.sptk.many b0;;")
-#define GOTO_LABEL_PARAM(n) asm volatile ("br.sptk.many " \
-                                          ASM_NAME(__op_gen_label) #n)
-#elif defined(__sparc__)
-#define EXIT_TB() asm volatile ("jmpl %i0 + 8, %g0; nop")
-#define GOTO_LABEL_PARAM(n) asm volatile ("ba " ASM_NAME(__op_gen_label) #n ";nop")
-#elif defined(__arm__)
-#define EXIT_TB() asm volatile ("b exec_loop")
-#define GOTO_LABEL_PARAM(n) asm volatile ("b " ASM_NAME(__op_gen_label) #n)
-#elif defined(__mc68000)
-#define EXIT_TB() asm volatile ("rts")
-#elif defined(__mips__)
-#define EXIT_TB() asm volatile ("jr $ra")
-#define GOTO_LABEL_PARAM(n) asm volatile (".set noat; la $1, " ASM_NAME(__op_gen_label) #n "; jr $1; .set at")
-#elif defined(__hppa__)
-#define GOTO_LABEL_PARAM(n) asm volatile ("b,n " ASM_NAME(__op_gen_label) #n)
-#else
-#error unsupported CPU
-#endif
 
 /* The return address may point to the start of the next instruction.

trunk/src/recompiler/exec-all.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -27 +27 @@
  * of the LGPL is applied is otherwise unspecified.
  */
+
+#ifndef _EXEC_ALL_H_
+#define _EXEC_ALL_H_
+
+#include "qemu-common.h"
 
 /* allow to see translation results - the slowdown should be negligible, so we leave it */
@@ -74 +79 @@
 extern target_ulong gen_opc_jump_pc[2];
 extern uint32_t gen_opc_hflags[OPC_BUF_SIZE];
-
-typedef void (GenOpFunc)(void);
-typedef void (GenOpFunc1)(long);
-typedef void (GenOpFunc2)(long, long);
-typedef void (GenOpFunc3)(long, long, long);
 
 #include "qemu-log.h"
@@ -103 +103 @@
                               int cflags);
 void cpu_exec_init(CPUState *env);
+void QEMU_NORETURN cpu_loop_exit(void);
 int page_unprotect(target_ulong address, unsigned long pc, void *puc);
 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
@@ -137 +138 @@
 #endif
 
-#if defined(__powerpc__) || defined(__x86_64__) || defined(__arm__)
+#if defined(_ARCH_PPC) || defined(__x86_64__) || defined(__arm__)
 #define USE_DIRECT_JUMP
 #endif
@@ -220 +221 @@
 #if defined(USE_DIRECT_JUMP)
 
-#if defined(__powerpc__)
+#if defined(_ARCH_PPC)
 extern void ppc_tb_set_jmp_target(unsigned long jmp_addr, unsigned long addr);
 #define tb_set_jmp_target1 ppc_tb_set_jmp_target
@@ -233 +234 @@
 static inline void tb_set_jmp_target1(unsigned long jmp_addr, unsigned long addr)
 {
+#if QEMU_GNUC_PREREQ(4, 1)
+    void __clear_cache(char *beg, char *end);
+#else
     register unsigned long _beg __asm ("a1");
     register unsigned long _end __asm ("a2");
     register unsigned long _flg __asm ("a3");
+#endif
 
     /* we could use a ldr pc, [pc, #-4] kind of branch and avoid the flush */
     *(uint32_t *)jmp_addr |= ((addr - (jmp_addr + 8)) >> 2) & 0xffffff;
 
+#if QEMU_GNUC_PREREQ(4, 1)
+    __clear_cache((char *) jmp_addr, (char *) jmp_addr + 4);
+#else
     /* flush icache */
     _beg = jmp_addr;
@@ -245 +253 @@
     _flg = 0;
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
+#endif
 }
 #endif
@@ -286 +295 @@
 
 TranslationBlock *tb_find_pc(unsigned long pc_ptr);
-
-#if defined(_WIN32)
-#define ASM_DATA_SECTION ".section \".data\"\n"
-#define ASM_PREVIOUS_SECTION ".section .text\n"
-#elif defined(__APPLE__)
-#define ASM_DATA_SECTION ".data\n"
-#define ASM_PREVIOUS_SECTION ".text\n"
-#else
-#define ASM_DATA_SECTION ".section \".data\"\n"
-#define ASM_PREVIOUS_SECTION ".previous\n"
-#endif
-
-#define ASM_OP_LABEL_NAME(n, opname) \
-    ASM_NAME(__op_label) #n "." ASM_NAME(opname)
 
 extern CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
@@ -432 +427 @@
 
 #endif
+
+typedef void (CPUDebugExcpHandler)(CPUState *env);
+
+CPUDebugExcpHandler *cpu_set_debug_excp_handler(CPUDebugExcpHandler *handler);
+#endif

trunk/src/recompiler/exec.c

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -61 +61 @@
 #endif
 #include "osdep.h"
+#include "kvm.h"
 #if defined(CONFIG_USER_ONLY)
 #include <qemu.h>
@@ -149 +150 @@
    causes some pages to fall outside the dirty map here. */
 RTGCPHYS phys_ram_dirty_size;
+uint8_t *phys_ram_dirty;
 #endif /* VBOX */
-#if !defined(VBOX)
-uint8_t *phys_ram_base;
-#endif
-uint8_t *phys_ram_dirty;
 
 CPUState *first_cpu;
@@ -182 +180 @@
     /* offset in host memory of the page + io_index in the low bits */
     ram_addr_t phys_offset;
+    ram_addr_t region_offset;
 } PhysPageDesc;
 
@@ -226 +225 @@
 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
-static int io_mem_nb;
+char io_mem_used[IO_MEM_NB_ENTRIES];
 static int io_mem_watch;
 #endif
@@ -257 +256 @@
     CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
     void *opaque[TARGET_PAGE_SIZE][2][4];
+    ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
 } subpage_t;
 
@@ -305 +305 @@
     {
         SYSTEM_INFO system_info;
-        DWORD old_protect;
 
         GetSystemInfo(&system_info);
@@ -314 +313 @@
 #endif
 #endif /* !VBOX */
-
     if (qemu_host_page_size == 0)
         qemu_host_page_size = qemu_real_host_page_size;
@@ -406 +404 @@
         /* allocate if not found */
 #if defined(CONFIG_USER_ONLY)
-        unsigned long addr;
         size_t len = sizeof(PageDesc) * L2_SIZE;
         /* Don't use qemu_malloc because it may recurse.  */
@@ -412 +409 @@
                  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
         *lp = p;
-        addr = h2g(p);
-        if (addr == (target_ulong)addr) {
+        if (h2g_valid(p)) {
+            unsigned long addr = h2g(p);
             page_set_flags(addr & TARGET_PAGE_MASK,
                            TARGET_PAGE_ALIGN(addr + len),
@@ -486 +483 @@
         pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
         *lp = pd;
-        for (i = 0; i < L2_SIZE; i++)
+        for (i = 0; i < L2_SIZE; i++) {
           pd[i].phys_offset = IO_MEM_UNASSIGNED;
+          pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
+        }
     }
     return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
@@ -545 +544 @@
         code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
 #else
-        /* XXX: needs adjustments */
+        /* XXX: needs ajustments */
         code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
 #endif
-
     }
     if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
         code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
 #endif /* VBOX */
-
     /* The code gen buffer location may have constraints depending on
        the host cpu and OS */
@@ -564 +561 @@
         return;
     }
-#else //!VBOX
+#else  /* !VBOX */
 #if defined(__linux__)
     {
@@ -582 +579 @@
         if (code_gen_buffer_size > (512 * 1024 * 1024))
             code_gen_buffer_size = (512 * 1024 * 1024);
+#elif defined(__arm__)
+        /* Map the buffer below 32M, so we can use direct calls and branches */
+        flags |= MAP_FIXED;
+        start = (void *) 0x01000000UL;
+        if (code_gen_buffer_size > 16 * 1024 * 1024)
+            code_gen_buffer_size = 16 * 1024 * 1024;
 #endif
         code_gen_buffer = mmap(start, code_gen_buffer_size,
@@ -615 +618 @@
 #else
     code_gen_buffer = qemu_malloc(code_gen_buffer_size);
-    if (!code_gen_buffer) {
-        fprintf(stderr, "Could not allocate dynamic translator buffer\n");
-        exit(1);
-    }
     map_exec(code_gen_buffer, code_gen_buffer_size);
 #endif
-    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
 #endif /* !VBOX */
 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
@@ -629 +627 @@
     map_exec(code_gen_prologue, _1K);
 #endif
-
     code_gen_buffer_max_size = code_gen_buffer_size -
         code_gen_max_block_size();
@@ -677 +674 @@
 }
 #endif
-#endif //!VBOX
+#endif /* !VBOX */
 
 void cpu_exec_init(CPUState *env)
@@ -692 +689 @@
     }
     env->cpu_index = cpu_index;
-    env->nb_watchpoints = 0;
+    TAILQ_INIT(&env->breakpoints);
+    TAILQ_INIT(&env->watchpoints);
     *penv = env;
 #ifndef VBOX
@@ -701 +699 @@
                     cpu_save, cpu_load, env);
 #endif
-#endif // !VBOX
+#endif /* !VBOX */
 }
 
@@ -780 +778 @@
 
 #ifdef DEBUG_TB_CHECK
+
 static void tb_invalidate_check(target_ulong address)
 {
@@ -1040 +1039 @@
 
     p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
-    if (!p->code_bitmap)
-        return;
 
     tb = p->first_tb;
@@ -1109 +1106 @@
                                    int is_cpu_write_access)
 {
-    int n, current_tb_modified, current_tb_not_found, current_flags;
+    TranslationBlock *tb, *tb_next, *saved_tb;
     CPUState *env = cpu_single_env;
+    target_ulong tb_start, tb_end;
     PageDesc *p;
-    TranslationBlock *tb, *tb_next, *current_tb, *saved_tb;
-    target_ulong tb_start, tb_end;
-    target_ulong current_pc, current_cs_base;
+    int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+    int current_tb_not_found = is_cpu_write_access;
+    TranslationBlock *current_tb = NULL;
+    int current_tb_modified = 0;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
+#endif /* TARGET_HAS_PRECISE_SMC */
 
     p = page_find(start >> TARGET_PAGE_BITS);
@@ -1128 +1132 @@
     /* we remove all the TBs in the range [start, end[ */
     /* XXX: see if in some cases it could be faster to invalidate all the code */
-    current_tb_not_found = is_cpu_write_access;
-    current_tb_modified = 0;
-    current_tb = NULL; /* avoid warning */
-    current_pc = 0; /* avoid warning */
-    current_cs_base = 0; /* avoid warning */
-    current_flags = 0; /* avoid warning */
     tb = p->first_tb;
     while (tb != NULL) {
@@ -1170 +1168 @@
                 cpu_restore_state(current_tb, env,
                                   env->mem_io_pc, NULL);
-#if defined(TARGET_I386)
-                current_flags = env->hflags;
-                current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
-                current_cs_base = (target_ulong)env->segs[R_CS].base;
-                current_pc = current_cs_base + env->eip;
-#else
-#error unsupported CPU
-#endif
+                cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                                     &current_flags);
             }
 #endif /* TARGET_HAS_PRECISE_SMC */
@@ -1217 +1209 @@
 }
 
-
 /* len must be <= 8 and start must be a multiple of len */
 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
@@ -1225 +1216 @@
 #if 0
     if (1) {
-        if (loglevel) {
-            fprintf(logfile, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
-                   cpu_single_env->mem_io_vaddr, len,
-                   cpu_single_env->eip,
-                   cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
-        }
+        qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
+                  cpu_single_env->mem_io_vaddr, len,
+                  cpu_single_env->eip,
+                  cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
     }
 #endif
@@ -1247 +1236 @@
 }
 
-
 #if !defined(CONFIG_SOFTMMU)
 static void tb_invalidate_phys_page(target_phys_addr_t addr,
                                     unsigned long pc, void *puc)
 {
-    int n, current_flags, current_tb_modified;
-    target_ulong current_pc, current_cs_base;
+    TranslationBlock *tb;
     PageDesc *p;
-    TranslationBlock *tb, *current_tb;
+    int n;
 #ifdef TARGET_HAS_PRECISE_SMC
+    TranslationBlock *current_tb = NULL;
     CPUState *env = cpu_single_env;
+    int current_tb_modified = 0;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
 #endif
 
@@ -1265 +1257 @@
         return;
     tb = p->first_tb;
-    current_tb_modified = 0;
-    current_tb = NULL;
-    current_pc = 0; /* avoid warning */
-    current_cs_base = 0; /* avoid warning */
-    current_flags = 0; /* avoid warning */
 #ifdef TARGET_HAS_PRECISE_SMC
     if (tb && pc != 0) {
@@ -1289 +1276 @@
             current_tb_modified = 1;
             cpu_restore_state(current_tb, env, pc, puc);
-#if defined(TARGET_I386)
-            current_flags = env->hflags;
-            current_flags |= (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
-            current_cs_base = (target_ulong)env->segs[R_CS].base;
-            current_pc = current_cs_base + env->eip;
-#else
-#error unsupported CPU
-#endif
+            cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                                 &current_flags);
         }
 #endif /* TARGET_HAS_PRECISE_SMC */
@@ -1540 +1521 @@
 
 /* Add a watchpoint.  */
-int cpu_watchpoint_insert(CPUState *env, target_ulong addr, int type)
-{
-    int i;
-
-    for (i = 0; i < env->nb_watchpoints; i++) {
-        if (addr == env->watchpoint[i].vaddr)
+int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
+                          int flags, CPUWatchpoint **watchpoint)
+{
+    target_ulong len_mask = ~(len - 1);
+    CPUWatchpoint *wp;
+
+    /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
+    if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
+        fprintf(stderr, "qemu: tried to set invalid watchpoint at "
+                TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
+#ifndef VBOX
+        return -EINVAL;
+#else
+        return VERR_INVALID_PARAMETER;
+#endif
+    }
+    wp = qemu_malloc(sizeof(*wp));
+
+    wp->vaddr = addr;
+    wp->len_mask = len_mask;
+    wp->flags = flags;
+
+    /* keep all GDB-injected watchpoints in front */
+    if (flags & BP_GDB)
+        TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
+    else
+        TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
+
+    tlb_flush_page(env, addr);
+
+    if (watchpoint)
+        *watchpoint = wp;
+    return 0;
+}
+
+/* Remove a specific watchpoint.  */
+int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
+                          int flags)
+{
+    target_ulong len_mask = ~(len - 1);
+    CPUWatchpoint *wp;
+
+    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
+        if (addr == wp->vaddr && len_mask == wp->len_mask
+                && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
+            cpu_watchpoint_remove_by_ref(env, wp);
             return 0;
-    }
-    if (env->nb_watchpoints >= MAX_WATCHPOINTS)
-        return -1;
-
-    i = env->nb_watchpoints++;
-    env->watchpoint[i].vaddr = addr;
-    env->watchpoint[i].type = type;
-    tlb_flush_page(env, addr);
-    /* FIXME: This flush is needed because of the hack to make memory ops
-       terminate the TB.  It can be removed once the proper IO trap and
-       re-execute bits are in.  */
-    tb_flush(env);
-    return i;
-}
-
-/* Remove a watchpoint.  */
-int cpu_watchpoint_remove(CPUState *env, target_ulong addr)
-{
-    int i;
-
-    for (i = 0; i < env->nb_watchpoints; i++) {
-        if (addr == env->watchpoint[i].vaddr) {
-            env->nb_watchpoints--;
-            env->watchpoint[i] = env->watchpoint[env->nb_watchpoints];
-            tlb_flush_page(env, addr);
+        }
+    }
+#ifndef VBOX
+    return -ENOENT;
+#else
+    return VERR_NOT_FOUND;
+#endif
+}
+
+/* Remove a specific watchpoint by reference.  */
+void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
+{
+    TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
+
+    tlb_flush_page(env, watchpoint->vaddr);
+
+    qemu_free(watchpoint);
+}
+
+/* Remove all matching watchpoints.  */
+void cpu_watchpoint_remove_all(CPUState *env, int mask)
+{
+    CPUWatchpoint *wp, *next;
+
+    TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
+        if (wp->flags & mask)
+            cpu_watchpoint_remove_by_ref(env, wp);
+    }
+}
+
+/* Add a breakpoint.  */
+int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
+                          CPUBreakpoint **breakpoint)
+{
+#if defined(TARGET_HAS_ICE)
+    CPUBreakpoint *bp;
+
+    bp = qemu_malloc(sizeof(*bp));
+
+    bp->pc = pc;
+    bp->flags = flags;
+
+    /* keep all GDB-injected breakpoints in front */
+    if (flags & BP_GDB)
+        TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
+    else
+        TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
+
+    breakpoint_invalidate(env, pc);
+
+    if (breakpoint)
+        *breakpoint = bp;
+    return 0;
+#else
+    return -ENOSYS;
+#endif
+}
+
+/* Remove a specific breakpoint.  */
+int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
+{
+#if defined(TARGET_HAS_ICE)
+    CPUBreakpoint *bp;
+
+    TAILQ_FOREACH(bp, &env->breakpoints, entry) {
+        if (bp->pc == pc && bp->flags == flags) {
+            cpu_breakpoint_remove_by_ref(env, bp);
             return 0;
         }
     }
-    return -1;
-}
-
-/* Remove all watchpoints. */
-void cpu_watchpoint_remove_all(CPUState *env) {
-    int i;
-
-    for (i = 0; i < env->nb_watchpoints; i++) {
-        tlb_flush_page(env, env->watchpoint[i].vaddr);
-    }
-    env->nb_watchpoints = 0;
-}
-
-/* add a breakpoint. EXCP_DEBUG is returned by the CPU loop if a
-   breakpoint is reached */
-int cpu_breakpoint_insert(CPUState *env, target_ulong pc)
+# ifndef VBOX
+    return -ENOENT;
+# else
+    return VERR_NOT_FOUND;
+# endif
+#else
+    return -ENOSYS;
+#endif
+}
+
+/* Remove a specific breakpoint by reference.  */
+void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
 {
 #if defined(TARGET_HAS_ICE)
-    int i;
-
-    for(i = 0; i < env->nb_breakpoints; i++) {
-        if (env->breakpoints[i] == pc)
-            return 0;
-    }
-
-    if (env->nb_breakpoints >= MAX_BREAKPOINTS)
-        return -1;
-    env->breakpoints[env->nb_breakpoints++] = pc;
-
-    breakpoint_invalidate(env, pc);
-    return 0;
-#else
-    return -1;
-#endif
-}
-
-/* remove all breakpoints */
-void cpu_breakpoint_remove_all(CPUState *env) {
+    TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
+
+    breakpoint_invalidate(env, breakpoint->pc);
+
+    qemu_free(breakpoint);
+#endif
+}
+
+/* Remove all matching breakpoints. */
+void cpu_breakpoint_remove_all(CPUState *env, int mask)
+{
 #if defined(TARGET_HAS_ICE)
-    int i;
-    for(i = 0; i < env->nb_breakpoints; i++) {
-        breakpoint_invalidate(env, env->breakpoints[i]);
-    }
-    env->nb_breakpoints = 0;
-#endif
-}
-
-/* remove a breakpoint */
-int cpu_breakpoint_remove(CPUState *env, target_ulong pc)
-{
-#if defined(TARGET_HAS_ICE)
-    int i;
-    for(i = 0; i < env->nb_breakpoints; i++) {
-        if (env->breakpoints[i] == pc)
-            goto found;
-    }
-    return -1;
- found:
-    env->nb_breakpoints--;
-    if (i < env->nb_breakpoints)
-      env->breakpoints[i] = env->breakpoints[env->nb_breakpoints];
-
-    breakpoint_invalidate(env, pc);
-    return 0;
-#else
-    return -1;
+    CPUBreakpoint *bp, *next;
+
+    TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
+        if (bp->flags & mask)
+            cpu_breakpoint_remove_by_ref(env, bp);
+    }
 #endif
 }
@@ -1758 +1787 @@
 
 #ifndef VBOX
-CPULogItem cpu_log_items[] = {
+const CPULogItem cpu_log_items[] = {
     { CPU_LOG_TB_OUT_ASM, "out_asm",
       "show generated host assembly code for each compiled TB" },
@@ -1780 +1809 @@
     { CPU_LOG_PCALL, "pcall",
       "show protected mode far calls/returns/exceptions" },
+    { CPU_LOG_RESET, "cpu_reset",
+      "show CPU state before CPU resets" },
 #endif
 #ifdef DEBUG_IOPORT
@@ -1845 +1876 @@
     cpu_dump_state(env, stderr, fprintf, 0);
 #endif
-    if (logfile) {
-        fprintf(logfile, "qemu: fatal: ");
-        vfprintf(logfile, fmt, ap2);
-        fprintf(logfile, "\n");
+    if (qemu_log_enabled()) {
+        qemu_log("qemu: fatal: ");
+        qemu_log_vprintf(fmt, ap2);
+        qemu_log("\n");
 #ifdef TARGET_I386
-        cpu_dump_state(env, logfile, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
-#else
-        cpu_dump_state(env, logfile, fprintf, 0);
-#endif
-        fflush(logfile);
-        fclose(logfile);
+        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
+#else
+        log_cpu_state(env, 0);
+#endif
+        qemu_log_flush();
+        qemu_log_close();
     }
     va_end(ap2);
@@ -1867 +1898 @@
 {
     CPUState *new_env = cpu_init(env->cpu_model_str);
-    /* preserve chaining and index */
     CPUState *next_cpu = new_env->next_cpu;
     int cpu_index = new_env->cpu_index;
+#if defined(TARGET_HAS_ICE)
+    CPUBreakpoint *bp;
+    CPUWatchpoint *wp;
+#endif
+
     memcpy(new_env, env, sizeof(CPUState));
+
+    /* Preserve chaining and index. */
     new_env->next_cpu = next_cpu;
     new_env->cpu_index = cpu_index;
+
+    /* Clone all break/watchpoints.
+       Note: Once we support ptrace with hw-debug register access, make sure
+       BP_CPU break/watchpoints are handled correctly on clone. */
+    TAILQ_INIT(&env->breakpoints);
+    TAILQ_INIT(&env->watchpoints);
+#if defined(TARGET_HAS_ICE)
+    TAILQ_FOREACH(bp, &env->breakpoints, entry) {
+        cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
+    }
+    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
+        cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
+                              wp->flags, NULL);
+    }
+#endif
+
     return new_env;
 }
@@ -2118 +2171 @@
     return in_migration;
 }
-#endif
+#endif /* !VBOX */
+
+void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
+{
+    if (kvm_enabled())
+        kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
+}
 
 #if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
@@ -2154 +2213 @@
     for(i = 0; i < CPU_TLB_SIZE; i++)
         tlb_update_dirty(&env->tlb_table[1][i], env->phys_addends[1][i]);
-#if (NB_MMU_MODES >= 3)
+# if (NB_MMU_MODES >= 3)
     for(i = 0; i < CPU_TLB_SIZE; i++)
         tlb_update_dirty(&env->tlb_table[2][i], env->phys_addends[2][i]);
-#if (NB_MMU_MODES == 4)
+#  if (NB_MMU_MODES == 4)
     for(i = 0; i < CPU_TLB_SIZE; i++)
         tlb_update_dirty(&env->tlb_table[3][i], env->phys_addends[3][i]);
-#endif
-#endif
+#  endif
+# endif
 #else /* VBOX */
     for(i = 0; i < CPU_TLB_SIZE; i++)
@@ -2218 +2277 @@
     int ret;
     CPUTLBEntry *te;
-    int i;
+    CPUWatchpoint *wp;
     target_phys_addr_t iotlb;
 #if defined(VBOX) && !defined(REM_PHYS_ADDR_IN_TLB)
@@ -2267 +2326 @@
            We can't use the high bits of pd for this because
            IO_MEM_ROMD uses these as a ram address.  */
-        iotlb = (pd & ~TARGET_PAGE_MASK) + paddr;
+        iotlb = (pd & ~TARGET_PAGE_MASK);
+#ifndef VBOX
+        if (p) {
+#else
+        if (   p->phys_offset
+            && (pd & ~TARGET_PAGE_MASK) != env->pVM->rem.s.iMMIOMemType
+            && (pd & ~TARGET_PAGE_MASK) != env->pVM->rem.s.iHandlerMemType) {
+#endif
+            iotlb += p->region_offset;
+        } else {
+            iotlb += paddr;
+        }
     }
 
@@ -2299 +2369 @@
     /* Make accesses to pages with watchpoints go via the
        watchpoint trap routines.  */
-    for (i = 0; i < env->nb_watchpoints; i++) {
-        if (vaddr == (env->watchpoint[i].vaddr & TARGET_PAGE_MASK)) {
+    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
+        if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
             iotlb = io_mem_watch + paddr;
             /* TODO: The memory case can be optimized by not trapping
@@ -2467 +2537 @@
     target_ulong addr;
 
+    if (start + len < start)
+        /* we've wrapped around */
+        return -1;
+
     end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
     start = start & TARGET_PAGE_MASK;
 
-    if( end < start )
-        /* we've wrapped around */
-        return -1;
     for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
         p = page_find(addr >> TARGET_PAGE_BITS);
@@ -2553 +2624 @@
 
 #if !defined(CONFIG_USER_ONLY)
+
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
-                             ram_addr_t memory);
+                             ram_addr_t memory, ram_addr_t region_offset);
 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
-                           ram_addr_t orig_memory);
+                           ram_addr_t orig_memory, ram_addr_t region_offset);
 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
                       need_subpage)                                     \
@@ -2579 +2651 @@
 /* register physical memory. 'size' must be a multiple of the target
    page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
-   io memory page */
-void cpu_register_physical_memory(target_phys_addr_t start_addr,
-                                  ram_addr_t size,
-                                  ram_addr_t phys_offset)
+   io memory page.  The address used when calling the IO function is
+   the offset from the start of the region, plus region_offset.  Both
+   start_region and regon_offset are rounded down to a page boundary
+   before calculating this offset.  This should not be a problem unless
+   the low bits of start_addr and region_offset differ.  */
+void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
+                                         ram_addr_t size,
+                                         ram_addr_t phys_offset,
+                                         ram_addr_t region_offset)
 {
     target_phys_addr_t addr, end_addr;
@@ -2597 +2674 @@
     }
 #endif
+    if (kvm_enabled())
+        kvm_set_phys_mem(start_addr, size, phys_offset);
+
+    if (phys_offset == IO_MEM_UNASSIGNED) {
+        region_offset = start_addr;
+    }
+    region_offset &= TARGET_PAGE_MASK;
     size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
     end_addr = start_addr + (target_phys_addr_t)size;
@@ -2611 +2695 @@
                 if (!(orig_memory & IO_MEM_SUBPAGE)) {
                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
-                                           &p->phys_offset, orig_memory);
+                                           &p->phys_offset, orig_memory,
+                                           p->region_offset);
                 } else {
                     subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
                                             >> IO_MEM_SHIFT];
                 }
-                subpage_register(subpage, start_addr2, end_addr2, phys_offset);
+                subpage_register(subpage, start_addr2, end_addr2, phys_offset,
+                                 region_offset);
+                p->region_offset = 0;
             } else {
                 p->phys_offset = phys_offset;
@@ -2626 +2713 @@
             p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
             p->phys_offset = phys_offset;
+            p->region_offset = region_offset;
             if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
-                (phys_offset & IO_MEM_ROMD))
+                (phys_offset & IO_MEM_ROMD)) {
                 phys_offset += TARGET_PAGE_SIZE;
-            else {
+            } else {
                 target_phys_addr_t start_addr2, end_addr2;
                 int need_subpage = 0;
@@ -2638 +2726 @@
                 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
                     subpage = subpage_init((addr & TARGET_PAGE_MASK),
-                                           &p->phys_offset, IO_MEM_UNASSIGNED);
+                                           &p->phys_offset, IO_MEM_UNASSIGNED,
+                                           addr & TARGET_PAGE_MASK);
                     subpage_register(subpage, start_addr2, end_addr2,
-                                     phys_offset);
+                                     phys_offset, region_offset);
+                    p->region_offset = 0;
                 }
             }
         }
+        region_offset += TARGET_PAGE_SIZE;
     }
 
@@ -2666 +2757 @@
 
 #ifndef VBOX
+void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
+{
+    if (kvm_enabled())
+        kvm_coalesce_mmio_region(addr, size);
+}
+
+void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
+{
+    if (kvm_enabled())
+        kvm_uncoalesce_mmio_region(addr, size);
+}
+
 /* XXX: better than nothing */
 ram_addr_t qemu_ram_alloc(ram_addr_t size)
@@ -2690 +2793 @@
     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
 #endif
-#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
+#if defined(TARGET_SPARC)
     do_unassigned_access(addr, 0, 0, 0, 1);
 #endif
@@ -2701 +2804 @@
     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
 #endif
-#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
+#if defined(TARGET_SPARC)
     do_unassigned_access(addr, 0, 0, 0, 2);
 #endif
@@ -2712 +2815 @@
     printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
 #endif
-#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
+#if defined(TARGET_SPARC)
     do_unassigned_access(addr, 0, 0, 0, 4);
 #endif
@@ -2723 +2826 @@
     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
 #endif
-#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
+#if defined(TARGET_SPARC)
     do_unassigned_access(addr, 1, 0, 0, 1);
 #endif
@@ -2733 +2836 @@
     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
 #endif
-#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
+#if defined(TARGET_SPARC)
     do_unassigned_access(addr, 1, 0, 0, 2);
 #endif
@@ -2743 +2846 @@
     printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
 #endif
-#if defined(TARGET_SPARC) || defined(TARGET_CRIS)
+#if defined(TARGET_SPARC)
     do_unassigned_access(addr, 1, 0, 0, 4);
 #endif
@@ -2828 +2931 @@
     stw_p(phys_ram_base + ram_addr, val);
 #endif
-
 #ifdef USE_KQEMU
     if (cpu_single_env->kqemu_enabled &&
@@ -2900 +3002 @@
 
 /* Generate a debug exception if a watchpoint has been hit.  */
-static void check_watchpoint(int offset, int flags)
+static void check_watchpoint(int offset, int len_mask, int flags)
 {
     CPUState *env = cpu_single_env;
+    target_ulong pc, cs_base;
+    TranslationBlock *tb;
     target_ulong vaddr;
-    int i;
-
+    CPUWatchpoint *wp;
+    int cpu_flags;
+
+    if (env->watchpoint_hit) {
+        /* We re-entered the check after replacing the TB. Now raise
+         * the debug interrupt so that is will trigger after the
+         * current instruction. */
+        cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
+        return;
+    }
     vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
-    for (i = 0; i < env->nb_watchpoints; i++) {
-        if (vaddr == env->watchpoint[i].vaddr
-                && (env->watchpoint[i].type & flags)) {
-            env->watchpoint_hit = i + 1;
-            cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
-            break;
+    TAILQ_FOREACH(wp, &env->watchpoints, entry) {
+        if ((vaddr == (wp->vaddr & len_mask) ||
+             (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
+            wp->flags |= BP_WATCHPOINT_HIT;
+            if (!env->watchpoint_hit) {
+                env->watchpoint_hit = wp;
+                tb = tb_find_pc(env->mem_io_pc);
+                if (!tb) {
+                    cpu_abort(env, "check_watchpoint: could not find TB for "
+                              "pc=%p", (void *)env->mem_io_pc);
+                }
+                cpu_restore_state(tb, env, env->mem_io_pc, NULL);
+                tb_phys_invalidate(tb, -1);
+                if (wp->flags & BP_STOP_BEFORE_ACCESS) {
+                    env->exception_index = EXCP_DEBUG;
+                } else {
+                    cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
+                    tb_gen_code(env, pc, cs_base, cpu_flags, 1);
+                }
+                cpu_resume_from_signal(env, NULL);
+            }
+        } else {
+            wp->flags &= ~BP_WATCHPOINT_HIT;
         }
     }
@@ -2922 +3051 @@
 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
     return ldub_phys(addr);
 }
@@ -2928 +3057 @@
 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
     return lduw_phys(addr);
 }
@@ -2934 +3063 @@
 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_READ);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
     return ldl_phys(addr);
 }
@@ -2941 +3070 @@
                              uint32_t val)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
     stb_phys(addr, val);
 }
@@ -2948 +3077 @@
                              uint32_t val)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
     stw_phys(addr, val);
 }
@@ -2955 +3084 @@
                              uint32_t val)
 {
-    check_watchpoint(addr & ~TARGET_PAGE_MASK, PAGE_WRITE);
+    check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
     stl_phys(addr, val);
 }
@@ -2977 +3106 @@
     unsigned int idx;
 
-    idx = SUBPAGE_IDX(addr - mmio->base);
+    idx = SUBPAGE_IDX(addr);
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
            mmio, len, addr, idx);
 #endif
-    ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len], addr);
+    ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
+                                       addr + mmio->region_offset[idx][0][len]);
 
     return ret;
@@ -2992 +3122 @@
     unsigned int idx;
 
-    idx = SUBPAGE_IDX(addr - mmio->base);
+    idx = SUBPAGE_IDX(addr);
 #if defined(DEBUG_SUBPAGE)
     printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
            mmio, len, addr, idx, value);
 #endif
-    (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len], addr, value);
+    (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
+                                  addr + mmio->region_offset[idx][1][len],
+                                  value);
 }
 
@@ -3067 +3199 @@
 
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
-                             ram_addr_t memory)
+                             ram_addr_t memory, ram_addr_t region_offset)
 {
     int idx, eidx;
@@ -3086 +3218 @@
                 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
                 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
+                mmio->region_offset[idx][0][i] = region_offset;
             }
             if (io_mem_write[memory][i]) {
                 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
                 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
+                mmio->region_offset[idx][1][i] = region_offset;
             }
         }
@@ -3098 +3232 @@
 
 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
-                           ram_addr_t orig_memory)
+                           ram_addr_t orig_memory, ram_addr_t region_offset)
 {
     subpage_t *mmio;
@@ -3104 +3238 @@
 
     mmio = qemu_mallocz(sizeof(subpage_t));
-    if (mmio != NULL) {
-        mmio->base = base;
-        subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
+
+    mmio->base = base;
+    subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
 #if defined(DEBUG_SUBPAGE)
-        printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
-               mmio, base, TARGET_PAGE_SIZE, subpage_memory);
-#endif
-        *phys = subpage_memory | IO_MEM_SUBPAGE;
-        subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory);
-    }
+    printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
+           mmio, base, TARGET_PAGE_SIZE, subpage_memory);
+#endif
+    *phys = subpage_memory | IO_MEM_SUBPAGE;
+    subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
+                         region_offset);
 
     return mmio;
 }
 
+static int get_free_io_mem_idx(void)
+{
+    int i;
+
+    for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
+        if (!io_mem_used[i]) {
+            io_mem_used[i] = 1;
+            return i;
+        }
+
+    return -1;
+}
+
 static void io_mem_init(void)
 {
+    int i;
+
     cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
     cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
     cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
-    io_mem_nb = 5;
+    for (i=0; i<5; i++)
+        io_mem_used[i] = 1;
 
     io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
@@ -3151 +3301 @@
 
     if (io_index <= 0) {
-        if (io_mem_nb >= IO_MEM_NB_ENTRIES)
-            return -1;
-        io_index = io_mem_nb++;
+        io_index = get_free_io_mem_idx();
+        if (io_index == -1)
+            return io_index;
     } else {
         if (io_index >= IO_MEM_NB_ENTRIES)
@@ -3167 +3317 @@
     io_mem_opaque[io_index] = opaque;
     return (io_index << IO_MEM_SHIFT) | subwidth;
+}
+
+void cpu_unregister_io_memory(int io_table_address)
+{
+    int i;
+    int io_index = io_table_address >> IO_MEM_SHIFT;
+
+    for (i=0;i < 3; i++) {
+        io_mem_read[io_index][i] = unassigned_mem_read[i];
+        io_mem_write[io_index][i] = unassigned_mem_write[i];
+    }
+    io_mem_opaque[io_index] = NULL;
+    io_mem_used[io_index] = 0;
 }
 
@@ -3248 +3411 @@
         if (is_write) {
             if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
+                target_phys_addr_t addr1 = addr;
                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+                if (p)
+                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
                 /* XXX: could force cpu_single_env to NULL to avoid
                    potential bugs */
-                if (l >= 4 && ((addr & 3) == 0)) {
+                if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit write access */
 #if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
@@ -3258 +3424 @@
                     val = *(const uint32_t *)buf;
 #endif
-                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
+                    io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
                     l = 4;
-                } else if (l >= 2 && ((addr & 1) == 0)) {
+                } else if (l >= 2 && ((addr1 & 1) == 0)) {
                     /* 16 bit write access */
 #if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
@@ -3267 +3433 @@
                     val = *(const uint16_t *)buf;
 #endif
-                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
+                    io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
                     l = 2;
                 } else {
@@ -3276 +3442 @@
                     val = *(const uint8_t *)buf;
 #endif
-                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
+                    io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
                     l = 1;
                 }
@@ -3303 +3469 @@
             if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
                 !(pd & IO_MEM_ROMD)) {
+                target_phys_addr_t addr1 = addr;
                 /* I/O case */
                 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
-                if (l >= 4 && ((addr & 3) == 0)) {
+                if (p)
+                    addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
+                if (l >= 4 && ((addr1 & 3) == 0)) {
                     /* 32 bit read access */
-                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
+                    val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
 #if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
                     stl_p(buf, val);
@@ -3314 +3483 @@
 #endif
                     l = 4;
-                } else if (l >= 2 && ((addr & 1) == 0)) {
+                } else if (l >= 2 && ((addr1 & 1) == 0)) {
                     /* 16 bit read access */
-                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
+                    val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
 #if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
                     stw_p(buf, val);
@@ -3325 +3494 @@
                 } else {
                     /* 8 bit read access */
-                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
+                    val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
 #if !defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB)
                     stb_p(buf, val);
@@ -3351 +3520 @@
 
 #ifndef VBOX
+
 /* used for ROM loading : can write in RAM and ROM */
 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
@@ -3389 +3559 @@
     }
 }
+
+typedef struct {
+    void *buffer;
+    target_phys_addr_t addr;
+    target_phys_addr_t len;
+} BounceBuffer;
+
+static BounceBuffer bounce;
+
+typedef struct MapClient {
+    void *opaque;
+    void (*callback)(void *opaque);
+    LIST_ENTRY(MapClient) link;
+} MapClient;
+
+static LIST_HEAD(map_client_list, MapClient) map_client_list
+    = LIST_HEAD_INITIALIZER(map_client_list);
+
+void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
+{
+    MapClient *client = qemu_malloc(sizeof(*client));
+
+    client->opaque = opaque;
+    client->callback = callback;
+    LIST_INSERT_HEAD(&map_client_list, client, link);
+    return client;
+}
+
+void cpu_unregister_map_client(void *_client)
+{
+    MapClient *client = (MapClient *)_client;
+
+    LIST_REMOVE(client, link);
+}
+
+static void cpu_notify_map_clients(void)
+{
+    MapClient *client;
+
+    while (!LIST_EMPTY(&map_client_list)) {
+        client = LIST_FIRST(&map_client_list);
+        client->callback(client->opaque);
+        LIST_REMOVE(client, link);
+    }
+}
+
+/* Map a physical memory region into a host virtual address.
+ * May map a subset of the requested range, given by and returned in *plen.
+ * May return NULL if resources needed to perform the mapping are exhausted.
+ * Use only for reads OR writes - not for read-modify-write operations.
+ * Use cpu_register_map_client() to know when retrying the map operation is
+ * likely to succeed.
+ */
+void *cpu_physical_memory_map(target_phys_addr_t addr,
+                              target_phys_addr_t *plen,
+                              int is_write)
+{
+    target_phys_addr_t len = *plen;
+    target_phys_addr_t done = 0;
+    int l;
+    uint8_t *ret = NULL;
+    uint8_t *ptr;
+    target_phys_addr_t page;
+    unsigned long pd;
+    PhysPageDesc *p;
+    unsigned long addr1;
+
+    while (len > 0) {
+        page = addr & TARGET_PAGE_MASK;
+        l = (page + TARGET_PAGE_SIZE) - addr;
+        if (l > len)
+            l = len;
+        p = phys_page_find(page >> TARGET_PAGE_BITS);
+        if (!p) {
+            pd = IO_MEM_UNASSIGNED;
+        } else {
+            pd = p->phys_offset;
+        }
+
+        if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
+            if (done || bounce.buffer) {
+                break;
+            }
+            bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
+            bounce.addr = addr;
+            bounce.len = l;
+            if (!is_write) {
+                cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
+            }
+            ptr = bounce.buffer;
+        } else {
+            addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+            ptr = phys_ram_base + addr1;
+        }
+        if (!done) {
+            ret = ptr;
+        } else if (ret + done != ptr) {
+            break;
+        }
+
+        len -= l;
+        addr += l;
+        done += l;
+    }
+    *plen = done;
+    return ret;
+}
+
+/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
+ * Will also mark the memory as dirty if is_write == 1.  access_len gives
+ * the amount of memory that was actually read or written by the caller.
+ */
+void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
+                               int is_write, target_phys_addr_t access_len)
+{
+    if (buffer != bounce.buffer) {
+        if (is_write) {
+            unsigned long addr1 = (uint8_t *)buffer - phys_ram_base;
+            while (access_len) {
+                unsigned l;
+                l = TARGET_PAGE_SIZE;
+                if (l > access_len)
+                    l = access_len;
+                if (!cpu_physical_memory_is_dirty(addr1)) {
+                    /* invalidate code */
+                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
+                    /* set dirty bit */
+                    phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
+                        (0xff & ~CODE_DIRTY_FLAG);
+                }
+                addr1 += l;
+                access_len -= l;
+            }
+        }
+        return;
+    }
+    if (is_write) {
+        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
+    }
+    qemu_free(bounce.buffer);
+    bounce.buffer = NULL;
+    cpu_notify_map_clients();
+}
+
 #endif /* !VBOX */
-
 
 /* warning: addr must be aligned */
@@ -3412 +3725 @@
         /* I/O case */
         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
         val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
     } else {
@@ -3446 +3761 @@
         /* I/O case */
         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
 #ifdef TARGET_WORDS_BIGENDIAN
         val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
@@ -3501 +3818 @@
     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
     } else {
@@ -3510 +3829 @@
         remR3PhysWriteU32((pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK), val); NOREF(ptr);
 #endif
+
 #ifndef VBOX
         if (unlikely(in_migration)) {
@@ -3540 +3860 @@
     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
 #ifdef TARGET_WORDS_BIGENDIAN
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
@@ -3575 +3897 @@
     if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
         io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+        if (p)
+            addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
         io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
     } else {

trunk/src/recompiler/fpu/softfloat-macros.h

@@ -718 +718 @@
 
 }
-

trunk/src/recompiler/fpu/softfloat-native.c

@@ -61 +61 @@
 #endif
 
-#if defined(__powerpc__)
+#if defined(_ARCH_PPC)
 
 /* correct (but slow) PowerPC rint() (glibc version is incorrect) */
-double qemu_rint(double x)
+static double qemu_rint(double x)
 {
     double y = 4503599627370496.0;
@@ -230 +230 @@
 {
     if (a < b) {
-        return -1;
+        return float_relation_less;
     } else if (a == b) {
-        return 0;
+        return float_relation_equal;
     } else if (a > b) {
-        return 1;
-    } else {
-        return 2;
+        return float_relation_greater;
+    } else {
+        return float_relation_unordered;
     }
 }
@@ -242 +242 @@
 {
     if (isless(a, b)) {
-        return -1;
+        return float_relation_less;
     } else if (a == b) {
-        return 0;
+        return float_relation_equal;
     } else if (isgreater(a, b)) {
-        return 1;
-    } else {
-        return 2;
+        return float_relation_greater;
+    } else {
+        return float_relation_unordered;
     }
 }
@@ -258 +258 @@
     a = u.i;
     return ( ( ( a>>22 ) & 0x1FF ) == 0x1FE ) && ( a & 0x003FFFFF );
+}
+
+int float32_is_nan( float32 a1 )
+{
+    float32u u;
+    uint64_t a;
+    u.f = a1;
+    a = u.i;
+    return ( 0xFF800000 < ( a<<1 ) );
 }
 
@@ -392 +401 @@
 {
     if (a < b) {
-        return -1;
+        return float_relation_less;
     } else if (a == b) {
-        return 0;
+        return float_relation_equal;
     } else if (a > b) {
-        return 1;
-    } else {
-        return 2;
+        return float_relation_greater;
+    } else {
+        return float_relation_unordered;
     }
 }
@@ -404 +413 @@
 {
     if (isless(a, b)) {
-        return -1;
+        return float_relation_less;
     } else if (a == b) {
-        return 0;
+        return float_relation_equal;
     } else if (isgreater(a, b)) {
-        return 1;
-    } else {
-        return 2;
+        return float_relation_greater;
+    } else {
+        return float_relation_unordered;
     }
 }
@@ -432 +441 @@
     a = u.i;
 
-    return ( LIT64( 0xFFE0000000000000 ) < (bits64) ( a<<1 ) );
+    return ( LIT64( 0xFFF0000000000000 ) < (bits64) ( a<<1 ) );
 
 }
@@ -484 +493 @@
 {
     if (a < b) {
-        return -1;
+        return float_relation_less;
     } else if (a == b) {
-        return 0;
+        return float_relation_equal;
     } else if (a > b) {
-        return 1;
-    } else {
-        return 2;
+        return float_relation_greater;
+    } else {
+        return float_relation_unordered;
     }
 }
@@ -496 +505 @@
 {
     if (isless(a, b)) {
-        return -1;
+        return float_relation_less;
     } else if (a == b) {
-        return 0;
+        return float_relation_equal;
     } else if (isgreater(a, b)) {
-        return 1;
-    } else {
-        return 2;
+        return float_relation_greater;
+    } else {
+        return float_relation_unordered;
     }
 }
 int floatx80_is_signaling_nan( floatx80 a1)
+{
+    floatx80u u;
+    uint64_t aLow;
+    u.f = a1;
+
+    aLow = u.i.low & ~ LIT64( 0x4000000000000000 );
+    return
+           ( ( u.i.high & 0x7FFF ) == 0x7FFF )
+        && (bits64) ( aLow<<1 )
+        && ( u.i.low == aLow );
+}
+
+int floatx80_is_nan( floatx80 a1 )
 {
     floatx80u u;

trunk/src/recompiler/fpu/softfloat-native.h

@@ -9 +9 @@
 #endif
 
-#ifdef __OpenBSD__
-/* Get OpenBSD version number */
+#if defined(__OpenBSD__) || defined(__NetBSD__)
 #include <sys/param.h>
 #endif
@@ -36 +35 @@
 #endif
 
+#ifdef __NetBSD__
+#ifndef isgreater
+#define isgreater(x, y)         __builtin_isgreater(x, y)
+#endif
+#ifndef isgreaterequal
+#define isgreaterequal(x, y)    __builtin_isgreaterequal(x, y)
+#endif
+#ifndef isless
+#define isless(x, y)            __builtin_isless(x, y)
+#endif
+#ifndef islessequal
+#define islessequal(x, y)       __builtin_islessequal(x, y)
+#endif
+#ifndef isunordered
+#define isunordered(x, y)       __builtin_isunordered(x, y)
+#endif
+#endif
+
+
 #define isnormal(x)             (fpclass(x) >= FP_NZERO)
 #define isgreater(x, y)         ((!unordered(x, y)) && ((x) > (y)))
@@ -123 +141 @@
 
 typedef struct float_status {
-    signed char float_rounding_mode;
-#ifdef FLOATX80
-    signed char floatx80_rounding_precision;
+    int float_rounding_mode;
+#ifdef FLOATX80
+    int floatx80_rounding_precision;
 #endif
 } float_status;
@@ -229 +247 @@
 int float32_compare_quiet( float32, float32 STATUS_PARAM );
 int float32_is_signaling_nan( float32 );
+int float32_is_nan( float32 );
 
 INLINE float32 float32_abs(float32 a)
@@ -238 +257 @@
 {
     return -a;
+}
+
+INLINE float32 float32_is_infinity(float32 a)
+{
+    return fpclassify(a) == FP_INFINITE;
+}
+
+INLINE float32 float32_is_neg(float32 a)
+{
+    float32u u;
+    u.f = a;
+    return u.i >> 31;
+}
+
+INLINE float32 float32_is_zero(float32 a)
+{
+    return fpclassify(a) == FP_ZERO;
 }
 
@@ -332 +368 @@
 }
 
+INLINE float64 float64_is_infinity(float64 a)
+{
+    return fpclassify(a) == FP_INFINITE;
+}
+
+INLINE float64 float64_is_neg(float64 a)
+{
+    float64u u;
+    u.f = a;
+    return u.i >> 63;
+}
+
+INLINE float64 float64_is_zero(float64 a)
+{
+    return fpclassify(a) == FP_ZERO;
+}
+
 INLINE float64 float64_scalbn(float64 a, int n)
 {
@@ -407 +460 @@
 int floatx80_compare_quiet( floatx80, floatx80 STATUS_PARAM );
 int floatx80_is_signaling_nan( floatx80 );
+int floatx80_is_nan( floatx80 );
 
 INLINE floatx80 floatx80_abs(floatx80 a)
@@ -418 +472 @@
 }
 
+INLINE floatx80 floatx80_is_infinity(floatx80 a)
+{
+    return fpclassify(a) == FP_INFINITE;
+}
+
+INLINE floatx80 floatx80_is_neg(floatx80 a)
+{
+    floatx80u u;
+    u.f = a;
+    return u.i.high >> 15;
+}
+
+INLINE floatx80 floatx80_is_zero(floatx80 a)
+{
+    return fpclassify(a) == FP_ZERO;
+}
+
 INLINE floatx80 floatx80_scalbn(floatx80 a, int n)
 {

trunk/src/recompiler/fpu/softfloat-specialize.h

@@ -38 +38 @@
 
 /*----------------------------------------------------------------------------
-| Underflow tininess-detection mode, statically initialized to default value.
-| (The declaration in `softfloat.h' must match the `int8' type here.)
-*----------------------------------------------------------------------------*/
-int8 float_detect_tininess = float_tininess_after_rounding;
-
-/*----------------------------------------------------------------------------
 | Raises the exceptions specified by `flags'.  Floating-point traps can be
 | defined here if desired.  It is currently not possible for such a trap
@@ -68 +62 @@
 #if defined(TARGET_SPARC)
 #define float32_default_nan make_float32(0x7FFFFFFF)
-#elif defined(TARGET_POWERPC)
+#elif defined(TARGET_POWERPC) || defined(TARGET_ARM)
 #define float32_default_nan make_float32(0x7FC00000)
 #elif defined(TARGET_HPPA)
@@ -150 +144 @@
     flag aIsNaN, aIsSignalingNaN, bIsNaN, bIsSignalingNaN;
     bits32 av, bv, res;
+
+    if ( STATUS(default_nan_mode) )
+        return float32_default_nan;
 
     aIsNaN = float32_is_nan( a );
@@ -193 +190 @@
 #if defined(TARGET_SPARC)
 #define float64_default_nan make_float64(LIT64( 0x7FFFFFFFFFFFFFFF ))
-#elif defined(TARGET_POWERPC)
+#elif defined(TARGET_POWERPC) || defined(TARGET_ARM)
 #define float64_default_nan make_float64(LIT64( 0x7FF8000000000000 ))
 #elif defined(TARGET_HPPA)
@@ -282 +279 @@
     flag aIsNaN, aIsSignalingNaN, bIsNaN, bIsSignalingNaN;
     bits64 av, bv, res;
+
+    if ( STATUS(default_nan_mode) )
+        return float64_default_nan;
 
     aIsNaN = float64_is_nan( a );
@@ -419 +419 @@
     flag aIsNaN, aIsSignalingNaN, bIsNaN, bIsSignalingNaN;
 
+    if ( STATUS(default_nan_mode) ) {
+        a.low = floatx80_default_nan_low;
+        a.high = floatx80_default_nan_high;
+        return a;
+    }
+
     aIsNaN = floatx80_is_nan( a );
     aIsSignalingNaN = floatx80_is_signaling_nan( a );
@@ -539 +545 @@
     flag aIsNaN, aIsSignalingNaN, bIsNaN, bIsSignalingNaN;
 
+    if ( STATUS(default_nan_mode) ) {
+        a.low = float128_default_nan_low;
+        a.high = float128_default_nan_high;
+        return a;
+    }
+
     aIsNaN = float128_is_nan( a );
     aIsSignalingNaN = float128_is_signaling_nan( a );

trunk/src/recompiler/fpu/softfloat.c

@@ -31 +31 @@
 =============================================================================*/
 
+/* FIXME: Flush-To-Zero only effects results.  Denormal inputs should also
+   be flushed to zero.  */
 #include "softfloat.h"
 
@@ -295 +297 @@
         }
         if ( zExp < 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloat32( zSign, 0, 0 );
             isTiny =
                    ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
@@ -458 +461 @@
         }
         if ( zExp < 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloat64( zSign, 0, 0 );
             isTiny =
                    ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
@@ -636 +640 @@
         }
         if ( zExp <= 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloatx80( zSign, 0, 0 );
             isTiny =
                    ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
@@ -966 +971 @@
         }
         if ( zExp < 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloat128( zSign, 0, 0, 0 );
             isTiny =
                    ( STATUS(float_detect_tininess) == float_tininess_before_rounding )
@@ -1638 +1644 @@
             return a;
         }
-        if ( aExp == 0 ) return packFloat32( zSign, 0, ( aSig + bSig )>>6 );
+        if ( aExp == 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloat32( zSign, 0, 0 );
+            return packFloat32( zSign, 0, ( aSig + bSig )>>6 );
+        }
         zSig = 0x40000000 + aSig + bSig;
         zExp = aExp;
@@ -2046 +2055 @@
     return roundAndPackFloat32( 0, zExp, zSig STATUS_VAR );
 
+}
+
+/*----------------------------------------------------------------------------
+| Returns the binary log of the single-precision floating-point value `a'.
+| The operation is performed according to the IEC/IEEE Standard for Binary
+| Floating-Point Arithmetic.
+*----------------------------------------------------------------------------*/
+float32 float32_log2( float32 a STATUS_PARAM )
+{
+    flag aSign, zSign;
+    int16 aExp;
+    bits32 aSig, zSig, i;
+
+    aSig = extractFloat32Frac( a );
+    aExp = extractFloat32Exp( a );
+    aSign = extractFloat32Sign( a );
+
+    if ( aExp == 0 ) {
+        if ( aSig == 0 ) return packFloat32( 1, 0xFF, 0 );
+        normalizeFloat32Subnormal( aSig, &aExp, &aSig );
+    }
+    if ( aSign ) {
+        float_raise( float_flag_invalid STATUS_VAR);
+        return float32_default_nan;
+    }
+    if ( aExp == 0xFF ) {
+        if ( aSig ) return propagateFloat32NaN( a, float32_zero STATUS_VAR );
+        return a;
+    }
+
+    aExp -= 0x7F;
+    aSig |= 0x00800000;
+    zSign = aExp < 0;
+    zSig = aExp << 23;
+
+    for (i = 1 << 22; i > 0; i >>= 1) {
+        aSig = ( (bits64)aSig * aSig ) >> 23;
+        if ( aSig & 0x01000000 ) {
+            aSig >>= 1;
+            zSig |= i;
+        }
+    }
+
+    if ( zSign )
+        zSig = -zSig;
+
+    return normalizeRoundAndPackFloat32( zSign, 0x85, zSig STATUS_VAR );
 }
 
@@ -2596 +2652 @@
             return a;
         }
-        if ( aExp == 0 ) return packFloat64( zSign, 0, ( aSig + bSig )>>9 );
+        if ( aExp == 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloat64( zSign, 0, 0 );
+            return packFloat64( zSign, 0, ( aSig + bSig )>>9 );
+        }
         zSig = LIT64( 0x4000000000000000 ) + aSig + bSig;
         zExp = aExp;
@@ -2992 +3051 @@
     return roundAndPackFloat64( 0, zExp, zSig STATUS_VAR );
 
+}
+
+/*----------------------------------------------------------------------------
+| Returns the binary log of the double-precision floating-point value `a'.
+| The operation is performed according to the IEC/IEEE Standard for Binary
+| Floating-Point Arithmetic.
+*----------------------------------------------------------------------------*/
+float64 float64_log2( float64 a STATUS_PARAM )
+{
+    flag aSign, zSign;
+    int16 aExp;
+    bits64 aSig, aSig0, aSig1, zSig, i;
+
+    aSig = extractFloat64Frac( a );
+    aExp = extractFloat64Exp( a );
+    aSign = extractFloat64Sign( a );
+
+    if ( aExp == 0 ) {
+        if ( aSig == 0 ) return packFloat64( 1, 0x7FF, 0 );
+        normalizeFloat64Subnormal( aSig, &aExp, &aSig );
+    }
+    if ( aSign ) {
+        float_raise( float_flag_invalid STATUS_VAR);
+        return float64_default_nan;
+    }
+    if ( aExp == 0x7FF ) {
+        if ( aSig ) return propagateFloat64NaN( a, float64_zero STATUS_VAR );
+        return a;
+    }
+
+    aExp -= 0x3FF;
+    aSig |= LIT64( 0x0010000000000000 );
+    zSign = aExp < 0;
+    zSig = (bits64)aExp << 52;
+    for (i = 1LL << 51; i > 0; i >>= 1) {
+        mul64To128( aSig, aSig, &aSig0, &aSig1 );
+        aSig = ( aSig0 << 12 ) | ( aSig1 >> 52 );
+        if ( aSig & LIT64( 0x0020000000000000 ) ) {
+            aSig >>= 1;
+            zSig |= i;
+        }
+    }
+
+    if ( zSign )
+        zSig = -zSig;
+    return normalizeRoundAndPackFloat64( zSign, 0x408, zSig STATUS_VAR );
 }
 
@@ -4598 +4703 @@
         }
         add128( aSig0, aSig1, bSig0, bSig1, &zSig0, &zSig1 );
-        if ( aExp == 0 ) return packFloat128( zSign, 0, zSig0, zSig1 );
+        if ( aExp == 0 ) {
+            if ( STATUS(flush_to_zero) ) return packFloat128( zSign, 0, 0, 0 );
+            return packFloat128( zSign, 0, zSig0, zSig1 );
+        }
         zSig2 = 0;
         zSig0 |= LIT64( 0x0002000000000000 );
@@ -5480 +5588 @@
         return a;
     }
-    aExp += n;
-    return roundAndPackFloat32( aSign, aExp, aSig STATUS_VAR );
+    if ( aExp != 0 )
+        aSig |= 0x00800000;
+    else if ( aSig == 0 )
+        return a;
+
+    aExp += n - 1;
+    aSig <<= 7;
+    return normalizeRoundAndPackFloat32( aSign, aExp, aSig STATUS_VAR );
 }
 
@@ -5497 +5611 @@
         return a;
     }
-    aExp += n;
-    return roundAndPackFloat64( aSign, aExp, aSig STATUS_VAR );
+    if ( aExp != 0 )
+        aSig |= LIT64( 0x0010000000000000 );
+    else if ( aSig == 0 )
+        return a;
+
+    aExp += n - 1;
+    aSig <<= 10;
+    return normalizeRoundAndPackFloat64( aSign, aExp, aSig STATUS_VAR );
 }
 
@@ -5515 +5635 @@
         return a;
     }
+    if (aExp == 0 && aSig == 0)
+        return a;
+
     aExp += n;
-    return roundAndPackFloatx80( STATUS(floatx80_rounding_precision),
-                                 aSign, aExp, aSig, 0 STATUS_VAR );
+    return normalizeRoundAndPackFloatx80( STATUS(floatx80_rounding_precision),
+                                          aSign, aExp, aSig, 0 STATUS_VAR );
 }
 #endif
@@ -5535 +5658 @@
         return a;
     }
-    aExp += n;
-    return roundAndPackFloat128( aSign, aExp, aSig0, aSig1, 0 STATUS_VAR );
+    if ( aExp != 0 )
+        aSig0 |= LIT64( 0x0001000000000000 );
+    else if ( aSig0 == 0 && aSig1 == 0 )
+        return a;
+
+    aExp += n - 1;
+    return normalizeRoundAndPackFloat128( aSign, aExp, aSig0, aSig1
+                                          STATUS_VAR );
 
 }

trunk/src/recompiler/fpu/softfloat.h

@@ -55 +55 @@
 typedef uint8_t uint8;
 typedef int8_t int8;
+#ifndef _AIX
 typedef int uint16;
 typedef int int16;
+#endif
 typedef unsigned int uint32;
 typedef signed int int32;
@@ -197 +199 @@
     signed char floatx80_rounding_precision;
 #endif
+    flag flush_to_zero;
+    flag default_nan_mode;
 } float_status;
 
 void set_float_rounding_mode(int val STATUS_PARAM);
 void set_float_exception_flags(int val STATUS_PARAM);
+INLINE void set_flush_to_zero(flag val STATUS_PARAM)
+{
+    STATUS(flush_to_zero) = val;
+}
+INLINE void set_default_nan_mode(flag val STATUS_PARAM)
+{
+    STATUS(default_nan_mode) = val;
+}
 INLINE int get_float_exception_flags(float_status *status)
 {
@@ -266 +278 @@
 float32 float32_rem( float32, float32 STATUS_PARAM );
 float32 float32_sqrt( float32 STATUS_PARAM );
+float32 float32_log2( float32 STATUS_PARAM );
 int float32_eq( float32, float32 STATUS_PARAM );
 int float32_le( float32, float32 STATUS_PARAM );
@@ -288 +301 @@
 }
 
+INLINE int float32_is_infinity(float32 a)
+{
+    return (float32_val(a) & 0x7fffffff) == 0x7f800000;
+}
+
+INLINE int float32_is_neg(float32 a)
+{
+    return float32_val(a) >> 31;
+}
+
+INLINE int float32_is_zero(float32 a)
+{
+    return (float32_val(a) & 0x7fffffff) == 0;
+}
+
 #define float32_zero make_float32(0)
+#define float32_one make_float32(0x3f800000)
 
 /*----------------------------------------------------------------------------
@@ -320 +349 @@
 float64 float64_rem( float64, float64 STATUS_PARAM );
 float64 float64_sqrt( float64 STATUS_PARAM );
+float64 float64_log2( float64 STATUS_PARAM );
 int float64_eq( float64, float64 STATUS_PARAM );
 int float64_le( float64, float64 STATUS_PARAM );
@@ -342 +372 @@
 }
 
+INLINE int float64_is_infinity(float64 a)
+{
+    return (float64_val(a) & 0x7fffffffffffffffLL ) == 0x7ff0000000000000LL;
+}
+
+INLINE int float64_is_neg(float64 a)
+{
+    return float64_val(a) >> 63;
+}
+
+INLINE int float64_is_zero(float64 a)
+{
+    return (float64_val(a) & 0x7fffffffffffffffLL) == 0;
+}
+
 #define float64_zero make_float64(0)
+#define float64_one make_float64(0x3ff0000000000000LL)
 
 #ifdef FLOATX80
@@ -389 +435 @@
     a.high ^= 0x8000;
     return a;
+}
+
+INLINE int floatx80_is_infinity(floatx80 a)
+{
+    return (a.high & 0x7fff) == 0x7fff && a.low == 0;
+}
+
+INLINE int floatx80_is_neg(floatx80 a)
+{
+    return a.high >> 15;
+}
+
+INLINE int floatx80_is_zero(floatx80 a)
+{
+    return (a.high & 0x7fff) == 0 && a.low == 0;
 }
 
@@ -442 +503 @@
 }
 
+INLINE int float128_is_infinity(float128 a)
+{
+    return (a.high & 0x7fffffffffffffffLL) == 0x7fff000000000000LL && a.low == 0;
+}
+
+INLINE int float128_is_neg(float128 a)
+{
+    return a.high >> 63;
+}
+
+INLINE int float128_is_zero(float128 a)
+{
+    return (a.high & 0x7fffffffffffffffLL) == 0 && a.low == 0;
+}
+
 #endif
 

trunk/src/recompiler/gen-icount.h

@@ -6 +6 @@
 static inline void gen_icount_start(void)
 {
-    TCGv count;
+    TCGv_i32 count;
 
     if (!use_icount)
@@ -16 +16 @@
        we allow the target to supply a convenient register temporary.  */
 #ifndef ICOUNT_TEMP
-    count = tcg_temp_local_new(TCG_TYPE_I32);
+    count = tcg_temp_local_new_i32();
 #else
     count = ICOUNT_TEMP;
@@ -28 +28 @@
     tcg_gen_st16_i32(count, cpu_env, offsetof(CPUState, icount_decr.u16.low));
 #ifndef ICOUNT_TEMP
-    tcg_temp_free(count);
+    tcg_temp_free_i32(count);
 #endif
 }
@@ -41 +41 @@
 }
 
-inline static void gen_io_start(void)
+static inline void gen_io_start(void)
 {
-    TCGv tmp = tcg_const_i32(1);
+    TCGv_i32 tmp = tcg_const_i32(1);
     tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUState, can_do_io));
-    tcg_temp_free(tmp);
+    tcg_temp_free_i32(tmp);
 }
 
 static inline void gen_io_end(void)
 {
-    TCGv tmp = tcg_const_i32(0);
+    TCGv_i32 tmp = tcg_const_i32(0);
     tcg_gen_st_i32(tmp, cpu_env, offsetof(CPUState, can_do_io));
-    tcg_temp_free(tmp);
+    tcg_temp_free_i32(tmp);
 }
-

trunk/src/recompiler/hostregs_helper.h

@@ -1 +1 @@
 /*
- *  Save/restore host registers.
+ *  Save/restore host registrs.
  *
  *  Copyright (c) 2007 CodeSourcery
@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -28 +28 @@
  */
 
-/* The GCC global register vairable extension is used to reserve some
-   host registers for use by dyngen.  However only the core parts of the
-   translation engine are compiled with these settings.  We must manually
+/* The GCC global register variable extension is used to reserve some
+   host registers for use by generated code.  However only the core parts of
+   the translation engine are compiled with these settings.  We must manually
    save/restore these registers when called from regular code.
    It is not sufficient to save/restore T0 et. al. as these may be declared

trunk/src/recompiler/osdep.h

@@ -87 +87 @@
 #define likely(x)   __builtin_expect(!!(x), 1)
 #define unlikely(x)   __builtin_expect(!!(x), 0)
-#endif /* !likely */
+#endif
 
 #ifndef offsetof
@@ -114 +114 @@
 #else
 #define always_inline __attribute__ (( always_inline )) __inline__
+#ifdef __OPTIMIZE__
 #define inline always_inline
+#endif
 #endif
 #else
@@ -130 +132 @@
 #endif
 
-#if defined (__GNUC__) && defined (__GNUC_MINOR_)
+#if defined (__GNUC__) && defined (__GNUC_MINOR__)
 # define QEMU_GNUC_PREREQ(maj, min) \
          ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
@@ -156 +158 @@
 #define qemu_gettimeofday(tp) gettimeofday(tp, NULL);
 #endif /* !_WIN32 */
-#endif /* !VBOX */
-
-#ifdef VBOX
-/** @todo why don't we go with dyngen-exec.h here? */
-#define FORCE_RET() ;
+#else  /* VBOX */
+# define qemu_memalign(alignment, size)  ( (alignment) <= PAGE_SIZE ? RTMemPageAlloc((size)) : NULL )
 #endif /* VBOX */
 

trunk/src/recompiler/qemu-common.h

@@ -7 +7 @@
 # include <string.h>
 # include <inttypes.h>
+# include <iprt/ctype.h>
+
+#define QEMU_NORETURN __attribute__ ((__noreturn__))
 
 void pstrcpy(char *buf, int buf_size, const char *str);
@@ -12 +15 @@
 # define snprintf RTStrPrintf
 
+#define qemu_isalnum(c)         RT_C_IS_ALNUM((unsigned char)(c))
+#define qemu_isalpha(c)         RT_C_IS_ALPHA((unsigned char)(c))
+#define qemu_iscntrl(c)         RT_C_IS_CNTRL((unsigned char)(c))
+#define qemu_isdigit(c)         RT_C_IS_DIGIT((unsigned char)(c))
+#define qemu_isgraph(c)         RT_C_IS_GRAPH((unsigned char)(c))
+#define qemu_islower(c)         RT_C_IS_LOWER((unsigned char)(c))
+#define qemu_isprint(c)         RT_C_IS_PRINT((unsigned char)(c))
+#define qemu_ispunct(c)         RT_C_IS_PUNCT((unsigned char)(c))
+#define qemu_isspace(c)         RT_C_IS_SPACE((unsigned char)(c))
+#define qemu_isupper(c)         RT_C_IS_UPPER((unsigned char)(c))
+#define qemu_isxdigit(c)        RT_C_IS_XDIGIT((unsigned char)(c))
+#define qemu_tolower(c)         RT_C_TO_LOWER((unsigned char)(c))
+#define qemu_toupper(c)         RT_C_TO_UPPER((unsigned char)(c))
+#define qemu_isascii(c)         RT_C_IS_ASCII((unsigned char)(c))
+#define qemu_toascii(c)         RT_C_TO_ASCII((unsigned char)(c))
+
 #else /* !VBOX */
+#ifdef _WIN32
+#define WIN32_LEAN_AND_MEAN
+#define WINVER 0x0501  /* needed for ipv6 bits */
+#include <windows.h>
+#endif
+
+#define QEMU_NORETURN __attribute__ ((__noreturn__))
+
+/* Hack around the mess dyngen-exec.h causes: We need QEMU_NORETURN in files that
+   cannot include the following headers without conflicts. This condition has
+   to be removed once dyngen is gone. */
+#ifndef __DYNGEN_EXEC_H__
+
 /* we put basic includes here to avoid repeating them in device drivers */
 #include <stdlib.h>
@@ -18 +50 @@
 #include <stdarg.h>
 #include <string.h>
+#include <strings.h>
 #include <inttypes.h>
 #include <limits.h>
@@ -26 +59 @@
 #include <fcntl.h>
 #include <sys/stat.h>
+#include "config-host.h"
 
 #ifndef O_LARGEFILE
@@ -38 +72 @@
 #endif
 
+#ifndef HAVE_IOVEC
+#define HAVE_IOVEC
+struct iovec {
+    void *iov_base;
+    size_t iov_len;
+};
+#else
+#include <sys/uio.h>
+#endif
+
 #ifdef _WIN32
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
 #define fsync _commit
 #define lseek _lseeki64
@@ -63 +105 @@
 #ifndef NEED_CPU_H
 
-#include "config-host.h"
 #include <setjmp.h>
 #include "osdep.h"
@@ -81 +122 @@
 QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
 void qemu_bh_schedule(QEMUBH *bh);
+/* Bottom halfs that are scheduled from a bottom half handler are instantly
+ * invoked.  This can create an infinite loop if a bottom half handler
+ * schedules itself.  qemu_bh_schedule_idle() avoids this infinite loop by
+ * ensuring that the bottom half isn't executed until the next main loop
+ * iteration.
+ */
+void qemu_bh_schedule_idle(QEMUBH *bh);
 void qemu_bh_cancel(QEMUBH *bh);
 void qemu_bh_delete(QEMUBH *bh);
@@ -96 +144 @@
 int stristart(const char *str, const char *val, const char **ptr);
 time_t mktimegm(struct tm *tm);
+int qemu_fls(int i);
+
+#define qemu_isalnum(c)         isalnum((unsigned char)(c))
+#define qemu_isalpha(c)         isalpha((unsigned char)(c))
+#define qemu_iscntrl(c)         iscntrl((unsigned char)(c))
+#define qemu_isdigit(c)         isdigit((unsigned char)(c))
+#define qemu_isgraph(c)         isgraph((unsigned char)(c))
+#define qemu_islower(c)         islower((unsigned char)(c))
+#define qemu_isprint(c)         isprint((unsigned char)(c))
+#define qemu_ispunct(c)         ispunct((unsigned char)(c))
+#define qemu_isspace(c)         isspace((unsigned char)(c))
+#define qemu_isupper(c)         isupper((unsigned char)(c))
+#define qemu_isxdigit(c)        isxdigit((unsigned char)(c))
+#define qemu_tolower(c)         tolower((unsigned char)(c))
+#define qemu_toupper(c)         toupper((unsigned char)(c))
+#define qemu_isascii(c)         isascii((unsigned char)(c))
+#define qemu_toascii(c)         toascii((unsigned char)(c))
 
 void *qemu_malloc(size_t size);
@@ -102 +167 @@
 void qemu_free(void *ptr);
 char *qemu_strdup(const char *str);
+char *qemu_strndup(const char *str, size_t size);
 
 void *get_mmap_addr(unsigned long size);
@@ -108 +174 @@
 /* Error handling.  */
 
-void hw_error(const char *fmt, ...)
-    __attribute__ ((__format__ (__printf__, 1, 2)))
-    __attribute__ ((__noreturn__));
+void QEMU_NORETURN hw_error(const char *fmt, ...)
+    __attribute__ ((__format__ (__printf__, 1, 2)));
 
 /* IO callbacks.  */
@@ -131 +196 @@
 typedef struct BlockDriverState BlockDriverState;
 typedef struct DisplayState DisplayState;
+typedef struct DisplayChangeListener DisplayChangeListener;
+typedef struct DisplaySurface DisplaySurface;
+typedef struct PixelFormat PixelFormat;
 typedef struct TextConsole TextConsole;
 typedef TextConsole QEMUConsole;
@@ -152 +220 @@
 /* Force QEMU to stop what it's doing and service IO */
 void qemu_service_io(void);
+
+typedef struct QEMUIOVector {
+    struct iovec *iov;
+    int niov;
+    int nalloc;
+    size_t size;
+} QEMUIOVector;
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
+void qemu_iovec_destroy(QEMUIOVector *qiov);
+void qemu_iovec_reset(QEMUIOVector *qiov);
+void qemu_iovec_to_buffer(QEMUIOVector *qiov, void *buf);
+void qemu_iovec_from_buffer(QEMUIOVector *qiov, const void *buf, size_t count);
+
+#endif /* dyngen-exec.h hack */
+
 #endif /* !VBOX */
 

trunk/src/recompiler/qemu-lock.h

@@ -14 +14 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -73 +73 @@
     return ASMAtomicCmpXchgU32((volatile uint32_t *)p, 1, 0) ? 0 : 1;
 }
-#elif defined(__powerpc__)
+#elif defined(_ARCH_PPC)
 static inline int testandset (int *p)
 {
     int ret;
     __asm__ __volatile__ (
-                          "0:    lwarx %0,0,%1\n"
+                          "      lwarx %0,0,%1\n"
                           "      xor. %0,%3,%0\n"
-                          "      bne 1f\n"
+                          "      bne $+12\n"
                           "      stwcx. %2,0,%1\n"
-                          "      bne- 0b\n"
-                          "1:    "
+                          "      bne- $-16\n"
                           : "=&r" (ret)
                           : "r" (p), "r" (1), "r" (0)

trunk/src/recompiler/qemu-log.h

@@ -2 +2 @@
 #define QEMU_LOG_H
 
+/* The deprecated global variables: */
 extern FILE *logfile;
 extern int loglevel;
 
+
+/*
+ * The new API:
+ *
+ */
+
+/* Log settings checking macros: */
+
+/* Returns true if qemu_log() will really write somewhere
+ */
+#define qemu_log_enabled() (logfile != NULL)
+
+/* Returns true if a bit is set in the current loglevel mask
+ */
+#define qemu_loglevel_mask(b) ((loglevel & (b)) != 0)
+
+
+/* Logging functions: */
+
+/* main logging function
+ */
+#define qemu_log(...) do {                 \
+        if (logfile)                       \
+            fprintf(logfile, ## __VA_ARGS__); \
+    } while (0)
+
+/* vfprintf-like logging function
+ */
+#define qemu_log_vprintf(fmt, va) do {     \
+        if (logfile)                       \
+            vfprintf(logfile, fmt, va);    \
+    } while (0)
+
+/* log only if a bit is set on the current loglevel mask
+ */
+#define qemu_log_mask(b, ...) do {         \
+        if (loglevel & (b))                \
+            fprintf(logfile, ## __VA_ARGS__); \
+    } while (0)
+
+
+
+
+/* Special cases: */
+
+/* cpu_dump_state() logging functions: */
+#define log_cpu_state(env, f) cpu_dump_state((env), logfile, fprintf, (f));
+#define log_cpu_state_mask(b, env, f) do {           \
+      if (loglevel & (b)) log_cpu_state((env), (f)); \
+  } while (0)
+
+/* disas() and target_disas() to logfile: */
+#define log_target_disas(start, len, flags) \
+        target_disas(logfile, (start), (len), (flags))
+#define log_disas(start, len) \
+        disas(logfile, (start), (len))
+
+/* page_dump() output to the log file: */
+#define log_page_dump() page_dump(logfile)
+
+
+
+/* Maintenance: */
+
+/* fflush() the log file */
+#define qemu_log_flush() fflush(logfile)
+
+/* Close the log file */
+#define qemu_log_close() do { \
+        fclose(logfile);      \
+        logfile = NULL;       \
+    } while (0)
+
+/* Set up a new log file */
+#define qemu_log_set_file(f) do { \
+        logfile = (f);            \
+    } while (0)
+
+/* Set up a new log file, only if none is set */
+#define qemu_log_try_set_file(f) do { \
+        if (!logfile)                 \
+            logfile = (f);            \
+    } while (0)
+
+
 #endif

trunk/src/recompiler/softmmu_exec.h

@@ -1 +1 @@
 /* Common softmmu definitions and inline routines.  */
 
+/* XXX: find something cleaner.
+ * Furthermore, this is false for 64 bits targets
+ */
 #define ldul_user       ldl_user
 #define ldul_kernel     ldl_kernel

trunk/src/recompiler/softmmu_header.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 

trunk/src/recompiler/softmmu_template.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -85 +85 @@
     }
 
+    env->mem_io_vaddr = addr;
 #if SHIFT <= 2
     res = io_mem_read[index][SHIFT](io_mem_opaque[index], physaddr);

trunk/src/recompiler/target-i386/cpu.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -179 +179 @@
 #define HF_EM_MASK           (1 << HF_EM_SHIFT)
 #define HF_TS_MASK           (1 << HF_TS_SHIFT)
+#define HF_IOPL_MASK         (3 << HF_IOPL_SHIFT)
 #define HF_LMA_MASK          (1 << HF_LMA_SHIFT)
 #define HF_CS64_MASK         (1 << HF_CS64_SHIFT)
 #define HF_OSFXSR_MASK       (1 << HF_OSFXSR_SHIFT)
+#define HF_VM_MASK           (1 << HF_VM_SHIFT)
 #define HF_SMM_MASK          (1 << HF_SMM_SHIFT)
 #define HF_SVME_MASK         (1 << HF_SVME_SHIFT)
@@ -197 +199 @@
 #define HF2_NMI_MASK          (1 << HF2_NMI_SHIFT)
 #define HF2_VINTR_MASK        (1 << HF2_VINTR_SHIFT)
+
+#define CR0_PE_SHIFT 0
+#define CR0_MP_SHIFT 1
 
 #define CR0_PE_MASK  (1 << 0)
@@ -216 +221 @@
 #define CR4_PGE_MASK  (1 << 7)
 #define CR4_PCE_MASK  (1 << 8)
-#define CR4_OSFXSR_MASK (1 << 9)
+#define CR4_OSFXSR_SHIFT 9
+#define CR4_OSFXSR_MASK (1 << CR4_OSFXSR_SHIFT)
 #define CR4_OSXMMEXCPT_MASK  (1 << 10)
+
+#define DR6_BD          (1 << 13)
+#define DR6_BS          (1 << 14)
+#define DR6_BT          (1 << 15)
+#define DR6_FIXED_1     0xffff0ff0
+
+#define DR7_GD          (1 << 13)
+#define DR7_TYPE_SHIFT  16
+#define DR7_LEN_SHIFT   18
+#define DR7_FIXED_1     0x00000400
 
 #define PG_PRESENT_BIT  0
@@ -249 +265 @@
 #define PG_ERROR_I_D_MASK  0x10
 
+#define MSR_IA32_TSC                    0x10
 #define MSR_IA32_APICBASE               0x1b
 #define MSR_IA32_APICBASE_BSP           (1<<8)
@@ -254 +271 @@
 #define MSR_IA32_APICBASE_BASE          (0xfffff<<12)
 
+#define MSR_MTRRcap                     0xfe
+#define MSR_MTRRcap_VCNT                8
+#define MSR_MTRRcap_FIXRANGE_SUPPORT    (1 << 8)
+#define MSR_MTRRcap_WC_SUPPORTED        (1 << 10)
+
 #define MSR_IA32_SYSENTER_CS            0x174
 #define MSR_IA32_SYSENTER_ESP           0x175
@@ -264 +286 @@
 #define MSR_IA32_PERF_STATUS            0x198
 
+#define MSR_MTRRphysBase(reg)           (0x200 + 2 * (reg))
+#define MSR_MTRRphysMask(reg)           (0x200 + 2 * (reg) + 1)
+
+#define MSR_MTRRfix64K_00000            0x250
+#define MSR_MTRRfix16K_80000            0x258
+#define MSR_MTRRfix16K_A0000            0x259
+#define MSR_MTRRfix4K_C0000             0x268
+#define MSR_MTRRfix4K_C8000             0x269
+#define MSR_MTRRfix4K_D0000             0x26a
+#define MSR_MTRRfix4K_D8000             0x26b
+#define MSR_MTRRfix4K_E0000             0x26c
+#define MSR_MTRRfix4K_E8000             0x26d
+#define MSR_MTRRfix4K_F0000             0x26e
+#define MSR_MTRRfix4K_F8000             0x26f
+
 #define MSR_PAT                         0x277
+
+#define MSR_MTRRdefType                 0x2ff
 
 #define MSR_EFER                        0xc0000080
@@ -380 +419 @@
 
 #define EXCP00_DIVZ     0
-#define EXCP01_SSTP     1
+#define EXCP01_DB       1
 #define EXCP02_NMI      2
 #define EXCP03_INT3     3
@@ -618 +657 @@
 #endif
 
+    uint64_t tsc;
+
     uint64_t pat;
 
@@ -625 +666 @@
     target_ulong exception_next_eip;
     target_ulong dr[8]; /* debug registers */
+    union {
+        CPUBreakpoint *cpu_breakpoint[4];
+        CPUWatchpoint *cpu_watchpoint[4];
+    }; /* break/watchpoints for dr[0..3] */
     uint32_t smbase;
     int old_exception;  /* exception in flight */
@@ -662 +707 @@
 
 #ifndef VBOX
+    /* MTRRs */
+    uint64_t mtrr_fixed[11];
+    uint64_t mtrr_deftype;
+    struct {
+        uint64_t base;
+        uint64_t mask;
+    } mtrr_var[8];
+
 #ifdef USE_KQEMU
     int kqemu_enabled;
     int last_io_time;
 #endif
+
+    /* For KVM */
+    uint64_t interrupt_bitmap[256 / 64];
+
     /* in order to simplify APIC support, we leave this pointer to the
        user */
@@ -874 +931 @@
 }
 
+/* op_helper.c */
 /* used for debug or cpu save/restore */
 void cpu_get_fp80(uint64_t *pmant, uint16_t *pexp, CPU86_LDouble f);
 CPU86_LDouble cpu_set_fp80(uint64_t mant, uint16_t upper);
 
+/* cpu-exec.c */
 /* the following helpers are only usable in user mode simulation as
    they can trigger unexpected exceptions */
@@ -889 +948 @@
 int cpu_x86_signal_handler(int host_signum, void *pinfo,
                            void *puc);
+
+/* helper.c */
+int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
+                             int is_write, int mmu_idx, int is_softmmu);
 void cpu_x86_set_a20(CPUX86State *env, int a20_state);
-
-uint64_t cpu_get_tsc(CPUX86State *env);
-
+void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
+                   uint32_t *eax, uint32_t *ebx,
+                   uint32_t *ecx, uint32_t *edx);
+
+static inline int hw_breakpoint_enabled(unsigned long dr7, int index)
+{
+    return (dr7 >> (index * 2)) & 3;
+}
+
+static inline int hw_breakpoint_type(unsigned long dr7, int index)
+{
+    return (dr7 >> (DR7_TYPE_SHIFT + (index * 2))) & 3;
+}
+
+static inline int hw_breakpoint_len(unsigned long dr7, int index)
+{
+    int len = ((dr7 >> (DR7_LEN_SHIFT + (index * 2))) & 3);
+    return (len == 2) ? 8 : len + 1;
+}
+
+void hw_breakpoint_insert(CPUX86State *env, int index);
+void hw_breakpoint_remove(CPUX86State *env, int index);
+int check_hw_breakpoints(CPUX86State *env, int force_dr6_update);
+
+/* will be suppressed */
+void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
+void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
+void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
+
+/* hw/apic.c */
 void cpu_set_apic_base(CPUX86State *env, uint64_t val);
 uint64_t cpu_get_apic_base(CPUX86State *env);
@@ -899 +989 @@
 uint8_t cpu_get_apic_tpr(CPUX86State *env);
 #endif
-#ifdef VBOX
-int cpu_rdmsr(CPUX86State *env, uint32_t idMsr, uint64_t *puValue);
-int cpu_wrmsr(CPUX86State *env, uint32_t idMsr, uint64_t uValue);
-#endif
+
+/* hw/pc.c */
 void cpu_smm_update(CPUX86State *env);
-
-/* will be suppressed */
-void cpu_x86_update_cr0(CPUX86State *env, uint32_t new_cr0);
+uint64_t cpu_get_tsc(CPUX86State *env);
 
 /* used to debug */
@@ -922 +1008 @@
 
 #ifdef VBOX
+int  cpu_rdmsr(CPUX86State *env, uint32_t idMsr, uint64_t *puValue);
+int  cpu_wrmsr(CPUX86State *env, uint32_t idMsr, uint64_t uValue);
 void cpu_trap_raw(CPUX86State *env1);
 
@@ -948 +1036 @@
 #define cpu_list x86_cpu_list
 
-#define CPU_SAVE_VERSION 7
+#define CPU_SAVE_VERSION 8
 
 /* MMU modes definitions */
@@ -959 +1047 @@
 }
 
+/* translate.c */
 void optimize_flags_init(void);
 
@@ -965 +1054 @@
     int (*compute_c)(void);  /* return the C flag */
 } CCTable;
-
-extern CCTable cc_table[];
 
 #if defined(CONFIG_USER_ONLY)
@@ -977 +1064 @@
 #endif
 
-#define CPU_PC_FROM_TB(env, tb) env->eip = tb->pc - tb->cs_base
-
 #include "cpu-all.h"
+#include "exec-all.h"
 
 #include "svm.h"
 
+static inline void cpu_pc_from_tb(CPUState *env, TranslationBlock *tb)
+{
+    env->eip = tb->pc - tb->cs_base;
+}
+
+static inline void cpu_get_tb_cpu_state(CPUState *env, target_ulong *pc,
+                                        target_ulong *cs_base, int *flags)
+{
+    *cs_base = env->segs[R_CS].base;
+    *pc = *cs_base + env->eip;
+    *flags = env->hflags | (env->eflags & (IOPL_MASK | TF_MASK | VM_MASK));
+}
+
 #endif /* CPU_I386_H */

trunk/src/recompiler/target-i386/exec.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -42 +42 @@
 register struct CPUX86State *env asm(AREG0);
 
+#include "qemu-common.h"
 #include "qemu-log.h"
 
@@ -68 +69 @@
 #include "exec-all.h"
 
-void cpu_x86_update_cr3(CPUX86State *env, target_ulong new_cr3);
-void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
-int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
-                             int is_write, int mmu_idx, int is_softmmu);
-void __hidden cpu_lock(void);
-void __hidden cpu_unlock(void);
+/* op_helper.c */
 void do_interrupt(int intno, int is_int, int error_code,
                   target_ulong next_eip, int is_hw);
 void do_interrupt_user(int intno, int is_int, int error_code,
                        target_ulong next_eip);
-void raise_interrupt(int intno, int is_int, int error_code,
-                     int next_eip_addend);
-void raise_exception_err(int exception_index, int error_code);
-void raise_exception(int exception_index);
+void QEMU_NORETURN raise_exception_err(int exception_index, int error_code);
+void QEMU_NORETURN raise_exception(int exception_index);
 void do_smm_enter(void);
-void __hidden cpu_loop_exit(void);
-
-void OPPROTO op_movl_eflags_T0(void);
-void OPPROTO op_movl_T0_eflags(void);
-#ifdef VBOX
-void OPPROTO op_movl_T0_eflags_vme(void);
-void OPPROTO op_movw_eflags_T0_vme(void);
-void OPPROTO op_cli_vme(void);
-void OPPROTO op_sti_vme(void);
-#endif
 
 /* n must be a constant to be efficient */
@@ -331 +315 @@
 #define FPUC_EM 0x3f
 
-extern const CPU86_LDouble f15rk[7];
-
-void fpu_raise_exception(void);
-void restore_native_fp_state(CPUState *env);
-void save_native_fp_state(CPUState *env);
-
-extern const uint8_t parity_table[256];
-extern const uint8_t rclw_table[32];
-extern const uint8_t rclb_table[32];
-
 static inline uint32_t compute_eflags(void)
 {
-    return env->eflags | cc_table[CC_OP].compute_all() | (DF & DF_MASK);
+    return env->eflags | helper_cc_compute_all(CC_OP) | (DF & DF_MASK);
 }
 

trunk/src/recompiler/target-i386/helper.c

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -33 +33 @@
 #include <string.h>
 #ifndef VBOX
-# include <inttypes.h>
-# include <signal.h>
-# include <assert.h>
-#endif
+#include <inttypes.h>
+#include <signal.h>
+#include <assert.h>
+#endif /* !VBOX */
 
 #include "cpu.h"
 #include "exec-all.h"
-#include "svm.h"
 #include "qemu-common.h"
+#include "kvm.h"
 
 //#define DEBUG_MMU
-
-static int cpu_x86_register (CPUX86State *env, const char *cpu_model);
 
 #ifndef VBOX
@@ -71 +69 @@
     static const char *ext2_feature_name[] = {
        "fpu", "vme", "de", "pse", "tsc", "msr", "pae", "mce",
-       "cx8" /* AMD CMPXCHG8B */, "apic", NULL, "syscall", "mttr", "pge", "mca", "cmov",
+       "cx8" /* AMD CMPXCHG8B */, "apic", NULL, "syscall", "mtrr", "pge", "mca", "cmov",
        "pat", "pse36", NULL, NULL /* Linux mp */, "nx" /* Intel xd */, NULL, "mmxext", "mmx",
        "fxsr", "fxsr_opt" /* AMD ffxsr */, "pdpe1gb" /* AMD Page1GB */, "rdtscp", NULL, "lm" /* Intel 64 */, "3dnowext", "3dnow",
@@ -105 +103 @@
 }
 #endif /* !VBOX */
-
-#ifndef VBOX
-CPUX86State *cpu_x86_init(const char *cpu_model)
-#else
-CPUX86State *cpu_x86_init(CPUX86State *env, const char *cpu_model)
-#endif
-{
-#ifndef VBOX
-    CPUX86State *env;
-#endif
-    static int inited;
-
-#ifndef VBOX
-    env = qemu_mallocz(sizeof(CPUX86State));
-    if (!env)
-        return NULL;
-#endif
-    cpu_exec_init(env);
-    env->cpu_model_str = cpu_model;
-
-    /* init various static tables */
-    if (!inited) {
-        inited = 1;
-        optimize_flags_init();
-    }
-    if (cpu_x86_register(env, cpu_model) < 0) {
-        cpu_x86_close(env);
-        return NULL;
-    }
-    cpu_reset(env);
-#ifdef USE_KQEMU
-    kqemu_init(env);
-#endif
-    return env;
-}
 
 typedef struct x86_def_t {
@@ -188 +151 @@
         .xlevel = 0x8000000A,
         .model_id = "QEMU Virtual CPU version " QEMU_VERSION,
+    },
+    {
+        .name = "phenom",
+        .level = 5,
+        .vendor1 = CPUID_VENDOR_AMD_1,
+        .vendor2 = CPUID_VENDOR_AMD_2,
+        .vendor3 = CPUID_VENDOR_AMD_3,
+        .family = 16,
+        .model = 2,
+        .stepping = 3,
+        /* Missing: CPUID_VME, CPUID_HT */
+        .features = PPRO_FEATURES |
+            CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
+            CPUID_PSE36,
+        /* Missing: CPUID_EXT_CX16, CPUID_EXT_POPCNT */
+        .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR,
+        /* Missing: CPUID_EXT2_PDPE1GB, CPUID_EXT2_RDTSCP */
+        .ext2_features = (PPRO_FEATURES & 0x0183F3FF) |
+            CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX |
+            CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT | CPUID_EXT2_MMXEXT |
+            CPUID_EXT2_FFXSR,
+        /* Missing: CPUID_EXT3_LAHF_LM, CPUID_EXT3_CMP_LEG, CPUID_EXT3_EXTAPIC,
+                    CPUID_EXT3_CR8LEG, CPUID_EXT3_ABM, CPUID_EXT3_SSE4A,
+                    CPUID_EXT3_MISALIGNSSE, CPUID_EXT3_3DNOWPREFETCH,
+                    CPUID_EXT3_OSVW, CPUID_EXT3_IBS */
+        .ext3_features = CPUID_EXT3_SVM,
+        .xlevel = 0x8000001A,
+        .model_id = "AMD Phenom(tm) 9550 Quad-Core Processor"
     },
     {
@@ -305 +296 @@
             /* Some CPUs got no CPUID_SEP */
         .ext_features = CPUID_EXT_MONITOR |
-            CPUID_EXT_SSE3 /* PNI */, CPUID_EXT_SSSE3,
+            CPUID_EXT_SSE3 /* PNI */ | CPUID_EXT_SSSE3,
             /* Missing: CPUID_EXT_DSCPL | CPUID_EXT_EST |
              * CPUID_EXT_TM2 | CPUID_EXT_XTPR */
@@ -327 +318 @@
 
     def = NULL;
-    for (i = 0; i < sizeof(x86_defs) / sizeof(x86_def_t); i++) {
+    for (i = 0; i < ARRAY_SIZE(x86_defs); i++) {
         if (strcmp(name, x86_defs[i].name) == 0) {
             def = &x86_defs[i];
@@ -358 +349 @@
                 char *err;
                 model = strtol(val, &err, 10);
-                if (!*val || *err || model < 0 || model > 0xf) {
+                if (!*val || *err || model < 0 || model > 0xff) {
                     fprintf(stderr, "bad numerical value %s\n", val);
                     goto error;
@@ -417 +408 @@
     unsigned int i;
 
-    for (i = 0; i < sizeof(x86_defs) / sizeof(x86_def_t); i++)
+    for (i = 0; i < ARRAY_SIZE(x86_defs); i++)
         (*cpu_fprintf)(f, "x86 %16s\n", x86_defs[i].name);
 }
@@ -439 +430 @@
     }
     env->cpuid_level = def->level;
-    env->cpuid_version = (def->family << 8) | (def->model << 4) | def->stepping;
+    if (def->family > 0x0f)
+        env->cpuid_version = 0xf00 | ((def->family - 0x0f) << 20);
+    else
+        env->cpuid_version = def->family << 8;
+    env->cpuid_version |= ((def->model & 0xf) << 4) | ((def->model >> 4) << 16);
+    env->cpuid_version |= def->stepping;
     env->cpuid_features = def->features;
     env->pat = 0x0007040600070406ULL;
@@ -469 +465 @@
     int i;
 
+    if (qemu_loglevel_mask(CPU_LOG_RESET)) {
+        qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
+        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
+    }
+
     memset(env, 0, offsetof(CPUX86State, breakpoints));
 
@@ -522 +523 @@
 
     env->mxcsr = 0x1f80;
+
+    memset(env->dr, 0, sizeof(env->dr));
+    env->dr[6] = DR6_FIXED_1;
+    env->dr[7] = DR7_FIXED_1;
+    cpu_breakpoint_remove_all(env, BP_CPU);
+    cpu_watchpoint_remove_all(env, BP_CPU);
 }
 
@@ -596 +603 @@
     char cc_op_name[32];
     static const char *seg_name[6] = { "ES", "CS", "SS", "DS", "FS", "GS" };
+
+    if (kvm_enabled())
+        kvm_arch_get_registers(env);
 
     eflags = env->eflags;
@@ -694 +704 @@
                     env->cr[3],
                     (uint32_t)env->cr[4]);
+        for(i = 0; i < 4; i++)
+            cpu_fprintf(f, "DR%d=%016" PRIx64 " ", i, env->dr[i]);
+        cpu_fprintf(f, "\nDR6=%016" PRIx64 " DR7=%016" PRIx64 "\n",
+                    env->dr[6], env->dr[7]);
     } else
 #endif
@@ -725 +739 @@
                     (uint32_t)env->cr[3],
                     (uint32_t)env->cr[4]);
+        for(i = 0; i < 4; i++)
+            cpu_fprintf(f, "DR%d=%08x ", i, env->dr[i]);
+        cpu_fprintf(f, "\nDR6=%08x DR7=%08x\n", env->dr[6], env->dr[7]);
     }
     if (flags & X86_DUMP_CCOP) {
@@ -899 +916 @@
 }
 
-/* XXX: also flush 4MB pages */
-void cpu_x86_flush_tlb(CPUX86State *env, target_ulong addr)
-{
-    tlb_flush_page(env, addr);
-}
-
 #if defined(CONFIG_USER_ONLY)
 
@@ -1327 +1338 @@
     return paddr;
 }
+
+void hw_breakpoint_insert(CPUState *env, int index)
+{
+    int type, err = 0;
+
+    switch (hw_breakpoint_type(env->dr[7], index)) {
+    case 0:
+        if (hw_breakpoint_enabled(env->dr[7], index))
+            err = cpu_breakpoint_insert(env, env->dr[index], BP_CPU,
+                                        &env->cpu_breakpoint[index]);
+        break;
+    case 1:
+        type = BP_CPU | BP_MEM_WRITE;
+        goto insert_wp;
+    case 2:
+         /* No support for I/O watchpoints yet */
+        break;
+    case 3:
+        type = BP_CPU | BP_MEM_ACCESS;
+    insert_wp:
+        err = cpu_watchpoint_insert(env, env->dr[index],
+                                    hw_breakpoint_len(env->dr[7], index),
+                                    type, &env->cpu_watchpoint[index]);
+        break;
+    }
+    if (err)
+        env->cpu_breakpoint[index] = NULL;
+}
+
+void hw_breakpoint_remove(CPUState *env, int index)
+{
+    if (!env->cpu_breakpoint[index])
+        return;
+    switch (hw_breakpoint_type(env->dr[7], index)) {
+    case 0:
+        if (hw_breakpoint_enabled(env->dr[7], index))
+            cpu_breakpoint_remove_by_ref(env, env->cpu_breakpoint[index]);
+        break;
+    case 1:
+    case 3:
+        cpu_watchpoint_remove_by_ref(env, env->cpu_watchpoint[index]);
+        break;
+    case 2:
+        /* No support for I/O watchpoints yet */
+        break;
+    }
+}
+
+int check_hw_breakpoints(CPUState *env, int force_dr6_update)
+{
+    target_ulong dr6;
+    int reg, type;
+    int hit_enabled = 0;
+
+    dr6 = env->dr[6] & ~0xf;
+    for (reg = 0; reg < 4; reg++) {
+        type = hw_breakpoint_type(env->dr[7], reg);
+        if ((type == 0 && env->dr[reg] == env->eip) ||
+            ((type & 1) && env->cpu_watchpoint[reg] &&
+             (env->cpu_watchpoint[reg]->flags & BP_WATCHPOINT_HIT))) {
+            dr6 |= 1 << reg;
+            if (hw_breakpoint_enabled(env->dr[7], reg))
+                hit_enabled = 1;
+        }
+    }
+    if (hit_enabled || force_dr6_update)
+        env->dr[6] = dr6;
+    return hit_enabled;
+}
+
+static CPUDebugExcpHandler *prev_debug_excp_handler;
+
+void raise_exception(int exception_index);
+
+static void breakpoint_handler(CPUState *env)
+{
+    CPUBreakpoint *bp;
+
+    if (env->watchpoint_hit) {
+        if (env->watchpoint_hit->flags & BP_CPU) {
+            env->watchpoint_hit = NULL;
+            if (check_hw_breakpoints(env, 0))
+                raise_exception(EXCP01_DB);
+            else
+                cpu_resume_from_signal(env, NULL);
+        }
+    } else {
+        TAILQ_FOREACH(bp, &env->breakpoints, entry)
+            if (bp->pc == env->eip) {
+                if (bp->flags & BP_CPU) {
+                    check_hw_breakpoints(env, 1);
+                    raise_exception(EXCP01_DB);
+                }
+                break;
+            }
+    }
+    if (prev_debug_excp_handler)
+        prev_debug_excp_handler(env);
+}
 #endif /* !CONFIG_USER_ONLY */
+
+#ifndef VBOX
+static void host_cpuid(uint32_t function, uint32_t count,
+                       uint32_t *eax, uint32_t *ebx,
+                       uint32_t *ecx, uint32_t *edx)
+{
+#if defined(CONFIG_KVM)
+    uint32_t vec[4];
+
+#ifdef __x86_64__
+    asm volatile("cpuid"
+                 : "=a"(vec[0]), "=b"(vec[1]),
+                   "=c"(vec[2]), "=d"(vec[3])
+                 : "0"(function), "c"(count) : "cc");
+#else
+    asm volatile("pusha \n\t"
+                 "cpuid \n\t"
+                 "mov %%eax, 0(%1) \n\t"
+                 "mov %%ebx, 4(%1) \n\t"
+                 "mov %%ecx, 8(%1) \n\t"
+                 "mov %%edx, 12(%1) \n\t"
+                 "popa"
+                 : : "a"(function), "c"(count), "S"(vec)
+                 : "memory", "cc");
+#endif
+
+    if (eax)
+        *eax = vec[0];
+    if (ebx)
+        *ebx = vec[1];
+    if (ecx)
+        *ecx = vec[2];
+    if (edx)
+        *edx = vec[3];
+#endif
+}
+
+void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
+                   uint32_t *eax, uint32_t *ebx,
+                   uint32_t *ecx, uint32_t *edx)
+{
+    /* test if maximum index reached */
+    if (index & 0x80000000) {
+        if (index > env->cpuid_xlevel)
+            index = env->cpuid_level;
+    } else {
+        if (index > env->cpuid_level)
+            index = env->cpuid_level;
+    }
+
+    switch(index) {
+    case 0:
+        *eax = env->cpuid_level;
+        *ebx = env->cpuid_vendor1;
+        *edx = env->cpuid_vendor2;
+        *ecx = env->cpuid_vendor3;
+
+        /* sysenter isn't supported on compatibility mode on AMD.  and syscall
+         * isn't supported in compatibility mode on Intel.  so advertise the
+         * actuall cpu, and say goodbye to migration between different vendors
+         * is you use compatibility mode. */
+        if (kvm_enabled())
+            host_cpuid(0, 0, NULL, ebx, ecx, edx);
+        break;
+    case 1:
+        *eax = env->cpuid_version;
+        *ebx = (env->cpuid_apic_id << 24) | 8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
+        *ecx = env->cpuid_ext_features;
+        *edx = env->cpuid_features;
+
+        /* "Hypervisor present" bit required for Microsoft SVVP */
+        if (kvm_enabled())
+            *ecx |= (1 << 31);
+        break;
+    case 2:
+        /* cache info: needed for Pentium Pro compatibility */
+        *eax = 1;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0x2c307d;
+        break;
+    case 4:
+        /* cache info: needed for Core compatibility */
+        switch (count) {
+            case 0: /* L1 dcache info */
+                *eax = 0x0000121;
+                *ebx = 0x1c0003f;
+                *ecx = 0x000003f;
+                *edx = 0x0000001;
+                break;
+            case 1: /* L1 icache info */
+                *eax = 0x0000122;
+                *ebx = 0x1c0003f;
+                *ecx = 0x000003f;
+                *edx = 0x0000001;
+                break;
+            case 2: /* L2 cache info */
+                *eax = 0x0000143;
+                *ebx = 0x3c0003f;
+                *ecx = 0x0000fff;
+                *edx = 0x0000001;
+                break;
+            default: /* end of info */
+                *eax = 0;
+                *ebx = 0;
+                *ecx = 0;
+                *edx = 0;
+                break;
+        }
+        break;
+    case 5:
+        /* mwait info: needed for Core compatibility */
+        *eax = 0; /* Smallest monitor-line size in bytes */
+        *ebx = 0; /* Largest monitor-line size in bytes */
+        *ecx = CPUID_MWAIT_EMX | CPUID_MWAIT_IBE;
+        *edx = 0;
+        break;
+    case 6:
+        /* Thermal and Power Leaf */
+        *eax = 0;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        break;
+    case 9:
+        /* Direct Cache Access Information Leaf */
+        *eax = 0; /* Bits 0-31 in DCA_CAP MSR */
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        break;
+    case 0xA:
+        /* Architectural Performance Monitoring Leaf */
+        *eax = 0;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        break;
+    case 0x80000000:
+        *eax = env->cpuid_xlevel;
+        *ebx = env->cpuid_vendor1;
+        *edx = env->cpuid_vendor2;
+        *ecx = env->cpuid_vendor3;
+        break;
+    case 0x80000001:
+        *eax = env->cpuid_features;
+        *ebx = 0;
+        *ecx = env->cpuid_ext3_features;
+        *edx = env->cpuid_ext2_features;
+
+        if (kvm_enabled()) {
+            uint32_t h_eax, h_edx;
+
+            host_cpuid(index, 0, &h_eax, NULL, NULL, &h_edx);
+
+            /* disable CPU features that the host does not support */
+
+            /* long mode */
+            if ((h_edx & 0x20000000) == 0 /* || !lm_capable_kernel */)
+                *edx &= ~0x20000000;
+            /* syscall */
+            if ((h_edx & 0x00000800) == 0)
+                *edx &= ~0x00000800;
+            /* nx */
+            if ((h_edx & 0x00100000) == 0)
+                *edx &= ~0x00100000;
+
+            /* disable CPU features that KVM cannot support */
+
+            /* svm */
+            *ecx &= ~4UL;
+            /* 3dnow */
+            *edx &= ~0xc0000000;
+        }
+        break;
+    case 0x80000002:
+    case 0x80000003:
+    case 0x80000004:
+        *eax = env->cpuid_model[(index - 0x80000002) * 4 + 0];
+        *ebx = env->cpuid_model[(index - 0x80000002) * 4 + 1];
+        *ecx = env->cpuid_model[(index - 0x80000002) * 4 + 2];
+        *edx = env->cpuid_model[(index - 0x80000002) * 4 + 3];
+        break;
+    case 0x80000005:
+        /* cache info (L1 cache) */
+        *eax = 0x01ff01ff;
+        *ebx = 0x01ff01ff;
+        *ecx = 0x40020140;
+        *edx = 0x40020140;
+        break;
+    case 0x80000006:
+        /* cache info (L2 cache) */
+        *eax = 0;
+        *ebx = 0x42004200;
+        *ecx = 0x02008140;
+        *edx = 0;
+        break;
+    case 0x80000008:
+        /* virtual & phys address size in low 2 bytes. */
+/* XXX: This value must match the one used in the MMU code. */
+        if (env->cpuid_ext2_features & CPUID_EXT2_LM) {
+            /* 64 bit processor */
+#if defined(USE_KQEMU)
+            *eax = 0x00003020;  /* 48 bits virtual, 32 bits physical */
+#else
+/* XXX: The physical address space is limited to 42 bits in exec.c. */
+            *eax = 0x00003028;  /* 48 bits virtual, 40 bits physical */
+#endif
+        } else {
+#if defined(USE_KQEMU)
+            *eax = 0x00000020;  /* 32 bits physical */
+#else
+            if (env->cpuid_features & CPUID_PSE36)
+                *eax = 0x00000024; /* 36 bits physical */
+            else
+                *eax = 0x00000020; /* 32 bits physical */
+#endif
+        }
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        break;
+    case 0x8000000A:
+        *eax = 0x00000001; /* SVM Revision */
+        *ebx = 0x00000010; /* nr of ASIDs */
+        *ecx = 0;
+        *edx = 0; /* optional features */
+        break;
+    default:
+        /* reserved values: zero */
+        *eax = 0;
+        *ebx = 0;
+        *ecx = 0;
+        *edx = 0;
+        break;
+    }
+}
+#endif /* !VBOX */
+
+#ifndef VBOX
+CPUX86State *cpu_x86_init(const char *cpu_model)
+#else
+CPUX86State *cpu_x86_init(CPUX86State *env, const char *cpu_model)
+#endif
+{
+#ifndef VBOX
+    CPUX86State *env;
+#endif
+    static int inited;
+
+#ifndef VBOX
+    env = qemu_mallocz(sizeof(CPUX86State));
+#endif
+    cpu_exec_init(env);
+    env->cpu_model_str = cpu_model;
+
+    /* init various static tables */
+    if (!inited) {
+        inited = 1;
+        optimize_flags_init();
+#ifndef CONFIG_USER_ONLY
+        prev_debug_excp_handler =
+            cpu_set_debug_excp_handler(breakpoint_handler);
+#endif
+    }
+    if (cpu_x86_register(env, cpu_model) < 0) {
+        cpu_x86_close(env);
+        return NULL;
+    }
+    cpu_reset(env);
+#ifdef USE_KQEMU
+    kqemu_init(env);
+#endif
+    if (kvm_enabled())
+        kvm_init_vcpu(env);
+    return env;
+}

trunk/src/recompiler/target-i386/helper.h

@@ -1 @@
-#ifndef DEF_HELPER
-#define DEF_HELPER(ret, name, params) ret name params;
-#endif
-
-DEF_HELPER(void, helper_lock, (void))
-DEF_HELPER(void, helper_unlock, (void))
-DEF_HELPER(void, helper_write_eflags, (target_ulong t0, uint32_t update_mask))
-DEF_HELPER(target_ulong, helper_read_eflags, (void))
-#ifdef VBOX
-DEF_HELPER(void, helper_write_eflags_vme, (target_ulong t0))
-DEF_HELPER(target_ulong, helper_read_eflags_vme, (void))
-#endif
-DEF_HELPER(void, helper_divb_AL, (target_ulong t0))
-DEF_HELPER(void, helper_idivb_AL, (target_ulong t0))
-DEF_HELPER(void, helper_divw_AX, (target_ulong t0))
-DEF_HELPER(void, helper_idivw_AX, (target_ulong t0))
-DEF_HELPER(void, helper_divl_EAX, (target_ulong t0))
-DEF_HELPER(void, helper_idivl_EAX, (target_ulong t0))
-#ifdef TARGET_X86_64
-DEF_HELPER(void, helper_mulq_EAX_T0, (target_ulong t0))
-DEF_HELPER(void, helper_imulq_EAX_T0, (target_ulong t0))
-DEF_HELPER(target_ulong, helper_imulq_T0_T1, (target_ulong t0, target_ulong t1))
-DEF_HELPER(void, helper_divq_EAX, (target_ulong t0))
-DEF_HELPER(void, helper_idivq_EAX, (target_ulong t0))
-#endif
-
-DEF_HELPER(void, helper_aam, (int base))
-DEF_HELPER(void, helper_aad, (int base))
-DEF_HELPER(void, helper_aaa, (void))
-DEF_HELPER(void, helper_aas, (void))
-DEF_HELPER(void, helper_daa, (void))
-DEF_HELPER(void, helper_das, (void))
-
-DEF_HELPER(target_ulong, helper_lsl, (target_ulong selector1))
-DEF_HELPER(target_ulong, helper_lar, (target_ulong selector1))
-DEF_HELPER(void, helper_verr, (target_ulong selector1))
-DEF_HELPER(void, helper_verw, (target_ulong selector1))
-DEF_HELPER(void, helper_lldt, (int selector))
-DEF_HELPER(void, helper_ltr, (int selector))
-DEF_HELPER(void, helper_load_seg, (int seg_reg, int selector))
-DEF_HELPER(void, helper_ljmp_protected, (int new_cs, target_ulong new_eip,
-                           int next_eip_addend))
-DEF_HELPER(void, helper_lcall_real, (int new_cs, target_ulong new_eip1,
-                       int shift, int next_eip))
-DEF_HELPER(void, helper_lcall_protected, (int new_cs, target_ulong new_eip,
-                            int shift, int next_eip_addend))
-DEF_HELPER(void, helper_iret_real, (int shift))
-DEF_HELPER(void, helper_iret_protected, (int shift, int next_eip))
-DEF_HELPER(void, helper_lret_protected, (int shift, int addend))
-DEF_HELPER(target_ulong, helper_read_crN, (int reg))
-DEF_HELPER(void, helper_write_crN, (int reg, target_ulong t0))
-DEF_HELPER(void, helper_lmsw, (target_ulong t0))
-DEF_HELPER(void, helper_clts, (void))
-DEF_HELPER(void, helper_movl_drN_T0, (int reg, target_ulong t0))
-DEF_HELPER(void, helper_invlpg, (target_ulong addr))
-
-DEF_HELPER(void, helper_enter_level, (int level, int data32, target_ulong t1))
-#ifdef TARGET_X86_64
-DEF_HELPER(void, helper_enter64_level, (int level, int data64, target_ulong t1))
-#endif
-DEF_HELPER(void, helper_sysenter, (void))
-DEF_HELPER(void, helper_sysexit, (int dflag))
-#ifdef TARGET_X86_64
-DEF_HELPER(void, helper_syscall, (int next_eip_addend))
-DEF_HELPER(void, helper_sysret, (int dflag))
-#endif
-DEF_HELPER(void, helper_hlt, (int next_eip_addend))
-DEF_HELPER(void, helper_monitor, (target_ulong ptr))
-DEF_HELPER(void, helper_mwait, (int next_eip_addend))
-DEF_HELPER(void, helper_debug, (void))
-DEF_HELPER(void, helper_raise_interrupt, (int intno, int next_eip_addend))
-DEF_HELPER(void, helper_raise_exception, (int exception_index))
-DEF_HELPER(void, helper_cli, (void))
-DEF_HELPER(void, helper_sti, (void))
-#ifdef VBOX
-DEF_HELPER(void, helper_cli_vme, (void))
-DEF_HELPER(void, helper_sti_vme, (void))
-#endif
-DEF_HELPER(void, helper_set_inhibit_irq, (void))
-DEF_HELPER(void, helper_reset_inhibit_irq, (void))
-DEF_HELPER(void, helper_boundw, (target_ulong a0, int v))
-DEF_HELPER(void, helper_boundl, (target_ulong a0, int v))
-DEF_HELPER(void, helper_rsm, (void))
-DEF_HELPER(void, helper_into, (int next_eip_addend))
-DEF_HELPER(void, helper_cmpxchg8b, (target_ulong a0))
-#ifdef TARGET_X86_64
-DEF_HELPER(void, helper_cmpxchg16b, (target_ulong a0))
-#endif
-DEF_HELPER(void, helper_single_step, (void))
-DEF_HELPER(void, helper_cpuid, (void))
-DEF_HELPER(void, helper_rdtsc, (void))
-DEF_HELPER(void, helper_rdpmc, (void))
-DEF_HELPER(void, helper_rdmsr, (void))
-DEF_HELPER(void, helper_wrmsr, (void))
-#ifdef VBOX
-DEF_HELPER(void, helper_rdtscp, (void))
-#endif
-
-DEF_HELPER(void, helper_check_iob, (uint32_t t0))
-DEF_HELPER(void, helper_check_iow, (uint32_t t0))
-DEF_HELPER(void, helper_check_iol, (uint32_t t0))
-#ifdef VBOX
-DEF_HELPER(void, helper_check_external_event, (void))
-DEF_HELPER(void, helper_dump_state, (void))
-DEF_HELPER(void, helper_sync_seg, (uint32_t t0))
-#endif
-DEF_HELPER(void, helper_outb, (uint32_t port, uint32_t data))
-DEF_HELPER(target_ulong, helper_inb, (uint32_t port))
-DEF_HELPER(void, helper_outw, (uint32_t port, uint32_t data))
-DEF_HELPER(target_ulong, helper_inw, (uint32_t port))
-DEF_HELPER(void, helper_outl, (uint32_t port, uint32_t data))
-DEF_HELPER(target_ulong, helper_inl, (uint32_t port))
-
-DEF_HELPER(void, helper_svm_check_intercept_param, (uint32_t type, uint64_t param))
-DEF_HELPER(void, helper_vmexit, (uint32_t exit_code, uint64_t exit_info_1))
-DEF_HELPER(void, helper_svm_check_io, (uint32_t port, uint32_t param,
-                         uint32_t next_eip_addend))
-DEF_HELPER(void, helper_vmrun, (int aflag, int next_eip_addend))
-DEF_HELPER(void, helper_vmmcall, (void))
-DEF_HELPER(void, helper_vmload, (int aflag))
-DEF_HELPER(void, helper_vmsave, (int aflag))
-DEF_HELPER(void, helper_stgi, (void))
-DEF_HELPER(void, helper_clgi, (void))
-DEF_HELPER(void, helper_skinit, (void))
-DEF_HELPER(void, helper_invlpga, (int aflag))
+#include "def-helper.h"
+
+DEF_HELPER_FLAGS_1(cc_compute_all, TCG_CALL_PURE, i32, int)
+DEF_HELPER_FLAGS_1(cc_compute_c, TCG_CALL_PURE, i32, int)
+
+DEF_HELPER_0(lock, void)
+DEF_HELPER_0(unlock, void)
+DEF_HELPER_2(write_eflags, void, tl, i32)
+DEF_HELPER_0(read_eflags, tl)
+DEF_HELPER_1(divb_AL, void, tl)
+DEF_HELPER_1(idivb_AL, void, tl)
+DEF_HELPER_1(divw_AX, void, tl)
+DEF_HELPER_1(idivw_AX, void, tl)
+DEF_HELPER_1(divl_EAX, void, tl)
+DEF_HELPER_1(idivl_EAX, void, tl)
+#ifdef TARGET_X86_64
+DEF_HELPER_1(mulq_EAX_T0, void, tl)
+DEF_HELPER_1(imulq_EAX_T0, void, tl)
+DEF_HELPER_2(imulq_T0_T1, tl, tl, tl)
+DEF_HELPER_1(divq_EAX, void, tl)
+DEF_HELPER_1(idivq_EAX, void, tl)
+#endif
+
+DEF_HELPER_1(aam, void, int)
+DEF_HELPER_1(aad, void, int)
+DEF_HELPER_0(aaa, void)
+DEF_HELPER_0(aas, void)
+DEF_HELPER_0(daa, void)
+DEF_HELPER_0(das, void)
+
+DEF_HELPER_1(lsl, tl, tl)
+DEF_HELPER_1(lar, tl, tl)
+DEF_HELPER_1(verr, void, tl)
+DEF_HELPER_1(verw, void, tl)
+DEF_HELPER_1(lldt, void, int)
+DEF_HELPER_1(ltr, void, int)
+DEF_HELPER_2(load_seg, void, int, int)
+DEF_HELPER_3(ljmp_protected, void, int, tl, int)
+DEF_HELPER_4(lcall_real, void, int, tl, int, int)
+DEF_HELPER_4(lcall_protected, void, int, tl, int, int)
+DEF_HELPER_1(iret_real, void, int)
+DEF_HELPER_2(iret_protected, void, int, int)
+DEF_HELPER_2(lret_protected, void, int, int)
+DEF_HELPER_1(read_crN, tl, int)
+DEF_HELPER_2(write_crN, void, int, tl)
+DEF_HELPER_1(lmsw, void, tl)
+DEF_HELPER_0(clts, void)
+DEF_HELPER_2(movl_drN_T0, void, int, tl)
+DEF_HELPER_1(invlpg, void, tl)
+
+DEF_HELPER_3(enter_level, void, int, int, tl)
+#ifdef TARGET_X86_64
+DEF_HELPER_3(enter64_level, void, int, int, tl)
+#endif
+DEF_HELPER_0(sysenter, void)
+DEF_HELPER_1(sysexit, void, int)
+#ifdef TARGET_X86_64
+DEF_HELPER_1(syscall, void, int)
+DEF_HELPER_1(sysret, void, int)
+#endif
+DEF_HELPER_1(hlt, void, int)
+DEF_HELPER_1(monitor, void, tl)
+DEF_HELPER_1(mwait, void, int)
+DEF_HELPER_0(debug, void)
+DEF_HELPER_2(raise_interrupt, void, int, int)
+DEF_HELPER_1(raise_exception, void, int)
+DEF_HELPER_0(cli, void)
+DEF_HELPER_0(sti, void)
+DEF_HELPER_0(set_inhibit_irq, void)
+DEF_HELPER_0(reset_inhibit_irq, void)
+DEF_HELPER_2(boundw, void, tl, int)
+DEF_HELPER_2(boundl, void, tl, int)
+DEF_HELPER_0(rsm, void)
+DEF_HELPER_1(into, void, int)
+DEF_HELPER_1(cmpxchg8b, void, tl)
+#ifdef TARGET_X86_64
+DEF_HELPER_1(cmpxchg16b, void, tl)
+#endif
+DEF_HELPER_0(single_step, void)
+DEF_HELPER_0(cpuid, void)
+DEF_HELPER_0(rdtsc, void)
+DEF_HELPER_0(rdpmc, void)
+DEF_HELPER_0(rdmsr, void)
+DEF_HELPER_0(wrmsr, void)
+
+DEF_HELPER_1(check_iob, void, i32)
+DEF_HELPER_1(check_iow, void, i32)
+DEF_HELPER_1(check_iol, void, i32)
+DEF_HELPER_2(outb, void, i32, i32)
+DEF_HELPER_1(inb, tl, i32)
+DEF_HELPER_2(outw, void, i32, i32)
+DEF_HELPER_1(inw, tl, i32)
+DEF_HELPER_2(outl, void, i32, i32)
+DEF_HELPER_1(inl, tl, i32)
+
+DEF_HELPER_2(svm_check_intercept_param, void, i32, i64)
+DEF_HELPER_2(vmexit, void, i32, i64)
+DEF_HELPER_3(svm_check_io, void, i32, i32, i32)
+DEF_HELPER_2(vmrun, void, int, int)
+DEF_HELPER_0(vmmcall, void)
+DEF_HELPER_1(vmload, void, int)
+DEF_HELPER_1(vmsave, void, int)
+DEF_HELPER_0(stgi, void)
+DEF_HELPER_0(clgi, void)
+DEF_HELPER_0(skinit, void)
+DEF_HELPER_1(invlpga, void, int)
 
 /* x86 FPU */
 
-DEF_HELPER(void, helper_flds_FT0, (uint32_t val))
-DEF_HELPER(void, helper_fldl_FT0, (uint64_t val))
-DEF_HELPER(void, helper_fildl_FT0, (int32_t val))
-DEF_HELPER(void, helper_flds_ST0, (uint32_t val))
-DEF_HELPER(void, helper_fldl_ST0, (uint64_t val))
-DEF_HELPER(void, helper_fildl_ST0, (int32_t val))
-DEF_HELPER(void, helper_fildll_ST0, (int64_t val))
+DEF_HELPER_1(flds_FT0, void, i32)
+DEF_HELPER_1(fldl_FT0, void, i64)
+DEF_HELPER_1(fildl_FT0, void, s32)
+DEF_HELPER_1(flds_ST0, void, i32)
+DEF_HELPER_1(fldl_ST0, void, i64)
+DEF_HELPER_1(fildl_ST0, void, s32)
+DEF_HELPER_1(fildll_ST0, void, s64)
 #ifndef VBOX
-DEF_HELPER(uint32_t, helper_fsts_ST0, (void))
-DEF_HELPER(uint64_t, helper_fstl_ST0, (void))
-DEF_HELPER(int32_t, helper_fist_ST0, (void))
-DEF_HELPER(int32_t, helper_fistl_ST0, (void))
-DEF_HELPER(int64_t, helper_fistll_ST0, (void))
-DEF_HELPER(int32_t, helper_fistt_ST0, (void))
-DEF_HELPER(int32_t, helper_fisttl_ST0, (void))
-DEF_HELPER(int64_t, helper_fisttll_ST0, (void))
+DEF_HELPER_0(fsts_ST0, i32)
+DEF_HELPER_0(fstl_ST0, i64)
+DEF_HELPER_0(fist_ST0, s32)
+DEF_HELPER_0(fistl_ST0, s32)
+DEF_HELPER_0(fistll_ST0, s64)
+DEF_HELPER_0(fistt_ST0, s32)
+DEF_HELPER_0(fisttl_ST0, s32)
+DEF_HELPER_0(fisttll_ST0, s64)
 #else  /* VBOX */
-DEF_HELPER(RTCCUINTREG, helper_fsts_ST0, (void))
-DEF_HELPER(uint64_t, helper_fstl_ST0, (void))
-DEF_HELPER(RTCCINTREG, helper_fist_ST0, (void))
-DEF_HELPER(RTCCINTREG, helper_fistl_ST0, (void))
-DEF_HELPER(int64_t, helper_fistll_ST0, (void))
-DEF_HELPER(RTCCINTREG, helper_fistt_ST0, (void))
-DEF_HELPER(RTCCINTREG, helper_fisttl_ST0, (void))
-DEF_HELPER(int64_t, helper_fisttll_ST0, (void))
+DEF_HELPER_0(fsts_ST0, RTCCUINTREG)
+DEF_HELPER_0(fstl_ST0, i64)
+DEF_HELPER_0(fist_ST0, RTCCINTREG)
+DEF_HELPER_0(fistl_ST0, RTCCINTREG)
+DEF_HELPER_0(fistll_ST0, s64)
+DEF_HELPER_0(fistt_ST0, RTCCINTREG)
+DEF_HELPER_0(fisttl_ST0, RTCCINTREG)
+DEF_HELPER_0(fisttll_ST0, s64)
 #endif /* VBOX */
-DEF_HELPER(void, helper_fldt_ST0, (target_ulong ptr))
-DEF_HELPER(void, helper_fstt_ST0, (target_ulong ptr))
-DEF_HELPER(void, helper_fpush, (void))
-DEF_HELPER(void, helper_fpop, (void))
-DEF_HELPER(void, helper_fdecstp, (void))
-DEF_HELPER(void, helper_fincstp, (void))
-DEF_HELPER(void, helper_ffree_STN, (int st_index))
-DEF_HELPER(void, helper_fmov_ST0_FT0, (void))
-DEF_HELPER(void, helper_fmov_FT0_STN, (int st_index))
-DEF_HELPER(void, helper_fmov_ST0_STN, (int st_index))
-DEF_HELPER(void, helper_fmov_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fxchg_ST0_STN, (int st_index))
-DEF_HELPER(void, helper_fcom_ST0_FT0, (void))
-DEF_HELPER(void, helper_fucom_ST0_FT0, (void))
-DEF_HELPER(void, helper_fcomi_ST0_FT0, (void))
-DEF_HELPER(void, helper_fucomi_ST0_FT0, (void))
-DEF_HELPER(void, helper_fadd_ST0_FT0, (void))
-DEF_HELPER(void, helper_fmul_ST0_FT0, (void))
-DEF_HELPER(void, helper_fsub_ST0_FT0, (void))
-DEF_HELPER(void, helper_fsubr_ST0_FT0, (void))
-DEF_HELPER(void, helper_fdiv_ST0_FT0, (void))
-DEF_HELPER(void, helper_fdivr_ST0_FT0, (void))
-DEF_HELPER(void, helper_fadd_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fmul_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fsub_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fsubr_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fdiv_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fdivr_STN_ST0, (int st_index))
-DEF_HELPER(void, helper_fchs_ST0, (void))
-DEF_HELPER(void, helper_fabs_ST0, (void))
-DEF_HELPER(void, helper_fxam_ST0, (void))
-DEF_HELPER(void, helper_fld1_ST0, (void))
-DEF_HELPER(void, helper_fldl2t_ST0, (void))
-DEF_HELPER(void, helper_fldl2e_ST0, (void))
-DEF_HELPER(void, helper_fldpi_ST0, (void))
-DEF_HELPER(void, helper_fldlg2_ST0, (void))
-DEF_HELPER(void, helper_fldln2_ST0, (void))
-DEF_HELPER(void, helper_fldz_ST0, (void))
-DEF_HELPER(void, helper_fldz_FT0, (void))
+DEF_HELPER_1(fldt_ST0, void, tl)
+DEF_HELPER_1(fstt_ST0, void, tl)
+DEF_HELPER_0(fpush, void)
+DEF_HELPER_0(fpop, void)
+DEF_HELPER_0(fdecstp, void)
+DEF_HELPER_0(fincstp, void)
+DEF_HELPER_1(ffree_STN, void, int)
+DEF_HELPER_0(fmov_ST0_FT0, void)
+DEF_HELPER_1(fmov_FT0_STN, void, int)
+DEF_HELPER_1(fmov_ST0_STN, void, int)
+DEF_HELPER_1(fmov_STN_ST0, void, int)
+DEF_HELPER_1(fxchg_ST0_STN, void, int)
+DEF_HELPER_0(fcom_ST0_FT0, void)
+DEF_HELPER_0(fucom_ST0_FT0, void)
+DEF_HELPER_0(fcomi_ST0_FT0, void)
+DEF_HELPER_0(fucomi_ST0_FT0, void)
+DEF_HELPER_0(fadd_ST0_FT0, void)
+DEF_HELPER_0(fmul_ST0_FT0, void)
+DEF_HELPER_0(fsub_ST0_FT0, void)
+DEF_HELPER_0(fsubr_ST0_FT0, void)
+DEF_HELPER_0(fdiv_ST0_FT0, void)
+DEF_HELPER_0(fdivr_ST0_FT0, void)
+DEF_HELPER_1(fadd_STN_ST0, void, int)
+DEF_HELPER_1(fmul_STN_ST0, void, int)
+DEF_HELPER_1(fsub_STN_ST0, void, int)
+DEF_HELPER_1(fsubr_STN_ST0, void, int)
+DEF_HELPER_1(fdiv_STN_ST0, void, int)
+DEF_HELPER_1(fdivr_STN_ST0, void, int)
+DEF_HELPER_0(fchs_ST0, void)
+DEF_HELPER_0(fabs_ST0, void)
+DEF_HELPER_0(fxam_ST0, void)
+DEF_HELPER_0(fld1_ST0, void)
+DEF_HELPER_0(fldl2t_ST0, void)
+DEF_HELPER_0(fldl2e_ST0, void)
+DEF_HELPER_0(fldpi_ST0, void)
+DEF_HELPER_0(fldlg2_ST0, void)
+DEF_HELPER_0(fldln2_ST0, void)
+DEF_HELPER_0(fldz_ST0, void)
+DEF_HELPER_0(fldz_FT0, void)
 #ifndef VBOX
-DEF_HELPER(uint32_t, helper_fnstsw, (void))
-DEF_HELPER(uint32_t, helper_fnstcw, (void))
+DEF_HELPER_0(fnstsw, i32)
+DEF_HELPER_0(fnstcw, i32)
 #else  /* VBOX */
-DEF_HELPER(RTCCUINTREG, helper_fnstsw, (void))
-DEF_HELPER(RTCCUINTREG, helper_fnstcw, (void))
+DEF_HELPER_0(fnstsw, RTCCUINTREG)
+DEF_HELPER_0(fnstcw, RTCCUINTREG)
 #endif /* VBOX */
-DEF_HELPER(void, helper_fldcw, (uint32_t val))
-DEF_HELPER(void, helper_fclex, (void))
-DEF_HELPER(void, helper_fwait, (void))
-DEF_HELPER(void, helper_fninit, (void))
-DEF_HELPER(void, helper_fbld_ST0, (target_ulong ptr))
-DEF_HELPER(void, helper_fbst_ST0, (target_ulong ptr))
-DEF_HELPER(void, helper_f2xm1, (void))
-DEF_HELPER(void, helper_fyl2x, (void))
-DEF_HELPER(void, helper_fptan, (void))
-DEF_HELPER(void, helper_fpatan, (void))
-DEF_HELPER(void, helper_fxtract, (void))
-DEF_HELPER(void, helper_fprem1, (void))
-DEF_HELPER(void, helper_fprem, (void))
-DEF_HELPER(void, helper_fyl2xp1, (void))
-DEF_HELPER(void, helper_fsqrt, (void))
-DEF_HELPER(void, helper_fsincos, (void))
-DEF_HELPER(void, helper_frndint, (void))
-DEF_HELPER(void, helper_fscale, (void))
-DEF_HELPER(void, helper_fsin, (void))
-DEF_HELPER(void, helper_fcos, (void))
-DEF_HELPER(void, helper_fstenv, (target_ulong ptr, int data32))
-DEF_HELPER(void, helper_fldenv, (target_ulong ptr, int data32))
-DEF_HELPER(void, helper_fsave, (target_ulong ptr, int data32))
-DEF_HELPER(void, helper_frstor, (target_ulong ptr, int data32))
-DEF_HELPER(void, helper_fxsave, (target_ulong ptr, int data64))
-DEF_HELPER(void, helper_fxrstor, (target_ulong ptr, int data64))
-DEF_HELPER(target_ulong, helper_bsf, (target_ulong t0))
-DEF_HELPER(target_ulong, helper_bsr, (target_ulong t0))
+DEF_HELPER_1(fldcw, void, i32)
+DEF_HELPER_0(fclex, void)
+DEF_HELPER_0(fwait, void)
+DEF_HELPER_0(fninit, void)
+DEF_HELPER_1(fbld_ST0, void, tl)
+DEF_HELPER_1(fbst_ST0, void, tl)
+DEF_HELPER_0(f2xm1, void)
+DEF_HELPER_0(fyl2x, void)
+DEF_HELPER_0(fptan, void)
+DEF_HELPER_0(fpatan, void)
+DEF_HELPER_0(fxtract, void)
+DEF_HELPER_0(fprem1, void)
+DEF_HELPER_0(fprem, void)
+DEF_HELPER_0(fyl2xp1, void)
+DEF_HELPER_0(fsqrt, void)
+DEF_HELPER_0(fsincos, void)
+DEF_HELPER_0(frndint, void)
+DEF_HELPER_0(fscale, void)
+DEF_HELPER_0(fsin, void)
+DEF_HELPER_0(fcos, void)
+DEF_HELPER_2(fstenv, void, tl, int)
+DEF_HELPER_2(fldenv, void, tl, int)
+DEF_HELPER_2(fsave, void, tl, int)
+DEF_HELPER_2(frstor, void, tl, int)
+DEF_HELPER_2(fxsave, void, tl, int)
+DEF_HELPER_2(fxrstor, void, tl, int)
+DEF_HELPER_1(bsf, tl, tl)
+DEF_HELPER_1(bsr, tl, tl)
 
 /* MMX/SSE */
 
-DEF_HELPER(void, helper_enter_mmx, (void))
-DEF_HELPER(void, helper_emms, (void))
-DEF_HELPER(void, helper_movq, (uint64_t *d, uint64_t *s))
+DEF_HELPER_0(enter_mmx, void)
+DEF_HELPER_0(emms, void)
+DEF_HELPER_2(movq, void, ptr, ptr)
 
 #define SHIFT 0
@@ -239 +220 @@
 #include "ops_sse_header.h"
 
-DEF_HELPER(target_ulong, helper_rclb, (target_ulong t0, target_ulong t1))
-DEF_HELPER(target_ulong, helper_rclw, (target_ulong t0, target_ulong t1))
-DEF_HELPER(target_ulong, helper_rcll, (target_ulong t0, target_ulong t1))
-DEF_HELPER(target_ulong, helper_rcrb, (target_ulong t0, target_ulong t1))
-DEF_HELPER(target_ulong, helper_rcrw, (target_ulong t0, target_ulong t1))
-DEF_HELPER(target_ulong, helper_rcrl, (target_ulong t0, target_ulong t1))
-#ifdef TARGET_X86_64
-DEF_HELPER(target_ulong, helper_rclq, (target_ulong t0, target_ulong t1))
-DEF_HELPER(target_ulong, helper_rcrq, (target_ulong t0, target_ulong t1))
+DEF_HELPER_2(rclb, tl, tl, tl)
+DEF_HELPER_2(rclw, tl, tl, tl)
+DEF_HELPER_2(rcll, tl, tl, tl)
+DEF_HELPER_2(rcrb, tl, tl, tl)
+DEF_HELPER_2(rcrw, tl, tl, tl)
+DEF_HELPER_2(rcrl, tl, tl, tl)
+#ifdef TARGET_X86_64
+DEF_HELPER_2(rclq, tl, tl, tl)
+DEF_HELPER_2(rcrq, tl, tl, tl)
 #endif
 
 #ifdef VBOX
+DEF_HELPER_1(write_eflags_vme, void, tl)
+DEF_HELPER_0(read_eflags_vme, tl)
+DEF_HELPER_0(cli_vme, void)
+DEF_HELPER_0(sti_vme, void)
+DEF_HELPER_0(rdtscp, void)
+DEF_HELPER_0(check_external_event, void)
+DEF_HELPER_0(dump_state, void)
+DEF_HELPER_1(sync_seg, void, i32)
+
 void helper_external_event(void);
 void helper_record_call(void);
@@ -259 +249 @@
 #endif /* VBOX */
 
-#undef DEF_HELPER
+#include "def-helper.h"

trunk/src/recompiler/target-i386/helper_template.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -291 +291 @@
 #endif
     if (count) {
-        eflags = cc_table[CC_OP].compute_all();
+        eflags = helper_cc_compute_all(CC_OP);
         t0 &= DATA_MASK;
         src = t0;
@@ -320 +320 @@
 #endif
     if (count) {
-        eflags = cc_table[CC_OP].compute_all();
+        eflags = helper_cc_compute_all(CC_OP);
         t0 &= DATA_MASK;
         src = t0;

trunk/src/recompiler/target-i386/op_helper.c

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -30 +30 @@
 #define CPU_NO_GLOBAL_REGS
 #include "exec.h"
+#include "exec-all.h"
 #include "host-utils.h"
 
@@ -39 +40 @@
 //#define DEBUG_PCALL
 
+
+#ifdef DEBUG_PCALL
+#  define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)
+#  define LOG_PCALL_STATE(env) \
+          log_cpu_state_mask(CPU_LOG_PCALL, (env), X86_DUMP_CCOP)
+#else
+#  define LOG_PCALL(...) do { } while (0)
+#  define LOG_PCALL_STATE(env) do { } while (0)
+#endif
+
+
 #if 0
 #define raise_exception_err(a, b)\
 do {\
-    if (logfile)\
-        fprintf(logfile, "raise_exception line=%d\n", __LINE__);\
+    qemu_log("raise_exception line=%d\n", __LINE__);\
     (raise_exception_err)(a, b);\
 } while (0)
 #endif
 
-const uint8_t parity_table[256] = {
+static const uint8_t parity_table[256] = {
     CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
     0, CC_P, CC_P, 0, CC_P, 0, 0, CC_P,
@@ -84 +95 @@
 
 /* modulo 17 table */
-const uint8_t rclw_table[32] = {
+static const uint8_t rclw_table[32] = {
     0, 1, 2, 3, 4, 5, 6, 7,
     8, 9,10,11,12,13,14,15,
@@ -92 +103 @@
 
 /* modulo 9 table */
-const uint8_t rclb_table[32] = {
+static const uint8_t rclb_table[32] = {
     0, 1, 2, 3, 4, 5, 6, 7,
     8, 0, 1, 2, 3, 4, 5, 6,
@@ -99 +110 @@
 };
 
-const CPU86_LDouble f15rk[7] =
+static const CPU86_LDouble f15rk[7] =
 {
     0.00000000000000000000L,
@@ -112 +123 @@
 /* broken thread support */
 
-spinlock_t global_cpu_lock = SPIN_LOCK_UNLOCKED;
+static spinlock_t global_cpu_lock = SPIN_LOCK_UNLOCKED;
 
 void helper_lock(void)
@@ -132 +143 @@
 {
     uint32_t eflags;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     eflags |= (DF & DF_MASK);
     eflags |= env->eflags & ~(VM_MASK | RF_MASK);
@@ -166 +177 @@
 {
     uint32_t eflags;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     eflags |= (DF & DF_MASK);
     eflags |= env->eflags & ~(VM_MASK | RF_MASK);
@@ -308 +319 @@
 
 #ifdef VBOX
-    e1 = e2 = 0;
+    e1 = e2 = 0; /* gcc warning? */
     cpl = env->hflags & HF_CPL_MASK;
     /* Trying to load a selector with CPL=1? */
@@ -392 +403 @@
 
     type = (e2 >> DESC_TYPE_SHIFT) & 0xf;
-#ifdef DEBUG_PCALL
-    if (loglevel & CPU_LOG_PCALL)
-        fprintf(logfile, "switch_tss: sel=0x%04x type=%d src=%d\n", tss_selector, type, source);
-#endif
-
-#if defined(VBOX) && defined(DEBUG)
-    printf("switch_tss %x %x %x %d %08x\n", tss_selector, e1, e2, source, next_eip);
-#endif
+    LOG_PCALL("switch_tss: sel=0x%04x type=%d src=%d\n", tss_selector, type, source);
 
     /* if task gate, we read the TSS segment and we load it */
@@ -628 +632 @@
         raise_exception_err(EXCP0D_GPF, 0);
     }
+
+#ifndef CONFIG_USER_ONLY
+    /* reset local breakpoints */
+    if (env->dr[7] & 0x55) {
+        for (i = 0; i < 4; i++) {
+            if (hw_breakpoint_enabled(env->dr[7], i) == 0x1)
+                hw_breakpoint_remove(env, i);
+        }
+        env->dr[7] &= ~0x55;
+    }
+#endif
 }
 
@@ -787 +802 @@
     int type, dpl, selector, ss_dpl, cpl;
     int has_error_code, new_stack, shift;
-    uint32_t e1, e2, offset, ss, esp, ss_e1, ss_e2;
+    uint32_t e1, e2, offset, ss = 0, esp, ss_e1 = 0, ss_e2 = 0;
     uint32_t old_eip, sp_mask;
 
 #ifdef VBOX
-    ss = ss_e1 = ss_e2 = 0;
     if (remR3NotifyTrap(env, intno, error_code, next_eip) != VINF_SUCCESS)
         cpu_loop_exit();
@@ -1271 +1285 @@
         env->eflags &= ~IF_MASK;
     }
-
 #ifndef VBOX
     env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);
@@ -1285 +1298 @@
 #endif
 
+#ifdef TARGET_X86_64
 #if defined(CONFIG_USER_ONLY)
 void helper_syscall(int next_eip_addend)
@@ -1301 +1315 @@
     }
     selector = (env->star >> 32) & 0xffff;
-#ifdef TARGET_X86_64
     if (env->hflags & HF_LMA_MASK) {
         int code64;
@@ -1327 +1340 @@
         else
             env->eip = env->cstar;
-    } else
-#endif
-    {
+    } else {
         ECX = (uint32_t)(env->eip + next_eip_addend);
 
@@ -1348 +1359 @@
 }
 #endif
-
+#endif
+
+#ifdef TARGET_X86_64
 void helper_sysret(int dflag)
 {
@@ -1361 +1374 @@
     }
     selector = (env->star >> 48) & 0xffff;
-#ifdef TARGET_X86_64
     if (env->hflags & HF_LMA_MASK) {
         if (dflag == 2) {
@@ -1387 +1399 @@
                     IF_MASK | IOPL_MASK | VM_MASK | RF_MASK | NT_MASK);
         cpu_x86_set_cpl(env, 3);
-    } else
-#endif
-    {
+    } else {
         cpu_x86_load_seg_cache(env, R_CS, selector | 3,
                                0, 0xffffffff,
@@ -1413 +1423 @@
 #endif
 }
+#endif
 
 #ifdef VBOX
@@ -1421 +1432 @@
 void helper_external_event(void)
 {
-#if defined(RT_OS_DARWIN) && defined(VBOX_STRICT)
+# if defined(RT_OS_DARWIN) && defined(VBOX_STRICT)
     uintptr_t uSP;
-# ifdef RT_ARCH_AMD64
+#  ifdef RT_ARCH_AMD64
     __asm__ __volatile__("movq %%rsp, %0" : "=r" (uSP));
-# else
+#  else
     __asm__ __volatile__("movl %%esp, %0" : "=r" (uSP));
+#  endif
+    AssertMsg(!(uSP & 15), ("xSP=%#p\n", uSP));
 # endif
-    AssertMsg(!(uSP & 15), ("xSP=%#p\n", uSP));
-#endif
     /* Keep in sync with flags checked by gen_check_external_event() */
     if (env->interrupt_request & CPU_INTERRUPT_EXTERNAL_HARD)
@@ -1546 +1557 @@
                   target_ulong next_eip, int is_hw)
 {
+    if (qemu_loglevel_mask(CPU_LOG_INT)) {
+        if ((env->cr[0] & CR0_PE_MASK)) {
+            static int count;
+            qemu_log("%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:" TARGET_FMT_lx " pc=" TARGET_FMT_lx " SP=%04x:" TARGET_FMT_lx,
+                    count, intno, error_code, is_int,
+                    env->hflags & HF_CPL_MASK,
+                    env->segs[R_CS].selector, EIP,
+                    (int)env->segs[R_CS].base + EIP,
+                    env->segs[R_SS].selector, ESP);
+            if (intno == 0x0e) {
+                qemu_log(" CR2=" TARGET_FMT_lx, env->cr[2]);
+            } else {
+                qemu_log(" EAX=" TARGET_FMT_lx, EAX);
+            }
+            qemu_log("\n");
+            log_cpu_state(env, X86_DUMP_CCOP);
+#if 0
+            {
+                int i;
+                uint8_t *ptr;
+                qemu_log("       code=");
+                ptr = env->segs[R_CS].base + env->eip;
+                for(i = 0; i < 16; i++) {
+                    qemu_log(" %02x", ldub(ptr + i));
+                }
+                qemu_log("\n");
+            }
+#endif
+            count++;
+        }
+    }
 #ifdef VBOX
     if (RT_UNLIKELY(env->state & CPU_EMULATE_SINGLE_STEP)) {
@@ -1557 +1599 @@
     }
 #endif
-
-    if (loglevel & CPU_LOG_INT) {
-        if ((env->cr[0] & CR0_PE_MASK)) {
-            static int count;
-            fprintf(logfile, "%6d: v=%02x e=%04x i=%d cpl=%d IP=%04x:" TARGET_FMT_lx " pc=" TARGET_FMT_lx " SP=%04x:" TARGET_FMT_lx,
-                    count, intno, error_code, is_int,
-                    env->hflags & HF_CPL_MASK,
-                    env->segs[R_CS].selector, EIP,
-                    (int)env->segs[R_CS].base + EIP,
-                    env->segs[R_SS].selector, ESP);
-            if (intno == 0x0e) {
-                fprintf(logfile, " CR2=" TARGET_FMT_lx, env->cr[2]);
-            } else {
-                fprintf(logfile, " EAX=" TARGET_FMT_lx, EAX);
-            }
-            fprintf(logfile, "\n");
-            cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
-#if 0
-            {
-                int i;
-                uint8_t *ptr;
-                fprintf(logfile, "       code=");
-                ptr = env->segs[R_CS].base + env->eip;
-                for(i = 0; i < 16; i++) {
-                    fprintf(logfile, " %02x", ldub(ptr + i));
-                }
-                fprintf(logfile, "\n");
-            }
-#endif
-            count++;
-        }
-    }
     if (env->cr[0] & CR0_PE_MASK) {
 #ifdef TARGET_X86_64
@@ -1614 +1624 @@
 }
 
+/* This should come from sysemu.h - if we could include it here... */
+void qemu_system_reset_request(void);
+
 /*
  * Check nested exceptions and change to double or triple fault if
@@ -1627 +1640 @@
                                (intno >= 10 && intno <= 13);
 
-    if (loglevel & CPU_LOG_INT)
-        fprintf(logfile, "check_exception old: 0x%x new 0x%x\n",
+    qemu_log_mask(CPU_LOG_INT, "check_exception old: 0x%x new 0x%x\n",
                 env->old_exception, intno);
 
-    if (env->old_exception == EXCP08_DBLE)
-        cpu_abort(env, "triple fault");
+#if !defined(CONFIG_USER_ONLY)
+    if (env->old_exception == EXCP08_DBLE) {
+        if (env->hflags & HF_SVMI_MASK)
+            helper_vmexit(SVM_EXIT_SHUTDOWN, 0); /* does not return */
+
+        qemu_log_mask(CPU_LOG_RESET, "Triple fault\n");
+
+# ifndef VBOX
+        qemu_system_reset_request();
+# else
+        remR3RaiseRC(env->pVM, VINF_EM_RESET); /** @todo test + improve tripple fault handling. */
+# endif
+        return EXCP_HLT;
+    }
+#endif
 
     if ((first_contributory && second_contributory)
@@ -1654 +1679 @@
  * is_int is TRUE.
  */
-void raise_interrupt(int intno, int is_int, int error_code,
-                     int next_eip_addend)
+static void QEMU_NORETURN raise_interrupt(int intno, int is_int, int error_code,
+                                          int next_eip_addend)
 {
 #if defined(VBOX) && defined(DEBUG)
@@ -1676 +1701 @@
 /* shortcuts to generate exceptions */
 
-void (raise_exception_err)(int exception_index, int error_code)
+void raise_exception_err(int exception_index, int error_code)
 {
     raise_interrupt(exception_index, 0, error_code, 0);
@@ -1712 +1737 @@
     int i, offset;
 
-    if (loglevel & CPU_LOG_INT) {
-        fprintf(logfile, "SMM: enter\n");
-        cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
-    }
+    qemu_log_mask(CPU_LOG_INT, "SMM: enter\n");
+    log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
 
     env->hflags |= HF_SMM_MASK;
@@ -1846 +1869 @@
    cpu_abort(env, "helper_rsm");
 #else /* !VBOX */
-    target_ulong sm_
-
     target_ulong sm_state;
     int i, offset;
@@ -1960 +1981 @@
     cpu_smm_update(env);
 
-    if (loglevel & CPU_LOG_INT) {
-        fprintf(logfile, "SMM: after RSM\n");
-        cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
-    }
+    qemu_log_mask(CPU_LOG_INT, "SMM: after RSM\n");
+    log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
 #endif /* !VBOX */
 }
@@ -2107 +2126 @@
     int eflags;
 
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     af = eflags & CC_A;
     al = EAX & 0xff;
@@ -2123 +2142 @@
     EAX = (EAX & ~0xffff) | al | (ah << 8);
     CC_SRC = eflags;
-    FORCE_RET();
 }
 
@@ -2132 +2150 @@
     int eflags;
 
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     af = eflags & CC_A;
     al = EAX & 0xff;
@@ -2148 +2166 @@
     EAX = (EAX & ~0xffff) | al | (ah << 8);
     CC_SRC = eflags;
-    FORCE_RET();
 }
 
@@ -2156 +2173 @@
     int eflags;
 
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     cf = eflags & CC_C;
     af = eflags & CC_A;
@@ -2176 +2193 @@
     eflags |= (al & 0x80); /* sf */
     CC_SRC = eflags;
-    FORCE_RET();
 }
 
@@ -2184 +2200 @@
     int eflags;
 
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     cf = eflags & CC_C;
     af = eflags & CC_A;
@@ -2207 +2223 @@
     eflags |= (al & 0x80); /* sf */
     CC_SRC = eflags;
-    FORCE_RET();
 }
 
@@ -2213 +2228 @@
 {
     int eflags;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     if (eflags & CC_O) {
         raise_interrupt(EXCP04_INTO, 1, 0, next_eip_addend);
@@ -2224 +2239 @@
     int eflags;
 
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     d = ldq(a0);
     if (d == (((uint64_t)EDX << 32) | (uint32_t)EAX)) {
@@ -2247 +2262 @@
     if ((a0 & 0xf) != 0)
         raise_exception(EXCP0D_GPF);
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     d0 = ldq(a0);
     d1 = ldq(a0 + 8);
@@ -2268 +2283 @@
 void helper_single_step(void)
 {
-    env->dr[6] |= 0x4000;
-    raise_exception(EXCP01_SSTP);
+#ifndef CONFIG_USER_ONLY
+    check_hw_breakpoints(env, 1);
+    env->dr[6] |= DR6_BS;
+#endif
+    raise_exception(EXCP01_DB);
 }
 
 void helper_cpuid(void)
 {
-#ifndef VBOX
-    uint32_t index;
+    uint32_t eax, ebx, ecx, edx;
 
     helper_svm_check_intercept_param(SVM_EXIT_CPUID, 0);
 
-    index = (uint32_t)EAX;
-    /* test if maximum index reached */
-    if (index & 0x80000000) {
-        if (index > env->cpuid_xlevel)
-            index = env->cpuid_level;
-    } else {
-        if (index > env->cpuid_level)
-            index = env->cpuid_level;
-    }
-
-    switch(index) {
-    case 0:
-        EAX = env->cpuid_level;
-        EBX = env->cpuid_vendor1;
-        EDX = env->cpuid_vendor2;
-        ECX = env->cpuid_vendor3;
-        break;
-    case 1:
-        EAX = env->cpuid_version;
-        EBX = (env->cpuid_apic_id << 24) | 8 << 8; /* CLFLUSH size in quad words, Linux wants it. */
-        ECX = env->cpuid_ext_features;
-        EDX = env->cpuid_features;
-        break;
-    case 2:
-        /* cache info: needed for Pentium Pro compatibility */
-        EAX = 1;
-        EBX = 0;
-        ECX = 0;
-        EDX = 0x2c307d;
-        break;
-    case 4:
-        /* cache info: needed for Core compatibility */
-        switch (ECX) {
-            case 0: /* L1 dcache info */
-                EAX = 0x0000121;
-                EBX = 0x1c0003f;
-                ECX = 0x000003f;
-                EDX = 0x0000001;
-                break;
-            case 1: /* L1 icache info */
-                EAX = 0x0000122;
-                EBX = 0x1c0003f;
-                ECX = 0x000003f;
-                EDX = 0x0000001;
-                break;
-            case 2: /* L2 cache info */
-                EAX = 0x0000143;
-                EBX = 0x3c0003f;
-                ECX = 0x0000fff;
-                EDX = 0x0000001;
-                break;
-            default: /* end of info */
-                EAX = 0;
-                EBX = 0;
-                ECX = 0;
-                EDX = 0;
-                break;
-        }
-
-        break;
-    case 5:
-        /* mwait info: needed for Core compatibility */
-        EAX = 0; /* Smallest monitor-line size in bytes */
-        EBX = 0; /* Largest monitor-line size in bytes */
-        ECX = CPUID_MWAIT_EMX | CPUID_MWAIT_IBE;
-        EDX = 0;
-        break;
-    case 6:
-        /* Thermal and Power Leaf */
-        EAX = 0;
-        EBX = 0;
-        ECX = 0;
-        EDX = 0;
-        break;
-    case 9:
-        /* Direct Cache Access Information Leaf */
-        EAX = 0; /* Bits 0-31 in DCA_CAP MSR */
-        EBX = 0;
-        ECX = 0;
-        EDX = 0;
-        break;
-    case 0xA:
-        /* Architectural Performance Monitoring Leaf */
-        EAX = 0;
-        EBX = 0;
-        ECX = 0;
-        EDX = 0;
-        break;
-    case 0x80000000:
-        EAX = env->cpuid_xlevel;
-        EBX = env->cpuid_vendor1;
-        EDX = env->cpuid_vendor2;
-        ECX = env->cpuid_vendor3;
-        break;
-    case 0x80000001:
-        EAX = env->cpuid_features;
-        EBX = 0;
-        ECX = env->cpuid_ext3_features;
-        EDX = env->cpuid_ext2_features;
-        break;
-    case 0x80000002:
-    case 0x80000003:
-    case 0x80000004:
-        EAX = env->cpuid_model[(index - 0x80000002) * 4 + 0];
-        EBX = env->cpuid_model[(index - 0x80000002) * 4 + 1];
-        ECX = env->cpuid_model[(index - 0x80000002) * 4 + 2];
-        EDX = env->cpuid_model[(index - 0x80000002) * 4 + 3];
-        break;
-    case 0x80000005:
-        /* cache info (L1 cache) */
-        EAX = 0x01ff01ff;
-        EBX = 0x01ff01ff;
-        ECX = 0x40020140;
-        EDX = 0x40020140;
-        break;
-    case 0x80000006:
-        /* cache info (L2 cache) */
-        EAX = 0;
-        EBX = 0x42004200;
-        ECX = 0x02008140;
-        EDX = 0;
-        break;
-    case 0x80000008:
-        /* virtual & phys address size in low 2 bytes. */
-/* XXX: This value must match the one used in the MMU code. */
-        if (env->cpuid_ext2_features & CPUID_EXT2_LM) {
-            /* 64 bit processor */
-#if defined(USE_KQEMU)
-            EAX = 0x00003020;   /* 48 bits virtual, 32 bits physical */
-#else
-/* XXX: The physical address space is limited to 42 bits in exec.c. */
-            EAX = 0x00003028;   /* 48 bits virtual, 40 bits physical */
-#endif
-        } else {
-#if defined(USE_KQEMU)
-            EAX = 0x00000020;   /* 32 bits physical */
-#else
-            if (env->cpuid_features & CPUID_PSE36)
-                EAX = 0x00000024; /* 36 bits physical */
-            else
-                EAX = 0x00000020; /* 32 bits physical */
-#endif
-        }
-        EBX = 0;
-        ECX = 0;
-        EDX = 0;
-        break;
-    case 0x8000000A:
-        EAX = 0x00000001;
-        EBX = 0;
-        ECX = 0;
-        EDX = 0;
-        break;
-    default:
-        /* reserved values: zero */
-        EAX = 0;
-        EBX = 0;
-        ECX = 0;
-        EDX = 0;
-        break;
-    }
-#else /* VBOX */
-    remR3CpuId(env, EAX, &EAX, &EBX, &ECX, &EDX);
-#endif /* VBOX */
+    cpu_x86_cpuid(env, (uint32_t)EAX, (uint32_t)ECX, &eax, &ebx, &ecx, &edx);
+    EAX = eax;
+    EBX = ebx;
+    ECX = ecx;
+    EDX = edx;
 }
 
@@ -2724 +2582 @@
                        e2);
 #if 0
-        fprintf(logfile, "load_seg: sel=0x%04x base=0x%08lx limit=0x%08lx flags=%08x\n",
+        qemu_log("load_seg: sel=0x%04x base=0x%08lx limit=0x%08lx flags=%08x\n",
                 selector, (unsigned long)sc->base, sc->limit, sc->flags);
 #endif
@@ -2738 +2596 @@
     target_ulong next_eip;
 
-#ifdef VBOX
+#ifdef VBOX /** @todo Why do we do this? */
     e1 = e2 = 0;
 #endif
@@ -2859 +2717 @@
     int new_stack, i;
     uint32_t e1, e2, cpl, dpl, rpl, selector, offset, param_count;
-    uint32_t ss, ss_e1, ss_e2, sp, type, ss_dpl, sp_mask;
+    uint32_t ss = 0, ss_e1 = 0, ss_e2 = 0, sp, type, ss_dpl, sp_mask;
     uint32_t val, limit, old_sp_mask;
     target_ulong ssp, old_ssp, next_eip;
 
-#ifdef VBOX
-    ss = ss_e1 = ss_e2 = e1 = e2 = 0;
+#ifdef VBOX /** @todo Why do we do this? */
+    e1 = e2 = 0;
 #endif
     next_eip = env->eip + next_eip_addend;
-#ifdef DEBUG_PCALL
-    if (loglevel & CPU_LOG_PCALL) {
-        fprintf(logfile, "lcall %04x:%08x s=%d\n",
-                new_cs, (uint32_t)new_eip, shift);
-        cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
-    }
-#endif
+    LOG_PCALL("lcall %04x:%08x s=%d\n", new_cs, (uint32_t)new_eip, shift);
+    LOG_PCALL_STATE(env);
     if ((new_cs & 0xfffc) == 0)
         raise_exception_err(EXCP0D_GPF, 0);
@@ -2879 +2732 @@
         raise_exception_err(EXCP0D_GPF, new_cs & 0xfffc);
     cpl = env->hflags & HF_CPL_MASK;
-#ifdef DEBUG_PCALL
-    if (loglevel & CPU_LOG_PCALL) {
-        fprintf(logfile, "desc=%08x:%08x\n", e1, e2);
-    }
-#endif
+    LOG_PCALL("desc=%08x:%08x\n", e1, e2);
     if (e2 & DESC_S_MASK) {
         if (!(e2 & DESC_CS_MASK))
@@ -2987 +2836 @@
             /* to inner privilege */
             get_ss_esp_from_tss(&ss, &sp, dpl);
-#ifdef DEBUG_PCALL
-            if (loglevel & CPU_LOG_PCALL)
-                fprintf(logfile, "new ss:esp=%04x:%08x param_count=%d ESP=" TARGET_FMT_lx "\n",
+            LOG_PCALL("new ss:esp=%04x:%08x param_count=%d ESP=" TARGET_FMT_lx "\n",
                         ss, sp, param_count, ESP);
-#endif
             if ((ss & 0xfffc) == 0)
                 raise_exception_err(EXCP0A_TSS, ss & 0xfffc);
@@ -3178 +3024 @@
     target_ulong ssp, sp, new_eip, new_esp, sp_mask;
 
-#ifdef VBOX
+#ifdef VBOX /** @todo Why do we do this? */
     ss_e1 = ss_e2 = e1 = e2 = 0;
 #endif
@@ -3233 +3079 @@
             POPW(ssp, sp, sp_mask, new_eflags);
     }
-#ifdef DEBUG_PCALL
-    if (loglevel & CPU_LOG_PCALL) {
-        fprintf(logfile, "lret new %04x:" TARGET_FMT_lx " s=%d addend=0x%x\n",
-                new_cs, new_eip, shift, addend);
-        cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
-    }
-#endif
+    LOG_PCALL("lret new %04x:" TARGET_FMT_lx " s=%d addend=0x%x\n",
+              new_cs, new_eip, shift, addend);
+    LOG_PCALL_STATE(env);
     if ((new_cs & 0xfffc) == 0)
     {
@@ -3324 +3166 @@
             POPW(ssp, sp, sp_mask, new_ss);
         }
-#ifdef DEBUG_PCALL
-        if (loglevel & CPU_LOG_PCALL) {
-            fprintf(logfile, "new ss:esp=%04x:" TARGET_FMT_lx "\n",
+        LOG_PCALL("new ss:esp=%04x:" TARGET_FMT_lx "\n",
                     new_ss, new_esp);
-        }
-#endif
         if ((new_ss & 0xfffc) == 0) {
 #ifdef TARGET_X86_64
@@ -3436 +3274 @@
 
 #ifdef VBOX
-    e1 = e2 = 0;
+    e1 = e2 = 0; /** @todo Why do we do this? */
     remR3TrapClear(env->pVM);
 #endif
@@ -3567 +3405 @@
 {
 }
+
+void helper_movl_drN_T0(int reg, target_ulong t0)
+{
+}
 #else
 target_ulong helper_read_crN(int reg)
@@ -3612 +3454 @@
     }
 }
+
+void helper_movl_drN_T0(int reg, target_ulong t0)
+{
+    int i;
+
+    if (reg < 4) {
+        hw_breakpoint_remove(env, reg);
+        env->dr[reg] = t0;
+        hw_breakpoint_insert(env, reg);
+    } else if (reg == 7) {
+        for (i = 0; i < 4; i++)
+            hw_breakpoint_remove(env, i);
+        env->dr[7] = t0;
+        for (i = 0; i < 4; i++)
+            hw_breakpoint_insert(env, i);
+    } else
+        env->dr[reg] = t0;
+}
 #endif
 
@@ -3626 +3486 @@
     env->cr[0] &= ~CR0_TS_MASK;
     env->hflags &= ~HF_TS_MASK;
-}
-
-/* XXX: do more */
-void helper_movl_drN_T0(int reg, target_ulong t0)
-{
-    env->dr[reg] = t0;
 }
 
@@ -3721 +3575 @@
         break;
     case MSR_IA32_APICBASE:
-#ifndef VBOX /* The CPUMSetGuestMsr call below does this now. */
+# ifndef VBOX /* The CPUMSetGuestMsr call below does this now. */
         cpu_set_apic_base(env, val);
-#endif
+# endif
         break;
     case MSR_EFER:
@@ -3739 +3593 @@
             if (env->cpuid_ext3_features & CPUID_EXT3_SVM)
                 update_mask |= MSR_EFER_SVME;
+            if (env->cpuid_ext2_features & CPUID_EXT2_FFXSR)
+                update_mask |= MSR_EFER_FFXSR;
             cpu_load_efer(env, (env->efer & ~update_mask) |
                           (val & update_mask));
@@ -3772 +3628 @@
         break;
 #endif
+# ifndef VBOX
+    case MSR_MTRRphysBase(0):
+    case MSR_MTRRphysBase(1):
+    case MSR_MTRRphysBase(2):
+    case MSR_MTRRphysBase(3):
+    case MSR_MTRRphysBase(4):
+    case MSR_MTRRphysBase(5):
+    case MSR_MTRRphysBase(6):
+    case MSR_MTRRphysBase(7):
+        env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysBase(0)) / 2].base = val;
+        break;
+    case MSR_MTRRphysMask(0):
+    case MSR_MTRRphysMask(1):
+    case MSR_MTRRphysMask(2):
+    case MSR_MTRRphysMask(3):
+    case MSR_MTRRphysMask(4):
+    case MSR_MTRRphysMask(5):
+    case MSR_MTRRphysMask(6):
+    case MSR_MTRRphysMask(7):
+        env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysMask(0)) / 2].mask = val;
+        break;
+    case MSR_MTRRfix64K_00000:
+        env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix64K_00000] = val;
+        break;
+    case MSR_MTRRfix16K_80000:
+    case MSR_MTRRfix16K_A0000:
+        env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix16K_80000 + 1] = val;
+        break;
+    case MSR_MTRRfix4K_C0000:
+    case MSR_MTRRfix4K_C8000:
+    case MSR_MTRRfix4K_D0000:
+    case MSR_MTRRfix4K_D8000:
+    case MSR_MTRRfix4K_E0000:
+    case MSR_MTRRfix4K_E8000:
+    case MSR_MTRRfix4K_F0000:
+    case MSR_MTRRfix4K_F8000:
+        env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix4K_C0000 + 3] = val;
+        break;
+    case MSR_MTRRdefType:
+        env->mtrr_deftype = val;
+        break;
+# endif /* !VBOX */
     default:
-#ifndef VBOX
+# ifndef VBOX
         /* XXX: exception ? */
-#endif
-        break;
-    }
-
-#ifdef VBOX
+# endif
+        break;
+    }
+
+# ifdef VBOX
     /* call CPUM. */
     if (cpu_wrmsr(env, (uint32_t)ECX, val) != 0)
@@ -3785 +3683 @@
         /** @todo be a brave man and raise a \#GP(0) here as we should... */
     }
-#endif
+# endif
 }
 
@@ -3791 +3689 @@
 {
     uint64_t val;
+
     helper_svm_check_intercept_param(SVM_EXIT_MSR, 0);
 
@@ -3818 +3717 @@
         val = env->vm_hsave;
         break;
-#ifndef VBOX /* forward to CPUMQueryGuestMsr. */
+# ifndef VBOX /* forward to CPUMQueryGuestMsr. */
     case MSR_IA32_PERF_STATUS:
         /* tsc_increment_by_tick */
@@ -3825 +3724 @@
         val |= (((uint64_t)4ULL) << 40);
         break;
-#endif /* !VBOX */
+# endif /* !VBOX */
 #ifdef TARGET_X86_64
     case MSR_LSTAR:
@@ -3855 +3754 @@
         break;
 #endif
+# ifndef VBOX
+    case MSR_MTRRphysBase(0):
+    case MSR_MTRRphysBase(1):
+    case MSR_MTRRphysBase(2):
+    case MSR_MTRRphysBase(3):
+    case MSR_MTRRphysBase(4):
+    case MSR_MTRRphysBase(5):
+    case MSR_MTRRphysBase(6):
+    case MSR_MTRRphysBase(7):
+        val = env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysBase(0)) / 2].base;
+        break;
+    case MSR_MTRRphysMask(0):
+    case MSR_MTRRphysMask(1):
+    case MSR_MTRRphysMask(2):
+    case MSR_MTRRphysMask(3):
+    case MSR_MTRRphysMask(4):
+    case MSR_MTRRphysMask(5):
+    case MSR_MTRRphysMask(6):
+    case MSR_MTRRphysMask(7):
+        val = env->mtrr_var[((uint32_t)ECX - MSR_MTRRphysMask(0)) / 2].mask;
+        break;
+    case MSR_MTRRfix64K_00000:
+        val = env->mtrr_fixed[0];
+        break;
+    case MSR_MTRRfix16K_80000:
+    case MSR_MTRRfix16K_A0000:
+        val = env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix16K_80000 + 1];
+        break;
+    case MSR_MTRRfix4K_C0000:
+    case MSR_MTRRfix4K_C8000:
+    case MSR_MTRRfix4K_D0000:
+    case MSR_MTRRfix4K_D8000:
+    case MSR_MTRRfix4K_E0000:
+    case MSR_MTRRfix4K_E8000:
+    case MSR_MTRRfix4K_F0000:
+    case MSR_MTRRfix4K_F8000:
+        val = env->mtrr_fixed[(uint32_t)ECX - MSR_MTRRfix4K_C0000 + 3];
+        break;
+    case MSR_MTRRdefType:
+        val = env->mtrr_deftype;
+        break;
+    case MSR_MTRRcap:
+        if (env->cpuid_features & CPUID_MTRR)
+            val = MSR_MTRRcap_VCNT | MSR_MTRRcap_FIXRANGE_SUPPORT | MSR_MTRRcap_WC_SUPPORTED;
+        else
+            /* XXX: exception ? */
+            val = 0;
+        break;
+# endif /* !VBOX */
     default:
-#ifndef VBOX
+# ifndef VBOX
         /* XXX: exception ? */
         val = 0;
-#else  /* VBOX */
+# else  /* VBOX */
         if (cpu_rdmsr(env, (uint32_t)ECX, &val) != 0)
         {
@@ -3865 +3813 @@
             val = 0;
         }
-#endif /* VBOX */
+# endif /* VBOX */
         break;
     }
@@ -3871 +3819 @@
     EDX = (uint32_t)(val >> 32);
 
-#ifdef VBOX_STRICT
+# ifdef VBOX_STRICT
     if (cpu_rdmsr(env, (uint32_t)ECX, &val) != 0)
         val = 0;
     AssertMsg(val == RT_MAKE_U64(EAX, EDX), ("idMsr=%#x val=%#llx eax:edx=%#llx\n", (uint32_t)ECX, val, RT_MAKE_U64(EAX, EDX)));
-#endif
+# endif
 }
 #endif
@@ -3886 +3834 @@
 
     selector = selector1 & 0xffff;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     if (load_segment(&e1, &e2, selector) != 0)
         goto fail;
@@ -3928 +3876 @@
 
     selector = selector1 & 0xffff;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     if ((selector & 0xfffc) == 0)
         goto fail;
@@ -3974 +3922 @@
 
     selector = selector1 & 0xffff;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     if ((selector & 0xfffc) == 0)
         goto fail;
@@ -4007 +3955 @@
 
     selector = selector1 & 0xffff;
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     if ((selector & 0xfffc) == 0)
         goto fail;
@@ -4047 +3995 @@
 }
 
-void fpu_raise_exception(void)
+static void fpu_raise_exception(void)
 {
     if (env->cr[0] & CR0_NE_MASK) {
@@ -4297 +4245 @@
     ret = floatx_compare(ST0, FT0, &env->fp_status);
     env->fpus = (env->fpus & ~0x4500) | fcom_ccval[ret + 1];
-    FORCE_RET();
 }
 
@@ -4306 +4253 @@
     ret = floatx_compare_quiet(ST0, FT0, &env->fp_status);
     env->fpus = (env->fpus & ~0x4500) | fcom_ccval[ret+ 1];
-    FORCE_RET();
 }
 
@@ -4317 +4263 @@
 
     ret = floatx_compare(ST0, FT0, &env->fp_status);
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     eflags = (eflags & ~(CC_Z | CC_P | CC_C)) | fcomi_ccval[ret + 1];
     CC_SRC = eflags;
-    FORCE_RET();
 }
 
@@ -4329 +4274 @@
 
     ret = floatx_compare_quiet(ST0, FT0, &env->fp_status);
-    eflags = cc_table[CC_OP].compute_all();
+    eflags = helper_cc_compute_all(CC_OP);
     eflags = (eflags & ~(CC_Z | CC_P | CC_C)) | fcomi_ccval[ret + 1];
     CC_SRC = eflags;
-    FORCE_RET();
 }
 
@@ -4525 +4469 @@
     if (env->fpus & FPUS_SE)
         fpu_raise_exception();
-    FORCE_RET();
 }
 
@@ -5040 +4983 @@
             nb_xmm_regs = 8;
         addr = ptr + 0xa0;
-        for(i = 0; i < nb_xmm_regs; i++) {
-            stq(addr, env->xmm_regs[i].XMM_Q(0));
-            stq(addr + 8, env->xmm_regs[i].XMM_Q(1));
-            addr += 16;
+        /* Fast FXSAVE leaves out the XMM registers */
+        if (!(env->efer & MSR_EFER_FFXSR)
+          || (env->hflags & HF_CPL_MASK)
+          || !(env->hflags & HF_LMA_MASK)) {
+            for(i = 0; i < nb_xmm_regs; i++) {
+                stq(addr, env->xmm_regs[i].XMM_Q(0));
+                stq(addr + 8, env->xmm_regs[i].XMM_Q(1));
+                addr += 16;
+            }
         }
     }
@@ -5080 +5028 @@
             nb_xmm_regs = 8;
         addr = ptr + 0xa0;
-        for(i = 0; i < nb_xmm_regs; i++) {
+        /* Fast FXRESTORE leaves out the XMM registers */
+        if (!(env->efer & MSR_EFER_FFXSR)
+          || (env->hflags & HF_CPL_MASK)
+          || !(env->hflags & HF_LMA_MASK)) {
+            for(i = 0; i < nb_xmm_regs; i++) {
 #if !defined(VBOX) || __GNUC__ < 4
-            env->xmm_regs[i].XMM_Q(0) = ldq(addr);
-            env->xmm_regs[i].XMM_Q(1) = ldq(addr + 8);
+                env->xmm_regs[i].XMM_Q(0) = ldq(addr);
+                env->xmm_regs[i].XMM_Q(1) = ldq(addr + 8);
 #else /* VBOX + __GNUC__ >= 4: gcc 4.x compiler bug - it runs out of registers for the 64-bit value. */
 # if 1
-            env->xmm_regs[i].XMM_L(0) = ldl(addr);
-            env->xmm_regs[i].XMM_L(1) = ldl(addr + 4);
-            env->xmm_regs[i].XMM_L(2) = ldl(addr + 8);
-            env->xmm_regs[i].XMM_L(3) = ldl(addr + 12);
+                env->xmm_regs[i].XMM_L(0) = ldl(addr);
+                env->xmm_regs[i].XMM_L(1) = ldl(addr + 4);
+                env->xmm_regs[i].XMM_L(2) = ldl(addr + 8);
+                env->xmm_regs[i].XMM_L(3) = ldl(addr + 12);
 # else
-            /* this works fine on Mac OS X, gcc 4.0.1 */
-            uint64_t u64 = ldq(addr);
-            env->xmm_regs[i].XMM_Q(0);
-            u64 = ldq(addr + 4);
-            env->xmm_regs[i].XMM_Q(1) = u64;
+                /* this works fine on Mac OS X, gcc 4.0.1 */
+                uint64_t u64 = ldq(addr);
+                env->xmm_regs[i].XMM_Q(0);
+                u64 = ldq(addr + 4);
+                env->xmm_regs[i].XMM_Q(1) = u64;
 # endif
 #endif
-            addr += 16;
+                addr += 16;
+            }
         }
     }
@@ -5429 +5382 @@
         raise_exception(EXCP05_BOUND);
     }
-    FORCE_RET();
 }
 
@@ -5440 +5392 @@
         raise_exception(EXCP05_BOUND);
     }
-    FORCE_RET();
 }
 
@@ -5520 +5471 @@
 #endif /* VBOX */
 
+#if !defined(CONFIG_USER_ONLY)
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
@@ -5552 +5504 @@
     env = saved_env;
 }
+#endif
 
 #ifdef VBOX
@@ -6148 +6101 @@
         addr = (uint32_t)EAX;
 
-    if (loglevel & CPU_LOG_TB_IN_ASM)
-        fprintf(logfile,"vmrun! " TARGET_FMT_lx "\n", addr);
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmrun! " TARGET_FMT_lx "\n", addr);
 
     env->vm_vmcb = addr;
@@ -6269 +6221 @@
         stl_phys(env->vm_vmcb + offsetof(struct vmcb, control.event_inj), event_inj & ~SVM_EVTINJ_VALID);
 
-        if (loglevel & CPU_LOG_TB_IN_ASM)
-            fprintf(logfile, "Injecting(%#hx): ", valid_err);
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, "Injecting(%#hx): ", valid_err);
         /* FIXME: need to implement valid_err */
         switch (event_inj & SVM_EVTINJ_TYPE_MASK) {
@@ -6278 +6229 @@
                 env->exception_is_int = 0;
                 env->exception_next_eip = -1;
-                if (loglevel & CPU_LOG_TB_IN_ASM)
-                    fprintf(logfile, "INTR");
+                qemu_log_mask(CPU_LOG_TB_IN_ASM, "INTR");
                 /* XXX: is it always correct ? */
                 do_interrupt(vector, 0, 0, 0, 1);
@@ -6288 +6238 @@
                 env->exception_is_int = 0;
                 env->exception_next_eip = EIP;
-                if (loglevel & CPU_LOG_TB_IN_ASM)
-                    fprintf(logfile, "NMI");
+                qemu_log_mask(CPU_LOG_TB_IN_ASM, "NMI");
                 cpu_loop_exit();
                 break;
@@ -6297 +6246 @@
                 env->exception_is_int = 0;
                 env->exception_next_eip = -1;
-                if (loglevel & CPU_LOG_TB_IN_ASM)
-                    fprintf(logfile, "EXEPT");
+                qemu_log_mask(CPU_LOG_TB_IN_ASM, "EXEPT");
                 cpu_loop_exit();
                 break;
@@ -6306 +6254 @@
                 env->exception_is_int = 1;
                 env->exception_next_eip = EIP;
-                if (loglevel & CPU_LOG_TB_IN_ASM)
-                    fprintf(logfile, "SOFT");
+                qemu_log_mask(CPU_LOG_TB_IN_ASM, "SOFT");
                 cpu_loop_exit();
                 break;
         }
-        if (loglevel & CPU_LOG_TB_IN_ASM)
-            fprintf(logfile, " %#x %#x\n", env->exception_index, env->error_code);
+        qemu_log_mask(CPU_LOG_TB_IN_ASM, " %#x %#x\n", env->exception_index, env->error_code);
     }
 }
@@ -6332 +6278 @@
         addr = (uint32_t)EAX;
 
-    if (loglevel & CPU_LOG_TB_IN_ASM)
-        fprintf(logfile,"vmload! " TARGET_FMT_lx "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmload! " TARGET_FMT_lx "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
                 addr, ldq_phys(addr + offsetof(struct vmcb, save.fs.base)),
                 env->segs[R_FS].base);
@@ -6368 +6313 @@
         addr = (uint32_t)EAX;
 
-    if (loglevel & CPU_LOG_TB_IN_ASM)
-        fprintf(logfile,"vmsave! " TARGET_FMT_lx "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmsave! " TARGET_FMT_lx "\nFS: %016" PRIx64 " | " TARGET_FMT_lx "\n",
                 addr, ldq_phys(addr + offsetof(struct vmcb, save.fs.base)),
                 env->segs[R_FS].base);
@@ -6521 +6465 @@
     uint32_t int_ctl;
 
-    if (loglevel & CPU_LOG_TB_IN_ASM)
-        fprintf(logfile,"vmexit(%08x, %016" PRIx64 ", %016" PRIx64 ", " TARGET_FMT_lx ")!\n",
+    qemu_log_mask(CPU_LOG_TB_IN_ASM, "vmexit(%08x, %016" PRIx64 ", %016" PRIx64 ", " TARGET_FMT_lx ")!\n",
                 exit_code, exit_info_1,
                 ldq_phys(env->vm_vmcb + offsetof(struct vmcb, control.exit_info_2)),
@@ -6668 +6611 @@
 
 /* XXX: suppress */
-void helper_movq(uint64_t *d, uint64_t *s)
-{
-    *d = *s;
+void helper_movq(void *d, void *s)
+{
+    *(uint64_t *)d = *(uint64_t *)s;
 }
 
@@ -6740 +6683 @@
 }
 
-CCTable cc_table[CC_OP_NB] = {
-    [CC_OP_DYNAMIC] = { /* should never happen */ },
-
-    [CC_OP_EFLAGS] = { compute_all_eflags, compute_c_eflags },
-
-    [CC_OP_MULB] = { compute_all_mulb, compute_c_mull },
-    [CC_OP_MULW] = { compute_all_mulw, compute_c_mull },
-    [CC_OP_MULL] = { compute_all_mull, compute_c_mull },
-
-    [CC_OP_ADDB] = { compute_all_addb, compute_c_addb },
-    [CC_OP_ADDW] = { compute_all_addw, compute_c_addw  },
-    [CC_OP_ADDL] = { compute_all_addl, compute_c_addl  },
-
-    [CC_OP_ADCB] = { compute_all_adcb, compute_c_adcb },
-    [CC_OP_ADCW] = { compute_all_adcw, compute_c_adcw  },
-    [CC_OP_ADCL] = { compute_all_adcl, compute_c_adcl  },
-
-    [CC_OP_SUBB] = { compute_all_subb, compute_c_subb  },
-    [CC_OP_SUBW] = { compute_all_subw, compute_c_subw  },
-    [CC_OP_SUBL] = { compute_all_subl, compute_c_subl  },
-
-    [CC_OP_SBBB] = { compute_all_sbbb, compute_c_sbbb  },
-    [CC_OP_SBBW] = { compute_all_sbbw, compute_c_sbbw  },
-    [CC_OP_SBBL] = { compute_all_sbbl, compute_c_sbbl  },
-
-    [CC_OP_LOGICB] = { compute_all_logicb, compute_c_logicb },
-    [CC_OP_LOGICW] = { compute_all_logicw, compute_c_logicw },
-    [CC_OP_LOGICL] = { compute_all_logicl, compute_c_logicl },
-
-    [CC_OP_INCB] = { compute_all_incb, compute_c_incl },
-    [CC_OP_INCW] = { compute_all_incw, compute_c_incl },
-    [CC_OP_INCL] = { compute_all_incl, compute_c_incl },
-
-    [CC_OP_DECB] = { compute_all_decb, compute_c_incl },
-    [CC_OP_DECW] = { compute_all_decw, compute_c_incl },
-    [CC_OP_DECL] = { compute_all_decl, compute_c_incl },
-
-    [CC_OP_SHLB] = { compute_all_shlb, compute_c_shlb },
-    [CC_OP_SHLW] = { compute_all_shlw, compute_c_shlw },
-    [CC_OP_SHLL] = { compute_all_shll, compute_c_shll },
-
-    [CC_OP_SARB] = { compute_all_sarb, compute_c_sarl },
-    [CC_OP_SARW] = { compute_all_sarw, compute_c_sarl },
-    [CC_OP_SARL] = { compute_all_sarl, compute_c_sarl },
+uint32_t helper_cc_compute_all(int op)
+{
+    switch (op) {
+    default: /* should never happen */ return 0;
+
+    case CC_OP_EFLAGS: return compute_all_eflags();
+
+    case CC_OP_MULB: return compute_all_mulb();
+    case CC_OP_MULW: return compute_all_mulw();
+    case CC_OP_MULL: return compute_all_mull();
+
+    case CC_OP_ADDB: return compute_all_addb();
+    case CC_OP_ADDW: return compute_all_addw();
+    case CC_OP_ADDL: return compute_all_addl();
+
+    case CC_OP_ADCB: return compute_all_adcb();
+    case CC_OP_ADCW: return compute_all_adcw();
+    case CC_OP_ADCL: return compute_all_adcl();
+
+    case CC_OP_SUBB: return compute_all_subb();
+    case CC_OP_SUBW: return compute_all_subw();
+    case CC_OP_SUBL: return compute_all_subl();
+
+    case CC_OP_SBBB: return compute_all_sbbb();
+    case CC_OP_SBBW: return compute_all_sbbw();
+    case CC_OP_SBBL: return compute_all_sbbl();
+
+    case CC_OP_LOGICB: return compute_all_logicb();
+    case CC_OP_LOGICW: return compute_all_logicw();
+    case CC_OP_LOGICL: return compute_all_logicl();
+
+    case CC_OP_INCB: return compute_all_incb();
+    case CC_OP_INCW: return compute_all_incw();
+    case CC_OP_INCL: return compute_all_incl();
+
+    case CC_OP_DECB: return compute_all_decb();
+    case CC_OP_DECW: return compute_all_decw();
+    case CC_OP_DECL: return compute_all_decl();
+
+    case CC_OP_SHLB: return compute_all_shlb();
+    case CC_OP_SHLW: return compute_all_shlw();
+    case CC_OP_SHLL: return compute_all_shll();
+
+    case CC_OP_SARB: return compute_all_sarb();
+    case CC_OP_SARW: return compute_all_sarw();
+    case CC_OP_SARL: return compute_all_sarl();
 
 #ifdef TARGET_X86_64
-    [CC_OP_MULQ] = { compute_all_mulq, compute_c_mull },
-
-    [CC_OP_ADDQ] = { compute_all_addq, compute_c_addq  },
-
-    [CC_OP_ADCQ] = { compute_all_adcq, compute_c_adcq  },
-
-    [CC_OP_SUBQ] = { compute_all_subq, compute_c_subq  },
-
-    [CC_OP_SBBQ] = { compute_all_sbbq, compute_c_sbbq  },
-
-    [CC_OP_LOGICQ] = { compute_all_logicq, compute_c_logicq },
-
-    [CC_OP_INCQ] = { compute_all_incq, compute_c_incl },
-
-    [CC_OP_DECQ] = { compute_all_decq, compute_c_incl },
-
-    [CC_OP_SHLQ] = { compute_all_shlq, compute_c_shlq },
-
-    [CC_OP_SARQ] = { compute_all_sarq, compute_c_sarl },
-#endif
-};
-
+    case CC_OP_MULQ: return compute_all_mulq();
+
+    case CC_OP_ADDQ: return compute_all_addq();
+
+    case CC_OP_ADCQ: return compute_all_adcq();
+
+    case CC_OP_SUBQ: return compute_all_subq();
+
+    case CC_OP_SBBQ: return compute_all_sbbq();
+
+    case CC_OP_LOGICQ: return compute_all_logicq();
+
+    case CC_OP_INCQ: return compute_all_incq();
+
+    case CC_OP_DECQ: return compute_all_decq();
+
+    case CC_OP_SHLQ: return compute_all_shlq();
+
+    case CC_OP_SARQ: return compute_all_sarq();
+#endif
+    }
+}
+
+uint32_t helper_cc_compute_c(int op)
+{
+    switch (op) {
+    default: /* should never happen */ return 0;
+
+    case CC_OP_EFLAGS: return compute_c_eflags();
+
+    case CC_OP_MULB: return compute_c_mull();
+    case CC_OP_MULW: return compute_c_mull();
+    case CC_OP_MULL: return compute_c_mull();
+
+    case CC_OP_ADDB: return compute_c_addb();
+    case CC_OP_ADDW: return compute_c_addw();
+    case CC_OP_ADDL: return compute_c_addl();
+
+    case CC_OP_ADCB: return compute_c_adcb();
+    case CC_OP_ADCW: return compute_c_adcw();
+    case CC_OP_ADCL: return compute_c_adcl();
+
+    case CC_OP_SUBB: return compute_c_subb();
+    case CC_OP_SUBW: return compute_c_subw();
+    case CC_OP_SUBL: return compute_c_subl();
+
+    case CC_OP_SBBB: return compute_c_sbbb();
+    case CC_OP_SBBW: return compute_c_sbbw();
+    case CC_OP_SBBL: return compute_c_sbbl();
+
+    case CC_OP_LOGICB: return compute_c_logicb();
+    case CC_OP_LOGICW: return compute_c_logicw();
+    case CC_OP_LOGICL: return compute_c_logicl();
+
+    case CC_OP_INCB: return compute_c_incl();
+    case CC_OP_INCW: return compute_c_incl();
+    case CC_OP_INCL: return compute_c_incl();
+
+    case CC_OP_DECB: return compute_c_incl();
+    case CC_OP_DECW: return compute_c_incl();
+    case CC_OP_DECL: return compute_c_incl();
+
+    case CC_OP_SHLB: return compute_c_shlb();
+    case CC_OP_SHLW: return compute_c_shlw();
+    case CC_OP_SHLL: return compute_c_shll();
+
+    case CC_OP_SARB: return compute_c_sarl();
+    case CC_OP_SARW: return compute_c_sarl();
+    case CC_OP_SARL: return compute_c_sarl();
+
+#ifdef TARGET_X86_64
+    case CC_OP_MULQ: return compute_c_mull();
+
+    case CC_OP_ADDQ: return compute_c_addq();
+
+    case CC_OP_ADCQ: return compute_c_adcq();
+
+    case CC_OP_SUBQ: return compute_c_subq();
+
+    case CC_OP_SBBQ: return compute_c_sbbq();
+
+    case CC_OP_LOGICQ: return compute_c_logicq();
+
+    case CC_OP_INCQ: return compute_c_incl();
+
+    case CC_OP_DECQ: return compute_c_incl();
+
+    case CC_OP_SHLQ: return compute_c_shlq();
+
+    case CC_OP_SARQ: return compute_c_sarl();
+#endif
+    }
+}

trunk/src/recompiler/target-i386/ops_sse.h

@@ -17 +17 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -69 +69 @@
 #endif
     }
-    FORCE_RET();
 }
 
@@ -115 +114 @@
 #endif
     }
-    FORCE_RET();
 }
 
@@ -136 +134 @@
 #endif
     }
-    FORCE_RET();
 }
 
@@ -174 +171 @@
 #endif
     }
-    FORCE_RET();
 }
 
@@ -193 +189 @@
 #endif
     }
-    FORCE_RET();
 }
 
@@ -212 +207 @@
 #endif
     }
-    FORCE_RET();
 }
 
@@ -227 +221 @@
     for(i = 16 - shift; i < 16; i++)
         d->B(i) = 0;
-    FORCE_RET();
 }
 
@@ -241 +234 @@
     for(i = 0; i < shift; i++)
         d->B(i) = 0;
-    FORCE_RET();
 }
 #endif
@@ -443 +435 @@
             (int16_t)s->W(2*i+1) * (int16_t)d->W(2*i+1);
     }
-    FORCE_RET();
 }
 
@@ -490 +481 @@
             stb(a0 + i, d->B(i));
     }
-    FORCE_RET();
 }
 
@@ -928 +918 @@
     ret = float32_compare_quiet(s0, s1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
-    FORCE_RET();
 }
 
@@ -940 +929 @@
     ret = float32_compare(s0, s1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
-    FORCE_RET();
 }
 
@@ -952 +940 @@
     ret = float64_compare_quiet(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
-    FORCE_RET();
 }
 
@@ -964 +951 @@
     ret = float64_compare(d0, d1, &env->sse_status);
     CC_SRC = comis_eflags[ret + 1];
-    FORCE_RET();
 }
 
@@ -991 +977 @@
     uint32_t val;
     val = 0;
-    val |= (s->XMM_B(0) >> 7);
-    val |= (s->XMM_B(1) >> 6) & 0x02;
-    val |= (s->XMM_B(2) >> 5) & 0x04;
-    val |= (s->XMM_B(3) >> 4) & 0x08;
-    val |= (s->XMM_B(4) >> 3) & 0x10;
-    val |= (s->XMM_B(5) >> 2) & 0x20;
-    val |= (s->XMM_B(6) >> 1) & 0x40;
-    val |= (s->XMM_B(7)) & 0x80;
-#if SHIFT == 1
-    val |= (s->XMM_B(8) << 1) & 0x0100;
-    val |= (s->XMM_B(9) << 2) & 0x0200;
-    val |= (s->XMM_B(10) << 3) & 0x0400;
-    val |= (s->XMM_B(11) << 4) & 0x0800;
-    val |= (s->XMM_B(12) << 5) & 0x1000;
-    val |= (s->XMM_B(13) << 6) & 0x2000;
-    val |= (s->XMM_B(14) << 7) & 0x4000;
-    val |= (s->XMM_B(15) << 8) & 0x8000;
+    val |= (s->B(0) >> 7);
+    val |= (s->B(1) >> 6) & 0x02;
+    val |= (s->B(2) >> 5) & 0x04;
+    val |= (s->B(3) >> 4) & 0x08;
+    val |= (s->B(4) >> 3) & 0x10;
+    val |= (s->B(5) >> 2) & 0x20;
+    val |= (s->B(6) >> 1) & 0x40;
+    val |= (s->B(7)) & 0x80;
+#if SHIFT == 1
+    val |= (s->B(8) << 1) & 0x0100;
+    val |= (s->B(9) << 2) & 0x0200;
+    val |= (s->B(10) << 3) & 0x0400;
+    val |= (s->B(11) << 4) & 0x0800;
+    val |= (s->B(12) << 5) & 0x1000;
+    val |= (s->B(13) << 6) & 0x2000;
+    val |= (s->B(14) << 7) & 0x4000;
+    val |= (s->B(15) << 8) & 0x8000;
 #endif
     return val;
@@ -1510 +1496 @@
     d->elem(0) = F(0);\
     d->elem(1) = F(1);\
-    d->elem(2) = F(2);\
-    d->elem(3) = F(3);\
-    if (num > 3) {\
-        d->elem(4) = F(4);\
-        d->elem(5) = F(5);\
-        if (num > 5) {\
+    if (num > 2) {\
+        d->elem(2) = F(2);\
+        d->elem(3) = F(3);\
+        if (num > 4) {\
+            d->elem(4) = F(4);\
+            d->elem(5) = F(5);\
             d->elem(6) = F(6);\
             d->elem(7) = F(7);\

trunk/src/recompiler/target-i386/ops_sse_header.h

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -36 +36 @@
 #endif
 
-DEF_HELPER(void, glue(helper_psrlw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psraw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psllw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psrld, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psrad, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pslld, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psrlq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psllq, SUFFIX), (Reg *d, Reg *s))
-
-#if SHIFT == 1
-DEF_HELPER(void, glue(helper_psrldq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pslldq, SUFFIX), (Reg *d, Reg *s))
+#define dh_alias_Reg ptr
+#define dh_alias_XMMReg ptr
+#define dh_alias_MMXReg ptr
+#define dh_ctype_Reg Reg *
+#define dh_ctype_XMMReg XMMReg *
+#define dh_ctype_MMXReg MMXReg *
+
+DEF_HELPER_2(glue(psrlw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psraw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psllw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psrld, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psrad, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pslld, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psrlq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psllq, SUFFIX), void, Reg, Reg)
+
+#if SHIFT == 1
+DEF_HELPER_2(glue(psrldq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pslldq, SUFFIX), void, Reg, Reg)
 #endif
 
 #define SSE_HELPER_B(name, F)\
-    DEF_HELPER(void, glue(name, SUFFIX), (Reg *d, Reg *s))
+    DEF_HELPER_2(glue(name, SUFFIX), void, Reg, Reg)
 
 #define SSE_HELPER_W(name, F)\
-    DEF_HELPER(void, glue(name, SUFFIX), (Reg *d, Reg *s))
+    DEF_HELPER_2(glue(name, SUFFIX), void, Reg, Reg)
 
 #define SSE_HELPER_L(name, F)\
-    DEF_HELPER(void, glue(name, SUFFIX), (Reg *d, Reg *s))
+    DEF_HELPER_2(glue(name, SUFFIX), void, Reg, Reg)
 
 #define SSE_HELPER_Q(name, F)\
-    DEF_HELPER(void, glue(name, SUFFIX), (Reg *d, Reg *s))
-
-SSE_HELPER_B(helper_paddb, FADD)
-SSE_HELPER_W(helper_paddw, FADD)
-SSE_HELPER_L(helper_paddl, FADD)
-SSE_HELPER_Q(helper_paddq, FADD)
-
-SSE_HELPER_B(helper_psubb, FSUB)
-SSE_HELPER_W(helper_psubw, FSUB)
-SSE_HELPER_L(helper_psubl, FSUB)
-SSE_HELPER_Q(helper_psubq, FSUB)
-
-SSE_HELPER_B(helper_paddusb, FADDUB)
-SSE_HELPER_B(helper_paddsb, FADDSB)
-SSE_HELPER_B(helper_psubusb, FSUBUB)
-SSE_HELPER_B(helper_psubsb, FSUBSB)
-
-SSE_HELPER_W(helper_paddusw, FADDUW)
-SSE_HELPER_W(helper_paddsw, FADDSW)
-SSE_HELPER_W(helper_psubusw, FSUBUW)
-SSE_HELPER_W(helper_psubsw, FSUBSW)
-
-SSE_HELPER_B(helper_pminub, FMINUB)
-SSE_HELPER_B(helper_pmaxub, FMAXUB)
-
-SSE_HELPER_W(helper_pminsw, FMINSW)
-SSE_HELPER_W(helper_pmaxsw, FMAXSW)
-
-SSE_HELPER_Q(helper_pand, FAND)
-SSE_HELPER_Q(helper_pandn, FANDN)
-SSE_HELPER_Q(helper_por, FOR)
-SSE_HELPER_Q(helper_pxor, FXOR)
-
-SSE_HELPER_B(helper_pcmpgtb, FCMPGTB)
-SSE_HELPER_W(helper_pcmpgtw, FCMPGTW)
-SSE_HELPER_L(helper_pcmpgtl, FCMPGTL)
-
-SSE_HELPER_B(helper_pcmpeqb, FCMPEQ)
-SSE_HELPER_W(helper_pcmpeqw, FCMPEQ)
-SSE_HELPER_L(helper_pcmpeql, FCMPEQ)
-
-SSE_HELPER_W(helper_pmullw, FMULLW)
+    DEF_HELPER_2(glue(name, SUFFIX), void, Reg, Reg)
+
+SSE_HELPER_B(paddb, FADD)
+SSE_HELPER_W(paddw, FADD)
+SSE_HELPER_L(paddl, FADD)
+SSE_HELPER_Q(paddq, FADD)
+
+SSE_HELPER_B(psubb, FSUB)
+SSE_HELPER_W(psubw, FSUB)
+SSE_HELPER_L(psubl, FSUB)
+SSE_HELPER_Q(psubq, FSUB)
+
+SSE_HELPER_B(paddusb, FADDUB)
+SSE_HELPER_B(paddsb, FADDSB)
+SSE_HELPER_B(psubusb, FSUBUB)
+SSE_HELPER_B(psubsb, FSUBSB)
+
+SSE_HELPER_W(paddusw, FADDUW)
+SSE_HELPER_W(paddsw, FADDSW)
+SSE_HELPER_W(psubusw, FSUBUW)
+SSE_HELPER_W(psubsw, FSUBSW)
+
+SSE_HELPER_B(pminub, FMINUB)
+SSE_HELPER_B(pmaxub, FMAXUB)
+
+SSE_HELPER_W(pminsw, FMINSW)
+SSE_HELPER_W(pmaxsw, FMAXSW)
+
+SSE_HELPER_Q(pand, FAND)
+SSE_HELPER_Q(pandn, FANDN)
+SSE_HELPER_Q(por, FOR)
+SSE_HELPER_Q(pxor, FXOR)
+
+SSE_HELPER_B(pcmpgtb, FCMPGTB)
+SSE_HELPER_W(pcmpgtw, FCMPGTW)
+SSE_HELPER_L(pcmpgtl, FCMPGTL)
+
+SSE_HELPER_B(pcmpeqb, FCMPEQ)
+SSE_HELPER_W(pcmpeqw, FCMPEQ)
+SSE_HELPER_L(pcmpeql, FCMPEQ)
+
+SSE_HELPER_W(pmullw, FMULLW)
 #if SHIFT == 0
-SSE_HELPER_W(helper_pmulhrw, FMULHRW)
-#endif
-SSE_HELPER_W(helper_pmulhuw, FMULHUW)
-SSE_HELPER_W(helper_pmulhw, FMULHW)
-
-SSE_HELPER_B(helper_pavgb, FAVG)
-SSE_HELPER_W(helper_pavgw, FAVG)
-
-DEF_HELPER(void, glue(helper_pmuludq, SUFFIX) , (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmaddwd, SUFFIX) , (Reg *d, Reg *s))
-
-DEF_HELPER(void, glue(helper_psadbw, SUFFIX) , (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_maskmov, SUFFIX) , (Reg *d, Reg *s, target_ulong a0))
-DEF_HELPER(void, glue(helper_movl_mm_T0, SUFFIX) , (Reg *d, uint32_t val))
+SSE_HELPER_W(pmulhrw, FMULHRW)
+#endif
+SSE_HELPER_W(pmulhuw, FMULHUW)
+SSE_HELPER_W(pmulhw, FMULHW)
+
+SSE_HELPER_B(pavgb, FAVG)
+SSE_HELPER_W(pavgw, FAVG)
+
+DEF_HELPER_2(glue(pmuludq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmaddwd, SUFFIX), void, Reg, Reg)
+
+DEF_HELPER_2(glue(psadbw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_3(glue(maskmov, SUFFIX), void, Reg, Reg, tl)
+DEF_HELPER_2(glue(movl_mm_T0, SUFFIX), void, Reg, i32)
 #ifdef TARGET_X86_64
-DEF_HELPER(void, glue(helper_movq_mm_T0, SUFFIX) , (Reg *d, uint64_t val))
+DEF_HELPER_2(glue(movq_mm_T0, SUFFIX), void, Reg, i64)
 #endif
 
 #if SHIFT == 0
-DEF_HELPER(void, glue(helper_pshufw, SUFFIX) , (Reg *d, Reg *s, int order))
+DEF_HELPER_3(glue(pshufw, SUFFIX), void, Reg, Reg, int)
 #else
-DEF_HELPER(void, helper_shufps, (Reg *d, Reg *s, int order))
-DEF_HELPER(void, helper_shufpd, (Reg *d, Reg *s, int order))
-DEF_HELPER(void, glue(helper_pshufd, SUFFIX) , (Reg *d, Reg *s, int order))
-DEF_HELPER(void, glue(helper_pshuflw, SUFFIX) , (Reg *d, Reg *s, int order))
-DEF_HELPER(void, glue(helper_pshufhw, SUFFIX) , (Reg *d, Reg *s, int order))
+DEF_HELPER_3(shufps, void, Reg, Reg, int)
+DEF_HELPER_3(shufpd, void, Reg, Reg, int)
+DEF_HELPER_3(glue(pshufd, SUFFIX), void, Reg, Reg, int)
+DEF_HELPER_3(glue(pshuflw, SUFFIX), void, Reg, Reg, int)
+DEF_HELPER_3(glue(pshufhw, SUFFIX), void, Reg, Reg, int)
 #endif
 
@@ -136 +143 @@
 
 #define SSE_HELPER_S(name, F)\
-    DEF_HELPER(void, helper_ ## name ## ps , (Reg *d, Reg *s))        \
-    DEF_HELPER(void, helper_ ## name ## ss , (Reg *d, Reg *s))        \
-    DEF_HELPER(void, helper_ ## name ## pd , (Reg *d, Reg *s))        \
-    DEF_HELPER(void, helper_ ## name ## sd , (Reg *d, Reg *s))
+    DEF_HELPER_2(name ## ps , void, Reg, Reg)        \
+    DEF_HELPER_2(name ## ss , void, Reg, Reg)        \
+    DEF_HELPER_2(name ## pd , void, Reg, Reg)        \
+    DEF_HELPER_2(name ## sd , void, Reg, Reg)
 
 SSE_HELPER_S(add, FPU_ADD)
@@ -150 +157 @@
 
 
-DEF_HELPER(void, helper_cvtps2pd, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_cvtpd2ps, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_cvtss2sd, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_cvtsd2ss, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_cvtdq2ps, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_cvtdq2pd, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_cvtpi2ps, (XMMReg *d, MMXReg *s))
-DEF_HELPER(void, helper_cvtpi2pd, (XMMReg *d, MMXReg *s))
-DEF_HELPER(void, helper_cvtsi2ss, (XMMReg *d, uint32_t val))
-DEF_HELPER(void, helper_cvtsi2sd, (XMMReg *d, uint32_t val))
+DEF_HELPER_2(cvtps2pd, void, Reg, Reg)
+DEF_HELPER_2(cvtpd2ps, void, Reg, Reg)
+DEF_HELPER_2(cvtss2sd, void, Reg, Reg)
+DEF_HELPER_2(cvtsd2ss, void, Reg, Reg)
+DEF_HELPER_2(cvtdq2ps, void, Reg, Reg)
+DEF_HELPER_2(cvtdq2pd, void, Reg, Reg)
+DEF_HELPER_2(cvtpi2ps, void, XMMReg, MMXReg)
+DEF_HELPER_2(cvtpi2pd, void, XMMReg, MMXReg)
+DEF_HELPER_2(cvtsi2ss, void, XMMReg, i32)
+DEF_HELPER_2(cvtsi2sd, void, XMMReg, i32)
 
 #ifdef TARGET_X86_64
-DEF_HELPER(void, helper_cvtsq2ss, (XMMReg *d, uint64_t val))
-DEF_HELPER(void, helper_cvtsq2sd, (XMMReg *d, uint64_t val))
-#endif
-
-DEF_HELPER(void, helper_cvtps2dq, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_cvtpd2dq, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_cvtps2pi, (MMXReg *d, XMMReg *s))
-DEF_HELPER(void, helper_cvtpd2pi, (MMXReg *d, XMMReg *s))
-DEF_HELPER(int32_t, helper_cvtss2si, (XMMReg *s))
-DEF_HELPER(int32_t, helper_cvtsd2si, (XMMReg *s))
+DEF_HELPER_2(cvtsq2ss, void, XMMReg, i64)
+DEF_HELPER_2(cvtsq2sd, void, XMMReg, i64)
+#endif
+
+DEF_HELPER_2(cvtps2dq, void, XMMReg, XMMReg)
+DEF_HELPER_2(cvtpd2dq, void, XMMReg, XMMReg)
+DEF_HELPER_2(cvtps2pi, void, MMXReg, XMMReg)
+DEF_HELPER_2(cvtpd2pi, void, MMXReg, XMMReg)
+DEF_HELPER_1(cvtss2si, s32, XMMReg)
+DEF_HELPER_1(cvtsd2si, s32, XMMReg)
 #ifdef TARGET_X86_64
-DEF_HELPER(int64_t, helper_cvtss2sq, (XMMReg *s))
-DEF_HELPER(int64_t, helper_cvtsd2sq, (XMMReg *s))
-#endif
-
-DEF_HELPER(void, helper_cvttps2dq, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_cvttpd2dq, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_cvttps2pi, (MMXReg *d, XMMReg *s))
-DEF_HELPER(void, helper_cvttpd2pi, (MMXReg *d, XMMReg *s))
-DEF_HELPER(int32_t, helper_cvttss2si, (XMMReg *s))
-DEF_HELPER(int32_t, helper_cvttsd2si, (XMMReg *s))
+DEF_HELPER_1(cvtss2sq, s64, XMMReg)
+DEF_HELPER_1(cvtsd2sq, s64, XMMReg)
+#endif
+
+DEF_HELPER_2(cvttps2dq, void, XMMReg, XMMReg)
+DEF_HELPER_2(cvttpd2dq, void, XMMReg, XMMReg)
+DEF_HELPER_2(cvttps2pi, void, MMXReg, XMMReg)
+DEF_HELPER_2(cvttpd2pi, void, MMXReg, XMMReg)
+DEF_HELPER_1(cvttss2si, s32, XMMReg)
+DEF_HELPER_1(cvttsd2si, s32, XMMReg)
 #ifdef TARGET_X86_64
-DEF_HELPER(int64_t, helper_cvttss2sq, (XMMReg *s))
-DEF_HELPER(int64_t, helper_cvttsd2sq, (XMMReg *s))
-#endif
-
-DEF_HELPER(void, helper_rsqrtps, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_rsqrtss, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_rcpps, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_rcpss, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_haddps, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_haddpd, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_hsubps, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_hsubpd, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_addsubps, (XMMReg *d, XMMReg *s))
-DEF_HELPER(void, helper_addsubpd, (XMMReg *d, XMMReg *s))
+DEF_HELPER_1(cvttss2sq, s64, XMMReg)
+DEF_HELPER_1(cvttsd2sq, s64, XMMReg)
+#endif
+
+DEF_HELPER_2(rsqrtps, void, XMMReg, XMMReg)
+DEF_HELPER_2(rsqrtss, void, XMMReg, XMMReg)
+DEF_HELPER_2(rcpps, void, XMMReg, XMMReg)
+DEF_HELPER_2(rcpss, void, XMMReg, XMMReg)
+DEF_HELPER_2(haddps, void, XMMReg, XMMReg)
+DEF_HELPER_2(haddpd, void, XMMReg, XMMReg)
+DEF_HELPER_2(hsubps, void, XMMReg, XMMReg)
+DEF_HELPER_2(hsubpd, void, XMMReg, XMMReg)
+DEF_HELPER_2(addsubps, void, XMMReg, XMMReg)
+DEF_HELPER_2(addsubpd, void, XMMReg, XMMReg)
 
 #define SSE_HELPER_CMP(name, F)\
-    DEF_HELPER(void, helper_ ## name ## ps , (Reg *d, Reg *s))        \
-    DEF_HELPER(void, helper_ ## name ## ss , (Reg *d, Reg *s))        \
-    DEF_HELPER(void, helper_ ## name ## pd , (Reg *d, Reg *s))        \
-    DEF_HELPER(void, helper_ ## name ## sd , (Reg *d, Reg *s))
+    DEF_HELPER_2( name ## ps , void, Reg, Reg)        \
+    DEF_HELPER_2( name ## ss , void, Reg, Reg)        \
+    DEF_HELPER_2( name ## pd , void, Reg, Reg)        \
+    DEF_HELPER_2( name ## sd , void, Reg, Reg)
 
 SSE_HELPER_CMP(cmpeq, FPU_CMPEQ)
@@ -214 +221 @@
 SSE_HELPER_CMP(cmpord, FPU_CMPORD)
 
-DEF_HELPER(void, helper_ucomiss, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_comiss, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_ucomisd, (Reg *d, Reg *s))
-DEF_HELPER(void, helper_comisd, (Reg *d, Reg *s))
-DEF_HELPER(uint32_t, helper_movmskps, (Reg *s))
-DEF_HELPER(uint32_t, helper_movmskpd, (Reg *s))
-#endif
-
-DEF_HELPER(uint32_t, glue(helper_pmovmskb, SUFFIX), (Reg *s))
-DEF_HELPER(void, glue(helper_packsswb, SUFFIX) , (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_packuswb, SUFFIX) , (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_packssdw, SUFFIX) , (Reg *d, Reg *s))
+DEF_HELPER_2(ucomiss, void, Reg, Reg)
+DEF_HELPER_2(comiss, void, Reg, Reg)
+DEF_HELPER_2(ucomisd, void, Reg, Reg)
+DEF_HELPER_2(comisd, void, Reg, Reg)
+DEF_HELPER_1(movmskps, i32, Reg)
+DEF_HELPER_1(movmskpd, i32, Reg)
+#endif
+
+DEF_HELPER_1(glue(pmovmskb, SUFFIX), i32, Reg)
+DEF_HELPER_2(glue(packsswb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(packuswb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(packssdw, SUFFIX), void, Reg, Reg)
 #define UNPCK_OP(base_name, base)                               \
-    DEF_HELPER(void, glue(helper_punpck ## base_name ## bw, SUFFIX) , (Reg *d, Reg *s)) \
-    DEF_HELPER(void, glue(helper_punpck ## base_name ## wd, SUFFIX) , (Reg *d, Reg *s)) \
-    DEF_HELPER(void, glue(helper_punpck ## base_name ## dq, SUFFIX) , (Reg *d, Reg *s))
+    DEF_HELPER_2(glue(punpck ## base_name ## bw, SUFFIX) , void, Reg, Reg) \
+    DEF_HELPER_2(glue(punpck ## base_name ## wd, SUFFIX) , void, Reg, Reg) \
+    DEF_HELPER_2(glue(punpck ## base_name ## dq, SUFFIX) , void, Reg, Reg)
 
 UNPCK_OP(l, 0)
@@ -235 +242 @@
 
 #if SHIFT == 1
-DEF_HELPER(void, glue(helper_punpcklqdq, SUFFIX) , (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_punpckhqdq, SUFFIX) , (Reg *d, Reg *s))
+DEF_HELPER_2(glue(punpcklqdq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(punpckhqdq, SUFFIX), void, Reg, Reg)
 #endif
 
 /* 3DNow! float ops */
 #if SHIFT == 0
-DEF_HELPER(void, helper_pi2fd, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pi2fw, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pf2id, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pf2iw, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfacc, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfadd, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfcmpeq, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfcmpge, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfcmpgt, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfmax, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfmin, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfmul, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfnacc, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfpnacc, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfrcp, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfrsqrt, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfsub, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pfsubr, (MMXReg *d, MMXReg *s))
-DEF_HELPER(void, helper_pswapd, (MMXReg *d, MMXReg *s))
+DEF_HELPER_2(pi2fd, void, MMXReg, MMXReg)
+DEF_HELPER_2(pi2fw, void, MMXReg, MMXReg)
+DEF_HELPER_2(pf2id, void, MMXReg, MMXReg)
+DEF_HELPER_2(pf2iw, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfacc, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfadd, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfcmpeq, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfcmpge, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfcmpgt, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfmax, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfmin, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfmul, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfnacc, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfpnacc, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfrcp, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfrsqrt, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfsub, void, MMXReg, MMXReg)
+DEF_HELPER_2(pfsubr, void, MMXReg, MMXReg)
+DEF_HELPER_2(pswapd, void, MMXReg, MMXReg)
 #endif
 
 /* SSSE3 op helpers */
-DEF_HELPER(void, glue(helper_phaddw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_phaddd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_phaddsw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_phsubw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_phsubd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_phsubsw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pabsb, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pabsw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pabsd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmaddubsw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmulhrsw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pshufb, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psignb, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psignw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_psignd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_palignr, SUFFIX), (Reg *d, Reg *s, int32_t shift))
+DEF_HELPER_2(glue(phaddw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(phaddd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(phaddsw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(phsubw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(phsubd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(phsubsw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pabsb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pabsw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pabsd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmaddubsw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmulhrsw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pshufb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psignb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psignw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(psignd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_3(glue(palignr, SUFFIX), void, Reg, Reg, s32)
 
 /* SSE4.1 op helpers */
 #if SHIFT == 1
-DEF_HELPER(void, glue(helper_pblendvb, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_blendvps, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_blendvpd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_ptest, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovsxbw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovsxbd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovsxbq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovsxwd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovsxwq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovsxdq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovzxbw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovzxbd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovzxbq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovzxwd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovzxwq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmovzxdq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmuldq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pcmpeqq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_packusdw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pminsb, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pminsd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pminuw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pminud, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmaxsb, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmaxsd, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmaxuw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmaxud, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pmulld, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_phminposuw, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_roundps, SUFFIX), (Reg *d, Reg *s, uint32_t mode))
-DEF_HELPER(void, glue(helper_roundpd, SUFFIX), (Reg *d, Reg *s, uint32_t mode))
-DEF_HELPER(void, glue(helper_roundss, SUFFIX), (Reg *d, Reg *s, uint32_t mode))
-DEF_HELPER(void, glue(helper_roundsd, SUFFIX), (Reg *d, Reg *s, uint32_t mode))
-DEF_HELPER(void, glue(helper_blendps, SUFFIX), (Reg *d, Reg *s, uint32_t imm))
-DEF_HELPER(void, glue(helper_blendpd, SUFFIX), (Reg *d, Reg *s, uint32_t imm))
-DEF_HELPER(void, glue(helper_pblendw, SUFFIX), (Reg *d, Reg *s, uint32_t imm))
-DEF_HELPER(void, glue(helper_dpps, SUFFIX), (Reg *d, Reg *s, uint32_t mask))
-DEF_HELPER(void, glue(helper_dppd, SUFFIX), (Reg *d, Reg *s, uint32_t mask))
-DEF_HELPER(void, glue(helper_mpsadbw, SUFFIX), (Reg *d, Reg *s, uint32_t off))
+DEF_HELPER_2(glue(pblendvb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(blendvps, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(blendvpd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(ptest, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovsxbw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovsxbd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovsxbq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovsxwd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovsxwq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovsxdq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovzxbw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovzxbd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovzxbq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovzxwd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovzxwq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmovzxdq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmuldq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pcmpeqq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(packusdw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pminsb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pminsd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pminuw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pminud, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmaxsb, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmaxsd, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmaxuw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmaxud, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(pmulld, SUFFIX), void, Reg, Reg)
+DEF_HELPER_2(glue(phminposuw, SUFFIX), void, Reg, Reg)
+DEF_HELPER_3(glue(roundps, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(roundpd, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(roundss, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(roundsd, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(blendps, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(blendpd, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(pblendw, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(dpps, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(dppd, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(mpsadbw, SUFFIX), void, Reg, Reg, i32)
 #endif
 
 /* SSE4.2 op helpers */
 #if SHIFT == 1
-DEF_HELPER(void, glue(helper_pcmpgtq, SUFFIX), (Reg *d, Reg *s))
-DEF_HELPER(void, glue(helper_pcmpestri, SUFFIX), (Reg *d, Reg *s, uint32_t ctl))
-DEF_HELPER(void, glue(helper_pcmpestrm, SUFFIX), (Reg *d, Reg *s, uint32_t ctl))
-DEF_HELPER(void, glue(helper_pcmpistri, SUFFIX), (Reg *d, Reg *s, uint32_t ctl))
-DEF_HELPER(void, glue(helper_pcmpistrm, SUFFIX), (Reg *d, Reg *s, uint32_t ctl))
-DEF_HELPER(target_ulong, helper_crc32,
-                (uint32_t crc1, target_ulong msg, uint32_t len))
-DEF_HELPER(target_ulong, helper_popcnt, (target_ulong n, uint32_t type))
+DEF_HELPER_2(glue(pcmpgtq, SUFFIX), void, Reg, Reg)
+DEF_HELPER_3(glue(pcmpestri, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(pcmpestrm, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(pcmpistri, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(glue(pcmpistrm, SUFFIX), void, Reg, Reg, i32)
+DEF_HELPER_3(crc32, tl, i32, tl, i32)
+DEF_HELPER_2(popcnt, tl, tl, i32)
 #endif
 

trunk/src/recompiler/target-i386/translate.c

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -33 +33 @@
 #include <string.h>
 #ifndef VBOX
-# include <inttypes.h>
-# include <signal.h>
-# include <assert.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <assert.h>
 #endif /* !VBOX */
 
@@ -41 +41 @@
 #include "exec-all.h"
 #include "disas.h"
+#include "tcg-op.h"
+
 #include "helper.h"
-#include "tcg-op.h"
+#define GEN_HELPER 1
+#include "helper.h"
 
 #define PREFIX_REPZ   0x01
@@ -71 +74 @@
 
 /* global register indexes */
-static TCGv cpu_env, cpu_A0, cpu_cc_op, cpu_cc_src, cpu_cc_dst, cpu_cc_tmp;
+static TCGv_ptr cpu_env;
+static TCGv cpu_A0, cpu_cc_src, cpu_cc_dst, cpu_cc_tmp;
+static TCGv_i32 cpu_cc_op;
 /* local temps */
 static TCGv cpu_T[2], cpu_T3;
 /* local register indexes (only used inside old micro ops) */
-static TCGv cpu_tmp0, cpu_tmp1_i64, cpu_tmp2_i32, cpu_tmp3_i32, cpu_tmp4, cpu_ptr0, cpu_ptr1;
+static TCGv cpu_tmp0, cpu_tmp4;
+static TCGv_ptr cpu_ptr0, cpu_ptr1;
+static TCGv_i32 cpu_tmp2_i32, cpu_tmp3_i32;
+static TCGv_i64 cpu_tmp1_i64;
 static TCGv cpu_tmp5, cpu_tmp6;
 
@@ -658 +666 @@
     default:
     case 3:
+        /* Should never happen on 32-bit targets.  */
+#ifdef TARGET_X86_64
         tcg_gen_qemu_ld64(t0, a0, mem_index);
+#endif
         break;
     }
@@ -694 +705 @@
     default:
     case 3:
+        /* Should never happen on 32-bit targets.  */
+#ifdef TARGET_X86_64
         tcg_gen_qemu_st64(t0, a0, mem_index);
+#endif
         break;
     }
@@ -716 +730 @@
     /** @todo: once TCG codegen improves, we may want to use version
         from else version */
-    tcg_gen_helper_0_0(helper_check_external_event);
+    gen_helper_check_external_event();
 # else
     int skip_label;
@@ -736 +750 @@
     tcg_temp_free(t0);
 
-    tcg_gen_helper_0_0(helper_check_external_event);
+    gen_helper_check_external_event();
 
    gen_set_label(skip_label);
@@ -745 +759 @@
 static void gen_check_external_event2()
 {
-    tcg_gen_helper_0_0(helper_check_external_event);
+    gen_helper_check_external_event();
 }
 # endif
@@ -762 +776 @@
     gen_jmp_im(pc);
 # ifdef VBOX_DUMP_STATE
-    tcg_gen_helper_0_0(helper_dump_state);
+    gen_helper_dump_state();
 # endif
 }
@@ -877 +891 @@
 }
 
-static void *helper_in_func[3] = {
-    helper_inb,
-    helper_inw,
-    helper_inl,
-};
-
-static void *helper_out_func[3] = {
-    helper_outb,
-    helper_outw,
-    helper_outl,
-};
-
-static void *gen_check_io_func[3] = {
-    helper_check_iob,
-    helper_check_iow,
-    helper_check_iol,
-};
+static void gen_helper_in_func(int ot, TCGv v, TCGv_i32 n)
+{
+    switch (ot) {
+    case 0: gen_helper_inb(v, n); break;
+    case 1: gen_helper_inw(v, n); break;
+    case 2: gen_helper_inl(v, n); break;
+    }
+
+}
+
+static void gen_helper_out_func(int ot, TCGv_i32 v, TCGv_i32 n)
+{
+    switch (ot) {
+    case 0: gen_helper_outb(v, n); break;
+    case 1: gen_helper_outw(v, n); break;
+    case 2: gen_helper_outl(v, n); break;
+    }
+
+}
 
 static void gen_check_io(DisasContext *s, int ot, target_ulong cur_eip,
@@ -908 +924 @@
         state_saved = 1;
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_helper_0_1(gen_check_io_func[ot],
-                           cpu_tmp2_i32);
+        switch (ot) {
+        case 0: gen_helper_check_iob(cpu_tmp2_i32); break;
+        case 1: gen_helper_check_iow(cpu_tmp2_i32); break;
+        case 2: gen_helper_check_iol(cpu_tmp2_i32); break;
+        }
     }
     if(s->flags & HF_SVMI_MASK) {
@@ -921 +940 @@
         next_eip = s->pc - s->cs_base;
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_helper_0_3(helper_svm_check_io,
-                           cpu_tmp2_i32,
-                           tcg_const_i32(svm_flags),
-                           tcg_const_i32(next_eip - cur_eip));
+        gen_helper_svm_check_io(cpu_tmp2_i32, tcg_const_i32(svm_flags),
+                                tcg_const_i32(next_eip - cur_eip));
     }
 }
@@ -980 +997 @@
 static void gen_compute_eflags_c(TCGv reg)
 {
-#if TCG_TARGET_REG_BITS == 32
-    tcg_gen_shli_i32(cpu_tmp2_i32, cpu_cc_op, 3);
-    tcg_gen_addi_i32(cpu_tmp2_i32, cpu_tmp2_i32,
-                     (long)cc_table + offsetof(CCTable, compute_c));
-    tcg_gen_ld_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0);
-    tcg_gen_call(&tcg_ctx, cpu_tmp2_i32, TCG_CALL_PURE,
-                 1, &cpu_tmp2_i32, 0, NULL);
-#else
-    tcg_gen_extu_i32_tl(cpu_tmp1_i64, cpu_cc_op);
-    tcg_gen_shli_i64(cpu_tmp1_i64, cpu_tmp1_i64, 4);
-    tcg_gen_addi_i64(cpu_tmp1_i64, cpu_tmp1_i64,
-                     (long)cc_table + offsetof(CCTable, compute_c));
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_tmp1_i64, 0);
-    tcg_gen_call(&tcg_ctx, cpu_tmp1_i64, TCG_CALL_PURE,
-                 1, &cpu_tmp2_i32, 0, NULL);
-#endif
+    gen_helper_cc_compute_c(cpu_tmp2_i32, cpu_cc_op);
     tcg_gen_extu_i32_tl(reg, cpu_tmp2_i32);
 }
@@ -1002 +1004 @@
 static void gen_compute_eflags(TCGv reg)
 {
-#if TCG_TARGET_REG_BITS == 32
-    tcg_gen_shli_i32(cpu_tmp2_i32, cpu_cc_op, 3);
-    tcg_gen_addi_i32(cpu_tmp2_i32, cpu_tmp2_i32,
-                     (long)cc_table + offsetof(CCTable, compute_all));
-    tcg_gen_ld_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0);
-    tcg_gen_call(&tcg_ctx, cpu_tmp2_i32, TCG_CALL_PURE,
-                 1, &cpu_tmp2_i32, 0, NULL);
-#else
-    tcg_gen_extu_i32_tl(cpu_tmp1_i64, cpu_cc_op);
-    tcg_gen_shli_i64(cpu_tmp1_i64, cpu_tmp1_i64, 4);
-    tcg_gen_addi_i64(cpu_tmp1_i64, cpu_tmp1_i64,
-                     (long)cc_table + offsetof(CCTable, compute_all));
-    tcg_gen_ld_i64(cpu_tmp1_i64, cpu_tmp1_i64, 0);
-    tcg_gen_call(&tcg_ctx, cpu_tmp1_i64, TCG_CALL_PURE,
-                 1, &cpu_tmp2_i32, 0, NULL);
-#endif
+    gen_helper_cc_compute_all(cpu_tmp2_i32, cpu_cc_op);
     tcg_gen_extu_i32_tl(reg, cpu_tmp2_i32);
 }
@@ -1391 +1378 @@
     tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[1]);
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
-    tcg_gen_helper_1_1(helper_in_func[ot], cpu_T[0], cpu_tmp2_i32);
+    gen_helper_in_func(ot, cpu_T[0], cpu_tmp2_i32);
     gen_op_st_T0_A0(ot + s->mem_index);
     gen_op_movl_T0_Dshift(ot);
@@ -1410 +1397 @@
     tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
     tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[0]);
-    tcg_gen_helper_0_2(helper_out_func[ot], cpu_tmp2_i32, cpu_tmp3_i32);
+    gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
 
     gen_op_movl_T0_Dshift(ot);
@@ -1462 +1449 @@
 GEN_REPZ2(cmps)
 
-static void *helper_fp_arith_ST0_FT0[8] = {
-    helper_fadd_ST0_FT0,
-    helper_fmul_ST0_FT0,
-    helper_fcom_ST0_FT0,
-    helper_fcom_ST0_FT0,
-    helper_fsub_ST0_FT0,
-    helper_fsubr_ST0_FT0,
-    helper_fdiv_ST0_FT0,
-    helper_fdivr_ST0_FT0,
-};
+static void gen_helper_fp_arith_ST0_FT0(int op)
+{
+    switch (op) {
+    case 0: gen_helper_fadd_ST0_FT0(); break;
+    case 1: gen_helper_fmul_ST0_FT0(); break;
+    case 2: gen_helper_fcom_ST0_FT0(); break;
+    case 3: gen_helper_fcom_ST0_FT0(); break;
+    case 4: gen_helper_fsub_ST0_FT0(); break;
+    case 5: gen_helper_fsubr_ST0_FT0(); break;
+    case 6: gen_helper_fdiv_ST0_FT0(); break;
+    case 7: gen_helper_fdivr_ST0_FT0(); break;
+    }
+}
 
 /* NOTE the exception in "r" op ordering */
-static void *helper_fp_arith_STN_ST0[8] = {
-    helper_fadd_STN_ST0,
-    helper_fmul_STN_ST0,
-    NULL,
-    NULL,
-    helper_fsubr_STN_ST0,
-    helper_fsub_STN_ST0,
-    helper_fdivr_STN_ST0,
-    helper_fdiv_STN_ST0,
-};
+static void gen_helper_fp_arith_STN_ST0(int op, int opreg)
+{
+    TCGv_i32 tmp = tcg_const_i32(opreg);
+    switch (op) {
+    case 0: gen_helper_fadd_STN_ST0(tmp); break;
+    case 1: gen_helper_fmul_STN_ST0(tmp); break;
+    case 4: gen_helper_fsubr_STN_ST0(tmp); break;
+    case 5: gen_helper_fsub_STN_ST0(tmp); break;
+    case 6: gen_helper_fdivr_STN_ST0(tmp); break;
+    case 7: gen_helper_fdiv_STN_ST0(tmp); break;
+    }
+}
 
 /* if d == OR_TMP0, it means memory operand (address in A0) */
@@ -1653 +1645 @@
 
     /* XXX: inefficient */
-    t0 = tcg_temp_local_new(TCG_TYPE_TL);
-    t1 = tcg_temp_local_new(TCG_TYPE_TL);
+    t0 = tcg_temp_local_new();
+    t1 = tcg_temp_local_new();
 
     tcg_gen_mov_tl(t0, cpu_T[0]);
@@ -1744 +1736 @@
 
     /* XXX: inefficient, but we must use local temps */
-    t0 = tcg_temp_local_new(TCG_TYPE_TL);
-    t1 = tcg_temp_local_new(TCG_TYPE_TL);
-    t2 = tcg_temp_local_new(TCG_TYPE_TL);
-    a0 = tcg_temp_local_new(TCG_TYPE_TL);
+    t0 = tcg_temp_local_new();
+    t1 = tcg_temp_local_new();
+    t2 = tcg_temp_local_new();
+    a0 = tcg_temp_local_new();
 
     if (ot == OT_QUAD)
@@ -1832 +1824 @@
 }
 
-static void *helper_rotc[8] = {
-    helper_rclb,
-    helper_rclw,
-    helper_rcll,
-    X86_64_ONLY(helper_rclq),
-    helper_rcrb,
-    helper_rcrw,
-    helper_rcrl,
-    X86_64_ONLY(helper_rcrq),
-};
-
 /* XXX: add faster immediate = 1 case */
 static void gen_rotc_rm_T1(DisasContext *s, int ot, int op1,
@@ -1858 +1839 @@
         gen_op_mov_TN_reg(ot, 0, op1);
 
-    tcg_gen_helper_1_2(helper_rotc[ot + (is_right * 4)],
-                       cpu_T[0], cpu_T[0], cpu_T[1]);
+    if (is_right) {
+        switch (ot) {
+        case 0: gen_helper_rcrb(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+        case 1: gen_helper_rcrw(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+        case 2: gen_helper_rcrl(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+#ifdef TARGET_X86_64
+        case 3: gen_helper_rcrq(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+#endif
+        }
+    } else {
+        switch (ot) {
+        case 0: gen_helper_rclb(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+        case 1: gen_helper_rclw(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+        case 2: gen_helper_rcll(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+#ifdef TARGET_X86_64
+        case 3: gen_helper_rclq(cpu_T[0], cpu_T[0], cpu_T[1]); break;
+#endif
+        }
+    }
     /* store */
     if (op1 == OR_TMP0)
@@ -1886 +1884 @@
     TCGv t0, t1, t2, a0;
 
-    t0 = tcg_temp_local_new(TCG_TYPE_TL);
-    t1 = tcg_temp_local_new(TCG_TYPE_TL);
-    t2 = tcg_temp_local_new(TCG_TYPE_TL);
-    a0 = tcg_temp_local_new(TCG_TYPE_TL);
+    t0 = tcg_temp_local_new();
+    t1 = tcg_temp_local_new();
+    t2 = tcg_temp_local_new();
+    a0 = tcg_temp_local_new();
 
     if (ot == OT_QUAD)
@@ -2449 +2447 @@
         /* nominal case: we use a jump */
         /* XXX: make it faster by adding new instructions in TCG */
-        t0 = tcg_temp_local_new(TCG_TYPE_TL);
+        t0 = tcg_temp_local_new();
         tcg_gen_movi_tl(t0, 0);
         l1 = gen_new_label();
@@ -2507 +2505 @@
         gen_jmp_im(cur_eip);
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_helper_0_2(helper_load_seg, tcg_const_i32(seg_reg), cpu_tmp2_i32);
+        gen_helper_load_seg(tcg_const_i32(seg_reg), cpu_tmp2_i32);
         /* abort translation because the addseg value may change or
            because ss32 may change. For R_SS, translation must always
@@ -2536 +2534 @@
         gen_op_set_cc_op(s->cc_op);
     gen_jmp_im(pc_start - s->cs_base);
-    tcg_gen_helper_0_2(helper_svm_check_intercept_param,
-                       tcg_const_i32(type), tcg_const_i64(param));
+    gen_helper_svm_check_intercept_param(tcg_const_i32(type),
+                                         tcg_const_i64(param));
 }
 
@@ -2744 +2742 @@
         if (level) {
             /* XXX: must save state */
-            tcg_gen_helper_0_3(helper_enter64_level,
-                               tcg_const_i32(level),
-                               tcg_const_i32((ot == OT_QUAD)),
-                               cpu_T[1]);
+            gen_helper_enter64_level(tcg_const_i32(level),
+                                     tcg_const_i32((ot == OT_QUAD)),
+                                     cpu_T[1]);
         }
         gen_op_mov_reg_T1(ot, R_EBP);
@@ -2770 +2767 @@
         if (level) {
             /* XXX: must save state */
-            tcg_gen_helper_0_3(helper_enter_level,
-                               tcg_const_i32(level),
-                               tcg_const_i32(s->dflag),
-                               cpu_T[1]);
+            gen_helper_enter_level(tcg_const_i32(level),
+                                   tcg_const_i32(s->dflag),
+                                   cpu_T[1]);
         }
         gen_op_mov_reg_T1(ot, R_EBP);
@@ -2786 +2782 @@
         gen_op_set_cc_op(s->cc_op);
     gen_jmp_im(cur_eip);
-    tcg_gen_helper_0_1(helper_raise_exception, tcg_const_i32(trapno));
+    gen_helper_raise_exception(tcg_const_i32(trapno));
     s->is_jmp = 3;
 }
@@ -2798 +2794 @@
         gen_op_set_cc_op(s->cc_op);
     gen_jmp_im(cur_eip);
-    tcg_gen_helper_0_2(helper_raise_interrupt,
-                       tcg_const_i32(intno),
-                       tcg_const_i32(next_eip - cur_eip));
+    gen_helper_raise_interrupt(tcg_const_i32(intno),
+                               tcg_const_i32(next_eip - cur_eip));
     s->is_jmp = 3;
 }
@@ -2809 +2804 @@
         gen_op_set_cc_op(s->cc_op);
     gen_jmp_im(cur_eip);
-    tcg_gen_helper_0_0(helper_debug);
+    gen_helper_debug();
     s->is_jmp = 3;
 }
@@ -2820 +2815 @@
         gen_op_set_cc_op(s->cc_op);
     if (s->tb->flags & HF_INHIBIT_IRQ_MASK) {
-        tcg_gen_helper_0_0(helper_reset_inhibit_irq);
+        gen_helper_reset_inhibit_irq();
     }
-
-#ifdef VBOX
-    gen_check_external_event();
-#endif /* VBOX */
-
     if (   s->singlestep_enabled
 #ifdef VBOX
@@ -2833 +2823 @@
 #endif
        ) {
-        tcg_gen_helper_0_0(helper_debug);
+        gen_helper_debug();
     } else if (s->tf) {
-        tcg_gen_helper_0_0(helper_single_step);
+        gen_helper_single_step();
     } else {
         tcg_gen_exit_tb(0);
@@ -2927 +2917 @@
 #define SSE_DUMMY ((void *)2)
 
-#define MMX_OP2(x) { helper_ ## x ## _mmx, helper_ ## x ## _xmm }
-#define SSE_FOP(x) { helper_ ## x ## ps, helper_ ## x ## pd, \
-                     helper_ ## x ## ss, helper_ ## x ## sd, }
+#define MMX_OP2(x) { gen_helper_ ## x ## _mmx, gen_helper_ ## x ## _xmm }
+#define SSE_FOP(x) { gen_helper_ ## x ## ps, gen_helper_ ## x ## pd, \
+                     gen_helper_ ## x ## ss, gen_helper_ ## x ## sd, }
 
 static void *sse_op_table1[256][4] = {
@@ -2940 +2930 @@
     [0x12] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movlps, movlpd, movsldup, movddup */
     [0x13] = { SSE_SPECIAL, SSE_SPECIAL },  /* movlps, movlpd */
-    [0x14] = { helper_punpckldq_xmm, helper_punpcklqdq_xmm },
-    [0x15] = { helper_punpckhdq_xmm, helper_punpckhqdq_xmm },
+    [0x14] = { gen_helper_punpckldq_xmm, gen_helper_punpcklqdq_xmm },
+    [0x15] = { gen_helper_punpckhdq_xmm, gen_helper_punpckhqdq_xmm },
     [0x16] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL },  /* movhps, movhpd, movshdup */
     [0x17] = { SSE_SPECIAL, SSE_SPECIAL },  /* movhps, movhpd */
@@ -2951 +2941 @@
     [0x2c] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvttps2pi, cvttpd2pi, cvttsd2si, cvttss2si */
     [0x2d] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* cvtps2pi, cvtpd2pi, cvtsd2si, cvtss2si */
-    [0x2e] = { helper_ucomiss, helper_ucomisd },
-    [0x2f] = { helper_comiss, helper_comisd },
+    [0x2e] = { gen_helper_ucomiss, gen_helper_ucomisd },
+    [0x2f] = { gen_helper_comiss, gen_helper_comisd },
     [0x50] = { SSE_SPECIAL, SSE_SPECIAL }, /* movmskps, movmskpd */
     [0x51] = SSE_FOP(sqrt),
-    [0x52] = { helper_rsqrtps, NULL, helper_rsqrtss, NULL },
-    [0x53] = { helper_rcpps, NULL, helper_rcpss, NULL },
-    [0x54] = { helper_pand_xmm, helper_pand_xmm }, /* andps, andpd */
-    [0x55] = { helper_pandn_xmm, helper_pandn_xmm }, /* andnps, andnpd */
-    [0x56] = { helper_por_xmm, helper_por_xmm }, /* orps, orpd */
-    [0x57] = { helper_pxor_xmm, helper_pxor_xmm }, /* xorps, xorpd */
+    [0x52] = { gen_helper_rsqrtps, NULL, gen_helper_rsqrtss, NULL },
+    [0x53] = { gen_helper_rcpps, NULL, gen_helper_rcpss, NULL },
+    [0x54] = { gen_helper_pand_xmm, gen_helper_pand_xmm }, /* andps, andpd */
+    [0x55] = { gen_helper_pandn_xmm, gen_helper_pandn_xmm }, /* andnps, andnpd */
+    [0x56] = { gen_helper_por_xmm, gen_helper_por_xmm }, /* orps, orpd */
+    [0x57] = { gen_helper_pxor_xmm, gen_helper_pxor_xmm }, /* xorps, xorpd */
     [0x58] = SSE_FOP(add),
     [0x59] = SSE_FOP(mul),
-    [0x5a] = { helper_cvtps2pd, helper_cvtpd2ps,
-               helper_cvtss2sd, helper_cvtsd2ss },
-    [0x5b] = { helper_cvtdq2ps, helper_cvtps2dq, helper_cvttps2dq },
+    [0x5a] = { gen_helper_cvtps2pd, gen_helper_cvtpd2ps,
+               gen_helper_cvtss2sd, gen_helper_cvtsd2ss },
+    [0x5b] = { gen_helper_cvtdq2ps, gen_helper_cvtps2dq, gen_helper_cvttps2dq },
     [0x5c] = SSE_FOP(sub),
     [0x5d] = SSE_FOP(min),
@@ -2972 +2962 @@
 
     [0xc2] = SSE_FOP(cmpeq),
-    [0xc6] = { helper_shufps, helper_shufpd },
+    [0xc6] = { gen_helper_shufps, gen_helper_shufpd },
 
     [0x38] = { SSE_SPECIAL, SSE_SPECIAL, NULL, SSE_SPECIAL }, /* SSSE3/SSE4 */
@@ -2990 +2980 @@
     [0x6a] = MMX_OP2(punpckhdq),
     [0x6b] = MMX_OP2(packssdw),
-    [0x6c] = { NULL, helper_punpcklqdq_xmm },
-    [0x6d] = { NULL, helper_punpckhqdq_xmm },
+    [0x6c] = { NULL, gen_helper_punpcklqdq_xmm },
+    [0x6d] = { NULL, gen_helper_punpckhqdq_xmm },
     [0x6e] = { SSE_SPECIAL, SSE_SPECIAL }, /* movd mm, ea */
     [0x6f] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movq, movdqa, , movqdu */
-    [0x70] = { helper_pshufw_mmx,
-               helper_pshufd_xmm,
-               helper_pshufhw_xmm,
-               helper_pshuflw_xmm },
+    [0x70] = { gen_helper_pshufw_mmx,
+               gen_helper_pshufd_xmm,
+               gen_helper_pshufhw_xmm,
+               gen_helper_pshuflw_xmm },
     [0x71] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftw */
     [0x72] = { SSE_SPECIAL, SSE_SPECIAL }, /* shiftd */
@@ -3005 +2995 @@
     [0x76] = MMX_OP2(pcmpeql),
     [0x77] = { SSE_DUMMY }, /* emms */
-    [0x7c] = { NULL, helper_haddpd, NULL, helper_haddps },
-    [0x7d] = { NULL, helper_hsubpd, NULL, helper_hsubps },
+    [0x7c] = { NULL, gen_helper_haddpd, NULL, gen_helper_haddps },
+    [0x7d] = { NULL, gen_helper_hsubpd, NULL, gen_helper_hsubps },
     [0x7e] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movd, movd, , movq */
     [0x7f] = { SSE_SPECIAL, SSE_SPECIAL, SSE_SPECIAL }, /* movq, movdqa, movdqu */
     [0xc4] = { SSE_SPECIAL, SSE_SPECIAL }, /* pinsrw */
     [0xc5] = { SSE_SPECIAL, SSE_SPECIAL }, /* pextrw */
-    [0xd0] = { NULL, helper_addsubpd, NULL, helper_addsubps },
+    [0xd0] = { NULL, gen_helper_addsubpd, NULL, gen_helper_addsubps },
     [0xd1] = MMX_OP2(psrlw),
     [0xd2] = MMX_OP2(psrld),
@@ -3033 +3023 @@
     [0xe4] = MMX_OP2(pmulhuw),
     [0xe5] = MMX_OP2(pmulhw),
-    [0xe6] = { NULL, helper_cvttpd2dq, helper_cvtdq2pd, helper_cvtpd2dq },
+    [0xe6] = { NULL, gen_helper_cvttpd2dq, gen_helper_cvtdq2pd, gen_helper_cvtpd2dq },
     [0xe7] = { SSE_SPECIAL , SSE_SPECIAL },  /* movntq, movntq */
     [0xe8] = MMX_OP2(psubsb),
@@ -3068 +3058 @@
     [8 + 6] = MMX_OP2(pslld),
     [16 + 2] = MMX_OP2(psrlq),
-    [16 + 3] = { NULL, helper_psrldq_xmm },
+    [16 + 3] = { NULL, gen_helper_psrldq_xmm },
     [16 + 6] = MMX_OP2(psllq),
-    [16 + 7] = { NULL, helper_pslldq_xmm },
+    [16 + 7] = { NULL, gen_helper_pslldq_xmm },
 };
 
 static void *sse_op_table3[4 * 3] = {
-    helper_cvtsi2ss,
-    helper_cvtsi2sd,
-    X86_64_ONLY(helper_cvtsq2ss),
-    X86_64_ONLY(helper_cvtsq2sd),
-
-    helper_cvttss2si,
-    helper_cvttsd2si,
-    X86_64_ONLY(helper_cvttss2sq),
-    X86_64_ONLY(helper_cvttsd2sq),
-
-    helper_cvtss2si,
-    helper_cvtsd2si,
-    X86_64_ONLY(helper_cvtss2sq),
-    X86_64_ONLY(helper_cvtsd2sq),
+    gen_helper_cvtsi2ss,
+    gen_helper_cvtsi2sd,
+    X86_64_ONLY(gen_helper_cvtsq2ss),
+    X86_64_ONLY(gen_helper_cvtsq2sd),
+
+    gen_helper_cvttss2si,
+    gen_helper_cvttsd2si,
+    X86_64_ONLY(gen_helper_cvttss2sq),
+    X86_64_ONLY(gen_helper_cvttsd2sq),
+
+    gen_helper_cvtss2si,
+    gen_helper_cvtsd2si,
+    X86_64_ONLY(gen_helper_cvtss2sq),
+    X86_64_ONLY(gen_helper_cvtsd2sq),
 };
 
@@ -3102 +3092 @@
 
 static void *sse_op_table5[256] = {
-    [0x0c] = helper_pi2fw,
-    [0x0d] = helper_pi2fd,
-    [0x1c] = helper_pf2iw,
-    [0x1d] = helper_pf2id,
-    [0x8a] = helper_pfnacc,
-    [0x8e] = helper_pfpnacc,
-    [0x90] = helper_pfcmpge,
-    [0x94] = helper_pfmin,
-    [0x96] = helper_pfrcp,
-    [0x97] = helper_pfrsqrt,
-    [0x9a] = helper_pfsub,
-    [0x9e] = helper_pfadd,
-    [0xa0] = helper_pfcmpgt,
-    [0xa4] = helper_pfmax,
-    [0xa6] = helper_movq, /* pfrcpit1; no need to actually increase precision */
-    [0xa7] = helper_movq, /* pfrsqit1 */
-    [0xaa] = helper_pfsubr,
-    [0xae] = helper_pfacc,
-    [0xb0] = helper_pfcmpeq,
-    [0xb4] = helper_pfmul,
-    [0xb6] = helper_movq, /* pfrcpit2 */
-    [0xb7] = helper_pmulhrw_mmx,
-    [0xbb] = helper_pswapd,
-    [0xbf] = helper_pavgb_mmx /* pavgusb */
+    [0x0c] = gen_helper_pi2fw,
+    [0x0d] = gen_helper_pi2fd,
+    [0x1c] = gen_helper_pf2iw,
+    [0x1d] = gen_helper_pf2id,
+    [0x8a] = gen_helper_pfnacc,
+    [0x8e] = gen_helper_pfpnacc,
+    [0x90] = gen_helper_pfcmpge,
+    [0x94] = gen_helper_pfmin,
+    [0x96] = gen_helper_pfrcp,
+    [0x97] = gen_helper_pfrsqrt,
+    [0x9a] = gen_helper_pfsub,
+    [0x9e] = gen_helper_pfadd,
+    [0xa0] = gen_helper_pfcmpgt,
+    [0xa4] = gen_helper_pfmax,
+    [0xa6] = gen_helper_movq, /* pfrcpit1; no need to actually increase precision */
+    [0xa7] = gen_helper_movq, /* pfrsqit1 */
+    [0xaa] = gen_helper_pfsubr,
+    [0xae] = gen_helper_pfacc,
+    [0xb0] = gen_helper_pfcmpeq,
+    [0xb4] = gen_helper_pfmul,
+    [0xb6] = gen_helper_movq, /* pfrcpit2 */
+    [0xb7] = gen_helper_pmulhrw_mmx,
+    [0xbb] = gen_helper_pswapd,
+    [0xbf] = gen_helper_pavgb_mmx /* pavgusb */
 };
 
@@ -3132 +3122 @@
 };
 #define SSSE3_OP(x) { MMX_OP2(x), CPUID_EXT_SSSE3 }
-#define SSE41_OP(x) { { NULL, helper_ ## x ## _xmm }, CPUID_EXT_SSE41 }
-#define SSE42_OP(x) { { NULL, helper_ ## x ## _xmm }, CPUID_EXT_SSE42 }
+#define SSE41_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_SSE41 }
+#define SSE42_OP(x) { { NULL, gen_helper_ ## x ## _xmm }, CPUID_EXT_SSE42 }
 #define SSE41_SPECIAL { { NULL, SSE_SPECIAL }, CPUID_EXT_SSE41 }
 static struct sse_op_helper_s sse_op_table6[256] = {
@@ -3254 +3244 @@
             goto illegal_op;
         /* femms */
-        tcg_gen_helper_0_0(helper_emms);
+        gen_helper_emms();
         return;
     }
     if (b == 0x77) {
         /* emms */
-        tcg_gen_helper_0_0(helper_emms);
+        gen_helper_emms();
         return;
     }
@@ -3265 +3255 @@
        the static cpu state) */
     if (!is_xmm) {
-        tcg_gen_helper_0_0(helper_enter_mmx);
+        gen_helper_enter_mmx();
     }
 
@@ -3302 +3292 @@
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
                                  offsetof(CPUX86State,fpregs[reg].mmx));
-                tcg_gen_helper_0_2(helper_movl_mm_T0_mmx, cpu_ptr0, cpu_T[0]);
+                tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
+                gen_helper_movl_mm_T0_mmx(cpu_ptr0, cpu_tmp2_i32);
             }
             break;
@@ -3311 +3302 @@
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
                                  offsetof(CPUX86State,xmm_regs[reg]));
-                tcg_gen_helper_0_2(helper_movq_mm_T0_xmm, cpu_ptr0, cpu_T[0]);
+                gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]);
             } else
 #endif
@@ -3319 +3310 @@
                                  offsetof(CPUX86State,xmm_regs[reg]));
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_2(helper_movl_mm_T0_xmm, cpu_ptr0, cpu_tmp2_i32);
+                gen_helper_movl_mm_T0_xmm(cpu_ptr0, cpu_tmp2_i32);
             }
             break;
@@ -3580 +3571 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op2_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op1_offset);
-            tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_ptr1);
+            ((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
             break;
         case 0x050: /* movmskps */
@@ -3586 +3577 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
                              offsetof(CPUX86State,xmm_regs[rm]));
-            tcg_gen_helper_1_1(helper_movmskps, cpu_tmp2_i32, cpu_ptr0);
+            gen_helper_movmskps(cpu_tmp2_i32, cpu_ptr0);
             tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
             gen_op_mov_reg_T0(OT_LONG, reg);
@@ -3594 +3585 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
                              offsetof(CPUX86State,xmm_regs[rm]));
-            tcg_gen_helper_1_1(helper_movmskpd, cpu_tmp2_i32, cpu_ptr0);
+            gen_helper_movmskpd(cpu_tmp2_i32, cpu_ptr0);
             tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
             gen_op_mov_reg_T0(OT_LONG, reg);
@@ -3600 +3591 @@
         case 0x02a: /* cvtpi2ps */
         case 0x12a: /* cvtpi2pd */
-            tcg_gen_helper_0_0(helper_enter_mmx);
+            gen_helper_enter_mmx();
             if (mod != 3) {
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
@@ -3614 +3605 @@
             switch(b >> 8) {
             case 0x0:
-                tcg_gen_helper_0_2(helper_cvtpi2ps, cpu_ptr0, cpu_ptr1);
+                gen_helper_cvtpi2ps(cpu_ptr0, cpu_ptr1);
                 break;
             default:
             case 0x1:
-                tcg_gen_helper_0_2(helper_cvtpi2pd, cpu_ptr0, cpu_ptr1);
+                gen_helper_cvtpi2pd(cpu_ptr0, cpu_ptr1);
                 break;
             }
@@ -3631 +3622 @@
             if (ot == OT_LONG) {
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_tmp2_i32);
+                ((void (*)(TCGv_ptr, TCGv_i32))sse_op2)(cpu_ptr0, cpu_tmp2_i32);
             } else {
-                tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_T[0]);
+                ((void (*)(TCGv_ptr, TCGv))sse_op2)(cpu_ptr0, cpu_T[0]);
             }
             break;
@@ -3640 +3631 @@
         case 0x02d: /* cvtps2pi */
         case 0x12d: /* cvtpd2pi */
-            tcg_gen_helper_0_0(helper_enter_mmx);
+            gen_helper_enter_mmx();
             if (mod != 3) {
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
@@ -3654 +3645 @@
             switch(b) {
             case 0x02c:
-                tcg_gen_helper_0_2(helper_cvttps2pi, cpu_ptr0, cpu_ptr1);
+                gen_helper_cvttps2pi(cpu_ptr0, cpu_ptr1);
                 break;
             case 0x12c:
-                tcg_gen_helper_0_2(helper_cvttpd2pi, cpu_ptr0, cpu_ptr1);
+                gen_helper_cvttpd2pi(cpu_ptr0, cpu_ptr1);
                 break;
             case 0x02d:
-                tcg_gen_helper_0_2(helper_cvtps2pi, cpu_ptr0, cpu_ptr1);
+                gen_helper_cvtps2pi(cpu_ptr0, cpu_ptr1);
                 break;
             case 0x12d:
-                tcg_gen_helper_0_2(helper_cvtpd2pi, cpu_ptr0, cpu_ptr1);
+                gen_helper_cvtpd2pi(cpu_ptr0, cpu_ptr1);
                 break;
             }
@@ -3689 +3680 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op2_offset);
             if (ot == OT_LONG) {
-                tcg_gen_helper_1_1(sse_op2, cpu_tmp2_i32, cpu_ptr0);
+                ((void (*)(TCGv_i32, TCGv_ptr))sse_op2)(cpu_tmp2_i32, cpu_ptr0);
                 tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
             } else {
-                tcg_gen_helper_1_1(sse_op2, cpu_T[0], cpu_ptr0);
+                ((void (*)(TCGv, TCGv_ptr))sse_op2)(cpu_T[0], cpu_ptr0);
             }
             gen_op_mov_reg_T0(ot, reg);
@@ -3743 +3734 @@
             break;
         case 0x2d6: /* movq2dq */
-            tcg_gen_helper_0_0(helper_enter_mmx);
+            gen_helper_enter_mmx();
             rm = (modrm & 7);
             gen_op_movq(offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)),
@@ -3750 +3741 @@
             break;
         case 0x3d6: /* movdq2q */
-            tcg_gen_helper_0_0(helper_enter_mmx);
+            gen_helper_enter_mmx();
             rm = (modrm & 7) | REX_B(s);
             gen_op_movq(offsetof(CPUX86State,fpregs[reg & 7].mmx),
@@ -3762 +3753 @@
                 rm = (modrm & 7) | REX_B(s);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, offsetof(CPUX86State,xmm_regs[rm]));
-                tcg_gen_helper_1_1(helper_pmovmskb_xmm, cpu_tmp2_i32, cpu_ptr0);
+                gen_helper_pmovmskb_xmm(cpu_tmp2_i32, cpu_ptr0);
             } else {
                 rm = (modrm & 7);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, offsetof(CPUX86State,fpregs[rm].mmx));
-                tcg_gen_helper_1_1(helper_pmovmskb_mmx, cpu_tmp2_i32, cpu_ptr0);
+                gen_helper_pmovmskb_mmx(cpu_tmp2_i32, cpu_ptr0);
             }
             tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
@@ -3804 +3795 @@
                     case 0x21: case 0x31: /* pmovsxbd, pmovzxbd */
                     case 0x24: case 0x34: /* pmovsxwq, pmovzxwq */
-                        tcg_gen_qemu_ld32u(cpu_tmp2_i32, cpu_A0,
+                        tcg_gen_qemu_ld32u(cpu_tmp0, cpu_A0,
                                           (s->mem_index >> 2) - 1);
+                        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp0);
                         tcg_gen_st_i32(cpu_tmp2_i32, cpu_env, op2_offset +
                                         offsetof(XMMReg, XMM_L(0)));
@@ -3837 +3829 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_ptr1);
+            ((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
 
             if (b == 0x17)
@@ -3866 +3858 @@
             tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
             gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
-            tcg_gen_helper_1_3(helper_crc32, cpu_T[0], cpu_tmp2_i32,
-                            cpu_T[0], tcg_const_i32(8 << ot));
+            gen_helper_crc32(cpu_T[0], cpu_tmp2_i32,
+                             cpu_T[0], tcg_const_i32(8 << ot));
 
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
@@ -3917 +3909 @@
                                         offsetof(CPUX86State,
                                                 xmm_regs[reg].XMM_L(val & 3)));
+                        tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                         if (mod == 3)
-                            gen_op_mov_reg_v(ot, rm, cpu_tmp2_i32);
+                            gen_op_mov_reg_v(ot, rm, cpu_T[0]);
                         else
-                            tcg_gen_qemu_st32(cpu_tmp2_i32, cpu_A0,
+                            tcg_gen_qemu_st32(cpu_T[0], cpu_A0,
                                             (s->mem_index >> 2) - 1);
                     } else { /* pextrq */
+#ifdef TARGET_X86_64
                         tcg_gen_ld_i64(cpu_tmp1_i64, cpu_env,
                                         offsetof(CPUX86State,
@@ -3931 +3925 @@
                             tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
                                             (s->mem_index >> 2) - 1);
+#else
+                        goto illegal_op;
+#endif
                     }
                     break;
@@ -3946 +3943 @@
                         gen_op_mov_TN_reg(OT_LONG, 0, rm);
                     else
-                        tcg_gen_qemu_ld8u(cpu_T[0], cpu_A0,
+                        tcg_gen_qemu_ld8u(cpu_tmp0, cpu_A0,
                                         (s->mem_index >> 2) - 1);
-                    tcg_gen_st8_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
+                    tcg_gen_st8_tl(cpu_tmp0, cpu_env, offsetof(CPUX86State,
                                             xmm_regs[reg].XMM_B(val & 15)));
                     break;
                 case 0x21: /* insertps */
-                    if (mod == 3)
+                    if (mod == 3) {
                         tcg_gen_ld_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,xmm_regs[rm]
                                                 .XMM_L((val >> 6) & 3)));
-                    else
-                        tcg_gen_qemu_ld32u(cpu_tmp2_i32, cpu_A0,
+                    } else {
+                        tcg_gen_qemu_ld32u(cpu_tmp0, cpu_A0,
                                         (s->mem_index >> 2) - 1);
+                        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp0);
+                    }
                     tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
                                     offsetof(CPUX86State,xmm_regs[reg]
@@ -3982 +3981 @@
                     if (ot == OT_LONG) { /* pinsrd */
                         if (mod == 3)
-                            gen_op_mov_v_reg(ot, cpu_tmp2_i32, rm);
+                            gen_op_mov_v_reg(ot, cpu_tmp0, rm);
                         else
-                            tcg_gen_qemu_ld32u(cpu_tmp2_i32, cpu_A0,
+                            tcg_gen_qemu_ld32u(cpu_tmp0, cpu_A0,
                                             (s->mem_index >> 2) - 1);
+                        tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_tmp0);
                         tcg_gen_st_i32(cpu_tmp2_i32, cpu_env,
                                         offsetof(CPUX86State,
                                                 xmm_regs[reg].XMM_L(val & 3)));
                     } else { /* pinsrq */
+#ifdef TARGET_X86_64
                         if (mod == 3)
                             gen_op_mov_v_reg(ot, cpu_tmp1_i64, rm);
@@ -3998 +3999 @@
                                         offsetof(CPUX86State,
                                                 xmm_regs[reg].XMM_Q(val & 1)));
+#else
+                        goto illegal_op;
+#endif
                     }
                     break;
@@ -4035 +4039 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_3(sse_op2, cpu_ptr0, cpu_ptr1, tcg_const_i32(val));
+            ((void (*)(TCGv_ptr, TCGv_ptr, TCGv_i32))sse_op2)(cpu_ptr0, cpu_ptr1, tcg_const_i32(val));
             break;
         default:
@@ -4095 +4099 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_ptr1);
+            ((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
             break;
         case 0x70: /* pshufx insn */
@@ -4102 +4106 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_3(sse_op2, cpu_ptr0, cpu_ptr1, tcg_const_i32(val));
+            ((void (*)(TCGv_ptr, TCGv_ptr, TCGv_i32))sse_op2)(cpu_ptr0, cpu_ptr1, tcg_const_i32(val));
             break;
         case 0xc2:
@@ -4112 +4116 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_ptr1);
+            ((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
             break;
         case 0xf7:
@@ -4132 +4136 @@
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_3(sse_op2, cpu_ptr0, cpu_ptr1, cpu_A0);
+            ((void (*)(TCGv_ptr, TCGv_ptr, TCGv))sse_op2)(cpu_ptr0, cpu_ptr1, cpu_A0);
             break;
         default:
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
-            tcg_gen_helper_0_2(sse_op2, cpu_ptr0, cpu_ptr1);
+            ((void (*)(TCGv_ptr, TCGv_ptr))sse_op2)(cpu_ptr0, cpu_ptr1);
             break;
         }
@@ -4298 +4302 @@
     int rex_w, rex_r;
 
-    if (unlikely(loglevel & CPU_LOG_TB_OP))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
         tcg_gen_debug_insn_start(pc_start);
     s->pc = pc_start;
@@ -4431 +4435 @@
 #ifndef VBOX
     if (prefixes & PREFIX_LOCK)
-        tcg_gen_helper_0_0(helper_lock);
+        gen_helper_lock();
 #else /* VBOX */
     if (prefixes & PREFIX_LOCK) {
@@ -4438 +4442 @@
             return s->pc;
         }
-        tcg_gen_helper_0_0(helper_lock);
+        gen_helper_lock();
     }
 #endif /* VBOX */
@@ -4659 +4663 @@
 #else
                 {
-                    TCGv t0, t1;
-                    t0 = tcg_temp_new(TCG_TYPE_I64);
-                    t1 = tcg_temp_new(TCG_TYPE_I64);
+                    TCGv_i64 t0, t1;
+                    t0 = tcg_temp_new_i64();
+                    t1 = tcg_temp_new_i64();
                     gen_op_mov_TN_reg(OT_LONG, 1, R_EAX);
                     tcg_gen_extu_i32_i64(t0, cpu_T[0]);
@@ -4679 +4683 @@
 #ifdef TARGET_X86_64
             case OT_QUAD:
-                tcg_gen_helper_0_1(helper_mulq_EAX_T0, cpu_T[0]);
+                gen_helper_mulq_EAX_T0(cpu_T[0]);
                 s->cc_op = CC_OP_MULQ;
                 break;
@@ -4728 +4732 @@
 #else
                 {
-                    TCGv t0, t1;
-                    t0 = tcg_temp_new(TCG_TYPE_I64);
-                    t1 = tcg_temp_new(TCG_TYPE_I64);
+                    TCGv_i64 t0, t1;
+                    t0 = tcg_temp_new_i64();
+                    t1 = tcg_temp_new_i64();
                     gen_op_mov_TN_reg(OT_LONG, 1, R_EAX);
                     tcg_gen_ext_i32_i64(t0, cpu_T[0]);
@@ -4749 +4753 @@
 #ifdef TARGET_X86_64
             case OT_QUAD:
-                tcg_gen_helper_0_1(helper_imulq_EAX_T0, cpu_T[0]);
+                gen_helper_imulq_EAX_T0(cpu_T[0]);
                 s->cc_op = CC_OP_MULQ;
                 break;
@@ -4759 +4763 @@
             case OT_BYTE:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_divb_AL, cpu_T[0]);
+                gen_helper_divb_AL(cpu_T[0]);
                 break;
             case OT_WORD:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_divw_AX, cpu_T[0]);
+                gen_helper_divw_AX(cpu_T[0]);
                 break;
             default:
             case OT_LONG:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_divl_EAX, cpu_T[0]);
+                gen_helper_divl_EAX(cpu_T[0]);
                 break;
 #ifdef TARGET_X86_64
             case OT_QUAD:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_divq_EAX, cpu_T[0]);
+                gen_helper_divq_EAX(cpu_T[0]);
                 break;
 #endif
@@ -4782 +4786 @@
             case OT_BYTE:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_idivb_AL, cpu_T[0]);
+                gen_helper_idivb_AL(cpu_T[0]);
                 break;
             case OT_WORD:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_idivw_AX, cpu_T[0]);
+                gen_helper_idivw_AX(cpu_T[0]);
                 break;
             default:
             case OT_LONG:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_idivl_EAX, cpu_T[0]);
+                gen_helper_idivl_EAX(cpu_T[0]);
                 break;
 #ifdef TARGET_X86_64
             case OT_QUAD:
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_idivq_EAX, cpu_T[0]);
+                gen_helper_idivq_EAX(cpu_T[0]);
                 break;
 #endif
@@ -4880 +4884 @@
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_4(helper_lcall_protected,
-                                   cpu_tmp2_i32, cpu_T[1],
-                                   tcg_const_i32(dflag),
-                                   tcg_const_i32(s->pc - pc_start));
+                gen_helper_lcall_protected(cpu_tmp2_i32, cpu_T[1],
+                                           tcg_const_i32(dflag),
+                                           tcg_const_i32(s->pc - pc_start));
             } else {
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_4(helper_lcall_real,
-                                   cpu_tmp2_i32, cpu_T[1],
-                                   tcg_const_i32(dflag),
-                                   tcg_const_i32(s->pc - s->cs_base));
+                gen_helper_lcall_real(cpu_tmp2_i32, cpu_T[1],
+                                      tcg_const_i32(dflag),
+                                      tcg_const_i32(s->pc - s->cs_base));
             }
             gen_eob(s);
@@ -4909 +4911 @@
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_3(helper_ljmp_protected,
-                                   cpu_tmp2_i32,
-                                   cpu_T[1],
-                                   tcg_const_i32(s->pc - pc_start));
+                gen_helper_ljmp_protected(cpu_tmp2_i32, cpu_T[1],
+                                          tcg_const_i32(s->pc - pc_start));
             } else {
                 gen_op_movl_seg_T0_vm(R_CS);
@@ -5021 +5021 @@
 #ifdef TARGET_X86_64
         if (ot == OT_QUAD) {
-            tcg_gen_helper_1_2(helper_imulq_T0_T1, cpu_T[0], cpu_T[0], cpu_T[1]);
+            gen_helper_imulq_T0_T1(cpu_T[0], cpu_T[0], cpu_T[1]);
         } else
 #endif
@@ -5034 +5034 @@
 #else
                 {
-                    TCGv t0, t1;
-                    t0 = tcg_temp_new(TCG_TYPE_I64);
-                    t1 = tcg_temp_new(TCG_TYPE_I64);
+                    TCGv_i64 t0, t1;
+                    t0 = tcg_temp_new_i64();
+                    t1 = tcg_temp_new_i64();
                     tcg_gen_ext_i32_i64(t0, cpu_T[0]);
                     tcg_gen_ext_i32_i64(t1, cpu_T[1]);
@@ -5100 +5100 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
-            t0 = tcg_temp_local_new(TCG_TYPE_TL);
-            t1 = tcg_temp_local_new(TCG_TYPE_TL);
-            t2 = tcg_temp_local_new(TCG_TYPE_TL);
-            a0 = tcg_temp_local_new(TCG_TYPE_TL);
+            t0 = tcg_temp_local_new();
+            t1 = tcg_temp_local_new();
+            t2 = tcg_temp_local_new();
+            a0 = tcg_temp_local_new();
             gen_op_mov_v_reg(ot, t1, reg);
             if (mod == 3) {
@@ -5155 +5155 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
-            tcg_gen_helper_0_1(helper_cmpxchg16b, cpu_A0);
+            gen_helper_cmpxchg16b(cpu_A0);
         } else
 #endif
@@ -5165 +5165 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
-            tcg_gen_helper_0_1(helper_cmpxchg8b, cpu_A0);
+            gen_helper_cmpxchg8b(cpu_A0);
         }
         s->cc_op = CC_OP_EFLAGS;
@@ -5291 +5291 @@
                _first_ does it */
             if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                tcg_gen_helper_0_0(helper_set_inhibit_irq);
+                gen_helper_set_inhibit_irq();
             s->tf = 0;
         }
@@ -5371 +5371 @@
                _first_ does it */
             if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                tcg_gen_helper_0_0(helper_set_inhibit_irq);
+                gen_helper_set_inhibit_irq();
             s->tf = 0;
         }
@@ -5565 +5565 @@
             /* for xchg, lock is implicit */
             if (!(prefixes & PREFIX_LOCK))
-                tcg_gen_helper_0_0(helper_lock);
+                gen_helper_lock();
             gen_op_ld_T1_A0(ot + s->mem_index);
             gen_op_st_T0_A0(ot + s->mem_index);
             if (!(prefixes & PREFIX_LOCK))
-                tcg_gen_helper_0_0(helper_unlock);
+                gen_helper_unlock();
             gen_op_mov_reg_T1(ot, reg);
         }
@@ -5728 +5728 @@
                         gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                        tcg_gen_helper_0_1(helper_flds_FT0, cpu_tmp2_i32);
+                        gen_helper_flds_FT0(cpu_tmp2_i32);
                         break;
                     case 1:
                         gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                        tcg_gen_helper_0_1(helper_fildl_FT0, cpu_tmp2_i32);
+                        gen_helper_fildl_FT0(cpu_tmp2_i32);
                         break;
                     case 2:
                         tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
                                           (s->mem_index >> 2) - 1);
-                        tcg_gen_helper_0_1(helper_fldl_FT0, cpu_tmp1_i64);
+                        gen_helper_fldl_FT0(cpu_tmp1_i64);
                         break;
                     case 3:
@@ -5744 +5744 @@
                         gen_op_lds_T0_A0(OT_WORD + s->mem_index);
                         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                        tcg_gen_helper_0_1(helper_fildl_FT0, cpu_tmp2_i32);
+                        gen_helper_fildl_FT0(cpu_tmp2_i32);
                         break;
                     }
 
-                    tcg_gen_helper_0_0(helper_fp_arith_ST0_FT0[op1]);
+                    gen_helper_fp_arith_ST0_FT0(op1);
                     if (op1 == 3) {
                         /* fcomp needs pop */
-                        tcg_gen_helper_0_0(helper_fpop);
+                        gen_helper_fpop();
                     }
                 }
@@ -5767 +5767 @@
                         gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                        tcg_gen_helper_0_1(helper_flds_ST0, cpu_tmp2_i32);
+                        gen_helper_flds_ST0(cpu_tmp2_i32);
                         break;
                     case 1:
                         gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                        tcg_gen_helper_0_1(helper_fildl_ST0, cpu_tmp2_i32);
+                        gen_helper_fildl_ST0(cpu_tmp2_i32);
                         break;
                     case 2:
                         tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
                                           (s->mem_index >> 2) - 1);
-                        tcg_gen_helper_0_1(helper_fldl_ST0, cpu_tmp1_i64);
+                        gen_helper_fldl_ST0(cpu_tmp1_i64);
                         break;
                     case 3:
@@ -5783 +5783 @@
                         gen_op_lds_T0_A0(OT_WORD + s->mem_index);
                         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                        tcg_gen_helper_0_1(helper_fildl_ST0, cpu_tmp2_i32);
+                        gen_helper_fildl_ST0(cpu_tmp2_i32);
                         break;
                     }
@@ -5791 +5791 @@
                     switch(op >> 4) {
                     case 1:
-                        tcg_gen_helper_1_0(helper_fisttl_ST0, cpu_tmp2_i32);
+                        gen_helper_fisttl_ST0(cpu_tmp2_i32);
                         tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                         gen_op_st_T0_A0(OT_LONG + s->mem_index);
                         break;
                     case 2:
-                        tcg_gen_helper_1_0(helper_fisttll_ST0, cpu_tmp1_i64);
+                        gen_helper_fisttll_ST0(cpu_tmp1_i64);
                         tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
                                           (s->mem_index >> 2) - 1);
@@ -5802 +5802 @@
                     case 3:
                     default:
-                        tcg_gen_helper_1_0(helper_fistt_ST0, cpu_tmp2_i32);
+                        gen_helper_fistt_ST0(cpu_tmp2_i32);
                         tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                         gen_op_st_T0_A0(OT_WORD + s->mem_index);
                         break;
                     }
-                    tcg_gen_helper_0_0(helper_fpop);
+                    gen_helper_fpop();
                     break;
                 default:
                     switch(op >> 4) {
                     case 0:
-                        tcg_gen_helper_1_0(helper_fsts_ST0, cpu_tmp2_i32);
+                        gen_helper_fsts_ST0(cpu_tmp2_i32);
                         tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                         gen_op_st_T0_A0(OT_LONG + s->mem_index);
                         break;
                     case 1:
-                        tcg_gen_helper_1_0(helper_fistl_ST0, cpu_tmp2_i32);
+                        gen_helper_fistl_ST0(cpu_tmp2_i32);
                         tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                         gen_op_st_T0_A0(OT_LONG + s->mem_index);
                         break;
                     case 2:
-                        tcg_gen_helper_1_0(helper_fstl_ST0, cpu_tmp1_i64);
+                        gen_helper_fstl_ST0(cpu_tmp1_i64);
                         tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
                                           (s->mem_index >> 2) - 1);
@@ -5828 +5828 @@
                     case 3:
                     default:
-                        tcg_gen_helper_1_0(helper_fist_ST0, cpu_tmp2_i32);
+                        gen_helper_fist_ST0(cpu_tmp2_i32);
                         tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                         gen_op_st_T0_A0(OT_WORD + s->mem_index);
@@ -5834 +5834 @@
                     }
                     if ((op & 7) == 3)
-                        tcg_gen_helper_0_0(helper_fpop);
+                        gen_helper_fpop();
                     break;
                 }
@@ -5842 +5842 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_2(helper_fldenv,
+                gen_helper_fldenv(
                                    cpu_A0, tcg_const_i32(s->dflag));
                 break;
@@ -5848 +5848 @@
                 gen_op_ld_T0_A0(OT_WORD + s->mem_index);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_1(helper_fldcw, cpu_tmp2_i32);
+                gen_helper_fldcw(cpu_tmp2_i32);
                 break;
             case 0x0e: /* fnstenv mem */
@@ -5854 +5854 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_2(helper_fstenv,
-                                   cpu_A0, tcg_const_i32(s->dflag));
+                gen_helper_fstenv(cpu_A0, tcg_const_i32(s->dflag));
                 break;
             case 0x0f: /* fnstcw mem */
-                tcg_gen_helper_1_0(helper_fnstcw, cpu_tmp2_i32);
+                gen_helper_fnstcw(cpu_tmp2_i32);
                 tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                 gen_op_st_T0_A0(OT_WORD + s->mem_index);
@@ -5866 +5865 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_fldt_ST0, cpu_A0);
+                gen_helper_fldt_ST0(cpu_A0);
                 break;
             case 0x1f: /* fstpt mem */
@@ -5872 +5871 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_fstt_ST0, cpu_A0);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fstt_ST0(cpu_A0);
+                gen_helper_fpop();
                 break;
             case 0x2c: /* frstor mem */
@@ -5879 +5878 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_2(helper_frstor,
-                                   cpu_A0, tcg_const_i32(s->dflag));
+                gen_helper_frstor(cpu_A0, tcg_const_i32(s->dflag));
                 break;
             case 0x2e: /* fnsave mem */
@@ -5886 +5884 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_2(helper_fsave,
-                                   cpu_A0, tcg_const_i32(s->dflag));
+                gen_helper_fsave(cpu_A0, tcg_const_i32(s->dflag));
                 break;
             case 0x2f: /* fnstsw mem */
-                tcg_gen_helper_1_0(helper_fnstsw, cpu_tmp2_i32);
+                gen_helper_fnstsw(cpu_tmp2_i32);
                 tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                 gen_op_st_T0_A0(OT_WORD + s->mem_index);
@@ -5898 +5895 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_fbld_ST0, cpu_A0);
+                gen_helper_fbld_ST0(cpu_A0);
                 break;
             case 0x3e: /* fbstp */
@@ -5904 +5901 @@
                     gen_op_set_cc_op(s->cc_op);
                 gen_jmp_im(pc_start - s->cs_base);
-                tcg_gen_helper_0_1(helper_fbst_ST0, cpu_A0);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fbst_ST0(cpu_A0);
+                gen_helper_fpop();
                 break;
             case 0x3d: /* fildll */
                 tcg_gen_qemu_ld64(cpu_tmp1_i64, cpu_A0,
                                   (s->mem_index >> 2) - 1);
-                tcg_gen_helper_0_1(helper_fildll_ST0, cpu_tmp1_i64);
+                gen_helper_fildll_ST0(cpu_tmp1_i64);
                 break;
             case 0x3f: /* fistpll */
-                tcg_gen_helper_1_0(helper_fistll_ST0, cpu_tmp1_i64);
+                gen_helper_fistll_ST0(cpu_tmp1_i64);
                 tcg_gen_qemu_st64(cpu_tmp1_i64, cpu_A0,
                                   (s->mem_index >> 2) - 1);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fpop();
                 break;
             default:
@@ -5927 +5924 @@
             switch(op) {
             case 0x08: /* fld sti */
-                tcg_gen_helper_0_0(helper_fpush);
-                tcg_gen_helper_0_1(helper_fmov_ST0_STN, tcg_const_i32((opreg + 1) & 7));
+                gen_helper_fpush();
+                gen_helper_fmov_ST0_STN(tcg_const_i32((opreg + 1) & 7));
                 break;
             case 0x09: /* fxchg sti */
             case 0x29: /* fxchg4 sti, undocumented op */
             case 0x39: /* fxchg7 sti, undocumented op */
-                tcg_gen_helper_0_1(helper_fxchg_ST0_STN, tcg_const_i32(opreg));
+                gen_helper_fxchg_ST0_STN(tcg_const_i32(opreg));
                 break;
             case 0x0a: /* grp d9/2 */
@@ -5942 +5939 @@
                         gen_op_set_cc_op(s->cc_op);
                     gen_jmp_im(pc_start - s->cs_base);
-                    tcg_gen_helper_0_0(helper_fwait);
+                    gen_helper_fwait();
                     break;
                 default:
@@ -5951 +5948 @@
                 switch(rm) {
                 case 0: /* fchs */
-                    tcg_gen_helper_0_0(helper_fchs_ST0);
+                    gen_helper_fchs_ST0();
                     break;
                 case 1: /* fabs */
-                    tcg_gen_helper_0_0(helper_fabs_ST0);
+                    gen_helper_fabs_ST0();
                     break;
                 case 4: /* ftst */
-                    tcg_gen_helper_0_0(helper_fldz_FT0);
-                    tcg_gen_helper_0_0(helper_fcom_ST0_FT0);
+                    gen_helper_fldz_FT0();
+                    gen_helper_fcom_ST0_FT0();
                     break;
                 case 5: /* fxam */
-                    tcg_gen_helper_0_0(helper_fxam_ST0);
+                    gen_helper_fxam_ST0();
                     break;
                 default:
@@ -5971 +5968 @@
                     switch(rm) {
                     case 0:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fld1_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fld1_ST0();
                         break;
                     case 1:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fldl2t_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fldl2t_ST0();
                         break;
                     case 2:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fldl2e_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fldl2e_ST0();
                         break;
                     case 3:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fldpi_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fldpi_ST0();
                         break;
                     case 4:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fldlg2_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fldlg2_ST0();
                         break;
                     case 5:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fldln2_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fldln2_ST0();
                         break;
                     case 6:
-                        tcg_gen_helper_0_0(helper_fpush);
-                        tcg_gen_helper_0_0(helper_fldz_ST0);
+                        gen_helper_fpush();
+                        gen_helper_fldz_ST0();
                         break;
                     default:
@@ -6006 +6003 @@
                 switch(rm) {
                 case 0: /* f2xm1 */
-                    tcg_gen_helper_0_0(helper_f2xm1);
+                    gen_helper_f2xm1();
                     break;
                 case 1: /* fyl2x */
-                    tcg_gen_helper_0_0(helper_fyl2x);
+                    gen_helper_fyl2x();
                     break;
                 case 2: /* fptan */
-                    tcg_gen_helper_0_0(helper_fptan);
+                    gen_helper_fptan();
                     break;
                 case 3: /* fpatan */
-                    tcg_gen_helper_0_0(helper_fpatan);
+                    gen_helper_fpatan();
                     break;
                 case 4: /* fxtract */
-                    tcg_gen_helper_0_0(helper_fxtract);
+                    gen_helper_fxtract();
                     break;
                 case 5: /* fprem1 */
-                    tcg_gen_helper_0_0(helper_fprem1);
+                    gen_helper_fprem1();
                     break;
                 case 6: /* fdecstp */
-                    tcg_gen_helper_0_0(helper_fdecstp);
+                    gen_helper_fdecstp();
                     break;
                 default:
                 case 7: /* fincstp */
-                    tcg_gen_helper_0_0(helper_fincstp);
+                    gen_helper_fincstp();
                     break;
                 }
@@ -6035 +6032 @@
                 switch(rm) {
                 case 0: /* fprem */
-                    tcg_gen_helper_0_0(helper_fprem);
+                    gen_helper_fprem();
                     break;
                 case 1: /* fyl2xp1 */
-                    tcg_gen_helper_0_0(helper_fyl2xp1);
+                    gen_helper_fyl2xp1();
                     break;
                 case 2: /* fsqrt */
-                    tcg_gen_helper_0_0(helper_fsqrt);
+                    gen_helper_fsqrt();
                     break;
                 case 3: /* fsincos */
-                    tcg_gen_helper_0_0(helper_fsincos);
+                    gen_helper_fsincos();
                     break;
                 case 5: /* fscale */
-                    tcg_gen_helper_0_0(helper_fscale);
+                    gen_helper_fscale();
                     break;
                 case 4: /* frndint */
-                    tcg_gen_helper_0_0(helper_frndint);
+                    gen_helper_frndint();
                     break;
                 case 6: /* fsin */
-                    tcg_gen_helper_0_0(helper_fsin);
+                    gen_helper_fsin();
                     break;
                 default:
                 case 7: /* fcos */
-                    tcg_gen_helper_0_0(helper_fcos);
+                    gen_helper_fcos();
                     break;
                 }
@@ -6069 +6066 @@
                     op1 = op & 7;
                     if (op >= 0x20) {
-                        tcg_gen_helper_0_1(helper_fp_arith_STN_ST0[op1], tcg_const_i32(opreg));
+                        gen_helper_fp_arith_STN_ST0(op1, opreg);
                         if (op >= 0x30)
-                            tcg_gen_helper_0_0(helper_fpop);
+                            gen_helper_fpop();
                     } else {
-                        tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                        tcg_gen_helper_0_0(helper_fp_arith_ST0_FT0[op1]);
+                        gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                        gen_helper_fp_arith_ST0_FT0(op1);
                     }
                 }
@@ -6080 +6077 @@
             case 0x02: /* fcom */
             case 0x22: /* fcom2, undocumented op */
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fcom_ST0_FT0);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fcom_ST0_FT0();
                 break;
             case 0x03: /* fcomp */
             case 0x23: /* fcomp3, undocumented op */
             case 0x32: /* fcomp5, undocumented op */
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fcom_ST0_FT0);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fcom_ST0_FT0();
+                gen_helper_fpop();
                 break;
             case 0x15: /* da/5 */
                 switch(rm) {
                 case 1: /* fucompp */
-                    tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(1));
-                    tcg_gen_helper_0_0(helper_fucom_ST0_FT0);
-                    tcg_gen_helper_0_0(helper_fpop);
-                    tcg_gen_helper_0_0(helper_fpop);
+                    gen_helper_fmov_FT0_STN(tcg_const_i32(1));
+                    gen_helper_fucom_ST0_FT0();
+                    gen_helper_fpop();
+                    gen_helper_fpop();
                     break;
                 default:
@@ -6109 +6106 @@
                     break;
                 case 2: /* fclex */
-                    tcg_gen_helper_0_0(helper_fclex);
+                    gen_helper_fclex();
                     break;
                 case 3: /* fninit */
-                    tcg_gen_helper_0_0(helper_fninit);
+                    gen_helper_fninit();
                     break;
                 case 4: /* fsetpm (287 only, just do nop here) */
@@ -6123 +6120 @@
                 if (s->cc_op != CC_OP_DYNAMIC)
                     gen_op_set_cc_op(s->cc_op);
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fucomi_ST0_FT0);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fucomi_ST0_FT0();
                 s->cc_op = CC_OP_EFLAGS;
                 break;
@@ -6130 +6127 @@
                 if (s->cc_op != CC_OP_DYNAMIC)
                     gen_op_set_cc_op(s->cc_op);
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fcomi_ST0_FT0);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fcomi_ST0_FT0();
                 s->cc_op = CC_OP_EFLAGS;
                 break;
             case 0x28: /* ffree sti */
-                tcg_gen_helper_0_1(helper_ffree_STN, tcg_const_i32(opreg));
+                gen_helper_ffree_STN(tcg_const_i32(opreg));
                 break;
             case 0x2a: /* fst sti */
-                tcg_gen_helper_0_1(helper_fmov_STN_ST0, tcg_const_i32(opreg));
+                gen_helper_fmov_STN_ST0(tcg_const_i32(opreg));
                 break;
             case 0x2b: /* fstp sti */
@@ -6144 +6141 @@
             case 0x3a: /* fstp8 sti, undocumented op */
             case 0x3b: /* fstp9 sti, undocumented op */
-                tcg_gen_helper_0_1(helper_fmov_STN_ST0, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fmov_STN_ST0(tcg_const_i32(opreg));
+                gen_helper_fpop();
                 break;
             case 0x2c: /* fucom st(i) */
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fucom_ST0_FT0);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fucom_ST0_FT0();
                 break;
             case 0x2d: /* fucomp st(i) */
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fucom_ST0_FT0);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fucom_ST0_FT0();
+                gen_helper_fpop();
                 break;
             case 0x33: /* de/3 */
                 switch(rm) {
                 case 1: /* fcompp */
-                    tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(1));
-                    tcg_gen_helper_0_0(helper_fcom_ST0_FT0);
-                    tcg_gen_helper_0_0(helper_fpop);
-                    tcg_gen_helper_0_0(helper_fpop);
+                    gen_helper_fmov_FT0_STN(tcg_const_i32(1));
+                    gen_helper_fcom_ST0_FT0();
+                    gen_helper_fpop();
+                    gen_helper_fpop();
                     break;
                 default:
@@ -6169 +6166 @@
                 break;
             case 0x38: /* ffreep sti, undocumented op */
-                tcg_gen_helper_0_1(helper_ffree_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_ffree_STN(tcg_const_i32(opreg));
+                gen_helper_fpop();
                 break;
             case 0x3c: /* df/4 */
                 switch(rm) {
                 case 0:
-                    tcg_gen_helper_1_0(helper_fnstsw, cpu_tmp2_i32);
+                    gen_helper_fnstsw(cpu_tmp2_i32);
                     tcg_gen_extu_i32_tl(cpu_T[0], cpu_tmp2_i32);
                     gen_op_mov_reg_T0(OT_WORD, R_EAX);
@@ -6186 +6183 @@
                 if (s->cc_op != CC_OP_DYNAMIC)
                     gen_op_set_cc_op(s->cc_op);
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fucomi_ST0_FT0);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fucomi_ST0_FT0();
+                gen_helper_fpop();
                 s->cc_op = CC_OP_EFLAGS;
                 break;
@@ -6194 +6191 @@
                 if (s->cc_op != CC_OP_DYNAMIC)
                     gen_op_set_cc_op(s->cc_op);
-                tcg_gen_helper_0_1(helper_fmov_FT0_STN, tcg_const_i32(opreg));
-                tcg_gen_helper_0_0(helper_fcomi_ST0_FT0);
-                tcg_gen_helper_0_0(helper_fpop);
+                gen_helper_fmov_FT0_STN(tcg_const_i32(opreg));
+                gen_helper_fcomi_ST0_FT0();
+                gen_helper_fpop();
                 s->cc_op = CC_OP_EFLAGS;
                 break;
@@ -6212 +6209 @@
                     l1 = gen_new_label();
                     gen_jcc1(s, s->cc_op, op1, l1);
-                    tcg_gen_helper_0_1(helper_fmov_ST0_STN, tcg_const_i32(opreg));
+                    gen_helper_fmov_ST0_STN(tcg_const_i32(opreg));
                     gen_set_label(l1);
                 }
@@ -6349 +6346 @@
             gen_io_start();
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_helper_1_1(helper_in_func[ot], cpu_T[1], cpu_tmp2_i32);
+        gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
         gen_op_mov_reg_T1(ot, R_EAX);
         if (use_icount) {
@@ -6366 +6363 @@
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
-#ifdef VBOX /* bird: linux is writing to this port for delaying I/O. */ /** @todo this breaks AIX, remove. */
+#ifdef VBOX /* bird: linux is writing to this port for delaying I/O. */ /** @todo YYY: this breaks AIX, remove. */
         if (val == 0x80)
             break;
@@ -6377 +6374 @@
         tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
         tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
-        tcg_gen_helper_0_2(helper_out_func[ot], cpu_tmp2_i32, cpu_tmp3_i32);
+        gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
         if (use_icount) {
             gen_io_end();
@@ -6396 +6393 @@
             gen_io_start();
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-        tcg_gen_helper_1_1(helper_in_func[ot], cpu_T[1], cpu_tmp2_i32);
+        gen_helper_in_func(ot, cpu_T[1], cpu_tmp2_i32);
         gen_op_mov_reg_T1(ot, R_EAX);
         if (use_icount) {
@@ -6420 +6417 @@
         tcg_gen_andi_i32(cpu_tmp2_i32, cpu_tmp2_i32, 0xffff);
         tcg_gen_trunc_tl_i32(cpu_tmp3_i32, cpu_T[1]);
-        tcg_gen_helper_0_2(helper_out_func[ot], cpu_tmp2_i32, cpu_tmp3_i32);
+        gen_helper_out_func(ot, cpu_tmp2_i32, cpu_tmp3_i32);
         if (use_icount) {
             gen_io_end();
@@ -6457 +6454 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_2(helper_lret_protected,
-                               tcg_const_i32(s->dflag),
-                               tcg_const_i32(val));
+            gen_helper_lret_protected(tcg_const_i32(s->dflag),
+                                      tcg_const_i32(val));
         } else {
             gen_stack_A0(s);
@@ -6485 +6481 @@
         if (!s->pe) {
             /* real mode */
-            tcg_gen_helper_0_1(helper_iret_real, tcg_const_i32(s->dflag));
+            gen_helper_iret_real(tcg_const_i32(s->dflag));
             s->cc_op = CC_OP_EFLAGS;
         } else if (s->vm86) {
@@ -6495 +6491 @@
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
-                tcg_gen_helper_0_1(helper_iret_real, tcg_const_i32(s->dflag));
+                gen_helper_iret_real(tcg_const_i32(s->dflag));
                 s->cc_op = CC_OP_EFLAGS;
             }
@@ -6502 +6498 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_2(helper_iret_protected,
-                               tcg_const_i32(s->dflag),
-                               tcg_const_i32(s->pc - s->cs_base));
+            gen_helper_iret_protected(tcg_const_i32(s->dflag),
+                                      tcg_const_i32(s->pc - s->cs_base));
             s->cc_op = CC_OP_EFLAGS;
         }
@@ -6519 +6514 @@
             if (s->dflag == 0)
                 tval &= 0xffff;
+#ifdef VBOX /* upstream fix */
             else if (!CODE64(s))
                 tval &= 0xffffffff;
+#endif
             gen_movtl_T0_im(next_eip);
             gen_push_T0(s);
@@ -6604 +6601 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
-            t0 = tcg_temp_local_new(TCG_TYPE_TL);
+            t0 = tcg_temp_local_new();
             if (mod != 3) {
                 gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
@@ -6648 +6645 @@
 #ifdef VBOX
             if (s->vm86 && s->vme && s->iopl != 3)
-                tcg_gen_helper_1_0(helper_read_eflags_vme, cpu_T[0]);
+                gen_helper_read_eflags_vme(cpu_T[0]);
             else
 #endif
-                tcg_gen_helper_1_0(helper_read_eflags, cpu_T[0]);
+            gen_helper_read_eflags(cpu_T[0]);
             gen_push_T0(s);
         }
@@ -6667 +6664 @@
             if (s->cpl == 0) {
                 if (s->dflag) {
-                    tcg_gen_helper_0_2(helper_write_eflags, cpu_T[0],
+                    gen_helper_write_eflags(cpu_T[0],
                                        tcg_const_i32((TF_MASK | AC_MASK | ID_MASK | NT_MASK | IF_MASK | IOPL_MASK)));
                 } else {
-                    tcg_gen_helper_0_2(helper_write_eflags, cpu_T[0],
+                    gen_helper_write_eflags(cpu_T[0],
                                        tcg_const_i32((TF_MASK | AC_MASK | ID_MASK | NT_MASK | IF_MASK | IOPL_MASK) & 0xffff));
                 }
@@ -6676 +6673 @@
                 if (s->cpl <= s->iopl) {
                     if (s->dflag) {
-                        tcg_gen_helper_0_2(helper_write_eflags, cpu_T[0],
+                        gen_helper_write_eflags(cpu_T[0],
                                            tcg_const_i32((TF_MASK | AC_MASK | ID_MASK | NT_MASK | IF_MASK)));
                     } else {
-                        tcg_gen_helper_0_2(helper_write_eflags, cpu_T[0],
+                        gen_helper_write_eflags(cpu_T[0],
                                            tcg_const_i32((TF_MASK | AC_MASK | ID_MASK | NT_MASK | IF_MASK) & 0xffff));
                     }
                 } else {
                     if (s->dflag) {
-                        tcg_gen_helper_0_2(helper_write_eflags, cpu_T[0],
+                        gen_helper_write_eflags(cpu_T[0],
                                            tcg_const_i32((TF_MASK | AC_MASK | ID_MASK | NT_MASK)));
                     } else {
 #ifdef VBOX
                         if (s->vm86 && s->vme)
-                            tcg_gen_helper_0_1(helper_write_eflags_vme, cpu_T[0]);
+                            gen_helper_write_eflags_vme(cpu_T[0]);
                         else
 #endif
-                            tcg_gen_helper_0_2(helper_write_eflags, cpu_T[0],
+                            gen_helper_write_eflags(cpu_T[0],
                                                tcg_const_i32((TF_MASK | AC_MASK | ID_MASK | NT_MASK) & 0xffff));
                     }
@@ -6858 +6855 @@
             label1 = gen_new_label();
             tcg_gen_movi_tl(cpu_cc_dst, 0);
-            t0 = tcg_temp_local_new(TCG_TYPE_TL);
+            t0 = tcg_temp_local_new();
             tcg_gen_mov_tl(t0, cpu_T[0]);
             tcg_gen_brcondi_tl(TCG_COND_EQ, t0, 0, label1);
             if (b & 1) {
-                tcg_gen_helper_1_1(helper_bsr, cpu_T[0], t0);
+                gen_helper_bsr(cpu_T[0], t0);
             } else {
-                tcg_gen_helper_1_1(helper_bsf, cpu_T[0], t0);
+                gen_helper_bsf(cpu_T[0], t0);
             }
             gen_op_mov_reg_T0(ot, reg);
@@ -6881 +6878 @@
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        tcg_gen_helper_0_0(helper_daa);
+        gen_helper_daa();
         s->cc_op = CC_OP_EFLAGS;
         break;
@@ -6889 +6886 @@
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        tcg_gen_helper_0_0(helper_das);
+        gen_helper_das();
         s->cc_op = CC_OP_EFLAGS;
         break;
@@ -6897 +6894 @@
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        tcg_gen_helper_0_0(helper_aaa);
+        gen_helper_aaa();
         s->cc_op = CC_OP_EFLAGS;
         break;
@@ -6905 +6902 @@
         if (s->cc_op != CC_OP_DYNAMIC)
             gen_op_set_cc_op(s->cc_op);
-        tcg_gen_helper_0_0(helper_aas);
+        gen_helper_aas();
         s->cc_op = CC_OP_EFLAGS;
         break;
@@ -6915 +6912 @@
             gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base);
         } else {
-            tcg_gen_helper_0_1(helper_aam, tcg_const_i32(val));
+            gen_helper_aam(tcg_const_i32(val));
             s->cc_op = CC_OP_LOGICB;
         }
@@ -6923 +6920 @@
             goto illegal_op;
         val = ldub_code(s->pc++);
-        tcg_gen_helper_0_1(helper_aad, tcg_const_i32(val));
+        gen_helper_aad(tcg_const_i32(val));
         s->cc_op = CC_OP_LOGICB;
         break;
@@ -6945 +6942 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_0(helper_fwait);
+            gen_helper_fwait();
         }
         break;
@@ -6974 +6971 @@
             gen_op_set_cc_op(s->cc_op);
         gen_jmp_im(pc_start - s->cs_base);
-        tcg_gen_helper_0_1(helper_into, tcg_const_i32(s->pc - pc_start));
-        break;
+        gen_helper_into(tcg_const_i32(s->pc - pc_start));
+        break;
+#ifdef WANT_ICEBP
     case 0xf1: /* icebp (undocumented, exits to external debugger) */
         gen_svm_check_intercept(s, pc_start, SVM_EXIT_ICEBP);
@@ -6986 +6984 @@
 #endif
         break;
+#endif
     case 0xfa: /* cli */
         if (!s->vm86) {
             if (s->cpl <= s->iopl) {
-                tcg_gen_helper_0_0(helper_cli);
+                gen_helper_cli();
             } else {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
@@ -6995 +6994 @@
         } else {
             if (s->iopl == 3) {
-                tcg_gen_helper_0_0(helper_cli);
+                gen_helper_cli();
 #ifdef VBOX
             } else if (s->iopl != 3 && s->vme) {
-                tcg_gen_helper_0_0(helper_cli_vme);
+                gen_helper_cli_vme();
 #endif
             } else {
@@ -7009 +7008 @@
             if (s->cpl <= s->iopl) {
             gen_sti:
-                tcg_gen_helper_0_0(helper_sti);
+                gen_helper_sti();
                 /* interruptions are enabled only the first insn after sti */
                 /* If several instructions disable interrupts, only the
                    _first_ does it */
                 if (!(s->tb->flags & HF_INHIBIT_IRQ_MASK))
-                    tcg_gen_helper_0_0(helper_set_inhibit_irq);
+                    gen_helper_set_inhibit_irq();
                 /* give a chance to handle pending irqs */
                 gen_jmp_im(s->pc - s->cs_base);
@@ -7026 +7025 @@
 #ifdef VBOX
             } else if (s->iopl != 3 && s->vme) {
-                tcg_gen_helper_0_0(helper_sti_vme);
+                gen_helper_sti_vme();
                 /* give a chance to handle pending irqs */
                 gen_jmp_im(s->pc - s->cs_base);
@@ -7050 +7049 @@
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         if (ot == OT_WORD)
-            tcg_gen_helper_0_2(helper_boundw, cpu_A0, cpu_tmp2_i32);
+            gen_helper_boundw(cpu_A0, cpu_tmp2_i32);
         else
-            tcg_gen_helper_0_2(helper_boundl, cpu_A0, cpu_tmp2_i32);
+            gen_helper_boundl(cpu_A0, cpu_tmp2_i32);
         break;
     case 0x1c8 ... 0x1cf: /* bswap reg */
@@ -7063 +7062 @@
         } else
         {
-            TCGv tmp0;
+            TCGv_i32 tmp0;
             gen_op_mov_TN_reg(OT_LONG, 0, reg);
 
-            tmp0 = tcg_temp_new(TCG_TYPE_I32);
+            tmp0 = tcg_temp_new_i32();
             tcg_gen_trunc_i64_i32(tmp0, cpu_T[0]);
             tcg_gen_bswap_i32(tmp0, tmp0);
@@ -7150 +7149 @@
             gen_jmp_im(pc_start - s->cs_base);
             if (b & 2) {
-                tcg_gen_helper_0_0(helper_rdmsr);
+                gen_helper_rdmsr();
             } else {
-                tcg_gen_helper_0_0(helper_wrmsr);
+                gen_helper_wrmsr();
             }
         }
@@ -7162 +7161 @@
         if (use_icount)
             gen_io_start();
-        tcg_gen_helper_0_0(helper_rdtsc);
+        gen_helper_rdtsc();
         if (use_icount) {
             gen_io_end();
@@ -7172 +7171 @@
             gen_op_set_cc_op(s->cc_op);
         gen_jmp_im(pc_start - s->cs_base);
-        tcg_gen_helper_0_0(helper_rdpmc);
+        gen_helper_rdpmc();
         break;
     case 0x134: /* sysenter */
@@ -7191 +7190 @@
             }
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_0(helper_sysenter);
+            gen_helper_sysenter();
             gen_eob(s);
         }
@@ -7212 +7211 @@
             }
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_1(helper_sysexit, tcg_const_i32(dflag));
+            gen_helper_sysexit(tcg_const_i32(dflag));
             gen_eob(s);
         }
@@ -7224 +7223 @@
         }
         gen_jmp_im(pc_start - s->cs_base);
-        tcg_gen_helper_0_1(helper_syscall, tcg_const_i32(s->pc - pc_start));
+        gen_helper_syscall(tcg_const_i32(s->pc - pc_start));
         gen_eob(s);
         break;
@@ -7236 +7235 @@
             }
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_1(helper_sysret, tcg_const_i32(s->dflag));
+            gen_helper_sysret(tcg_const_i32(s->dflag));
             /* condition codes are modified only in long mode */
             if (s->lma)
@@ -7248 +7247 @@
             gen_op_set_cc_op(s->cc_op);
         gen_jmp_im(pc_start - s->cs_base);
-        tcg_gen_helper_0_0(helper_cpuid);
+        gen_helper_cpuid();
         break;
     case 0xf4: /* hlt */
@@ -7257 +7256 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_1(helper_hlt, tcg_const_i32(s->pc - pc_start));
+            gen_helper_hlt(tcg_const_i32(s->pc - pc_start));
             s->is_jmp = 3;
         }
@@ -7286 +7285 @@
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_1(helper_lldt, cpu_tmp2_i32);
+                gen_helper_lldt(cpu_tmp2_i32);
             }
             break;
@@ -7309 +7308 @@
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-                tcg_gen_helper_0_1(helper_ltr, cpu_tmp2_i32);
+                gen_helper_ltr(cpu_tmp2_i32);
             }
             break;
@@ -7320 +7319 @@
                 gen_op_set_cc_op(s->cc_op);
             if (op == 4)
-                tcg_gen_helper_0_1(helper_verr, cpu_T[0]);
+                gen_helper_verr(cpu_T[0]);
             else
-                tcg_gen_helper_0_1(helper_verw, cpu_T[0]);
+                gen_helper_verw(cpu_T[0]);
             s->cc_op = CC_OP_EFLAGS;
             break;
@@ -7341 +7340 @@
                 goto illegal_op;
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_0(helper_rdtscp);
+            gen_helper_rdtscp();
             break;
         }
@@ -7380 +7379 @@
                     }
                     gen_add_A0_ds_seg(s);
-                    tcg_gen_helper_0_1(helper_monitor, cpu_A0);
+                    gen_helper_monitor(cpu_A0);
                     break;
                 case 1: /* mwait */
@@ -7391 +7390 @@
                     }
                     gen_jmp_im(pc_start - s->cs_base);
-                    tcg_gen_helper_0_1(helper_mwait, tcg_const_i32(s->pc - pc_start));
+                    gen_helper_mwait(tcg_const_i32(s->pc - pc_start));
                     gen_eob(s);
                     break;
@@ -7423 +7422 @@
                         break;
                     } else {
-                        tcg_gen_helper_0_2(helper_vmrun,
-                                           tcg_const_i32(s->aflag),
-                                           tcg_const_i32(s->pc - pc_start));
+                        gen_helper_vmrun(tcg_const_i32(s->aflag),
+                                         tcg_const_i32(s->pc - pc_start));
                         tcg_gen_exit_tb(0);
                         s->is_jmp = 3;
@@ -7433 +7431 @@
                     if (!(s->flags & HF_SVME_MASK))
                         goto illegal_op;
-                    tcg_gen_helper_0_0(helper_vmmcall);
+                    gen_helper_vmmcall();
                     break;
                 case 2: /* VMLOAD */
@@ -7442 +7440 @@
                         break;
                     } else {
-                        tcg_gen_helper_0_1(helper_vmload,
-                                           tcg_const_i32(s->aflag));
+                        gen_helper_vmload(tcg_const_i32(s->aflag));
                     }
                     break;
@@ -7453 +7450 @@
                         break;
                     } else {
-                        tcg_gen_helper_0_1(helper_vmsave,
-                                           tcg_const_i32(s->aflag));
+                        gen_helper_vmsave(tcg_const_i32(s->aflag));
                     }
                     break;
@@ -7466 +7462 @@
                         break;
                     } else {
-                        tcg_gen_helper_0_0(helper_stgi);
+                        gen_helper_stgi();
                     }
                     break;
@@ -7476 +7472 @@
                         break;
                     } else {
-                        tcg_gen_helper_0_0(helper_clgi);
+                        gen_helper_clgi();
                     }
                     break;
@@ -7484 +7480 @@
                         !s->pe)
                         goto illegal_op;
-                    tcg_gen_helper_0_0(helper_skinit);
+                    gen_helper_skinit();
                     break;
                 case 7: /* INVLPGA */
@@ -7493 +7489 @@
                         break;
                     } else {
-                        tcg_gen_helper_0_1(helper_invlpga,
-                                           tcg_const_i32(s->aflag));
+                        gen_helper_invlpga(tcg_const_i32(s->aflag));
                     }
                     break;
@@ -7522 +7517 @@
         case 4: /* smsw */
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_READ_CR0);
+#if defined TARGET_X86_64 && defined WORDS_BIGENDIAN
+            tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]) + 4);
+#else
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
+#endif
             gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 1);
             break;
@@ -7531 +7530 @@
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
                 gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
-                tcg_gen_helper_0_1(helper_lmsw, cpu_T[0]);
+                gen_helper_lmsw(cpu_T[0]);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
@@ -7558 +7557 @@
                     gen_jmp_im(pc_start - s->cs_base);
                     gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
-                    tcg_gen_helper_0_1(helper_invlpg, cpu_A0);
+                    gen_helper_invlpg(cpu_A0);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
@@ -7615 +7614 @@
             if (!s->pe || s->vm86)
                 goto illegal_op;
-            t0 = tcg_temp_local_new(TCG_TYPE_TL);
-            t1 = tcg_temp_local_new(TCG_TYPE_TL);
-            t2 = tcg_temp_local_new(TCG_TYPE_TL);
+            t0 = tcg_temp_local_new();
+            t1 = tcg_temp_local_new();
+            t2 = tcg_temp_local_new();
 #ifdef VBOX
-            a0 = tcg_temp_local_new(TCG_TYPE_TL);
+            a0 = tcg_temp_local_new();
 #endif
             ot = OT_WORD;
@@ -7680 +7679 @@
             reg = ((modrm >> 3) & 7) | rex_r;
             gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
-            t0 = tcg_temp_local_new(TCG_TYPE_TL);
+            t0 = tcg_temp_local_new();
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             if (b == 0x102)
-                tcg_gen_helper_1_1(helper_lar, t0, cpu_T[0]);
+                gen_helper_lar(t0, cpu_T[0]);
             else
-                tcg_gen_helper_1_1(helper_lsl, t0, cpu_T[0]);
+                gen_helper_lsl(t0, cpu_T[0]);
             tcg_gen_andi_tl(cpu_tmp0, cpu_cc_src, CC_Z);
             label1 = gen_new_label();
@@ -7746 +7745 @@
                 if (b & 2) {
                     gen_op_mov_TN_reg(ot, 0, rm);
-                    tcg_gen_helper_0_2(helper_write_crN,
-                                       tcg_const_i32(reg), cpu_T[0]);
+                    gen_helper_write_crN(tcg_const_i32(reg), cpu_T[0]);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
                 } else {
-                    tcg_gen_helper_1_1(helper_read_crN,
-                                       cpu_T[0], tcg_const_i32(reg));
+                    gen_helper_read_crN(cpu_T[0], tcg_const_i32(reg));
                     gen_op_mov_reg_T0(ot, rm);
                 }
@@ -7783 +7780 @@
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_DR0 + reg);
                 gen_op_mov_TN_reg(ot, 0, rm);
-                tcg_gen_helper_0_2(helper_movl_drN_T0,
-                                   tcg_const_i32(reg), cpu_T[0]);
+                gen_helper_movl_drN_T0(tcg_const_i32(reg), cpu_T[0]);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
@@ -7799 +7795 @@
         } else {
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
-            tcg_gen_helper_0_0(helper_clts);
+            gen_helper_clts();
             /* abort block because static cpu state changed */
             gen_jmp_im(s->pc - s->cs_base);
@@ -7835 +7831 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_2(helper_fxsave,
-                               cpu_A0, tcg_const_i32((s->dflag == 2)));
+            gen_helper_fxsave(cpu_A0, tcg_const_i32((s->dflag == 2)));
             break;
         case 1: /* fxrstor */
@@ -7850 +7845 @@
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
-            tcg_gen_helper_0_2(helper_fxrstor,
-                               cpu_A0, tcg_const_i32((s->dflag == 2)));
+            gen_helper_fxrstor(cpu_A0, tcg_const_i32((s->dflag == 2)));
             break;
         case 2: /* ldmxcsr */
@@ -7910 +7904 @@
         }
         gen_jmp_im(s->pc - s->cs_base);
-        tcg_gen_helper_0_0(helper_rsm);
+        gen_helper_rsm();
         gen_eob(s);
         break;
@@ -7931 +7925 @@
 
         gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
-        tcg_gen_helper_1_2(helper_popcnt,
-                cpu_T[0], cpu_T[0], tcg_const_i32(ot));
+        gen_helper_popcnt(cpu_T[0], cpu_T[0], tcg_const_i32(ot));
         gen_op_mov_reg_T0(ot, reg);
 
@@ -7955 +7948 @@
     /* lock generation */
     if (s->prefix & PREFIX_LOCK)
-        tcg_gen_helper_0_0(helper_unlock);
+        gen_helper_unlock();
     return s->pc;
  illegal_op:
     if (s->prefix & PREFIX_LOCK)
-        tcg_gen_helper_0_0(helper_unlock);
+        gen_helper_unlock();
     /* XXX: ensure that no lock was generated */
     gen_exception(s, EXCP06_ILLOP, pc_start - s->cs_base);
@@ -7972 +7965 @@
     assert(sizeof(CCTable) == (1 << 4));
 #endif
-    cpu_env = tcg_global_reg_new(TCG_TYPE_PTR, TCG_AREG0, "env");
-    cpu_cc_op = tcg_global_mem_new(TCG_TYPE_I32,
-                                   TCG_AREG0, offsetof(CPUState, cc_op), "cc_op");
-    cpu_cc_src = tcg_global_mem_new(TCG_TYPE_TL,
-                                    TCG_AREG0, offsetof(CPUState, cc_src), "cc_src");
-    cpu_cc_dst = tcg_global_mem_new(TCG_TYPE_TL,
-                                    TCG_AREG0, offsetof(CPUState, cc_dst), "cc_dst");
-    cpu_cc_tmp = tcg_global_mem_new(TCG_TYPE_TL,
-                                    TCG_AREG0, offsetof(CPUState, cc_tmp), "cc_tmp");
+    cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
+    cpu_cc_op = tcg_global_mem_new_i32(TCG_AREG0,
+                                       offsetof(CPUState, cc_op), "cc_op");
+    cpu_cc_src = tcg_global_mem_new(TCG_AREG0, offsetof(CPUState, cc_src),
+                                    "cc_src");
+    cpu_cc_dst = tcg_global_mem_new(TCG_AREG0, offsetof(CPUState, cc_dst),
+                                    "cc_dst");
+    cpu_cc_tmp = tcg_global_mem_new(TCG_AREG0, offsetof(CPUState, cc_tmp),
+                                    "cc_tmp");
 
     /* register helpers */
-
-#define DEF_HELPER(ret, name, params) tcg_register_helper(name, #name);
+#define GEN_HELPER 2
 #include "helper.h"
 }
@@ -7998 +7990 @@
     target_ulong pc_ptr;
     uint16_t *gen_opc_end;
+    CPUBreakpoint *bp;
     int j, lj, cflags;
     uint64_t flags;
@@ -8029 +8022 @@
         dc->record_call = 0;
 # endif
-#endif
+#endif /* VBOX */
     dc->cpl = (flags >> HF_CPL_SHIFT) & 3;
     dc->iopl = (flags >> IOPL_SHIFT) & 3;
@@ -8067 +8060 @@
 #endif
 
-    cpu_T[0] = tcg_temp_new(TCG_TYPE_TL);
-    cpu_T[1] = tcg_temp_new(TCG_TYPE_TL);
-    cpu_A0 = tcg_temp_new(TCG_TYPE_TL);
-    cpu_T3 = tcg_temp_new(TCG_TYPE_TL);
-
-    cpu_tmp0 = tcg_temp_new(TCG_TYPE_TL);
-    cpu_tmp1_i64 = tcg_temp_new(TCG_TYPE_I64);
-    cpu_tmp2_i32 = tcg_temp_new(TCG_TYPE_I32);
-    cpu_tmp3_i32 = tcg_temp_new(TCG_TYPE_I32);
-    cpu_tmp4 = tcg_temp_new(TCG_TYPE_TL);
-    cpu_tmp5 = tcg_temp_new(TCG_TYPE_TL);
-    cpu_tmp6 = tcg_temp_new(TCG_TYPE_TL);
-    cpu_ptr0 = tcg_temp_new(TCG_TYPE_PTR);
-    cpu_ptr1 = tcg_temp_new(TCG_TYPE_PTR);
+    cpu_T[0] = tcg_temp_new();
+    cpu_T[1] = tcg_temp_new();
+    cpu_A0 = tcg_temp_new();
+    cpu_T3 = tcg_temp_new();
+
+    cpu_tmp0 = tcg_temp_new();
+    cpu_tmp1_i64 = tcg_temp_new_i64();
+    cpu_tmp2_i32 = tcg_temp_new_i32();
+    cpu_tmp3_i32 = tcg_temp_new_i32();
+    cpu_tmp4 = tcg_temp_new();
+    cpu_tmp5 = tcg_temp_new();
+    cpu_tmp6 = tcg_temp_new();
+    cpu_ptr0 = tcg_temp_new_ptr();
+    cpu_ptr1 = tcg_temp_new_ptr();
 
     gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
@@ -8094 +8087 @@
     gen_icount_start();
     for(;;) {
-        if (env->nb_breakpoints > 0) {
-            for(j = 0; j < env->nb_breakpoints; j++) {
-                if (env->breakpoints[j] == pc_ptr) {
+        if (unlikely(!TAILQ_EMPTY(&env->breakpoints))) {
+            TAILQ_FOREACH(bp, &env->breakpoints, entry) {
+                if (bp->pc == pc_ptr) {
                     gen_debug(dc, pc_ptr - dc->cs_base);
                     break;
@@ -8174 +8167 @@
 
 #ifdef DEBUG_DISAS
-    if (loglevel & CPU_LOG_TB_CPU) {
-        cpu_dump_state(env, logfile, fprintf, X86_DUMP_CCOP);
-    }
-    if (loglevel & CPU_LOG_TB_IN_ASM) {
+    log_cpu_state_mask(CPU_LOG_TB_CPU, env, X86_DUMP_CCOP);
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         int disas_flags;
-        fprintf(logfile, "----------------\n");
-        fprintf(logfile, "IN: %s\n", lookup_symbol(pc_start));
+        qemu_log("----------------\n");
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
 #ifdef TARGET_X86_64
         if (dc->code64)
@@ -8187 +8178 @@
 #endif
             disas_flags = !dc->code32;
-        target_disas(logfile, pc_start, pc_ptr - pc_start, disas_flags);
-        fprintf(logfile, "\n");
+        log_target_disas(pc_start, pc_ptr - pc_start, disas_flags);
+        qemu_log("\n");
     }
 #endif
@@ -8213 +8204 @@
     int cc_op;
 #ifdef DEBUG_DISAS
-    if (loglevel & CPU_LOG_TB_OP) {
+    if (qemu_loglevel_mask(CPU_LOG_TB_OP)) {
         int i;
-        fprintf(logfile, "RESTORE:\n");
+        qemu_log("RESTORE:\n");
         for(i = 0;i <= pc_pos; i++) {
             if (gen_opc_instr_start[i]) {
-                fprintf(logfile, "0x%04x: " TARGET_FMT_lx "\n", i, gen_opc_pc[i]);
-            }
-        }
-        fprintf(logfile, "spc=0x%08lx pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
+                qemu_log("0x%04x: " TARGET_FMT_lx "\n", i, gen_opc_pc[i]);
+            }
+        }
+        qemu_log("spc=0x%08lx pc_pos=0x%x eip=" TARGET_FMT_lx " cs_base=%x\n",
                 searched_pc, pc_pos, gen_opc_pc[pc_pos] - tb->cs_base,
                 (uint32_t)tb->cs_base);

trunk/src/recompiler/tcg/README

@@ -61 +61 @@
 - Basic blocks end after branches (e.g. brcond_i32 instruction),
   goto_tb and exit_tb instructions.
-- Basic blocks end before legacy dyngen operations.
-- Basic blocks start after the end of a previous basic block, at a
-  set_label instruction or after a legacy dyngen operation.
+- Basic blocks start after the end of a previous basic block, or at a
+  set_label instruction.
 
 After the end of a basic block, the content of temporaries is
@@ -206 +205 @@
 t0=~t1
 
-********* Shifts
+* andc_i32/i64 t0, t1, t2
+
+t0=t1&~t2
+
+* eqv_i32/i64 t0, t1, t2
+
+t0=~(t1^t2)
+
+* nand_i32/i64 t0, t1, t2
+
+t0=~(t1&t2)
+
+* nor_i32/i64 t0, t1, t2
+
+t0=~(t1|t2)
+
+* orc_i32/i64 t0, t1, t2
+
+t0=t1|~t2
+
+********* Shifts/Rotates
 
 * shl_i32/i64 t0, t1, t2
@@ -219 +238 @@
 
 t0=t1 >> t2 (signed). Undefined behavior if t2 < 0 or t2 >= 32 (resp 64)
+
+* rotl_i32/i64 t0, t1, t2
+
+Rotation of t2 bits to the left. Undefined behavior if t2 < 0 or t2 >= 32 (resp 64)
+
+* rotr_i32/i64 t0, t1, t2
+
+Rotation of t2 bits to the right. Undefined behavior if t2 < 0 or t2 >= 32 (resp 64)
 
 ********* Misc
@@ -308 +335 @@
 instructions.
 
-* qemu_ld_i32/i64 t0, t1, flags
-qemu_ld8u_i32/i64 t0, t1, flags
-qemu_ld8s_i32/i64 t0, t1, flags
-qemu_ld16u_i32/i64 t0, t1, flags
-qemu_ld16s_i32/i64 t0, t1, flags
-qemu_ld32u_i64 t0, t1, flags
-qemu_ld32s_i64 t0, t1, flags
+* qemu_ld8u t0, t1, flags
+qemu_ld8s t0, t1, flags
+qemu_ld16u t0, t1, flags
+qemu_ld16s t0, t1, flags
+qemu_ld32u t0, t1, flags
+qemu_ld32s t0, t1, flags
+qemu_ld64 t0, t1, flags
 
 Load data at the QEMU CPU address t1 into t0. t1 has the QEMU CPU
@@ -320 +347 @@
 kernel access) for example.
 
-* qemu_st_i32/i64 t0, t1, flags
-qemu_st8_i32/i64 t0, t1, flags
-qemu_st16_i32/i64 t0, t1, flags
-qemu_st32_i64 t0, t1, flags
+* qemu_st8 t0, t1, flags
+qemu_st16 t0, t1, flags
+qemu_st32 t0, t1, flags
+qemu_st64 t0, t1, flags
 
 Store the data t0 at the QEMU CPU Address t1. t1 has the QEMU CPU
@@ -362 +389 @@
 instruction. Memory constraints are not supported in this
 version. Aliases are specified in the input operands as for GCC.
+
+The same register may be used for both an input and an output, even when
+they are not explicitly aliased.  If an op expands to multiple target
+instructions then care must be taken to avoid clobbering input values.
+GCC style "early clobber" outputs are not currently supported.
 
 A target can define specific register or constant constraints. If an
@@ -391 +423 @@
   64 bit return type.
 
-5) Migration from dyngen to TCG
-
-TCG is backward compatible with QEMU "dyngen" operations. It means
-that TCG instructions can be freely mixed with dyngen operations. It
-is expected that QEMU targets will be progressively fully converted to
-TCG. Once a target is fully converted to TCG, it will be possible
-to apply more optimizations because more registers will be free for
-the generated code.
-
-The exception model is the same as the dyngen one.
-
-6) Recommended coding rules for best performance
+5) Recommended coding rules for best performance
 
 - Use globals to represent the parts of the QEMU CPU state which are
@@ -410 +431 @@
 - Avoid globals stored in fixed registers. They must be used only to
   store the pointer to the CPU state and possibly to store a pointer
-  to a register window. The other uses are to ensure backward
-  compatibility with dyngen during the porting a new target to TCG.
+  to a register window.
 
 - Use temporaries. Use local temporaries only when really needed,

trunk/src/recompiler/tcg/i386/tcg-target.c

@@ -302 +302 @@
 }
 
-void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
+static void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
 {
     if (val != 0)
@@ -309 +309 @@
 
 #ifdef VBOX
-void tcg_out_subi(TCGContext *s, int reg, tcg_target_long val)
+static void tcg_out_subi(TCGContext *s, int reg, tcg_target_long val)
 {
     if (val != 0)
@@ -666 +666 @@
         break;
     case 0:
+        /* movzbl */
+        tcg_out_modrm(s, 0xb6 | P_EXT, data_reg, TCG_REG_EAX);
+        break;
     case 1:
+        /* movzwl */
+        tcg_out_modrm(s, 0xb7 | P_EXT, data_reg, TCG_REG_EAX);
+        break;
     case 2:
     default:
@@ -961 +967 @@
         tcg_gen_stack_alignment_check(s);
 # endif
-
         tcg_out8(s, 0xe8);
         tcg_out32(s, (tcg_target_long)qemu_st_helpers[s_bits] -

trunk/src/recompiler/tcg/i386/tcg-target.h

@@ -22 +22 @@
  * THE SOFTWARE.
  */
-
 #define TCG_TARGET_I386 1
 

trunk/src/recompiler/tcg/tcg-op.h

@@ -22 +22 @@
  * THE SOFTWARE.
  */
-
 #include "tcg.h"
 
-#ifdef CONFIG_DYNGEN_OP
-/* legacy dyngen operations */
-#include "gen-op.h"
-#endif
-
 int gen_new_label(void);
 
-static inline void tcg_gen_op1(int opc, TCGv arg1)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
+static inline void tcg_gen_op1_i32(int opc, TCGv_i32 arg1)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+}
+
+static inline void tcg_gen_op1_i64(int opc, TCGv_i64 arg1)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
 }
 
@@ -44 +44 @@
 }
 
-static inline void tcg_gen_op2(int opc, TCGv arg1, TCGv arg2)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-}
-
-static inline void tcg_gen_op2i(int opc, TCGv arg1, TCGArg arg2)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
+static inline void tcg_gen_op2_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+}
+
+static inline void tcg_gen_op2_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+}
+
+static inline void tcg_gen_op2i_i32(int opc, TCGv_i32 arg1, TCGArg arg2)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = arg2;
+}
+
+static inline void tcg_gen_op2i_i64(int opc, TCGv_i64 arg1, TCGArg arg2)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
     *gen_opparam_ptr++ = arg2;
 }
@@ -65 +79 @@
 }
 
-static inline void tcg_gen_op3(int opc, TCGv arg1, TCGv arg2, TCGv arg3)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
-}
-
-static inline void tcg_gen_op3i(int opc, TCGv arg1, TCGv arg2, TCGArg arg3)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
+static inline void tcg_gen_op3_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                   TCGv_i32 arg3)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+}
+
+static inline void tcg_gen_op3_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                   TCGv_i64 arg3)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+}
+
+static inline void tcg_gen_op3i_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                    TCGArg arg3)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
     *gen_opparam_ptr++ = arg3;
 }
 
-static inline void tcg_gen_op4(int opc, TCGv arg1, TCGv arg2, TCGv arg3,
-                               TCGv arg4)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
-    *gen_opparam_ptr++ = GET_TCGV(arg4);
-}
-
-static inline void tcg_gen_op4i(int opc, TCGv arg1, TCGv arg2, TCGv arg3,
-                                TCGArg arg4)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
+static inline void tcg_gen_op3i_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                    TCGArg arg3)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = arg3;
+}
+
+static inline void tcg_gen_ldst_op_i32(int opc, TCGv_i32 val, TCGv_ptr base,
+                                       TCGArg offset)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(val);
+    *gen_opparam_ptr++ = GET_TCGV_PTR(base);
+    *gen_opparam_ptr++ = offset;
+}
+
+static inline void tcg_gen_ldst_op_i64(int opc, TCGv_i64 val, TCGv_ptr base,
+                                       TCGArg offset)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *gen_opparam_ptr++ = GET_TCGV_PTR(base);
+    *gen_opparam_ptr++ = offset;
+}
+
+static inline void tcg_gen_qemu_ldst_op_i64_i32(int opc, TCGv_i64 val, TCGv_i32 addr,
+                                                TCGArg mem_index)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *gen_opparam_ptr++ = GET_TCGV_I32(addr);
+    *gen_opparam_ptr++ = mem_index;
+}
+
+static inline void tcg_gen_qemu_ldst_op_i64_i64(int opc, TCGv_i64 val, TCGv_i64 addr,
+                                                TCGArg mem_index)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *gen_opparam_ptr++ = GET_TCGV_I64(addr);
+    *gen_opparam_ptr++ = mem_index;
+}
+
+static inline void tcg_gen_op4_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                   TCGv_i32 arg3, TCGv_i32 arg4)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+}
+
+static inline void tcg_gen_op4_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                   TCGv_i64 arg3, TCGv_i64 arg4)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+}
+
+static inline void tcg_gen_op4i_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                    TCGv_i32 arg3, TCGArg arg4)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
     *gen_opparam_ptr++ = arg4;
 }
 
-static inline void tcg_gen_op4ii(int opc, TCGv arg1, TCGv arg2, TCGArg arg3,
-                                 TCGArg arg4)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
+static inline void tcg_gen_op4i_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                    TCGv_i64 arg3, TCGArg arg4)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = arg4;
+}
+
+static inline void tcg_gen_op4ii_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                     TCGArg arg3, TCGArg arg4)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
     *gen_opparam_ptr++ = arg3;
     *gen_opparam_ptr++ = arg4;
 }
 
-static inline void tcg_gen_op5(int opc, TCGv arg1, TCGv arg2,
-                               TCGv arg3, TCGv arg4,
-                               TCGv arg5)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
-    *gen_opparam_ptr++ = GET_TCGV(arg4);
-    *gen_opparam_ptr++ = GET_TCGV(arg5);
-}
-
-static inline void tcg_gen_op5i(int opc, TCGv arg1, TCGv arg2,
-                                TCGv arg3, TCGv arg4,
-                                TCGArg arg5)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
-    *gen_opparam_ptr++ = GET_TCGV(arg4);
+static inline void tcg_gen_op4ii_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                     TCGArg arg3, TCGArg arg4)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = arg3;
+    *gen_opparam_ptr++ = arg4;
+}
+
+static inline void tcg_gen_op5_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                   TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+}
+
+static inline void tcg_gen_op5_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                   TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+}
+
+static inline void tcg_gen_op5i_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGArg arg5)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
     *gen_opparam_ptr++ = arg5;
 }
 
-static inline void tcg_gen_op6(int opc, TCGv arg1, TCGv arg2,
-                               TCGv arg3, TCGv arg4,
-                               TCGv arg5, TCGv arg6)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
-    *gen_opparam_ptr++ = GET_TCGV(arg4);
-    *gen_opparam_ptr++ = GET_TCGV(arg5);
-    *gen_opparam_ptr++ = GET_TCGV(arg6);
-}
-
-static inline void tcg_gen_op6ii(int opc, TCGv arg1, TCGv arg2,
-                                 TCGv arg3, TCGv arg4,
-                                 TCGArg arg5, TCGArg arg6)
-{
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV(arg1);
-    *gen_opparam_ptr++ = GET_TCGV(arg2);
-    *gen_opparam_ptr++ = GET_TCGV(arg3);
-    *gen_opparam_ptr++ = GET_TCGV(arg4);
+static inline void tcg_gen_op5i_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGArg arg5)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *gen_opparam_ptr++ = arg5;
+}
+
+static inline void tcg_gen_op6_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                   TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5,
+                                   TCGv_i32 arg6)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg6);
+}
+
+static inline void tcg_gen_op6_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                   TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5,
+                                   TCGv_i64 arg6)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg6);
+}
+
+static inline void tcg_gen_op6ii_i32(int opc, TCGv_i32 arg1, TCGv_i32 arg2,
+                                     TCGv_i32 arg3, TCGv_i32 arg4, TCGArg arg5,
+                                     TCGArg arg6)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
     *gen_opparam_ptr++ = arg5;
     *gen_opparam_ptr++ = arg6;
 }
 
+static inline void tcg_gen_op6ii_i64(int opc, TCGv_i64 arg1, TCGv_i64 arg2,
+                                     TCGv_i64 arg3, TCGv_i64 arg4, TCGArg arg5,
+                                     TCGArg arg6)
+{
+    *gen_opc_ptr++ = opc;
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *gen_opparam_ptr++ = arg5;
+    *gen_opparam_ptr++ = arg6;
+}
+
 static inline void gen_set_label(int n)
 {
@@ -171 +317 @@
 }
 
-static inline void tcg_gen_mov_i32(TCGv ret, TCGv arg)
-{
-    if (GET_TCGV(ret) != GET_TCGV(arg))
-        tcg_gen_op2(INDEX_op_mov_i32, ret, arg);
-}
-
-static inline void tcg_gen_movi_i32(TCGv ret, int32_t arg)
-{
-    tcg_gen_op2i(INDEX_op_movi_i32, ret, arg);
+static inline void tcg_gen_mov_i32(TCGv_i32 ret, TCGv_i32 arg)
+{
+    if (GET_TCGV_I32(ret) != GET_TCGV_I32(arg))
+        tcg_gen_op2_i32(INDEX_op_mov_i32, ret, arg);
+}
+
+static inline void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
+{
+    tcg_gen_op2i_i32(INDEX_op_movi_i32, ret, arg);
 }
 
 /* helper calls */
-#define TCG_HELPER_CALL_FLAGS 0
-
-static inline void tcg_gen_helper_0_0(void *func)
-{
-    TCGv t0;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 0, NULL, 0, NULL);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_0_1(void *func, TCGv arg)
-{
-    TCGv t0;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 0, NULL, 1, &arg);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_0_2(void *func, TCGv arg1, TCGv arg2)
-{
-    TCGv args[2];
-    TCGv t0;
-    args[0] = arg1;
-    args[1] = arg2;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 0, NULL, 2, args);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_0_3(void *func,
-                                      TCGv arg1, TCGv arg2, TCGv arg3)
-{
-    TCGv args[3];
-    TCGv t0;
-    args[0] = arg1;
-    args[1] = arg2;
-    args[2] = arg3;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 0, NULL, 3, args);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_0_4(void *func, TCGv arg1, TCGv arg2,
-                                      TCGv arg3, TCGv arg4)
-{
-    TCGv args[4];
-    TCGv t0;
-    args[0] = arg1;
-    args[1] = arg2;
-    args[2] = arg3;
-    args[3] = arg4;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 0, NULL, 4, args);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_1_0(void *func, TCGv ret)
-{
-    TCGv t0;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 1, &ret, 0, NULL);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_1_1(void *func, TCGv ret, TCGv arg1)
-{
-    TCGv t0;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 1, &ret, 1, &arg1);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_1_2(void *func, TCGv ret,
-                                      TCGv arg1, TCGv arg2)
-{
-    TCGv args[2];
-    TCGv t0;
-    args[0] = arg1;
-    args[1] = arg2;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 1, &ret, 2, args);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_1_3(void *func, TCGv ret,
-                                      TCGv arg1, TCGv arg2, TCGv arg3)
-{
-    TCGv args[3];
-    TCGv t0;
-    args[0] = arg1;
-    args[1] = arg2;
-    args[2] = arg3;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 1, &ret, 3, args);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_helper_1_4(void *func, TCGv ret,
-                                      TCGv arg1, TCGv arg2, TCGv arg3,
-                                      TCGv arg4)
-{
-    TCGv args[4];
-    TCGv t0;
-    args[0] = arg1;
-    args[1] = arg2;
-    args[2] = arg3;
-    args[3] = arg4;
-    t0 = tcg_const_ptr((tcg_target_long)func);
-    tcg_gen_call(&tcg_ctx,
-                 t0, TCG_HELPER_CALL_FLAGS,
-                 1, &ret, 4, args);
-    tcg_temp_free(t0);
+static inline void tcg_gen_helperN(void *func, int flags, int sizemask,
+                                   TCGArg ret, int nargs, TCGArg *args)
+{
+    TCGv_ptr fn;
+    fn = tcg_const_ptr((tcg_target_long)func);
+    tcg_gen_callN(&tcg_ctx, fn, flags, sizemask, ret,
+                  nargs, args);
+    tcg_temp_free_ptr(fn);
+}
+
+/* FIXME: Should this be pure?  */
+static inline void tcg_gen_helper64(void *func, TCGv_i64 ret,
+                                    TCGv_i64 a, TCGv_i64 b)
+{
+    TCGv_ptr fn;
+    TCGArg args[2];
+    fn = tcg_const_ptr((tcg_target_long)func);
+    args[0] = GET_TCGV_I64(a);
+    args[1] = GET_TCGV_I64(b);
+    tcg_gen_callN(&tcg_ctx, fn, 0, 7, GET_TCGV_I64(ret), 2, args);
+    tcg_temp_free_ptr(fn);
 }
 
 /* 32 bit ops */
 
-static inline void tcg_gen_ld8u_i32(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_ld8u_i32, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld8s_i32(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_ld8s_i32, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld16u_i32(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_ld16u_i32, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld16s_i32(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_ld16s_i32, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld_i32(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_ld_i32, ret, arg2, offset);
-}
-
-static inline void tcg_gen_st8_i32(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_st8_i32, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st16_i32(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_st16_i32, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st_i32(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_st_i32, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_add_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_add_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_addi_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_ld8u_i32, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld8s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_ld8s_i32, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld16u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_ld16u_i32, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld16s_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_ld16s_i32, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_ld_i32, ret, arg2, offset);
+}
+
+static inline void tcg_gen_st8_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_st8_i32, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_st16_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_st16_i32, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_st_i32(TCGv_i32 arg1, TCGv_ptr arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i32(INDEX_op_st_i32, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_add_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_add_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     /* some cases can be optimized here */
@@ -368 +405 @@
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_add_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_sub_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_sub_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_subi_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_sub_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_sub_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0 = tcg_const_i32(arg1);
+    tcg_gen_sub_i32(ret, t0, arg2);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     /* some cases can be optimized here */
@@ -385 +429 @@
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_sub_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_and_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_and_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_andi_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_and_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     /* some cases can be optimized here */
@@ -404 +448 @@
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_and_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_or_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_or_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_ori_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_or_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     /* some cases can be optimized here */
@@ -423 +467 @@
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_or_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_xor_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_xor_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_xori_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_xor_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     /* some cases can be optimized here */
@@ -440 +484 @@
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_xor_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_shl_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_shl_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_shli_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_shl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_shl_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_shl_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_shr_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_shr_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_shri_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_shr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_shr_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_shr_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_sar_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_sar_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_sari_i32(TCGv ret, TCGv arg1, int32_t arg2)
+static inline void tcg_gen_sar_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_sar_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
     if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i32(arg2);
+        TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_sar_i32(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i32(t0);
     }
 }
 
-static inline void tcg_gen_brcond_i32(int cond, TCGv arg1, TCGv arg2,
+static inline void tcg_gen_brcond_i32(int cond, TCGv_i32 arg1, TCGv_i32 arg2,
                                       int label_index)
 {
-    tcg_gen_op4ii(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
-}
-
-static inline void tcg_gen_brcondi_i32(int cond, TCGv arg1, int32_t arg2,
+    tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
+}
+
+static inline void tcg_gen_brcondi_i32(int cond, TCGv_i32 arg1, int32_t arg2,
                                        int label_index)
 {
-    TCGv t0 = tcg_const_i32(arg2);
+    TCGv_i32 t0 = tcg_const_i32(arg2);
     tcg_gen_brcond_i32(cond, arg1, t0, label_index);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_mul_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_mul_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_muli_i32(TCGv ret, TCGv arg1, int32_t arg2)
-{
-    TCGv t0 = tcg_const_i32(arg2);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_mul_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
+{
+    TCGv_i32 t0 = tcg_const_i32(arg2);
     tcg_gen_mul_i32(ret, arg1, t0);
-    tcg_temp_free(t0);
+    tcg_temp_free_i32(t0);
 }
 
 #ifdef TCG_TARGET_HAS_div_i32
-static inline void tcg_gen_div_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_div_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_rem_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_rem_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_divu_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_divu_i32, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_remu_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_remu_i32, ret, arg1, arg2);
-}
-#else
-static inline void tcg_gen_div_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
+static inline void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
+}
+#else
+static inline void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
     tcg_gen_sari_i32(t0, arg1, 31);
-    tcg_gen_op5(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_rem_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
+    tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
     tcg_gen_sari_i32(t0, arg1, 31);
-    tcg_gen_op5(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_divu_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
+    tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
     tcg_gen_movi_i32(t0, 0);
-    tcg_gen_op5(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_remu_i32(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
+    tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
     tcg_gen_movi_i32(t0, 0);
-    tcg_gen_op5(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
-    tcg_temp_free(t0);
+    tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
+    tcg_temp_free_i32(t0);
 }
 #endif
@@ -580 +624 @@
 #if TCG_TARGET_REG_BITS == 32
 
-static inline void tcg_gen_mov_i64(TCGv ret, TCGv arg)
-{
-    if (GET_TCGV(ret) != GET_TCGV(arg)) {
-        tcg_gen_mov_i32(ret, arg);
+static inline void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    if (GET_TCGV_I64(ret) != GET_TCGV_I64(arg)) {
+        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
     }
 }
 
-static inline void tcg_gen_movi_i64(TCGv ret, int64_t arg)
-{
-    tcg_gen_movi_i32(ret, arg);
+static inline void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
+{
+    tcg_gen_movi_i32(TCGV_LOW(ret), arg);
     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
 }
 
-static inline void tcg_gen_ld8u_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_ld8u_i32(ret, arg2, offset);
+static inline void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                    tcg_target_long offset)
+{
+    tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_ld8s_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_ld8s_i32(ret, arg2, offset);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_ld16u_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_ld16u_i32(ret, arg2, offset);
+static inline void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                    tcg_target_long offset)
+{
+    tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), 31);
+}
+
+static inline void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                     tcg_target_long offset)
+{
+    tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_ld16s_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_ld16s_i32(ret, arg2, offset);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_ld32u_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_ld_i32(ret, arg2, offset);
+static inline void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                     tcg_target_long offset)
+{
+    tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+}
+
+static inline void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                     tcg_target_long offset)
+{
+    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_ld32s_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_ld_i32(ret, arg2, offset);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_ld_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
+static inline void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                     tcg_target_long offset)
+{
+    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+}
+
+static inline void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2,
+                                  tcg_target_long offset)
 {
     /* since arg2 and ret have different types, they cannot be the
@@ -636 +687 @@
 #ifdef TCG_TARGET_WORDS_BIGENDIAN
     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
-    tcg_gen_ld_i32(ret, arg2, offset + 4);
-#else
-    tcg_gen_ld_i32(ret, arg2, offset);
+    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
+#else
+    tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
 #endif
 }
 
-static inline void tcg_gen_st8_i64(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_st8_i32(arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st16_i64(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_st16_i32(arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st32_i64(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_st_i32(arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st_i64(TCGv arg1, TCGv arg2, tcg_target_long offset)
+static inline void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2,
+                                   tcg_target_long offset)
+{
+    tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset);
+}
+
+static inline void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2,
+                                    tcg_target_long offset)
+{
+    tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset);
+}
+
+static inline void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2,
+                                    tcg_target_long offset)
+{
+    tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
+}
+
+static inline void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2,
+                                  tcg_target_long offset)
 {
 #ifdef TCG_TARGET_WORDS_BIGENDIAN
     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
-    tcg_gen_st_i32(arg1, arg2, offset + 4);
-#else
-    tcg_gen_st_i32(arg1, arg2, offset);
+    tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
+#else
+    tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
 #endif
 }
 
-static inline void tcg_gen_add_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op6(INDEX_op_add2_i32, ret, TCGV_HIGH(ret),
-                arg1, TCGV_HIGH(arg1), arg2, TCGV_HIGH(arg2));
-}
-
-static inline void tcg_gen_addi_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
-    tcg_gen_add_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_sub_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op6(INDEX_op_sub2_i32, ret, TCGV_HIGH(ret),
-                arg1, TCGV_HIGH(arg1), arg2, TCGV_HIGH(arg2));
-}
-
-static inline void tcg_gen_subi_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
-    tcg_gen_sub_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_and_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_and_i32(ret, arg1, arg2);
+static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op6_i32(INDEX_op_add2_i32, TCGV_LOW(ret), TCGV_HIGH(ret),
+                    TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
+                    TCGV_HIGH(arg2));
+}
+
+static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op6_i32(INDEX_op_sub2_i32, TCGV_LOW(ret), TCGV_HIGH(ret),
+                    TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
+                    TCGV_HIGH(arg2));
+}
+
+static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
 }
 
-static inline void tcg_gen_andi_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    tcg_gen_andi_i32(ret, arg1, arg2);
+static inline void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
     tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
 }
 
-static inline void tcg_gen_or_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_or_i32(ret, arg1, arg2);
+static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
 }
 
-static inline void tcg_gen_ori_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    tcg_gen_ori_i32(ret, arg1, arg2);
+static inline void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
     tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
 }
 
-static inline void tcg_gen_xor_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_xor_i32(ret, arg1, arg2);
+static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
 }
 
-static inline void tcg_gen_xori_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    tcg_gen_xori_i32(ret, arg1, arg2);
+static inline void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
     tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
 }
@@ -733 +776 @@
 /* XXX: use generic code when basic block handling is OK or CPU
    specific code (x86) */
-static inline void tcg_gen_shl_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_shl_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_shli_i64(TCGv ret, TCGv arg1, int64_t arg2)
+static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_shl_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
     tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
 }
 
-static inline void tcg_gen_shr_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_shr_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_shri_i64(TCGv ret, TCGv arg1, int64_t arg2)
+static inline void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_shr_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
     tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
 }
 
-static inline void tcg_gen_sar_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_sar_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_sari_i64(TCGv ret, TCGv arg1, int64_t arg2)
+static inline void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_sar_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
     tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
 }
 
-static inline void tcg_gen_brcond_i64(int cond, TCGv arg1, TCGv arg2,
+static inline void tcg_gen_brcond_i64(int cond, TCGv_i64 arg1, TCGv_i64 arg2,
                                       int label_index)
 {
-    tcg_gen_op6ii(INDEX_op_brcond2_i32,
-                  arg1, TCGV_HIGH(arg1), arg2, TCGV_HIGH(arg2),
-                  cond, label_index);
-}
-
-static inline void tcg_gen_mul_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0, t1;
-
-    t0 = tcg_temp_new(TCG_TYPE_I64);
-    t1 = tcg_temp_new(TCG_TYPE_I32);
-
-    tcg_gen_op4(INDEX_op_mulu2_i32, t0, TCGV_HIGH(t0), arg1, arg2);
-
-    tcg_gen_mul_i32(t1, arg1, TCGV_HIGH(arg2));
+    tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
+                      TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
+                      TCGV_HIGH(arg2), cond, label_index);
+}
+
+static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    TCGv_i32 t1;
+
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i32();
+
+    tcg_gen_op4_i32(INDEX_op_mulu2_i32, TCGV_LOW(t0), TCGV_HIGH(t0),
+                    TCGV_LOW(arg1), TCGV_LOW(arg2));
+
+    tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
-    tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), arg2);
+    tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
 
     tcg_gen_mov_i64(ret, t0);
-    tcg_temp_free(t0);
-    tcg_temp_free(t1);
-}
-
-static inline void tcg_gen_muli_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
-    tcg_gen_mul_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_div_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_div_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_rem_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_rem_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_divu_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_divu_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_remu_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_helper_1_2(tcg_helper_remu_i64, ret, arg1, arg2);
-}
-
-#else
-
-static inline void tcg_gen_mov_i64(TCGv ret, TCGv arg)
-{
-    if (GET_TCGV(ret) != GET_TCGV(arg))
-        tcg_gen_op2(INDEX_op_mov_i64, ret, arg);
-}
-
-static inline void tcg_gen_movi_i64(TCGv ret, int64_t arg)
-{
-    tcg_gen_op2i(INDEX_op_movi_i64, ret, arg);
-}
-
-static inline void tcg_gen_ld8u_i64(TCGv ret, TCGv arg2,
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i32(t1);
+}
+
+static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_div_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_rem_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_divu_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_helper64(tcg_helper_remu_i64, ret, arg1, arg2);
+}
+
+#else
+
+static inline void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    if (GET_TCGV_I64(ret) != GET_TCGV_I64(arg))
+        tcg_gen_op2_i64(INDEX_op_mov_i64, ret, arg);
+}
+
+static inline void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
+{
+    tcg_gen_op2i_i64(INDEX_op_movi_i64, ret, arg);
+}
+
+static inline void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_i64 arg2,
                                     tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_ld8u_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld8s_i64(TCGv ret, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_ld8u_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_i64 arg2,
                                     tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_ld8s_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld16u_i64(TCGv ret, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_ld8s_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_i64 arg2,
                                      tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_ld16u_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld16s_i64(TCGv ret, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_ld16u_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_i64 arg2,
                                      tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_ld16s_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld32u_i64(TCGv ret, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_ld16s_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_i64 arg2,
                                      tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_ld32u_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld32s_i64(TCGv ret, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_ld32u_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_i64 arg2,
                                      tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_ld32s_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_ld_i64(TCGv ret, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_ld_i64, ret, arg2, offset);
-}
-
-static inline void tcg_gen_st8_i64(TCGv arg1, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_ld32s_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_i64 arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i64(INDEX_op_ld_i64, ret, arg2, offset);
+}
+
+static inline void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_i64 arg2,
                                    tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_st8_i64, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st16_i64(TCGv arg1, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_st8_i64, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_i64 arg2,
                                     tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_st16_i64, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st32_i64(TCGv arg1, TCGv arg2,
+    tcg_gen_ldst_op_i64(INDEX_op_st16_i64, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_i64 arg2,
                                     tcg_target_long offset)
 {
-    tcg_gen_op3i(INDEX_op_st32_i64, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_st_i64(TCGv arg1, TCGv arg2, tcg_target_long offset)
-{
-    tcg_gen_op3i(INDEX_op_st_i64, arg1, arg2, offset);
-}
-
-static inline void tcg_gen_add_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_add_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_addi_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
-    tcg_gen_add_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_sub_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_sub_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_subi_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
-    tcg_gen_sub_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_and_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_and_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_andi_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
+    tcg_gen_ldst_op_i64(INDEX_op_st32_i64, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_i64 arg2, tcg_target_long offset)
+{
+    tcg_gen_ldst_op_i64(INDEX_op_st_i64, arg1, arg2, offset);
+}
+
+static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_add_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_sub_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_and_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg2);
     tcg_gen_and_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_or_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_or_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_ori_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_or_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg2);
     tcg_gen_or_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_xor_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_xor_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_xori_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_xor_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg2);
     tcg_gen_xor_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_shl_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_shl_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_shli_i64(TCGv ret, TCGv arg1, int64_t arg2)
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_shl_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
     if (arg2 == 0) {
         tcg_gen_mov_i64(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i64(arg2);
+        TCGv_i64 t0 = tcg_const_i64(arg2);
         tcg_gen_shl_i64(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i64(t0);
     }
 }
 
-static inline void tcg_gen_shr_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_shr_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_shri_i64(TCGv ret, TCGv arg1, int64_t arg2)
+static inline void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_shr_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
     if (arg2 == 0) {
         tcg_gen_mov_i64(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i64(arg2);
+        TCGv_i64 t0 = tcg_const_i64(arg2);
         tcg_gen_shr_i64(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i64(t0);
     }
 }
 
-static inline void tcg_gen_sar_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_sar_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_sari_i64(TCGv ret, TCGv arg1, int64_t arg2)
+static inline void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_sar_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
     if (arg2 == 0) {
         tcg_gen_mov_i64(ret, arg1);
     } else {
-        TCGv t0 = tcg_const_i64(arg2);
+        TCGv_i64 t0 = tcg_const_i64(arg2);
         tcg_gen_sar_i64(ret, arg1, t0);
-        tcg_temp_free(t0);
+        tcg_temp_free_i64(t0);
     }
 }
 
-static inline void tcg_gen_brcond_i64(int cond, TCGv arg1, TCGv arg2,
+static inline void tcg_gen_brcond_i64(int cond, TCGv_i64 arg1, TCGv_i64 arg2,
                                       int label_index)
 {
-    tcg_gen_op4ii(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
-}
-
-static inline void tcg_gen_mul_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_mul_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_muli_i64(TCGv ret, TCGv arg1, int64_t arg2)
-{
-    TCGv t0 = tcg_const_i64(arg2);
+    tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
+}
+
+static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2);
+}
+
+#ifdef TCG_TARGET_HAS_div_i64
+static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
+}
+
+static inline void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
+}
+#else
+static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_sari_i64(t0, arg1, 63);
+    tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_sari_i64(t0, arg1, 63);
+    tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_movi_i64(t0, 0);
+    tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_movi_i64(t0, 0);
+    tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
+    tcg_temp_free_i64(t0);
+}
+#endif
+
+#endif
+
+static inline void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    /* some cases can be optimized here */
+    if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_add_i64(ret, arg1, t0);
+        tcg_temp_free_i64(t0);
+    }
+}
+
+static inline void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg1);
+    tcg_gen_sub_i64(ret, t0, arg2);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    /* some cases can be optimized here */
+    if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_sub_i64(ret, arg1, t0);
+        tcg_temp_free_i64(t0);
+    }
+}
+static inline void tcg_gen_brcondi_i64(int cond, TCGv_i64 arg1, int64_t arg2,
+                                       int label_index)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg2);
+    tcg_gen_brcond_i64(cond, arg1, t0, label_index);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    TCGv_i64 t0 = tcg_const_i64(arg2);
     tcg_gen_mul_i64(ret, arg1, t0);
-    tcg_temp_free(t0);
-}
-
-#ifdef TCG_TARGET_HAS_div_i64
-static inline void tcg_gen_div_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_div_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_rem_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_rem_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_divu_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_divu_i64, ret, arg1, arg2);
-}
-
-static inline void tcg_gen_remu_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    tcg_gen_op3(INDEX_op_remu_i64, ret, arg1, arg2);
-}
-#else
-static inline void tcg_gen_div_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I64);
-    tcg_gen_sari_i64(t0, arg1, 63);
-    tcg_gen_op5(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_rem_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I64);
-    tcg_gen_sari_i64(t0, arg1, 63);
-    tcg_gen_op5(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_divu_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I64);
-    tcg_gen_movi_i64(t0, 0);
-    tcg_gen_op5(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-
-static inline void tcg_gen_remu_i64(TCGv ret, TCGv arg1, TCGv arg2)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I64);
-    tcg_gen_movi_i64(t0, 0);
-    tcg_gen_op5(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
-    tcg_temp_free(t0);
-}
-#endif
-
-#endif
-
-static inline void tcg_gen_brcondi_i64(int cond, TCGv arg1, int64_t arg2,
-                                       int label_index)
-{
-    TCGv t0 = tcg_const_i64(arg2);
-    tcg_gen_brcond_i64(cond, arg1, t0, label_index);
-    tcg_temp_free(t0);
-}
+    tcg_temp_free_i64(t0);
+}
+
 
 /***************************************/
 /* optional operations */
 
-static inline void tcg_gen_ext8s_i32(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
 #ifdef TCG_TARGET_HAS_ext8s_i32
-    tcg_gen_op2(INDEX_op_ext8s_i32, ret, arg);
+    tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
 #else
     tcg_gen_shli_i32(ret, arg, 24);
@@ -1101 +1156 @@
 }
 
-static inline void tcg_gen_ext16s_i32(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
 #ifdef TCG_TARGET_HAS_ext16s_i32
-    tcg_gen_op2(INDEX_op_ext16s_i32, ret, arg);
+    tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
 #else
     tcg_gen_shli_i32(ret, arg, 16);
@@ -1113 +1168 @@
 /* These are currently just for convenience.
    We assume a target will recognise these automatically .  */
-static inline void tcg_gen_ext8u_i32(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
     tcg_gen_andi_i32(ret, arg, 0xffu);
 }
 
-static inline void tcg_gen_ext16u_i32(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
     tcg_gen_andi_i32(ret, arg, 0xffffu);
@@ -1124 +1179 @@
 
 /* Note: we assume the two high bytes are set to zero */
-static inline void tcg_gen_bswap16_i32(TCGv ret, TCGv arg)
+static inline void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
 #ifdef TCG_TARGET_HAS_bswap16_i32
-    tcg_gen_op2(INDEX_op_bswap16_i32, ret, arg);
-#else
-    TCGv t0, t1;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
-    t1 = tcg_temp_new(TCG_TYPE_I32);
+    tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
+#else
+    TCGv_i32 t0, t1;
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new_i32();
 
     tcg_gen_shri_i32(t0, arg, 8);
@@ -1137 +1192 @@
     tcg_gen_shli_i32(t1, t1, 8);
     tcg_gen_or_i32(ret, t0, t1);
-    tcg_temp_free(t0);
-    tcg_temp_free(t1);
-#endif
-}
-
-static inline void tcg_gen_bswap_i32(TCGv ret, TCGv arg)
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+#endif
+}
+
+static inline void tcg_gen_bswap_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
 #ifdef TCG_TARGET_HAS_bswap_i32
-    tcg_gen_op2(INDEX_op_bswap_i32, ret, arg);
-#else
-    TCGv t0, t1;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
-    t1 = tcg_temp_new(TCG_TYPE_I32);
+    tcg_gen_op2_i32(INDEX_op_bswap_i32, ret, arg);
+#else
+    TCGv_i32 t0, t1;
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new_i32();
 
     tcg_gen_shli_i32(t0, arg, 24);
@@ -1163 +1218 @@
     tcg_gen_shri_i32(t1, arg, 24);
     tcg_gen_or_i32(ret, t0, t1);
-    tcg_temp_free(t0);
-    tcg_temp_free(t1);
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
 #endif
 }
 
 #if TCG_TARGET_REG_BITS == 32
-static inline void tcg_gen_ext8s_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_ext8s_i32(ret, arg);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_ext16s_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_ext16s_i32(ret, arg);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_ext32s_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_mov_i32(ret, arg);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_ext8u_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_ext8u_i32(ret, arg);
+static inline void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+}
+
+static inline void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+}
+
+static inline void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+}
+
+static inline void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_ext16u_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_ext16u_i32(ret, arg);
+static inline void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_ext32u_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_mov_i32(ret, arg);
+static inline void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_trunc_i64_i32(TCGv ret, TCGv arg)
-{
-    tcg_gen_mov_i32(ret, arg);
-}
-
-static inline void tcg_gen_extu_i32_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_mov_i32(ret, arg);
+static inline void tcg_gen_trunc_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
+{
+    tcg_gen_mov_i32(ret, TCGV_LOW(arg));
+}
+
+static inline void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
+{
+    tcg_gen_mov_i32(TCGV_LOW(ret), arg);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
-static inline void tcg_gen_ext_i32_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_mov_i32(ret, arg);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-}
-
-static inline void tcg_gen_bswap_i64(TCGv ret, TCGv arg)
-{
-    TCGv t0, t1;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
-    t1 = tcg_temp_new(TCG_TYPE_I32);
-
-    tcg_gen_bswap_i32(t0, arg);
+static inline void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
+{
+    tcg_gen_mov_i32(TCGV_LOW(ret), arg);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+}
+
+static inline void tcg_gen_bswap_i64(TCGv_i64 ret, TCGv_i64 arg)
+{
+    TCGv_i32 t0, t1;
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new_i32();
+
+    tcg_gen_bswap_i32(t0, TCGV_LOW(arg));
     tcg_gen_bswap_i32(t1, TCGV_HIGH(arg));
-    tcg_gen_mov_i32(ret, t1);
+    tcg_gen_mov_i32(TCGV_LOW(ret), t1);
     tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
-    tcg_temp_free(t0);
-    tcg_temp_free(t1);
-}
-#else
-
-static inline void tcg_gen_ext8s_i64(TCGv ret, TCGv arg)
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+}
+#else
+
+static inline void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
 #ifdef TCG_TARGET_HAS_ext8s_i64
-    tcg_gen_op2(INDEX_op_ext8s_i64, ret, arg);
+    tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
 #else
     tcg_gen_shli_i64(ret, arg, 56);
@@ -1247 +1302 @@
 }
 
-static inline void tcg_gen_ext16s_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
 #ifdef TCG_TARGET_HAS_ext16s_i64
-    tcg_gen_op2(INDEX_op_ext16s_i64, ret, arg);
+    tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
 #else
     tcg_gen_shli_i64(ret, arg, 48);
@@ -1257 +1312 @@
 }
 
-static inline void tcg_gen_ext32s_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
 #ifdef TCG_TARGET_HAS_ext32s_i64
-    tcg_gen_op2(INDEX_op_ext32s_i64, ret, arg);
+    tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
 #else
     tcg_gen_shli_i64(ret, arg, 32);
@@ -1267 +1322 @@
 }
 
-static inline void tcg_gen_ext8u_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
     tcg_gen_andi_i64(ret, arg, 0xffu);
 }
 
-static inline void tcg_gen_ext16u_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
     tcg_gen_andi_i64(ret, arg, 0xffffu);
 }
 
-static inline void tcg_gen_ext32u_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
     tcg_gen_andi_i64(ret, arg, 0xffffffffu);
@@ -1284 +1339 @@
 /* Note: we assume the target supports move between 32 and 64 bit
    registers.  This will probably break MIPS64 targets.  */
-static inline void tcg_gen_trunc_i64_i32(TCGv ret, TCGv arg)
-{
-    tcg_gen_mov_i32(ret, arg);
+static inline void tcg_gen_trunc_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
+{
+    tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
 }
 
 /* Note: we assume the target supports move between 32 and 64 bit
    registers */
-static inline void tcg_gen_extu_i32_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_andi_i64(ret, arg, 0xffffffffu);
+static inline void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
+{
+    tcg_gen_andi_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)), 0xffffffffu);
 }
 
 /* Note: we assume the target supports move between 32 and 64 bit
    registers */
-static inline void tcg_gen_ext_i32_i64(TCGv ret, TCGv arg)
-{
-    tcg_gen_ext32s_i64(ret, arg);
-}
-
-static inline void tcg_gen_bswap_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
+{
+    tcg_gen_ext32s_i64(ret, MAKE_TCGV_I64(GET_TCGV_I32(arg)));
+}
+
+static inline void tcg_gen_bswap_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
 #ifdef TCG_TARGET_HAS_bswap_i64
-    tcg_gen_op2(INDEX_op_bswap_i64, ret, arg);
-#else
-    TCGv t0, t1;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
-    t1 = tcg_temp_new(TCG_TYPE_I32);
+    tcg_gen_op2_i64(INDEX_op_bswap_i64, ret, arg);
+#else
+    TCGv_i32 t0, t1;
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new_i32();
 
     tcg_gen_shli_i64(t0, arg, 56);
@@ -1340 +1395 @@
     tcg_gen_shri_i64(t1, arg, 56);
     tcg_gen_or_i64(ret, t0, t1);
-    tcg_temp_free(t0);
-    tcg_temp_free(t1);
-#endif
-}
-
-#endif
-
-static inline void tcg_gen_neg_i32(TCGv ret, TCGv arg)
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+#endif
+}
+
+#endif
+
+static inline void tcg_gen_neg_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
 #ifdef TCG_TARGET_HAS_neg_i32
-    tcg_gen_op2(INDEX_op_neg_i32, ret, arg);
-#else
-    TCGv t0 = tcg_const_i32(0);
+    tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg);
+#else
+    TCGv_i32 t0 = tcg_const_i32(0);
     tcg_gen_sub_i32(ret, t0, arg);
-    tcg_temp_free(t0);
-#endif
-}
-
-static inline void tcg_gen_neg_i64(TCGv ret, TCGv arg)
+    tcg_temp_free_i32(t0);
+#endif
+}
+
+static inline void tcg_gen_neg_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
 #ifdef TCG_TARGET_HAS_neg_i64
-    tcg_gen_op2(INDEX_op_neg_i64, ret, arg);
-#else
-    TCGv t0 = tcg_const_i64(0);
+    tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg);
+#else
+    TCGv_i64 t0 = tcg_const_i64(0);
     tcg_gen_sub_i64(ret, t0, arg);
-    tcg_temp_free(t0);
-#endif
-}
-
-static inline void tcg_gen_not_i32(TCGv ret, TCGv arg)
+    tcg_temp_free_i64(t0);
+#endif
+}
+
+static inline void tcg_gen_not_i32(TCGv_i32 ret, TCGv_i32 arg)
 {
     tcg_gen_xori_i32(ret, arg, -1);
 }
 
-static inline void tcg_gen_not_i64(TCGv ret, TCGv arg)
+static inline void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
 {
     tcg_gen_xori_i64(ret, arg, -1);
 }
 
-static inline void tcg_gen_discard_i32(TCGv arg)
-{
-    tcg_gen_op1(INDEX_op_discard, arg);
+static inline void tcg_gen_discard_i32(TCGv_i32 arg)
+{
+    tcg_gen_op1_i32(INDEX_op_discard, arg);
 }
 
 #if TCG_TARGET_REG_BITS == 32
-static inline void tcg_gen_discard_i64(TCGv arg)
-{
-    tcg_gen_discard_i32(arg);
+static inline void tcg_gen_discard_i64(TCGv_i64 arg)
+{
+    tcg_gen_discard_i32(TCGV_LOW(arg));
     tcg_gen_discard_i32(TCGV_HIGH(arg));
 }
 #else
-static inline void tcg_gen_discard_i64(TCGv arg)
-{
-    tcg_gen_op1(INDEX_op_discard, arg);
-}
-#endif
-
-static inline void tcg_gen_concat_i32_i64(TCGv dest, TCGv low, TCGv high)
+static inline void tcg_gen_discard_i64(TCGv_i64 arg)
+{
+    tcg_gen_op1_i64(INDEX_op_discard, arg);
+}
+#endif
+
+static inline void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
 {
 #if TCG_TARGET_REG_BITS == 32
-    tcg_gen_mov_i32(dest, low);
+    tcg_gen_mov_i32(TCGV_LOW(dest), low);
     tcg_gen_mov_i32(TCGV_HIGH(dest), high);
 #else
-    TCGv tmp = tcg_temp_new (TCG_TYPE_I64);
+    TCGv_i64 tmp = tcg_temp_new_i64();
     /* This extension is only needed for type correctness.
        We may be able to do better given target specific information.  */
@@ -1410 +1465 @@
     tcg_gen_extu_i32_i64(dest, low);
     tcg_gen_or_i64(dest, dest, tmp);
-    tcg_temp_free(tmp);
-#endif
-}
-
-static inline void tcg_gen_concat32_i64(TCGv dest, TCGv low, TCGv high)
+    tcg_temp_free_i64(tmp);
+#endif
+}
+
+static inline void tcg_gen_concat32_i64(TCGv_i64 dest, TCGv_i64 low, TCGv_i64 high)
 {
 #if TCG_TARGET_REG_BITS == 32
-    tcg_gen_concat_i32_i64(dest, low, high);
-#else
-    TCGv tmp = tcg_temp_new(TCG_TYPE_I64);
+    tcg_gen_concat_i32_i64(dest, TCGV_LOW(low), TCGV_LOW(high));
+#else
+    TCGv_i64 tmp = tcg_temp_new_i64();
     tcg_gen_ext32u_i64(dest, low);
     tcg_gen_shli_i64(tmp, high, 32);
     tcg_gen_or_i64(dest, dest, tmp);
-    tcg_temp_free(tmp);
-#endif
+    tcg_temp_free_i64(tmp);
+#endif
+}
+
+static inline void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
+    tcg_gen_not_i32(t0, arg2);
+    tcg_gen_and_i32(ret, arg1, t0);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_not_i64(t0, arg2);
+    tcg_gen_and_i64(ret, arg1, t0);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
+    tcg_gen_xor_i32(t0, arg1, arg2);
+    tcg_gen_not_i32(ret, t0);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_xor_i64(t0, arg1, arg2);
+    tcg_gen_not_i64(ret, t0);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
+    tcg_gen_and_i32(t0, arg1, arg2);
+    tcg_gen_not_i32(ret, t0);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_and_i64(t0, arg1, arg2);
+    tcg_gen_not_i64(ret, t0);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
+    tcg_gen_or_i32(t0, arg1, arg2);
+    tcg_gen_not_i32(ret, t0);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_or_i64(t0, arg1, arg2);
+    tcg_gen_not_i64(ret, t0);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
+    tcg_gen_not_i32(t0, arg2);
+    tcg_gen_or_i32(ret, arg1, t0);
+    tcg_temp_free_i32(t0);
+}
+
+static inline void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_not_i64(t0, arg2);
+    tcg_gen_or_i64(ret, arg1, t0);
+    tcg_temp_free_i64(t0);
+}
+
+static inline void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0, t1;
+
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new_i32();
+    tcg_gen_shl_i32(t0, arg1, arg2);
+    tcg_gen_subfi_i32(t1, 32, arg2);
+    tcg_gen_shr_i32(t1, arg1, t1);
+    tcg_gen_or_i32(ret, t0, t1);
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+}
+
+static inline void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0, t1;
+
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
+    tcg_gen_shl_i64(t0, arg1, arg2);
+    tcg_gen_subfi_i64(t1, 64, arg2);
+    tcg_gen_shr_i64(t1, arg1, t1);
+    tcg_gen_or_i64(ret, t0, t1);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
+static inline void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
+{
+    /* some cases can be optimized here */
+    if (arg2 == 0) {
+        tcg_gen_mov_i32(ret, arg1);
+    } else {
+        TCGv_i32 t0, t1;
+        t0 = tcg_temp_new_i32();
+        t1 = tcg_temp_new_i32();
+        tcg_gen_shli_i32(t0, arg1, arg2);
+        tcg_gen_shri_i32(t1, arg1, 32 - arg2);
+        tcg_gen_or_i32(ret, t0, t1);
+        tcg_temp_free_i32(t0);
+        tcg_temp_free_i32(t1);
+    }
+}
+
+static inline void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    /* some cases can be optimized here */
+    if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else {
+        TCGv_i64 t0, t1;
+        t0 = tcg_temp_new_i64();
+        t1 = tcg_temp_new_i64();
+        tcg_gen_shli_i64(t0, arg1, arg2);
+        tcg_gen_shri_i64(t1, arg1, 64 - arg2);
+        tcg_gen_or_i64(ret, t0, t1);
+        tcg_temp_free_i64(t0);
+        tcg_temp_free_i64(t1);
+    }
+}
+
+static inline void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
+{
+    TCGv_i32 t0, t1;
+
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new_i32();
+    tcg_gen_shr_i32(t0, arg1, arg2);
+    tcg_gen_subfi_i32(t1, 32, arg2);
+    tcg_gen_shl_i32(t1, arg1, t1);
+    tcg_gen_or_i32(ret, t0, t1);
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+}
+
+static inline void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
+{
+    TCGv_i64 t0, t1;
+
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
+    tcg_gen_shl_i64(t0, arg1, arg2);
+    tcg_gen_subfi_i64(t1, 64, arg2);
+    tcg_gen_shl_i64(t1, arg1, t1);
+    tcg_gen_or_i64(ret, t0, t1);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
+static inline void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
+{
+    /* some cases can be optimized here */
+    if (arg2 == 0) {
+        tcg_gen_mov_i32(ret, arg1);
+    } else {
+        tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
+    }
+}
+
+static inline void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+{
+    /* some cases can be optimized here */
+    if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else {
+        tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
+    }
 }
 
@@ -1434 +1689 @@
 #endif
 
+#if TARGET_LONG_BITS == 32
+#define TCGv TCGv_i32
+#define tcg_temp_new() tcg_temp_new_i32()
+#define tcg_global_reg_new tcg_global_reg_new_i32
+#define tcg_global_mem_new tcg_global_mem_new_i32
+#define tcg_temp_local_new() tcg_temp_local_new_i32()
+#define tcg_temp_free tcg_temp_free_i32
+#define tcg_gen_qemu_ldst_op tcg_gen_op3i_i32
+#define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i32
+#define TCGV_UNUSED(x) TCGV_UNUSED_I32(x)
+#define TCGV_EQUAL(a, b) (GET_TCGV_I32(a) == GET_TCGV_I32(b))
+#else
+#define TCGv TCGv_i64
+#define tcg_temp_new() tcg_temp_new_i64()
+#define tcg_global_reg_new tcg_global_reg_new_i64
+#define tcg_global_mem_new tcg_global_mem_new_i64
+#define tcg_temp_local_new() tcg_temp_local_new_i64()
+#define tcg_temp_free tcg_temp_free_i64
+#define tcg_gen_qemu_ldst_op tcg_gen_op3i_i64
+#define tcg_gen_qemu_ldst_op_i64 tcg_gen_qemu_ldst_op_i64_i64
+#define TCGV_UNUSED(x) TCGV_UNUSED_I64(x)
+#define TCGV_EQUAL(a, b) (GET_TCGV_I64(a) == GET_TCGV_I64(b))
+#endif
+
 /* debug info: write the PC of the corresponding QEMU CPU instruction */
 static inline void tcg_gen_debug_insn_start(uint64_t pc)
@@ -1460 +1739 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_ld8u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_ld8u, ret, addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld8u, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld8u, TCGV_LOW(ret), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 #endif
@@ -1470 +1750 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_ld8s, ret, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_ld8s, ret, addr, TCGV_HIGH(addr), mem_index);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld8s, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld8s, TCGV_LOW(ret), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
 #endif
 }
@@ -1480 +1761 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_ld16u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_ld16u, ret, addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld16u, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld16u, TCGV_LOW(ret), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 #endif
@@ -1490 +1772 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_ld16s, ret, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_ld16s, ret, addr, TCGV_HIGH(addr), mem_index);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld16s, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld16s, TCGV_LOW(ret), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
 #endif
 }
@@ -1500 +1783 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_ld32u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_ld32u, ret, addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld32u, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld32u, TCGV_LOW(ret), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 #endif
@@ -1510 +1794 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_ld32u, ret, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_ld32u, ret, addr, TCGV_HIGH(addr), mem_index);
-    tcg_gen_sari_i32(TCGV_HIGH(ret), ret, 31);
-#endif
-}
-
-static inline void tcg_gen_qemu_ld64(TCGv ret, TCGv addr, int mem_index)
+    tcg_gen_op3i_i32(INDEX_op_qemu_ld32u, ret, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld32u, TCGV_LOW(ret), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
+    tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
+#endif
+}
+
+static inline void tcg_gen_qemu_ld64(TCGv_i64 ret, TCGv addr, int mem_index)
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op4i(INDEX_op_qemu_ld64, ret, TCGV_HIGH(ret), addr, mem_index);
-#else
-    tcg_gen_op5i(INDEX_op_qemu_ld64, ret, TCGV_HIGH(ret),
-                 addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op4i_i32(INDEX_op_qemu_ld64, TCGV_LOW(ret), TCGV_HIGH(ret), addr, mem_index);
+#else
+    tcg_gen_op5i_i32(INDEX_op_qemu_ld64, TCGV_LOW(ret), TCGV_HIGH(ret),
+                     TCGV_LOW(addr), TCGV_HIGH(addr), mem_index);
 #endif
 }
@@ -1530 +1815 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_st8, arg, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_st8, arg, addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op3i_i32(INDEX_op_qemu_st8, arg, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_st8, TCGV_LOW(arg), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
 #endif
 }
@@ -1539 +1825 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_st16, arg, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_st16, arg, addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op3i_i32(INDEX_op_qemu_st16, arg, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_st16, TCGV_LOW(arg), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
 #endif
 }
@@ -1548 +1835 @@
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op3i(INDEX_op_qemu_st32, arg, addr, mem_index);
-#else
-    tcg_gen_op4i(INDEX_op_qemu_st32, arg, addr, TCGV_HIGH(addr), mem_index);
-#endif
-}
-
-static inline void tcg_gen_qemu_st64(TCGv arg, TCGv addr, int mem_index)
+    tcg_gen_op3i_i32(INDEX_op_qemu_st32, arg, addr, mem_index);
+#else
+    tcg_gen_op4i_i32(INDEX_op_qemu_st32, TCGV_LOW(arg), TCGV_LOW(addr),
+                     TCGV_HIGH(addr), mem_index);
+#endif
+}
+
+static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 {
 #if TARGET_LONG_BITS == 32
-    tcg_gen_op4i(INDEX_op_qemu_st64, arg, TCGV_HIGH(arg), addr, mem_index);
-#else
-    tcg_gen_op5i(INDEX_op_qemu_st64, arg, TCGV_HIGH(arg),
-                 addr, TCGV_HIGH(addr), mem_index);
+    tcg_gen_op4i_i32(INDEX_op_qemu_st64, TCGV_LOW(arg), TCGV_HIGH(arg), addr,
+                     mem_index);
+#else
+    tcg_gen_op5i_i32(INDEX_op_qemu_st64, TCGV_LOW(arg), TCGV_HIGH(arg),
+                     TCGV_LOW(addr), TCGV_HIGH(addr), mem_index);
 #endif
 }
@@ -1571 +1860 @@
 static inline void tcg_gen_qemu_ld8u(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_ld8u, ret, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld8u, ret, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_ld8s(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_ld8s, ret, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld8s, ret, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_ld16u(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_ld16u, ret, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld16u, ret, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_ld16s(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_ld16s, ret, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld16s, ret, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_ld32u(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_ld32u, ret, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32u, ret, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_ld32s(TCGv ret, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_ld32s, ret, addr, mem_index);
-}
-
-static inline void tcg_gen_qemu_ld64(TCGv ret, TCGv addr, int mem_index)
-{
-    tcg_gen_op3i(INDEX_op_qemu_ld64, ret, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_ld32s, ret, addr, mem_index);
+}
+
+static inline void tcg_gen_qemu_ld64(TCGv_i64 ret, TCGv addr, int mem_index)
+{
+    tcg_gen_qemu_ldst_op_i64(INDEX_op_qemu_ld64, ret, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_st8(TCGv arg, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_st8, arg, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_st8, arg, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_st16(TCGv arg, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_st16, arg, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_st16, arg, addr, mem_index);
 }
 
 static inline void tcg_gen_qemu_st32(TCGv arg, TCGv addr, int mem_index)
 {
-    tcg_gen_op3i(INDEX_op_qemu_st32, arg, addr, mem_index);
-}
-
-static inline void tcg_gen_qemu_st64(TCGv arg, TCGv addr, int mem_index)
-{
-    tcg_gen_op3i(INDEX_op_qemu_st64, arg, addr, mem_index);
+    tcg_gen_qemu_ldst_op(INDEX_op_qemu_st32, arg, addr, mem_index);
+}
+
+static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
+{
+    tcg_gen_qemu_ldst_op_i64(INDEX_op_qemu_st64, arg, addr, mem_index);
 }
 
@@ -1648 +1937 @@
 #define tcg_gen_sub_tl tcg_gen_sub_i64
 #define tcg_gen_neg_tl tcg_gen_neg_i64
+#define tcg_gen_subfi_tl tcg_gen_subfi_i64
 #define tcg_gen_subi_tl tcg_gen_subi_i64
 #define tcg_gen_and_tl tcg_gen_and_i64
@@ -1680 +1970 @@
 #define tcg_gen_ext32s_tl tcg_gen_ext32s_i64
 #define tcg_gen_concat_tl_i64 tcg_gen_concat32_i64
+#define tcg_gen_andc_tl tcg_gen_andc_i64
+#define tcg_gen_eqv_tl tcg_gen_eqv_i64
+#define tcg_gen_nand_tl tcg_gen_nand_i64
+#define tcg_gen_nor_tl tcg_gen_nor_i64
+#define tcg_gen_orc_tl tcg_gen_orc_i64
+#define tcg_gen_rotl_tl tcg_gen_rotl_i64
+#define tcg_gen_rotli_tl tcg_gen_rotli_i64
+#define tcg_gen_rotr_tl tcg_gen_rotr_i64
+#define tcg_gen_rotri_tl tcg_gen_rotri_i64
 #define tcg_const_tl tcg_const_i64
+#define tcg_const_local_tl tcg_const_local_i64
 #else
 #define TCG_TYPE_TL TCG_TYPE_I32
@@ -1700 +2000 @@
 #define tcg_gen_sub_tl tcg_gen_sub_i32
 #define tcg_gen_neg_tl tcg_gen_neg_i32
+#define tcg_gen_subfi_tl tcg_gen_subfi_i32
 #define tcg_gen_subi_tl tcg_gen_subi_i32
 #define tcg_gen_and_tl tcg_gen_and_i32
@@ -1732 +2033 @@
 #define tcg_gen_ext32s_tl tcg_gen_mov_i32
 #define tcg_gen_concat_tl_i64 tcg_gen_concat_i32_i64
+#define tcg_gen_andc_tl tcg_gen_andc_i32
+#define tcg_gen_eqv_tl tcg_gen_eqv_i32
+#define tcg_gen_nand_tl tcg_gen_nand_i32
+#define tcg_gen_nor_tl tcg_gen_nor_i32
+#define tcg_gen_orc_tl tcg_gen_orc_i32
+#define tcg_gen_rotl_tl tcg_gen_rotl_i32
+#define tcg_gen_rotli_tl tcg_gen_rotli_i32
+#define tcg_gen_rotr_tl tcg_gen_rotr_i32
+#define tcg_gen_rotri_tl tcg_gen_rotri_i32
 #define tcg_const_tl tcg_const_i32
+#define tcg_const_local_tl tcg_const_local_i32
 #endif
 
@@ -1744 +2055 @@
 #define tcg_gen_ext_i32_ptr tcg_gen_ext_i32_i64
 #endif /* TCG_TARGET_REG_BITS != 32 */
-

trunk/src/recompiler/tcg/tcg-opc.h

@@ -22 +22 @@
  * THE SOFTWARE.
  */
-
-#ifdef CONFIG_DYNGEN_OP
-#include "dyngen-opc.h"
-#endif
-
 #ifndef DEF2
 #define DEF2(name, oargs, iargs, cargs, flags) DEF(name, oargs + iargs + cargs, 0)

trunk/src/recompiler/tcg/tcg-runtime.c

@@ -22 +22 @@
  * THE SOFTWARE.
  */
-
 #include <stdarg.h>
 #include <stdlib.h>
@@ -31 +30 @@
 #include "config.h"
 #include "osdep.h"
+#include "cpu.h" // For TARGET_LONG_BITS
 #include "tcg.h"
 
@@ -67 +67 @@
     return arg1 % arg2;
 }
-

trunk/src/recompiler/tcg/tcg.c

@@ -43 +43 @@
 #include <malloc.h>
 #endif
+#ifdef _AIX
+#include <alloca.h>
+#endif
 
 #include "config.h"
 #include "qemu-common.h"
+#include "cache-utils.h"
 
 /* Note: the long term plan is to reduce the dependancies on the QEMU
@@ -73 +77 @@
                         tcg_target_long value, tcg_target_long addend);
 
-TCGOpDef tcg_op_defs[] = {
+static TCGOpDef tcg_op_defs[] = {
 #define DEF(s, n, copy_size) { #s, 0, 0, n, n, 0, copy_size },
 #ifndef VBOX
@@ -85 +89 @@
 };
 
-TCGRegSet tcg_target_available_regs[2];
-TCGRegSet tcg_target_call_clobber_regs;
+static TCGRegSet tcg_target_available_regs[2];
+static TCGRegSet tcg_target_call_clobber_regs;
 
 /* XXX: move that inside the context */
@@ -289 +293 @@
 }
 
-TCGv tcg_global_reg_new(TCGType type, int reg, const char *name)
+static inline int tcg_global_reg_new_internal(TCGType type, int reg,
+                                              const char *name)
 {
     TCGContext *s = &tcg_ctx;
@@ -311 +316 @@
     s->nb_globals++;
     tcg_regset_set_reg(s->reserved_regs, reg);
-    return MAKE_TCGV(idx);
-}
-
-#if TCG_TARGET_REG_BITS == 32
-/* temporary hack to avoid register shortage for tcg_qemu_st64() */
-TCGv tcg_global_reg2_new_hack(TCGType type, int reg1, int reg2,
-                              const char *name)
-{
-    TCGContext *s = &tcg_ctx;
-    TCGTemp *ts;
+    return idx;
+}
+
+TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name)
+{
     int idx;
-    char buf[64];
-
-    if (type != TCG_TYPE_I64)
-        tcg_abort();
-    idx = s->nb_globals;
-    tcg_temp_alloc(s, s->nb_globals + 2);
-    ts = &s->temps[s->nb_globals];
-    ts->base_type = type;
-    ts->type = TCG_TYPE_I32;
-    ts->fixed_reg = 1;
-    ts->reg = reg1;
-    pstrcpy(buf, sizeof(buf), name);
-    pstrcat(buf, sizeof(buf), "_0");
-    ts->name = strdup(buf);
-
-    ts++;
-    ts->base_type = type;
-    ts->type = TCG_TYPE_I32;
-    ts->fixed_reg = 1;
-    ts->reg = reg2;
-    pstrcpy(buf, sizeof(buf), name);
-    pstrcat(buf, sizeof(buf), "_1");
-    ts->name = strdup(buf);
-
-    s->nb_globals += 2;
-    return MAKE_TCGV(idx);
-}
-#endif
-
-TCGv tcg_global_mem_new(TCGType type, int reg, tcg_target_long offset,
-                        const char *name)
+
+    idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name);
+    return MAKE_TCGV_I32(idx);
+}
+
+TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name)
+{
+    int idx;
+
+    idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name);
+    return MAKE_TCGV_I64(idx);
+}
+
+static inline int tcg_global_mem_new_internal(TCGType type, int reg,
+                                              tcg_target_long offset,
+                                              const char *name)
 {
     TCGContext *s = &tcg_ctx;
@@ -408 +393 @@
         s->nb_globals++;
     }
-    return MAKE_TCGV(idx);
-}
-
-TCGv tcg_temp_new_internal(TCGType type, int temp_local)
+    return idx;
+}
+
+TCGv_i32 tcg_global_mem_new_i32(int reg, tcg_target_long offset,
+                                const char *name)
+{
+    int idx;
+
+    idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
+    return MAKE_TCGV_I32(idx);
+}
+
+TCGv_i64 tcg_global_mem_new_i64(int reg, tcg_target_long offset,
+                                const char *name)
+{
+    int idx;
+
+    idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
+    return MAKE_TCGV_I64(idx);
+}
+
+static inline int tcg_temp_new_internal(TCGType type, int temp_local)
 {
     TCGContext *s = &tcg_ctx;
@@ -459 +462 @@
         }
     }
-    return MAKE_TCGV(idx);
-}
-
-void tcg_temp_free(TCGv arg)
+    return idx;
+}
+
+TCGv_i32 tcg_temp_new_internal_i32(int temp_local)
+{
+    int idx;
+
+    idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local);
+    return MAKE_TCGV_I32(idx);
+}
+
+TCGv_i64 tcg_temp_new_internal_i64(int temp_local)
+{
+    int idx;
+
+    idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local);
+    return MAKE_TCGV_I64(idx);
+}
+
+static inline void tcg_temp_free_internal(int idx)
 {
     TCGContext *s = &tcg_ctx;
     TCGTemp *ts;
-    int idx = GET_TCGV(arg);
     int k;
 
@@ -480 +498 @@
 }
 
-
-TCGv tcg_const_i32(int32_t val)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I32);
+void tcg_temp_free_i32(TCGv_i32 arg)
+{
+    tcg_temp_free_internal(GET_TCGV_I32(arg));
+}
+
+void tcg_temp_free_i64(TCGv_i64 arg)
+{
+    tcg_temp_free_internal(GET_TCGV_I64(arg));
+}
+
+TCGv_i32 tcg_const_i32(int32_t val)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_new_i32();
     tcg_gen_movi_i32(t0, val);
     return t0;
 }
 
-TCGv tcg_const_i64(int64_t val)
-{
-    TCGv t0;
-    t0 = tcg_temp_new(TCG_TYPE_I64);
+TCGv_i64 tcg_const_i64(int64_t val)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_new_i64();
+    tcg_gen_movi_i64(t0, val);
+    return t0;
+}
+
+TCGv_i32 tcg_const_local_i32(int32_t val)
+{
+    TCGv_i32 t0;
+    t0 = tcg_temp_local_new_i32();
+    tcg_gen_movi_i32(t0, val);
+    return t0;
+}
+
+TCGv_i64 tcg_const_local_i64(int64_t val)
+{
+    TCGv_i64 t0;
+    t0 = tcg_temp_local_new_i64();
     tcg_gen_movi_i64(t0, val);
     return t0;
@@ -520 +563 @@
 }
 
-static inline TCGType tcg_get_base_type(TCGContext *s, TCGv arg)
-{
-    return s->temps[GET_TCGV(arg)].base_type;
-}
-
-static void tcg_gen_call_internal(TCGContext *s, TCGv func,
-                                  unsigned int flags,
-                                  unsigned int nb_rets, const TCGv *rets,
-                                  unsigned int nb_params, const TCGv *params)
-{
-#ifndef VBOX
-   int i;
-#else
-   unsigned int i;
-#endif
-    *gen_opc_ptr++ = INDEX_op_call;
-    *gen_opparam_ptr++ = (nb_rets << 16) | (nb_params + 1);
-    for(i = 0; i < nb_rets; i++) {
-        *gen_opparam_ptr++ = GET_TCGV(rets[i]);
-    }
-    for(i = 0; i < nb_params; i++) {
-        *gen_opparam_ptr++ = GET_TCGV(params[i]);
-    }
-    *gen_opparam_ptr++ = GET_TCGV(func);
-
-    *gen_opparam_ptr++ = flags;
-    /* total parameters, needed to go backward in the instruction stream */
-    *gen_opparam_ptr++ = 1 + nb_rets + nb_params + 3;
-}
-
-
-#if TCG_TARGET_REG_BITS < 64
 /* Note: we convert the 64 bit args to 32 bit and do some alignment
    and endian swap. Maybe it would be better to do the alignment
    and endian swap in tcg_reg_alloc_call(). */
-void tcg_gen_call(TCGContext *s, TCGv func, unsigned int flags,
-                  unsigned int nb_rets, const TCGv *rets,
-                  unsigned int nb_params, const TCGv *args1)
-{
-    TCGv ret, *args2, rets_2[2], arg;
-    int j, i, call_type;
-
-    if (nb_rets == 1) {
-        ret = rets[0];
-        if (tcg_get_base_type(s, ret) == TCG_TYPE_I64) {
+void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
+                   int sizemask, TCGArg ret, int nargs, TCGArg *args)
+{
+    int call_type;
+    int i;
+    int real_args;
+    int nb_rets;
+    TCGArg *nparam;
+    *gen_opc_ptr++ = INDEX_op_call;
+    nparam = gen_opparam_ptr++;
+    call_type = (flags & TCG_CALL_TYPE_MASK);
+    if (ret != TCG_CALL_DUMMY_ARG) {
+#if TCG_TARGET_REG_BITS < 64
+        if (sizemask & 1) {
+#ifdef TCG_TARGET_WORDS_BIGENDIAN
+            *gen_opparam_ptr++ = ret + 1;
+            *gen_opparam_ptr++ = ret;
+#else
+            *gen_opparam_ptr++ = ret;
+            *gen_opparam_ptr++ = ret + 1;
+#endif
             nb_rets = 2;
-#ifdef TCG_TARGET_WORDS_BIGENDIAN
-            rets_2[0] = TCGV_HIGH(ret);
-            rets_2[1] = ret;
-#else
-            rets_2[0] = ret;
-            rets_2[1] = TCGV_HIGH(ret);
-#endif
-            rets = rets_2;
-        }
-    }
-    args2 = alloca((nb_params * 3) * sizeof(TCGv));
-    j = 0;
-    call_type = (flags & TCG_CALL_TYPE_MASK);
-    for(i = 0; i < nb_params; i++) {
-        arg = args1[i];
-        if (tcg_get_base_type(s, arg) == TCG_TYPE_I64) {
+        } else
+#endif
+        {
+            *gen_opparam_ptr++ = ret;
+            nb_rets = 1;
+        }
+    } else {
+        nb_rets = 0;
+    }
+    real_args = 0;
+    for (i = 0; i < nargs; i++) {
+#if TCG_TARGET_REG_BITS < 64
+        if (sizemask & (2 << i)) {
 #ifdef TCG_TARGET_I386
             /* REGPARM case: if the third parameter is 64 bit, it is
                allocated on the stack */
-            if (j == 2 && call_type == TCG_CALL_TYPE_REGPARM) {
+            if (i == 2 && call_type == TCG_CALL_TYPE_REGPARM) {
                 call_type = TCG_CALL_TYPE_REGPARM_2;
                 flags = (flags & ~TCG_CALL_TYPE_MASK) | call_type;
             }
-            args2[j++] = arg;
-            args2[j++] = TCGV_HIGH(arg);
-#else
+#endif
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
             /* some targets want aligned 64 bit args */
-            if (j & 1) {
-                args2[j++] = TCG_CALL_DUMMY_ARG;
+            if (real_args & 1) {
+                *gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
+                real_args++;
             }
 #endif
 #ifdef TCG_TARGET_WORDS_BIGENDIAN
-            args2[j++] = TCGV_HIGH(arg);
-            args2[j++] = arg;
+            *gen_opparam_ptr++ = args[i] + 1;
+            *gen_opparam_ptr++ = args[i];
 #else
-            args2[j++] = arg;
-            args2[j++] = TCGV_HIGH(arg);
-#endif
-#endif
-        } else {
-            args2[j++] = arg;
-        }
-    }
-    tcg_gen_call_internal(s, func, flags,
-                          nb_rets, rets, j, args2);
-}
-#else
-void tcg_gen_call(TCGContext *s, TCGv func, unsigned int flags,
-                  unsigned int nb_rets, const TCGv *rets,
-                  unsigned int nb_params, const TCGv *args1)
-{
-    tcg_gen_call_internal(s, func, flags,
-                          nb_rets, rets, nb_params, args1);
-}
-#endif
+            *gen_opparam_ptr++ = args[i];
+            *gen_opparam_ptr++ = args[i] + 1;
+#endif
+            real_args += 2;
+        } else
+#endif
+        {
+            *gen_opparam_ptr++ = args[i];
+            real_args++;
+        }
+    }
+    *gen_opparam_ptr++ = GET_TCGV_PTR(func);
+
+    *gen_opparam_ptr++ = flags;
+
+    *nparam = (nb_rets << 16) | (real_args + 1);
+
+    /* total parameters, needed to go backward in the instruction stream */
+    *gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
+}
 
 #if TCG_TARGET_REG_BITS == 32
-void tcg_gen_shifti_i64(TCGv ret, TCGv arg1,
+void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
                         int c, int right, int arith)
 {
     if (c == 0) {
-        tcg_gen_mov_i32(ret, arg1);
+        tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
     } else if (c >= 32) {
@@ -634 +652 @@
         if (right) {
             if (arith) {
-                tcg_gen_sari_i32(ret, TCGV_HIGH(arg1), c);
+                tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
             } else {
-                tcg_gen_shri_i32(ret, TCGV_HIGH(arg1), c);
+                tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
             }
         } else {
-            tcg_gen_shli_i32(TCGV_HIGH(ret), arg1, c);
-            tcg_gen_movi_i32(ret, 0);
+            tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
+            tcg_gen_movi_i32(TCGV_LOW(ret), 0);
         }
     } else {
-        TCGv t0, t1;
-
-        t0 = tcg_temp_new(TCG_TYPE_I32);
-        t1 = tcg_temp_new(TCG_TYPE_I32);
+        TCGv_i32 t0, t1;
+
+        t0 = tcg_temp_new_i32();
+        t1 = tcg_temp_new_i32();
         if (right) {
             tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
@@ -655 +673 @@
             else
                 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
-            tcg_gen_shri_i32(ret, arg1, c);
-            tcg_gen_or_i32(ret, ret, t0);
+            tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
+            tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
             tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
         } else {
-            tcg_gen_shri_i32(t0, arg1, 32 - c);
+            tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
             /* Note: ret can be the same as arg1, so we use t1 */
-            tcg_gen_shli_i32(t1, arg1, c);
+            tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
-            tcg_gen_mov_i32(ret, t1);
-        }
-        tcg_temp_free(t0);
-        tcg_temp_free(t1);
+            tcg_gen_mov_i32(TCGV_LOW(ret), t1);
+        }
+        tcg_temp_free_i32(t0);
+        tcg_temp_free_i32(t1);
     }
 }
@@ -712 +730 @@
 }
 
-char *tcg_get_arg_str(TCGContext *s, char *buf, int buf_size, TCGv arg)
-{
-    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV(arg));
+char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg)
+{
+    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg));
+}
+
+char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg)
+{
+    return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg));
 }
 
@@ -857 +880 @@
             th = tcg_find_helper(s, val);
             if (th) {
-                fprintf(outfile, th->name);
+                fprintf(outfile, "%s", th->name);
             } else {
                 if (c == INDEX_op_movi_i32)
@@ -1174 +1197 @@
             /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
         default:
-            if (op > INDEX_op_end) {
-                args -= def->nb_args;
-                nb_iargs = def->nb_iargs;
-                nb_oargs = def->nb_oargs;
-
-                /* Test if the operation can be removed because all
-                   its outputs are dead. We assume that nb_oargs == 0
-                   implies side effects */
-                if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
-                    for(i = 0; i < nb_oargs; i++) {
-                        arg = args[i];
-                        if (!dead_temps[arg])
-                            goto do_not_remove;
+            args -= def->nb_args;
+            nb_iargs = def->nb_iargs;
+            nb_oargs = def->nb_oargs;
+
+            /* Test if the operation can be removed because all
+               its outputs are dead. We assume that nb_oargs == 0
+               implies side effects */
+            if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
+                for(i = 0; i < nb_oargs; i++) {
+                    arg = args[i];
+                    if (!dead_temps[arg])
+                        goto do_not_remove;
+                }
+                tcg_set_nop(s, gen_opc_buf + op_index, args, def->nb_args);
+#ifdef CONFIG_PROFILER
+                s->del_op_count++;
+#endif
+            } else {
+            do_not_remove:
+
+                /* output args are dead */
+                for(i = 0; i < nb_oargs; i++) {
+                    arg = args[i];
+                    dead_temps[arg] = 1;
+                }
+
+                /* if end of basic block, update */
+                if (def->flags & TCG_OPF_BB_END) {
+                    tcg_la_bb_end(s, dead_temps);
+                } else if (def->flags & TCG_OPF_CALL_CLOBBER) {
+                    /* globals are live */
+                    memset(dead_temps, 0, s->nb_globals);
+                }
+
+                /* input args are live */
+                dead_iargs = 0;
+                for(i = 0; i < nb_iargs; i++) {
+                    arg = args[i + nb_oargs];
+                    if (dead_temps[arg]) {
+                        dead_iargs |= (1 << i);
                     }
-                    tcg_set_nop(s, gen_opc_buf + op_index, args, def->nb_args);
-#ifdef CONFIG_PROFILER
-                    s->del_op_count++;
-#endif
-                } else {
-                do_not_remove:
-
-                    /* output args are dead */
-                    for(i = 0; i < nb_oargs; i++) {
-                        arg = args[i];
-                        dead_temps[arg] = 1;
-                    }
-
-                    /* if end of basic block, update */
-                    if (def->flags & TCG_OPF_BB_END) {
-                        tcg_la_bb_end(s, dead_temps);
-                    } else if (def->flags & TCG_OPF_CALL_CLOBBER) {
-                        /* globals are live */
-                        memset(dead_temps, 0, s->nb_globals);
-                    }
-
-                    /* input args are live */
-                    dead_iargs = 0;
-                    for(i = 0; i < nb_iargs; i++) {
-                        arg = args[i + nb_oargs];
-                        if (dead_temps[arg]) {
-                            dead_iargs |= (1 << i);
-                        }
-                        dead_temps[arg] = 0;
-                    }
-                    s->op_dead_iargs[op_index] = dead_iargs;
+                    dead_temps[arg] = 0;
                 }
-            } else {
-                /* legacy dyngen operations */
-                args -= def->nb_args;
-                /* mark end of basic block */
-                tcg_la_bb_end(s, dead_temps);
+                s->op_dead_iargs[op_index] = dead_iargs;
             }
             break;
@@ -1877 +1893 @@
 #ifdef CONFIG_PROFILER
 
-static int64_t dyngen_table_op_count[NB_OPS];
+static int64_t tcg_table_op_count[NB_OPS];
 
 void dump_op_count(void)
@@ -1883 +1899 @@
     int i;
     FILE *f;
-    f = fopen("/tmp/op1.log", "w");
-    for(i = 0; i < INDEX_op_end; i++) {
-        fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, dyngen_table_op_count[i]);
-    }
-    fclose(f);
-    f = fopen("/tmp/op2.log", "w");
+    f = fopen("/tmp/op.log", "w");
     for(i = INDEX_op_end; i < NB_OPS; i++) {
-        fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, dyngen_table_op_count[i]);
+        fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]);
     }
     fclose(f);
@@ -1906 +1917 @@
 
 #ifdef DEBUG_DISAS
-    if (unlikely(loglevel & CPU_LOG_TB_OP)) {
-        fprintf(logfile, "OP:\n");
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        qemu_log("OP:\n");
         tcg_dump_ops(s, logfile);
-        fprintf(logfile, "\n");
+        qemu_log("\n");
     }
 #endif
@@ -1922 +1933 @@
 
 #ifdef DEBUG_DISAS
-    if (unlikely(loglevel & CPU_LOG_TB_OP_OPT)) {
-        fprintf(logfile, "OP after la:\n");
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) {
+        qemu_log("OP after la:\n");
         tcg_dump_ops(s, logfile);
-        fprintf(logfile, "\n");
+        qemu_log("\n");
     }
 #endif
@@ -1940 +1951 @@
         opc = gen_opc_buf[op_index];
 #ifdef CONFIG_PROFILER
-        dyngen_table_op_count[opc]++;
+        tcg_table_op_count[opc]++;
 #endif
         def = &tcg_op_defs[opc];
@@ -1995 +2006 @@
         case INDEX_op_end:
             goto the_end;
-
-#ifdef CONFIG_DYNGEN_OP
-        case 0 ... INDEX_op_end - 1:
-            /* legacy dyngen ops */
-#ifdef CONFIG_PROFILER
-            s->old_op_count++;
-#endif
-            tcg_reg_alloc_bb_end(s, s->reserved_regs);
-            if (search_pc >= 0) {
-                s->code_ptr += def->copy_size;
-                args += def->nb_args;
-            } else {
-                args = dyngen_op(s, opc, args);
-            }
-            goto next;
-#endif
         default:
             /* Note: in order to speed up the code, it would be much
@@ -2033 +2028 @@
 }
 
-int dyngen_code(TCGContext *s, uint8_t *gen_code_buf)
+int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf)
 {
 #ifdef CONFIG_PROFILER
@@ -2061 +2056 @@
    not be changed, though writing the same values is ok.
    Return -1 if not found. */
-int dyngen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset)
+int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset)
 {
     return tcg_gen_code_common(s, gen_code_buf, offset);
@@ -2082 +2077 @@
     cpu_fprintf(f, "avg ops/TB          %0.1f max=%d\n",
                 s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max);
-    cpu_fprintf(f, "old ops/total ops   %0.1f%%\n",
-                s->op_count ? (double)s->old_op_count / s->op_count * 100.0 : 0);
     cpu_fprintf(f, "deleted ops/TB      %0.2f\n",
                 s->tb_count ?

trunk/src/recompiler/tcg/tcg.h

@@ -115 +115 @@
    We use plain int by default to avoid this runtime overhead.
    Users of tcg_gen_* don't need to know about any of this, and should
-   treat TCGv as an opaque type.  */
+   treat TCGv as an opaque type.
+   In additon we do typechecking for different types of variables.  TCGv_i32
+   and TCGv_i64 are 32/64-bit variables respectively.  TCGv and TCGv_ptr
+   are aliases for target_ulong and host pointer sized values respectively.
+ */
 
 //#define DEBUG_TCGV 1
@@ -123 +127 @@
 typedef struct
 {
-    int n;
-} TCGv;
-
-#define MAKE_TCGV(i) __extension__ \
-  ({ TCGv make_tcgv_tmp = {i}; make_tcgv_tmp;})
-#define GET_TCGV(t) ((t).n)
+    int i32;
+} TCGv_i32;
+
+typedef struct
+{
+    int i64;
+} TCGv_i64;
+
+#define MAKE_TCGV_I32(i) __extension__                  \
+    ({ TCGv_i32 make_tcgv_tmp = {i}; make_tcgv_tmp;})
+#define MAKE_TCGV_I64(i) __extension__                  \
+    ({ TCGv_i64 make_tcgv_tmp = {i}; make_tcgv_tmp;})
+#define GET_TCGV_I32(t) ((t).i32)
+#define GET_TCGV_I64(t) ((t).i64)
 #if TCG_TARGET_REG_BITS == 32
-#define TCGV_HIGH(t) MAKE_TCGV(GET_TCGV(t) + 1)
+#define TCGV_LOW(t) MAKE_TCGV_I32(GET_TCGV_I64(t))
+#define TCGV_HIGH(t) MAKE_TCGV_I32(GET_TCGV_I64(t) + 1)
 #endif
 
 #else /* !DEBUG_TCGV */
 
-typedef int TCGv;
-#define MAKE_TCGV(x) (x)
-#define GET_TCGV(t) (t)
+typedef int TCGv_i32;
+typedef int TCGv_i64;
+#define MAKE_TCGV_I32(x) (x)
+#define MAKE_TCGV_I64(x) (x)
+#define GET_TCGV_I32(t) (t)
+#define GET_TCGV_I64(t) (t)
 #if TCG_TARGET_REG_BITS == 32
+#define TCGV_LOW(t) (t)
 #define TCGV_HIGH(t) ((t) + 1)
 #endif
@@ -145 +162 @@
 
 /* Dummy definition to avoid compiler warnings.  */
-#define TCGV_UNUSED(x) x = MAKE_TCGV(-1)
+#define TCGV_UNUSED_I32(x) x = MAKE_TCGV_I32(-1)
+#define TCGV_UNUSED_I64(x) x = MAKE_TCGV_I64(-1)
 
 /* call flags */
@@ -159 +177 @@
 
 /* used to align parameters */
-#define TCG_CALL_DUMMY_TCGV     MAKE_TCGV(-1)
+#define TCG_CALL_DUMMY_TCGV     MAKE_TCGV_I32(-1)
 #define TCG_CALL_DUMMY_ARG      ((TCGArg)(-1))
 
@@ -255 +273 @@
     int64_t temp_count;
     int temp_count_max;
-    int64_t old_op_count;
     int64_t del_op_count;
     int64_t code_in_len;
@@ -297 +314 @@
 void tcg_func_start(TCGContext *s);
 
-int dyngen_code(TCGContext *s, uint8_t *gen_code_buf);
-int dyngen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset);
+int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf);
+int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset);
 
 void tcg_set_frame(TCGContext *s, int reg,
                    tcg_target_long start, tcg_target_long size);
-TCGv tcg_global_reg_new(TCGType type, int reg, const char *name);
-TCGv tcg_global_reg2_new_hack(TCGType type, int reg1, int reg2,
-                              const char *name);
-TCGv tcg_global_mem_new(TCGType type, int reg, tcg_target_long offset,
-                        const char *name);
-TCGv tcg_temp_new_internal(TCGType type, int temp_local);
-static inline TCGv tcg_temp_new(TCGType type)
-{
-    return tcg_temp_new_internal(type, 0);
+
+TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name);
+TCGv_i32 tcg_global_mem_new_i32(int reg, tcg_target_long offset,
+                                const char *name);
+TCGv_i32 tcg_temp_new_internal_i32(int temp_local);
+static inline TCGv_i32 tcg_temp_new_i32(void)
+{
+    return tcg_temp_new_internal_i32(0);
 }
-static inline TCGv tcg_temp_local_new(TCGType type)
-{
-    return tcg_temp_new_internal(type, 1);
+static inline TCGv_i32 tcg_temp_local_new_i32(void)
+{
+    return tcg_temp_new_internal_i32(1);
 }
-void tcg_temp_free(TCGv arg);
-char *tcg_get_arg_str(TCGContext *s, char *buf, int buf_size, TCGv arg);
+void tcg_temp_free_i32(TCGv_i32 arg);
+char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg);
+
+TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name);
+TCGv_i64 tcg_global_mem_new_i64(int reg, tcg_target_long offset,
+                                const char *name);
+TCGv_i64 tcg_temp_new_internal_i64(int temp_local);
+static inline TCGv_i64 tcg_temp_new_i64(void)
+{
+    return tcg_temp_new_internal_i64(0);
+}
+static inline TCGv_i64 tcg_temp_local_new_i64(void)
+{
+    return tcg_temp_new_internal_i64(1);
+}
+void tcg_temp_free_i64(TCGv_i64 arg);
+char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg);
+
 void tcg_dump_info(FILE *f,
                    int (*cpu_fprintf)(FILE *f, const char *fmt, ...));
@@ -357 +389 @@
     const char *args_ct_str[TCG_MAX_OP_ARGS];
 } TCGTargetOpDef;
-
-extern TCGOpDef tcg_op_defs[];
 
 void tcg_target_init(TCGContext *s);
@@ -384 +414 @@
 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs);
 
-void tcg_gen_call(TCGContext *s, TCGv func, unsigned int flags,
-                  unsigned int nb_rets, const TCGv *rets,
-                  unsigned int nb_params, const TCGv *args1);
-void tcg_gen_shifti_i64(TCGv ret, TCGv arg1,
-                        int c, int right, int arith);
-
-/* only used for debugging purposes */
-void tcg_register_helper(void *func, const char *name);
-#define TCG_HELPER(func) tcg_register_helper(func, #func)
-const char *tcg_helper_get_name(TCGContext *s, void *func);
-void tcg_dump_ops(TCGContext *s, FILE *outfile);
-
-void dump_ops(const uint16_t *opc_buf, const TCGArg *opparam_buf);
-TCGv tcg_const_i32(int32_t val);
-TCGv tcg_const_i64(int64_t val);
-
 #if TCG_TARGET_REG_BITS == 32
 #define tcg_const_ptr tcg_const_i32
 #define tcg_add_ptr tcg_add_i32
 #define tcg_sub_ptr tcg_sub_i32
+#define TCGv_ptr TCGv_i32
+#define GET_TCGV_PTR GET_TCGV_I32
+#define tcg_global_reg_new_ptr tcg_global_reg_new_i32
+#define tcg_global_mem_new_ptr tcg_global_mem_new_i32
+#define tcg_temp_new_ptr tcg_temp_new_i32
+#define tcg_temp_free_ptr tcg_temp_free_i32
 #else
 #define tcg_const_ptr tcg_const_i64
 #define tcg_add_ptr tcg_add_i64
 #define tcg_sub_ptr tcg_sub_i64
-#endif
+#define TCGv_ptr TCGv_i64
+#define GET_TCGV_PTR GET_TCGV_I64
+#define tcg_global_reg_new_ptr tcg_global_reg_new_i64
+#define tcg_global_mem_new_ptr tcg_global_mem_new_i64
+#define tcg_temp_new_ptr tcg_temp_new_i64
+#define tcg_temp_free_ptr tcg_temp_free_i64
+#endif
+
+void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
+                   int sizemask, TCGArg ret, int nargs, TCGArg *args);
+
+void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
+                        int c, int right, int arith);
+
+/* only used for debugging purposes */
+void tcg_register_helper(void *func, const char *name);
+const char *tcg_helper_get_name(TCGContext *s, void *func);
+void tcg_dump_ops(TCGContext *s, FILE *outfile);
+
+void dump_ops(const uint16_t *opc_buf, const TCGArg *opparam_buf);
+TCGv_i32 tcg_const_i32(int32_t val);
+TCGv_i64 tcg_const_i64(int64_t val);
+TCGv_i32 tcg_const_local_i32(int32_t val);
+TCGv_i64 tcg_const_local_i64(int64_t val);
 
 void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
@@ -414 +457 @@
 const TCGArg *tcg_gen_code_op(TCGContext *s, int opc, const TCGArg *args1,
                               unsigned int dead_iargs);
-
-const TCGArg *dyngen_op(TCGContext *s, int opc, const TCGArg *opparam_ptr);
 
 /* tcg-runtime.c */
@@ -431 +472 @@
 extern uint8_t* code_gen_prologue;
 #endif
-
-#if defined(__powerpc__) && !defined(__powerpc64__)
+#if defined(_ARCH_PPC) && !defined(_ARCH_PPC64)
 #define tcg_qemu_tb_exec(tb_ptr) \
     ((long REGPARM __attribute__ ((longcall)) (*)(void *))code_gen_prologue)(tb_ptr)
 #else
-
 # if defined(VBOX) && defined(GCC_WITH_BUGGY_REGPARM)
 #  define tcg_qemu_tb_exec(tb_ptr, ret)        \

trunk/src/recompiler/tcg/x86_64/tcg-target.c

@@ -700 +700 @@
         break;
     case 0:
+        /* movzbq */
+        tcg_out_modrm(s, 0xb6 | P_EXT | P_REXW, data_reg, TCG_REG_RAX);
+        break;
     case 1:
+        /* movzwq */
+        tcg_out_modrm(s, 0xb7 | P_EXT | P_REXW, data_reg, TCG_REG_RAX);
+        break;
     case 2:
     default:
@@ -733 +739 @@
     bswap = 0;
 #endif
-
     switch(opc) {
     case 0:

trunk/src/recompiler/tests/linux-test.c

@@ -1 +1 @@
 /*
  *  linux and CPU test
- * 
+ *
  *  Copyright (c) 2003 Fabrice Bellard
  *
@@ -16 +16 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *  Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston,
+ *  MA 02110-1301, USA.
  */
 
@@ -69 +70 @@
 {
     if (ret < 0) {
-        error1(filename, line, "%m (ret=%d, errno=%d)", 
+        error1(filename, line, "%m (ret=%d, errno=%d)",
                ret, errno);
     }
@@ -104 +105 @@
     if (getcwd(cur_dir, sizeof(cur_dir)) == NULL)
         error("getcwd");
-    
+
     chk_error(mkdir(TESTPATH, 0755));
-    
+
     chk_error(chdir(TESTPATH));
-    
+
     /* open/read/write/close/readv/writev/lseek */
 
@@ -135 +136 @@
     if (memcmp(buf, buf2, FILE_BUF_SIZE) != 0)
         error("memcmp");
-    
+
 #define FOFFSET 16
     ret = chk_error(lseek(fd, FOFFSET, SEEK_SET));
@@ -149 +150 @@
     if (memcmp(buf + FOFFSET, buf3, FILE_BUF_SIZE - FOFFSET) != 0)
         error("memcmp");
-    
+
     chk_error(close(fd));
 
@@ -182 +183 @@
     chk_error(fstat(fd, &st));
     chk_error(close(fd));
-    
+
     if (st.st_size != 50)
         error("stat size");
     if (!S_ISREG(st.st_mode))
         error("stat mode");
-    
+
     /* symlink/lstat */
     chk_error(symlink("file2", "file3"));
@@ -193 +194 @@
     if (!S_ISLNK(st.st_mode))
         error("stat mode");
-    
+
     /* getdents */
     dir = opendir(TESTPATH);
@@ -252 +253 @@
     if (ti >= 2)
         error("gettimeofday");
-    
+
     chk_error(getrusage(RUSAGE_SELF, &rusg1));
     for(i = 0;i < 10000; i++);
@@ -283 +284 @@
     int len;
     len = strlen(buf);
-    if (len < buf_size) 
+    if (len < buf_size)
         pstrcpy(buf + len, buf_size - len, s);
     return buf;
@@ -338 +339 @@
     if (val != SOCK_STREAM)
         error("getsockopt");
-    
+
     pid = chk_error(fork());
     if (pid == 0) {
@@ -430 +431 @@
 
     stack1 = malloc(STACK_SIZE);
-    pid1 = chk_error(clone(thread1_func, stack1 + STACK_SIZE, 
+    pid1 = chk_error(clone(thread1_func, stack1 + STACK_SIZE,
                            CLONE_VM | CLONE_FS | CLONE_FILES | SIGCHLD, "hello1"));
 
     stack2 = malloc(STACK_SIZE);
-    pid2 = chk_error(clone(thread2_func, stack2 + STACK_SIZE, 
+    pid2 = chk_error(clone(thread2_func, stack2 + STACK_SIZE,
                            CLONE_VM | CLONE_FS | CLONE_FILES | SIGCHLD, "hello2"));
 
@@ -476 +477 @@
     act.sa_flags = 0;
     chk_error(sigaction(SIGALRM, &act, NULL));
-    
+
     it.it_interval.tv_sec = 0;
     it.it_interval.tv_usec = 10 * 1000;
@@ -486 +487 @@
         oit.it_value.tv_usec != it.it_value.tv_usec)
         error("itimer");
-    
+
     while (alarm_count < 5) {
         usleep(10 * 1000);
@@ -509 +510 @@
         *(uint8_t *)0 = 0;
     }
-    
+
     act.sa_handler = SIG_DFL;
     sigemptyset(&act.sa_mask);

trunk/src/recompiler/tests/qruncom.c

@@ -199 +199 @@
 
     env = cpu_init("qemu32");
-
-    /* set user mode state (XXX: should be done automatically by
-       cpu_init ?) */
-    env->user_mode_only = 1;
 
     cpu_x86_set_cpl(env, 3);

trunk/src/recompiler/tests/sha1.c

@@ -239 +239 @@
     return 0;
 }
-
-

trunk/src/recompiler/tests/test-i386-shift.h

@@ -184 +184 @@
 #undef OP_NOBYTE
 #undef EXECSHIFT
-

trunk/src/recompiler/tests/test-i386.c

@@ -16 +16 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, write to the Free Software
- *  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *  Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston,
+ *  MA 02110-1301, USA.
  */
 

trunk/src/recompiler/tests/test-mmap.c

@@ -18 +18 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
- * 
+ *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ * Foundation, Inc., 51 Franklin Street - Fifth Floor, Boston,
+ *  MA 02110-1301, USA.
  */
 
@@ -65 +66 @@
 
                 len = pagesize + (pagesize * i & 7);
-                p1 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-                p2 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-                p3 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-                p4 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-                p5 = mmap(NULL, len, PROT_READ, 
+                p1 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                p2 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                p3 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                p4 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+                p5 = mmap(NULL, len, PROT_READ,
                           MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 
@@ -120 +121 @@
 
         len = 0x02000000;
-        p1 = mmap(NULL, len, PROT_READ, 
+        p1 = mmap(NULL, len, PROT_READ,
                   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 
@@ -128 +129 @@
         p = (uintptr_t) p1;
         fail_unless ((p & pagemask) == 0);
-        
+
         /* Make sure we can read from the entire area.  */
         memcpy (dummybuf, p1, pagesize);
@@ -147 +148 @@
         {
                 int nlen;
-                p1 = mmap(NULL, pagesize, PROT_READ, 
+                p1 = mmap(NULL, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
                 fail_unless (p1 != MAP_FAILED);
@@ -154 +155 @@
                 memcpy (dummybuf, p1, pagesize);
 
-                p2 = mmap(NULL, pagesize, PROT_READ, 
+                p2 = mmap(NULL, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
                 fail_unless (p2 != MAP_FAILED);
@@ -164 +165 @@
                 munmap (p1, pagesize);
                 nlen = pagesize * 8;
-                p3 = mmap(NULL, nlen, PROT_READ, 
+                p3 = mmap(NULL, nlen, PROT_READ,
                           MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
 
                 /* Check if the mmaped areas collide.  */
-                if (p3 < p2 
+                if (p3 < p2
                     && (p3 + nlen) > p2)
                         fail_unless (0);
@@ -193 +194 @@
 
         /* Find a suitable address to start with.  */
-        addr = mmap(NULL, pagesize * 40, PROT_READ | PROT_WRITE, 
+        addr = mmap(NULL, pagesize * 40, PROT_READ | PROT_WRITE,
                     MAP_PRIVATE | MAP_ANONYMOUS,
                     -1, 0);
@@ -202 +203 @@
         {
                 /* Create submaps within our unfixed map.  */
-                p1 = mmap(addr, pagesize, PROT_READ, 
+                p1 = mmap(addr, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
                           -1, 0);
-                /* Make sure we get pages aligned with the pagesize. 
+                /* Make sure we get pages aligned with the pagesize.
                    The target expects this.  */
                 p = (uintptr_t) p1;
                 fail_unless (p1 == addr);
-                fail_unless ((p & pagemask) == 0);              
+                fail_unless ((p & pagemask) == 0);
                 memcpy (dummybuf, p1, pagesize);
                 munmap (p1, pagesize);
@@ -233 +234 @@
         {
                 /* Create submaps within our unfixed map.  */
-                p1 = mmap(addr, pagesize, PROT_READ | PROT_WRITE, 
+                p1 = mmap(addr, pagesize, PROT_READ | PROT_WRITE,
                           MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED,
                           -1, 0);
-                /* Make sure we get pages aligned with the pagesize. 
+                /* Make sure we get pages aligned with the pagesize.
                    The target expects this.  */
                 p = (uintptr_t) p1;
                 fail_unless (p1 == addr);
-                fail_unless ((p & pagemask) == 0);              
+                fail_unless ((p & pagemask) == 0);
                 memcpy (p1, dummybuf, pagesize);
                 munmap (p1, pagesize);
@@ -260 +261 @@
 
                 len = pagesize;
-                p1 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE, 
+                p1 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE,
                           test_fd, 0);
-                p2 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE, 
+                p2 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE,
                           test_fd, pagesize);
-                p3 = mmap(NULL, len, PROT_READ, 
-                          MAP_PRIVATE, 
+                p3 = mmap(NULL, len, PROT_READ,
+                          MAP_PRIVATE,
                           test_fd, pagesize * 2);
 
@@ -309 +310 @@
         for (i = 0; i < 0x10; i++)
         {
-                p1 = mmap(NULL, pagesize, PROT_READ, 
-                          MAP_PRIVATE, 
-                          test_fd, 
+                p1 = mmap(NULL, pagesize, PROT_READ,
+                          MAP_PRIVATE,
+                          test_fd,
                           (test_fsize - sizeof *p1) & ~pagemask);
 
@@ -341 +342 @@
 
         /* Find a suitable address to start with.  */
-        addr = mmap(NULL, pagesize * 44, PROT_READ, 
+        addr = mmap(NULL, pagesize * 44, PROT_READ,
                     MAP_PRIVATE | MAP_ANONYMOUS,
                     -1, 0);
@@ -351 +352 @@
         {
                 /* Create submaps within our unfixed map.  */
-                p1 = mmap(addr, pagesize, PROT_READ, 
-                          MAP_PRIVATE | MAP_FIXED, 
-                          test_fd, 
+                p1 = mmap(addr, pagesize, PROT_READ,
+                          MAP_PRIVATE | MAP_FIXED,
+                          test_fd,
                           (test_fsize - sizeof *p1) & ~pagemask);
 
@@ -383 +384 @@
 
         /* Find a suitable address to start with.  */
-        addr = mmap(NULL, pagesize * 40 * 4, PROT_READ, 
+        addr = mmap(NULL, pagesize * 40 * 4, PROT_READ,
                     MAP_PRIVATE | MAP_ANONYMOUS,
                     -1, 0);
@@ -391 +392 @@
         for (i = 0; i < 40; i++)
         {
-                p1 = mmap(addr, pagesize, PROT_READ, 
+                p1 = mmap(addr, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_FIXED,
                           test_fd, 0);
-                p2 = mmap(addr + pagesize, pagesize, PROT_READ, 
+                p2 = mmap(addr + pagesize, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_FIXED,
                           test_fd, pagesize);
-                p3 = mmap(addr + pagesize * 2, pagesize, PROT_READ, 
+                p3 = mmap(addr + pagesize * 2, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_FIXED,
                           test_fd, pagesize * 2);
-                p4 = mmap(addr + pagesize * 3, pagesize, PROT_READ, 
+                p4 = mmap(addr + pagesize * 3, pagesize, PROT_READ,
                           MAP_PRIVATE | MAP_FIXED,
                           test_fd, pagesize * 3);
 
-                /* Make sure we get pages aligned with the pagesize. 
+                /* Make sure we get pages aligned with the pagesize.
                    The target expects this.  */
                 fail_unless (p1 == (void *)addr);
@@ -454 +455 @@
         for (i = 0; i < (pagesize * 4) / sizeof i; i++)
                 write (test_fd, &i, sizeof i);
-        /* Append a few extra writes to make the file end at non 
+        /* Append a few extra writes to make the file end at non
            page boundary.  */
         write (test_fd, &i, sizeof i); i++;

trunk/src/recompiler/tests/test_path.c

@@ -150 +150 @@
     return 0;
 }
-

trunk/src/recompiler/translate-all.c

@@ -16 +16 @@
  * You should have received a copy of the GNU Lesser General Public
  * License along with this library; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA  02110-1301 USA
  */
 
@@ -139 +139 @@
     s->code_time -= profile_getclock();
 #endif
-    gen_code_size = dyngen_code(s, gen_code_buf);
+    gen_code_size = tcg_gen_code(s, gen_code_buf);
     *gen_code_size_ptr = gen_code_size;
 #ifdef CONFIG_PROFILER
@@ -152 +152 @@
 
 #ifdef DEBUG_DISAS
-    if (loglevel & CPU_LOG_TB_OUT_ASM) {
-        fprintf(logfile, "OUT: [size=%d]\n", *gen_code_size_ptr);
-        disas(logfile, tb->tc_ptr, *gen_code_size_ptr);
-        fprintf(logfile, "\n");
-        fflush(logfile);
+    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
+        qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
+        log_disas(tb->tc_ptr, *gen_code_size_ptr);
+        qemu_log("\n");
+        qemu_log_flush();
     }
 #endif
@@ -202 +202 @@
     s->tb_next = tb->tb_next;
 #endif
-    j = dyngen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
+    j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
     if (j < 0)
         return -1;