Changeset 13337 in vbox for trunk/src

Timestamp:

Oct 16, 2008 11:59:21 AM (16 years ago)

Author:

vboxsync

Message:

more recompiler work

Location:

trunk/src/recompiler_new

Files:

• Makefile.kmk (modified) (1 diff)
• VBoxRecompiler.c (modified) (1 diff)
• dyngen-exec.h (modified) (3 diffs)
• exec-all.h (modified) (6 diffs)
• exec.c (modified) (10 diffs)
• osdep.h (modified) (2 diffs)
• softmmu_header.h (modified) (12 diffs)
• softmmu_template.h (modified) (15 diffs)
• target-i386/cpu.h (modified) (6 diffs)
• target-i386/helper.c (modified) (1 diff)
• target-i386/ops_sse.h (modified) (60 diffs)
• translate-all.c (modified) (7 diffs)
• translate-op.c (deleted)

trunk/src/recompiler_new/Makefile.kmk

@@ -92 +92 @@
         exec.c                  \
         translate-all.c         \
-        translate-op.c          \
-        tcg/tcg.c               \
-        tcg/tcg-dyngen.c        \
-        tcg/tcg-runtime.c       \
-        fpu/softfloat-native.c  \
+        host-utils.c            \
+        tcg/tcg.c               \
+        tcg/tcg-dyngen.c        \
+        tcg/tcg-runtime.c       \
+        fpu/softfloat-native.c  \
+        target-i386/op_helper.c \
         target-i386/helper.c    \
         target-i386/helper2.c   \

trunk/src/recompiler_new/VBoxRecompiler.c

@@ -26 +26 @@
 #define LOG_GROUP LOG_GROUP_REM
 #include "vl.h"
+#include "osdep.h"
 #include "exec-all.h"
 

trunk/src/recompiler_new/dyngen-exec.h

@@ -90 +90 @@
 #define UINT64_MAX              ((uint64_t)(18446744073709551615))
 
+#ifdef _BSD
+typedef struct __sFILE FILE;
+#else
 typedef struct FILE FILE;
+#endif
 extern int fprintf(FILE *, const char *, ...);
+extern int fputs(const char *, FILE *);
 extern int printf(const char *, ...);
 #undef NULL
@@ -234 +239 @@
 #define tostring(s)     #s
 
-#ifdef __alpha__
+#if defined(__alpha__) || defined(__s390__)
 /* the symbols are considered non exported so a br immediate is generated */
 #define __hidden __attribute__((visibility("hidden")))
@@ -308 +313 @@
 #endif
 
+
+/* The return address may point to the start of the next instruction.
+   Subtracting one gets us the call instruction itself.  */
+#if defined(__s390__)
+# define GETPC() ((void*)(((unsigned long)__builtin_return_address(0) & 0x7fffffffUL) - 1))
+#elif defined(__arm__)
+/* Thumb return addresses have the low bit set, so we need to subtract two.
+   This is still safe in ARM mode because instructions are 4 bytes.  */
+# define GETPC() ((void *)((unsigned long)__builtin_return_address(0) - 2))
+#else
+# define GETPC() ((void *)((unsigned long)__builtin_return_address(0) - 1))
+#endif
 #endif /* !defined(__DYNGEN_EXEC_H__) */

trunk/src/recompiler_new/exec-all.h

@@ -44 +44 @@
 #endif /* VBOX */
 
-#ifndef glue
-#define xglue(x, y) x ## y
-#define glue(x, y) xglue(x, y)
-#define stringify(s)    tostring(s)
-#define tostring(s)     #s
-#endif
-
-#if __GNUC__ < 3
-#define __builtin_expect(x, n) (x)
-#endif
-
-#ifdef __i386__
-#define REGPARM(n) __attribute((regparm(n)))
-#else
-#define REGPARM(n)
-#endif
-
 /* is_jmp field values */
 #define DISAS_NEXT    0 /* next instruction can be analyzed */
@@ -97 +80 @@
 typedef void (GenOpFunc3)(long, long, long);
 
-#ifndef VBOX
-extern FILE *logfile;
-extern int loglevel;
-#endif
+#include "qemu-log.h"
 
 void gen_intermediate_code(CPUState *env, struct TranslationBlock *tb);
@@ -171 +151 @@
     target_ulong pc;   /* simulated PC corresponding to this block (EIP + CS base) */
     target_ulong cs_base; /* CS base for this block */
-    unsigned int flags; /* flags defining in which context the code was generated */
+    uint64_t flags; /* flags defining in which context the code was generated */
     uint16_t size;      /* size of target code for this block (1 <=
                            size <= TARGET_PAGE_SIZE) */
@@ -199 +179 @@
 #  error "First 4GB aren't reachable. jmp dword [tb_next] wont work."
 # endif
-    uint32_t tb_next[2]; /* address of jump generated code */
+    unsigned long tb_next[2]; /* address of jump generated code */
 #endif
     /* list of TBs jumping to this one. This is a circular list using
@@ -331 +311 @@
 extern void *io_mem_opaque[IO_MEM_NB_ENTRIES];
 
-#ifdef __powerpc__
-static inline int testandset (int *p)
-{
-    int ret;
-    __asm__ __volatile__ (
-                          "0:    lwarx %0,0,%1\n"
-                          "      xor. %0,%3,%0\n"
-                          "      bne 1f\n"
-                          "      stwcx. %2,0,%1\n"
-                          "      bne- 0b\n"
-                          "1:    "
-                          : "=&r" (ret)
-                          : "r" (p), "r" (1), "r" (0)
-                          : "cr0", "memory");
-    return ret;
-}
-#endif
-
-#ifdef __i386__
-static inline int testandset (int *p)
-{
-    long int readval = 0;
-
-    __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
-                          : "+m" (*p), "+a" (readval)
-                          : "r" (1)
-                          : "cc");
-    return readval;
-}
-#endif
-
-#ifdef __x86_64__
-static inline int testandset (int *p)
-{
-    long int readval = 0;
-
-    __asm__ __volatile__ ("lock; cmpxchgl %2, %0"
-                          : "+m" (*p), "+a" (readval)
-                          : "r" (1)
-                          : "cc");
-    return readval;
-}
-#endif
-
-#ifdef __s390__
-static inline int testandset (int *p)
-{
-    int ret;
-
-    __asm__ __volatile__ ("0: cs    %0,%1,0(%2)\n"
-                          "   jl    0b"
-                          : "=&d" (ret)
-                          : "r" (1), "a" (p), "0" (*p)
-                          : "cc", "memory" );
-    return ret;
-}
-#endif
-
-#ifdef __alpha__
-static inline int testandset (int *p)
-{
-    int ret;
-    unsigned long one;
-
-    __asm__ __volatile__ ("0:   mov 1,%2\n"
-                          "     ldl_l %0,%1\n"
-                          "     stl_c %2,%1\n"
-                          "     beq %2,1f\n"
-                          ".subsection 2\n"
-                          "1:   br 0b\n"
-                          ".previous"
-                          : "=r" (ret), "=m" (*p), "=r" (one)
-                          : "m" (*p));
-    return ret;
-}
-#endif
-
-#ifdef __sparc__
-static inline int testandset (int *p)
-{
-        int ret;
-
-        __asm__ __volatile__("ldstub    [%1], %0"
-                             : "=r" (ret)
-                             : "r" (p)
-                             : "memory");
-
-        return (ret ? 1 : 0);
-}
-#endif
-
-#ifdef __arm__
-static inline int testandset (int *spinlock)
-{
-    register unsigned int ret;
-    __asm__ __volatile__("swp %0, %1, [%2]"
-                         : "=r"(ret)
-                         : "0"(1), "r"(spinlock));
-
-    return ret;
-}
-#endif
-
-#ifdef __mc68000
-static inline int testandset (int *p)
-{
-    char ret;
-    __asm__ __volatile__("tas %1; sne %0"
-                         : "=r" (ret)
-                         : "m" (p)
-                         : "cc","memory");
-    return ret;
-}
-#endif
-
-#ifdef __ia64
-#include <ia64intrin.h>
-
-static inline int testandset (int *p)
-{
-    return __sync_lock_test_and_set (p, 1);
-}
-#endif
-
-typedef int spinlock_t;
-
-#define SPIN_LOCK_UNLOCKED 0
-
-#if defined(CONFIG_USER_ONLY)
-static inline void spin_lock(spinlock_t *lock)
-{
-    while (testandset(lock));
-}
-
-static inline void spin_unlock(spinlock_t *lock)
-{
-    *lock = 0;
-}
-
-static inline int spin_trylock(spinlock_t *lock)
-{
-    return !testandset(lock);
-}
-#else
-static inline void spin_lock(spinlock_t *lock)
-{
-}
-
-static inline void spin_unlock(spinlock_t *lock)
-{
-}
-
-static inline int spin_trylock(spinlock_t *lock)
-{
-    return 1;
-}
-#endif
+#include "qemu-lock.h"
 
 extern spinlock_t tb_lock;
@@ -497 +321 @@
 void tlb_fill(target_ulong addr, int is_write, int is_user,
               void *retaddr);
+
+#include "softmmu_defs.h"
 
 #define ACCESS_TYPE (NB_MMU_MODES + 1)

trunk/src/recompiler_new/exec.c

@@ -212 +212 @@
 FILE *logfile;
 int loglevel;
+#ifndef VBOX
 static int log_append = 0;
+#endif
 
 /* statistics */
@@ -1829 +1831 @@
 {
     int i;
-    TranslationBlock *tb;
 
 #if defined(DEBUG_TLB)
@@ -1954 +1955 @@
 }
 
+#ifndef VBOX
 int cpu_physical_memory_set_dirty_tracking(int enable)
 {
@@ -1964 +1966 @@
     return in_migration;
 }
-
+#endif
 
 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
@@ -2036 +2038 @@
 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
                       target_phys_addr_t paddr, int prot,
-                      int is_user, int is_softmmu)
+                      int mmu_idx, int is_softmmu)
 {
     PhysPageDesc *p;
@@ -2042 +2044 @@
     unsigned int index;
     target_ulong address;
+    target_ulong code_address;
     target_phys_addr_t addend;
     int ret;
     CPUTLBEntry *te;
+    int i;
+    target_phys_addr_t iotlb;
 
     p = phys_page_find(paddr >> TARGET_PAGE_BITS);
@@ -2053 +2058 @@
     }
 #if defined(DEBUG_TLB)
-    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x u=%d smmu=%d pd=0x%08lx\n",
-           vaddr, (int)paddr, prot, is_user, is_softmmu, pd);
+    printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
+           vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
 #endif
 
     ret = 0;
-#if !defined(CONFIG_SOFTMMU)
-    if (is_softmmu)
-#endif
-    {
-        if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
-            /* IO memory case */
-            address = vaddr | pd;
-            addend = paddr;
+    address = vaddr;
+    if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
+        /* IO memory case (romd handled later) */
+        address |= TLB_MMIO;
+    }
+#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
+    addend = pd & TARGET_PAGE_MASK;
+#elif !defined(VBOX)
+    addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
+#else
+    addend = (unsigned long)remR3GCPhys2HCVirt(env, pd & TARGET_PAGE_MASK);
+#endif
+    if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
+        /* Normal RAM.  */
+        iotlb = pd & TARGET_PAGE_MASK;
+        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
+            iotlb |= IO_MEM_NOTDIRTY;
+        else
+            iotlb |= IO_MEM_ROM;
+    } else {
+        /* IO handlers are currently passed a phsical address.
+           It would be nice to pass an offset from the base address
+           of that region.  This would avoid having to special case RAM,
+           and avoid full address decoding in every device.
+           We can't use the high bits of pd for this because
+           IO_MEM_ROMD uses these as a ram address.  */
+        iotlb = (pd & ~TARGET_PAGE_MASK) + paddr;
+    }
+
+    code_address = address;
+    /* Make accesses to pages with watchpoints go via the
+       watchpoint trap routines.  */
+    for (i = 0; i < env->nb_watchpoints; i++) {
+        if (vaddr == (env->watchpoint[i].vaddr & TARGET_PAGE_MASK)) {
+            iotlb = io_mem_watch + paddr;
+            /* TODO: The memory case can be optimized by not trapping
+               reads of pages with a write breakpoint.  */
+            address |= TLB_MMIO;
+        }
+    }
+
+    index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+    env->iotlb[mmu_idx][index] = iotlb - vaddr;
+    te = &env->tlb_table[mmu_idx][index];
+    te->addend = addend - vaddr;
+    if (prot & PAGE_READ) {
+        te->addr_read = address;
+    } else {
+        te->addr_read = -1;
+    }
+
+    if (prot & PAGE_EXEC) {
+        te->addr_code = code_address;
+    } else {
+        te->addr_code = -1;
+    }
+    if (prot & PAGE_WRITE) {
+        if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
+            (pd & IO_MEM_ROMD)) {
+            /* Write access calls the I/O callback.  */
+            te->addr_write = address | TLB_MMIO;
+        } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
+                   !cpu_physical_memory_is_dirty(pd)) {
+            te->addr_write = address | TLB_NOTDIRTY;
         } else {
-            /* standard memory */
-            address = vaddr;
-#if defined(VBOX) && defined(REM_PHYS_ADDR_IN_TLB)
-            addend = pd & TARGET_PAGE_MASK;
-#elif !defined(VBOX)
-            addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
-#else
-            addend = (unsigned long)remR3GCPhys2HCVirt(env, pd & TARGET_PAGE_MASK);
-#endif
-        }
-
-        index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-        addend -= vaddr;
-        te = &env->tlb_table[is_user][index];
-        te->addend = addend;
-        if (prot & PAGE_READ) {
-            te->addr_read = address;
-        } else {
-            te->addr_read = -1;
-        }
-        if (prot & PAGE_EXEC) {
-            te->addr_code = address;
-        } else {
-            te->addr_code = -1;
-        }
-        if (prot & PAGE_WRITE) {
-            if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
-                (pd & IO_MEM_ROMD)) {
-                /* write access calls the I/O callback */
-                te->addr_write = vaddr |
-                    (pd & ~(TARGET_PAGE_MASK | IO_MEM_ROMD));
-            } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
-                       !cpu_physical_memory_is_dirty(pd)) {
-                te->addr_write = vaddr | IO_MEM_NOTDIRTY;
-            } else {
-                te->addr_write = address;
-            }
-        } else {
-            te->addr_write = -1;
-        }
+            te->addr_write = address;
+        }
+    } else {
+        te->addr_write = -1;
+    }
 #ifdef VBOX
-        /* inform raw mode about TLB page change */
-        remR3FlushPage(env, vaddr);
-#endif
-    }
-#if !defined(CONFIG_SOFTMMU)
-    else {
-        if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM) {
-            /* IO access: no mapping is done as it will be handled by the
-               soft MMU */
-            if (!(env->hflags & HF_SOFTMMU_MASK))
-                ret = 2;
-        } else {
-            void *map_addr;
-
-            if (vaddr >= MMAP_AREA_END) {
-                ret = 2;
-            } else {
-                if (prot & PROT_WRITE) {
-                    if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
-#if defined(TARGET_HAS_SMC) || 1
-                        first_tb ||
-#endif
-                        ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
-                         !cpu_physical_memory_is_dirty(pd))) {
-                        /* ROM: we do as if code was inside */
-                        /* if code is present, we only map as read only and save the
-                           original mapping */
-                        VirtPageDesc *vp;
-
-                        vp = virt_page_find_alloc(vaddr >> TARGET_PAGE_BITS, 1);
-                        vp->phys_addr = pd;
-                        vp->prot = prot;
-                        vp->valid_tag = virt_valid_tag;
-                        prot &= ~PAGE_WRITE;
-                    }
-                }
-                map_addr = mmap((void *)vaddr, TARGET_PAGE_SIZE, prot,
-                                MAP_SHARED | MAP_FIXED, phys_ram_fd, (pd & TARGET_PAGE_MASK));
-                if (map_addr == MAP_FAILED) {
-                    cpu_abort(env, "mmap failed when mapped physical address 0x%08x to virtual address 0x%08x\n",
-                              paddr, vaddr);
-                }
-            }
-        }
-    }
+    /* inform raw mode about TLB page change */
+    remR3FlushPage(env, vaddr);
 #endif
     return ret;
@@ -2484 +2467 @@
        flushed */
     if (dirty_flags == 0xff)
-        tlb_set_dirty(cpu_single_env, addr, cpu_single_env->mem_write_vaddr);
+        tlb_set_dirty(cpu_single_env, addr, cpu_single_env->mem_io_vaddr);
 }
 
@@ -2529 +2512 @@
        flushed */
     if (dirty_flags == 0xff)
-        tlb_set_dirty(cpu_single_env, addr, cpu_single_env->mem_write_vaddr);
+        tlb_set_dirty(cpu_single_env, addr, cpu_single_env->mem_io_vaddr);
 }
 
@@ -2574 +2557 @@
        flushed */
     if (dirty_flags == 0xff)
-        tlb_set_dirty(cpu_single_env, addr, cpu_single_env->mem_write_vaddr);
+        tlb_set_dirty(cpu_single_env, addr, cpu_single_env->mem_io_vaddr);
 }
 

trunk/src/recompiler_new/osdep.h

@@ -26 +26 @@
 #endif
 
-#define unlikely(cond)      RT_UNLIKELY(cond)
-
 #else /* !VBOX */
 
@@ -50 +48 @@
 #endif /* !VBOX */
 
+#ifdef __OpenBSD__
+#include <sys/types.h>
+#include <sys/signal.h>
 #endif
+
+#ifndef glue
+#define xglue(x, y) x ## y
+#define glue(x, y) xglue(x, y)
+#define stringify(s)    tostring(s)
+#define tostring(s)     #s
+#endif
+
+#ifndef likely
+#ifndef VBOX
+#if __GNUC__ < 3
+#define __builtin_expect(x, n) (x)
+#endif
+
+#define likely(x)   __builtin_expect(!!(x), 1)
+#define unlikely(x)   __builtin_expect(!!(x), 0)
+#else // VBOX
+#define likely(cond)        RT_LIKELY(cond)
+#define unlikely(cond)      RT_UNLIKELY(cond)
+#endif
+#endif // !likely
+
+#ifndef offsetof
+#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *) 0)->MEMBER)
+#endif
+#ifndef container_of
+#define container_of(ptr, type, member) ({                      \
+        const typeof(((type *) 0)->member) *__mptr = (ptr);     \
+        (type *) ((char *) __mptr - offsetof(type, member));})
+#endif
+
+#ifndef MIN
+#define MIN(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+#ifndef MAX
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+#endif
+
+#ifndef ARRAY_SIZE
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof((x)[0]))
+#endif
+
+#ifndef always_inline
+#if (__GNUC__ < 3) || defined(__APPLE__)
+#define always_inline inline
+#else
+#define always_inline __attribute__ (( always_inline )) __inline__
+#define inline always_inline
+#endif
+#else
+#define inline always_inline
+#endif
+
+#ifdef __i386__
+#define REGPARM __attribute((regparm(3)))
+#else
+#define REGPARM
+#endif
+
+#if defined (__GNUC__) && defined (__GNUC_MINOR_)
+# define QEMU_GNUC_PREREQ(maj, min) \
+         ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min))
+#else
+# define QEMU_GNUC_PREREQ(maj, min) 0
+#endif
+
+void *qemu_memalign(size_t alignment, size_t size);
+void *qemu_vmalloc(size_t size);
+void qemu_vfree(void *ptr);
+
+#ifndef VBOX
+int qemu_create_pidfile(const char *filename);
+
+#ifdef _WIN32
+int ffs(int i);
+
+typedef struct {
+    long tv_sec;
+    long tv_usec;
+} qemu_timeval;
+int qemu_gettimeofday(qemu_timeval *tp);
+#else
+typedef struct timeval qemu_timeval;
+#define qemu_gettimeofday(tp) gettimeofday(tp, NULL);
+#endif /* !_WIN32 */
+#endif // !VBOX
+
+#endif

trunk/src/recompiler_new/softmmu_header.h

@@ -49 +49 @@
 #endif
 
-#if ACCESS_TYPE == 0
-
-#define CPU_MEM_INDEX 0
+#if ACCESS_TYPE < (NB_MMU_MODES)
+
+#define CPU_MMU_INDEX ACCESS_TYPE
 #define MMUSUFFIX _mmu
 
-#elif ACCESS_TYPE == 1
-
-#define CPU_MEM_INDEX 1
+#elif ACCESS_TYPE == (NB_MMU_MODES)
+
+#define CPU_MMU_INDEX (cpu_mmu_index(env))
 #define MMUSUFFIX _mmu
 
-#elif ACCESS_TYPE == 2
-
-#ifdef TARGET_I386
-#define CPU_MEM_INDEX ((env->hflags & HF_CPL_MASK) == 3)
-#elif defined (TARGET_PPC)
-#define CPU_MEM_INDEX (msr_pr)
-#elif defined (TARGET_MIPS)
-#define CPU_MEM_INDEX ((env->hflags & MIPS_HFLAG_MODE) == MIPS_HFLAG_UM)
-#elif defined (TARGET_SPARC)
-#define CPU_MEM_INDEX ((env->psrs) == 0)
-#elif defined (TARGET_ARM)
-#define CPU_MEM_INDEX ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)
-#elif defined (TARGET_SH4)
-#define CPU_MEM_INDEX ((env->sr & SR_MD) == 0)
-#else
-#error unsupported CPU
-#endif
-#define MMUSUFFIX _mmu
-
-#elif ACCESS_TYPE == 3
-
-#ifdef TARGET_I386
-#define CPU_MEM_INDEX ((env->hflags & HF_CPL_MASK) == 3)
-#elif defined (TARGET_PPC)
-#define CPU_MEM_INDEX (msr_pr)
-#elif defined (TARGET_MIPS)
-#define CPU_MEM_INDEX ((env->hflags & MIPS_HFLAG_MODE) == MIPS_HFLAG_UM)
-#elif defined (TARGET_SPARC)
-#define CPU_MEM_INDEX ((env->psrs) == 0)
-#elif defined (TARGET_ARM)
-#define CPU_MEM_INDEX ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)
-#elif defined (TARGET_SH4)
-#define CPU_MEM_INDEX ((env->sr & SR_MD) == 0)
-#else
-#error unsupported CPU
-#endif
+#elif ACCESS_TYPE == (NB_MMU_MODES + 1)
+
+#define CPU_MMU_INDEX (cpu_mmu_index(env))
 #define MMUSUFFIX _cmmu
 
@@ -107 +74 @@
 #endif
 
-#if ACCESS_TYPE == 3
+#if ACCESS_TYPE == (NB_MMU_MODES + 1)
 #define ADDR_READ addr_code
 #else
@@ -113 +80 @@
 #endif
 
-DATA_TYPE REGPARM(1) glue(glue(__ld, SUFFIX), MMUSUFFIX)(target_ulong addr,
-                                                         int is_user);
-void REGPARM(2) glue(glue(__st, SUFFIX), MMUSUFFIX)(target_ulong addr, DATA_TYPE v, int is_user);
-
 #if (DATA_SIZE <= 4) && (TARGET_LONG_BITS == 32) && defined(__i386__) && \
-    (ACCESS_TYPE <= 1) && defined(ASM_SOFTMMU) && (!defined(VBOX) || !defined(REM_PHYS_ADDR_IN_TLB))
-
-#define CPU_TLB_ENTRY_BITS 4
-
-static inline RES_TYPE glue(glue(ld, USUFFIX), MEMSUFFIX)(target_ulong ptr)
-{
-    int res;
-
-    asm volatile ("movl %1, %%edx\n"
-                  "movl %1, %%eax\n"
-                  "shrl %3, %%edx\n"
-                  "andl %4, %%eax\n"
-                  "andl %2, %%edx\n"
-                  "leal %5(%%edx, %%ebp), %%edx\n"
-                  "cmpl (%%edx), %%eax\n"
-                  "movl %1, %%eax\n"
-                  "je 1f\n"
-                  "pushl %6\n"
-                  "call %7\n"
-                  "popl %%edx\n"
-                  "movl %%eax, %0\n"
-                  "jmp 2f\n"
-                  "1:\n"
-                  "addl 12(%%edx), %%eax\n"
-#if DATA_SIZE == 1
-                  "movzbl (%%eax), %0\n"
-#elif DATA_SIZE == 2
-                  "movzwl (%%eax), %0\n"
-#elif DATA_SIZE == 4
-                  "movl (%%eax), %0\n"
-#else
-#error unsupported size
-#endif
-                  "2:\n"
-                  : "=r" (res)
-                  : "r" (ptr), 
-                  "i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS), 
-                  "i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS), 
-                  "i" (TARGET_PAGE_MASK | (DATA_SIZE - 1)),
-                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_read)),
-                  "i" (CPU_MEM_INDEX),
-                  "m" (*(uint8_t *)&glue(glue(__ld, SUFFIX), MMUSUFFIX))
-                  : "%eax", "%ecx", "%edx", "memory", "cc");
-    return res;
-}
-
-#if DATA_SIZE <= 2
-static inline int glue(glue(lds, SUFFIX), MEMSUFFIX)(target_ulong ptr)
-{
-    int res;
-
-    asm volatile ("movl %1, %%edx\n"
-                  "movl %1, %%eax\n"
-                  "shrl %3, %%edx\n"
-                  "andl %4, %%eax\n"
-                  "andl %2, %%edx\n"
-                  "leal %5(%%edx, %%ebp), %%edx\n"
-                  "cmpl (%%edx), %%eax\n"
-                  "movl %1, %%eax\n"
-                  "je 1f\n"
-                  "pushl %6\n"
-                  "call %7\n"
-                  "popl %%edx\n"
-#if DATA_SIZE == 1
-                  "movsbl %%al, %0\n"
-#elif DATA_SIZE == 2
-                  "movswl %%ax, %0\n"
-#else
-#error unsupported size
-#endif
-                  "jmp 2f\n"
-                  "1:\n"
-                  "addl 12(%%edx), %%eax\n"
-#if DATA_SIZE == 1
-                  "movsbl (%%eax), %0\n"
-#elif DATA_SIZE == 2
-                  "movswl (%%eax), %0\n"
-#else
-#error unsupported size
-#endif
-                  "2:\n"
-                  : "=r" (res)
-                  : "r" (ptr), 
-                  "i" ((CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS), 
-                  "i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS), 
-                  "i" (TARGET_PAGE_MASK | (DATA_SIZE - 1)),
-                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_read)),
-                  "i" (CPU_MEM_INDEX),
-                  "m" (*(uint8_t *)&glue(glue(__ld, SUFFIX), MMUSUFFIX))
-                  : "%eax", "%ecx", "%edx", "memory", "cc");
-    return res;
-}
-#endif
+    (ACCESS_TYPE < NB_MMU_MODES) && defined(ASM_SOFTMMU)
 
 #ifdef VBOX
@@ -224 +95 @@
     addr = ptr;
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    is_user = CPU_MEM_INDEX;
+    is_user = CPU_MMU_INDEX;
     if (__builtin_expect(env->tlb_table[is_user][index].addr_write != 
                          (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
@@ -280 +151 @@
                   "i" (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS), 
                   "i" (TARGET_PAGE_MASK | (DATA_SIZE - 1)),
-                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MEM_INDEX][0].addr_write)),
-                  "i" (CPU_MEM_INDEX),
+                  "m" (*(uint32_t *)offsetof(CPUState, tlb_table[CPU_MMU_INDEX][0].addr_write)),
+                  "i" (CPU_MMU_INDEX),
                   "m" (*(uint8_t *)&glue(glue(__st, SUFFIX), MMUSUFFIX))
                   : "%eax", "%ecx", "%edx", "memory", "cc");
@@ -301 +172 @@
     addr = ptr;
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    is_user = CPU_MEM_INDEX;
+    is_user = CPU_MMU_INDEX;
     if (__builtin_expect(env->tlb_table[is_user][index].ADDR_READ != 
                          (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
@@ -322 +193 @@
     addr = ptr;
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    is_user = CPU_MEM_INDEX;
+    is_user = CPU_MMU_INDEX;
     if (__builtin_expect(env->tlb_table[is_user][index].ADDR_READ != 
                          (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
@@ -334 +205 @@
 #endif
 
-#if ACCESS_TYPE != 3
+#if ACCESS_TYPE != (NB_MMU_MODES + 1)
 
 /* generic store macro */
@@ -347 +218 @@
     addr = ptr;
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    is_user = CPU_MEM_INDEX;
+    is_user = CPU_MMU_INDEX;
     if (__builtin_expect(env->tlb_table[is_user][index].addr_write != 
                          (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))), 0)) {
@@ -357 +228 @@
 }
 
-#endif /* ACCESS_TYPE != 3 */
+#endif /* ACCESS_TYPE != (NB_MMU_MODES + 1) */
 
 #endif /* !asm */
 
-#if ACCESS_TYPE != 3
+#if ACCESS_TYPE != (NB_MMU_MODES + 1)
 
 #if DATA_SIZE == 8
@@ -407 +278 @@
 #endif /* DATA_SIZE == 4 */
 
-#endif /* ACCESS_TYPE != 3 */
+#endif /* ACCESS_TYPE != (NB_MMU_MODES + 1) */
 
 #undef RES_TYPE
@@ -415 +286 @@
 #undef USUFFIX
 #undef DATA_SIZE
-#undef CPU_MEM_INDEX
+#undef CPU_MMU_INDEX
 #undef MMUSUFFIX
 #undef ADDR_READ

trunk/src/recompiler_new/softmmu_template.h

@@ -61 +61 @@
                                                         void *retaddr);
 static inline DATA_TYPE glue(io_read, SUFFIX)(target_phys_addr_t physaddr, 
-                                              target_ulong tlb_addr)
+                                              target_ulong addr,
+                                              void *retaddr)
 {
     DATA_TYPE res;
     int index;
-
-    index = (tlb_addr >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+    index = (physaddr >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
+    env->mem_io_pc = (unsigned long)retaddr;
+    if (index > (IO_MEM_NOTDIRTY >> IO_MEM_SHIFT)
+            && !can_do_io(env)) {
+        cpu_io_recompile(env, retaddr);
+    }
+
 #if SHIFT <= 2
     res = io_mem_read[index][SHIFT](io_mem_opaque[index], physaddr);
@@ -85 +92 @@
 
 /* handle all cases except unaligned access which span two pages */
-DATA_TYPE REGPARM(1) glue(glue(__ld, SUFFIX), MMUSUFFIX)(target_ulong addr,
-                                                         int is_user)
+DATA_TYPE REGPARM glue(glue(__ld, SUFFIX), MMUSUFFIX)(target_ulong addr,
+                                                      int mmu_idx)
 {
     DATA_TYPE res;
     int index;
     target_ulong tlb_addr;
-    target_phys_addr_t physaddr;
+    target_phys_addr_t addend;
     void *retaddr;
-    
+
     /* test if there is match for unaligned or IO access */
     /* XXX: could done more in memory macro in a non portable way */
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
  redo:
-    tlb_addr = env->tlb_table[is_user][index].ADDR_READ;
+    tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
     if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        physaddr = addr + env->tlb_table[is_user][index].addend;
         if (tlb_addr & ~TARGET_PAGE_MASK) {
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            res = glue(io_read, SUFFIX)(physaddr, tlb_addr);
+            retaddr = GETPC();
+            addend = env->iotlb[mmu_idx][index];
+            res = glue(io_read, SUFFIX)(addend, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
             /* slow unaligned access (it spans two pages or IO) */
@@ -111 +119 @@
             retaddr = GETPC();
 #ifdef ALIGNED_ONLY
-            do_unaligned_access(addr, READ_ACCESS_TYPE, is_user, retaddr);
-#endif
-            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(addr, 
-                                                         is_user, retaddr);
+            do_unaligned_access(addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+#endif
+            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(addr,
+                                                         mmu_idx, retaddr);
         } else {
             /* unaligned/aligned access in the same page */
@@ -120 +128 @@
             if ((addr & (DATA_SIZE - 1)) != 0) {
                 retaddr = GETPC();
-                do_unaligned_access(addr, READ_ACCESS_TYPE, is_user, retaddr);
+                do_unaligned_access(addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
             }
 #endif
-            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(long)physaddr);
+            addend = env->tlb_table[mmu_idx][index].addend;
+            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(long)(addr+addend));
         }
     } else {
@@ -130 +139 @@
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0)
-            do_unaligned_access(addr, READ_ACCESS_TYPE, is_user, retaddr);
-#endif
-        tlb_fill(addr, READ_ACCESS_TYPE, is_user, retaddr);
+            do_unaligned_access(addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+#endif
+        tlb_fill(addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
         goto redo;
     }
@@ -139 +148 @@
 
 /* handle all unaligned cases */
-static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(target_ulong addr, 
-                                                        int is_user,
+static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(target_ulong addr,
+                                                        int mmu_idx,
                                                         void *retaddr)
 {
     DATA_TYPE res, res1, res2;
     int index, shift;
-    target_phys_addr_t physaddr;
+    target_phys_addr_t addend;
     target_ulong tlb_addr, addr1, addr2;
 
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
  redo:
-    tlb_addr = env->tlb_table[is_user][index].ADDR_READ;
+    tlb_addr = env->tlb_table[mmu_idx][index].ADDR_READ;
     if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        physaddr = addr + env->tlb_table[is_user][index].addend;
         if (tlb_addr & ~TARGET_PAGE_MASK) {
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            res = glue(io_read, SUFFIX)(physaddr, tlb_addr);
+            retaddr = GETPC();
+            addend = env->iotlb[mmu_idx][index];
+            res = glue(io_read, SUFFIX)(addend, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
@@ -163 +173 @@
             addr1 = addr & ~(DATA_SIZE - 1);
             addr2 = addr1 + DATA_SIZE;
-            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(addr1, 
-                                                          is_user, retaddr);
-            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(addr2, 
-                                                          is_user, retaddr);
+            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(addr1,
+                                                          mmu_idx, retaddr);
+            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(addr2,
+                                                          mmu_idx, retaddr);
             shift = (addr & (DATA_SIZE - 1)) * 8;
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -176 +186 @@
         } else {
             /* unaligned/aligned access in the same page */
-            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(long)physaddr);
+            addend = env->tlb_table[mmu_idx][index].addend;
+            res = glue(glue(ld, USUFFIX), _raw)((uint8_t *)(long)(addr+addend));
         }
     } else {
         /* the page is not in the TLB : fill it */
-        tlb_fill(addr, READ_ACCESS_TYPE, is_user, retaddr);
+        tlb_fill(addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
         goto redo;
     }
@@ -195 +206 @@
 static inline void glue(io_write, SUFFIX)(target_phys_addr_t physaddr, 
                                           DATA_TYPE val,
-                                          target_ulong tlb_addr,
+                                          target_ulong addr,
                                           void *retaddr)
 {
     int index;
-
-    index = (tlb_addr >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
-    env->mem_write_vaddr = tlb_addr;
-    env->mem_write_pc = (unsigned long)retaddr;
+    index = (physaddr >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
+    physaddr = (physaddr & TARGET_PAGE_MASK) + addr;
+    if (index > (IO_MEM_NOTDIRTY >> IO_MEM_SHIFT)
+            && !can_do_io(env)) {
+        cpu_io_recompile(env, retaddr);
+    }
+
+    env->mem_io_vaddr = addr;
+    env->mem_io_pc = (unsigned long)retaddr;
 #if SHIFT <= 2
     io_mem_write[index][SHIFT](io_mem_opaque[index], physaddr, val);
@@ -219 +235 @@
 }
 
-void REGPARM(2) glue(glue(__st, SUFFIX), MMUSUFFIX)(target_ulong addr, 
-                                                    DATA_TYPE val,
-                                                    int is_user)
-{
-    target_phys_addr_t physaddr;
+void REGPARM glue(glue(__st, SUFFIX), MMUSUFFIX)(target_ulong addr,
+                                                 DATA_TYPE val,
+                                                 int mmu_idx)
+{
+    target_phys_addr_t addend;
     target_ulong tlb_addr;
     void *retaddr;
     int index;
-    
+
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
  redo:
-    tlb_addr = env->tlb_table[is_user][index].addr_write;
+    tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
     if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        physaddr = addr + env->tlb_table[is_user][index].addend;
         if (tlb_addr & ~TARGET_PAGE_MASK) {
             /* IO access */
@@ -238 +253 @@
                 goto do_unaligned_access;
             retaddr = GETPC();
-            glue(io_write, SUFFIX)(physaddr, val, tlb_addr, retaddr);
+            addend = env->iotlb[mmu_idx][index];
+            glue(io_write, SUFFIX)(addend, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
             retaddr = GETPC();
 #ifdef ALIGNED_ONLY
-            do_unaligned_access(addr, 1, is_user, retaddr);
-#endif
-            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(addr, val, 
-                                                   is_user, retaddr);
+            do_unaligned_access(addr, 1, mmu_idx, retaddr);
+#endif
+            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(addr, val,
+                                                   mmu_idx, retaddr);
         } else {
             /* aligned/unaligned access in the same page */
@@ -252 +268 @@
             if ((addr & (DATA_SIZE - 1)) != 0) {
                 retaddr = GETPC();
-                do_unaligned_access(addr, 1, is_user, retaddr);
+                do_unaligned_access(addr, 1, mmu_idx, retaddr);
             }
 #endif
-            glue(glue(st, SUFFIX), _raw)((uint8_t *)(long)physaddr, val);
+            addend = env->tlb_table[mmu_idx][index].addend;
+            glue(glue(st, SUFFIX), _raw)((uint8_t *)(long)(addr+addend), val);
         }
     } else {
@@ -262 +279 @@
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0)
-            do_unaligned_access(addr, 1, is_user, retaddr);
-#endif
-        tlb_fill(addr, 1, is_user, retaddr);
+            do_unaligned_access(addr, 1, mmu_idx, retaddr);
+#endif
+        tlb_fill(addr, 1, mmu_idx, retaddr);
         goto redo;
     }
@@ -270 +287 @@
 
 /* handles all unaligned cases */
-static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(target_ulong addr, 
+/* handles all unaligned cases */
+static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(target_ulong addr,
                                                    DATA_TYPE val,
-                                                   int is_user,
+                                                   int mmu_idx,
                                                    void *retaddr)
 {
-    target_phys_addr_t physaddr;
+    target_phys_addr_t addend;
     target_ulong tlb_addr;
     int index, i;
@@ -281 +299 @@
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
  redo:
-    tlb_addr = env->tlb_table[is_user][index].addr_write;
+    tlb_addr = env->tlb_table[mmu_idx][index].addr_write;
     if ((addr & TARGET_PAGE_MASK) == (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        physaddr = addr + env->tlb_table[is_user][index].addend;
         if (tlb_addr & ~TARGET_PAGE_MASK) {
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            glue(io_write, SUFFIX)(physaddr, val, tlb_addr, retaddr);
+            addend = env->iotlb[mmu_idx][index];
+            glue(io_write, SUFFIX)(addend, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
             /* XXX: not efficient, but simple */
-            for(i = 0;i < DATA_SIZE; i++) {
+            /* Note: relies on the fact that tlb_fill() does not remove the
+             * previous page from the TLB cache.  */
+            for(i = DATA_SIZE - 1; i >= 0; i--) {
 #ifdef TARGET_WORDS_BIGENDIAN
-                glue(slow_stb, MMUSUFFIX)(addr + i, val >> (((DATA_SIZE - 1) * 8) - (i * 8)), 
-                                          is_user, retaddr);
-#else
-                glue(slow_stb, MMUSUFFIX)(addr + i, val >> (i * 8), 
-                                          is_user, retaddr);
+                glue(slow_stb, MMUSUFFIX)(addr + i, val >> (((DATA_SIZE - 1) * 8) - (i * 8)),
+                                          mmu_idx, retaddr);
+#else
+                glue(slow_stb, MMUSUFFIX)(addr + i, val >> (i * 8),
+                                          mmu_idx, retaddr);
 #endif
             }
         } else {
             /* aligned/unaligned access in the same page */
-            glue(glue(st, SUFFIX), _raw)((uint8_t *)(long)physaddr, val);
+            addend = env->tlb_table[mmu_idx][index].addend;
+            glue(glue(st, SUFFIX), _raw)((uint8_t *)(long)(addr+addend), val);
         }
     } else {
         /* the page is not in the TLB : fill it */
-        tlb_fill(addr, 1, is_user, retaddr);
+        tlb_fill(addr, 1, mmu_idx, retaddr);
         goto redo;
     }

trunk/src/recompiler_new/target-i386/cpu.h

@@ -267 +267 @@
 #define MSR_MCG_CTL                     0x17b
 
+#define MSR_IA32_PERF_STATUS            0x198
+
 #define MSR_PAT                         0x277
 
@@ -494 +496 @@
     uint16_t _w[2];
     uint32_t _l[1];
+    float32 _s[2];
     uint64_t q;
 } MMXReg;
@@ -508 +511 @@
 #define MMX_W(n) _w[3 - (n)]
 #define MMX_L(n) _l[1 - (n)]
+#define MMX_S(n) _s[1 - (n)]
 #else
 #define XMM_B(n) _b[n]
@@ -519 +523 @@
 #define MMX_W(n) _w[n]
 #define MMX_L(n) _l[n]
+#define MMX_S(n) _s[n]
 #endif
 #define MMX_Q(n) q
@@ -555 +560 @@
 
     target_ulong cr[5]; /* NOTE: cr1 is unused */
-    uint32_t a20_mask;
+    uint64_t a20_mask;
 
     /* FPU state */
@@ -657 +662 @@
     uint32_t cpuid_ext2_features;
     uint32_t cpuid_ext3_features;
+    uint32_t cpuid_apic_id;
 
 #ifndef VBOX

trunk/src/recompiler_new/target-i386/helper.c

@@ -30 +30 @@
 # include <VBox/err.h>
 #endif
-#include "exec.h"
+
+#ifndef VBOX
+#include <stdarg.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <assert.h>
+#endif
+
+#include "cpu.h"
+#include "exec-all.h"
+#include "svm.h"
+#include "qemu-common.h"
 
 //#define DEBUG_PCALL

trunk/src/recompiler_new/target-i386/ops_sse.h

@@ -1 +1 @@
 /*
- *  MMX/SSE/SSE2/PNI support
- * 
+ *  MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support
+ *
  *  Copyright (c) 2005 Fabrice Bellard
+ *  Copyright (c) 2008 Intel Corporation  <[email protected]>
  *
  * This library is free software; you can redistribute it and/or
@@ -18 +19 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
-
 /*
  * Sun LGPL Disclaimer: For the avoidance of doubt, except that if any license choice
@@ -27 +27 @@
  * of the LGPL is applied is otherwise unspecified.
  */
+
 #if SHIFT == 0
 #define Reg MMXReg
@@ -45 +46 @@
 #endif
 
-void OPPROTO glue(op_psrlw, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psrlw, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 15) {
@@ -74 +71 @@
 }
 
-void OPPROTO glue(op_psraw, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psraw, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 15) {
@@ -99 +92 @@
 }
 
-void OPPROTO glue(op_psllw, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psllw, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 15) {
@@ -128 +117 @@
 }
 
-void OPPROTO glue(op_psrld, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psrld, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 31) {
@@ -153 +138 @@
 }
 
-void OPPROTO glue(op_psrad, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psrad, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 31) {
@@ -174 +155 @@
 }
 
-void OPPROTO glue(op_pslld, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_pslld, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 31) {
@@ -199 +176 @@
 }
 
-void OPPROTO glue(op_psrlq, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psrlq, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 63) {
@@ -222 +195 @@
 }
 
-void OPPROTO glue(op_psllq, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psllq, SUFFIX)(Reg *d, Reg *s)
+{
     int shift;
-
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     if (s->Q(0) > 63) {
@@ -246 +215 @@
 
 #if SHIFT == 1
-void OPPROTO glue(op_psrldq, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_psrldq, SUFFIX)(Reg *d, Reg *s)
+{
     int shift, i;
 
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
     shift = s->L(0);
     if (shift > 16)
@@ -263 +229 @@
 }
 
-void OPPROTO glue(op_pslldq, SUFFIX)(void)
-{
-    Reg *d, *s;
+void glue(helper_pslldq, SUFFIX)(Reg *d, Reg *s)
+{
     int shift, i;
 
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
     shift = s->L(0);
     if (shift > 16)
@@ -281 +244 @@
 #endif
 
-#define SSE_OP_B(name, F)\
-void OPPROTO glue(name, SUFFIX) (void)\
+#define SSE_HELPER_B(name, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->B(0) = F(d->B(0), s->B(0));\
     d->B(1) = F(d->B(1), s->B(1));\
@@ -307 +267 @@
 }
 
-#define SSE_OP_W(name, F)\
-void OPPROTO glue(name, SUFFIX) (void)\
+#define SSE_HELPER_W(name, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->W(0) = F(d->W(0), s->W(0));\
     d->W(1) = F(d->W(1), s->W(1));\
@@ -325 +282 @@
 }
 
-#define SSE_OP_L(name, F)\
-void OPPROTO glue(name, SUFFIX) (void)\
+#define SSE_HELPER_L(name, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->L(0) = F(d->L(0), s->L(0));\
     d->L(1) = F(d->L(1), s->L(1));\
@@ -339 +293 @@
 }
 
-#define SSE_OP_Q(name, F)\
-void OPPROTO glue(name, SUFFIX) (void)\
+#define SSE_HELPER_Q(name, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->Q(0) = F(d->Q(0), s->Q(0));\
     XMM_ONLY(\
@@ -419 +370 @@
 
 #define FMULLW(a, b) (a) * (b)
+#define FMULHRW(a, b) ((int16_t)(a) * (int16_t)(b) + 0x8000) >> 16
 #define FMULHUW(a, b) (a) * (b) >> 16
 #define FMULHW(a, b) (int16_t)(a) * (int16_t)(b) >> 16
@@ -425 +377 @@
 #endif
 
-SSE_OP_B(op_paddb, FADD)
-SSE_OP_W(op_paddw, FADD)
-SSE_OP_L(op_paddl, FADD)
-SSE_OP_Q(op_paddq, FADD)
-
-SSE_OP_B(op_psubb, FSUB)
-SSE_OP_W(op_psubw, FSUB)
-SSE_OP_L(op_psubl, FSUB)
-SSE_OP_Q(op_psubq, FSUB)
-
-SSE_OP_B(op_paddusb, FADDUB)
-SSE_OP_B(op_paddsb, FADDSB)
-SSE_OP_B(op_psubusb, FSUBUB)
-SSE_OP_B(op_psubsb, FSUBSB)
-
-SSE_OP_W(op_paddusw, FADDUW)
-SSE_OP_W(op_paddsw, FADDSW)
-SSE_OP_W(op_psubusw, FSUBUW)
-SSE_OP_W(op_psubsw, FSUBSW)
-
-SSE_OP_B(op_pminub, FMINUB)
-SSE_OP_B(op_pmaxub, FMAXUB)
-
-SSE_OP_W(op_pminsw, FMINSW)
-SSE_OP_W(op_pmaxsw, FMAXSW)
-
-SSE_OP_Q(op_pand, FAND)
-SSE_OP_Q(op_pandn, FANDN)
-SSE_OP_Q(op_por, FOR)
-SSE_OP_Q(op_pxor, FXOR)
-
-SSE_OP_B(op_pcmpgtb, FCMPGTB)
-SSE_OP_W(op_pcmpgtw, FCMPGTW)
-SSE_OP_L(op_pcmpgtl, FCMPGTL)
-
-SSE_OP_B(op_pcmpeqb, FCMPEQ)
-SSE_OP_W(op_pcmpeqw, FCMPEQ)
-SSE_OP_L(op_pcmpeql, FCMPEQ)
-
-SSE_OP_W(op_pmullw, FMULLW)
-SSE_OP_W(op_pmulhuw, FMULHUW)
-SSE_OP_W(op_pmulhw, FMULHW)
-
-SSE_OP_B(op_pavgb, FAVG)
-SSE_OP_W(op_pavgw, FAVG)
-
-void OPPROTO glue(op_pmuludq, SUFFIX) (void)
-{
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-
+SSE_HELPER_B(helper_paddb, FADD)
+SSE_HELPER_W(helper_paddw, FADD)
+SSE_HELPER_L(helper_paddl, FADD)
+SSE_HELPER_Q(helper_paddq, FADD)
+
+SSE_HELPER_B(helper_psubb, FSUB)
+SSE_HELPER_W(helper_psubw, FSUB)
+SSE_HELPER_L(helper_psubl, FSUB)
+SSE_HELPER_Q(helper_psubq, FSUB)
+
+SSE_HELPER_B(helper_paddusb, FADDUB)
+SSE_HELPER_B(helper_paddsb, FADDSB)
+SSE_HELPER_B(helper_psubusb, FSUBUB)
+SSE_HELPER_B(helper_psubsb, FSUBSB)
+
+SSE_HELPER_W(helper_paddusw, FADDUW)
+SSE_HELPER_W(helper_paddsw, FADDSW)
+SSE_HELPER_W(helper_psubusw, FSUBUW)
+SSE_HELPER_W(helper_psubsw, FSUBSW)
+
+SSE_HELPER_B(helper_pminub, FMINUB)
+SSE_HELPER_B(helper_pmaxub, FMAXUB)
+
+SSE_HELPER_W(helper_pminsw, FMINSW)
+SSE_HELPER_W(helper_pmaxsw, FMAXSW)
+
+SSE_HELPER_Q(helper_pand, FAND)
+SSE_HELPER_Q(helper_pandn, FANDN)
+SSE_HELPER_Q(helper_por, FOR)
+SSE_HELPER_Q(helper_pxor, FXOR)
+
+SSE_HELPER_B(helper_pcmpgtb, FCMPGTB)
+SSE_HELPER_W(helper_pcmpgtw, FCMPGTW)
+SSE_HELPER_L(helper_pcmpgtl, FCMPGTL)
+
+SSE_HELPER_B(helper_pcmpeqb, FCMPEQ)
+SSE_HELPER_W(helper_pcmpeqw, FCMPEQ)
+SSE_HELPER_L(helper_pcmpeql, FCMPEQ)
+
+SSE_HELPER_W(helper_pmullw, FMULLW)
+#if SHIFT == 0
+SSE_HELPER_W(helper_pmulhrw, FMULHRW)
+#endif
+SSE_HELPER_W(helper_pmulhuw, FMULHUW)
+SSE_HELPER_W(helper_pmulhw, FMULHW)
+
+SSE_HELPER_B(helper_pavgb, FAVG)
+SSE_HELPER_W(helper_pavgw, FAVG)
+
+void glue(helper_pmuludq, SUFFIX) (Reg *d, Reg *s)
+{
     d->Q(0) = (uint64_t)s->L(0) * (uint64_t)d->L(0);
 #if SHIFT == 1
@@ -483 +434 @@
 }
 
-void OPPROTO glue(op_pmaddwd, SUFFIX) (void)
+void glue(helper_pmaddwd, SUFFIX) (Reg *d, Reg *s)
 {
     int i;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     for(i = 0; i < (2 << SHIFT); i++) {
@@ -506 +454 @@
 }
 #endif
-void OPPROTO glue(op_psadbw, SUFFIX) (void)
+void glue(helper_psadbw, SUFFIX) (Reg *d, Reg *s)
 {
     unsigned int val;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     val = 0;
@@ -537 +482 @@
 }
 
-void OPPROTO glue(op_maskmov, SUFFIX) (void)
+void glue(helper_maskmov, SUFFIX) (Reg *d, Reg *s, target_ulong a0)
 {
     int i;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
     for(i = 0; i < (8 << SHIFT); i++) {
         if (s->B(i) & 0x80)
-            stb(A0 + i, d->B(i));
+            stb(a0 + i, d->B(i));
     }
     FORCE_RET();
 }
 
-void OPPROTO glue(op_movl_mm_T0, SUFFIX) (void)
-{
-    Reg *d;
-    d = (Reg *)((char *)env + PARAM1);
-    d->L(0) = T0;
+void glue(helper_movl_mm_T0, SUFFIX) (Reg *d, uint32_t val)
+{
+    d->L(0) = val;
     d->L(1) = 0;
 #if SHIFT == 1
@@ -561 +501 @@
 }
 
-void OPPROTO glue(op_movl_T0_mm, SUFFIX) (void)
-{
-    Reg *s;
-    s = (Reg *)((char *)env + PARAM1);
-    T0 = s->L(0);
-}
-
 #ifdef TARGET_X86_64
-void OPPROTO glue(op_movq_mm_T0, SUFFIX) (void)
-{
-    Reg *d;
-    d = (Reg *)((char *)env + PARAM1);
-    d->Q(0) = T0;
+void glue(helper_movq_mm_T0, SUFFIX) (Reg *d, uint64_t val)
+{
+    d->Q(0) = val;
 #if SHIFT == 1
     d->Q(1) = 0;
 #endif
 }
-
-void OPPROTO glue(op_movq_T0_mm, SUFFIX) (void)
-{
-    Reg *s;
-    s = (Reg *)((char *)env + PARAM1);
-    T0 = s->Q(0);
-}
 #endif
 
 #if SHIFT == 0
-void OPPROTO glue(op_pshufw, SUFFIX) (void)
-{
-#if __GCC__ == 3 || defined(RT_ARCH_AMD64) /* VBOX hack in #else */
-    Reg r, *d, *s;
-    int order;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
+void glue(helper_pshufw, SUFFIX) (Reg *d, Reg *s, int order)
+{
+    Reg r;
     r.W(0) = s->W(order & 3);
     r.W(1) = s->W((order >> 2) & 3);
@@ -601 +520 @@
     r.W(3) = s->W((order >> 6) & 3);
     *d = r;
+}
 #else
-    Reg *s;
-    int order;
-    uint32_t l0, l1;
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
-    l0 = s->W(order & 3);
-    l0 |= (uint32_t)s->W((order >> 2) & 3) << 16;
-    l1 = s->W((order >> 4) & 3);
-    l1 |= (uint32_t)s->W((order >> 6) & 3) << 16;
-
-    s = (Reg *)((char *)env + PARAM1);
-    s->_l[0] = l0;
-    s->_l[1] = l1;
-#endif 
-}
-#else
-void OPPROTO op_shufps(void)
-{
-    Reg r, *d, *s;
-    int order;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
+void helper_shufps(Reg *d, Reg *s, int order)
+{
+    Reg r;
     r.L(0) = d->L(order & 3);
     r.L(1) = d->L((order >> 2) & 3);
@@ -632 +532 @@
 }
 
-void OPPROTO op_shufpd(void)
-{
-    Reg r, *d, *s;
-    int order;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
+void helper_shufpd(Reg *d, Reg *s, int order)
+{
+    Reg r;
     r.Q(0) = d->Q(order & 1);
     r.Q(1) = s->Q((order >> 1) & 1);
@@ -644 +540 @@
 }
 
-void OPPROTO glue(op_pshufd, SUFFIX) (void)
-{
-    Reg r, *d, *s;
-    int order;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
+void glue(helper_pshufd, SUFFIX) (Reg *d, Reg *s, int order)
+{
+    Reg r;
     r.L(0) = s->L(order & 3);
     r.L(1) = s->L((order >> 2) & 3);
@@ -658 +550 @@
 }
 
-void OPPROTO glue(op_pshuflw, SUFFIX) (void)
-{
-    Reg r, *d, *s;
-    int order;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
+void glue(helper_pshuflw, SUFFIX) (Reg *d, Reg *s, int order)
+{
+    Reg r;
     r.W(0) = s->W(order & 3);
     r.W(1) = s->W((order >> 2) & 3);
@@ -673 +561 @@
 }
 
-void OPPROTO glue(op_pshufhw, SUFFIX) (void)
-{
-    Reg r, *d, *s;
-    int order;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
-    order = PARAM3;
+void glue(helper_pshufhw, SUFFIX) (Reg *d, Reg *s, int order)
+{
+    Reg r;
     r.Q(0) = s->Q(0);
     r.W(4) = s->W(4 + (order & 3));
@@ -693 +577 @@
 /* XXX: not accurate */
 
-#define SSE_OP_S(name, F)\
-void OPPROTO op_ ## name ## ps (void)\
+#define SSE_HELPER_S(name, F)\
+void helper_ ## name ## ps (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
     d->XMM_S(1) = F(32, d->XMM_S(1), s->XMM_S(1));\
@@ -705 +586 @@
 }\
 \
-void OPPROTO op_ ## name ## ss (void)\
+void helper_ ## name ## ss (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
 }\
-void OPPROTO op_ ## name ## pd (void)\
+void helper_ ## name ## pd (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
     d->XMM_D(1) = F(64, d->XMM_D(1), s->XMM_D(1));\
 }\
 \
-void OPPROTO op_ ## name ## sd (void)\
+void helper_ ## name ## sd (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
 }
@@ -737 +609 @@
 #define FPU_SQRT(size, a, b) float ## size ## _sqrt(b, &env->sse_status)
 
-SSE_OP_S(add, FPU_ADD)
-SSE_OP_S(sub, FPU_SUB)
-SSE_OP_S(mul, FPU_MUL)
-SSE_OP_S(div, FPU_DIV)
-SSE_OP_S(min, FPU_MIN)
-SSE_OP_S(max, FPU_MAX)
-SSE_OP_S(sqrt, FPU_SQRT)
+SSE_HELPER_S(add, FPU_ADD)
+SSE_HELPER_S(sub, FPU_SUB)
+SSE_HELPER_S(mul, FPU_MUL)
+SSE_HELPER_S(div, FPU_DIV)
+SSE_HELPER_S(min, FPU_MIN)
+SSE_HELPER_S(max, FPU_MAX)
+SSE_HELPER_S(sqrt, FPU_SQRT)
 
 
 /* float to float conversions */
-void OPPROTO op_cvtps2pd(void)
+void helper_cvtps2pd(Reg *d, Reg *s)
 {
     float32 s0, s1;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
     s0 = s->XMM_S(0);
     s1 = s->XMM_S(1);
@@ -759 +628 @@
 }
 
-void OPPROTO op_cvtpd2ps(void)
-{
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
+void helper_cvtpd2ps(Reg *d, Reg *s)
+{
     d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
     d->XMM_S(1) = float64_to_float32(s->XMM_D(1), &env->sse_status);
@@ -769 +635 @@
 }
 
-void OPPROTO op_cvtss2sd(void)
-{
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
+void helper_cvtss2sd(Reg *d, Reg *s)
+{
     d->XMM_D(0) = float32_to_float64(s->XMM_S(0), &env->sse_status);
 }
 
-void OPPROTO op_cvtsd2ss(void)
-{
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
+void helper_cvtsd2ss(Reg *d, Reg *s)
+{
     d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
 }
 
 /* integer to float */
-void OPPROTO op_cvtdq2ps(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvtdq2ps(Reg *d, Reg *s)
+{
     d->XMM_S(0) = int32_to_float32(s->XMM_L(0), &env->sse_status);
     d->XMM_S(1) = int32_to_float32(s->XMM_L(1), &env->sse_status);
@@ -796 +654 @@
 }
 
-void OPPROTO op_cvtdq2pd(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvtdq2pd(Reg *d, Reg *s)
+{
     int32_t l0, l1;
     l0 = (int32_t)s->XMM_L(0);
@@ -807 +663 @@
 }
 
-void OPPROTO op_cvtpi2ps(void)
-{
-    XMMReg *d = (Reg *)((char *)env + PARAM1);
-    MMXReg *s = (MMXReg *)((char *)env + PARAM2);
+void helper_cvtpi2ps(XMMReg *d, MMXReg *s)
+{
     d->XMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
     d->XMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
 }
 
-void OPPROTO op_cvtpi2pd(void)
-{
-    XMMReg *d = (Reg *)((char *)env + PARAM1);
-    MMXReg *s = (MMXReg *)((char *)env + PARAM2);
+void helper_cvtpi2pd(XMMReg *d, MMXReg *s)
+{
     d->XMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
     d->XMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
 }
 
-void OPPROTO op_cvtsi2ss(void)
-{
-    XMMReg *d = (Reg *)((char *)env + PARAM1);
-    d->XMM_S(0) = int32_to_float32(T0, &env->sse_status);
-}
-
-void OPPROTO op_cvtsi2sd(void)
-{
-    XMMReg *d = (Reg *)((char *)env + PARAM1);
-    d->XMM_D(0) = int32_to_float64(T0, &env->sse_status);
+void helper_cvtsi2ss(XMMReg *d, uint32_t val)
+{
+    d->XMM_S(0) = int32_to_float32(val, &env->sse_status);
+}
+
+void helper_cvtsi2sd(XMMReg *d, uint32_t val)
+{
+    d->XMM_D(0) = int32_to_float64(val, &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-void OPPROTO op_cvtsq2ss(void)
-{
-    XMMReg *d = (Reg *)((char *)env + PARAM1);
-    d->XMM_S(0) = int64_to_float32(T0, &env->sse_status);
-}
-
-void OPPROTO op_cvtsq2sd(void)
-{
-    XMMReg *d = (Reg *)((char *)env + PARAM1);
-    d->XMM_D(0) = int64_to_float64(T0, &env->sse_status);
+void helper_cvtsq2ss(XMMReg *d, uint64_t val)
+{
+    d->XMM_S(0) = int64_to_float32(val, &env->sse_status);
+}
+
+void helper_cvtsq2sd(XMMReg *d, uint64_t val)
+{
+    d->XMM_D(0) = int64_to_float64(val, &env->sse_status);
 }
 #endif
 
 /* float to integer */
-void OPPROTO op_cvtps2dq(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvtps2dq(XMMReg *d, XMMReg *s)
+{
     d->XMM_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
     d->XMM_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
@@ -860 +706 @@
 }
 
-void OPPROTO op_cvtpd2dq(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvtpd2dq(XMMReg *d, XMMReg *s)
+{
     d->XMM_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
     d->XMM_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
@@ -869 +713 @@
 }
 
-void OPPROTO op_cvtps2pi(void)
-{
-    MMXReg *d = (MMXReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvtps2pi(MMXReg *d, XMMReg *s)
+{
     d->MMX_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
     d->MMX_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
 }
 
-void OPPROTO op_cvtpd2pi(void)
-{
-    MMXReg *d = (MMXReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvtpd2pi(MMXReg *d, XMMReg *s)
+{
     d->MMX_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
     d->MMX_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
 }
 
-void OPPROTO op_cvtss2si(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float32_to_int32(s->XMM_S(0), &env->sse_status);
-}
-
-void OPPROTO op_cvtsd2si(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float64_to_int32(s->XMM_D(0), &env->sse_status);
+int32_t helper_cvtss2si(XMMReg *s)
+{
+    return float32_to_int32(s->XMM_S(0), &env->sse_status);
+}
+
+int32_t helper_cvtsd2si(XMMReg *s)
+{
+    return float64_to_int32(s->XMM_D(0), &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-void OPPROTO op_cvtss2sq(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float32_to_int64(s->XMM_S(0), &env->sse_status);
-}
-
-void OPPROTO op_cvtsd2sq(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float64_to_int64(s->XMM_D(0), &env->sse_status);
+int64_t helper_cvtss2sq(XMMReg *s)
+{
+    return float32_to_int64(s->XMM_S(0), &env->sse_status);
+}
+
+int64_t helper_cvtsd2sq(XMMReg *s)
+{
+    return float64_to_int64(s->XMM_D(0), &env->sse_status);
 }
 #endif
 
 /* float to integer truncated */
-void OPPROTO op_cvttps2dq(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvttps2dq(XMMReg *d, XMMReg *s)
+{
     d->XMM_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
     d->XMM_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
@@ -922 +756 @@
 }
 
-void OPPROTO op_cvttpd2dq(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvttpd2dq(XMMReg *d, XMMReg *s)
+{
     d->XMM_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
     d->XMM_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
@@ -931 +763 @@
 }
 
-void OPPROTO op_cvttps2pi(void)
-{
-    MMXReg *d = (MMXReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvttps2pi(MMXReg *d, XMMReg *s)
+{
     d->MMX_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
     d->MMX_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
 }
 
-void OPPROTO op_cvttpd2pi(void)
-{
-    MMXReg *d = (MMXReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_cvttpd2pi(MMXReg *d, XMMReg *s)
+{
     d->MMX_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
     d->MMX_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
 }
 
-void OPPROTO op_cvttss2si(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
-}
-
-void OPPROTO op_cvttsd2si(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
+int32_t helper_cvttss2si(XMMReg *s)
+{
+    return float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
+}
+
+int32_t helper_cvttsd2si(XMMReg *s)
+{
+    return float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
 }
 
 #ifdef TARGET_X86_64
-void OPPROTO op_cvttss2sq(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float32_to_int64_round_to_zero(s->XMM_S(0), &env->sse_status);
-}
-
-void OPPROTO op_cvttsd2sq(void)
-{
-    XMMReg *s = (XMMReg *)((char *)env + PARAM1);
-    T0 = float64_to_int64_round_to_zero(s->XMM_D(0), &env->sse_status);
-}
-#endif
-
-void OPPROTO op_rsqrtps(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+int64_t helper_cvttss2sq(XMMReg *s)
+{
+    return float32_to_int64_round_to_zero(s->XMM_S(0), &env->sse_status);
+}
+
+int64_t helper_cvttsd2sq(XMMReg *s)
+{
+    return float64_to_int64_round_to_zero(s->XMM_D(0), &env->sse_status);
+}
+#endif
+
+void helper_rsqrtps(XMMReg *d, XMMReg *s)
+{
     d->XMM_S(0) = approx_rsqrt(s->XMM_S(0));
     d->XMM_S(1) = approx_rsqrt(s->XMM_S(1));
@@ -983 +805 @@
 }
 
-void OPPROTO op_rsqrtss(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_rsqrtss(XMMReg *d, XMMReg *s)
+{
     d->XMM_S(0) = approx_rsqrt(s->XMM_S(0));
 }
 
-void OPPROTO op_rcpps(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_rcpps(XMMReg *d, XMMReg *s)
+{
     d->XMM_S(0) = approx_rcp(s->XMM_S(0));
     d->XMM_S(1) = approx_rcp(s->XMM_S(1));
@@ -1000 +818 @@
 }
 
-void OPPROTO op_rcpss(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_rcpss(XMMReg *d, XMMReg *s)
+{
     d->XMM_S(0) = approx_rcp(s->XMM_S(0));
 }
 
-void OPPROTO op_haddps(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_haddps(XMMReg *d, XMMReg *s)
+{
     XMMReg r;
     r.XMM_S(0) = d->XMM_S(0) + d->XMM_S(1);
@@ -1019 +833 @@
 }
 
-void OPPROTO op_haddpd(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_haddpd(XMMReg *d, XMMReg *s)
+{
     XMMReg r;
     r.XMM_D(0) = d->XMM_D(0) + d->XMM_D(1);
@@ -1029 +841 @@
 }
 
-void OPPROTO op_hsubps(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_hsubps(XMMReg *d, XMMReg *s)
+{
     XMMReg r;
     r.XMM_S(0) = d->XMM_S(0) - d->XMM_S(1);
@@ -1041 +851 @@
 }
 
-void OPPROTO op_hsubpd(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_hsubpd(XMMReg *d, XMMReg *s)
+{
     XMMReg r;
     r.XMM_D(0) = d->XMM_D(0) - d->XMM_D(1);
@@ -1051 +859 @@
 }
 
-void OPPROTO op_addsubps(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_addsubps(XMMReg *d, XMMReg *s)
+{
     d->XMM_S(0) = d->XMM_S(0) - s->XMM_S(0);
     d->XMM_S(1) = d->XMM_S(1) + s->XMM_S(1);
@@ -1061 +867 @@
 }
 
-void OPPROTO op_addsubpd(void)
-{
-    XMMReg *d = (XMMReg *)((char *)env + PARAM1);
-    XMMReg *s = (XMMReg *)((char *)env + PARAM2);
+void helper_addsubpd(XMMReg *d, XMMReg *s)
+{
     d->XMM_D(0) = d->XMM_D(0) - s->XMM_D(0);
     d->XMM_D(1) = d->XMM_D(1) + s->XMM_D(1);
@@ -1070 +874 @@
 
 /* XXX: unordered */
-#define SSE_OP_CMP(name, F)\
-void OPPROTO op_ ## name ## ps (void)\
+#define SSE_HELPER_CMP(name, F)\
+void helper_ ## name ## ps (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
     d->XMM_L(1) = F(32, d->XMM_S(1), s->XMM_S(1));\
@@ -1082 +883 @@
 }\
 \
-void OPPROTO op_ ## name ## ss (void)\
+void helper_ ## name ## ss (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0));\
 }\
-void OPPROTO op_ ## name ## pd (void)\
+void helper_ ## name ## pd (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
     d->XMM_Q(1) = F(64, d->XMM_D(1), s->XMM_D(1));\
 }\
 \
-void OPPROTO op_ ## name ## sd (void)\
+void helper_ ## name ## sd (Reg *d, Reg *s)\
 {\
-    Reg *d, *s;\
-    d = (Reg *)((char *)env + PARAM1);\
-    s = (Reg *)((char *)env + PARAM2);\
     d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0));\
 }
@@ -1115 +907 @@
 #define FPU_CMPORD(size, a, b) float ## size ## _unordered(a, b, &env->sse_status) ? 0 : -1
 
-SSE_OP_CMP(cmpeq, FPU_CMPEQ)
-SSE_OP_CMP(cmplt, FPU_CMPLT)
-SSE_OP_CMP(cmple, FPU_CMPLE)
-SSE_OP_CMP(cmpunord, FPU_CMPUNORD)
-SSE_OP_CMP(cmpneq, FPU_CMPNEQ)
-SSE_OP_CMP(cmpnlt, FPU_CMPNLT)
-SSE_OP_CMP(cmpnle, FPU_CMPNLE)
-SSE_OP_CMP(cmpord, FPU_CMPORD)
+SSE_HELPER_CMP(cmpeq, FPU_CMPEQ)
+SSE_HELPER_CMP(cmplt, FPU_CMPLT)
+SSE_HELPER_CMP(cmple, FPU_CMPLE)
+SSE_HELPER_CMP(cmpunord, FPU_CMPUNORD)
+SSE_HELPER_CMP(cmpneq, FPU_CMPNEQ)
+SSE_HELPER_CMP(cmpnlt, FPU_CMPNLT)
+SSE_HELPER_CMP(cmpnle, FPU_CMPNLE)
+SSE_HELPER_CMP(cmpord, FPU_CMPORD)
 
 const int comis_eflags[4] = {CC_C, CC_Z, 0, CC_Z | CC_P | CC_C};
 
-void OPPROTO op_ucomiss(void)
+void helper_ucomiss(Reg *d, Reg *s)
 {
     int ret;
     float32 s0, s1;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     s0 = d->XMM_S(0);
@@ -1141 +930 @@
 }
 
-void OPPROTO op_comiss(void)
+void helper_comiss(Reg *d, Reg *s)
 {
     int ret;
     float32 s0, s1;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     s0 = d->XMM_S(0);
@@ -1156 +942 @@
 }
 
-void OPPROTO op_ucomisd(void)
+void helper_ucomisd(Reg *d, Reg *s)
 {
     int ret;
     float64 d0, d1;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     d0 = d->XMM_D(0);
@@ -1171 +954 @@
 }
 
-void OPPROTO op_comisd(void)
+void helper_comisd(Reg *d, Reg *s)
 {
     int ret;
     float64 d0, d1;
-    Reg *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
 
     d0 = d->XMM_D(0);
@@ -1186 +966 @@
 }
 
-void OPPROTO op_movmskps(void)
+uint32_t helper_movmskps(Reg *s)
 {
     int b0, b1, b2, b3;
-    Reg *s;
-    s = (Reg *)((char *)env + PARAM1);
     b0 = s->XMM_L(0) >> 31;
     b1 = s->XMM_L(1) >> 31;
     b2 = s->XMM_L(2) >> 31;
     b3 = s->XMM_L(3) >> 31;
-    T0 = b0 | (b1 << 1) | (b2 << 2) | (b3 << 3);
-}
-
-void OPPROTO op_movmskpd(void)
+    return b0 | (b1 << 1) | (b2 << 2) | (b3 << 3);
+}
+
+uint32_t helper_movmskpd(Reg *s)
 {
     int b0, b1;
-    Reg *s;
-    s = (Reg *)((char *)env + PARAM1);
     b0 = s->XMM_L(1) >> 31;
     b1 = s->XMM_L(3) >> 31;
-    T0 = b0 | (b1 << 1);
-}
-
-#endif
-
-void OPPROTO glue(op_pmovmskb, SUFFIX)(void)
-{
-    Reg *s;
-    s = (Reg *)((char *)env + PARAM1);
-    T0 = 0;
-    T0 |= (s->XMM_B(0) >> 7);
-    T0 |= (s->XMM_B(1) >> 6) & 0x02;
-    T0 |= (s->XMM_B(2) >> 5) & 0x04;
-    T0 |= (s->XMM_B(3) >> 4) & 0x08;
-    T0 |= (s->XMM_B(4) >> 3) & 0x10;
-    T0 |= (s->XMM_B(5) >> 2) & 0x20;
-    T0 |= (s->XMM_B(6) >> 1) & 0x40;
-    T0 |= (s->XMM_B(7)) & 0x80;
-#if SHIFT == 1
-    T0 |= (s->XMM_B(8) << 1) & 0x0100;
-    T0 |= (s->XMM_B(9) << 2) & 0x0200;
-    T0 |= (s->XMM_B(10) << 3) & 0x0400;
-    T0 |= (s->XMM_B(11) << 4) & 0x0800;
-    T0 |= (s->XMM_B(12) << 5) & 0x1000;
-    T0 |= (s->XMM_B(13) << 6) & 0x2000;
-    T0 |= (s->XMM_B(14) << 7) & 0x4000;
-    T0 |= (s->XMM_B(15) << 8) & 0x8000;
-#endif
-}
-
-void OPPROTO glue(op_pinsrw, SUFFIX) (void)
-{
-    Reg *d = (Reg *)((char *)env + PARAM1);
-    int pos = PARAM2;
-    
-    d->W(pos) = T0;
-}
-
-void OPPROTO glue(op_pextrw, SUFFIX) (void)
-{
-    Reg *s = (Reg *)((char *)env + PARAM1);
-    int pos = PARAM2;
-    
-    T0 = s->W(pos);
-}
-
-void OPPROTO glue(op_packsswb, SUFFIX) (void)
-{
-    Reg r, *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
+    return b0 | (b1 << 1);
+}
+
+#endif
+
+uint32_t glue(helper_pmovmskb, SUFFIX)(Reg *s)
+{
+    uint32_t val;
+    val = 0;
+    val |= (s->XMM_B(0) >> 7);
+    val |= (s->XMM_B(1) >> 6) & 0x02;
+    val |= (s->XMM_B(2) >> 5) & 0x04;
+    val |= (s->XMM_B(3) >> 4) & 0x08;
+    val |= (s->XMM_B(4) >> 3) & 0x10;
+    val |= (s->XMM_B(5) >> 2) & 0x20;
+    val |= (s->XMM_B(6) >> 1) & 0x40;
+    val |= (s->XMM_B(7)) & 0x80;
+#if SHIFT == 1
+    val |= (s->XMM_B(8) << 1) & 0x0100;
+    val |= (s->XMM_B(9) << 2) & 0x0200;
+    val |= (s->XMM_B(10) << 3) & 0x0400;
+    val |= (s->XMM_B(11) << 4) & 0x0800;
+    val |= (s->XMM_B(12) << 5) & 0x1000;
+    val |= (s->XMM_B(13) << 6) & 0x2000;
+    val |= (s->XMM_B(14) << 7) & 0x4000;
+    val |= (s->XMM_B(15) << 8) & 0x8000;
+#endif
+    return val;
+}
+
+void glue(helper_packsswb, SUFFIX) (Reg *d, Reg *s)
+{
+    Reg r;
 
     r.B(0) = satsb((int16_t)d->W(0));
@@ -1280 +1038 @@
 }
 
-void OPPROTO glue(op_packuswb, SUFFIX) (void)
-{
-    Reg r, *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
+void glue(helper_packuswb, SUFFIX) (Reg *d, Reg *s)
+{
+    Reg r;
 
     r.B(0) = satub((int16_t)d->W(0));
@@ -1309 +1065 @@
 }
 
-void OPPROTO glue(op_packssdw, SUFFIX) (void)
-{
-    Reg r, *d, *s;
-    d = (Reg *)((char *)env + PARAM1);
-    s = (Reg *)((char *)env + PARAM2);
+void glue(helper_packssdw, SUFFIX) (Reg *d, Reg *s)
+{
+    Reg r;
 
     r.W(0) = satsw(d->L(0));
@@ -1332 +1086 @@
 #define UNPCK_OP(base_name, base)                               \
                                                                 \
-void OPPROTO glue(op_punpck ## base_name ## bw, SUFFIX) (void)   \
+void glue(helper_punpck ## base_name ## bw, SUFFIX) (Reg *d, Reg *s)   \
 {                                                               \
-    Reg r, *d, *s;                                              \
-    d = (Reg *)((char *)env + PARAM1);                          \
-    s = (Reg *)((char *)env + PARAM2);                          \
+    Reg r;                                              \
                                                                 \
     r.B(0) = d->B((base << (SHIFT + 2)) + 0);                   \
@@ -1359 +1111 @@
 }                                                               \
                                                                 \
-void OPPROTO glue(op_punpck ## base_name ## wd, SUFFIX) (void)   \
+void glue(helper_punpck ## base_name ## wd, SUFFIX) (Reg *d, Reg *s)   \
 {                                                               \
-    Reg r, *d, *s;                                              \
-    d = (Reg *)((char *)env + PARAM1);                          \
-    s = (Reg *)((char *)env + PARAM2);                          \
+    Reg r;                                              \
                                                                 \
     r.W(0) = d->W((base << (SHIFT + 1)) + 0);                   \
@@ -1378 +1128 @@
 }                                                               \
                                                                 \
-void OPPROTO glue(op_punpck ## base_name ## dq, SUFFIX) (void)   \
+void glue(helper_punpck ## base_name ## dq, SUFFIX) (Reg *d, Reg *s)   \
 {                                                               \
-    Reg r, *d, *s;                                              \
-    d = (Reg *)((char *)env + PARAM1);                          \
-    s = (Reg *)((char *)env + PARAM2);                          \
+    Reg r;                                              \
                                                                 \
     r.L(0) = d->L((base << SHIFT) + 0);                         \
@@ -1394 +1142 @@
                                                                 \
 XMM_ONLY(                                                       \
-void OPPROTO glue(op_punpck ## base_name ## qdq, SUFFIX) (void)  \
+void glue(helper_punpck ## base_name ## qdq, SUFFIX) (Reg *d, Reg *s)  \
 {                                                               \
-    Reg r, *d, *s;                                              \
-    d = (Reg *)((char *)env + PARAM1);                          \
-    s = (Reg *)((char *)env + PARAM2);                          \
+    Reg r;                                              \
                                                                 \
     r.Q(0) = d->Q(base);                                        \
@@ -1408 +1154 @@
 UNPCK_OP(l, 0)
 UNPCK_OP(h, 1)
+
+/* 3DNow! float ops */
+#if SHIFT == 0
+void helper_pi2fd(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = int32_to_float32(s->MMX_L(0), &env->mmx_status);
+    d->MMX_S(1) = int32_to_float32(s->MMX_L(1), &env->mmx_status);
+}
+
+void helper_pi2fw(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = int32_to_float32((int16_t)s->MMX_W(0), &env->mmx_status);
+    d->MMX_S(1) = int32_to_float32((int16_t)s->MMX_W(2), &env->mmx_status);
+}
+
+void helper_pf2id(MMXReg *d, MMXReg *s)
+{
+    d->MMX_L(0) = float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status);
+    d->MMX_L(1) = float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status);
+}
+
+void helper_pf2iw(MMXReg *d, MMXReg *s)
+{
+    d->MMX_L(0) = satsw(float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status));
+    d->MMX_L(1) = satsw(float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status));
+}
+
+void helper_pfacc(MMXReg *d, MMXReg *s)
+{
+    MMXReg r;
+    r.MMX_S(0) = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
+    r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
+    *d = r;
+}
+
+void helper_pfadd(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = float32_add(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
+    d->MMX_S(1) = float32_add(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
+}
+
+void helper_pfcmpeq(MMXReg *d, MMXReg *s)
+{
+    d->MMX_L(0) = float32_eq(d->MMX_S(0), s->MMX_S(0), &env->mmx_status) ? -1 : 0;
+    d->MMX_L(1) = float32_eq(d->MMX_S(1), s->MMX_S(1), &env->mmx_status) ? -1 : 0;
+}
+
+void helper_pfcmpge(MMXReg *d, MMXReg *s)
+{
+    d->MMX_L(0) = float32_le(s->MMX_S(0), d->MMX_S(0), &env->mmx_status) ? -1 : 0;
+    d->MMX_L(1) = float32_le(s->MMX_S(1), d->MMX_S(1), &env->mmx_status) ? -1 : 0;
+}
+
+void helper_pfcmpgt(MMXReg *d, MMXReg *s)
+{
+    d->MMX_L(0) = float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status) ? -1 : 0;
+    d->MMX_L(1) = float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status) ? -1 : 0;
+}
+
+void helper_pfmax(MMXReg *d, MMXReg *s)
+{
+    if (float32_lt(d->MMX_S(0), s->MMX_S(0), &env->mmx_status))
+        d->MMX_S(0) = s->MMX_S(0);
+    if (float32_lt(d->MMX_S(1), s->MMX_S(1), &env->mmx_status))
+        d->MMX_S(1) = s->MMX_S(1);
+}
+
+void helper_pfmin(MMXReg *d, MMXReg *s)
+{
+    if (float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status))
+        d->MMX_S(0) = s->MMX_S(0);
+    if (float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status))
+        d->MMX_S(1) = s->MMX_S(1);
+}
+
+void helper_pfmul(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = float32_mul(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
+    d->MMX_S(1) = float32_mul(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
+}
+
+void helper_pfnacc(MMXReg *d, MMXReg *s)
+{
+    MMXReg r;
+    r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
+    r.MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
+    *d = r;
+}
+
+void helper_pfpnacc(MMXReg *d, MMXReg *s)
+{
+    MMXReg r;
+    r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
+    r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
+    *d = r;
+}
+
+void helper_pfrcp(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = approx_rcp(s->MMX_S(0));
+    d->MMX_S(1) = d->MMX_S(0);
+}
+
+void helper_pfrsqrt(MMXReg *d, MMXReg *s)
+{
+    d->MMX_L(1) = s->MMX_L(0) & 0x7fffffff;
+    d->MMX_S(1) = approx_rsqrt(d->MMX_S(1));
+    d->MMX_L(1) |= s->MMX_L(0) & 0x80000000;
+    d->MMX_L(0) = d->MMX_L(1);
+}
+
+void helper_pfsub(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = float32_sub(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
+    d->MMX_S(1) = float32_sub(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
+}
+
+void helper_pfsubr(MMXReg *d, MMXReg *s)
+{
+    d->MMX_S(0) = float32_sub(s->MMX_S(0), d->MMX_S(0), &env->mmx_status);
+    d->MMX_S(1) = float32_sub(s->MMX_S(1), d->MMX_S(1), &env->mmx_status);
+}
+
+void helper_pswapd(MMXReg *d, MMXReg *s)
+{
+    MMXReg r;
+    r.MMX_L(0) = s->MMX_L(1);
+    r.MMX_L(1) = s->MMX_L(0);
+    *d = r;
+}
+#endif
+
+/* SSSE3 op helpers */
+void glue(helper_pshufb, SUFFIX) (Reg *d, Reg *s)
+{
+    int i;
+    Reg r;
+
+    for (i = 0; i < (8 << SHIFT); i++)
+        r.B(i) = (s->B(i) & 0x80) ? 0 : (d->B(s->B(i) & ((8 << SHIFT) - 1)));
+
+    *d = r;
+}
+
+void glue(helper_phaddw, SUFFIX) (Reg *d, Reg *s)
+{
+    d->W(0) = (int16_t)d->W(0) + (int16_t)d->W(1);
+    d->W(1) = (int16_t)d->W(2) + (int16_t)d->W(3);
+    XMM_ONLY(d->W(2) = (int16_t)d->W(4) + (int16_t)d->W(5));
+    XMM_ONLY(d->W(3) = (int16_t)d->W(6) + (int16_t)d->W(7));
+    d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) + (int16_t)s->W(1);
+    d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) + (int16_t)s->W(3);
+    XMM_ONLY(d->W(6) = (int16_t)s->W(4) + (int16_t)s->W(5));
+    XMM_ONLY(d->W(7) = (int16_t)s->W(6) + (int16_t)s->W(7));
+}
+
+void glue(helper_phaddd, SUFFIX) (Reg *d, Reg *s)
+{
+    d->L(0) = (int32_t)d->L(0) + (int32_t)d->L(1);
+    XMM_ONLY(d->L(1) = (int32_t)d->L(2) + (int32_t)d->L(3));
+    d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) + (int32_t)s->L(1);
+    XMM_ONLY(d->L(3) = (int32_t)s->L(2) + (int32_t)s->L(3));
+}
+
+void glue(helper_phaddsw, SUFFIX) (Reg *d, Reg *s)
+{
+    d->W(0) = satsw((int16_t)d->W(0) + (int16_t)d->W(1));
+    d->W(1) = satsw((int16_t)d->W(2) + (int16_t)d->W(3));
+    XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) + (int16_t)d->W(5)));
+    XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) + (int16_t)d->W(7)));
+    d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) + (int16_t)s->W(1));
+    d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) + (int16_t)s->W(3));
+    XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) + (int16_t)s->W(5)));
+    XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) + (int16_t)s->W(7)));
+}
+
+void glue(helper_pmaddubsw, SUFFIX) (Reg *d, Reg *s)
+{
+    d->W(0) = satsw((int8_t)s->B( 0) * (uint8_t)d->B( 0) +
+                    (int8_t)s->B( 1) * (uint8_t)d->B( 1));
+    d->W(1) = satsw((int8_t)s->B( 2) * (uint8_t)d->B( 2) +
+                    (int8_t)s->B( 3) * (uint8_t)d->B( 3));
+    d->W(2) = satsw((int8_t)s->B( 4) * (uint8_t)d->B( 4) +
+                    (int8_t)s->B( 5) * (uint8_t)d->B( 5));
+    d->W(3) = satsw((int8_t)s->B( 6) * (uint8_t)d->B( 6) +
+                    (int8_t)s->B( 7) * (uint8_t)d->B( 7));
+#if SHIFT == 1
+    d->W(4) = satsw((int8_t)s->B( 8) * (uint8_t)d->B( 8) +
+                    (int8_t)s->B( 9) * (uint8_t)d->B( 9));
+    d->W(5) = satsw((int8_t)s->B(10) * (uint8_t)d->B(10) +
+                    (int8_t)s->B(11) * (uint8_t)d->B(11));
+    d->W(6) = satsw((int8_t)s->B(12) * (uint8_t)d->B(12) +
+                    (int8_t)s->B(13) * (uint8_t)d->B(13));
+    d->W(7) = satsw((int8_t)s->B(14) * (uint8_t)d->B(14) +
+                    (int8_t)s->B(15) * (uint8_t)d->B(15));
+#endif
+}
+
+void glue(helper_phsubw, SUFFIX) (Reg *d, Reg *s)
+{
+    d->W(0) = (int16_t)d->W(0) - (int16_t)d->W(1);
+    d->W(1) = (int16_t)d->W(2) - (int16_t)d->W(3);
+    XMM_ONLY(d->W(2) = (int16_t)d->W(4) - (int16_t)d->W(5));
+    XMM_ONLY(d->W(3) = (int16_t)d->W(6) - (int16_t)d->W(7));
+    d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) - (int16_t)s->W(1);
+    d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) - (int16_t)s->W(3);
+    XMM_ONLY(d->W(6) = (int16_t)s->W(4) - (int16_t)s->W(5));
+    XMM_ONLY(d->W(7) = (int16_t)s->W(6) - (int16_t)s->W(7));
+}
+
+void glue(helper_phsubd, SUFFIX) (Reg *d, Reg *s)
+{
+    d->L(0) = (int32_t)d->L(0) - (int32_t)d->L(1);
+    XMM_ONLY(d->L(1) = (int32_t)d->L(2) - (int32_t)d->L(3));
+    d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) - (int32_t)s->L(1);
+    XMM_ONLY(d->L(3) = (int32_t)s->L(2) - (int32_t)s->L(3));
+}
+
+void glue(helper_phsubsw, SUFFIX) (Reg *d, Reg *s)
+{
+    d->W(0) = satsw((int16_t)d->W(0) - (int16_t)d->W(1));
+    d->W(1) = satsw((int16_t)d->W(2) - (int16_t)d->W(3));
+    XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) - (int16_t)d->W(5)));
+    XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) - (int16_t)d->W(7)));
+    d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) - (int16_t)s->W(1));
+    d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) - (int16_t)s->W(3));
+    XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) - (int16_t)s->W(5)));
+    XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) - (int16_t)s->W(7)));
+}
+
+#define FABSB(_, x) x > INT8_MAX  ? -(int8_t ) x : x
+#define FABSW(_, x) x > INT16_MAX ? -(int16_t) x : x
+#define FABSL(_, x) x > INT32_MAX ? -(int32_t) x : x
+SSE_HELPER_B(helper_pabsb, FABSB)
+SSE_HELPER_W(helper_pabsw, FABSW)
+SSE_HELPER_L(helper_pabsd, FABSL)
+
+#define FMULHRSW(d, s) ((int16_t) d * (int16_t) s + 0x4000) >> 15
+SSE_HELPER_W(helper_pmulhrsw, FMULHRSW)
+
+#define FSIGNB(d, s) s <= INT8_MAX  ? s ? d : 0 : -(int8_t ) d
+#define FSIGNW(d, s) s <= INT16_MAX ? s ? d : 0 : -(int16_t) d
+#define FSIGNL(d, s) s <= INT32_MAX ? s ? d : 0 : -(int32_t) d
+SSE_HELPER_B(helper_psignb, FSIGNB)
+SSE_HELPER_W(helper_psignw, FSIGNW)
+SSE_HELPER_L(helper_psignd, FSIGNL)
+
+void glue(helper_palignr, SUFFIX) (Reg *d, Reg *s, int32_t shift)
+{
+    Reg r;
+
+    /* XXX could be checked during translation */
+    if (shift >= (16 << SHIFT)) {
+        r.Q(0) = 0;
+        XMM_ONLY(r.Q(1) = 0);
+    } else {
+        shift <<= 3;
+#define SHR(v, i) (i < 64 && i > -64 ? i > 0 ? v >> (i) : (v << -(i)) : 0)
+#if SHIFT == 0
+        r.Q(0) = SHR(s->Q(0), shift -   0) |
+                 SHR(d->Q(0), shift -  64);
+#else
+        r.Q(0) = SHR(s->Q(0), shift -   0) |
+                 SHR(s->Q(1), shift -  64) |
+                 SHR(d->Q(0), shift - 128) |
+                 SHR(d->Q(1), shift - 192);
+        r.Q(1) = SHR(s->Q(0), shift +  64) |
+                 SHR(s->Q(1), shift -   0) |
+                 SHR(d->Q(0), shift -  64) |
+                 SHR(d->Q(1), shift - 128);
+#endif
+#undef SHR
+    }
+
+    *d = r;
+}
+
+#define XMM0 env->xmm_regs[0]
+
+#if SHIFT == 1
+#define SSE_HELPER_V(name, elem, num, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s)\
+{\
+    d->elem(0) = F(d->elem(0), s->elem(0), XMM0.elem(0));\
+    d->elem(1) = F(d->elem(1), s->elem(1), XMM0.elem(1));\
+    if (num > 2) {\
+        d->elem(2) = F(d->elem(2), s->elem(2), XMM0.elem(2));\
+        d->elem(3) = F(d->elem(3), s->elem(3), XMM0.elem(3));\
+        if (num > 4) {\
+            d->elem(4) = F(d->elem(4), s->elem(4), XMM0.elem(4));\
+            d->elem(5) = F(d->elem(5), s->elem(5), XMM0.elem(5));\
+            d->elem(6) = F(d->elem(6), s->elem(6), XMM0.elem(6));\
+            d->elem(7) = F(d->elem(7), s->elem(7), XMM0.elem(7));\
+            if (num > 8) {\
+                d->elem(8) = F(d->elem(8), s->elem(8), XMM0.elem(8));\
+                d->elem(9) = F(d->elem(9), s->elem(9), XMM0.elem(9));\
+                d->elem(10) = F(d->elem(10), s->elem(10), XMM0.elem(10));\
+                d->elem(11) = F(d->elem(11), s->elem(11), XMM0.elem(11));\
+                d->elem(12) = F(d->elem(12), s->elem(12), XMM0.elem(12));\
+                d->elem(13) = F(d->elem(13), s->elem(13), XMM0.elem(13));\
+                d->elem(14) = F(d->elem(14), s->elem(14), XMM0.elem(14));\
+                d->elem(15) = F(d->elem(15), s->elem(15), XMM0.elem(15));\
+            }\
+        }\
+    }\
+}
+
+#define SSE_HELPER_I(name, elem, num, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s, uint32_t imm)\
+{\
+    d->elem(0) = F(d->elem(0), s->elem(0), ((imm >> 0) & 1));\
+    d->elem(1) = F(d->elem(1), s->elem(1), ((imm >> 1) & 1));\
+    if (num > 2) {\
+        d->elem(2) = F(d->elem(2), s->elem(2), ((imm >> 2) & 1));\
+        d->elem(3) = F(d->elem(3), s->elem(3), ((imm >> 3) & 1));\
+        if (num > 4) {\
+            d->elem(4) = F(d->elem(4), s->elem(4), ((imm >> 4) & 1));\
+            d->elem(5) = F(d->elem(5), s->elem(5), ((imm >> 5) & 1));\
+            d->elem(6) = F(d->elem(6), s->elem(6), ((imm >> 6) & 1));\
+            d->elem(7) = F(d->elem(7), s->elem(7), ((imm >> 7) & 1));\
+            if (num > 8) {\
+                d->elem(8) = F(d->elem(8), s->elem(8), ((imm >> 8) & 1));\
+                d->elem(9) = F(d->elem(9), s->elem(9), ((imm >> 9) & 1));\
+                d->elem(10) = F(d->elem(10), s->elem(10), ((imm >> 10) & 1));\
+                d->elem(11) = F(d->elem(11), s->elem(11), ((imm >> 11) & 1));\
+                d->elem(12) = F(d->elem(12), s->elem(12), ((imm >> 12) & 1));\
+                d->elem(13) = F(d->elem(13), s->elem(13), ((imm >> 13) & 1));\
+                d->elem(14) = F(d->elem(14), s->elem(14), ((imm >> 14) & 1));\
+                d->elem(15) = F(d->elem(15), s->elem(15), ((imm >> 15) & 1));\
+            }\
+        }\
+    }\
+}
+
+/* SSE4.1 op helpers */
+#define FBLENDVB(d, s, m) (m & 0x80) ? s : d
+#define FBLENDVPS(d, s, m) (m & 0x80000000) ? s : d
+#define FBLENDVPD(d, s, m) (m & 0x8000000000000000LL) ? s : d
+SSE_HELPER_V(helper_pblendvb, B, 16, FBLENDVB)
+SSE_HELPER_V(helper_blendvps, L, 4, FBLENDVPS)
+SSE_HELPER_V(helper_blendvpd, Q, 2, FBLENDVPD)
+
+void glue(helper_ptest, SUFFIX) (Reg *d, Reg *s)
+{
+    uint64_t zf = (s->Q(0) &  d->Q(0)) | (s->Q(1) &  d->Q(1));
+    uint64_t cf = (s->Q(0) & ~d->Q(0)) | (s->Q(1) & ~d->Q(1));
+
+    CC_SRC = (zf ? 0 : CC_Z) | (cf ? 0 : CC_C);
+}
+
+#define SSE_HELPER_F(name, elem, num, F)\
+void glue(name, SUFFIX) (Reg *d, Reg *s)\
+{\
+    d->elem(0) = F(0);\
+    d->elem(1) = F(1);\
+    d->elem(2) = F(2);\
+    d->elem(3) = F(3);\
+    if (num > 3) {\
+        d->elem(4) = F(4);\
+        d->elem(5) = F(5);\
+        if (num > 5) {\
+            d->elem(6) = F(6);\
+            d->elem(7) = F(7);\
+        }\
+    }\
+}
+
+SSE_HELPER_F(helper_pmovsxbw, W, 8, (int8_t) s->B)
+SSE_HELPER_F(helper_pmovsxbd, L, 4, (int8_t) s->B)
+SSE_HELPER_F(helper_pmovsxbq, Q, 2, (int8_t) s->B)
+SSE_HELPER_F(helper_pmovsxwd, L, 4, (int16_t) s->W)
+SSE_HELPER_F(helper_pmovsxwq, Q, 2, (int16_t) s->W)
+SSE_HELPER_F(helper_pmovsxdq, Q, 2, (int32_t) s->L)
+SSE_HELPER_F(helper_pmovzxbw, W, 8, s->B)
+SSE_HELPER_F(helper_pmovzxbd, L, 4, s->B)
+SSE_HELPER_F(helper_pmovzxbq, Q, 2, s->B)
+SSE_HELPER_F(helper_pmovzxwd, L, 4, s->W)
+SSE_HELPER_F(helper_pmovzxwq, Q, 2, s->W)
+SSE_HELPER_F(helper_pmovzxdq, Q, 2, s->L)
+
+void glue(helper_pmuldq, SUFFIX) (Reg *d, Reg *s)
+{
+    d->Q(0) = (int64_t) (int32_t) d->L(0) * (int32_t) s->L(0);
+    d->Q(1) = (int64_t) (int32_t) d->L(2) * (int32_t) s->L(2);
+}
+
+#define FCMPEQQ(d, s) d == s ? -1 : 0
+SSE_HELPER_Q(helper_pcmpeqq, FCMPEQQ)
+
+void glue(helper_packusdw, SUFFIX) (Reg *d, Reg *s)
+{
+    d->W(0) = satuw((int32_t) d->L(0));
+    d->W(1) = satuw((int32_t) d->L(1));
+    d->W(2) = satuw((int32_t) d->L(2));
+    d->W(3) = satuw((int32_t) d->L(3));
+    d->W(4) = satuw((int32_t) s->L(0));
+    d->W(5) = satuw((int32_t) s->L(1));
+    d->W(6) = satuw((int32_t) s->L(2));
+    d->W(7) = satuw((int32_t) s->L(3));
+}
+
+#define FMINSB(d, s) MIN((int8_t) d, (int8_t) s)
+#define FMINSD(d, s) MIN((int32_t) d, (int32_t) s)
+#define FMAXSB(d, s) MAX((int8_t) d, (int8_t) s)
+#define FMAXSD(d, s) MAX((int32_t) d, (int32_t) s)
+SSE_HELPER_B(helper_pminsb, FMINSB)
+SSE_HELPER_L(helper_pminsd, FMINSD)
+SSE_HELPER_W(helper_pminuw, MIN)
+SSE_HELPER_L(helper_pminud, MIN)
+SSE_HELPER_B(helper_pmaxsb, FMAXSB)
+SSE_HELPER_L(helper_pmaxsd, FMAXSD)
+SSE_HELPER_W(helper_pmaxuw, MAX)
+SSE_HELPER_L(helper_pmaxud, MAX)
+
+#define FMULLD(d, s) (int32_t) d * (int32_t) s
+SSE_HELPER_L(helper_pmulld, FMULLD)
+
+void glue(helper_phminposuw, SUFFIX) (Reg *d, Reg *s)
+{
+    int idx = 0;
+
+    if (s->W(1) < s->W(idx))
+        idx = 1;
+    if (s->W(2) < s->W(idx))
+        idx = 2;
+    if (s->W(3) < s->W(idx))
+        idx = 3;
+    if (s->W(4) < s->W(idx))
+        idx = 4;
+    if (s->W(5) < s->W(idx))
+        idx = 5;
+    if (s->W(6) < s->W(idx))
+        idx = 6;
+    if (s->W(7) < s->W(idx))
+        idx = 7;
+
+    d->Q(1) = 0;
+    d->L(1) = 0;
+    d->W(1) = idx;
+    d->W(0) = s->W(idx);
+}
+
+void glue(helper_roundps, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
+{
+    signed char prev_rounding_mode;
+
+    prev_rounding_mode = env->sse_status.float_rounding_mode;
+    if (!(mode & (1 << 2)))
+        switch (mode & 3) {
+        case 0:
+            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
+            break;
+        case 1:
+            set_float_rounding_mode(float_round_down, &env->sse_status);
+            break;
+        case 2:
+            set_float_rounding_mode(float_round_up, &env->sse_status);
+            break;
+        case 3:
+            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
+            break;
+        }
+
+    d->L(0) = float64_round_to_int(s->L(0), &env->sse_status);
+    d->L(1) = float64_round_to_int(s->L(1), &env->sse_status);
+    d->L(2) = float64_round_to_int(s->L(2), &env->sse_status);
+    d->L(3) = float64_round_to_int(s->L(3), &env->sse_status);
+
+#if 0 /* TODO */
+    if (mode & (1 << 3))
+        set_float_exception_flags(
+                        get_float_exception_flags(&env->sse_status) &
+                        ~float_flag_inexact,
+                        &env->sse_status);
+#endif
+    env->sse_status.float_rounding_mode = prev_rounding_mode;
+}
+
+void glue(helper_roundpd, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
+{
+    signed char prev_rounding_mode;
+
+    prev_rounding_mode = env->sse_status.float_rounding_mode;
+    if (!(mode & (1 << 2)))
+        switch (mode & 3) {
+        case 0:
+            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
+            break;
+        case 1:
+            set_float_rounding_mode(float_round_down, &env->sse_status);
+            break;
+        case 2:
+            set_float_rounding_mode(float_round_up, &env->sse_status);
+            break;
+        case 3:
+            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
+            break;
+        }
+
+    d->Q(0) = float64_round_to_int(s->Q(0), &env->sse_status);
+    d->Q(1) = float64_round_to_int(s->Q(1), &env->sse_status);
+
+#if 0 /* TODO */
+    if (mode & (1 << 3))
+        set_float_exception_flags(
+                        get_float_exception_flags(&env->sse_status) &
+                        ~float_flag_inexact,
+                        &env->sse_status);
+#endif
+    env->sse_status.float_rounding_mode = prev_rounding_mode;
+}
+
+void glue(helper_roundss, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
+{
+    signed char prev_rounding_mode;
+
+    prev_rounding_mode = env->sse_status.float_rounding_mode;
+    if (!(mode & (1 << 2)))
+        switch (mode & 3) {
+        case 0:
+            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
+            break;
+        case 1:
+            set_float_rounding_mode(float_round_down, &env->sse_status);
+            break;
+        case 2:
+            set_float_rounding_mode(float_round_up, &env->sse_status);
+            break;
+        case 3:
+            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
+            break;
+        }
+
+    d->L(0) = float64_round_to_int(s->L(0), &env->sse_status);
+
+#if 0 /* TODO */
+    if (mode & (1 << 3))
+        set_float_exception_flags(
+                        get_float_exception_flags(&env->sse_status) &
+                        ~float_flag_inexact,
+                        &env->sse_status);
+#endif
+    env->sse_status.float_rounding_mode = prev_rounding_mode;
+}
+
+void glue(helper_roundsd, SUFFIX) (Reg *d, Reg *s, uint32_t mode)
+{
+    signed char prev_rounding_mode;
+
+    prev_rounding_mode = env->sse_status.float_rounding_mode;
+    if (!(mode & (1 << 2)))
+        switch (mode & 3) {
+        case 0:
+            set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
+            break;
+        case 1:
+            set_float_rounding_mode(float_round_down, &env->sse_status);
+            break;
+        case 2:
+            set_float_rounding_mode(float_round_up, &env->sse_status);
+            break;
+        case 3:
+            set_float_rounding_mode(float_round_to_zero, &env->sse_status);
+            break;
+        }
+
+    d->Q(0) = float64_round_to_int(s->Q(0), &env->sse_status);
+
+#if 0 /* TODO */
+    if (mode & (1 << 3))
+        set_float_exception_flags(
+                        get_float_exception_flags(&env->sse_status) &
+                        ~float_flag_inexact,
+                        &env->sse_status);
+#endif
+    env->sse_status.float_rounding_mode = prev_rounding_mode;
+}
+
+#define FBLENDP(d, s, m) m ? s : d
+SSE_HELPER_I(helper_blendps, L, 4, FBLENDP)
+SSE_HELPER_I(helper_blendpd, Q, 2, FBLENDP)
+SSE_HELPER_I(helper_pblendw, W, 8, FBLENDP)
+
+void glue(helper_dpps, SUFFIX) (Reg *d, Reg *s, uint32_t mask)
+{
+    float32 iresult = 0 /*float32_zero*/;
+
+    if (mask & (1 << 4))
+        iresult = float32_add(iresult,
+                        float32_mul(d->L(0), s->L(0), &env->sse_status),
+                        &env->sse_status);
+    if (mask & (1 << 5))
+        iresult = float32_add(iresult,
+                        float32_mul(d->L(1), s->L(1), &env->sse_status),
+                        &env->sse_status);
+    if (mask & (1 << 6))
+        iresult = float32_add(iresult,
+                        float32_mul(d->L(2), s->L(2), &env->sse_status),
+                        &env->sse_status);
+    if (mask & (1 << 7))
+        iresult = float32_add(iresult,
+                        float32_mul(d->L(3), s->L(3), &env->sse_status),
+                        &env->sse_status);
+    d->L(0) = (mask & (1 << 0)) ? iresult : 0 /*float32_zero*/;
+    d->L(1) = (mask & (1 << 1)) ? iresult : 0 /*float32_zero*/;
+    d->L(2) = (mask & (1 << 2)) ? iresult : 0 /*float32_zero*/;
+    d->L(3) = (mask & (1 << 3)) ? iresult : 0 /*float32_zero*/;
+}
+
+void glue(helper_dppd, SUFFIX) (Reg *d, Reg *s, uint32_t mask)
+{
+    float64 iresult = 0 /*float64_zero*/;
+
+    if (mask & (1 << 4))
+        iresult = float64_add(iresult,
+                        float64_mul(d->Q(0), s->Q(0), &env->sse_status),
+                        &env->sse_status);
+    if (mask & (1 << 5))
+        iresult = float64_add(iresult,
+                        float64_mul(d->Q(1), s->Q(1), &env->sse_status),
+                        &env->sse_status);
+    d->Q(0) = (mask & (1 << 0)) ? iresult : 0 /*float64_zero*/;
+    d->Q(1) = (mask & (1 << 1)) ? iresult : 0 /*float64_zero*/;
+}
+
+void glue(helper_mpsadbw, SUFFIX) (Reg *d, Reg *s, uint32_t offset)
+{
+    int s0 = (offset & 3) << 2;
+    int d0 = (offset & 4) << 0;
+    int i;
+    Reg r;
+
+    for (i = 0; i < 8; i++, d0++) {
+        r.W(i) = 0;
+        r.W(i) += abs1(d->B(d0 + 0) - s->B(s0 + 0));
+        r.W(i) += abs1(d->B(d0 + 1) - s->B(s0 + 1));
+        r.W(i) += abs1(d->B(d0 + 2) - s->B(s0 + 2));
+        r.W(i) += abs1(d->B(d0 + 3) - s->B(s0 + 3));
+    }
+
+    *d = r;
+}
+
+/* SSE4.2 op helpers */
+/* it's unclear whether signed or unsigned */
+#define FCMPGTQ(d, s) d > s ? -1 : 0
+SSE_HELPER_Q(helper_pcmpgtq, FCMPGTQ)
+
+static inline int pcmp_elen(int reg, uint32_t ctrl)
+{
+    int val;
+
+    /* Presence of REX.W is indicated by a bit higher than 7 set */
+    if (ctrl >> 8)
+        val = abs1((int64_t) env->regs[reg]);
+    else
+        val = abs1((int32_t) env->regs[reg]);
+
+    if (ctrl & 1) {
+        if (val > 8)
+            return 8;
+    } else
+        if (val > 16)
+            return 16;
+
+    return val;
+}
+
+static inline int pcmp_ilen(Reg *r, uint8_t ctrl)
+{
+    int val = 0;
+
+    if (ctrl & 1) {
+        while (val < 8 && r->W(val))
+            val++;
+    } else
+        while (val < 16 && r->B(val))
+            val++;
+
+    return val;
+}
+
+static inline int pcmp_val(Reg *r, uint8_t ctrl, int i)
+{
+    switch ((ctrl >> 0) & 3) {
+    case 0:
+        return r->B(i);
+    case 1:
+        return r->W(i);
+    case 2:
+        return (int8_t) r->B(i);
+    case 3:
+    default:
+        return (int16_t) r->W(i);
+    }
+}
+
+static inline unsigned pcmpxstrx(Reg *d, Reg *s,
+                int8_t ctrl, int valids, int validd)
+{
+    unsigned int res = 0;
+    int v;
+    int j, i;
+    int upper = (ctrl & 1) ? 7 : 15;
+
+    valids--;
+    validd--;
+
+    CC_SRC = (valids < upper ? CC_Z : 0) | (validd < upper ? CC_S : 0);
+
+    switch ((ctrl >> 2) & 3) {
+    case 0:
+        for (j = valids; j >= 0; j--) {
+            res <<= 1;
+            v = pcmp_val(s, ctrl, j);
+            for (i = validd; i >= 0; i--)
+                res |= (v == pcmp_val(d, ctrl, i));
+        }
+        break;
+    case 1:
+        for (j = valids; j >= 0; j--) {
+            res <<= 1;
+            v = pcmp_val(s, ctrl, j);
+            for (i = ((validd - 1) | 1); i >= 0; i -= 2)
+                res |= (pcmp_val(d, ctrl, i - 0) <= v &&
+                        pcmp_val(d, ctrl, i - 1) >= v);
+        }
+        break;
+    case 2:
+        res = (2 << (upper - MAX(valids, validd))) - 1;
+        res <<= MAX(valids, validd) - MIN(valids, validd);
+        for (i = MIN(valids, validd); i >= 0; i--) {
+            res <<= 1;
+            v = pcmp_val(s, ctrl, i);
+            res |= (v == pcmp_val(d, ctrl, i));
+        }
+        break;
+    case 3:
+        for (j = valids - validd; j >= 0; j--) {
+            res <<= 1;
+            res |= 1;
+            for (i = MIN(upper - j, validd); i >= 0; i--)
+                res &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i));
+        }
+        break;
+    }
+
+    switch ((ctrl >> 4) & 3) {
+    case 1:
+        res ^= (2 << upper) - 1;
+        break;
+    case 3:
+        res ^= (2 << valids) - 1;
+        break;
+    }
+
+    if (res)
+       CC_SRC |= CC_C;
+    if (res & 1)
+       CC_SRC |= CC_O;
+
+    return res;
+}
+
+static inline int rffs1(unsigned int val)
+{
+    int ret = 1, hi;
+
+    for (hi = sizeof(val) * 4; hi; hi /= 2)
+        if (val >> hi) {
+            val >>= hi;
+            ret += hi;
+        }
+
+    return ret;
+}
+
+static inline int ffs1(unsigned int val)
+{
+    int ret = 1, hi;
+
+    for (hi = sizeof(val) * 4; hi; hi /= 2)
+        if (val << hi) {
+            val <<= hi;
+            ret += hi;
+        }
+
+    return ret;
+}
+
+void glue(helper_pcmpestri, SUFFIX) (Reg *d, Reg *s, uint32_t ctrl)
+{
+    unsigned int res = pcmpxstrx(d, s, ctrl,
+                    pcmp_elen(R_EDX, ctrl),
+                    pcmp_elen(R_EAX, ctrl));
+
+    if (res)
+        env->regs[R_ECX] = ((ctrl & (1 << 6)) ? rffs1 : ffs1)(res) - 1;
+    else
+        env->regs[R_ECX] = 16 >> (ctrl & (1 << 0));
+}
+
+void glue(helper_pcmpestrm, SUFFIX) (Reg *d, Reg *s, uint32_t ctrl)
+{
+    int i;
+    unsigned int res = pcmpxstrx(d, s, ctrl,
+                    pcmp_elen(R_EDX, ctrl),
+                    pcmp_elen(R_EAX, ctrl));
+
+    if ((ctrl >> 6) & 1) {
+        if (ctrl & 1)
+            for (i = 0; i <= 8; i--, res >>= 1)
+                d->W(i) = (res & 1) ? ~0 : 0;
+        else
+            for (i = 0; i <= 16; i--, res >>= 1)
+                d->B(i) = (res & 1) ? ~0 : 0;
+    } else {
+        d->Q(1) = 0;
+        d->Q(0) = res;
+    }
+}
+
+void glue(helper_pcmpistri, SUFFIX) (Reg *d, Reg *s, uint32_t ctrl)
+{
+    unsigned int res = pcmpxstrx(d, s, ctrl,
+                    pcmp_ilen(s, ctrl),
+                    pcmp_ilen(d, ctrl));
+
+    if (res)
+        env->regs[R_ECX] = ((ctrl & (1 << 6)) ? rffs1 : ffs1)(res) - 1;
+    else
+        env->regs[R_ECX] = 16 >> (ctrl & (1 << 0));
+}
+
+void glue(helper_pcmpistrm, SUFFIX) (Reg *d, Reg *s, uint32_t ctrl)
+{
+    int i;
+    unsigned int res = pcmpxstrx(d, s, ctrl,
+                    pcmp_ilen(s, ctrl),
+                    pcmp_ilen(d, ctrl));
+
+    if ((ctrl >> 6) & 1) {
+        if (ctrl & 1)
+            for (i = 0; i <= 8; i--, res >>= 1)
+                d->W(i) = (res & 1) ? ~0 : 0;
+        else
+            for (i = 0; i <= 16; i--, res >>= 1)
+                d->B(i) = (res & 1) ? ~0 : 0;
+    } else {
+        d->Q(1) = 0;
+        d->Q(0) = res;
+    }
+}
+
+#define CRCPOLY        0x1edc6f41
+#define CRCPOLY_BITREV 0x82f63b78
+target_ulong helper_crc32(uint32_t crc1, target_ulong msg, uint32_t len)
+{
+    target_ulong crc = (msg & ((target_ulong) -1 >>
+                            (TARGET_LONG_BITS - len))) ^ crc1;
+
+    while (len--)
+        crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_BITREV : 0);
+
+    return crc;
+}
+
+#define POPMASK(i)     ((target_ulong) -1 / ((1LL << (1 << i)) + 1))
+#define POPCOUNT(n, i) (n & POPMASK(i)) + ((n >> (1 << i)) & POPMASK(i))
+target_ulong helper_popcnt(target_ulong n, uint32_t type)
+{
+    CC_SRC = n ? 0 : CC_Z;
+
+    n = POPCOUNT(n, 0);
+    n = POPCOUNT(n, 1);
+    n = POPCOUNT(n, 2);
+    n = POPCOUNT(n, 3);
+    if (type == 1)
+        return n & 0xff;
+
+    n = POPCOUNT(n, 4);
+#ifndef TARGET_X86_64
+    return n;
+#else
+    if (type == 2)
+        return n & 0xff;
+
+    return POPCOUNT(n, 5);
+#endif
+}
+#endif
 
 #undef SHIFT

trunk/src/recompiler_new/translate-all.c

@@ -41 +41 @@
 #include "tcg.h"
 
-#ifndef VBOX
 /* code generation context */
 TCGContext tcg_ctx;
-#else
-TCGContext g_tcg_ctx;
-static TCGContext* getCompilerCtx(CPUState *env)
-{
-    /** @todo nike: should be field in CPU env */
-    //return env->tcg_context;
-    return &g_tcg_ctx;
-}
-#endif
 
 uint16_t gen_opc_buf[OPC_BUF_SIZE];
-uint32_t gen_opparam_buf[OPPARAM_BUF_SIZE];
-long gen_labels[OPC_BUF_SIZE];
-int nb_gen_labels;
+TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
 
 target_ulong gen_opc_pc[OPC_BUF_SIZE];
+uint16_t gen_opc_icount[OPC_BUF_SIZE];
 uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
 #if defined(TARGET_I386)
@@ -86 +75 @@
 }
 
-#ifndef VBOX
 void cpu_gen_init()
 {
@@ -93 +81 @@
                   CPU_TEMP_BUF_NLONGS * sizeof(long));
 }
-#else
-void cpu_gen_init(CPUState *env)
-{
-    TCGContext* tcg_ctx = getCompilerCtx(env);
-    tcg_context_init(tcg_ctx);
-    tcg_set_frame(tcg_ctx, TCG_AREG0, offsetof(CPUState, temp_buf),
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
-}
-#endif
 
 /* return non zero if the very first instruction is invalid so that
@@ -110 +89 @@
 */
 int cpu_gen_code(CPUState *env, TranslationBlock *tb,
-                 int max_code_size, int *gen_code_size_ptr)
-{
-#ifdef VBOX
-    TCGContext *s = getCompilerCtx(env);
-#else
+                 int *gen_code_size_ptr)
+{
     TCGContext *s = &tcg_ctx;
-#endif
     uint8_t *gen_code_buf;
     int gen_code_size;
@@ -133 +108 @@
     tcg_func_start(s);
 
-    if (gen_intermediate_code(env, tb) < 0)
-    {
-        RAWEx_ProfileStop(env, STATS_QEMU_COMPILATION);
-        return -1;
-    }
+    gen_intermediate_code(env, tb);
 #else /* !VBOX */
     tcg_func_start(s);
 
-    if (gen_intermediate_code(env, tb) < 0)
-        return -1;
+    gen_intermediate_code(env, tb);
 #endif /* !VBOX */
 
@@ -196 +166 @@
                       void *puc)
 {
-#ifndef VBOX
     TCGContext *s = &tcg_ctx;
-#else
-    TCGContext *s = getCompilerCtx(env);
-#endif
     int j;
     unsigned long tc_ptr;
@@ -213 +179 @@
 
 #ifdef VBOX
-    /** @todo: what's right here? */
-    if (gen_intermediate_code_pc(env, tb) < 0)
-        return -1;
+    gen_intermediate_code_pc(env, tb);
 #else
     gen_intermediate_code_pc(env, tb);