Changeset 102022 in vbox for trunk/src/VBox/VMM/include
- Timestamp:
- Nov 9, 2023 11:38:47 AM (15 months ago)
- Location:
- trunk/src/VBox/VMM/include
- Files:
-
- 1 edited
- 1 copied
-
IEMN8veRecompiler.h (modified) (1 diff)
-
IEMN8veRecompilerEmit.h (copied) (copied from trunk/src/VBox/VMM/include/IEMN8veRecompiler.h ) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/src/VBox/VMM/include/IEMN8veRecompiler.h
r101984 r102022 809 809 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked)) 810 810 811 812 /**813 * Emit a simple marker instruction to more easily tell where something starts814 * in the disassembly.815 */816 DECL_INLINE_THROW(uint32_t)817 iemNativeEmitMarker(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t uInfo)818 {819 #ifdef RT_ARCH_AMD64820 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);821 if (uInfo == 0)822 {823 /* nop */824 pbCodeBuf[off++] = 0x90;825 }826 else827 {828 /* nop [disp32] */829 pbCodeBuf[off++] = 0x0f;830 pbCodeBuf[off++] = 0x1f;831 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, 0, 5);832 pbCodeBuf[off++] = RT_BYTE1(uInfo);833 pbCodeBuf[off++] = RT_BYTE2(uInfo);834 pbCodeBuf[off++] = RT_BYTE3(uInfo);835 pbCodeBuf[off++] = RT_BYTE4(uInfo);836 }837 #elif RT_ARCH_ARM64838 /* nop */839 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);840 pu32CodeBuf[off++] = 0xd503201f;841 842 RT_NOREF(uInfo);843 #else844 # error "port me"845 #endif846 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);847 return off;848 }849 850 851 /*********************************************************************************************************************************852 * Loads, Stores and Related Stuff. *853 *********************************************************************************************************************************/854 855 /**856 * Emits setting a GPR to zero.857 */858 DECL_INLINE_THROW(uint32_t)859 iemNativeEmitGprZero(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr)860 {861 #ifdef RT_ARCH_AMD64862 /* xor gpr32, gpr32 */863 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);864 if (iGpr >= 8)865 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;866 pbCodeBuf[off++] = 0x33;867 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGpr & 7, iGpr & 7);868 869 #elif RT_ARCH_ARM64870 /* mov gpr, #0x0 */871 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);872 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | iGpr;873 874 #else875 # error "port me"876 #endif877 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);878 return off;879 }880 881 882 /**883 * Emits loading a constant into a 64-bit GPR884 */885 DECL_INLINE_THROW(uint32_t)886 iemNativeEmitLoadGprImm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint64_t uImm64)887 {888 if (!uImm64)889 return iemNativeEmitGprZero(pReNative, off, iGpr);890 891 #ifdef RT_ARCH_AMD64892 if (uImm64 <= UINT32_MAX)893 {894 /* mov gpr, imm32 */895 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);896 if (iGpr >= 8)897 pbCodeBuf[off++] = X86_OP_REX_B;898 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);899 pbCodeBuf[off++] = RT_BYTE1(uImm64);900 pbCodeBuf[off++] = RT_BYTE2(uImm64);901 pbCodeBuf[off++] = RT_BYTE3(uImm64);902 pbCodeBuf[off++] = RT_BYTE4(uImm64);903 }904 else905 {906 /* mov gpr, imm64 */907 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 10);908 if (iGpr < 8)909 pbCodeBuf[off++] = X86_OP_REX_W;910 else911 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;912 pbCodeBuf[off++] = 0xb8 + (iGpr & 7);913 pbCodeBuf[off++] = RT_BYTE1(uImm64);914 pbCodeBuf[off++] = RT_BYTE2(uImm64);915 pbCodeBuf[off++] = RT_BYTE3(uImm64);916 pbCodeBuf[off++] = RT_BYTE4(uImm64);917 pbCodeBuf[off++] = RT_BYTE5(uImm64);918 pbCodeBuf[off++] = RT_BYTE6(uImm64);919 pbCodeBuf[off++] = RT_BYTE7(uImm64);920 pbCodeBuf[off++] = RT_BYTE8(uImm64);921 }922 923 #elif RT_ARCH_ARM64924 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);925 926 /*927 * We need to start this sequence with a 'mov grp, imm16, lsl #x' and928 * supply remaining bits using 'movk grp, imm16, lsl #x'.929 *930 * The mov instruction is encoded 0xd2800000 + shift + imm16 + grp,931 * while the movk is 0xf2800000 + shift + imm16 + grp, meaning the diff932 * is 0x20000000 (bit 29). So, we keep this bit in a variable and set it933 * after the first non-zero immediate component so we switch to movk for934 * the remainder.935 */936 uint32_t fMovK = 0;937 /* mov gpr, imm16 */938 uint32_t uImmPart = ((uint32_t)((uImm64 >> 0) & UINT32_C(0xffff)) << 5);939 if (uImmPart)940 {941 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | uImmPart | iGpr;942 fMovK |= RT_BIT_32(29);943 }944 /* mov[k] gpr, imm16, lsl #16 */945 uImmPart = ((uint32_t)((uImm64 >> 16) & UINT32_C(0xffff)) << 5);946 if (uImmPart)947 {948 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(1) << 21) | uImmPart | iGpr;949 fMovK |= RT_BIT_32(29);950 }951 /* mov[k] gpr, imm16, lsl #32 */952 uImmPart = ((uint32_t)((uImm64 >> 32) & UINT32_C(0xffff)) << 5);953 if (uImmPart)954 {955 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(2) << 21) | uImmPart | iGpr;956 fMovK |= RT_BIT_32(29);957 }958 /* mov[k] gpr, imm16, lsl #48 */959 uImmPart = ((uint32_t)((uImm64 >> 48) & UINT32_C(0xffff)) << 5);960 if (uImmPart)961 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | fMovK | (UINT32_C(3) << 21) | uImmPart | iGpr;962 963 /** @todo there is an inverted mask variant we might want to explore if it964 * reduces the number of instructions... */965 /** @todo load into 'w' register instead of 'x' when imm64 <= UINT32_MAX?966 * clang 12.x does that, only to use the 'x' version for the967 * addressing in the following ldr). */968 969 #else970 # error "port me"971 #endif972 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);973 return off;974 }975 976 977 /**978 * Emits loading a constant into a 8-bit GPR979 * @note The AMD64 version does *NOT* clear any bits in the 8..63 range,980 * only the ARM64 version does that.981 */982 DECL_INLINE_THROW(uint32_t)983 iemNativeEmitLoadGpr8Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint8_t uImm8)984 {985 #ifdef RT_ARCH_AMD64986 /* mov gpr, imm8 */987 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);988 if (iGpr >= 8)989 pbCodeBuf[off++] = X86_OP_REX_B;990 else if (iGpr >= 4)991 pbCodeBuf[off++] = X86_OP_REX;992 pbCodeBuf[off++] = 0xb0 + (iGpr & 7);993 pbCodeBuf[off++] = RT_BYTE1(uImm8);994 995 #elif RT_ARCH_ARM64996 /* movz gpr, imm16, lsl #0 */997 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);998 pu32CodeBuf[off++] = UINT32_C(0xd2800000) | (UINT32_C(0) << 21) | ((uint32_t)uImm8 << 5) | iGpr;999 1000 #else1001 # error "port me"1002 #endif1003 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1004 return off;1005 }1006 1007 1008 #ifdef RT_ARCH_AMD641009 /**1010 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.1011 */1012 DECL_FORCE_INLINE(uint32_t)1013 iemNativeEmitGprByVCpuDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint32_t offVCpu)1014 {1015 if (offVCpu < 128)1016 {1017 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);1018 pbCodeBuf[off++] = (uint8_t)(int8_t)offVCpu;1019 }1020 else1021 {1022 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, IEMNATIVE_REG_FIXED_PVMCPU);1023 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offVCpu);1024 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offVCpu);1025 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offVCpu);1026 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offVCpu);1027 }1028 return off;1029 }1030 #elif RT_ARCH_ARM641031 /**1032 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.1033 */1034 DECL_FORCE_INLINE_THROW(uint32_t)1035 iemNativeEmitGprByVCpuLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,1036 uint32_t offVCpu, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)1037 {1038 /*1039 * There are a couple of ldr variants that takes an immediate offset, so1040 * try use those if we can, otherwise we have to use the temporary register1041 * help with the addressing.1042 */1043 if (offVCpu < _4K * cbData && !(offVCpu & (cbData - 1)))1044 {1045 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */1046 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1047 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu / cbData);1048 }1049 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)(_4K * cbData) && !(offVCpu & (cbData - 1)))1050 {1051 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1052 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PCPUMCTX,1053 (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx)) / cbData);1054 }1055 else1056 {1057 /* The offset is too large, so we must load it into a register and use1058 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */1059 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */1060 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, offVCpu);1061 1062 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1063 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, IEMNATIVE_REG_FIXED_PVMCPU,1064 IEMNATIVE_REG_FIXED_TMP0);1065 }1066 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1067 return off;1068 }1069 #endif1070 1071 1072 /**1073 * Emits a 64-bit GPR load of a VCpu value.1074 */1075 DECL_INLINE_THROW(uint32_t)1076 iemNativeEmitLoadGprFromVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1077 {1078 #ifdef RT_ARCH_AMD641079 /* mov reg64, mem64 */1080 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1081 if (iGpr < 8)1082 pbCodeBuf[off++] = X86_OP_REX_W;1083 else1084 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;1085 pbCodeBuf[off++] = 0x8b;1086 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off,iGpr, offVCpu);1087 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1088 1089 #elif RT_ARCH_ARM641090 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));1091 1092 #else1093 # error "port me"1094 #endif1095 return off;1096 }1097 1098 1099 /**1100 * Emits a 32-bit GPR load of a VCpu value.1101 * @note Bits 32 thru 63 in the GPR will be zero after the operation.1102 */1103 DECL_INLINE_THROW(uint32_t)1104 iemNativeEmitLoadGprFromVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1105 {1106 #ifdef RT_ARCH_AMD641107 /* mov reg32, mem32 */1108 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1109 if (iGpr >= 8)1110 pbCodeBuf[off++] = X86_OP_REX_R;1111 pbCodeBuf[off++] = 0x8b;1112 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);1113 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1114 1115 #elif RT_ARCH_ARM641116 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));1117 1118 #else1119 # error "port me"1120 #endif1121 return off;1122 }1123 1124 1125 /**1126 * Emits a 16-bit GPR load of a VCpu value.1127 * @note Bits 16 thru 63 in the GPR will be zero after the operation.1128 */1129 DECL_INLINE_THROW(uint32_t)1130 iemNativeEmitLoadGprFromVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1131 {1132 #ifdef RT_ARCH_AMD641133 /* movzx reg32, mem16 */1134 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);1135 if (iGpr >= 8)1136 pbCodeBuf[off++] = X86_OP_REX_R;1137 pbCodeBuf[off++] = 0x0f;1138 pbCodeBuf[off++] = 0xb7;1139 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);1140 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1141 1142 #elif RT_ARCH_ARM641143 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Half, sizeof(uint16_t));1144 1145 #else1146 # error "port me"1147 #endif1148 return off;1149 }1150 1151 1152 /**1153 * Emits a 8-bit GPR load of a VCpu value.1154 * @note Bits 8 thru 63 in the GPR will be zero after the operation.1155 */1156 DECL_INLINE_THROW(uint32_t)1157 iemNativeEmitLoadGprFromVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1158 {1159 #ifdef RT_ARCH_AMD641160 /* movzx reg32, mem8 */1161 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);1162 if (iGpr >= 8)1163 pbCodeBuf[off++] = X86_OP_REX_R;1164 pbCodeBuf[off++] = 0x0f;1165 pbCodeBuf[off++] = 0xb6;1166 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);1167 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1168 1169 #elif RT_ARCH_ARM641170 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_Ld_Byte, sizeof(uint8_t));1171 1172 #else1173 # error "port me"1174 #endif1175 return off;1176 }1177 1178 1179 /**1180 * Emits a store of a GPR value to a 64-bit VCpu field.1181 */1182 DECL_INLINE_THROW(uint32_t)1183 iemNativeEmitStoreGprToVCpuU64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1184 {1185 #ifdef RT_ARCH_AMD641186 /* mov mem64, reg64 */1187 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1188 if (iGpr < 8)1189 pbCodeBuf[off++] = X86_OP_REX_W;1190 else1191 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;1192 pbCodeBuf[off++] = 0x89;1193 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf,off,iGpr, offVCpu);1194 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1195 1196 #elif RT_ARCH_ARM641197 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));1198 1199 #else1200 # error "port me"1201 #endif1202 return off;1203 }1204 1205 1206 /**1207 * Emits a store of a GPR value to a 32-bit VCpu field.1208 */1209 DECL_INLINE_THROW(uint32_t)1210 iemNativeEmitStoreGprToVCpuU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1211 {1212 #ifdef RT_ARCH_AMD641213 /* mov mem32, reg32 */1214 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1215 if (iGpr >= 8)1216 pbCodeBuf[off++] = X86_OP_REX_R;1217 pbCodeBuf[off++] = 0x89;1218 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);1219 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1220 1221 #elif RT_ARCH_ARM641222 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));1223 1224 #else1225 # error "port me"1226 #endif1227 return off;1228 }1229 1230 1231 /**1232 * Emits a store of a GPR value to a 16-bit VCpu field.1233 */1234 DECL_INLINE_THROW(uint32_t)1235 iemNativeEmitStoreGprToVCpuU16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1236 {1237 #ifdef RT_ARCH_AMD641238 /* mov mem16, reg16 */1239 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);1240 pbCodeBuf[off++] = X86_OP_PRF_SIZE_OP;1241 if (iGpr >= 8)1242 pbCodeBuf[off++] = X86_OP_REX_R;1243 pbCodeBuf[off++] = 0x89;1244 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);1245 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1246 1247 #elif RT_ARCH_ARM641248 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Half, sizeof(uint16_t));1249 1250 #else1251 # error "port me"1252 #endif1253 return off;1254 }1255 1256 1257 /**1258 * Emits a store of a GPR value to a 8-bit VCpu field.1259 */1260 DECL_INLINE_THROW(uint32_t)1261 iemNativeEmitStoreGprToVCpuU8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGpr, uint32_t offVCpu)1262 {1263 #ifdef RT_ARCH_AMD641264 /* mov mem8, reg8 */1265 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1266 if (iGpr >= 8)1267 pbCodeBuf[off++] = X86_OP_REX_R;1268 pbCodeBuf[off++] = 0x88;1269 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGpr, offVCpu);1270 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1271 1272 #elif RT_ARCH_ARM641273 off = iemNativeEmitGprByVCpuLdSt(pReNative, off, iGpr, offVCpu, kArmv8A64InstrLdStType_St_Byte, sizeof(uint8_t));1274 1275 #else1276 # error "port me"1277 #endif1278 return off;1279 }1280 1281 1282 /**1283 * Emits a load effective address to a GRP of a VCpu field.1284 */1285 DECL_INLINE_THROW(uint32_t)1286 iemNativeEmitLeaGprByVCpu(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t offVCpu)1287 {1288 #ifdef RT_ARCH_AMD641289 /* lea gprdst, [rbx + offDisp] */1290 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1291 if (iGprDst < 8)1292 pbCodeBuf[off++] = X86_OP_REX_W;1293 else1294 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;1295 pbCodeBuf[off++] = 0x8d;1296 off = iemNativeEmitGprByVCpuDisp(pbCodeBuf, off, iGprDst, offVCpu);1297 1298 #elif defined(RT_ARCH_ARM64)1299 if (offVCpu < (unsigned)_4K)1300 {1301 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1302 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PVMCPU, offVCpu);1303 }1304 else if (offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx) < (unsigned)_4K)1305 {1306 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1307 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX,1308 offVCpu - RT_UOFFSETOF(VMCPU, cpum.GstCtx));1309 }1310 else1311 {1312 Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);1313 off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offVCpu);1314 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1315 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, IEMNATIVE_REG_FIXED_PCPUMCTX, iGprDst);1316 }1317 1318 #else1319 # error "port me"1320 #endif1321 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1322 return off;1323 }1324 1325 1326 /**1327 * Emits a gprdst = gprsrc load.1328 */1329 DECL_INLINE_THROW(uint32_t)1330 iemNativeEmitLoadGprFromGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)1331 {1332 #ifdef RT_ARCH_AMD641333 /* mov gprdst, gprsrc */1334 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);1335 if ((iGprDst | iGprSrc) >= 8)1336 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W | X86_OP_REX_B1337 : iGprSrc >= 8 ? X86_OP_REX_W | X86_OP_REX_R | X86_OP_REX_B1338 : X86_OP_REX_W | X86_OP_REX_R;1339 else1340 pbCodeBuf[off++] = X86_OP_REX_W;1341 pbCodeBuf[off++] = 0x8b;1342 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);1343 1344 #elif RT_ARCH_ARM641345 /* mov dst, src; alias for: orr dst, xzr, src */1346 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1347 pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_XZR, iGprSrc);1348 1349 #else1350 # error "port me"1351 #endif1352 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1353 return off;1354 }1355 1356 1357 /**1358 * Emits a gprdst = gprsrc[31:0] load.1359 * @note Bits 63 thru 32 are cleared.1360 */1361 DECL_INLINE_THROW(uint32_t)1362 iemNativeEmitLoadGprFromGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)1363 {1364 #ifdef RT_ARCH_AMD641365 /* mov gprdst, gprsrc */1366 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);1367 if ((iGprDst | iGprSrc) >= 8)1368 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B1369 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B1370 : X86_OP_REX_R;1371 pbCodeBuf[off++] = 0x8b;1372 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);1373 1374 #elif RT_ARCH_ARM641375 /* mov dst32, src32; alias for: orr dst32, wzr, src32 */1376 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1377 pu32CodeBuf[off++] = Armv8A64MkInstrOrr(iGprDst, ARMV8_A64_REG_WZR, iGprSrc, false /*f64bit*/);1378 1379 #else1380 # error "port me"1381 #endif1382 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1383 return off;1384 }1385 1386 1387 /**1388 * Emits a gprdst = gprsrc[15:0] load.1389 * @note Bits 63 thru 15 are cleared.1390 */1391 DECL_INLINE_THROW(uint32_t)1392 iemNativeEmitLoadGprFromGpr16(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)1393 {1394 #ifdef RT_ARCH_AMD641395 /* movzx Gv,Ew */1396 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);1397 if ((iGprDst | iGprSrc) >= 8)1398 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B1399 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B1400 : X86_OP_REX_R;1401 pbCodeBuf[off++] = 0x0f;1402 pbCodeBuf[off++] = 0xb7;1403 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);1404 1405 #elif RT_ARCH_ARM641406 /* and gprdst, gprsrc, #0xffff */1407 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1408 # if 11409 Assert(Armv8A64ConvertImmRImmS2Mask32(0x0f, 0) == UINT16_MAX);1410 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x0f, 0, false /*f64Bit*/);1411 # else1412 Assert(Armv8A64ConvertImmRImmS2Mask64(0x4f, 0) == UINT16_MAX);1413 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x4f, 0);1414 # endif1415 1416 #else1417 # error "port me"1418 #endif1419 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1420 return off;1421 }1422 1423 1424 /**1425 * Emits a gprdst = gprsrc[7:0] load.1426 * @note Bits 63 thru 8 are cleared.1427 */1428 DECL_INLINE_THROW(uint32_t)1429 iemNativeEmitLoadGprFromGpr8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)1430 {1431 #ifdef RT_ARCH_AMD641432 /* movzx Gv,Eb */1433 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);1434 if ((iGprDst | iGprSrc) >= 8)1435 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B1436 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B1437 : X86_OP_REX_R;1438 pbCodeBuf[off++] = 0x0f;1439 pbCodeBuf[off++] = 0xb6;1440 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);1441 1442 #elif RT_ARCH_ARM641443 /* and gprdst, gprsrc, #0xff */1444 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1445 # if 11446 Assert(Armv8A64ConvertImmRImmS2Mask32(0x07, 0) == UINT8_MAX);1447 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x07, 0, false /*f64Bit*/);1448 # else1449 Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);1450 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprSrc, 0x47, 0);1451 # endif1452 1453 #else1454 # error "port me"1455 #endif1456 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1457 return off;1458 }1459 1460 #if 0 /** @todo */1461 /**1462 * Emits a gprdst = gprsrc[15:8] load (ah, ch, dh, bh).1463 * @note Bits 63 thru 8 are cleared.1464 */1465 DECL_INLINE_THROW(uint32_t)1466 iemNativeEmitLoadGprFromGpr8hi(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)1467 {1468 #ifdef RT_ARCH_AMD641469 /* movzx Gv,Eb */1470 /** @todo */1471 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);1472 if ((iGprDst | iGprSrc) >= 8)1473 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_B1474 : iGprSrc >= 8 ? X86_OP_REX_R | X86_OP_REX_B1475 : X86_OP_REX_R;1476 pbCodeBuf[off++] = 0x0f;1477 pbCodeBuf[off++] = 0xb6;1478 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);1479 1480 #elif RT_ARCH_ARM641481 /* ubfx gprdst, gprsrc, #8, #8 */1482 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1483 Assert(Armv8A64ConvertImmRImmS2Mask64(0x47, 0) == UINT8_MAX);1484 pu32CodeBuf[off++] = /** @todo ubfx */;1485 1486 #else1487 # error "port me"1488 #endif1489 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1490 return off;1491 }1492 #endif1493 1494 #ifdef RT_ARCH_AMD641495 /**1496 * Common bit of iemNativeEmitLoadGprByBp and friends.1497 */1498 DECL_FORCE_INLINE(uint32_t) iemNativeEmitGprByBpDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, int32_t offDisp,1499 PIEMRECOMPILERSTATE pReNativeAssert)1500 {1501 if (offDisp < 128 && offDisp >= -128)1502 {1503 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, X86_GREG_xBP);1504 pbCodeBuf[off++] = (uint8_t)(int8_t)offDisp;1505 }1506 else1507 {1508 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, iGprReg & 7, X86_GREG_xBP);1509 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);1510 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);1511 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);1512 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);1513 }1514 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNativeAssert, off); RT_NOREF(pReNativeAssert);1515 return off;1516 }1517 #elif defined(RT_ARCH_ARM64)1518 /**1519 * Common bit of iemNativeEmitLoadGprByBp and friends.1520 */1521 DECL_FORCE_INLINE_THROW(uint32_t)1522 iemNativeEmitGprByBpLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,1523 int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)1524 {1525 if ((uint32_t)offDisp < 4096U * cbData && !((uint32_t)offDisp & (cbData - 1)))1526 {1527 /* str w/ unsigned imm12 (scaled) */1528 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1529 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, ARMV8_A64_REG_BP, (uint32_t)offDisp / cbData);1530 }1531 else if (offDisp >= -256 && offDisp <= 256)1532 {1533 /* stur w/ signed imm9 (unscaled) */1534 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1535 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(enmOperation, iGprReg, ARMV8_A64_REG_BP, offDisp);1536 }1537 else1538 {1539 /* Use temporary indexing register. */1540 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);1541 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1542 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, ARMV8_A64_REG_BP,1543 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);1544 }1545 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1546 return off;1547 }1548 #endif1549 1550 1551 /**1552 * Emits a 64-bit GRP load instruction with an BP relative source address.1553 */1554 DECL_INLINE_THROW(uint32_t)1555 iemNativeEmitLoadGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)1556 {1557 #ifdef RT_ARCH_AMD641558 /* mov gprdst, qword [rbp + offDisp] */1559 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1560 if (iGprDst < 8)1561 pbCodeBuf[off++] = X86_OP_REX_W;1562 else1563 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;1564 pbCodeBuf[off++] = 0x8b;1565 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);1566 1567 #elif defined(RT_ARCH_ARM64)1568 return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_St_Dword, sizeof(uint64_t));1569 1570 #else1571 # error "port me"1572 #endif1573 }1574 1575 1576 /**1577 * Emits a 32-bit GRP load instruction with an BP relative source address.1578 */1579 DECL_INLINE_THROW(uint32_t)1580 iemNativeEmitLoadGprByBpU32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)1581 {1582 #ifdef RT_ARCH_AMD641583 /* mov gprdst, dword [rbp + offDisp] */1584 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1585 if (iGprDst >= 8)1586 pbCodeBuf[off++] = X86_OP_REX_R;1587 pbCodeBuf[off++] = 0x8b;1588 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);1589 1590 #elif defined(RT_ARCH_ARM64)1591 return iemNativeEmitGprByBpLdSt(pReNative, off, iGprDst, offDisp, kArmv8A64InstrLdStType_St_Word, sizeof(uint32_t));1592 1593 #else1594 # error "port me"1595 #endif1596 }1597 1598 1599 /**1600 * Emits a load effective address to a GRP with an BP relative source address.1601 */1602 DECL_INLINE_THROW(uint32_t)1603 iemNativeEmitLeaGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t offDisp)1604 {1605 #ifdef RT_ARCH_AMD641606 /* lea gprdst, [rbp + offDisp] */1607 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1608 if (iGprDst < 8)1609 pbCodeBuf[off++] = X86_OP_REX_W;1610 else1611 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;1612 pbCodeBuf[off++] = 0x8d;1613 off = iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprDst, offDisp, pReNative);1614 1615 #elif defined(RT_ARCH_ARM64)1616 if ((uint32_t)offDisp < (unsigned)_4K)1617 {1618 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1619 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)offDisp);1620 }1621 else if ((uint32_t)-offDisp < (unsigned)_4K)1622 {1623 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1624 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, (uint32_t)-offDisp);1625 }1626 else1627 {1628 Assert(iGprDst != IEMNATIVE_REG_FIXED_PVMCPU);1629 off = iemNativeEmitLoadGprImm64(pReNative, off, iGprDst, offDisp >= 0 ? (uint32_t)offDisp : (uint32_t)-offDisp);1630 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1631 if (offDisp >= 0)1632 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);1633 else1634 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, iGprDst, ARMV8_A64_REG_SP, iGprDst);1635 }1636 1637 #else1638 # error "port me"1639 #endif1640 1641 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1642 return off;1643 }1644 1645 1646 /**1647 * Emits a 64-bit GPR store with an BP relative destination address.1648 *1649 * @note May trash IEMNATIVE_REG_FIXED_TMP0.1650 */1651 DECL_INLINE_THROW(uint32_t)1652 iemNativeEmitStoreGprByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint8_t iGprSrc)1653 {1654 #ifdef RT_ARCH_AMD641655 /* mov qword [rbp + offDisp], gprdst */1656 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1657 if (iGprSrc < 8)1658 pbCodeBuf[off++] = X86_OP_REX_W;1659 else1660 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_R;1661 pbCodeBuf[off++] = 0x89;1662 return iemNativeEmitGprByBpDisp(pbCodeBuf, off, iGprSrc, offDisp, pReNative);1663 1664 #elif defined(RT_ARCH_ARM64)1665 if (offDisp >= 0 && offDisp < 4096 * 8 && !((uint32_t)offDisp & 7))1666 {1667 /* str w/ unsigned imm12 (scaled) */1668 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1669 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(kArmv8A64InstrLdStType_St_Dword, iGprSrc,1670 ARMV8_A64_REG_BP, (uint32_t)offDisp / 8);1671 }1672 else if (offDisp >= -256 && offDisp <= 256)1673 {1674 /* stur w/ signed imm9 (unscaled) */1675 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1676 pu32CodeBuf[off++] = Armv8A64MkInstrSturLdur(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP, offDisp);1677 }1678 else1679 {1680 /* Use temporary indexing register. */1681 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, (uint32_t)offDisp);1682 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1683 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(kArmv8A64InstrLdStType_St_Dword, iGprSrc, ARMV8_A64_REG_BP,1684 IEMNATIVE_REG_FIXED_TMP0, kArmv8A64InstrLdStExtend_Sxtw);1685 }1686 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1687 return off;1688 1689 #else1690 # error "Port me!"1691 #endif1692 }1693 1694 1695 /**1696 * Emits a 64-bit immediate store with an BP relative destination address.1697 *1698 * @note May trash IEMNATIVE_REG_FIXED_TMP0.1699 */1700 DECL_INLINE_THROW(uint32_t)1701 iemNativeEmitStoreImm64ByBp(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offDisp, uint64_t uImm64)1702 {1703 #ifdef RT_ARCH_AMD641704 if ((int64_t)uImm64 == (int32_t)uImm64)1705 {1706 /* mov qword [rbp + offDisp], imm32 - sign extended */1707 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 11);1708 pbCodeBuf[off++] = X86_OP_REX_W;1709 pbCodeBuf[off++] = 0xc7;1710 if (offDisp < 128 && offDisp >= -128)1711 {1712 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, 0, X86_GREG_xBP);1713 pbCodeBuf[off++] = (uint8_t)offDisp;1714 }1715 else1716 {1717 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM4, 0, X86_GREG_xBP);1718 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);1719 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);1720 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);1721 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);1722 }1723 pbCodeBuf[off++] = RT_BYTE1(uImm64);1724 pbCodeBuf[off++] = RT_BYTE2(uImm64);1725 pbCodeBuf[off++] = RT_BYTE3(uImm64);1726 pbCodeBuf[off++] = RT_BYTE4(uImm64);1727 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1728 return off;1729 }1730 #endif1731 1732 /* Load tmp0, imm64; Store tmp to bp+disp. */1733 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uImm64);1734 return iemNativeEmitStoreGprByBp(pReNative, off, offDisp, IEMNATIVE_REG_FIXED_TMP0);1735 }1736 1737 1738 #ifdef RT_ARCH_AMD641739 /**1740 * Common bit of iemNativeEmitLoadGprByGpr and friends.1741 */1742 DECL_FORCE_INLINE(uint32_t)1743 iemNativeEmitGprByGprDisp(uint8_t *pbCodeBuf, uint32_t off, uint8_t iGprReg, uint8_t iGprBase, int32_t offDisp)1744 {1745 if (offDisp == 0 && (iGprBase & 7) != X86_GREG_xBP) /* Can use encoding w/o displacement field. */1746 {1747 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM0, iGprReg & 7, iGprBase & 7);1748 if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */1749 pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */1750 }1751 else if (offDisp == (int8_t)offDisp)1752 {1753 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);1754 if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */1755 pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */1756 pbCodeBuf[off++] = (uint8_t)offDisp;1757 }1758 else1759 {1760 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_MEM1, iGprReg & 7, iGprBase & 7);1761 if ((iGprBase & 7) == X86_GREG_xSP) /* for RSP/R12 relative addressing we have to use a SIB byte. */1762 pbCodeBuf[off++] = X86_SIB_MAKE(X86_GREG_xSP, X86_GREG_xSP, 0); /* -> [RSP/R12] */1763 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offDisp);1764 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offDisp);1765 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offDisp);1766 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offDisp);1767 }1768 return off;1769 }1770 #elif RT_ARCH_ARM641771 /**1772 * Common bit of iemNativeEmitLoadGprFromVCpuU64 and friends.1773 */1774 DECL_FORCE_INLINE_THROW(uint32_t)1775 iemNativeEmitGprByGprLdSt(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprReg,1776 uint8_t iGprBase, int32_t offDisp, ARMV8A64INSTRLDSTTYPE enmOperation, unsigned cbData)1777 {1778 /*1779 * There are a couple of ldr variants that takes an immediate offset, so1780 * try use those if we can, otherwise we have to use the temporary register1781 * help with the addressing.1782 */1783 if ((uint32_t)offDisp < _4K * cbData && !((uint32_t)offDisp & (cbData - 1)))1784 {1785 /* Use the unsigned variant of ldr Wt, [<Xn|SP>, #off]. */1786 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1787 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRUOff(enmOperation, iGprReg, iGprBase, (uint32_t)offDisp / cbData);1788 }1789 else1790 {1791 /* The offset is too large, so we must load it into a register and use1792 ldr Wt, [<Xn|SP>, (<Wm>|<Xm>)]. */1793 /** @todo reduce by offVCpu by >> 3 or >> 2? if it saves instructions? */1794 uint8_t const idxTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)offDisp);1795 1796 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1797 pu32CodeBuf[off++] = Armv8A64MkInstrStLdRegIdx(enmOperation, iGprReg, iGprBase, idxTmpReg);1798 1799 iemNativeRegFreeTmpImm(pReNative, idxTmpReg);1800 }1801 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1802 return off;1803 }1804 #endif1805 1806 1807 /**1808 * Emits a 64-bit GPR load via a GPR base address with a displacement.1809 */1810 DECL_INLINE_THROW(uint32_t)1811 iemNativeEmitLoadGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)1812 {1813 #ifdef RT_ARCH_AMD641814 /* mov reg64, mem64 */1815 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);1816 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);1817 pbCodeBuf[off++] = 0x8b;1818 off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);1819 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1820 1821 #elif RT_ARCH_ARM641822 off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Dword, sizeof(uint64_t));1823 1824 #else1825 # error "port me"1826 #endif1827 return off;1828 }1829 1830 1831 /**1832 * Emits a 32-bit GPR load via a GPR base address with a displacement.1833 * @note Bits 63 thru 32 in @a iGprDst will be cleared.1834 */1835 DECL_INLINE_THROW(uint32_t)1836 iemNativeEmitLoadGpr32ByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprBase, int32_t offDisp)1837 {1838 #ifdef RT_ARCH_AMD641839 /* mov reg32, mem32 */1840 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 8);1841 if (iGprDst >= 8 || iGprBase >= 8)1842 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprBase < 8 ? 0 : X86_OP_REX_B);1843 pbCodeBuf[off++] = 0x8b;1844 off = iemNativeEmitGprByGprDisp(pbCodeBuf, off, iGprDst, iGprBase, offDisp);1845 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1846 1847 #elif RT_ARCH_ARM641848 off = iemNativeEmitGprByGprLdSt(pReNative, off, iGprDst, iGprBase, offDisp, kArmv8A64InstrLdStType_Ld_Word, sizeof(uint32_t));1849 1850 #else1851 # error "port me"1852 #endif1853 return off;1854 }1855 1856 1857 /*********************************************************************************************************************************1858 * Subtraction and Additions *1859 *********************************************************************************************************************************/1860 1861 1862 #ifdef RT_ARCH_AMD641863 /**1864 * Emits a 64-bit GPR subtract with a signed immediate subtrahend.1865 */1866 DECL_INLINE_THROW(uint32_t)1867 iemNativeEmitSubGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iSubtrahend)1868 {1869 /* sub gprdst, imm8/imm32 */1870 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);1871 if (iGprDst < 8)1872 pbCodeBuf[off++] = X86_OP_REX_W;1873 else1874 pbCodeBuf[off++] = X86_OP_REX_W | X86_OP_REX_B;1875 if (iSubtrahend < 128 && iSubtrahend >= -128)1876 {1877 pbCodeBuf[off++] = 0x83;1878 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);1879 pbCodeBuf[off++] = (uint8_t)iSubtrahend;1880 }1881 else1882 {1883 pbCodeBuf[off++] = 0x81;1884 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);1885 pbCodeBuf[off++] = RT_BYTE1(iSubtrahend);1886 pbCodeBuf[off++] = RT_BYTE2(iSubtrahend);1887 pbCodeBuf[off++] = RT_BYTE3(iSubtrahend);1888 pbCodeBuf[off++] = RT_BYTE4(iSubtrahend);1889 }1890 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1891 return off;1892 }1893 #endif1894 1895 1896 /**1897 * Emits adding a 64-bit GPR to another, storing the result in the frist.1898 * @note The AMD64 version sets flags.1899 */1900 DECL_INLINE_THROW(uint32_t)1901 iemNativeEmitAddTwoGprs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprAddend)1902 {1903 #if defined(RT_ARCH_AMD64)1904 /* add Gv,Ev */1905 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);1906 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)1907 | (iGprAddend < 8 ? 0 : X86_OP_REX_B);1908 pbCodeBuf[off++] = 0x04;1909 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprAddend & 7);1910 1911 #elif defined(RT_ARCH_ARM64)1912 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1913 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iGprAddend);1914 1915 #else1916 # error "Port me"1917 #endif1918 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1919 return off;1920 }1921 1922 1923 /**1924 * Emits a 64-bit GPR additions with a 8-bit signed immediate.1925 */1926 DECL_INLINE_THROW(uint32_t)1927 iemNativeEmitAddGprImm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)1928 {1929 #if defined(RT_ARCH_AMD64)1930 /* add or inc */1931 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);1932 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;1933 if (iImm8 != 1)1934 {1935 pbCodeBuf[off++] = 0x83;1936 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);1937 pbCodeBuf[off++] = (uint8_t)iImm8;1938 }1939 else1940 {1941 pbCodeBuf[off++] = 0xff;1942 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);1943 }1944 1945 #elif defined(RT_ARCH_ARM64)1946 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1947 if (iImm8 >= 0)1948 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8);1949 else1950 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8);1951 1952 #else1953 # error "Port me"1954 #endif1955 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1956 return off;1957 }1958 1959 1960 /**1961 * Emits a 32-bit GPR additions with a 8-bit signed immediate.1962 * @note Bits 32 thru 63 in the GPR will be zero after the operation.1963 */1964 DECL_INLINE_THROW(uint32_t)1965 iemNativeEmitAddGpr32Imm8(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int8_t iImm8)1966 {1967 #if defined(RT_ARCH_AMD64)1968 /* add or inc */1969 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);1970 if (iGprDst >= 8)1971 pbCodeBuf[off++] = X86_OP_REX_B;1972 if (iImm8 != 1)1973 {1974 pbCodeBuf[off++] = 0x83;1975 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);1976 pbCodeBuf[off++] = (uint8_t)iImm8;1977 }1978 else1979 {1980 pbCodeBuf[off++] = 0xff;1981 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);1982 }1983 1984 #elif defined(RT_ARCH_ARM64)1985 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);1986 if (iImm8 >= 0)1987 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint8_t)iImm8, false /*f64Bit*/);1988 else1989 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint8_t)-iImm8, false /*f64Bit*/);1990 1991 #else1992 # error "Port me"1993 #endif1994 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);1995 return off;1996 }1997 1998 1999 /**2000 * Emits a 64-bit GPR additions with a 64-bit signed addend.2001 */2002 DECL_INLINE_THROW(uint32_t)2003 iemNativeEmitAddGprImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int64_t iAddend)2004 {2005 #if defined(RT_ARCH_AMD64)2006 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)2007 return iemNativeEmitAddGprImm8(pReNative, off, iGprDst, (int8_t)iAddend);2008 2009 if (iAddend <= INT32_MAX && iAddend >= INT32_MIN)2010 {2011 /* add grp, imm32 */2012 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);2013 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;2014 pbCodeBuf[off++] = 0x81;2015 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);2016 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);2017 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);2018 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);2019 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);2020 }2021 else2022 {2023 /* Best to use a temporary register to deal with this in the simplest way: */2024 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);2025 2026 /* add dst, tmpreg */2027 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2028 pbCodeBuf[off++] = (iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_R)2029 | (iTmpReg < 8 ? 0 : X86_OP_REX_B);2030 pbCodeBuf[off++] = 0x03;2031 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iTmpReg & 7);2032 2033 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2034 }2035 2036 #elif defined(RT_ARCH_ARM64)2037 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))2038 {2039 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2040 if (iAddend >= 0)2041 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend);2042 else2043 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend);2044 }2045 else2046 {2047 /* Use temporary register for the immediate. */2048 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint64_t)iAddend);2049 2050 /* add gprdst, gprdst, tmpreg */2051 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2052 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg);2053 2054 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2055 }2056 2057 #else2058 # error "Port me"2059 #endif2060 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2061 return off;2062 }2063 2064 2065 /**2066 * Emits a 32-bit GPR additions with a 32-bit signed immediate.2067 * @note Bits 32 thru 63 in the GPR will be zero after the operation.2068 */2069 DECL_INLINE_THROW(uint32_t)2070 iemNativeEmitAddGpr32Imm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, int32_t iAddend)2071 {2072 #if defined(RT_ARCH_AMD64)2073 if (iAddend <= INT8_MAX && iAddend >= INT8_MIN)2074 return iemNativeEmitAddGpr32Imm8(pReNative, off, iGprDst, (int8_t)iAddend);2075 2076 /* add grp, imm32 */2077 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);2078 if (iGprDst >= 8)2079 pbCodeBuf[off++] = X86_OP_REX_B;2080 pbCodeBuf[off++] = 0x81;2081 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprDst & 7);2082 pbCodeBuf[off++] = RT_BYTE1((uint32_t)iAddend);2083 pbCodeBuf[off++] = RT_BYTE2((uint32_t)iAddend);2084 pbCodeBuf[off++] = RT_BYTE3((uint32_t)iAddend);2085 pbCodeBuf[off++] = RT_BYTE4((uint32_t)iAddend);2086 2087 #elif defined(RT_ARCH_ARM64)2088 if ((uint64_t)RT_ABS(iAddend) < RT_BIT_32(12))2089 {2090 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2091 if (iAddend >= 0)2092 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(false /*fSub*/, iGprDst, iGprDst, (uint32_t)iAddend, false /*f64Bit*/);2093 else2094 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, iGprDst, iGprDst, (uint32_t)-iAddend, false /*f64Bit*/);2095 }2096 else2097 {2098 /* Use temporary register for the immediate. */2099 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, (uint32_t)iAddend);2100 2101 /* add gprdst, gprdst, tmpreg */2102 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2103 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(false /*fSub*/, iGprDst, iGprDst, iTmpReg, false /*f64Bit*/);2104 2105 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2106 }2107 2108 #else2109 # error "Port me"2110 #endif2111 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2112 return off;2113 }2114 2115 2116 2117 /*********************************************************************************************************************************2118 * Bit Operations *2119 *********************************************************************************************************************************/2120 2121 /**2122 * Emits code for clearing bits 16 thru 63 in the GPR.2123 */2124 DECL_INLINE_THROW(uint32_t)2125 iemNativeEmitClear16UpGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst)2126 {2127 #if defined(RT_ARCH_AMD64)2128 /* movzx Gv,Ew */2129 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2130 if (iGprDst >= 8)2131 pbCodeBuf[off++] = X86_OP_REX_B | X86_OP_REX_R;2132 pbCodeBuf[off++] = 0x0f;2133 pbCodeBuf[off++] = 0xb7;2134 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprDst & 7);2135 2136 #elif defined(RT_ARCH_ARM64)2137 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2138 # if 12139 pu32CodeBuf[off++] = Armv8A64MkInstrUxth(iGprDst, iGprDst);2140 # else2141 ///* This produces 0xffff; 0x4f: N=1 imms=001111 (immr=0) => size=64 length=15 */2142 //pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, 0x4f);2143 # endif2144 #else2145 # error "Port me"2146 #endif2147 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2148 return off;2149 }2150 2151 2152 /**2153 * Emits code for AND'ing two 64-bit GPRs.2154 *2155 * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD642156 * and ARM64 hosts.2157 */2158 DECL_INLINE_THROW(uint32_t)2159 iemNativeEmitAndGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)2160 {2161 #if defined(RT_ARCH_AMD64)2162 /* and Gv, Ev */2163 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2164 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);2165 pbCodeBuf[off++] = 0x23;2166 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);2167 RT_NOREF(fSetFlags);2168 2169 #elif defined(RT_ARCH_ARM64)2170 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2171 if (!fSetFlags)2172 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc);2173 else2174 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc);2175 2176 #else2177 # error "Port me"2178 #endif2179 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2180 return off;2181 }2182 2183 2184 /**2185 * Emits code for AND'ing two 32-bit GPRs.2186 */2187 DECL_INLINE_THROW(uint32_t)2188 iemNativeEmitAndGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc, bool fSetFlags = false)2189 {2190 #if defined(RT_ARCH_AMD64)2191 /* and Gv, Ev */2192 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2193 if (iGprDst >= 8 || iGprSrc >= 8)2194 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);2195 pbCodeBuf[off++] = 0x23;2196 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);2197 RT_NOREF(fSetFlags);2198 2199 #elif defined(RT_ARCH_ARM64)2200 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2201 if (fSetFlags)2202 pu32CodeBuf[off++] = Armv8A64MkInstrAnd(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);2203 else2204 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);2205 2206 #else2207 # error "Port me"2208 #endif2209 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2210 return off;2211 }2212 2213 2214 /**2215 * Emits code for AND'ing a 64-bit GPRs with a constant.2216 *2217 * @note When fSetFlags=true, JZ/JNZ jumps can be used afterwards on both AMD642218 * and ARM64 hosts.2219 */2220 DECL_INLINE_THROW(uint32_t)2221 iemNativeEmitAndGprByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint64_t uImm, bool fSetFlags = false)2222 {2223 #if defined(RT_ARCH_AMD64)2224 if ((int64_t)uImm == (int8_t)uImm)2225 {2226 /* and Ev, imm8 */2227 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2228 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_B);2229 pbCodeBuf[off++] = 0x83;2230 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2231 pbCodeBuf[off++] = (uint8_t)uImm;2232 }2233 else if ((int64_t)uImm == (int32_t)uImm)2234 {2235 /* and Ev, imm32 */2236 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);2237 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_B);2238 pbCodeBuf[off++] = 0x81;2239 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2240 pbCodeBuf[off++] = RT_BYTE1(uImm);2241 pbCodeBuf[off++] = RT_BYTE2(uImm);2242 pbCodeBuf[off++] = RT_BYTE3(uImm);2243 pbCodeBuf[off++] = RT_BYTE4(uImm);2244 }2245 else2246 {2247 /* Use temporary register for the 64-bit immediate. */2248 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);2249 off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg);2250 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2251 }2252 RT_NOREF(fSetFlags);2253 2254 #elif defined(RT_ARCH_ARM64)2255 uint32_t uImmR = 0;2256 uint32_t uImmNandS = 0;2257 if (Armv8A64ConvertMask64ToImmRImmS(uImm, &uImmNandS, &uImmR))2258 {2259 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2260 if (!fSetFlags)2261 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR);2262 else2263 pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR);2264 }2265 else2266 {2267 /* Use temporary register for the 64-bit immediate. */2268 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);2269 off = iemNativeEmitAndGprByGpr(pReNative, off, iGprDst, iTmpReg, fSetFlags);2270 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2271 }2272 2273 #else2274 # error "Port me"2275 #endif2276 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2277 return off;2278 }2279 2280 2281 /**2282 * Emits code for AND'ing an 32-bit GPRs with a constant.2283 */2284 DECL_INLINE_THROW(uint32_t)2285 iemNativeEmitAndGpr32ByImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint32_t uImm, bool fSetFlags = false)2286 {2287 #if defined(RT_ARCH_AMD64)2288 /* and Ev, imm */2289 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);2290 if (iGprDst >= 8)2291 pbCodeBuf[off++] = X86_OP_REX_B;2292 if ((int32_t)uImm == (int8_t)uImm)2293 {2294 pbCodeBuf[off++] = 0x83;2295 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2296 pbCodeBuf[off++] = (uint8_t)uImm;2297 }2298 else2299 {2300 pbCodeBuf[off++] = 0x81;2301 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2302 pbCodeBuf[off++] = RT_BYTE1(uImm);2303 pbCodeBuf[off++] = RT_BYTE2(uImm);2304 pbCodeBuf[off++] = RT_BYTE3(uImm);2305 pbCodeBuf[off++] = RT_BYTE4(uImm);2306 }2307 RT_NOREF(fSetFlags);2308 2309 #elif defined(RT_ARCH_ARM64)2310 uint32_t uImmR = 0;2311 uint32_t uImmNandS = 0;2312 if (Armv8A64ConvertMask32ToImmRImmS(uImm, &uImmNandS, &uImmR))2313 {2314 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2315 if (!fSetFlags)2316 pu32CodeBuf[off++] = Armv8A64MkInstrAndImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);2317 else2318 pu32CodeBuf[off++] = Armv8A64MkInstrAndsImm(iGprDst, iGprDst, uImmNandS, uImmR, false /*f64Bit*/);2319 }2320 else2321 {2322 /* Use temporary register for the 64-bit immediate. */2323 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);2324 off = iemNativeEmitAndGpr32ByGpr32(pReNative, off, iGprDst, iTmpReg, fSetFlags);2325 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2326 }2327 2328 #else2329 # error "Port me"2330 #endif2331 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2332 return off;2333 }2334 2335 2336 /**2337 * Emits code for XOR'ing two 64-bit GPRs.2338 */2339 DECL_INLINE_THROW(uint32_t)2340 iemNativeEmitXorGprByGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)2341 {2342 #if defined(RT_ARCH_AMD64)2343 /* and Gv, Ev */2344 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2345 pbCodeBuf[off++] = X86_OP_REX_W | (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);2346 pbCodeBuf[off++] = 0x33;2347 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);2348 2349 #elif defined(RT_ARCH_ARM64)2350 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2351 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc);2352 2353 #else2354 # error "Port me"2355 #endif2356 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2357 return off;2358 }2359 2360 2361 /**2362 * Emits code for XOR'ing two 32-bit GPRs.2363 */2364 DECL_INLINE_THROW(uint32_t)2365 iemNativeEmitXorGpr32ByGpr32(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t iGprSrc)2366 {2367 #if defined(RT_ARCH_AMD64)2368 /* and Gv, Ev */2369 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2370 if (iGprDst >= 8 || iGprSrc >= 8)2371 pbCodeBuf[off++] = (iGprDst < 8 ? 0 : X86_OP_REX_R) | (iGprSrc < 8 ? 0 : X86_OP_REX_B);2372 pbCodeBuf[off++] = 0x33;2373 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprDst & 7, iGprSrc & 7);2374 2375 #elif defined(RT_ARCH_ARM64)2376 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2377 pu32CodeBuf[off++] = Armv8A64MkInstrEor(iGprDst, iGprDst, iGprSrc, false /*f64Bit*/);2378 2379 #else2380 # error "Port me"2381 #endif2382 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2383 return off;2384 }2385 2386 2387 /*********************************************************************************************************************************2388 * Shifting *2389 *********************************************************************************************************************************/2390 2391 /**2392 * Emits code for shifting a GPR a fixed number of bits to the left.2393 */2394 DECL_INLINE_THROW(uint32_t)2395 iemNativeEmitShiftGprLeft(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)2396 {2397 Assert(cShift > 0 && cShift < 64);2398 2399 #if defined(RT_ARCH_AMD64)2400 /* shl dst, cShift */2401 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2402 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;2403 if (cShift != 1)2404 {2405 pbCodeBuf[off++] = 0xc1;2406 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2407 pbCodeBuf[off++] = cShift;2408 }2409 else2410 {2411 pbCodeBuf[off++] = 0xd1;2412 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2413 }2414 2415 #elif defined(RT_ARCH_ARM64)2416 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2417 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift);2418 2419 #else2420 # error "Port me"2421 #endif2422 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2423 return off;2424 }2425 2426 2427 /**2428 * Emits code for shifting a 32-bit GPR a fixed number of bits to the left.2429 */2430 DECL_INLINE_THROW(uint32_t)2431 iemNativeEmitShiftGpr32Left(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)2432 {2433 Assert(cShift > 0 && cShift < 32);2434 2435 #if defined(RT_ARCH_AMD64)2436 /* shl dst, cShift */2437 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2438 if (iGprDst >= 8)2439 pbCodeBuf[off++] = X86_OP_REX_B;2440 if (cShift != 1)2441 {2442 pbCodeBuf[off++] = 0xc1;2443 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2444 pbCodeBuf[off++] = cShift;2445 }2446 else2447 {2448 pbCodeBuf[off++] = 0xd1;2449 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprDst & 7);2450 }2451 2452 #elif defined(RT_ARCH_ARM64)2453 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2454 pu32CodeBuf[off++] = Armv8A64MkInstrLslImm(iGprDst, iGprDst, cShift, false /*64Bit*/);2455 2456 #else2457 # error "Port me"2458 #endif2459 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2460 return off;2461 }2462 2463 2464 /**2465 * Emits code for (unsigned) shifting a GPR a fixed number of bits to the right.2466 */2467 DECL_INLINE_THROW(uint32_t)2468 iemNativeEmitShiftGprRight(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)2469 {2470 Assert(cShift > 0 && cShift < 64);2471 2472 #if defined(RT_ARCH_AMD64)2473 /* shr dst, cShift */2474 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2475 pbCodeBuf[off++] = iGprDst < 8 ? X86_OP_REX_W : X86_OP_REX_W | X86_OP_REX_B;2476 if (cShift != 1)2477 {2478 pbCodeBuf[off++] = 0xc1;2479 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);2480 pbCodeBuf[off++] = cShift;2481 }2482 else2483 {2484 pbCodeBuf[off++] = 0xd1;2485 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);2486 }2487 2488 #elif defined(RT_ARCH_ARM64)2489 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2490 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift);2491 2492 #else2493 # error "Port me"2494 #endif2495 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2496 return off;2497 }2498 2499 2500 /**2501 * Emits code for (unsigned) shifting a 32-bit GPR a fixed number of bits to the2502 * right.2503 */2504 DECL_INLINE_THROW(uint32_t)2505 iemNativeEmitShiftGpr32Right(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprDst, uint8_t cShift)2506 {2507 Assert(cShift > 0 && cShift < 32);2508 2509 #if defined(RT_ARCH_AMD64)2510 /* shr dst, cShift */2511 uint8_t *pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2512 if (iGprDst >= 8)2513 pbCodeBuf[off++] = X86_OP_REX_B;2514 if (cShift != 1)2515 {2516 pbCodeBuf[off++] = 0xc1;2517 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);2518 pbCodeBuf[off++] = cShift;2519 }2520 else2521 {2522 pbCodeBuf[off++] = 0xd1;2523 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 5, iGprDst & 7);2524 }2525 2526 #elif defined(RT_ARCH_ARM64)2527 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2528 pu32CodeBuf[off++] = Armv8A64MkInstrLsrImm(iGprDst, iGprDst, cShift, false /*64Bit*/);2529 2530 #else2531 # error "Port me"2532 #endif2533 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2534 return off;2535 }2536 2537 2538 2539 /*********************************************************************************************************************************2540 * Compare and Testing *2541 *********************************************************************************************************************************/2542 2543 2544 #ifdef RT_ARCH_ARM642545 /**2546 * Emits an ARM64 compare instruction.2547 */2548 DECL_INLINE_THROW(uint32_t)2549 iemNativeEmitCmpArm64(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight,2550 bool f64Bit = true, uint32_t cShift = 0, ARMV8A64INSTRSHIFT enmShift = kArmv8A64InstrShift_Lsr)2551 {2552 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2553 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubReg(true /*fSub*/, ARMV8_A64_REG_XZR /*iRegResult*/, iGprLeft, iGprRight,2554 f64Bit, true /*fSetFlags*/, cShift, enmShift);2555 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2556 return off;2557 }2558 #endif2559 2560 2561 /**2562 * Emits a compare of two 64-bit GPRs, settings status flags/whatever for use2563 * with conditional instruction.2564 */2565 DECL_INLINE_THROW(uint32_t)2566 iemNativeEmitCmpGprWithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)2567 {2568 #ifdef RT_ARCH_AMD642569 /* cmp Gv, Ev */2570 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2571 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);2572 pbCodeBuf[off++] = 0x3b;2573 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);2574 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2575 2576 #elif defined(RT_ARCH_ARM64)2577 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);2578 2579 #else2580 # error "Port me!"2581 #endif2582 return off;2583 }2584 2585 2586 /**2587 * Emits a compare of two 32-bit GPRs, settings status flags/whatever for use2588 * with conditional instruction.2589 */2590 DECL_INLINE_THROW(uint32_t)2591 iemNativeEmitCmpGpr32WithGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint8_t iGprRight)2592 {2593 #ifdef RT_ARCH_AMD642594 /* cmp Gv, Ev */2595 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);2596 if (iGprLeft >= 8 || iGprRight >= 8)2597 pbCodeBuf[off++] = (iGprLeft >= 8 ? X86_OP_REX_R : 0) | (iGprRight >= 8 ? X86_OP_REX_B : 0);2598 pbCodeBuf[off++] = 0x3b;2599 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprLeft & 7, iGprRight & 7);2600 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2601 2602 #elif defined(RT_ARCH_ARM64)2603 off = iemNativeEmitCmpArm64(pReNative, off, iGprLeft, iGprRight, false /*f64Bit*/);2604 2605 #else2606 # error "Port me!"2607 #endif2608 return off;2609 }2610 2611 2612 /**2613 * Emits a compare of a 64-bit GPR with a constant value, settings status2614 * flags/whatever for use with conditional instruction.2615 */2616 DECL_INLINE_THROW(uint32_t)2617 iemNativeEmitCmpGprWithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint64_t uImm)2618 {2619 #ifdef RT_ARCH_AMD642620 if (uImm <= UINT32_C(0xff))2621 {2622 /* cmp Ev, Ib */2623 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 4);2624 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);2625 pbCodeBuf[off++] = 0x83;2626 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);2627 pbCodeBuf[off++] = (uint8_t)uImm;2628 }2629 else if ((int64_t)uImm == (int32_t)uImm)2630 {2631 /* cmp Ev, imm */2632 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);2633 pbCodeBuf[off++] = X86_OP_REX_W | (iGprLeft >= 8 ? X86_OP_REX_B : 0);2634 pbCodeBuf[off++] = 0x81;2635 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);2636 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2637 pbCodeBuf[off++] = RT_BYTE1(uImm);2638 pbCodeBuf[off++] = RT_BYTE2(uImm);2639 pbCodeBuf[off++] = RT_BYTE3(uImm);2640 pbCodeBuf[off++] = RT_BYTE4(uImm);2641 }2642 else2643 {2644 /* Use temporary register for the immediate. */2645 uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);2646 off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);2647 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2648 }2649 2650 #elif defined(RT_ARCH_ARM64)2651 /** @todo guess there are clevere things we can do here... */2652 if (uImm < _4K)2653 {2654 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2655 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,2656 true /*64Bit*/, true /*fSetFlags*/);2657 }2658 else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)2659 {2660 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2661 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,2662 true /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);2663 }2664 else2665 {2666 /* Use temporary register for the immediate. */2667 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);2668 off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iTmpReg);2669 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2670 }2671 2672 #else2673 # error "Port me!"2674 #endif2675 2676 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2677 return off;2678 }2679 2680 2681 /**2682 * Emits a compare of a 32-bit GPR with a constant value, settings status2683 * flags/whatever for use with conditional instruction.2684 */2685 DECL_INLINE_THROW(uint32_t)2686 iemNativeEmitCmpGpr32WithImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprLeft, uint32_t uImm)2687 {2688 #ifdef RT_ARCH_AMD642689 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);2690 if (iGprLeft >= 8)2691 pbCodeBuf[off++] = X86_OP_REX_B;2692 if (uImm <= UINT32_C(0xff))2693 {2694 /* cmp Ev, Ib */2695 pbCodeBuf[off++] = 0x83;2696 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);2697 pbCodeBuf[off++] = (uint8_t)uImm;2698 }2699 else2700 {2701 /* cmp Ev, imm */2702 pbCodeBuf[off++] = 0x81;2703 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 7, iGprLeft & 7);2704 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2705 pbCodeBuf[off++] = RT_BYTE1(uImm);2706 pbCodeBuf[off++] = RT_BYTE2(uImm);2707 pbCodeBuf[off++] = RT_BYTE3(uImm);2708 pbCodeBuf[off++] = RT_BYTE4(uImm);2709 }2710 2711 #elif defined(RT_ARCH_ARM64)2712 /** @todo guess there are clevere things we can do here... */2713 if (uImm < _4K)2714 {2715 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2716 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,2717 false /*64Bit*/, true /*fSetFlags*/);2718 }2719 else if (uImm < RT_BIT_32(12+12) && (uImm & (_4K - 1)) == 0)2720 {2721 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2722 pu32CodeBuf[off++] = Armv8A64MkInstrAddSubUImm12(true /*fSub*/, ARMV8_A64_REG_XZR, iGprLeft, (uint32_t)uImm,2723 false /*64Bit*/, true /*fSetFlags*/, true /*fShift12*/);2724 }2725 else2726 {2727 /* Use temporary register for the immediate. */2728 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, uImm);2729 off = iemNativeEmitCmpGpr32WithGpr(pReNative, off, iGprLeft, iTmpReg);2730 iemNativeRegFreeTmpImm(pReNative, iTmpReg);2731 }2732 2733 #else2734 # error "Port me!"2735 #endif2736 2737 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2738 return off;2739 }2740 2741 2742 2743 /*********************************************************************************************************************************2744 * Branching *2745 *********************************************************************************************************************************/2746 2747 /**2748 * Emits a JMP rel32 / B imm19 to the given label.2749 */2750 DECL_INLINE_THROW(uint32_t)2751 iemNativeEmitJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)2752 {2753 Assert(idxLabel < pReNative->cLabels);2754 2755 #ifdef RT_ARCH_AMD642756 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);2757 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)2758 {2759 uint32_t offRel = pReNative->paLabels[idxLabel].off - (off + 2);2760 if ((int32_t)offRel < 128 && (int32_t)offRel >= -128)2761 {2762 pbCodeBuf[off++] = 0xeb; /* jmp rel8 */2763 pbCodeBuf[off++] = (uint8_t)offRel;2764 }2765 else2766 {2767 offRel -= 3;2768 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */2769 pbCodeBuf[off++] = RT_BYTE1(offRel);2770 pbCodeBuf[off++] = RT_BYTE2(offRel);2771 pbCodeBuf[off++] = RT_BYTE3(offRel);2772 pbCodeBuf[off++] = RT_BYTE4(offRel);2773 }2774 }2775 else2776 {2777 pbCodeBuf[off++] = 0xe9; /* jmp rel32 */2778 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);2779 pbCodeBuf[off++] = 0xfe;2780 pbCodeBuf[off++] = 0xff;2781 pbCodeBuf[off++] = 0xff;2782 pbCodeBuf[off++] = 0xff;2783 }2784 pbCodeBuf[off++] = 0xcc; /* int3 poison */2785 2786 #elif defined(RT_ARCH_ARM64)2787 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2788 if (pReNative->paLabels[idxLabel].off != UINT32_MAX)2789 pu32CodeBuf[off++] = Armv8A64MkInstrB(pReNative->paLabels[idxLabel].off - off);2790 else2791 {2792 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm26At0);2793 pu32CodeBuf[off++] = Armv8A64MkInstrB(-1);2794 }2795 2796 #else2797 # error "Port me!"2798 #endif2799 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2800 return off;2801 }2802 2803 2804 /**2805 * Emits a JMP rel32 / B imm19 to a new undefined label.2806 */2807 DECL_INLINE_THROW(uint32_t)2808 iemNativeEmitJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)2809 {2810 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);2811 return iemNativeEmitJmpToLabel(pReNative, off, idxLabel);2812 }2813 2814 /** Condition type. */2815 #ifdef RT_ARCH_AMD642816 typedef enum IEMNATIVEINSTRCOND : uint8_t2817 {2818 kIemNativeInstrCond_o = 0,2819 kIemNativeInstrCond_no,2820 kIemNativeInstrCond_c,2821 kIemNativeInstrCond_nc,2822 kIemNativeInstrCond_e,2823 kIemNativeInstrCond_ne,2824 kIemNativeInstrCond_be,2825 kIemNativeInstrCond_nbe,2826 kIemNativeInstrCond_s,2827 kIemNativeInstrCond_ns,2828 kIemNativeInstrCond_p,2829 kIemNativeInstrCond_np,2830 kIemNativeInstrCond_l,2831 kIemNativeInstrCond_nl,2832 kIemNativeInstrCond_le,2833 kIemNativeInstrCond_nle2834 } IEMNATIVEINSTRCOND;2835 #elif defined(RT_ARCH_ARM64)2836 typedef ARMV8INSTRCOND IEMNATIVEINSTRCOND;2837 #else2838 # error "Port me!"2839 #endif2840 2841 2842 /**2843 * Emits a Jcc rel32 / B.cc imm19 to the given label (ASSUMED requiring fixup).2844 */2845 DECL_INLINE_THROW(uint32_t)2846 iemNativeEmitJccToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel, IEMNATIVEINSTRCOND enmCond)2847 {2848 Assert(idxLabel < pReNative->cLabels);2849 2850 #ifdef RT_ARCH_AMD642851 /* jcc rel32 */2852 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);2853 pbCodeBuf[off++] = 0x0f;2854 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;2855 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_Rel32, -4);2856 pbCodeBuf[off++] = 0x00;2857 pbCodeBuf[off++] = 0x00;2858 pbCodeBuf[off++] = 0x00;2859 pbCodeBuf[off++] = 0x00;2860 2861 #elif defined(RT_ARCH_ARM64)2862 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);2863 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);2864 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, -1);2865 2866 #else2867 # error "Port me!"2868 #endif2869 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);2870 return off;2871 }2872 2873 2874 /**2875 * Emits a Jcc rel32 / B.cc imm19 to a new label.2876 */2877 DECL_INLINE_THROW(uint32_t)2878 iemNativeEmitJccToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,2879 IEMNATIVELABELTYPE enmLabelType, uint16_t uData, IEMNATIVEINSTRCOND enmCond)2880 {2881 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);2882 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, enmCond);2883 }2884 2885 2886 /**2887 * Emits a JZ/JE rel32 / B.EQ imm19 to the given label.2888 */2889 DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)2890 {2891 #ifdef RT_ARCH_AMD642892 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_e);2893 #elif defined(RT_ARCH_ARM64)2894 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Eq);2895 #else2896 # error "Port me!"2897 #endif2898 }2899 2900 /**2901 * Emits a JZ/JE rel32 / B.EQ imm19 to a new label.2902 */2903 DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,2904 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)2905 {2906 #ifdef RT_ARCH_AMD642907 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_e);2908 #elif defined(RT_ARCH_ARM64)2909 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Eq);2910 #else2911 # error "Port me!"2912 #endif2913 }2914 2915 2916 /**2917 * Emits a JNZ/JNE rel32 / B.NE imm19 to the given label.2918 */2919 DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)2920 {2921 #ifdef RT_ARCH_AMD642922 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_ne);2923 #elif defined(RT_ARCH_ARM64)2924 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ne);2925 #else2926 # error "Port me!"2927 #endif2928 }2929 2930 /**2931 * Emits a JNZ/JNE rel32 / B.NE imm19 to a new label.2932 */2933 DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,2934 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)2935 {2936 #ifdef RT_ARCH_AMD642937 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_ne);2938 #elif defined(RT_ARCH_ARM64)2939 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ne);2940 #else2941 # error "Port me!"2942 #endif2943 }2944 2945 2946 /**2947 * Emits a JBE/JNA rel32 / B.LS imm19 to the given label.2948 */2949 DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)2950 {2951 #ifdef RT_ARCH_AMD642952 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_be);2953 #elif defined(RT_ARCH_ARM64)2954 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Ls);2955 #else2956 # error "Port me!"2957 #endif2958 }2959 2960 /**2961 * Emits a JBE/JNA rel32 / B.LS imm19 to a new label.2962 */2963 DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,2964 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)2965 {2966 #ifdef RT_ARCH_AMD642967 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_be);2968 #elif defined(RT_ARCH_ARM64)2969 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Ls);2970 #else2971 # error "Port me!"2972 #endif2973 }2974 2975 2976 /**2977 * Emits a JA/JNBE rel32 / B.HI imm19 to the given label.2978 */2979 DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t idxLabel)2980 {2981 #ifdef RT_ARCH_AMD642982 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kIemNativeInstrCond_nbe);2983 #elif defined(RT_ARCH_ARM64)2984 return iemNativeEmitJccToLabel(pReNative, off, idxLabel, kArmv8InstrCond_Hi);2985 #else2986 # error "Port me!"2987 #endif2988 }2989 2990 /**2991 * Emits a JA/JNBE rel32 / B.HI imm19 to a new label.2992 */2993 DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,2994 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)2995 {2996 #ifdef RT_ARCH_AMD642997 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kIemNativeInstrCond_nbe);2998 #elif defined(RT_ARCH_ARM64)2999 return iemNativeEmitJccToNewLabel(pReNative, off, enmLabelType, uData, kArmv8InstrCond_Hi);3000 #else3001 # error "Port me!"3002 #endif3003 }3004 3005 3006 /**3007 * Emits a Jcc rel32 / B.cc imm19 with a fixed displacement.3008 * How @a offJmp is applied is are target specific.3009 */3010 DECL_INLINE_THROW(uint32_t)3011 iemNativeEmitJccToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget, IEMNATIVEINSTRCOND enmCond)3012 {3013 #ifdef RT_ARCH_AMD643014 /* jcc rel32 */3015 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 6);3016 if (offTarget < 128 && offTarget >= -128)3017 {3018 pbCodeBuf[off++] = (uint8_t)enmCond | 0x70;3019 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);3020 }3021 else3022 {3023 pbCodeBuf[off++] = 0x0f;3024 pbCodeBuf[off++] = (uint8_t)enmCond | 0x80;3025 pbCodeBuf[off++] = RT_BYTE1((uint32_t)offTarget);3026 pbCodeBuf[off++] = RT_BYTE2((uint32_t)offTarget);3027 pbCodeBuf[off++] = RT_BYTE3((uint32_t)offTarget);3028 pbCodeBuf[off++] = RT_BYTE4((uint32_t)offTarget);3029 }3030 3031 #elif defined(RT_ARCH_ARM64)3032 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);3033 pu32CodeBuf[off++] = Armv8A64MkInstrBCond(enmCond, offTarget);3034 3035 #else3036 # error "Port me!"3037 #endif3038 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);3039 return off;3040 }3041 3042 3043 /**3044 * Emits a JZ/JE rel32 / B.EQ imm19 with a fixed displacement.3045 * How @a offJmp is applied is are target specific.3046 */3047 DECL_INLINE_THROW(uint32_t) iemNativeEmitJzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)3048 {3049 #ifdef RT_ARCH_AMD643050 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_e);3051 #elif defined(RT_ARCH_ARM64)3052 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Eq);3053 #else3054 # error "Port me!"3055 #endif3056 }3057 3058 3059 /**3060 * Emits a JNZ/JNE rel32 / B.NE imm19 with a fixed displacement.3061 * How @a offJmp is applied is are target specific.3062 */3063 DECL_INLINE_THROW(uint32_t) iemNativeEmitJnzToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)3064 {3065 #ifdef RT_ARCH_AMD643066 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_ne);3067 #elif defined(RT_ARCH_ARM64)3068 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ne);3069 #else3070 # error "Port me!"3071 #endif3072 }3073 3074 3075 /**3076 * Emits a JBE/JNA rel32 / B.LS imm19 with a fixed displacement.3077 * How @a offJmp is applied is are target specific.3078 */3079 DECL_INLINE_THROW(uint32_t) iemNativeEmitJbeToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)3080 {3081 #ifdef RT_ARCH_AMD643082 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_be);3083 #elif defined(RT_ARCH_ARM64)3084 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Ls);3085 #else3086 # error "Port me!"3087 #endif3088 }3089 3090 3091 /**3092 * Emits a JA/JNBE rel32 / B.EQ imm19 with a fixed displacement.3093 * How @a offJmp is applied is are target specific.3094 */3095 DECL_INLINE_THROW(uint32_t) iemNativeEmitJaToFixed(PIEMRECOMPILERSTATE pReNative, uint32_t off, int32_t offTarget)3096 {3097 #ifdef RT_ARCH_AMD643098 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kIemNativeInstrCond_nbe);3099 #elif defined(RT_ARCH_ARM64)3100 return iemNativeEmitJccToFixed(pReNative, off, offTarget, kArmv8InstrCond_Hi);3101 #else3102 # error "Port me!"3103 #endif3104 }3105 3106 3107 /**3108 * Fixes up a conditional jump to a fixed label.3109 * @see iemNativeEmitJnzToFixed, iemNativeEmitJzToFixed, ...3110 */3111 DECLINLINE(void) iemNativeFixupFixedJump(PIEMRECOMPILERSTATE pReNative, uint32_t offFixup, uint32_t offTarget)3112 {3113 # if defined(RT_ARCH_AMD64)3114 uint8_t * const pbCodeBuf = pReNative->pInstrBuf;3115 if (pbCodeBuf[offFixup] != 0x0f)3116 {3117 Assert((uint8_t)(pbCodeBuf[offFixup] - 0x70) <= 0x10);3118 pbCodeBuf[offFixup + 1] = (uint8_t)(offTarget - (offFixup + 2));3119 Assert(pbCodeBuf[offFixup + 1] == offTarget - (offFixup + 2));3120 }3121 else3122 {3123 Assert((uint8_t)(pbCodeBuf[offFixup + 1] - 0x80) <= 0x10);3124 uint32_t const offRel32 = offTarget - (offFixup + 6);3125 pbCodeBuf[offFixup + 2] = RT_BYTE1(offRel32);3126 pbCodeBuf[offFixup + 3] = RT_BYTE2(offRel32);3127 pbCodeBuf[offFixup + 4] = RT_BYTE3(offRel32);3128 pbCodeBuf[offFixup + 5] = RT_BYTE4(offRel32);3129 }3130 3131 # elif defined(RT_ARCH_ARM64)3132 uint32_t * const pu32CodeBuf = pReNative->pInstrBuf;3133 3134 int32_t const offDisp = offTarget - offFixup;3135 Assert(offDisp >= -262144 && offDisp < 262144);3136 Assert((pu32CodeBuf[offFixup] & UINT32_C(0xff000000)) == UINT32_C(0x54000000)); /* B.COND + BC.COND */3137 3138 pu32CodeBuf[offFixup] = (pu32CodeBuf[offFixup] & UINT32_C(0xff00001f))3139 | (((uint32_t)offDisp & UINT32_C(0x0007ffff)) << 5);3140 3141 # endif3142 }3143 3144 3145 /**3146 * Internal helper, don't call directly.3147 */3148 DECL_INLINE_THROW(uint32_t)3149 iemNativeEmitTestBitInGprAndJmpToLabelIfCc(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc,3150 uint8_t iBitNo, uint32_t idxLabel, bool fJmpIfSet)3151 {3152 Assert(iBitNo < 64);3153 #ifdef RT_ARCH_AMD643154 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);3155 if (iBitNo < 8)3156 {3157 /* test Eb, imm8 */3158 if (iGprSrc >= 4)3159 pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;3160 pbCodeBuf[off++] = 0xf6;3161 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);3162 pbCodeBuf[off++] = (uint8_t)1 << iBitNo;3163 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_ne : kIemNativeInstrCond_e);3164 }3165 else3166 {3167 /* bt Ev, imm8 */3168 if (iBitNo >= 32)3169 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_B);3170 else if (iGprSrc >= 8)3171 pbCodeBuf[off++] = X86_OP_REX_B;3172 pbCodeBuf[off++] = 0x0f;3173 pbCodeBuf[off++] = 0xba;3174 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 4, iGprSrc & 7);3175 pbCodeBuf[off++] = iBitNo;3176 off = iemNativeEmitJccToLabel(pReNative, off, idxLabel, fJmpIfSet ? kIemNativeInstrCond_c : kIemNativeInstrCond_nc);3177 }3178 3179 #elif defined(RT_ARCH_ARM64)3180 /* Use the TBNZ instruction here. */3181 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);3182 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm14At5);3183 pu32CodeBuf[off++] = Armv8A64MkInstrTbzTbnz(fJmpIfSet, 0, iGprSrc, iBitNo);3184 3185 #else3186 # error "Port me!"3187 #endif3188 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);3189 return off;3190 }3191 3192 3193 /**3194 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _set_ in3195 * @a iGprSrc.3196 *3197 * @note On ARM64 the range is only +/-8191 instructions.3198 */3199 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,3200 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)3201 {3202 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, true /*fJmpIfSet*/);3203 }3204 3205 3206 /**3207 * Emits a jump to @a idxLabel on the condition that bit @a iBitNo _is_ _not_3208 * _set_ in @a iGprSrc.3209 *3210 * @note On ARM64 the range is only +/-8191 instructions.3211 */3212 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestBitInGprAndJmpToLabelIfNotSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,3213 uint8_t iGprSrc, uint8_t iBitNo, uint32_t idxLabel)3214 {3215 return iemNativeEmitTestBitInGprAndJmpToLabelIfCc(pReNative, off, iGprSrc, iBitNo, idxLabel, false /*fJmpIfSet*/);3216 }3217 3218 3219 /**3220 * Emits a test for any of the bits from @a fBits in @a iGprSrc, setting CPU3221 * flags accordingly.3222 */3223 DECL_INLINE_THROW(uint32_t)3224 iemNativeEmitTestAnyBitsInGpr(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, uint64_t fBits)3225 {3226 Assert(fBits != 0);3227 #ifdef RT_ARCH_AMD643228 3229 if (fBits >= UINT32_MAX)3230 {3231 uint8_t iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);3232 3233 /* test Ev,Gv */3234 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 5);3235 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R) | (iTmpReg < 8 ? 0 : X86_OP_REX_B);3236 pbCodeBuf[off++] = 0x85;3237 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 8, iTmpReg & 7);3238 3239 iemNativeRegFreeTmpImm(pReNative, iTmpReg);3240 }3241 else if (fBits <= UINT32_MAX)3242 {3243 /* test Eb, imm8 or test Ev, imm32 */3244 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 7);3245 if (fBits <= UINT8_MAX)3246 {3247 if (iGprSrc >= 4)3248 pbCodeBuf[off++] = iGprSrc >= 8 ? X86_OP_REX_B : X86_OP_REX;3249 pbCodeBuf[off++] = 0xf6;3250 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);3251 pbCodeBuf[off++] = (uint8_t)fBits;3252 }3253 else3254 {3255 if (iGprSrc >= 8)3256 pbCodeBuf[off++] = X86_OP_REX_B;3257 pbCodeBuf[off++] = 0xf7;3258 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 0, iGprSrc & 7);3259 pbCodeBuf[off++] = RT_BYTE1(fBits);3260 pbCodeBuf[off++] = RT_BYTE2(fBits);3261 pbCodeBuf[off++] = RT_BYTE3(fBits);3262 pbCodeBuf[off++] = RT_BYTE4(fBits);3263 }3264 }3265 /** @todo implement me. */3266 else3267 AssertFailedStmt(IEMNATIVE_DO_LONGJMP(pReNative, VERR_IEM_EMIT_CASE_NOT_IMPLEMENTED_1));3268 3269 #elif defined(RT_ARCH_ARM64)3270 3271 if (false)3272 {3273 /** @todo figure out how to work the immr / N:imms constants. */3274 }3275 else3276 {3277 /* ands Zr, iGprSrc, iTmpReg */3278 uint8_t const iTmpReg = iemNativeRegAllocTmpImm(pReNative, &off, fBits);3279 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);3280 pu32CodeBuf[off++] = Armv8A64MkInstrAnds(ARMV8_A64_REG_XZR, iGprSrc, iTmpReg);3281 iemNativeRegFreeTmpImm(pReNative, iTmpReg);3282 }3283 3284 #else3285 # error "Port me!"3286 #endif3287 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);3288 return off;3289 }3290 3291 3292 /**3293 * Emits a jump to @a idxLabel on the condition _any_ of the bits in @a fBits3294 * are set in @a iGprSrc.3295 */3296 DECL_INLINE_THROW(uint32_t)3297 iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfAnySet(PIEMRECOMPILERSTATE pReNative, uint32_t off,3298 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)3299 {3300 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));3301 3302 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);3303 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);3304 3305 return off;3306 }3307 3308 3309 /**3310 * Emits a jump to @a idxLabel on the condition _none_ of the bits in @a fBits3311 * are set in @a iGprSrc.3312 */3313 DECL_INLINE_THROW(uint32_t)3314 iemNativeEmitTestAnyBitsInGprAndJmpToLabelIfNoneSet(PIEMRECOMPILERSTATE pReNative, uint32_t off,3315 uint8_t iGprSrc, uint64_t fBits, uint32_t idxLabel)3316 {3317 Assert(fBits); Assert(!RT_IS_POWER_OF_TWO(fBits));3318 3319 off = iemNativeEmitTestAnyBitsInGpr(pReNative, off, iGprSrc, fBits);3320 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);3321 3322 return off;3323 }3324 3325 3326 /**3327 * Emits code that jumps to @a idxLabel if @a iGprSrc is zero.3328 *3329 * The operand size is given by @a f64Bit.3330 */3331 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGprIsZeroAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3332 uint8_t iGprSrc, bool f64Bit, uint32_t idxLabel)3333 {3334 Assert(idxLabel < pReNative->cLabels);3335 3336 #ifdef RT_ARCH_AMD643337 /* test reg32,reg32 / test reg64,reg64 */3338 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 3);3339 if (f64Bit)3340 pbCodeBuf[off++] = X86_OP_REX_W | (iGprSrc < 8 ? 0 : X86_OP_REX_R | X86_OP_REX_B);3341 else if (iGprSrc >= 8)3342 pbCodeBuf[off++] = X86_OP_REX_R | X86_OP_REX_B;3343 pbCodeBuf[off++] = 0x85;3344 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, iGprSrc & 7, iGprSrc & 7);3345 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);3346 3347 /* jz idxLabel */3348 off = iemNativeEmitJzToLabel(pReNative, off, idxLabel);3349 3350 #elif defined(RT_ARCH_ARM64)3351 uint32_t *pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);3352 iemNativeAddFixup(pReNative, off, idxLabel, kIemNativeFixupType_RelImm19At5);3353 pu32CodeBuf[off++] = Armv8A64MkInstrCbzCbnz(false /*fJmpIfNotZero*/, 0, iGprSrc, f64Bit);3354 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);3355 3356 #else3357 # error "Port me!"3358 #endif3359 return off;3360 }3361 3362 3363 /**3364 * Emits code that jumps to a new label if @a iGprSrc is zero.3365 *3366 * The operand size is given by @a f64Bit.3367 */3368 DECL_INLINE_THROW(uint32_t)3369 iemNativeEmitTestIfGprIsZeroAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t iGprSrc, bool f64Bit,3370 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)3371 {3372 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);3373 return iemNativeEmitTestIfGprIsZeroAndJmpToLabel(pReNative, off, iGprSrc, f64Bit, idxLabel);3374 }3375 3376 3377 /**3378 * Emits code that jumps to the given label if @a iGprLeft and @a iGprRight3379 * differs.3380 */3381 DECL_INLINE_THROW(uint32_t)3382 iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3383 uint8_t iGprLeft, uint8_t iGprRight, uint32_t idxLabel)3384 {3385 off = iemNativeEmitCmpGprWithGpr(pReNative, off, iGprLeft, iGprRight);3386 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);3387 return off;3388 }3389 3390 3391 /**3392 * Emits code that jumps to a new label if @a iGprLeft and @a iGprRight differs.3393 */3394 DECL_INLINE_THROW(uint32_t)3395 iemNativeEmitTestIfGprNotEqualGprAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3396 uint8_t iGprLeft, uint8_t iGprRight,3397 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)3398 {3399 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);3400 return iemNativeEmitTestIfGprNotEqualGprAndJmpToLabel(pReNative, off, iGprLeft, iGprRight, idxLabel);3401 }3402 3403 3404 /**3405 * Emits code that jumps to the given label if @a iGprSrc differs from @a uImm.3406 */3407 DECL_INLINE_THROW(uint32_t)3408 iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3409 uint8_t iGprSrc, uint64_t uImm, uint32_t idxLabel)3410 {3411 off = iemNativeEmitCmpGprWithImm(pReNative, off, iGprSrc, uImm);3412 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);3413 return off;3414 }3415 3416 3417 /**3418 * Emits code that jumps to a new label if @a iGprSrc differs from @a uImm.3419 */3420 DECL_INLINE_THROW(uint32_t)3421 iemNativeEmitTestIfGprNotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3422 uint8_t iGprSrc, uint64_t uImm,3423 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)3424 {3425 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);3426 return iemNativeEmitTestIfGprNotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);3427 }3428 3429 3430 /**3431 * Emits code that jumps to the given label if 32-bit @a iGprSrc differs from3432 * @a uImm.3433 */3434 DECL_INLINE_THROW(uint32_t) iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3435 uint8_t iGprSrc, uint32_t uImm, uint32_t idxLabel)3436 {3437 off = iemNativeEmitCmpGpr32WithImm(pReNative, off, iGprSrc, uImm);3438 off = iemNativeEmitJnzToLabel(pReNative, off, idxLabel);3439 return off;3440 }3441 3442 3443 /**3444 * Emits code that jumps to a new label if 32-bit @a iGprSrc differs from3445 * @a uImm.3446 */3447 DECL_INLINE_THROW(uint32_t)3448 iemNativeEmitTestIfGpr32NotEqualImmAndJmpToNewLabel(PIEMRECOMPILERSTATE pReNative, uint32_t off,3449 uint8_t iGprSrc, uint32_t uImm,3450 IEMNATIVELABELTYPE enmLabelType, uint16_t uData = 0)3451 {3452 uint32_t const idxLabel = iemNativeLabelCreate(pReNative, enmLabelType, UINT32_MAX /*offWhere*/, uData);3453 return iemNativeEmitTestIfGpr32NotEqualImmAndJmpToLabel(pReNative, off, iGprSrc, uImm, idxLabel);3454 }3455 3456 3457 3458 /**3459 * Emits a call to a 64-bit address.3460 */3461 DECL_INLINE_THROW(uint32_t) iemNativeEmitCallImm(PIEMRECOMPILERSTATE pReNative, uint32_t off, uintptr_t uPfn)3462 {3463 #ifdef RT_ARCH_AMD643464 off = iemNativeEmitLoadGprImm64(pReNative, off, X86_GREG_xAX, uPfn);3465 3466 /* call rax */3467 uint8_t * const pbCodeBuf = iemNativeInstrBufEnsure(pReNative, off, 2);3468 pbCodeBuf[off++] = 0xff;3469 pbCodeBuf[off++] = X86_MODRM_MAKE(X86_MOD_REG, 2, X86_GREG_xAX);3470 3471 #elif defined(RT_ARCH_ARM64)3472 off = iemNativeEmitLoadGprImm64(pReNative, off, IEMNATIVE_REG_FIXED_TMP0, uPfn);3473 3474 uint32_t * const pu32CodeBuf = iemNativeInstrBufEnsure(pReNative, off, 1);3475 pu32CodeBuf[off++] = Armv8A64MkInstrBlr(IEMNATIVE_REG_FIXED_TMP0);3476 3477 #else3478 # error "port me"3479 #endif3480 IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(pReNative, off);3481 return off;3482 }3483 3484 3485 811 /** @} */ 3486 812 -
trunk/src/VBox/VMM/include/IEMN8veRecompilerEmit.h
r102011 r102022 1 1 /* $Id$ */ 2 2 /** @file 3 * IEM - Interpreted Execution Manager - Native Recompiler In ternals.3 * IEM - Interpreted Execution Manager - Native Recompiler Inlined Emitters. 4 4 */ 5 5 6 6 /* 7 * Copyright (C) 20 11-2023 Oracle and/or its affiliates.7 * Copyright (C) 2023 Oracle and/or its affiliates. 8 8 * 9 9 * This file is part of VirtualBox base platform packages, as … … 26 26 */ 27 27 28 #ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompiler _h29 #define VMM_INCLUDED_SRC_include_IEMN8veRecompiler _h28 #ifndef VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h 29 #define VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h 30 30 #ifndef RT_WITHOUT_PRAGMA_ONCE 31 31 # pragma once 32 32 #endif 33 33 34 35 /** @defgroup grp_iem_n8ve_re Native Recompiler Internals. 36 * @ingroup grp_iem_int 34 #include "IEMN8veRecompiler.h" 35 36 37 /** @defgroup grp_iem_n8ve_re_inline Native Recompiler Inlined Emitters 38 * @ingroup grp_iem_n8ve_re 37 39 * @{ 38 40 */ 39 40 /** @def IEMNATIVE_WITH_TB_DEBUG_INFO41 * Enables generating internal debug info for better TB disassembly dumping. */42 #if defined(DEBUG) || defined(DOXYGEN_RUNNING)43 # define IEMNATIVE_WITH_TB_DEBUG_INFO44 #endif45 46 47 /** @name Stack Frame Layout48 *49 * @{ */50 /** The size of the area for stack variables and spills and stuff.51 * @note This limit is duplicated in the python script(s). We add 0x40 for52 * alignment padding. */53 #define IEMNATIVE_FRAME_VAR_SIZE (0xc0 + 0x40)54 /** Number of 64-bit variable slots (0x100 / 8 = 32. */55 #define IEMNATIVE_FRAME_VAR_SLOTS (IEMNATIVE_FRAME_VAR_SIZE / 8)56 AssertCompile(IEMNATIVE_FRAME_VAR_SLOTS == 32);57 58 #ifdef RT_ARCH_AMD6459 /** An stack alignment adjustment (between non-volatile register pushes and60 * the stack variable area, so the latter better aligned). */61 # define IEMNATIVE_FRAME_ALIGN_SIZE 862 63 /** Number of stack arguments slots for calls made from the frame. */64 # ifdef RT_OS_WINDOWS65 # define IEMNATIVE_FRAME_STACK_ARG_COUNT 466 # else67 # define IEMNATIVE_FRAME_STACK_ARG_COUNT 268 # endif69 /** Number of any shadow arguments (spill area) for calls we make. */70 # ifdef RT_OS_WINDOWS71 # define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 472 # else73 # define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 074 # endif75 76 /** Frame pointer (RBP) relative offset of the last push. */77 # ifdef RT_OS_WINDOWS78 # define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)79 # else80 # define IEMNATIVE_FP_OFF_LAST_PUSH (5 * -8)81 # endif82 /** Frame pointer (RBP) relative offset of the stack variable area (the lowest83 * address for it). */84 # define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)85 /** Frame pointer (RBP) relative offset of the first stack argument for calls. */86 # define IEMNATIVE_FP_OFF_STACK_ARG0 (IEMNATIVE_FP_OFF_STACK_VARS - IEMNATIVE_FRAME_STACK_ARG_COUNT * 8)87 /** Frame pointer (RBP) relative offset of the second stack argument for calls. */88 # define IEMNATIVE_FP_OFF_STACK_ARG1 (IEMNATIVE_FP_OFF_STACK_ARG0 + 8)89 # ifdef RT_OS_WINDOWS90 /** Frame pointer (RBP) relative offset of the third stack argument for calls. */91 # define IEMNATIVE_FP_OFF_STACK_ARG2 (IEMNATIVE_FP_OFF_STACK_ARG0 + 16)92 /** Frame pointer (RBP) relative offset of the fourth stack argument for calls. */93 # define IEMNATIVE_FP_OFF_STACK_ARG3 (IEMNATIVE_FP_OFF_STACK_ARG0 + 24)94 # endif95 96 # ifdef RT_OS_WINDOWS97 /** Frame pointer (RBP) relative offset of the first incoming shadow argument. */98 # define IEMNATIVE_FP_OFF_IN_SHADOW_ARG0 (16)99 /** Frame pointer (RBP) relative offset of the second incoming shadow argument. */100 # define IEMNATIVE_FP_OFF_IN_SHADOW_ARG1 (24)101 /** Frame pointer (RBP) relative offset of the third incoming shadow argument. */102 # define IEMNATIVE_FP_OFF_IN_SHADOW_ARG2 (32)103 /** Frame pointer (RBP) relative offset of the fourth incoming shadow argument. */104 # define IEMNATIVE_FP_OFF_IN_SHADOW_ARG3 (40)105 # endif106 107 #elif RT_ARCH_ARM64108 /** No alignment padding needed for arm64. */109 # define IEMNATIVE_FRAME_ALIGN_SIZE 0110 /** No stack argument slots, got 8 registers for arguments will suffice. */111 # define IEMNATIVE_FRAME_STACK_ARG_COUNT 0112 /** There are no argument spill area. */113 # define IEMNATIVE_FRAME_SHADOW_ARG_COUNT 0114 115 /** Number of saved registers at the top of our stack frame.116 * This includes the return address and old frame pointer, so x19 thru x30. */117 # define IEMNATIVE_FRAME_SAVE_REG_COUNT (12)118 /** The size of the save registered (IEMNATIVE_FRAME_SAVE_REG_COUNT). */119 # define IEMNATIVE_FRAME_SAVE_REG_SIZE (IEMNATIVE_FRAME_SAVE_REG_COUNT * 8)120 121 /** Frame pointer (BP) relative offset of the last push. */122 # define IEMNATIVE_FP_OFF_LAST_PUSH (7 * -8)123 124 /** Frame pointer (BP) relative offset of the stack variable area (the lowest125 * address for it). */126 # define IEMNATIVE_FP_OFF_STACK_VARS (IEMNATIVE_FP_OFF_LAST_PUSH - IEMNATIVE_FRAME_ALIGN_SIZE - IEMNATIVE_FRAME_VAR_SIZE)127 128 #else129 # error "port me"130 #endif131 /** @} */132 133 134 /** @name Fixed Register Allocation(s)135 * @{ */136 /** @def IEMNATIVE_REG_FIXED_PVMCPU137 * The number of the register holding the pVCpu pointer. */138 /** @def IEMNATIVE_REG_FIXED_PCPUMCTX139 * The number of the register holding the &pVCpu->cpum.GstCtx pointer.140 * @note This not available on AMD64, only ARM64. */141 /** @def IEMNATIVE_REG_FIXED_TMP0142 * Dedicated temporary register.143 * @todo replace this by a register allocator and content tracker. */144 /** @def IEMNATIVE_REG_FIXED_MASK145 * Mask GPRs with fixes assignments, either by us or dictated by the CPU/OS146 * architecture. */147 #if defined(RT_ARCH_AMD64) && !defined(DOXYGEN_RUNNING)148 # define IEMNATIVE_REG_FIXED_PVMCPU X86_GREG_xBX149 # define IEMNATIVE_REG_FIXED_TMP0 X86_GREG_x11150 # define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \151 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) \152 | RT_BIT_32(X86_GREG_xSP) \153 | RT_BIT_32(X86_GREG_xBP) )154 155 #elif defined(RT_ARCH_ARM64) || defined(DOXYGEN_RUNNING)156 # define IEMNATIVE_REG_FIXED_PVMCPU ARMV8_A64_REG_X28157 # define IEMNATIVE_REG_FIXED_PCPUMCTX ARMV8_A64_REG_X27158 # define IEMNATIVE_REG_FIXED_TMP0 ARMV8_A64_REG_X15159 # define IEMNATIVE_REG_FIXED_MASK ( RT_BIT_32(ARMV8_A64_REG_SP) \160 | RT_BIT_32(ARMV8_A64_REG_LR) \161 | RT_BIT_32(ARMV8_A64_REG_BP) \162 | RT_BIT_32(IEMNATIVE_REG_FIXED_PVMCPU) \163 | RT_BIT_32(IEMNATIVE_REG_FIXED_PCPUMCTX) \164 | RT_BIT_32(ARMV8_A64_REG_X18) \165 | RT_BIT_32(IEMNATIVE_REG_FIXED_TMP0) )166 167 #else168 # error "port me"169 #endif170 /** @} */171 172 /** @name Call related registers.173 * @{ */174 /** @def IEMNATIVE_CALL_RET_GREG175 * The return value register. */176 /** @def IEMNATIVE_CALL_ARG_GREG_COUNT177 * Number of arguments in registers. */178 /** @def IEMNATIVE_CALL_ARG0_GREG179 * The general purpose register carrying argument \#0. */180 /** @def IEMNATIVE_CALL_ARG1_GREG181 * The general purpose register carrying argument \#1. */182 /** @def IEMNATIVE_CALL_ARG2_GREG183 * The general purpose register carrying argument \#2. */184 /** @def IEMNATIVE_CALL_ARG3_GREG185 * The general purpose register carrying argument \#3. */186 /** @def IEMNATIVE_CALL_VOLATILE_GREG_MASK187 * Mask of registers the callee will not save and may trash. */188 #ifdef RT_ARCH_AMD64189 # define IEMNATIVE_CALL_RET_GREG X86_GREG_xAX190 191 # ifdef RT_OS_WINDOWS192 # define IEMNATIVE_CALL_ARG_GREG_COUNT 4193 # define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xCX194 # define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xDX195 # define IEMNATIVE_CALL_ARG2_GREG X86_GREG_x8196 # define IEMNATIVE_CALL_ARG3_GREG X86_GREG_x9197 # define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \198 | RT_BIT_32(X86_GREG_xCX) \199 | RT_BIT_32(X86_GREG_xDX) \200 | RT_BIT_32(X86_GREG_x8) \201 | RT_BIT_32(X86_GREG_x9) \202 | RT_BIT_32(X86_GREG_x10) \203 | RT_BIT_32(X86_GREG_x11) )204 # else205 # define IEMNATIVE_CALL_ARG_GREG_COUNT 6206 # define IEMNATIVE_CALL_ARG0_GREG X86_GREG_xDI207 # define IEMNATIVE_CALL_ARG1_GREG X86_GREG_xSI208 # define IEMNATIVE_CALL_ARG2_GREG X86_GREG_xDX209 # define IEMNATIVE_CALL_ARG3_GREG X86_GREG_xCX210 # define IEMNATIVE_CALL_ARG4_GREG X86_GREG_x8211 # define IEMNATIVE_CALL_ARG5_GREG X86_GREG_x9212 # define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(X86_GREG_xAX) \213 | RT_BIT_32(X86_GREG_xCX) \214 | RT_BIT_32(X86_GREG_xDX) \215 | RT_BIT_32(X86_GREG_xDI) \216 | RT_BIT_32(X86_GREG_xSI) \217 | RT_BIT_32(X86_GREG_x8) \218 | RT_BIT_32(X86_GREG_x9) \219 | RT_BIT_32(X86_GREG_x10) \220 | RT_BIT_32(X86_GREG_x11) )221 # endif222 223 #elif defined(RT_ARCH_ARM64)224 # define IEMNATIVE_CALL_RET_GREG ARMV8_A64_REG_X0225 # define IEMNATIVE_CALL_ARG_GREG_COUNT 8226 # define IEMNATIVE_CALL_ARG0_GREG ARMV8_A64_REG_X0227 # define IEMNATIVE_CALL_ARG1_GREG ARMV8_A64_REG_X1228 # define IEMNATIVE_CALL_ARG2_GREG ARMV8_A64_REG_X2229 # define IEMNATIVE_CALL_ARG3_GREG ARMV8_A64_REG_X3230 # define IEMNATIVE_CALL_ARG4_GREG ARMV8_A64_REG_X4231 # define IEMNATIVE_CALL_ARG5_GREG ARMV8_A64_REG_X5232 # define IEMNATIVE_CALL_ARG6_GREG ARMV8_A64_REG_X6233 # define IEMNATIVE_CALL_ARG7_GREG ARMV8_A64_REG_X7234 # define IEMNATIVE_CALL_VOLATILE_GREG_MASK ( RT_BIT_32(ARMV8_A64_REG_X0) \235 | RT_BIT_32(ARMV8_A64_REG_X1) \236 | RT_BIT_32(ARMV8_A64_REG_X2) \237 | RT_BIT_32(ARMV8_A64_REG_X3) \238 | RT_BIT_32(ARMV8_A64_REG_X4) \239 | RT_BIT_32(ARMV8_A64_REG_X5) \240 | RT_BIT_32(ARMV8_A64_REG_X6) \241 | RT_BIT_32(ARMV8_A64_REG_X7) \242 | RT_BIT_32(ARMV8_A64_REG_X8) \243 | RT_BIT_32(ARMV8_A64_REG_X9) \244 | RT_BIT_32(ARMV8_A64_REG_X10) \245 | RT_BIT_32(ARMV8_A64_REG_X11) \246 | RT_BIT_32(ARMV8_A64_REG_X12) \247 | RT_BIT_32(ARMV8_A64_REG_X13) \248 | RT_BIT_32(ARMV8_A64_REG_X14) \249 | RT_BIT_32(ARMV8_A64_REG_X15) \250 | RT_BIT_32(ARMV8_A64_REG_X16) \251 | RT_BIT_32(ARMV8_A64_REG_X17) )252 253 #endif254 255 /** This is the maximum argument count we'll ever be needing. */256 #define IEMNATIVE_CALL_MAX_ARG_COUNT 7257 /** @} */258 259 260 /** @def IEMNATIVE_HST_GREG_COUNT261 * Number of host general purpose registers we tracker. */262 /** @def IEMNATIVE_HST_GREG_MASK263 * Mask corresponding to IEMNATIVE_HST_GREG_COUNT that can be applied to264 * inverted register masks and such to get down to a correct set of regs. */265 #ifdef RT_ARCH_AMD64266 # define IEMNATIVE_HST_GREG_COUNT 16267 # define IEMNATIVE_HST_GREG_MASK UINT32_C(0xffff)268 269 #elif defined(RT_ARCH_ARM64)270 # define IEMNATIVE_HST_GREG_COUNT 32271 # define IEMNATIVE_HST_GREG_MASK UINT32_MAX272 #else273 # error "Port me!"274 #endif275 276 277 /** Native code generator label types. */278 typedef enum279 {280 kIemNativeLabelType_Invalid = 0,281 /* Labels w/o data, only once instance per TB: */282 kIemNativeLabelType_Return,283 kIemNativeLabelType_ReturnBreak,284 kIemNativeLabelType_ReturnWithFlags,285 kIemNativeLabelType_NonZeroRetOrPassUp,286 kIemNativeLabelType_RaiseGp0,287 /* Labels with data, potentially multiple instances per TB: */288 kIemNativeLabelType_If,289 kIemNativeLabelType_Else,290 kIemNativeLabelType_Endif,291 kIemNativeLabelType_CheckIrq,292 kIemNativeLabelType_End293 } IEMNATIVELABELTYPE;294 295 /** Native code generator label definition. */296 typedef struct IEMNATIVELABEL297 {298 /** Code offset if defined, UINT32_MAX if it needs to be generated after/in299 * the epilog. */300 uint32_t off;301 /** The type of label (IEMNATIVELABELTYPE). */302 uint16_t enmType;303 /** Additional label data, type specific. */304 uint16_t uData;305 } IEMNATIVELABEL;306 /** Pointer to a label. */307 typedef IEMNATIVELABEL *PIEMNATIVELABEL;308 309 310 /** Native code generator fixup types. */311 typedef enum312 {313 kIemNativeFixupType_Invalid = 0,314 #if defined(RT_ARCH_AMD64) || defined(RT_ARCH_X86)315 /** AMD64 fixup: PC relative 32-bit with addend in bData. */316 kIemNativeFixupType_Rel32,317 #elif defined(RT_ARCH_ARM64)318 /** ARM64 fixup: PC relative offset at bits 25:0 (B, BL). */319 kIemNativeFixupType_RelImm26At0,320 /** ARM64 fixup: PC relative offset at bits 23:5 (CBZ, CBNZ, B.CC). */321 kIemNativeFixupType_RelImm19At5,322 /** ARM64 fixup: PC relative offset at bits 18:5 (TBZ, TBNZ). */323 kIemNativeFixupType_RelImm14At5,324 #endif325 kIemNativeFixupType_End326 } IEMNATIVEFIXUPTYPE;327 328 /** Native code generator fixup. */329 typedef struct IEMNATIVEFIXUP330 {331 /** Code offset of the fixup location. */332 uint32_t off;333 /** The IEMNATIVELABEL this is a fixup for. */334 uint16_t idxLabel;335 /** The fixup type (IEMNATIVEFIXUPTYPE). */336 uint8_t enmType;337 /** Addend or other data. */338 int8_t offAddend;339 } IEMNATIVEFIXUP;340 /** Pointer to a native code generator fixup. */341 typedef IEMNATIVEFIXUP *PIEMNATIVEFIXUP;342 343 344 /**345 * Guest registers that can be shadowed in GPRs.346 */347 typedef enum IEMNATIVEGSTREG : uint8_t348 {349 kIemNativeGstReg_GprFirst = 0,350 kIemNativeGstReg_GprLast = kIemNativeGstReg_GprFirst + 15,351 kIemNativeGstReg_Pc,352 kIemNativeGstReg_EFlags, /**< 32-bit, includes internal flags. */353 kIemNativeGstReg_SegSelFirst,354 kIemNativeGstReg_SegSelLast = kIemNativeGstReg_SegSelFirst + 5,355 kIemNativeGstReg_SegBaseFirst,356 kIemNativeGstReg_SegBaseLast = kIemNativeGstReg_SegBaseFirst + 5,357 kIemNativeGstReg_SegLimitFirst,358 kIemNativeGstReg_SegLimitLast = kIemNativeGstReg_SegLimitFirst + 5,359 kIemNativeGstReg_End360 } IEMNATIVEGSTREG;361 362 /**363 * Intended use statement for iemNativeRegAllocTmpForGuestReg().364 */365 typedef enum IEMNATIVEGSTREGUSE366 {367 /** The usage is read-only, the register holding the guest register368 * shadow copy will not be modified by the caller. */369 kIemNativeGstRegUse_ReadOnly = 0,370 /** The caller will update the guest register (think: PC += cbInstr).371 * The guest shadow copy will follow the returned register. */372 kIemNativeGstRegUse_ForUpdate,373 /** The caller will use the guest register value as input in a calculation374 * and the host register will be modified.375 * This means that the returned host register will not be marked as a shadow376 * copy of the guest register. */377 kIemNativeGstRegUse_Calculation378 } IEMNATIVEGSTREGUSE;379 380 /**381 * Guest registers (classes) that can be referenced.382 */383 typedef enum IEMNATIVEGSTREGREF : uint8_t384 {385 kIemNativeGstRegRef_Invalid = 0,386 kIemNativeGstRegRef_Gpr,387 kIemNativeGstRegRef_GprHighByte, /**< AH, CH, DH, BH*/388 kIemNativeGstRegRef_EFlags,389 kIemNativeGstRegRef_MxCsr,390 kIemNativeGstRegRef_FpuReg,391 kIemNativeGstRegRef_MReg,392 kIemNativeGstRegRef_XReg,393 //kIemNativeGstRegRef_YReg, - doesn't work.394 kIemNativeGstRegRef_End395 } IEMNATIVEGSTREGREF;396 397 398 /** Variable kinds. */399 typedef enum IEMNATIVEVARKIND : uint8_t400 {401 /** Customary invalid zero value. */402 kIemNativeVarKind_Invalid = 0,403 /** This is either in a register or on the stack. */404 kIemNativeVarKind_Stack,405 /** Immediate value - loaded into register when needed, or can live on the406 * stack if referenced (in theory). */407 kIemNativeVarKind_Immediate,408 /** Variable reference - loaded into register when needed, never stack. */409 kIemNativeVarKind_VarRef,410 /** Guest register reference - loaded into register when needed, never stack. */411 kIemNativeVarKind_GstRegRef,412 /** End of valid values. */413 kIemNativeVarKind_End414 } IEMNATIVEVARKIND;415 416 417 /** Variable or argument. */418 typedef struct IEMNATIVEVAR419 {420 /** The kind of variable. */421 IEMNATIVEVARKIND enmKind;422 /** The variable size in bytes. */423 uint8_t cbVar;424 /** The first stack slot (uint64_t), except for immediate and references425 * where it usually is UINT8_MAX. */426 uint8_t idxStackSlot;427 /** The host register allocated for the variable, UINT8_MAX if not. */428 uint8_t idxReg;429 /** The argument number if argument, UINT8_MAX if regular variable. */430 uint8_t uArgNo;431 /** If referenced, the index of the variable referencing this one, otherwise432 * UINT8_MAX. A referenced variable must only be placed on the stack and433 * must be either kIemNativeVarKind_Stack or kIemNativeVarKind_Immediate. */434 uint8_t idxReferrerVar;435 /** Guest register being shadowed here, kIemNativeGstReg_End(/UINT8_MAX) if not. */436 IEMNATIVEGSTREG enmGstReg;437 uint8_t bAlign;438 439 union440 {441 /** kIemNativeVarKind_Immediate: The immediate value. */442 uint64_t uValue;443 /** kIemNativeVarKind_VarRef: The index of the variable being referenced. */444 uint8_t idxRefVar;445 /** kIemNativeVarKind_GstRegRef: The guest register being referrenced. */446 struct447 {448 /** The class of register. */449 IEMNATIVEGSTREGREF enmClass;450 /** Index within the class. */451 uint8_t idx;452 } GstRegRef;453 } u;454 } IEMNATIVEVAR;455 456 /** What is being kept in a host register. */457 typedef enum IEMNATIVEWHAT : uint8_t458 {459 /** The traditional invalid zero value. */460 kIemNativeWhat_Invalid = 0,461 /** Mapping a variable (IEMNATIVEHSTREG::idxVar). */462 kIemNativeWhat_Var,463 /** Temporary register, this is typically freed when a MC completes. */464 kIemNativeWhat_Tmp,465 /** Call argument w/o a variable mapping. This is free (via466 * IEMNATIVE_CALL_VOLATILE_GREG_MASK) after the call is emitted. */467 kIemNativeWhat_Arg,468 /** Return status code.469 * @todo not sure if we need this... */470 kIemNativeWhat_rc,471 /** The fixed pVCpu (PVMCPUCC) register.472 * @todo consider offsetting this on amd64 to use negative offsets to access473 * more members using 8-byte disp. */474 kIemNativeWhat_pVCpuFixed,475 /** The fixed pCtx (PCPUMCTX) register.476 * @todo consider offsetting this on amd64 to use negative offsets to access477 * more members using 8-byte disp. */478 kIemNativeWhat_pCtxFixed,479 /** Fixed temporary register. */480 kIemNativeWhat_FixedTmp,481 /** Register reserved by the CPU or OS architecture. */482 kIemNativeWhat_FixedReserved,483 /** End of valid values. */484 kIemNativeWhat_End485 } IEMNATIVEWHAT;486 487 /**488 * Host general register entry.489 *490 * The actual allocation status is kept in IEMRECOMPILERSTATE::bmHstRegs.491 *492 * @todo Track immediate values in host registers similarlly to how we track the493 * guest register shadow copies. For it to be real helpful, though,494 * we probably need to know which will be reused and put them into495 * non-volatile registers, otherwise it's going to be more or less496 * restricted to an instruction or two.497 */498 typedef struct IEMNATIVEHSTREG499 {500 /** Set of guest registers this one shadows.501 *502 * Using a bitmap here so we can designate the same host register as a copy503 * for more than one guest register. This is expected to be useful in504 * situations where one value is copied to several registers in a sequence.505 * If the mapping is 1:1, then we'd have to pick which side of a 'MOV SRC,DST'506 * sequence we'd want to let this register follow to be a copy of and there507 * will always be places where we'd be picking the wrong one.508 */509 uint64_t fGstRegShadows;510 /** What is being kept in this register. */511 IEMNATIVEWHAT enmWhat;512 /** Variable index if holding a variable, otherwise UINT8_MAX. */513 uint8_t idxVar;514 /** Alignment padding. */515 uint8_t abAlign[6];516 } IEMNATIVEHSTREG;517 518 519 /**520 * Core state for the native recompiler, that is, things that needs careful521 * handling when dealing with branches.522 */523 typedef struct IEMNATIVECORESTATE524 {525 /** Allocation bitmap for aHstRegs. */526 uint32_t bmHstRegs;527 528 /** Bitmap marking which host register contains guest register shadow copies.529 * This is used during register allocation to try preserve copies. */530 uint32_t bmHstRegsWithGstShadow;531 /** Bitmap marking valid entries in aidxGstRegShadows. */532 uint64_t bmGstRegShadows;533 534 union535 {536 /** Index of variable arguments, UINT8_MAX if not valid. */537 uint8_t aidxArgVars[8];538 /** For more efficient resetting. */539 uint64_t u64ArgVars;540 };541 542 /** Allocation bitmap for the stack. */543 uint32_t bmStack;544 /** Allocation bitmap for aVars. */545 uint32_t bmVars;546 547 /** Maps a guest register to a host GPR (index by IEMNATIVEGSTREG).548 * Entries are only valid if the corresponding bit in bmGstRegShadows is set.549 * (A shadow copy of a guest register can only be held in a one host register,550 * there are no duplicate copies or ambiguities like that). */551 uint8_t aidxGstRegShadows[kIemNativeGstReg_End];552 553 /** Host register allocation tracking. */554 IEMNATIVEHSTREG aHstRegs[IEMNATIVE_HST_GREG_COUNT];555 556 /** Variables and arguments. */557 IEMNATIVEVAR aVars[9];558 } IEMNATIVECORESTATE;559 /** Pointer to core state. */560 typedef IEMNATIVECORESTATE *PIEMNATIVECORESTATE;561 /** Pointer to const core state. */562 typedef IEMNATIVECORESTATE const *PCIEMNATIVECORESTATE;563 564 565 /**566 * Conditional stack entry.567 */568 typedef struct IEMNATIVECOND569 {570 /** Set if we're in the "else" part, clear if we're in the "if" before it. */571 bool fInElse;572 /** The label for the IEM_MC_ELSE. */573 uint32_t idxLabelElse;574 /** The label for the IEM_MC_ENDIF. */575 uint32_t idxLabelEndIf;576 /** The initial state snapshot as the if-block starts executing. */577 IEMNATIVECORESTATE InitialState;578 /** The state snapshot at the end of the if-block. */579 IEMNATIVECORESTATE IfFinalState;580 } IEMNATIVECOND;581 /** Pointer to a condition stack entry. */582 typedef IEMNATIVECOND *PIEMNATIVECOND;583 584 585 /**586 * Native recompiler state.587 */588 typedef struct IEMRECOMPILERSTATE589 {590 /** Size of the buffer that pbNativeRecompileBufR3 points to in591 * IEMNATIVEINSTR units. */592 uint32_t cInstrBufAlloc;593 #ifdef VBOX_STRICT594 /** Strict: How far the last iemNativeInstrBufEnsure() checked. */595 uint32_t offInstrBufChecked;596 #else597 uint32_t uPadding1; /* We don't keep track of the size here... */598 #endif599 /** Fixed temporary code buffer for native recompilation. */600 PIEMNATIVEINSTR pInstrBuf;601 602 /** Bitmaps with the label types used. */603 uint64_t bmLabelTypes;604 /** Actual number of labels in paLabels. */605 uint32_t cLabels;606 /** Max number of entries allowed in paLabels before reallocating it. */607 uint32_t cLabelsAlloc;608 /** Labels defined while recompiling (referenced by fixups). */609 PIEMNATIVELABEL paLabels;610 611 /** Actual number of fixups paFixups. */612 uint32_t cFixups;613 /** Max number of entries allowed in paFixups before reallocating it. */614 uint32_t cFixupsAlloc;615 /** Buffer used by the recompiler for recording fixups when generating code. */616 PIEMNATIVEFIXUP paFixups;617 618 #ifdef IEMNATIVE_WITH_TB_DEBUG_INFO619 /** Number of debug info entries allocated for pDbgInfo. */620 uint32_t cDbgInfoAlloc;621 uint32_t uPadding;622 /** Debug info. */623 PIEMTBDBG pDbgInfo;624 #endif625 626 /** The translation block being recompiled. */627 PCIEMTB pTbOrg;628 629 /** The current condition stack depth (aCondStack). */630 uint8_t cCondDepth;631 uint8_t bPadding2;632 /** Condition sequence number (for generating unique labels). */633 uint16_t uCondSeqNo;634 /** Check IRQ seqeunce number (for generating unique lables). */635 uint16_t uCheckIrqSeqNo;636 uint8_t bPadding3;637 638 /** The argument count + hidden regs from the IEM_MC_BEGIN statement. */639 uint8_t cArgs;640 /** The IEM_CIMPL_F_XXX flags from the IEM_MC_BEGIN statement. */641 uint32_t fCImpl;642 /** The IEM_MC_F_XXX flags from the IEM_MC_BEGIN statement. */643 uint32_t fMc;644 645 /** Core state requiring care with branches. */646 IEMNATIVECORESTATE Core;647 648 /** The condition nesting stack. */649 IEMNATIVECOND aCondStack[2];650 651 #ifndef IEM_WITH_THROW_CATCH652 /** Pointer to the setjmp/longjmp buffer if we're not using C++ exceptions653 * for recompilation error handling. */654 jmp_buf JmpBuf;655 #endif656 } IEMRECOMPILERSTATE;657 /** Pointer to a native recompiler state. */658 typedef IEMRECOMPILERSTATE *PIEMRECOMPILERSTATE;659 660 661 /** @def IEMNATIVE_TRY_SETJMP662 * Wrapper around setjmp / try, hiding all the ugly differences.663 *664 * @note Use with extreme care as this is a fragile macro.665 * @param a_pReNative The native recompile state.666 * @param a_rcTarget The variable that should receive the status code in case667 * of a longjmp/throw.668 */669 /** @def IEMNATIVE_CATCH_LONGJMP_BEGIN670 * Start wrapper for catch / setjmp-else.671 *672 * This will set up a scope.673 *674 * @note Use with extreme care as this is a fragile macro.675 * @param a_pReNative The native recompile state.676 * @param a_rcTarget The variable that should receive the status code in case677 * of a longjmp/throw.678 */679 /** @def IEMNATIVE_CATCH_LONGJMP_END680 * End wrapper for catch / setjmp-else.681 *682 * This will close the scope set up by IEMNATIVE_CATCH_LONGJMP_BEGIN and clean683 * up the state.684 *685 * @note Use with extreme care as this is a fragile macro.686 * @param a_pReNative The native recompile state.687 */688 /** @def IEMNATIVE_DO_LONGJMP689 *690 * Wrapper around longjmp / throw.691 *692 * @param a_pReNative The native recompile state.693 * @param a_rc The status code jump back with / throw.694 */695 #ifdef IEM_WITH_THROW_CATCH696 # define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \697 a_rcTarget = VINF_SUCCESS; \698 try699 # define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \700 catch (int rcThrown) \701 { \702 a_rcTarget = rcThrown703 # define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \704 } \705 ((void)0)706 # define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) throw int(a_rc)707 #else /* !IEM_WITH_THROW_CATCH */708 # define IEMNATIVE_TRY_SETJMP(a_pReNative, a_rcTarget) \709 if ((a_rcTarget = setjmp((a_pReNative)->JmpBuf)) == 0)710 # define IEMNATIVE_CATCH_LONGJMP_BEGIN(a_pReNative, a_rcTarget) \711 else \712 { \713 ((void)0)714 # define IEMNATIVE_CATCH_LONGJMP_END(a_pReNative) \715 }716 # define IEMNATIVE_DO_LONGJMP(a_pReNative, a_rc) longjmp((a_pReNative)->JmpBuf, (a_rc))717 #endif /* !IEM_WITH_THROW_CATCH */718 719 720 /**721 * Native recompiler worker for a threaded function.722 *723 * @returns New code buffer offset; throws VBox status code in case of a failure.724 * @param pReNative The native recompiler state.725 * @param off The current code buffer offset.726 * @param pCallEntry The threaded call entry.727 *728 * @note This may throw/longjmp VBox status codes (int) to abort compilation, so no RT_NOEXCEPT!729 */730 typedef uint32_t (VBOXCALL FNIEMNATIVERECOMPFUNC)(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry);731 /** Pointer to a native recompiler worker for a threaded function. */732 typedef FNIEMNATIVERECOMPFUNC *PFNIEMNATIVERECOMPFUNC;733 734 /** Defines a native recompiler worker for a threaded function.735 * @see FNIEMNATIVERECOMPFUNC */736 #define IEM_DECL_IEMNATIVERECOMPFUNC_DEF(a_Name) \737 uint32_t VBOXCALL a_Name(PIEMRECOMPILERSTATE pReNative, uint32_t off, PCIEMTHRDEDCALLENTRY pCallEntry)738 739 /** Prototypes a native recompiler function for a threaded function.740 * @see FNIEMNATIVERECOMPFUNC */741 #define IEM_DECL_IEMNATIVERECOMPFUNC_PROTO(a_Name) FNIEMNATIVERECOMPFUNC a_Name742 743 DECL_HIDDEN_THROW(uint32_t) iemNativeLabelCreate(PIEMRECOMPILERSTATE pReNative, IEMNATIVELABELTYPE enmType,744 uint32_t offWhere = UINT32_MAX, uint16_t uData = 0);745 DECL_HIDDEN_THROW(void) iemNativeLabelDefine(PIEMRECOMPILERSTATE pReNative, uint32_t idxLabel, uint32_t offWhere);746 DECL_HIDDEN_THROW(void) iemNativeAddFixup(PIEMRECOMPILERSTATE pReNative, uint32_t offWhere, uint32_t idxLabel,747 IEMNATIVEFIXUPTYPE enmType, int8_t offAddend = 0);748 DECL_HIDDEN_THROW(PIEMNATIVEINSTR) iemNativeInstrBufEnsureSlow(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq);749 750 DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmp(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, bool fPreferVolatile = true);751 DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpImm(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint64_t uImm,752 bool fPreferVolatile = true);753 DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestReg(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,754 IEMNATIVEGSTREG enmGstReg, IEMNATIVEGSTREGUSE enmIntendedUse);755 DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocTmpForGuestRegIfAlreadyPresent(PIEMRECOMPILERSTATE pReNative, uint32_t *poff,756 IEMNATIVEGSTREG enmGstReg);757 758 DECL_HIDDEN_THROW(uint8_t) iemNativeRegAllocVar(PIEMRECOMPILERSTATE pReNative, uint32_t *poff, uint8_t idxVar);759 DECL_HIDDEN_THROW(uint32_t) iemNativeRegAllocArgs(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t cArgs);760 DECL_HIDDEN_THROW(uint8_t) iemNativeRegAssignRc(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg);761 DECLHIDDEN(void) iemNativeRegFree(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;762 DECLHIDDEN(void) iemNativeRegFreeTmp(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;763 DECLHIDDEN(void) iemNativeRegFreeTmpImm(PIEMRECOMPILERSTATE pReNative, uint8_t idxHstReg) RT_NOEXCEPT;764 DECLHIDDEN(void) iemNativeRegFreeAndFlushMask(PIEMRECOMPILERSTATE pReNative, uint32_t fHstRegMask) RT_NOEXCEPT;765 DECL_HIDDEN_THROW(uint32_t) iemNativeRegFlushPendingWrites(PIEMRECOMPILERSTATE pReNative, uint32_t off);766 767 DECL_HIDDEN_THROW(uint32_t) iemNativeEmitLoadGprWithGstShadowReg(PIEMRECOMPILERSTATE pReNative, uint32_t off,768 uint8_t idxHstReg, IEMNATIVEGSTREG enmGstReg);769 DECL_HIDDEN_THROW(uint32_t) iemNativeEmitCheckCallRetAndPassUp(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint8_t idxInstr);770 771 extern DECL_HIDDEN_DATA(const char * const) g_apszIemNativeHstRegNames[];772 773 774 /**775 * Ensures that there is sufficient space in the instruction output buffer.776 *777 * This will reallocate the buffer if needed and allowed.778 *779 * @note Always use IEMNATIVE_ASSERT_INSTR_BUF_ENSURE when done to check the780 * allocation size.781 *782 * @returns Pointer to the instruction output buffer on success; throws VBox783 * status code on failure, so no need to check it.784 * @param pReNative The native recompile state.785 * @param off Current instruction offset. Works safely for UINT32_MAX786 * as well.787 * @param cInstrReq Number of instruction about to be added. It's okay to788 * overestimate this a bit.789 */790 DECL_FORCE_INLINE_THROW(PIEMNATIVEINSTR)791 iemNativeInstrBufEnsure(PIEMRECOMPILERSTATE pReNative, uint32_t off, uint32_t cInstrReq)792 {793 uint64_t const offChecked = off + (uint64_t)cInstrReq; /** @todo may reconsider the need for UINT32_MAX safety... */794 if (RT_LIKELY(offChecked <= pReNative->cInstrBufAlloc))795 {796 #ifdef VBOX_STRICT797 pReNative->offInstrBufChecked = offChecked;798 #endif799 return pReNative->pInstrBuf;800 }801 return iemNativeInstrBufEnsureSlow(pReNative, off, cInstrReq);802 }803 804 /**805 * Checks that we didn't exceed the space requested in the last806 * iemNativeInstrBufEnsure() call. */807 #define IEMNATIVE_ASSERT_INSTR_BUF_ENSURE(a_pReNative, a_off) \808 AssertMsg((a_off) <= (a_pReNative)->offInstrBufChecked, \809 ("off=%#x offInstrBufChecked=%#x\n", (a_off), (a_pReNative)->offInstrBufChecked))810 811 41 812 42 /** … … 3485 2715 /** @} */ 3486 2716 3487 #endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompiler _h */3488 2717 #endif /* !VMM_INCLUDED_SRC_include_IEMN8veRecompilerEmit_h */ 2718
Note:
See TracChangeset
for help on using the changeset viewer.
