IEMAllAImplC.cpp@ 94224

Last change on this file since 94224 was 94224, checked in by vboxsync, 3 years ago
VMM/IEM: Attempt at doxygen fix. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 128.9 KB

Line
1	/* $Id: IEMAllAImplC.cpp 94224 2022-03-14 13:23:00Z vboxsync $ */
2	/** @file
3	* IEM - Instruction Implementation in Assembly, portable C variant.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#include "IEMInternal.h"
23	#include <VBox/vmm/vmcc.h>
24	#include <iprt/errcore.h>
25	#include <iprt/x86.h>
26	#include <iprt/uint128.h>
27
28
29	/*********************************************************************************************************************************
30	* Defined Constants And Macros *
31	*********************************************************************************************************************************/
32	/** @def IEM_WITHOUT_ASSEMBLY
33	* Enables all the code in this file.
34	*/
35	#if !defined(IEM_WITHOUT_ASSEMBLY)
36	# if defined(RT_ARCH_ARM32) \|\| defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
37	# define IEM_WITHOUT_ASSEMBLY
38	# endif
39	#endif
40	/* IEM_WITH_ASSEMBLY trumps IEM_WITHOUT_ASSEMBLY for tstIEMAImplAsm purposes. */
41	#ifdef IEM_WITH_ASSEMBLY
42	# undef IEM_WITHOUT_ASSEMBLY
43	#endif
44
45	/**
46	* Calculates the signed flag value given a result and it's bit width.
47	*
48	* The signed flag (SF) is a duplication of the most significant bit in the
49	* result.
50	*
51	* @returns X86_EFL_SF or 0.
52	* @param a_uResult Unsigned result value.
53	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
54	*/
55	#define X86_EFL_CALC_SF(a_uResult, a_cBitsWidth) \
56	( (uint32_t)((a_uResult) >> ((a_cBitsWidth) - X86_EFL_SF_BIT - 1)) & X86_EFL_SF )
57
58	/**
59	* Calculates the zero flag value given a result.
60	*
61	* The zero flag (ZF) indicates whether the result is zero or not.
62	*
63	* @returns X86_EFL_ZF or 0.
64	* @param a_uResult Unsigned result value.
65	*/
66	#define X86_EFL_CALC_ZF(a_uResult) \
67	( (uint32_t)((a_uResult) == 0) << X86_EFL_ZF_BIT )
68
69	/**
70	* Extracts the OF flag from a OF calculation result.
71	*
72	* These are typically used by concating with a bitcount. The problem is that
73	* 8-bit values needs shifting in the other direction than the others.
74	*/
75	#define X86_EFL_GET_OF_8(a_uValue) (((uint32_t)(a_uValue) << (X86_EFL_OF_BIT - 8 + 1)) & X86_EFL_OF)
76	#define X86_EFL_GET_OF_16(a_uValue) ((uint32_t)((a_uValue) >> (16 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
77	#define X86_EFL_GET_OF_32(a_uValue) ((uint32_t)((a_uValue) >> (32 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
78	#define X86_EFL_GET_OF_64(a_uValue) ((uint32_t)((a_uValue) >> (64 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
79
80	/**
81	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) after arithmetic op.
82	*
83	* @returns Status bits.
84	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
85	* @param a_uResult Unsigned result value.
86	* @param a_uSrc The source value (for AF calc).
87	* @param a_uDst The original destination value (for AF calc).
88	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
89	* @param a_CfExpr Bool expression for the carry flag (CF).
90	* @param a_uSrcOf The a_uSrc value to use for overflow calculation.
91	*/
92	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(a_pfEFlags, a_uResult, a_uDst, a_uSrc, a_cBitsWidth, a_CfExpr, a_uSrcOf) \
93	do { \
94	uint32_t fEflTmp = *(a_pfEFlags); \
95	fEflTmp &= ~X86_EFL_STATUS_BITS; \
96	fEflTmp \|= (a_CfExpr) << X86_EFL_CF_BIT; \
97	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
98	fEflTmp \|= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uSrc) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
99	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
100	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
101	\
102	/* Overflow during ADDition happens when both inputs have the same signed \
103	bit value and the result has a different sign bit value. \
104	\
105	Since subtraction can be rewritten as addition: 2 - 1 == 2 + -1, it \
106	follows that for SUBtraction the signed bit value must differ between \
107	the two inputs and the result's signed bit diff from the first input. \
108	Note! Must xor with sign bit to convert, not do (0 - a_uSrc). \
109	\
110	See also: http://teaching.idallen.com/dat2343/10f/notes/040_overflow.txt */ \
111	fEflTmp \|= X86_EFL_GET_OF_ ## a_cBitsWidth( ( ((uint ## a_cBitsWidth ## _t)~((a_uDst) ^ (a_uSrcOf))) \
112	& RT_BIT_64(a_cBitsWidth - 1)) \
113	& ((a_uResult) ^ (a_uDst)) ); \
114	*(a_pfEFlags) = fEflTmp; \
115	} while (0)
116
117	/**
118	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) after a logical op.
119	*
120	* CF and OF are defined to be 0 by logical operations. AF on the other hand is
121	* undefined. We do not set AF, as that seems to make the most sense (which
122	* probably makes it the most wrong in real life).
123	*
124	* @returns Status bits.
125	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
126	* @param a_uResult Unsigned result value.
127	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
128	* @param a_fExtra Additional bits to set.
129	*/
130	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(a_pfEFlags, a_uResult, a_cBitsWidth, a_fExtra) \
131	do { \
132	uint32_t fEflTmp = *(a_pfEFlags); \
133	fEflTmp &= ~X86_EFL_STATUS_BITS; \
134	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
135	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
136	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
137	fEflTmp \|= (a_fExtra); \
138	*(a_pfEFlags) = fEflTmp; \
139	} while (0)
140
141
142	/*********************************************************************************************************************************
143	* Global Variables *
144	*********************************************************************************************************************************/
145	/**
146	* Parity calculation table.
147	*
148	* This is also used by iemAllAImpl.asm.
149	*
150	* The generator code:
151	* @code
152	* #include <stdio.h>
153	*
154	* int main()
155	* {
156	* unsigned b;
157	* for (b = 0; b < 256; b++)
158	* {
159	* int cOnes = ( b & 1)
160	* + ((b >> 1) & 1)
161	* + ((b >> 2) & 1)
162	* + ((b >> 3) & 1)
163	* + ((b >> 4) & 1)
164	* + ((b >> 5) & 1)
165	* + ((b >> 6) & 1)
166	* + ((b >> 7) & 1);
167	* printf(" /" "* %#04x = %u%u%u%u%u%u%u%ub *" "/ %s,\n",
168	* b,
169	* (b >> 7) & 1,
170	* (b >> 6) & 1,
171	* (b >> 5) & 1,
172	* (b >> 4) & 1,
173	* (b >> 3) & 1,
174	* (b >> 2) & 1,
175	* (b >> 1) & 1,
176	* b & 1,
177	* cOnes & 1 ? "0" : "X86_EFL_PF");
178	* }
179	* return 0;
180	* }
181	* @endcode
182	*/
183	uint8_t const g_afParity[256] =
184	{
185	/* 0000 = 00000000b */ X86_EFL_PF,
186	/* 0x01 = 00000001b */ 0,
187	/* 0x02 = 00000010b */ 0,
188	/* 0x03 = 00000011b */ X86_EFL_PF,
189	/* 0x04 = 00000100b */ 0,
190	/* 0x05 = 00000101b */ X86_EFL_PF,
191	/* 0x06 = 00000110b */ X86_EFL_PF,
192	/* 0x07 = 00000111b */ 0,
193	/* 0x08 = 00001000b */ 0,
194	/* 0x09 = 00001001b */ X86_EFL_PF,
195	/* 0x0a = 00001010b */ X86_EFL_PF,
196	/* 0x0b = 00001011b */ 0,
197	/* 0x0c = 00001100b */ X86_EFL_PF,
198	/* 0x0d = 00001101b */ 0,
199	/* 0x0e = 00001110b */ 0,
200	/* 0x0f = 00001111b */ X86_EFL_PF,
201	/* 0x10 = 00010000b */ 0,
202	/* 0x11 = 00010001b */ X86_EFL_PF,
203	/* 0x12 = 00010010b */ X86_EFL_PF,
204	/* 0x13 = 00010011b */ 0,
205	/* 0x14 = 00010100b */ X86_EFL_PF,
206	/* 0x15 = 00010101b */ 0,
207	/* 0x16 = 00010110b */ 0,
208	/* 0x17 = 00010111b */ X86_EFL_PF,
209	/* 0x18 = 00011000b */ X86_EFL_PF,
210	/* 0x19 = 00011001b */ 0,
211	/* 0x1a = 00011010b */ 0,
212	/* 0x1b = 00011011b */ X86_EFL_PF,
213	/* 0x1c = 00011100b */ 0,
214	/* 0x1d = 00011101b */ X86_EFL_PF,
215	/* 0x1e = 00011110b */ X86_EFL_PF,
216	/* 0x1f = 00011111b */ 0,
217	/* 0x20 = 00100000b */ 0,
218	/* 0x21 = 00100001b */ X86_EFL_PF,
219	/* 0x22 = 00100010b */ X86_EFL_PF,
220	/* 0x23 = 00100011b */ 0,
221	/* 0x24 = 00100100b */ X86_EFL_PF,
222	/* 0x25 = 00100101b */ 0,
223	/* 0x26 = 00100110b */ 0,
224	/* 0x27 = 00100111b */ X86_EFL_PF,
225	/* 0x28 = 00101000b */ X86_EFL_PF,
226	/* 0x29 = 00101001b */ 0,
227	/* 0x2a = 00101010b */ 0,
228	/* 0x2b = 00101011b */ X86_EFL_PF,
229	/* 0x2c = 00101100b */ 0,
230	/* 0x2d = 00101101b */ X86_EFL_PF,
231	/* 0x2e = 00101110b */ X86_EFL_PF,
232	/* 0x2f = 00101111b */ 0,
233	/* 0x30 = 00110000b */ X86_EFL_PF,
234	/* 0x31 = 00110001b */ 0,
235	/* 0x32 = 00110010b */ 0,
236	/* 0x33 = 00110011b */ X86_EFL_PF,
237	/* 0x34 = 00110100b */ 0,
238	/* 0x35 = 00110101b */ X86_EFL_PF,
239	/* 0x36 = 00110110b */ X86_EFL_PF,
240	/* 0x37 = 00110111b */ 0,
241	/* 0x38 = 00111000b */ 0,
242	/* 0x39 = 00111001b */ X86_EFL_PF,
243	/* 0x3a = 00111010b */ X86_EFL_PF,
244	/* 0x3b = 00111011b */ 0,
245	/* 0x3c = 00111100b */ X86_EFL_PF,
246	/* 0x3d = 00111101b */ 0,
247	/* 0x3e = 00111110b */ 0,
248	/* 0x3f = 00111111b */ X86_EFL_PF,
249	/* 0x40 = 01000000b */ 0,
250	/* 0x41 = 01000001b */ X86_EFL_PF,
251	/* 0x42 = 01000010b */ X86_EFL_PF,
252	/* 0x43 = 01000011b */ 0,
253	/* 0x44 = 01000100b */ X86_EFL_PF,
254	/* 0x45 = 01000101b */ 0,
255	/* 0x46 = 01000110b */ 0,
256	/* 0x47 = 01000111b */ X86_EFL_PF,
257	/* 0x48 = 01001000b */ X86_EFL_PF,
258	/* 0x49 = 01001001b */ 0,
259	/* 0x4a = 01001010b */ 0,
260	/* 0x4b = 01001011b */ X86_EFL_PF,
261	/* 0x4c = 01001100b */ 0,
262	/* 0x4d = 01001101b */ X86_EFL_PF,
263	/* 0x4e = 01001110b */ X86_EFL_PF,
264	/* 0x4f = 01001111b */ 0,
265	/* 0x50 = 01010000b */ X86_EFL_PF,
266	/* 0x51 = 01010001b */ 0,
267	/* 0x52 = 01010010b */ 0,
268	/* 0x53 = 01010011b */ X86_EFL_PF,
269	/* 0x54 = 01010100b */ 0,
270	/* 0x55 = 01010101b */ X86_EFL_PF,
271	/* 0x56 = 01010110b */ X86_EFL_PF,
272	/* 0x57 = 01010111b */ 0,
273	/* 0x58 = 01011000b */ 0,
274	/* 0x59 = 01011001b */ X86_EFL_PF,
275	/* 0x5a = 01011010b */ X86_EFL_PF,
276	/* 0x5b = 01011011b */ 0,
277	/* 0x5c = 01011100b */ X86_EFL_PF,
278	/* 0x5d = 01011101b */ 0,
279	/* 0x5e = 01011110b */ 0,
280	/* 0x5f = 01011111b */ X86_EFL_PF,
281	/* 0x60 = 01100000b */ X86_EFL_PF,
282	/* 0x61 = 01100001b */ 0,
283	/* 0x62 = 01100010b */ 0,
284	/* 0x63 = 01100011b */ X86_EFL_PF,
285	/* 0x64 = 01100100b */ 0,
286	/* 0x65 = 01100101b */ X86_EFL_PF,
287	/* 0x66 = 01100110b */ X86_EFL_PF,
288	/* 0x67 = 01100111b */ 0,
289	/* 0x68 = 01101000b */ 0,
290	/* 0x69 = 01101001b */ X86_EFL_PF,
291	/* 0x6a = 01101010b */ X86_EFL_PF,
292	/* 0x6b = 01101011b */ 0,
293	/* 0x6c = 01101100b */ X86_EFL_PF,
294	/* 0x6d = 01101101b */ 0,
295	/* 0x6e = 01101110b */ 0,
296	/* 0x6f = 01101111b */ X86_EFL_PF,
297	/* 0x70 = 01110000b */ 0,
298	/* 0x71 = 01110001b */ X86_EFL_PF,
299	/* 0x72 = 01110010b */ X86_EFL_PF,
300	/* 0x73 = 01110011b */ 0,
301	/* 0x74 = 01110100b */ X86_EFL_PF,
302	/* 0x75 = 01110101b */ 0,
303	/* 0x76 = 01110110b */ 0,
304	/* 0x77 = 01110111b */ X86_EFL_PF,
305	/* 0x78 = 01111000b */ X86_EFL_PF,
306	/* 0x79 = 01111001b */ 0,
307	/* 0x7a = 01111010b */ 0,
308	/* 0x7b = 01111011b */ X86_EFL_PF,
309	/* 0x7c = 01111100b */ 0,
310	/* 0x7d = 01111101b */ X86_EFL_PF,
311	/* 0x7e = 01111110b */ X86_EFL_PF,
312	/* 0x7f = 01111111b */ 0,
313	/* 0x80 = 10000000b */ 0,
314	/* 0x81 = 10000001b */ X86_EFL_PF,
315	/* 0x82 = 10000010b */ X86_EFL_PF,
316	/* 0x83 = 10000011b */ 0,
317	/* 0x84 = 10000100b */ X86_EFL_PF,
318	/* 0x85 = 10000101b */ 0,
319	/* 0x86 = 10000110b */ 0,
320	/* 0x87 = 10000111b */ X86_EFL_PF,
321	/* 0x88 = 10001000b */ X86_EFL_PF,
322	/* 0x89 = 10001001b */ 0,
323	/* 0x8a = 10001010b */ 0,
324	/* 0x8b = 10001011b */ X86_EFL_PF,
325	/* 0x8c = 10001100b */ 0,
326	/* 0x8d = 10001101b */ X86_EFL_PF,
327	/* 0x8e = 10001110b */ X86_EFL_PF,
328	/* 0x8f = 10001111b */ 0,
329	/* 0x90 = 10010000b */ X86_EFL_PF,
330	/* 0x91 = 10010001b */ 0,
331	/* 0x92 = 10010010b */ 0,
332	/* 0x93 = 10010011b */ X86_EFL_PF,
333	/* 0x94 = 10010100b */ 0,
334	/* 0x95 = 10010101b */ X86_EFL_PF,
335	/* 0x96 = 10010110b */ X86_EFL_PF,
336	/* 0x97 = 10010111b */ 0,
337	/* 0x98 = 10011000b */ 0,
338	/* 0x99 = 10011001b */ X86_EFL_PF,
339	/* 0x9a = 10011010b */ X86_EFL_PF,
340	/* 0x9b = 10011011b */ 0,
341	/* 0x9c = 10011100b */ X86_EFL_PF,
342	/* 0x9d = 10011101b */ 0,
343	/* 0x9e = 10011110b */ 0,
344	/* 0x9f = 10011111b */ X86_EFL_PF,
345	/* 0xa0 = 10100000b */ X86_EFL_PF,
346	/* 0xa1 = 10100001b */ 0,
347	/* 0xa2 = 10100010b */ 0,
348	/* 0xa3 = 10100011b */ X86_EFL_PF,
349	/* 0xa4 = 10100100b */ 0,
350	/* 0xa5 = 10100101b */ X86_EFL_PF,
351	/* 0xa6 = 10100110b */ X86_EFL_PF,
352	/* 0xa7 = 10100111b */ 0,
353	/* 0xa8 = 10101000b */ 0,
354	/* 0xa9 = 10101001b */ X86_EFL_PF,
355	/* 0xaa = 10101010b */ X86_EFL_PF,
356	/* 0xab = 10101011b */ 0,
357	/* 0xac = 10101100b */ X86_EFL_PF,
358	/* 0xad = 10101101b */ 0,
359	/* 0xae = 10101110b */ 0,
360	/* 0xaf = 10101111b */ X86_EFL_PF,
361	/* 0xb0 = 10110000b */ 0,
362	/* 0xb1 = 10110001b */ X86_EFL_PF,
363	/* 0xb2 = 10110010b */ X86_EFL_PF,
364	/* 0xb3 = 10110011b */ 0,
365	/* 0xb4 = 10110100b */ X86_EFL_PF,
366	/* 0xb5 = 10110101b */ 0,
367	/* 0xb6 = 10110110b */ 0,
368	/* 0xb7 = 10110111b */ X86_EFL_PF,
369	/* 0xb8 = 10111000b */ X86_EFL_PF,
370	/* 0xb9 = 10111001b */ 0,
371	/* 0xba = 10111010b */ 0,
372	/* 0xbb = 10111011b */ X86_EFL_PF,
373	/* 0xbc = 10111100b */ 0,
374	/* 0xbd = 10111101b */ X86_EFL_PF,
375	/* 0xbe = 10111110b */ X86_EFL_PF,
376	/* 0xbf = 10111111b */ 0,
377	/* 0xc0 = 11000000b */ X86_EFL_PF,
378	/* 0xc1 = 11000001b */ 0,
379	/* 0xc2 = 11000010b */ 0,
380	/* 0xc3 = 11000011b */ X86_EFL_PF,
381	/* 0xc4 = 11000100b */ 0,
382	/* 0xc5 = 11000101b */ X86_EFL_PF,
383	/* 0xc6 = 11000110b */ X86_EFL_PF,
384	/* 0xc7 = 11000111b */ 0,
385	/* 0xc8 = 11001000b */ 0,
386	/* 0xc9 = 11001001b */ X86_EFL_PF,
387	/* 0xca = 11001010b */ X86_EFL_PF,
388	/* 0xcb = 11001011b */ 0,
389	/* 0xcc = 11001100b */ X86_EFL_PF,
390	/* 0xcd = 11001101b */ 0,
391	/* 0xce = 11001110b */ 0,
392	/* 0xcf = 11001111b */ X86_EFL_PF,
393	/* 0xd0 = 11010000b */ 0,
394	/* 0xd1 = 11010001b */ X86_EFL_PF,
395	/* 0xd2 = 11010010b */ X86_EFL_PF,
396	/* 0xd3 = 11010011b */ 0,
397	/* 0xd4 = 11010100b */ X86_EFL_PF,
398	/* 0xd5 = 11010101b */ 0,
399	/* 0xd6 = 11010110b */ 0,
400	/* 0xd7 = 11010111b */ X86_EFL_PF,
401	/* 0xd8 = 11011000b */ X86_EFL_PF,
402	/* 0xd9 = 11011001b */ 0,
403	/* 0xda = 11011010b */ 0,
404	/* 0xdb = 11011011b */ X86_EFL_PF,
405	/* 0xdc = 11011100b */ 0,
406	/* 0xdd = 11011101b */ X86_EFL_PF,
407	/* 0xde = 11011110b */ X86_EFL_PF,
408	/* 0xdf = 11011111b */ 0,
409	/* 0xe0 = 11100000b */ 0,
410	/* 0xe1 = 11100001b */ X86_EFL_PF,
411	/* 0xe2 = 11100010b */ X86_EFL_PF,
412	/* 0xe3 = 11100011b */ 0,
413	/* 0xe4 = 11100100b */ X86_EFL_PF,
414	/* 0xe5 = 11100101b */ 0,
415	/* 0xe6 = 11100110b */ 0,
416	/* 0xe7 = 11100111b */ X86_EFL_PF,
417	/* 0xe8 = 11101000b */ X86_EFL_PF,
418	/* 0xe9 = 11101001b */ 0,
419	/* 0xea = 11101010b */ 0,
420	/* 0xeb = 11101011b */ X86_EFL_PF,
421	/* 0xec = 11101100b */ 0,
422	/* 0xed = 11101101b */ X86_EFL_PF,
423	/* 0xee = 11101110b */ X86_EFL_PF,
424	/* 0xef = 11101111b */ 0,
425	/* 0xf0 = 11110000b */ X86_EFL_PF,
426	/* 0xf1 = 11110001b */ 0,
427	/* 0xf2 = 11110010b */ 0,
428	/* 0xf3 = 11110011b */ X86_EFL_PF,
429	/* 0xf4 = 11110100b */ 0,
430	/* 0xf5 = 11110101b */ X86_EFL_PF,
431	/* 0xf6 = 11110110b */ X86_EFL_PF,
432	/* 0xf7 = 11110111b */ 0,
433	/* 0xf8 = 11111000b */ 0,
434	/* 0xf9 = 11111001b */ X86_EFL_PF,
435	/* 0xfa = 11111010b */ X86_EFL_PF,
436	/* 0xfb = 11111011b */ 0,
437	/* 0xfc = 11111100b */ X86_EFL_PF,
438	/* 0xfd = 11111101b */ 0,
439	/* 0xfe = 11111110b */ 0,
440	/* 0xff = 11111111b */ X86_EFL_PF,
441	};
442
443
444	/*
445	* There are a few 64-bit on 32-bit things we'd rather do in C. Actually, doing
446	* it all in C is probably safer atm., optimize what's necessary later, maybe.
447	*/
448	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
449
450
451	/*********************************************************************************************************************************
452	* Binary Operations *
453	*********************************************************************************************************************************/
454
455	/*
456	* ADD
457	*/
458
459	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
460	{
461	uint64_t uDst = *puDst;
462	uint64_t uResult = uDst + uSrc;
463	*puDst = uResult;
464	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult < uDst, uSrc);
465	}
466
467	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
468
469	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
470	{
471	uint32_t uDst = *puDst;
472	uint32_t uResult = uDst + uSrc;
473	*puDst = uResult;
474	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult < uDst, uSrc);
475	}
476
477
478	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
479	{
480	uint16_t uDst = *puDst;
481	uint16_t uResult = uDst + uSrc;
482	*puDst = uResult;
483	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult < uDst, uSrc);
484	}
485
486
487	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
488	{
489	uint8_t uDst = *puDst;
490	uint8_t uResult = uDst + uSrc;
491	*puDst = uResult;
492	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult < uDst, uSrc);
493	}
494
495	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
496
497	/*
498	* ADC
499	*/
500
501	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
502	{
503	if (!(*pfEFlags & X86_EFL_CF))
504	iemAImpl_add_u64(puDst, uSrc, pfEFlags);
505	else
506	{
507	uint64_t uDst = *puDst;
508	uint64_t uResult = uDst + uSrc + 1;
509	*puDst = uResult;
510	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult <= uDst, uSrc);
511	}
512	}
513
514	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
515
516	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
517	{
518	if (!(*pfEFlags & X86_EFL_CF))
519	iemAImpl_add_u32(puDst, uSrc, pfEFlags);
520	else
521	{
522	uint32_t uDst = *puDst;
523	uint32_t uResult = uDst + uSrc + 1;
524	*puDst = uResult;
525	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult <= uDst, uSrc);
526	}
527	}
528
529
530	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
531	{
532	if (!(*pfEFlags & X86_EFL_CF))
533	iemAImpl_add_u16(puDst, uSrc, pfEFlags);
534	else
535	{
536	uint16_t uDst = *puDst;
537	uint16_t uResult = uDst + uSrc + 1;
538	*puDst = uResult;
539	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult <= uDst, uSrc);
540	}
541	}
542
543
544	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
545	{
546	if (!(*pfEFlags & X86_EFL_CF))
547	iemAImpl_add_u8(puDst, uSrc, pfEFlags);
548	else
549	{
550	uint8_t uDst = *puDst;
551	uint8_t uResult = uDst + uSrc + 1;
552	*puDst = uResult;
553	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult <= uDst, uSrc);
554	}
555	}
556
557	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
558
559	/*
560	* SUB
561	*/
562
563	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
564	{
565	uint64_t uDst = *puDst;
566	uint64_t uResult = uDst - uSrc;
567	*puDst = uResult;
568	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst < uSrc, uSrc ^ RT_BIT_64(63));
569	}
570
571	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
572
573	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
574	{
575	uint32_t uDst = *puDst;
576	uint32_t uResult = uDst - uSrc;
577	*puDst = uResult;
578	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst < uSrc, uSrc ^ RT_BIT_32(31));
579	}
580
581
582	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
583	{
584	uint16_t uDst = *puDst;
585	uint16_t uResult = uDst - uSrc;
586	*puDst = uResult;
587	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst < uSrc, uSrc ^ (uint16_t)0x8000);
588	}
589
590
591	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
592	{
593	uint8_t uDst = *puDst;
594	uint8_t uResult = uDst - uSrc;
595	*puDst = uResult;
596	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst < uSrc, uSrc ^ (uint8_t)0x80);
597	}
598
599	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
600
601	/*
602	* SBB
603	*/
604
605	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
606	{
607	if (!(*pfEFlags & X86_EFL_CF))
608	iemAImpl_sub_u64(puDst, uSrc, pfEFlags);
609	else
610	{
611	uint64_t uDst = *puDst;
612	uint64_t uResult = uDst - uSrc - 1;
613	*puDst = uResult;
614	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst <= uSrc, uSrc ^ RT_BIT_64(63));
615	}
616	}
617
618	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
619
620	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
621	{
622	if (!(*pfEFlags & X86_EFL_CF))
623	iemAImpl_sub_u32(puDst, uSrc, pfEFlags);
624	else
625	{
626	uint32_t uDst = *puDst;
627	uint32_t uResult = uDst - uSrc - 1;
628	*puDst = uResult;
629	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst <= uSrc, uSrc ^ RT_BIT_32(31));
630	}
631	}
632
633
634	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
635	{
636	if (!(*pfEFlags & X86_EFL_CF))
637	iemAImpl_sub_u16(puDst, uSrc, pfEFlags);
638	else
639	{
640	uint16_t uDst = *puDst;
641	uint16_t uResult = uDst - uSrc - 1;
642	*puDst = uResult;
643	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst <= uSrc, uSrc ^ (uint16_t)0x8000);
644	}
645	}
646
647
648	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
649	{
650	if (!(*pfEFlags & X86_EFL_CF))
651	iemAImpl_sub_u8(puDst, uSrc, pfEFlags);
652	else
653	{
654	uint8_t uDst = *puDst;
655	uint8_t uResult = uDst - uSrc - 1;
656	*puDst = uResult;
657	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst <= uSrc, uSrc ^ (uint8_t)0x80);
658	}
659	}
660
661	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
662
663
664	/*
665	* OR
666	*/
667
668	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
669	{
670	uint64_t uResult = *puDst \| uSrc;
671	*puDst = uResult;
672	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
673	}
674
675	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
676
677	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
678	{
679	uint32_t uResult = *puDst \| uSrc;
680	*puDst = uResult;
681	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
682	}
683
684
685	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
686	{
687	uint16_t uResult = *puDst \| uSrc;
688	*puDst = uResult;
689	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
690	}
691
692
693	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
694	{
695	uint8_t uResult = *puDst \| uSrc;
696	*puDst = uResult;
697	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
698	}
699
700	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
701
702	/*
703	* XOR
704	*/
705
706	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
707	{
708	uint64_t uResult = *puDst ^ uSrc;
709	*puDst = uResult;
710	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
711	}
712
713	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
714
715	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
716	{
717	uint32_t uResult = *puDst ^ uSrc;
718	*puDst = uResult;
719	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
720	}
721
722
723	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
724	{
725	uint16_t uResult = *puDst ^ uSrc;
726	*puDst = uResult;
727	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
728	}
729
730
731	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
732	{
733	uint8_t uResult = *puDst ^ uSrc;
734	*puDst = uResult;
735	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
736	}
737
738	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
739
740	/*
741	* AND
742	*/
743
744	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
745	{
746	uint64_t uResult = *puDst & uSrc;
747	*puDst = uResult;
748	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
749	}
750
751	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
752
753	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
754	{
755	uint32_t uResult = *puDst & uSrc;
756	*puDst = uResult;
757	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
758	}
759
760
761	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
762	{
763	uint16_t uResult = *puDst & uSrc;
764	*puDst = uResult;
765	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
766	}
767
768
769	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
770	{
771	uint8_t uResult = *puDst & uSrc;
772	*puDst = uResult;
773	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
774	}
775
776	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
777
778	/*
779	* CMP
780	*/
781
782	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
783	{
784	uint64_t uDstTmp = *puDst;
785	iemAImpl_sub_u64(&uDstTmp, uSrc, pfEFlags);
786	}
787
788	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
789
790	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
791	{
792	uint32_t uDstTmp = *puDst;
793	iemAImpl_sub_u32(&uDstTmp, uSrc, pfEFlags);
794	}
795
796
797	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
798	{
799	uint16_t uDstTmp = *puDst;
800	iemAImpl_sub_u16(&uDstTmp, uSrc, pfEFlags);
801	}
802
803
804	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
805	{
806	uint8_t uDstTmp = *puDst;
807	iemAImpl_sub_u8(&uDstTmp, uSrc, pfEFlags);
808	}
809
810	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
811
812	/*
813	* TEST
814	*/
815
816	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
817	{
818	uint64_t uResult = *puDst & uSrc;
819	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
820	}
821
822	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
823
824	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
825	{
826	uint32_t uResult = *puDst & uSrc;
827	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
828	}
829
830
831	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
832	{
833	uint16_t uResult = *puDst & uSrc;
834	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
835	}
836
837
838	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
839	{
840	uint8_t uResult = *puDst & uSrc;
841	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
842	}
843
844	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
845
846
847	/*
848	* LOCK prefixed variants of the above
849	*/
850
851	/** 64-bit locked binary operand operation. */
852	# define DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \
853	do { \
854	uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
855	uint ## a_cBitsWidth ## _t uTmp; \
856	uint32_t fEflTmp; \
857	do \
858	{ \
859	uTmp = uOld; \
860	fEflTmp = *pfEFlags; \
861	iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, uSrc, &fEflTmp); \
862	} while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \
863	*pfEFlags = fEflTmp; \
864	} while (0)
865
866
867	#define EMIT_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \
868	IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \
869	uint ## a_cBitsWidth ## _t uSrc, \
870	uint32_t *pfEFlags)) \
871	{ \
872	DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth); \
873	}
874
875	EMIT_LOCKED_BIN_OP(add, 64)
876	EMIT_LOCKED_BIN_OP(adc, 64)
877	EMIT_LOCKED_BIN_OP(sub, 64)
878	EMIT_LOCKED_BIN_OP(sbb, 64)
879	EMIT_LOCKED_BIN_OP(or, 64)
880	EMIT_LOCKED_BIN_OP(xor, 64)
881	EMIT_LOCKED_BIN_OP(and, 64)
882	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
883	EMIT_LOCKED_BIN_OP(add, 32)
884	EMIT_LOCKED_BIN_OP(adc, 32)
885	EMIT_LOCKED_BIN_OP(sub, 32)
886	EMIT_LOCKED_BIN_OP(sbb, 32)
887	EMIT_LOCKED_BIN_OP(or, 32)
888	EMIT_LOCKED_BIN_OP(xor, 32)
889	EMIT_LOCKED_BIN_OP(and, 32)
890
891	EMIT_LOCKED_BIN_OP(add, 16)
892	EMIT_LOCKED_BIN_OP(adc, 16)
893	EMIT_LOCKED_BIN_OP(sub, 16)
894	EMIT_LOCKED_BIN_OP(sbb, 16)
895	EMIT_LOCKED_BIN_OP(or, 16)
896	EMIT_LOCKED_BIN_OP(xor, 16)
897	EMIT_LOCKED_BIN_OP(and, 16)
898
899	EMIT_LOCKED_BIN_OP(add, 8)
900	EMIT_LOCKED_BIN_OP(adc, 8)
901	EMIT_LOCKED_BIN_OP(sub, 8)
902	EMIT_LOCKED_BIN_OP(sbb, 8)
903	EMIT_LOCKED_BIN_OP(or, 8)
904	EMIT_LOCKED_BIN_OP(xor, 8)
905	EMIT_LOCKED_BIN_OP(and, 8)
906	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
907
908
909	/*
910	* Bit operations (same signature as above).
911	*/
912
913	/*
914	* BT
915	*/
916
917	IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
918	{
919	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
920	not modified by either AMD (3990x) or Intel (i9-9980HK). */
921	Assert(uSrc < 64);
922	uint64_t uDst = *puDst;
923	if (uDst & RT_BIT_64(uSrc))
924	*pfEFlags \|= X86_EFL_CF;
925	else
926	*pfEFlags &= ~X86_EFL_CF;
927	}
928
929	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
930
931	IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
932	{
933	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
934	not modified by either AMD (3990x) or Intel (i9-9980HK). */
935	Assert(uSrc < 32);
936	uint32_t uDst = *puDst;
937	if (uDst & RT_BIT_32(uSrc))
938	*pfEFlags \|= X86_EFL_CF;
939	else
940	*pfEFlags &= ~X86_EFL_CF;
941	}
942
943	IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
944	{
945	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
946	not modified by either AMD (3990x) or Intel (i9-9980HK). */
947	Assert(uSrc < 16);
948	uint16_t uDst = *puDst;
949	if (uDst & RT_BIT_32(uSrc))
950	*pfEFlags \|= X86_EFL_CF;
951	else
952	*pfEFlags &= ~X86_EFL_CF;
953	}
954
955	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
956
957	/*
958	* BTC
959	*/
960
961	IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
962	{
963	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
964	not modified by either AMD (3990x) or Intel (i9-9980HK). */
965	Assert(uSrc < 64);
966	uint64_t fMask = RT_BIT_64(uSrc);
967	uint64_t uDst = *puDst;
968	if (uDst & fMask)
969	{
970	uDst &= ~fMask;
971	*puDst = uDst;
972	*pfEFlags \|= X86_EFL_CF;
973	}
974	else
975	{
976	uDst \|= fMask;
977	*puDst = uDst;
978	*pfEFlags &= ~X86_EFL_CF;
979	}
980	}
981
982	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
983
984	IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
985	{
986	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
987	not modified by either AMD (3990x) or Intel (i9-9980HK). */
988	Assert(uSrc < 32);
989	uint32_t fMask = RT_BIT_32(uSrc);
990	uint32_t uDst = *puDst;
991	if (uDst & fMask)
992	{
993	uDst &= ~fMask;
994	*puDst = uDst;
995	*pfEFlags \|= X86_EFL_CF;
996	}
997	else
998	{
999	uDst \|= fMask;
1000	*puDst = uDst;
1001	*pfEFlags &= ~X86_EFL_CF;
1002	}
1003	}
1004
1005
1006	IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1007	{
1008	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
1009	not modified by either AMD (3990x) or Intel (i9-9980HK). */
1010	Assert(uSrc < 16);
1011	uint16_t fMask = RT_BIT_32(uSrc);
1012	uint16_t uDst = *puDst;
1013	if (uDst & fMask)
1014	{
1015	uDst &= ~fMask;
1016	*puDst = uDst;
1017	*pfEFlags \|= X86_EFL_CF;
1018	}
1019	else
1020	{
1021	uDst \|= fMask;
1022	*puDst = uDst;
1023	*pfEFlags &= ~X86_EFL_CF;
1024	}
1025	}
1026
1027	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1028
1029	/*
1030	* BTR
1031	*/
1032
1033	IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1034	{
1035	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1036	logical operation (AND/OR/whatever). */
1037	Assert(uSrc < 64);
1038	uint64_t fMask = RT_BIT_64(uSrc);
1039	uint64_t uDst = *puDst;
1040	if (uDst & fMask)
1041	{
1042	uDst &= ~fMask;
1043	*puDst = uDst;
1044	*pfEFlags \|= X86_EFL_CF;
1045	}
1046	else
1047	*pfEFlags &= ~X86_EFL_CF;
1048	}
1049
1050	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1051
1052	IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1053	{
1054	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1055	logical operation (AND/OR/whatever). */
1056	Assert(uSrc < 32);
1057	uint32_t fMask = RT_BIT_32(uSrc);
1058	uint32_t uDst = *puDst;
1059	if (uDst & fMask)
1060	{
1061	uDst &= ~fMask;
1062	*puDst = uDst;
1063	*pfEFlags \|= X86_EFL_CF;
1064	}
1065	else
1066	*pfEFlags &= ~X86_EFL_CF;
1067	}
1068
1069
1070	IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1071	{
1072	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1073	logical operation (AND/OR/whatever). */
1074	Assert(uSrc < 16);
1075	uint16_t fMask = RT_BIT_32(uSrc);
1076	uint16_t uDst = *puDst;
1077	if (uDst & fMask)
1078	{
1079	uDst &= ~fMask;
1080	*puDst = uDst;
1081	*pfEFlags \|= X86_EFL_CF;
1082	}
1083	else
1084	*pfEFlags &= ~X86_EFL_CF;
1085	}
1086
1087	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1088
1089	/*
1090	* BTS
1091	*/
1092
1093	IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1094	{
1095	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1096	logical operation (AND/OR/whatever). */
1097	Assert(uSrc < 64);
1098	uint64_t fMask = RT_BIT_64(uSrc);
1099	uint64_t uDst = *puDst;
1100	if (uDst & fMask)
1101	*pfEFlags \|= X86_EFL_CF;
1102	else
1103	{
1104	uDst \|= fMask;
1105	*puDst = uDst;
1106	*pfEFlags &= ~X86_EFL_CF;
1107	}
1108	}
1109
1110	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1111
1112	IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1113	{
1114	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1115	logical operation (AND/OR/whatever). */
1116	Assert(uSrc < 32);
1117	uint32_t fMask = RT_BIT_32(uSrc);
1118	uint32_t uDst = *puDst;
1119	if (uDst & fMask)
1120	*pfEFlags \|= X86_EFL_CF;
1121	else
1122	{
1123	uDst \|= fMask;
1124	*puDst = uDst;
1125	*pfEFlags &= ~X86_EFL_CF;
1126	}
1127	}
1128
1129
1130	IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1131	{
1132	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1133	logical operation (AND/OR/whatever). */
1134	Assert(uSrc < 16);
1135	uint16_t fMask = RT_BIT_32(uSrc);
1136	uint32_t uDst = *puDst;
1137	if (uDst & fMask)
1138	*pfEFlags \|= X86_EFL_CF;
1139	else
1140	{
1141	uDst \|= fMask;
1142	*puDst = uDst;
1143	*pfEFlags &= ~X86_EFL_CF;
1144	}
1145	}
1146
1147	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1148
1149
1150	EMIT_LOCKED_BIN_OP(btc, 64)
1151	EMIT_LOCKED_BIN_OP(btr, 64)
1152	EMIT_LOCKED_BIN_OP(bts, 64)
1153	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1154	EMIT_LOCKED_BIN_OP(btc, 32)
1155	EMIT_LOCKED_BIN_OP(btr, 32)
1156	EMIT_LOCKED_BIN_OP(bts, 32)
1157
1158	EMIT_LOCKED_BIN_OP(btc, 16)
1159	EMIT_LOCKED_BIN_OP(btr, 16)
1160	EMIT_LOCKED_BIN_OP(bts, 16)
1161	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1162
1163
1164	/*
1165	* Helpers for BSR and BSF.
1166	*
1167	* Note! "undefined" flags: OF, SF, AF, PF, CF.
1168	* Intel behavior modelled on 10980xe, AMD on 3990X. Other marchs may
1169	* produce different result (see https://www.sandpile.org/x86/flags.htm),
1170	* but we restrict ourselves to emulating these recent marchs.
1171	*/
1172	#define SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlag, a_iBit) do { \
1173	unsigned iBit = (a_iBit); \
1174	uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF \| X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_AF \| X86_EFL_PF \| X86_EFL_CF); \
1175	if (iBit) \
1176	{ \
1177	*puDst = --iBit; \
1178	fEfl \|= g_afParity[iBit]; \
1179	} \
1180	else \
1181	fEfl \|= X86_EFL_ZF \| X86_EFL_PF; \
1182	*pfEFlags = fEfl; \
1183	} while (0)
1184	#define SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlag, a_iBit) do { \
1185	unsigned const iBit = (a_iBit); \
1186	if (iBit) \
1187	{ \
1188	*puDst = iBit - 1; \
1189	*pfEFlags &= ~X86_EFL_ZF; \
1190	} \
1191	else \
1192	*pfEFlags \|= X86_EFL_ZF; \
1193	} while (0)
1194
1195
1196	/*
1197	* BSF - first (least significant) bit set
1198	*/
1199	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1200	{
1201	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
1202	}
1203
1204	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_intel,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1205	{
1206	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
1207	}
1208
1209	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_amd,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1210	{
1211	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
1212	}
1213
1214	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1215
1216	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1217	{
1218	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
1219	}
1220
1221	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_intel,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1222	{
1223	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
1224	}
1225
1226	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_amd,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1227	{
1228	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
1229	}
1230
1231
1232	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1233	{
1234	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
1235	}
1236
1237	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_intel,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1238	{
1239	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
1240	}
1241
1242	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_amd,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1243	{
1244	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
1245	}
1246
1247	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1248
1249
1250	/*
1251	* BSR - last (most significant) bit set
1252	*/
1253	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1254	{
1255	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
1256	}
1257
1258	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_intel,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1259	{
1260	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
1261	}
1262
1263	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_amd,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1264	{
1265	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
1266	}
1267
1268	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1269
1270	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1271	{
1272	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
1273	}
1274
1275	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_intel,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1276	{
1277	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
1278	}
1279
1280	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_amd,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1281	{
1282	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
1283	}
1284
1285
1286	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1287	{
1288	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
1289	}
1290
1291	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_intel,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1292	{
1293	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
1294	}
1295
1296	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_amd,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1297	{
1298	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
1299	}
1300
1301	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1302
1303
1304	/*
1305	* XCHG
1306	*/
1307
1308	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t puMem, uint64_t puReg))
1309	{
1310	#if ARCH_BITS >= 64
1311	puReg = ASMAtomicXchgU64(puMem, puReg);
1312	#else
1313	uint64_t uOldMem = *puMem;
1314	while (!ASMAtomicCmpXchgExU64(puMem, *puReg, uOldMem, &uOldMem))
1315	ASMNopPause();
1316	*puReg = uOldMem;
1317	#endif
1318	}
1319
1320	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1321
1322	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t puMem, uint32_t puReg))
1323	{
1324	puReg = ASMAtomicXchgU32(puMem, puReg);
1325	}
1326
1327
1328	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t puMem, uint16_t puReg))
1329	{
1330	puReg = ASMAtomicXchgU16(puMem, puReg);
1331	}
1332
1333
1334	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked,(uint8_t puMem, uint8_t puReg))
1335	{
1336	puReg = ASMAtomicXchgU8(puMem, puReg);
1337	}
1338
1339	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1340
1341
1342	/* Unlocked variants for fDisregardLock mode: */
1343
1344	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t puMem, uint64_t puReg))
1345	{
1346	uint64_t const uOld = *puMem;
1347	puMem = puReg;
1348	*puReg = uOld;
1349	}
1350
1351	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1352
1353	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t puMem, uint32_t puReg))
1354	{
1355	uint32_t const uOld = *puMem;
1356	puMem = puReg;
1357	*puReg = uOld;
1358	}
1359
1360
1361	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t puMem, uint16_t puReg))
1362	{
1363	uint16_t const uOld = *puMem;
1364	puMem = puReg;
1365	*puReg = uOld;
1366	}
1367
1368
1369	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked,(uint8_t puMem, uint8_t puReg))
1370	{
1371	uint8_t const uOld = *puMem;
1372	puMem = puReg;
1373	*puReg = uOld;
1374	}
1375
1376	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1377
1378
1379	/*
1380	* XADD and LOCK XADD.
1381	*/
1382	#define EMIT_XADD(a_cBitsWidth, a_Type) \
1383	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth,(a_Type puDst, a_Type puReg, uint32_t *pfEFlags)) \
1384	{ \
1385	a_Type uDst = *puDst; \
1386	a_Type uResult = uDst; \
1387	iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, pfEFlags); \
1388	*puDst = uResult; \
1389	*puReg = uDst; \
1390	} \
1391	\
1392	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth ## _locked,(a_Type puDst, a_Type puReg, uint32_t *pfEFlags)) \
1393	{ \
1394	a_Type uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
1395	a_Type uResult; \
1396	uint32_t fEflTmp; \
1397	do \
1398	{ \
1399	uResult = uOld; \
1400	fEflTmp = *pfEFlags; \
1401	iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, &fEflTmp); \
1402	} while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uResult, uOld, &uOld)); \
1403	*puReg = uOld; \
1404	*pfEFlags = fEflTmp; \
1405	}
1406	EMIT_XADD(64, uint64_t)
1407	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1408	EMIT_XADD(32, uint32_t)
1409	EMIT_XADD(16, uint16_t)
1410	EMIT_XADD(8, uint8_t)
1411	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1412
1413	#endif
1414
1415	/*
1416	* CMPXCHG, CMPXCHG8B, CMPXCHG16B
1417	*
1418	* Note! We don't have non-locking/atomic cmpxchg primitives, so all cmpxchg
1419	* instructions are emulated as locked.
1420	*/
1421	#if defined(IEM_WITHOUT_ASSEMBLY)
1422
1423	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t pu8Dst, uint8_t puAl, uint8_t uSrcReg, uint32_t *pEFlags))
1424	{
1425	uint8_t uOld = *puAl;
1426	if (ASMAtomicCmpXchgExU8(pu8Dst, uSrcReg, uOld, puAl))
1427	Assert(*puAl == uOld);
1428	iemAImpl_cmp_u8(&uOld, *puAl, pEFlags);
1429	}
1430
1431
1432	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t pu16Dst, uint16_t puAx, uint16_t uSrcReg, uint32_t *pEFlags))
1433	{
1434	uint16_t uOld = *puAx;
1435	if (ASMAtomicCmpXchgExU16(pu16Dst, uSrcReg, uOld, puAx))
1436	Assert(*puAx == uOld);
1437	iemAImpl_cmp_u16(&uOld, *puAx, pEFlags);
1438	}
1439
1440
1441	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t pu32Dst, uint32_t puEax, uint32_t uSrcReg, uint32_t *pEFlags))
1442	{
1443	uint32_t uOld = *puEax;
1444	if (ASMAtomicCmpXchgExU32(pu32Dst, uSrcReg, uOld, puEax))
1445	Assert(*puEax == uOld);
1446	iemAImpl_cmp_u32(&uOld, *puEax, pEFlags);
1447	}
1448
1449
1450	# if ARCH_BITS == 32
1451	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t pu64Dst, uint64_t puRax, uint64_t puSrcReg, uint32_t pEFlags))
1452	# else
1453	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t pu64Dst, uint64_t puRax, uint64_t uSrcReg, uint32_t *pEFlags))
1454	# endif
1455	{
1456	# if ARCH_BITS == 32
1457	uint64_t const uSrcReg = *puSrcReg;
1458	# endif
1459	uint64_t uOld = *puRax;
1460	if (ASMAtomicCmpXchgExU64(pu64Dst, uSrcReg, uOld, puRax))
1461	Assert(*puRax == uOld);
1462	iemAImpl_cmp_u64(&uOld, *puRax, pEFlags);
1463	}
1464
1465
1466	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1467	uint32_t *pEFlags))
1468	{
1469	uint64_t const uNew = pu64EbxEcx->u;
1470	uint64_t const uOld = pu64EaxEdx->u;
1471	if (ASMAtomicCmpXchgExU64(pu64Dst, uNew, uOld, &pu64EaxEdx->u))
1472	{
1473	Assert(pu64EaxEdx->u == uOld);
1474	*pEFlags \|= X86_EFL_ZF;
1475	}
1476	else
1477	*pEFlags &= ~X86_EFL_ZF;
1478	}
1479
1480
1481	# if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
1482	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1483	uint32_t *pEFlags))
1484	{
1485	# ifdef VBOX_STRICT
1486	RTUINT128U const uOld = *pu128RaxRdx;
1487	# endif
1488	# if defined(RT_ARCH_AMD64)
1489	if (ASMAtomicCmpXchgU128v2(&pu128Dst->u, pu128RbxRcx->s.Hi, pu128RbxRcx->s.Lo, pu128RaxRdx->s.Hi, pu128RaxRdx->s.Lo,
1490	&pu128RaxRdx->u))
1491	# else
1492	if (ASMAtomicCmpXchgU128(&pu128Dst->u, pu128RbxRcx->u, pu128RaxRdx->u, &pu128RaxRdx->u))
1493	# endif
1494	{
1495	Assert(pu128RaxRdx->s.Lo == uOld.s.Lo && pu128RaxRdx->s.Hi == uOld.s.Hi);
1496	*pEFlags \|= X86_EFL_ZF;
1497	}
1498	else
1499	*pEFlags &= ~X86_EFL_ZF;
1500	}
1501	# endif
1502
1503	#endif /* defined(IEM_WITHOUT_ASSEMBLY) */
1504
1505	# if !defined(RT_ARCH_ARM64) /** @todo may need this for unaligned accesses... */
1506	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
1507	PRTUINT128U pu128RbxRcx, uint32_t *pEFlags))
1508	{
1509	RTUINT128U u128Tmp = *pu128Dst;
1510	if ( u128Tmp.s.Lo == pu128RaxRdx->s.Lo
1511	&& u128Tmp.s.Hi == pu128RaxRdx->s.Hi)
1512	{
1513	pu128Dst = pu128RbxRcx;
1514	*pEFlags \|= X86_EFL_ZF;
1515	}
1516	else
1517	{
1518	*pu128RaxRdx = u128Tmp;
1519	*pEFlags &= ~X86_EFL_ZF;
1520	}
1521	}
1522	#endif /* !RT_ARCH_ARM64 */
1523
1524	#if defined(IEM_WITHOUT_ASSEMBLY)
1525
1526	/* Unlocked versions mapped to the locked ones: */
1527
1528	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t pu8Dst, uint8_t puAl, uint8_t uSrcReg, uint32_t *pEFlags))
1529	{
1530	iemAImpl_cmpxchg_u8_locked(pu8Dst, puAl, uSrcReg, pEFlags);
1531	}
1532
1533
1534	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t pu16Dst, uint16_t puAx, uint16_t uSrcReg, uint32_t *pEFlags))
1535	{
1536	iemAImpl_cmpxchg_u16_locked(pu16Dst, puAx, uSrcReg, pEFlags);
1537	}
1538
1539
1540	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t pu32Dst, uint32_t puEax, uint32_t uSrcReg, uint32_t *pEFlags))
1541	{
1542	iemAImpl_cmpxchg_u32_locked(pu32Dst, puEax, uSrcReg, pEFlags);
1543	}
1544
1545
1546	# if ARCH_BITS == 32
1547	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t pu64Dst, uint64_t puRax, uint64_t puSrcReg, uint32_t pEFlags))
1548	{
1549	iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, puSrcReg, pEFlags);
1550	}
1551	# else
1552	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t pu64Dst, uint64_t puRax, uint64_t uSrcReg, uint32_t *pEFlags))
1553	{
1554	iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, uSrcReg, pEFlags);
1555	}
1556	# endif
1557
1558
1559	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx, uint32_t pEFlags))
1560	{
1561	iemAImpl_cmpxchg8b_locked(pu64Dst, pu64EaxEdx, pu64EbxEcx, pEFlags);
1562	}
1563
1564
1565	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1566	uint32_t *pEFlags))
1567	{
1568	iemAImpl_cmpxchg16b_locked(pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags);
1569	}
1570
1571	#endif /* defined(IEM_WITHOUT_ASSEMBLY) */
1572
1573	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1574
1575	/*
1576	* MUL, IMUL, DIV and IDIV helpers.
1577	*
1578	* - The U64 versions must use 128-bit intermediates, so we need to abstract the
1579	* division step so we can select between using C operators and
1580	* RTUInt128DivRem/RTUInt128MulU64ByU64.
1581	*
1582	* - The U8 versions work returns output in AL + AH instead of xDX + xAX, with the
1583	* IDIV/DIV taking all the input in AX too. This means we have to abstract some
1584	* input loads and the result storing.
1585	*/
1586
1587	DECLINLINE(void) RTUInt128DivRemByU64(PRTUINT128U pQuotient, PRTUINT128U pRemainder, PCRTUINT128U pDividend, uint64_t u64Divisor)
1588	{
1589	# ifdef __GNUC__ /* GCC maybe really annoying in function. */
1590	pQuotient->s.Lo = 0;
1591	pQuotient->s.Hi = 0;
1592	# endif
1593	RTUINT128U Divisor;
1594	Divisor.s.Lo = u64Divisor;
1595	Divisor.s.Hi = 0;
1596	RTUInt128DivRem(pQuotient, pRemainder, pDividend, &Divisor);
1597	}
1598
1599	# define DIV_LOAD(a_Dividend) \
1600	a_Dividend.s.Lo = puA, a_Dividend.s.Hi = puD
1601	# define DIV_LOAD_U8(a_Dividend) \
1602	a_Dividend.u = *puAX
1603
1604	# define DIV_STORE(a_Quotient, a_uReminder) puA = (a_Quotient), puD = (a_uReminder)
1605	# define DIV_STORE_U8(a_Quotient, a_uReminder) *puAX = (uint8_t)(a_Quotient) \| ((uint16_t)(a_uReminder) << 8)
1606
1607	# define MUL_LOAD_F1() *puA
1608	# define MUL_LOAD_F1_U8() ((uint8_t)*puAX)
1609
1610	# define MUL_STORE(a_Result) puA = (a_Result).s.Lo, puD = (a_Result).s.Hi
1611	# define MUL_STORE_U8(a_Result) *puAX = a_Result.u
1612
1613	# define MULDIV_NEG(a_Value, a_cBitsWidth2x) \
1614	(a_Value).u = UINT ## a_cBitsWidth2x ## _C(0) - (a_Value).u
1615	# define MULDIV_NEG_U128(a_Value, a_cBitsWidth2x) \
1616	RTUInt128AssignNeg(&(a_Value))
1617
1618	# define MULDIV_MUL(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \
1619	(a_Result).u = (uint ## a_cBitsWidth2x ## _t)(a_Factor1) * (a_Factor2)
1620	# define MULDIV_MUL_U128(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \
1621	RTUInt128MulU64ByU64(&(a_Result), a_Factor1, a_Factor2);
1622
1623	# define MULDIV_MODDIV(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \
1624	a_Quotient.u = (a_Dividend).u / (a_uDivisor), \
1625	a_Remainder.u = (a_Dividend).u % (a_uDivisor)
1626	# define MULDIV_MODDIV_U128(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \
1627	RTUInt128DivRemByU64(&a_Quotient, &a_Remainder, &a_Dividend, a_uDivisor)
1628
1629
1630	/*
1631	* MUL
1632	*/
1633	# define EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, a_Suffix, a_fIntelFlags) \
1634	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_mul_u,a_cBitsWidth,a_Suffix), a_Args) \
1635	{ \
1636	RTUINT ## a_cBitsWidth2x ## U Result; \
1637	a_fnMul(Result, a_fnLoadF1(), uFactor, a_cBitsWidth2x); \
1638	a_fnStore(Result); \
1639	\
1640	/* Calc EFLAGS: */ \
1641	uint32_t fEfl = *pfEFlags; \
1642	if (a_fIntelFlags) \
1643	{ /* Intel: 6700K and 10980XE behavior */ \
1644	fEfl &= ~(X86_EFL_SF \| X86_EFL_CF \| X86_EFL_OF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_PF); \
1645	if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \
1646	fEfl \|= X86_EFL_SF; \
1647	fEfl \|= g_afParity[Result.s.Lo & 0xff]; \
1648	if (Result.s.Hi != 0) \
1649	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1650	} \
1651	else \
1652	{ /* AMD: 3990X */ \
1653	if (Result.s.Hi != 0) \
1654	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1655	else \
1656	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
1657	} \
1658	*pfEFlags = fEfl; \
1659	return 0; \
1660	} \
1661
1662	# define EMIT_MUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul) \
1663	EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, RT_NOTHING, 1) \
1664	EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _intel, 1) \
1665	EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _amd, 0) \
1666
1667	EMIT_MUL(64, 128, (uint64_t puA, uint64_t puD, uint64_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
1668	MUL_LOAD_F1, MUL_STORE, MULDIV_MUL_U128)
1669	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1670	EMIT_MUL(32, 64, (uint32_t puA, uint32_t puD, uint32_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
1671	MUL_LOAD_F1, MUL_STORE, MULDIV_MUL)
1672	EMIT_MUL(16, 32, (uint16_t puA, uint16_t puD, uint16_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
1673	MUL_LOAD_F1, MUL_STORE, MULDIV_MUL)
1674	EMIT_MUL(8, 16, (uint16_t puAX, uint8_t uFactor, uint32_t pfEFlags), (puAX, uFactor, pfEFlags),
1675	MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_MUL)
1676	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1677
1678
1679	/*
1680	* IMUL
1681	*
1682	* The SF, ZF, AF and PF flags are "undefined". AMD (3990x) leaves these
1683	* flags as is. Whereas Intel skylake (6700K and 10980X (Cascade Lake)) always
1684	* clear AF and ZF and calculates SF and PF as per the lower half of the result.
1685	*/
1686	# define EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, \
1687	a_Suffix, a_fIntelFlags) \
1688	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_imul_u,a_cBitsWidth,a_Suffix),a_Args) \
1689	{ \
1690	RTUINT ## a_cBitsWidth2x ## U Result; \
1691	uint32_t fEfl = *pfEFlags & ~(X86_EFL_CF \| X86_EFL_OF); \
1692	\
1693	uint ## a_cBitsWidth ## _t const uFactor1 = a_fnLoadF1(); \
1694	if (!(uFactor1 & RT_BIT_64(a_cBitsWidth - 1))) \
1695	{ \
1696	if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \
1697	{ \
1698	a_fnMul(Result, uFactor1, uFactor2, a_cBitsWidth2x); \
1699	if (Result.s.Hi != 0 \|\| Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \
1700	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1701	} \
1702	else \
1703	{ \
1704	uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \
1705	a_fnMul(Result, uFactor1, uPositiveFactor2, a_cBitsWidth2x); \
1706	if (Result.s.Hi != 0 \|\| Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \
1707	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1708	a_fnNeg(Result, a_cBitsWidth2x); \
1709	} \
1710	} \
1711	else \
1712	{ \
1713	if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \
1714	{ \
1715	uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \
1716	a_fnMul(Result, uPositiveFactor1, uFactor2, a_cBitsWidth2x); \
1717	if (Result.s.Hi != 0 \|\| Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \
1718	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1719	a_fnNeg(Result, a_cBitsWidth2x); \
1720	} \
1721	else \
1722	{ \
1723	uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \
1724	uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \
1725	a_fnMul(Result, uPositiveFactor1, uPositiveFactor2, a_cBitsWidth2x); \
1726	if (Result.s.Hi != 0 \|\| Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \
1727	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1728	} \
1729	} \
1730	a_fnStore(Result); \
1731	\
1732	if (a_fIntelFlags) \
1733	{ \
1734	fEfl &= ~(X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF \| X86_EFL_PF); \
1735	if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \
1736	fEfl \|= X86_EFL_SF; \
1737	fEfl \|= g_afParity[Result.s.Lo & 0xff]; \
1738	} \
1739	*pfEFlags = fEfl; \
1740	return 0; \
1741	}
1742	# define EMIT_IMUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul) \
1743	EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, RT_NOTHING, 1) \
1744	EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _intel, 1) \
1745	EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _amd, 0)
1746
1747	EMIT_IMUL(64, 128, (uint64_t puA, uint64_t puD, uint64_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
1748	MUL_LOAD_F1, MUL_STORE, MULDIV_NEG_U128, MULDIV_MUL_U128)
1749	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1750	EMIT_IMUL(32, 64, (uint32_t puA, uint32_t puD, uint32_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
1751	MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL)
1752	EMIT_IMUL(16, 32, (uint16_t puA, uint16_t puD, uint16_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
1753	MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL)
1754	EMIT_IMUL(8, 16, (uint16_t puAX, uint8_t uFactor2, uint32_t pfEFlags), (puAX, uFactor2, pfEFlags),
1755	MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_NEG, MULDIV_MUL)
1756	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1757
1758
1759	/*
1760	* IMUL with two operands are mapped onto the three operand variant, ignoring
1761	* the high part of the product.
1762	*/
1763	# define EMIT_IMUL_TWO(a_cBits, a_uType) \
1764	IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits,(a_uType puDst, a_uType uSrc, uint32_t pfEFlags)) \
1765	{ \
1766	a_uType uIgn; \
1767	iemAImpl_imul_u ## a_cBits(puDst, &uIgn, uSrc, pfEFlags); \
1768	} \
1769	\
1770	IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _intel,(a_uType puDst, a_uType uSrc, uint32_t pfEFlags)) \
1771	{ \
1772	a_uType uIgn; \
1773	iemAImpl_imul_u ## a_cBits ## _intel(puDst, &uIgn, uSrc, pfEFlags); \
1774	} \
1775	\
1776	IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _amd,(a_uType puDst, a_uType uSrc, uint32_t pfEFlags)) \
1777	{ \
1778	a_uType uIgn; \
1779	iemAImpl_imul_u ## a_cBits ## _amd(puDst, &uIgn, uSrc, pfEFlags); \
1780	}
1781
1782	EMIT_IMUL_TWO(64, uint64_t)
1783	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1784	EMIT_IMUL_TWO(32, uint32_t)
1785	EMIT_IMUL_TWO(16, uint16_t)
1786	# endif
1787
1788
1789	/*
1790	* DIV
1791	*/
1792	# define EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, \
1793	a_Suffix, a_fIntelFlags) \
1794	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_div_u,a_cBitsWidth,a_Suffix),a_Args) \
1795	{ \
1796	RTUINT ## a_cBitsWidth2x ## U Dividend; \
1797	a_fnLoad(Dividend); \
1798	if ( uDivisor != 0 \
1799	&& Dividend.s.Hi < uDivisor) \
1800	{ \
1801	RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \
1802	a_fnDivRem(Quotient, Remainder, Dividend, uDivisor); \
1803	a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \
1804	\
1805	/* Calc EFLAGS: Intel 6700K and 10980XE leaves them alone. AMD 3990X sets AF and clears PF, ZF and SF. */ \
1806	if (!a_fIntelFlags) \
1807	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1808	return 0; \
1809	} \
1810	/* #DE */ \
1811	return -1; \
1812	}
1813	# define EMIT_DIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem) \
1814	EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, RT_NOTHING, 1) \
1815	EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _intel, 1) \
1816	EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _amd, 0)
1817
1818	EMIT_DIV(64,128,(uint64_t puA, uint64_t puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1819	DIV_LOAD, DIV_STORE, MULDIV_MODDIV_U128)
1820	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1821	EMIT_DIV(32,64, (uint32_t puA, uint32_t puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1822	DIV_LOAD, DIV_STORE, MULDIV_MODDIV)
1823	EMIT_DIV(16,32, (uint16_t puA, uint16_t puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1824	DIV_LOAD, DIV_STORE, MULDIV_MODDIV)
1825	EMIT_DIV(8,16, (uint16_t puAX, uint8_t uDivisor, uint32_t pfEFlags), (puAX, uDivisor, pfEFlags),
1826	DIV_LOAD_U8, DIV_STORE_U8, MULDIV_MODDIV)
1827	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1828
1829
1830	/*
1831	* IDIV
1832	*
1833	* EFLAGS are ignored and left as-is by Intel 6700K and 10980XE. AMD 3990X will
1834	* set AF and clear PF, ZF and SF just like it does for DIV.
1835	*
1836	*/
1837	# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
1838	# define EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, \
1839	a_Suffix, a_fIntelFlags) \
1840	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_idiv_u,a_cBitsWidth,a_Suffix),a_Args) \
1841	{ \
1842	/* Note! Skylake leaves all flags alone. */ \
1843	\
1844	/** @todo overflow checks */ \
1845	if (uDivisor != 0) \
1846	{ \
1847	/* \
1848	* Convert to unsigned division. \
1849	*/ \
1850	RTUINT ## a_cBitsWidth2x ## U Dividend; \
1851	a_fnLoad(Dividend); \
1852	bool const fSignedDividend = RT_BOOL(Dividend.s.Hi & RT_BIT_64(a_cBitsWidth - 1)); \
1853	if (fSignedDividend) \
1854	a_fnNeg(Dividend, a_cBitsWidth2x); \
1855	\
1856	uint ## a_cBitsWidth ## _t uDivisorPositive; \
1857	if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \
1858	uDivisorPositive = uDivisor; \
1859	else \
1860	uDivisorPositive = UINT ## a_cBitsWidth ## _C(0) - uDivisor; \
1861	\
1862	RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \
1863	a_fnDivRem(Quotient, Remainder, Dividend, uDivisorPositive); \
1864	\
1865	/* \
1866	* Setup the result, checking for overflows. \
1867	*/ \
1868	if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \
1869	{ \
1870	if (!fSignedDividend) \
1871	{ \
1872	/* Positive divisor, positive dividend => result positive. */ \
1873	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \
1874	{ \
1875	a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \
1876	if (!a_fIntelFlags) \
1877	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1878	return 0; \
1879	} \
1880	} \
1881	else \
1882	{ \
1883	/* Positive divisor, negative dividend => result negative. */ \
1884	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \
1885	{ \
1886	a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \
1887	if (!a_fIntelFlags) \
1888	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1889	return 0; \
1890	} \
1891	} \
1892	} \
1893	else \
1894	{ \
1895	if (!fSignedDividend) \
1896	{ \
1897	/* Negative divisor, positive dividend => negative quotient, positive remainder. */ \
1898	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \
1899	{ \
1900	a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, Remainder.s.Lo); \
1901	if (!a_fIntelFlags) \
1902	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1903	return 0; \
1904	} \
1905	} \
1906	else \
1907	{ \
1908	/* Negative divisor, negative dividend => positive quotient, negative remainder. */ \
1909	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \
1910	{ \
1911	a_fnStore(Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \
1912	if (!a_fIntelFlags) \
1913	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1914	return 0; \
1915	} \
1916	} \
1917	} \
1918	} \
1919	/* #DE */ \
1920	return -1; \
1921	}
1922	# define EMIT_IDIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem) \
1923	EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, RT_NOTHING, 1) \
1924	EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _intel, 1) \
1925	EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _amd, 0)
1926
1927	EMIT_IDIV(64,128,(uint64_t puA, uint64_t puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1928	DIV_LOAD, DIV_STORE, MULDIV_NEG_U128, MULDIV_MODDIV_U128)
1929	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1930	EMIT_IDIV(32,64,(uint32_t puA, uint32_t puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1931	DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV)
1932	EMIT_IDIV(16,32,(uint16_t puA, uint16_t puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1933	DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV)
1934	EMIT_IDIV(8,16,(uint16_t puAX, uint8_t uDivisor, uint32_t pfEFlags), (puAX, uDivisor, pfEFlags),
1935	DIV_LOAD_U8, DIV_STORE_U8, MULDIV_NEG, MULDIV_MODDIV)
1936	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1937	# endif /* DOXYGEN_RUNNING */
1938
1939
1940	/*********************************************************************************************************************************
1941	* Unary operations. *
1942	*********************************************************************************************************************************/
1943
1944	/** @def IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC
1945	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an INC or DEC instruction.
1946	*
1947	* CF is NOT modified for hysterical raisins (allegedly for carrying and
1948	* borrowing in arithmetic loops on intel 8008).
1949	*
1950	* @returns Status bits.
1951	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
1952	* @param a_uResult Unsigned result value.
1953	* @param a_uDst The original destination value (for AF calc).
1954	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
1955	* @param a_OfMethod 0 for INC-style, 1 for DEC-style.
1956	*/
1957	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth, a_OfMethod) \
1958	do { \
1959	uint32_t fEflTmp = *(a_pfEFlags); \
1960	fEflTmp &= ~X86_EFL_STATUS_BITS \| X86_EFL_CF; \
1961	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
1962	fEflTmp \|= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
1963	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
1964	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
1965	fEflTmp \|= X86_EFL_GET_OF_ ## a_cBitsWidth(a_OfMethod == 0 ? (((a_uDst) ^ RT_BIT_64(a_cBitsWidth - 1)) & (a_uResult)) \
1966	: ((a_uDst) & ((a_uResult) ^ RT_BIT_64(a_cBitsWidth - 1))) ); \
1967	*(a_pfEFlags) = fEflTmp; \
1968	} while (0)
1969
1970	/*
1971	* INC
1972	*/
1973
1974	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u64,(uint64_t puDst, uint32_t pfEFlags))
1975	{
1976	uint64_t uDst = *puDst;
1977	uint64_t uResult = uDst + 1;
1978	*puDst = uResult;
1979	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 0 /INC/);
1980	}
1981
1982	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1983
1984	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u32,(uint32_t puDst, uint32_t pfEFlags))
1985	{
1986	uint32_t uDst = *puDst;
1987	uint32_t uResult = uDst + 1;
1988	*puDst = uResult;
1989	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 0 /INC/);
1990	}
1991
1992
1993	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u16,(uint16_t puDst, uint32_t pfEFlags))
1994	{
1995	uint16_t uDst = *puDst;
1996	uint16_t uResult = uDst + 1;
1997	*puDst = uResult;
1998	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 0 /INC/);
1999	}
2000
2001	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u8,(uint8_t puDst, uint32_t pfEFlags))
2002	{
2003	uint8_t uDst = *puDst;
2004	uint8_t uResult = uDst + 1;
2005	*puDst = uResult;
2006	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 0 /INC/);
2007	}
2008
2009	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2010
2011
2012	/*
2013	* DEC
2014	*/
2015
2016	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u64,(uint64_t puDst, uint32_t pfEFlags))
2017	{
2018	uint64_t uDst = *puDst;
2019	uint64_t uResult = uDst - 1;
2020	*puDst = uResult;
2021	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 1 /INC/);
2022	}
2023
2024	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2025
2026	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u32,(uint32_t puDst, uint32_t pfEFlags))
2027	{
2028	uint32_t uDst = *puDst;
2029	uint32_t uResult = uDst - 1;
2030	*puDst = uResult;
2031	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 1 /INC/);
2032	}
2033
2034
2035	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u16,(uint16_t puDst, uint32_t pfEFlags))
2036	{
2037	uint16_t uDst = *puDst;
2038	uint16_t uResult = uDst - 1;
2039	*puDst = uResult;
2040	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 1 /INC/);
2041	}
2042
2043
2044	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u8,(uint8_t puDst, uint32_t pfEFlags))
2045	{
2046	uint8_t uDst = *puDst;
2047	uint8_t uResult = uDst - 1;
2048	*puDst = uResult;
2049	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 1 /INC/);
2050	}
2051
2052	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2053
2054
2055	/*
2056	* NOT
2057	*/
2058
2059	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u64,(uint64_t puDst, uint32_t pfEFlags))
2060	{
2061	uint64_t uDst = *puDst;
2062	uint64_t uResult = ~uDst;
2063	*puDst = uResult;
2064	/* EFLAGS are not modified. */
2065	RT_NOREF_PV(pfEFlags);
2066	}
2067
2068	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2069
2070	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u32,(uint32_t puDst, uint32_t pfEFlags))
2071	{
2072	uint32_t uDst = *puDst;
2073	uint32_t uResult = ~uDst;
2074	*puDst = uResult;
2075	/* EFLAGS are not modified. */
2076	RT_NOREF_PV(pfEFlags);
2077	}
2078
2079	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u16,(uint16_t puDst, uint32_t pfEFlags))
2080	{
2081	uint16_t uDst = *puDst;
2082	uint16_t uResult = ~uDst;
2083	*puDst = uResult;
2084	/* EFLAGS are not modified. */
2085	RT_NOREF_PV(pfEFlags);
2086	}
2087
2088	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u8,(uint8_t puDst, uint32_t pfEFlags))
2089	{
2090	uint8_t uDst = *puDst;
2091	uint8_t uResult = ~uDst;
2092	*puDst = uResult;
2093	/* EFLAGS are not modified. */
2094	RT_NOREF_PV(pfEFlags);
2095	}
2096
2097	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2098
2099
2100	/*
2101	* NEG
2102	*/
2103
2104	/**
2105	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an NEG instruction.
2106	*
2107	* @returns Status bits.
2108	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
2109	* @param a_uResult Unsigned result value.
2110	* @param a_uDst The original destination value (for AF calc).
2111	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
2112	*/
2113	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth) \
2114	do { \
2115	uint32_t fEflTmp = *(a_pfEFlags); \
2116	fEflTmp &= ~X86_EFL_STATUS_BITS & ~X86_EFL_CF; \
2117	fEflTmp \|= ((a_uDst) != 0) << X86_EFL_CF_BIT; \
2118	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
2119	fEflTmp \|= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
2120	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
2121	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
2122	fEflTmp \|= X86_EFL_GET_OF_ ## a_cBitsWidth((a_uDst) & (a_uResult)); \
2123	*(a_pfEFlags) = fEflTmp; \
2124	} while (0)
2125
2126	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u64,(uint64_t puDst, uint32_t pfEFlags))
2127	{
2128	uint64_t uDst = *puDst;
2129	uint64_t uResult = (uint64_t)0 - uDst;
2130	*puDst = uResult;
2131	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 64);
2132	}
2133
2134	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2135
2136	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u32,(uint32_t puDst, uint32_t pfEFlags))
2137	{
2138	uint32_t uDst = *puDst;
2139	uint32_t uResult = (uint32_t)0 - uDst;
2140	*puDst = uResult;
2141	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 32);
2142	}
2143
2144
2145	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u16,(uint16_t puDst, uint32_t pfEFlags))
2146	{
2147	uint16_t uDst = *puDst;
2148	uint16_t uResult = (uint16_t)0 - uDst;
2149	*puDst = uResult;
2150	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 16);
2151	}
2152
2153
2154	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u8,(uint8_t puDst, uint32_t pfEFlags))
2155	{
2156	uint8_t uDst = *puDst;
2157	uint8_t uResult = (uint8_t)0 - uDst;
2158	*puDst = uResult;
2159	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 8);
2160	}
2161
2162	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2163
2164	/*
2165	* Locked variants.
2166	*/
2167
2168	/** Emit a function for doing a locked unary operand operation. */
2169	# define EMIT_LOCKED_UNARY_OP(a_Mnemonic, a_cBitsWidth) \
2170	IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \
2171	uint32_t *pfEFlags)) \
2172	{ \
2173	uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
2174	uint ## a_cBitsWidth ## _t uTmp; \
2175	uint32_t fEflTmp; \
2176	do \
2177	{ \
2178	uTmp = uOld; \
2179	fEflTmp = *pfEFlags; \
2180	iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, &fEflTmp); \
2181	} while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \
2182	*pfEFlags = fEflTmp; \
2183	}
2184
2185	EMIT_LOCKED_UNARY_OP(inc, 64)
2186	EMIT_LOCKED_UNARY_OP(dec, 64)
2187	EMIT_LOCKED_UNARY_OP(not, 64)
2188	EMIT_LOCKED_UNARY_OP(neg, 64)
2189	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2190	EMIT_LOCKED_UNARY_OP(inc, 32)
2191	EMIT_LOCKED_UNARY_OP(dec, 32)
2192	EMIT_LOCKED_UNARY_OP(not, 32)
2193	EMIT_LOCKED_UNARY_OP(neg, 32)
2194
2195	EMIT_LOCKED_UNARY_OP(inc, 16)
2196	EMIT_LOCKED_UNARY_OP(dec, 16)
2197	EMIT_LOCKED_UNARY_OP(not, 16)
2198	EMIT_LOCKED_UNARY_OP(neg, 16)
2199
2200	EMIT_LOCKED_UNARY_OP(inc, 8)
2201	EMIT_LOCKED_UNARY_OP(dec, 8)
2202	EMIT_LOCKED_UNARY_OP(not, 8)
2203	EMIT_LOCKED_UNARY_OP(neg, 8)
2204	# endif
2205
2206	#endif /* !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2207
2208
2209	/*********************************************************************************************************************************
2210	* Shifting and Rotating *
2211	*********************************************************************************************************************************/
2212
2213	/*
2214	* ROL
2215	*/
2216	#define EMIT_ROL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \
2217	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rol_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2218	{ \
2219	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2220	if (cShift) \
2221	{ \
2222	if (a_cBitsWidth < 32) \
2223	cShift &= a_cBitsWidth - 1; \
2224	a_uType const uDst = *puDst; \
2225	a_uType const uResult = a_fnHlp(uDst, cShift); \
2226	*puDst = uResult; \
2227	\
2228	/* Calc EFLAGS. The OF bit is undefined if cShift > 1, we implement \
2229	it the same way as for 1 bit shifts. */ \
2230	AssertCompile(X86_EFL_CF_BIT == 0); \
2231	uint32_t fEfl = *pfEFlags; \
2232	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2233	uint32_t const fCarry = (uResult & X86_EFL_CF); \
2234	fEfl \|= fCarry; \
2235	if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
2236	fEfl \|= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; \
2237	else /* Intel 10980XE: According to the first sub-shift: */ \
2238	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
2239	*pfEFlags = fEfl; \
2240	} \
2241	}
2242
2243	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2244	EMIT_ROL(64, uint64_t, RT_NOTHING, 1, ASMRotateLeftU64)
2245	#endif
2246	EMIT_ROL(64, uint64_t, _intel, 1, ASMRotateLeftU64)
2247	EMIT_ROL(64, uint64_t, _amd, 0, ASMRotateLeftU64)
2248
2249	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2250	EMIT_ROL(32, uint32_t, RT_NOTHING, 1, ASMRotateLeftU32)
2251	#endif
2252	EMIT_ROL(32, uint32_t, _intel, 1, ASMRotateLeftU32)
2253	EMIT_ROL(32, uint32_t, _amd, 0, ASMRotateLeftU32)
2254
2255	DECL_FORCE_INLINE(uint16_t) iemAImpl_rol_u16_hlp(uint16_t uValue, uint8_t cShift)
2256	{
2257	return (uValue << cShift) \| (uValue >> (16 - cShift));
2258	}
2259	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2260	EMIT_ROL(16, uint16_t, RT_NOTHING, 1, iemAImpl_rol_u16_hlp)
2261	#endif
2262	EMIT_ROL(16, uint16_t, _intel, 1, iemAImpl_rol_u16_hlp)
2263	EMIT_ROL(16, uint16_t, _amd, 0, iemAImpl_rol_u16_hlp)
2264
2265	DECL_FORCE_INLINE(uint8_t) iemAImpl_rol_u8_hlp(uint8_t uValue, uint8_t cShift)
2266	{
2267	return (uValue << cShift) \| (uValue >> (8 - cShift));
2268	}
2269	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2270	EMIT_ROL(8, uint8_t, RT_NOTHING, 1, iemAImpl_rol_u8_hlp)
2271	#endif
2272	EMIT_ROL(8, uint8_t, _intel, 1, iemAImpl_rol_u8_hlp)
2273	EMIT_ROL(8, uint8_t, _amd, 0, iemAImpl_rol_u8_hlp)
2274
2275
2276	/*
2277	* ROR
2278	*/
2279	#define EMIT_ROR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \
2280	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_ror_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2281	{ \
2282	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2283	if (cShift) \
2284	{ \
2285	if (a_cBitsWidth < 32) \
2286	cShift &= a_cBitsWidth - 1; \
2287	a_uType const uDst = *puDst; \
2288	a_uType const uResult = a_fnHlp(uDst, cShift); \
2289	*puDst = uResult; \
2290	\
2291	/* Calc EFLAGS: */ \
2292	AssertCompile(X86_EFL_CF_BIT == 0); \
2293	uint32_t fEfl = *pfEFlags; \
2294	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2295	uint32_t const fCarry = (uResult >> ((a_cBitsWidth) - 1)) & X86_EFL_CF; \
2296	fEfl \|= fCarry; \
2297	if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
2298	fEfl \|= (((uResult >> ((a_cBitsWidth) - 2)) ^ fCarry) & 1) << X86_EFL_OF_BIT; \
2299	else /* Intel 10980XE: According to the first sub-shift: */ \
2300	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << (a_cBitsWidth - 1))); \
2301	*pfEFlags = fEfl; \
2302	} \
2303	}
2304
2305	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2306	EMIT_ROR(64, uint64_t, RT_NOTHING, 1, ASMRotateRightU64)
2307	#endif
2308	EMIT_ROR(64, uint64_t, _intel, 1, ASMRotateRightU64)
2309	EMIT_ROR(64, uint64_t, _amd, 0, ASMRotateRightU64)
2310
2311	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2312	EMIT_ROR(32, uint32_t, RT_NOTHING, 1, ASMRotateRightU32)
2313	#endif
2314	EMIT_ROR(32, uint32_t, _intel, 1, ASMRotateRightU32)
2315	EMIT_ROR(32, uint32_t, _amd, 0, ASMRotateRightU32)
2316
2317	DECL_FORCE_INLINE(uint16_t) iemAImpl_ror_u16_hlp(uint16_t uValue, uint8_t cShift)
2318	{
2319	return (uValue >> cShift) \| (uValue << (16 - cShift));
2320	}
2321	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2322	EMIT_ROR(16, uint16_t, RT_NOTHING, 1, iemAImpl_ror_u16_hlp)
2323	#endif
2324	EMIT_ROR(16, uint16_t, _intel, 1, iemAImpl_ror_u16_hlp)
2325	EMIT_ROR(16, uint16_t, _amd, 0, iemAImpl_ror_u16_hlp)
2326
2327	DECL_FORCE_INLINE(uint8_t) iemAImpl_ror_u8_hlp(uint8_t uValue, uint8_t cShift)
2328	{
2329	return (uValue >> cShift) \| (uValue << (8 - cShift));
2330	}
2331	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2332	EMIT_ROR(8, uint8_t, RT_NOTHING, 1, iemAImpl_ror_u8_hlp)
2333	#endif
2334	EMIT_ROR(8, uint8_t, _intel, 1, iemAImpl_ror_u8_hlp)
2335	EMIT_ROR(8, uint8_t, _amd, 0, iemAImpl_ror_u8_hlp)
2336
2337
2338	/*
2339	* RCL
2340	*/
2341	#define EMIT_RCL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2342	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcl_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2343	{ \
2344	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2345	if (a_cBitsWidth < 32 && a_fIntelFlags) \
2346	cShift %= a_cBitsWidth + 1; \
2347	if (cShift) \
2348	{ \
2349	if (a_cBitsWidth < 32 && !a_fIntelFlags) \
2350	cShift %= a_cBitsWidth + 1; \
2351	a_uType const uDst = *puDst; \
2352	a_uType uResult = uDst << cShift; \
2353	if (cShift > 1) \
2354	uResult \|= uDst >> (a_cBitsWidth + 1 - cShift); \
2355	\
2356	AssertCompile(X86_EFL_CF_BIT == 0); \
2357	uint32_t fEfl = *pfEFlags; \
2358	uint32_t fInCarry = fEfl & X86_EFL_CF; \
2359	uResult \|= (a_uType)fInCarry << (cShift - 1); \
2360	\
2361	*puDst = uResult; \
2362	\
2363	/* Calc EFLAGS. */ \
2364	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2365	uint32_t const fOutCarry = a_cBitsWidth >= 32 \|\| cShift \
2366	? (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF : fInCarry; \
2367	fEfl \|= fOutCarry; \
2368	if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
2369	fEfl \|= ((uResult >> (a_cBitsWidth - 1)) ^ fOutCarry) << X86_EFL_OF_BIT; \
2370	else /* Intel 10980XE: According to the first sub-shift: */ \
2371	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
2372	*pfEFlags = fEfl; \
2373	} \
2374	}
2375
2376	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2377	EMIT_RCL(64, uint64_t, RT_NOTHING, 1)
2378	#endif
2379	EMIT_RCL(64, uint64_t, _intel, 1)
2380	EMIT_RCL(64, uint64_t, _amd, 0)
2381
2382	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2383	EMIT_RCL(32, uint32_t, RT_NOTHING, 1)
2384	#endif
2385	EMIT_RCL(32, uint32_t, _intel, 1)
2386	EMIT_RCL(32, uint32_t, _amd, 0)
2387
2388	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2389	EMIT_RCL(16, uint16_t, RT_NOTHING, 1)
2390	#endif
2391	EMIT_RCL(16, uint16_t, _intel, 1)
2392	EMIT_RCL(16, uint16_t, _amd, 0)
2393
2394	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2395	EMIT_RCL(8, uint8_t, RT_NOTHING, 1)
2396	#endif
2397	EMIT_RCL(8, uint8_t, _intel, 1)
2398	EMIT_RCL(8, uint8_t, _amd, 0)
2399
2400
2401	/*
2402	* RCR
2403	*/
2404	#define EMIT_RCR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2405	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcr_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2406	{ \
2407	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2408	if (a_cBitsWidth < 32 && a_fIntelFlags) \
2409	cShift %= a_cBitsWidth + 1; \
2410	if (cShift) \
2411	{ \
2412	if (a_cBitsWidth < 32 && !a_fIntelFlags) \
2413	cShift %= a_cBitsWidth + 1; \
2414	a_uType const uDst = *puDst; \
2415	a_uType uResult = uDst >> cShift; \
2416	if (cShift > 1) \
2417	uResult \|= uDst << (a_cBitsWidth + 1 - cShift); \
2418	\
2419	AssertCompile(X86_EFL_CF_BIT == 0); \
2420	uint32_t fEfl = *pfEFlags; \
2421	uint32_t fInCarry = fEfl & X86_EFL_CF; \
2422	uResult \|= (a_uType)fInCarry << (a_cBitsWidth - cShift); \
2423	*puDst = uResult; \
2424	\
2425	/* Calc EFLAGS. The OF bit is undefined if cShift > 1, we implement \
2426	it the same way as for 1 bit shifts. */ \
2427	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2428	uint32_t const fOutCarry = (uDst >> (cShift - 1)) & X86_EFL_CF; \
2429	fEfl \|= fOutCarry; \
2430	if (!a_fIntelFlags) /* AMD 3990X: XOR two most signficant bits of the result: */ \
2431	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uResult ^ (uResult << 1)); \
2432	else /* Intel 10980XE: same as AMD, but only for the first sub-shift: */ \
2433	fEfl \|= (fInCarry ^ (uint32_t)(uDst >> (a_cBitsWidth - 1))) << X86_EFL_OF_BIT; \
2434	*pfEFlags = fEfl; \
2435	} \
2436	}
2437
2438	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2439	EMIT_RCR(64, uint64_t, RT_NOTHING, 1)
2440	#endif
2441	EMIT_RCR(64, uint64_t, _intel, 1)
2442	EMIT_RCR(64, uint64_t, _amd, 0)
2443
2444	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2445	EMIT_RCR(32, uint32_t, RT_NOTHING, 1)
2446	#endif
2447	EMIT_RCR(32, uint32_t, _intel, 1)
2448	EMIT_RCR(32, uint32_t, _amd, 0)
2449
2450	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2451	EMIT_RCR(16, uint16_t, RT_NOTHING, 1)
2452	#endif
2453	EMIT_RCR(16, uint16_t, _intel, 1)
2454	EMIT_RCR(16, uint16_t, _amd, 0)
2455
2456	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2457	EMIT_RCR(8, uint8_t, RT_NOTHING, 1)
2458	#endif
2459	EMIT_RCR(8, uint8_t, _intel, 1)
2460	EMIT_RCR(8, uint8_t, _amd, 0)
2461
2462
2463	/*
2464	* SHL
2465	*/
2466	#define EMIT_SHL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2467	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shl_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2468	{ \
2469	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2470	if (cShift) \
2471	{ \
2472	a_uType const uDst = *puDst; \
2473	a_uType uResult = uDst << cShift; \
2474	*puDst = uResult; \
2475	\
2476	/* Calc EFLAGS. */ \
2477	AssertCompile(X86_EFL_CF_BIT == 0); \
2478	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2479	uint32_t fCarry = (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; \
2480	fEfl \|= fCarry; \
2481	if (!a_fIntelFlags) \
2482	fEfl \|= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; /* AMD 3990X: Last shift result. */ \
2483	else \
2484	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); /* Intel 10980XE: First shift result. */ \
2485	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2486	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2487	fEfl \|= g_afParity[uResult & 0xff]; \
2488	if (!a_fIntelFlags) \
2489	fEfl \|= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
2490	*pfEFlags = fEfl; \
2491	} \
2492	}
2493
2494	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2495	EMIT_SHL(64, uint64_t, RT_NOTHING, 1)
2496	#endif
2497	EMIT_SHL(64, uint64_t, _intel, 1)
2498	EMIT_SHL(64, uint64_t, _amd, 0)
2499
2500	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2501	EMIT_SHL(32, uint32_t, RT_NOTHING, 1)
2502	#endif
2503	EMIT_SHL(32, uint32_t, _intel, 1)
2504	EMIT_SHL(32, uint32_t, _amd, 0)
2505
2506	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2507	EMIT_SHL(16, uint16_t, RT_NOTHING, 1)
2508	#endif
2509	EMIT_SHL(16, uint16_t, _intel, 1)
2510	EMIT_SHL(16, uint16_t, _amd, 0)
2511
2512	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2513	EMIT_SHL(8, uint8_t, RT_NOTHING, 1)
2514	#endif
2515	EMIT_SHL(8, uint8_t, _intel, 1)
2516	EMIT_SHL(8, uint8_t, _amd, 0)
2517
2518
2519	/*
2520	* SHR
2521	*/
2522	#define EMIT_SHR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2523	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shr_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2524	{ \
2525	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2526	if (cShift) \
2527	{ \
2528	a_uType const uDst = *puDst; \
2529	a_uType uResult = uDst >> cShift; \
2530	*puDst = uResult; \
2531	\
2532	/* Calc EFLAGS. */ \
2533	AssertCompile(X86_EFL_CF_BIT == 0); \
2534	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2535	fEfl \|= (uDst >> (cShift - 1)) & X86_EFL_CF; \
2536	if (a_fIntelFlags \|\| cShift == 1) /* AMD 3990x does what intel documents; Intel 10980XE does this for all shift counts. */ \
2537	fEfl \|= (uDst >> (a_cBitsWidth - 1)) << X86_EFL_OF_BIT; \
2538	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2539	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2540	fEfl \|= g_afParity[uResult & 0xff]; \
2541	if (!a_fIntelFlags) \
2542	fEfl \|= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
2543	*pfEFlags = fEfl; \
2544	} \
2545	}
2546
2547	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2548	EMIT_SHR(64, uint64_t, RT_NOTHING, 1)
2549	#endif
2550	EMIT_SHR(64, uint64_t, _intel, 1)
2551	EMIT_SHR(64, uint64_t, _amd, 0)
2552
2553	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2554	EMIT_SHR(32, uint32_t, RT_NOTHING, 1)
2555	#endif
2556	EMIT_SHR(32, uint32_t, _intel, 1)
2557	EMIT_SHR(32, uint32_t, _amd, 0)
2558
2559	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2560	EMIT_SHR(16, uint16_t, RT_NOTHING, 1)
2561	#endif
2562	EMIT_SHR(16, uint16_t, _intel, 1)
2563	EMIT_SHR(16, uint16_t, _amd, 0)
2564
2565	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2566	EMIT_SHR(8, uint8_t, RT_NOTHING, 1)
2567	#endif
2568	EMIT_SHR(8, uint8_t, _intel, 1)
2569	EMIT_SHR(8, uint8_t, _amd, 0)
2570
2571
2572	/*
2573	* SAR
2574	*/
2575	#define EMIT_SAR(a_cBitsWidth, a_uType, a_iType, a_Suffix, a_fIntelFlags) \
2576	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_sar_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2577	{ \
2578	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2579	if (cShift) \
2580	{ \
2581	a_iType const iDst = (a_iType)*puDst; \
2582	a_uType uResult = iDst >> cShift; \
2583	*puDst = uResult; \
2584	\
2585	/* Calc EFLAGS. \
2586	Note! The OF flag is always zero because the result never differs from the input. */ \
2587	AssertCompile(X86_EFL_CF_BIT == 0); \
2588	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2589	fEfl \|= (iDst >> (cShift - 1)) & X86_EFL_CF; \
2590	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2591	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2592	fEfl \|= g_afParity[uResult & 0xff]; \
2593	if (!a_fIntelFlags) \
2594	fEfl \|= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
2595	*pfEFlags = fEfl; \
2596	} \
2597	}
2598
2599	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2600	EMIT_SAR(64, uint64_t, int64_t, RT_NOTHING, 1)
2601	#endif
2602	EMIT_SAR(64, uint64_t, int64_t, _intel, 1)
2603	EMIT_SAR(64, uint64_t, int64_t, _amd, 0)
2604
2605	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2606	EMIT_SAR(32, uint32_t, int32_t, RT_NOTHING, 1)
2607	#endif
2608	EMIT_SAR(32, uint32_t, int32_t, _intel, 1)
2609	EMIT_SAR(32, uint32_t, int32_t, _amd, 0)
2610
2611	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2612	EMIT_SAR(16, uint16_t, int16_t, RT_NOTHING, 1)
2613	#endif
2614	EMIT_SAR(16, uint16_t, int16_t, _intel, 1)
2615	EMIT_SAR(16, uint16_t, int16_t, _amd, 0)
2616
2617	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2618	EMIT_SAR(8, uint8_t, int8_t, RT_NOTHING, 1)
2619	#endif
2620	EMIT_SAR(8, uint8_t, int8_t, _intel, 1)
2621	EMIT_SAR(8, uint8_t, int8_t, _amd, 0)
2622
2623
2624	/*
2625	* SHLD
2626	*
2627	* - CF is the last bit shifted out of puDst.
2628	* - AF is always cleared by Intel 10980XE.
2629	* - AF is always set by AMD 3990X.
2630	* - OF is set according to the first shift on Intel 10980XE, it seems.
2631	* - OF is set according to the last sub-shift on AMD 3990X.
2632	* - ZF, SF and PF are calculated according to the result by both vendors.
2633	*
2634	* For 16-bit shifts the count mask isn't 15, but 31, and the CPU will
2635	* pick either the source register or the destination register for input bits
2636	* when going beyond 16. According to https://www.sandpile.org/x86/flags.htm
2637	* intel has changed behaviour here several times. We implement what current
2638	* skylake based does for now, we can extend this later as needed.
2639	*/
2640	#define EMIT_SHLD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2641	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shld_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, uint8_t cShift, \
2642	uint32_t *pfEFlags)) \
2643	{ \
2644	cShift &= a_cBitsWidth - 1; \
2645	if (cShift) \
2646	{ \
2647	a_uType const uDst = *puDst; \
2648	a_uType uResult = uDst << cShift; \
2649	uResult \|= uSrc >> (a_cBitsWidth - cShift); \
2650	*puDst = uResult; \
2651	\
2652	/* CALC EFLAGS: */ \
2653	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2654	if (a_fIntelFlags) \
2655	/* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
2656	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
2657	else \
2658	{ /* AMD 3990X: Set according to last shift. AF always set. */ \
2659	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth((uDst << (cShift - 1)) ^ uResult); \
2660	fEfl \|= X86_EFL_AF; \
2661	} \
2662	AssertCompile(X86_EFL_CF_BIT == 0); \
2663	fEfl \|= (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; /* CF = last bit shifted out */ \
2664	fEfl \|= g_afParity[uResult & 0xff]; \
2665	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2666	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2667	*pfEFlags = fEfl; \
2668	} \
2669	}
2670
2671	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2672	EMIT_SHLD(64, uint64_t, RT_NOTHING, 1)
2673	#endif
2674	EMIT_SHLD(64, uint64_t, _intel, 1)
2675	EMIT_SHLD(64, uint64_t, _amd, 0)
2676
2677	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2678	EMIT_SHLD(32, uint32_t, RT_NOTHING, 1)
2679	#endif
2680	EMIT_SHLD(32, uint32_t, _intel, 1)
2681	EMIT_SHLD(32, uint32_t, _amd, 0)
2682
2683	#define EMIT_SHLD_16(a_Suffix, a_fIntelFlags) \
2684	IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shld_u16,a_Suffix),(uint16_t puDst, uint16_t uSrc, uint8_t cShift, uint32_t pfEFlags)) \
2685	{ \
2686	cShift &= 31; \
2687	if (cShift) \
2688	{ \
2689	uint16_t const uDst = *puDst; \
2690	uint64_t const uTmp = a_fIntelFlags \
2691	? ((uint64_t)uDst << 32) \| ((uint32_t)uSrc << 16) \| uDst \
2692	: ((uint64_t)uDst << 32) \| ((uint32_t)uSrc << 16) \| uSrc; \
2693	uint16_t const uResult = (uint16_t)((uTmp << cShift) >> 32); \
2694	*puDst = uResult; \
2695	\
2696	/* CALC EFLAGS: */ \
2697	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2698	AssertCompile(X86_EFL_CF_BIT == 0); \
2699	if (a_fIntelFlags) \
2700	{ \
2701	fEfl \|= (uTmp >> (48 - cShift)) & X86_EFL_CF; /* CF = last bit shifted out of the combined operand */ \
2702	/* Intel 6700K & 10980XE: OF is et according to the first shift. AF always cleared. */ \
2703	fEfl \|= X86_EFL_GET_OF_16(uDst ^ (uDst << 1)); \
2704	} \
2705	else \
2706	{ \
2707	/* AMD 3990X: OF is set according to last shift, with some weirdness. AF always set. CF = last bit shifted out of uDst. */ \
2708	if (cShift < 16) \
2709	{ \
2710	fEfl \|= (uDst >> (16 - cShift)) & X86_EFL_CF; \
2711	fEfl \|= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ uResult); \
2712	} \
2713	else \
2714	fEfl \|= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ 0); \
2715	fEfl \|= X86_EFL_AF; \
2716	} \
2717	fEfl \|= g_afParity[uResult & 0xff]; \
2718	fEfl \|= X86_EFL_CALC_SF(uResult, 16); \
2719	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2720	*pfEFlags = fEfl; \
2721	} \
2722	}
2723
2724	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2725	EMIT_SHLD_16(RT_NOTHING, 1)
2726	#endif
2727	EMIT_SHLD_16(_intel, 1)
2728	EMIT_SHLD_16(_amd, 0)
2729
2730
2731	/*
2732	* SHRD
2733	*
2734	* EFLAGS behaviour seems to be the same as with SHLD:
2735	* - CF is the last bit shifted out of puDst.
2736	* - AF is always cleared by Intel 10980XE.
2737	* - AF is always set by AMD 3990X.
2738	* - OF is set according to the first shift on Intel 10980XE, it seems.
2739	* - OF is set according to the last sub-shift on AMD 3990X.
2740	* - ZF, SF and PF are calculated according to the result by both vendors.
2741	*
2742	* For 16-bit shifts the count mask isn't 15, but 31, and the CPU will
2743	* pick either the source register or the destination register for input bits
2744	* when going beyond 16. According to https://www.sandpile.org/x86/flags.htm
2745	* intel has changed behaviour here several times. We implement what current
2746	* skylake based does for now, we can extend this later as needed.
2747	*/
2748	#define EMIT_SHRD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2749	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shrd_u,a_cBitsWidth,a_Suffix),(a_uType puDst, a_uType uSrc, uint8_t cShift, uint32_t pfEFlags)) \
2750	{ \
2751	cShift &= a_cBitsWidth - 1; \
2752	if (cShift) \
2753	{ \
2754	a_uType const uDst = *puDst; \
2755	a_uType uResult = uDst >> cShift; \
2756	uResult \|= uSrc << (a_cBitsWidth - cShift); \
2757	*puDst = uResult; \
2758	\
2759	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2760	AssertCompile(X86_EFL_CF_BIT == 0); \
2761	fEfl \|= (uDst >> (cShift - 1)) & X86_EFL_CF; \
2762	if (a_fIntelFlags) \
2763	/* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
2764	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uSrc << (a_cBitsWidth - 1))); \
2765	else \
2766	{ /* AMD 3990X: Set according to last shift. AF always set. */ \
2767	if (cShift > 1) /* Set according to last shift. */ \
2768	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth((uSrc << (a_cBitsWidth - cShift + 1)) ^ uResult); \
2769	else \
2770	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ uResult); \
2771	fEfl \|= X86_EFL_AF; \
2772	} \
2773	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2774	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2775	fEfl \|= g_afParity[uResult & 0xff]; \
2776	*pfEFlags = fEfl; \
2777	} \
2778	}
2779
2780	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2781	EMIT_SHRD(64, uint64_t, RT_NOTHING, 1)
2782	#endif
2783	EMIT_SHRD(64, uint64_t, _intel, 1)
2784	EMIT_SHRD(64, uint64_t, _amd, 0)
2785
2786	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2787	EMIT_SHRD(32, uint32_t, RT_NOTHING, 1)
2788	#endif
2789	EMIT_SHRD(32, uint32_t, _intel, 1)
2790	EMIT_SHRD(32, uint32_t, _amd, 0)
2791
2792	#define EMIT_SHRD_16(a_Suffix, a_fIntelFlags) \
2793	IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shrd_u16,a_Suffix),(uint16_t puDst, uint16_t uSrc, uint8_t cShift, uint32_t pfEFlags)) \
2794	{ \
2795	cShift &= 31; \
2796	if (cShift) \
2797	{ \
2798	uint16_t const uDst = *puDst; \
2799	uint64_t const uTmp = a_fIntelFlags \
2800	? uDst \| ((uint32_t)uSrc << 16) \| ((uint64_t)uDst << 32) \
2801	: uDst \| ((uint32_t)uSrc << 16) \| ((uint64_t)uSrc << 32); \
2802	uint16_t const uResult = (uint16_t)(uTmp >> cShift); \
2803	*puDst = uResult; \
2804	\
2805	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2806	AssertCompile(X86_EFL_CF_BIT == 0); \
2807	if (a_fIntelFlags) \
2808	{ \
2809	/* Intel 10980XE: The CF is the last shifted out of the combined uTmp operand. */ \
2810	fEfl \|= (uTmp >> (cShift - 1)) & X86_EFL_CF; \
2811	/* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
2812	fEfl \|= X86_EFL_GET_OF_16(uDst ^ (uSrc << 15)); \
2813	} \
2814	else \
2815	{ \
2816	/* AMD 3990X: CF flag seems to be last bit shifted out of uDst, not the combined uSrc:uSrc:uDst operand. */ \
2817	fEfl \|= (uDst >> (cShift - 1)) & X86_EFL_CF; \
2818	/* AMD 3990X: Set according to last shift. AF always set. */ \
2819	if (cShift > 1) /* Set according to last shift. */ \
2820	fEfl \|= X86_EFL_GET_OF_16((uint16_t)(uTmp >> (cShift - 1)) ^ uResult); \
2821	else \
2822	fEfl \|= X86_EFL_GET_OF_16(uDst ^ uResult); \
2823	fEfl \|= X86_EFL_AF; \
2824	} \
2825	fEfl \|= X86_EFL_CALC_SF(uResult, 16); \
2826	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2827	fEfl \|= g_afParity[uResult & 0xff]; \
2828	*pfEFlags = fEfl; \
2829	} \
2830	}
2831
2832	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2833	EMIT_SHRD_16(RT_NOTHING, 1)
2834	#endif
2835	EMIT_SHRD_16(_intel, 1)
2836	EMIT_SHRD_16(_amd, 0)
2837
2838
2839	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2840
2841	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2842	/*
2843	* BSWAP
2844	*/
2845
2846	IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u64,(uint64_t *puDst))
2847	{
2848	puDst = ASMByteSwapU64(puDst);
2849	}
2850
2851
2852	IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u32,(uint32_t *puDst))
2853	{
2854	puDst = ASMByteSwapU32(puDst);
2855	}
2856
2857
2858	/* Note! undocument, so 32-bit arg */
2859	IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u16,(uint32_t *puDst))
2860	{
2861	#if 0
2862	(uint16_t )puDst = ASMByteSwapU16((uint16_t )puDst);
2863	#else
2864	/* This is the behaviour AMD 3990x (64-bit mode): */
2865	(uint16_t )puDst = 0;
2866	#endif
2867	}
2868
2869	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2870
2871
2872
2873	# if defined(IEM_WITHOUT_ASSEMBLY)
2874
2875	/*
2876	* LFENCE, SFENCE & MFENCE.
2877	*/
2878
2879	IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void))
2880	{
2881	ASMReadFence();
2882	}
2883
2884
2885	IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void))
2886	{
2887	ASMWriteFence();
2888	}
2889
2890
2891	IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void))
2892	{
2893	ASMMemoryFence();
2894	}
2895
2896
2897	# ifndef RT_ARCH_ARM64
2898	IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void))
2899	{
2900	ASMMemoryFence();
2901	}
2902	# endif
2903
2904	# endif
2905
2906	#endif /* !RT_ARCH_AMD64 \|\| IEM_WITHOUT_ASSEMBLY */
2907
2908
2909	IEM_DECL_IMPL_DEF(void, iemAImpl_arpl,(uint16_t pu16Dst, uint16_t u16Src, uint32_t pfEFlags))
2910	{
2911	if ((*pu16Dst & X86_SEL_RPL) < (u16Src & X86_SEL_RPL))
2912	{
2913	*pu16Dst &= X86_SEL_MASK_OFF_RPL;
2914	*pu16Dst \|= u16Src & X86_SEL_RPL;
2915
2916	*pfEFlags \|= X86_EFL_ZF;
2917	}
2918	else
2919	*pfEFlags &= ~X86_EFL_ZF;
2920	}
2921
2922
2923	/*********************************************************************************************************************************
2924	* x87 FPU *
2925	*********************************************************************************************************************************/
2926	#if defined(IEM_WITHOUT_ASSEMBLY)
2927
2928	IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
2929	{
2930	RT_NOREF(pFpuState, pFpuRes, pr80Val);
2931	AssertReleaseFailed();
2932	}
2933
2934
2935	IEM_DECL_IMPL_DEF(void, iemAImpl_fabs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
2936	{
2937	RT_NOREF(pFpuState, pFpuRes, pr80Val);
2938	AssertReleaseFailed();
2939	}
2940
2941
2942	IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
2943	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
2944	{
2945	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
2946	AssertReleaseFailed();
2947	}
2948
2949
2950	IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
2951	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
2952	{
2953	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
2954	AssertReleaseFailed();
2955	}
2956
2957
2958	IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
2959	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
2960	{
2961	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
2962	AssertReleaseFailed();
2963	}
2964
2965
2966	IEM_DECL_IMPL_DEF(void, iemAImpl_fchs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
2967	{
2968	RT_NOREF(pFpuState, pFpuRes, pr80Val);
2969	AssertReleaseFailed();
2970	}
2971
2972
2973	IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r32,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
2974	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
2975	{
2976	RT_NOREF(pFpuState, pFSW, pr80Val1, pr32Val2);
2977	AssertReleaseFailed();
2978	}
2979
2980
2981	IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r64,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
2982	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
2983	{
2984	RT_NOREF(pFpuState, pFSW, pr80Val1, pr64Val2);
2985	AssertReleaseFailed();
2986	}
2987
2988
2989	IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
2990	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
2991	{
2992	RT_NOREF(pFpuState, pFSW, pr80Val1, pr80Val2);
2993	AssertReleaseFailed();
2994	}
2995
2996
2997	IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fcomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
2998	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
2999	{
3000	RT_NOREF(pFpuState, pFSW, pr80Val1, pr80Val2);
3001	AssertReleaseFailed();
3002	return 0;
3003	}
3004
3005
3006	IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
3007	{
3008	RT_NOREF(pFpuState, pFpuRes, pr80Val);
3009	AssertReleaseFailed();
3010	}
3011
3012
3013	IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3014	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
3015	{
3016	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
3017	AssertReleaseFailed();
3018	}
3019
3020
3021	IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3022	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
3023	{
3024	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
3025	AssertReleaseFailed();
3026	}
3027
3028
3029	IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3030	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3031	{
3032	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3033	AssertReleaseFailed();
3034	}
3035
3036
3037	IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3038	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
3039	{
3040	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
3041	AssertReleaseFailed();
3042	}
3043
3044
3045	IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3046	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
3047	{
3048	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
3049	AssertReleaseFailed();
3050	}
3051
3052
3053	IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3054	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3055	{
3056	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3057	AssertReleaseFailed();
3058	}
3059
3060
3061	IEM_DECL_IMPL_DEF(void, iemAImpl_fiadd_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3062	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3063	{
3064	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
3065	AssertReleaseFailed();
3066	}
3067
3068
3069	IEM_DECL_IMPL_DEF(void, iemAImpl_fiadd_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3070	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3071	{
3072	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
3073	AssertReleaseFailed();
3074	}
3075
3076
3077	IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
3078	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3079	{
3080	RT_NOREF(pFpuState, pu16Fsw, pr80Val1, pi16Val2);
3081	AssertReleaseFailed();
3082	}
3083
3084
3085	IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
3086	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3087	{
3088	RT_NOREF(pFpuState, pu16Fsw, pr80Val1, pi32Val2);
3089	AssertReleaseFailed();
3090	}
3091
3092
3093	IEM_DECL_IMPL_DEF(void, iemAImpl_fidiv_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3094	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3095	{
3096	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
3097	AssertReleaseFailed();
3098	}
3099
3100
3101	IEM_DECL_IMPL_DEF(void, iemAImpl_fidiv_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3102	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3103	{
3104	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
3105	AssertReleaseFailed();
3106	}
3107
3108
3109	IEM_DECL_IMPL_DEF(void, iemAImpl_fidivr_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3110	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3111	{
3112	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
3113	AssertReleaseFailed();
3114	}
3115
3116
3117	IEM_DECL_IMPL_DEF(void, iemAImpl_fidivr_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3118	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3119	{
3120	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
3121	AssertReleaseFailed();
3122	}
3123
3124
3125	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_i16_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int16_t const *pi16Val))
3126	{
3127	RT_NOREF(pFpuState, pFpuRes, pi16Val);
3128	AssertReleaseFailed();
3129	}
3130
3131
3132	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_i32_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int32_t const *pi32Val))
3133	{
3134	RT_NOREF(pFpuState, pFpuRes, pi32Val);
3135	AssertReleaseFailed();
3136	}
3137
3138
3139	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_i64_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, int64_t const *pi64Val))
3140	{
3141	RT_NOREF(pFpuState, pFpuRes, pi64Val);
3142	AssertReleaseFailed();
3143	}
3144
3145
3146	IEM_DECL_IMPL_DEF(void, iemAImpl_fimul_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3147	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3148	{
3149	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
3150	AssertReleaseFailed();
3151	}
3152
3153
3154	IEM_DECL_IMPL_DEF(void, iemAImpl_fimul_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3155	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3156	{
3157	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
3158	AssertReleaseFailed();
3159	}
3160
3161
3162	IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3163	int16_t *pi16Val, PCRTFLOAT80U pr80Val))
3164	{
3165	RT_NOREF(pFpuState, pu16FSW, pi16Val, pr80Val);
3166	AssertReleaseFailed();
3167	}
3168
3169
3170	IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3171	int32_t *pi32Val, PCRTFLOAT80U pr80Val))
3172	{
3173	RT_NOREF(pFpuState, pu16FSW, pi32Val, pr80Val);
3174	AssertReleaseFailed();
3175	}
3176
3177
3178	IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3179	int64_t *pi64Val, PCRTFLOAT80U pr80Val))
3180	{
3181	RT_NOREF(pFpuState, pu16FSW, pi64Val, pr80Val);
3182	AssertReleaseFailed();
3183	}
3184
3185
3186	IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3187	int16_t *pi16Val, PCRTFLOAT80U pr80Val))
3188	{
3189	RT_NOREF(pFpuState, pu16FSW, pi16Val, pr80Val);
3190	AssertReleaseFailed();
3191	}
3192
3193
3194	IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3195	int32_t *pi32Val, PCRTFLOAT80U pr80Val))
3196	{
3197	RT_NOREF(pFpuState, pu16FSW, pi32Val, pr80Val);
3198	AssertReleaseFailed();
3199	}
3200
3201
3202	IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3203	int64_t *pi64Val, PCRTFLOAT80U pr80Val))
3204	{
3205	RT_NOREF(pFpuState, pu16FSW, pi64Val, pr80Val);
3206	AssertReleaseFailed();
3207	}
3208
3209
3210	IEM_DECL_IMPL_DEF(void, iemAImpl_fisub_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3211	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3212	{
3213	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
3214	AssertReleaseFailed();
3215	}
3216
3217
3218	IEM_DECL_IMPL_DEF(void, iemAImpl_fisub_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3219	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3220	{
3221	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
3222	AssertReleaseFailed();
3223	}
3224
3225
3226	IEM_DECL_IMPL_DEF(void, iemAImpl_fisubr_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3227	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
3228	{
3229	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
3230	AssertReleaseFailed();
3231	}
3232
3233
3234	IEM_DECL_IMPL_DEF(void, iemAImpl_fisubr_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3235	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
3236	{
3237	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
3238	AssertReleaseFailed();
3239	}
3240
3241
3242	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r32_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val))
3243	{
3244	RT_NOREF(pFpuState, pFpuRes, pr32Val);
3245	AssertReleaseFailed();
3246	}
3247
3248
3249	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r64_to_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val))
3250	{
3251	RT_NOREF(pFpuState, pFpuRes, pr64Val);
3252	AssertReleaseFailed();
3253	}
3254
3255	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
3256	{
3257	RT_NOREF(pFpuState, pFpuRes, pr80Val);
3258	AssertReleaseFailed();
3259	}
3260
3261
3262	IEM_DECL_IMPL_DEF(void, iemAImpl_fld1,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3263	{
3264	RT_NOREF(pFpuState, pFpuRes);
3265	AssertReleaseFailed();
3266	}
3267
3268
3269	IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2e,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3270	{
3271	RT_NOREF(pFpuState, pFpuRes);
3272	AssertReleaseFailed();
3273	}
3274
3275
3276	IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2t,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3277	{
3278	RT_NOREF(pFpuState, pFpuRes);
3279	AssertReleaseFailed();
3280	}
3281
3282
3283	IEM_DECL_IMPL_DEF(void, iemAImpl_fldlg2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3284	{
3285	RT_NOREF(pFpuState, pFpuRes);
3286	AssertReleaseFailed();
3287	}
3288
3289
3290	IEM_DECL_IMPL_DEF(void, iemAImpl_fldln2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3291	{
3292	RT_NOREF(pFpuState, pFpuRes);
3293	AssertReleaseFailed();
3294	}
3295
3296
3297	IEM_DECL_IMPL_DEF(void, iemAImpl_fldpi,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3298	{
3299	RT_NOREF(pFpuState, pFpuRes);
3300	AssertReleaseFailed();
3301	}
3302
3303
3304	IEM_DECL_IMPL_DEF(void, iemAImpl_fldz,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3305	{
3306	RT_NOREF(pFpuState, pFpuRes);
3307	AssertReleaseFailed();
3308	}
3309
3310
3311	IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3312	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
3313	{
3314	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
3315	AssertReleaseFailed();
3316	}
3317
3318
3319	IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3320	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
3321	{
3322	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
3323	AssertReleaseFailed();
3324	}
3325
3326
3327	IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3328	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3329	{
3330	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3331	AssertReleaseFailed();
3332	}
3333
3334
3335	IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3336	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3337	{
3338	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3339	AssertReleaseFailed();
3340	}
3341
3342
3343	IEM_DECL_IMPL_DEF(void, iemAImpl_fprem_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3344	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3345	{
3346	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3347	AssertReleaseFailed();
3348	}
3349
3350
3351	IEM_DECL_IMPL_DEF(void, iemAImpl_fprem1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3352	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3353	{
3354	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3355	AssertReleaseFailed();
3356	}
3357
3358
3359	IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
3360	{
3361	RT_NOREF(pFpuState, pFpuResTwo, pr80Val);
3362	AssertReleaseFailed();
3363	}
3364
3365
3366	IEM_DECL_IMPL_DEF(void, iemAImpl_frndint_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
3367	{
3368	RT_NOREF(pFpuState, pFpuRes, pr80Val);
3369	AssertReleaseFailed();
3370	}
3371
3372
3373	IEM_DECL_IMPL_DEF(void, iemAImpl_fscale_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3374	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3375	{
3376	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3377	AssertReleaseFailed();
3378	}
3379
3380
3381	IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
3382	{
3383	RT_NOREF(pFpuState, pFpuRes, pr80Val);
3384	AssertReleaseFailed();
3385	}
3386
3387
3388	IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
3389	{
3390	RT_NOREF(pFpuState, pFpuResTwo, pr80Val);
3391	AssertReleaseFailed();
3392	}
3393
3394
3395	IEM_DECL_IMPL_DEF(void, iemAImpl_fsqrt_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
3396	{
3397	RT_NOREF(pFpuState, pFpuRes, pr80Val);
3398	AssertReleaseFailed();
3399	}
3400
3401
3402	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3403	PRTFLOAT32U pr32Dst, PCRTFLOAT80U pr80Src))
3404	{
3405	RT_NOREF(pFpuState, pu16FSW, pr32Dst, pr80Src);
3406	AssertReleaseFailed();
3407	}
3408
3409
3410	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3411	PRTFLOAT64U pr64Dst, PCRTFLOAT80U pr80Src))
3412	{
3413	RT_NOREF(pFpuState, pu16FSW, pr64Dst, pr80Src);
3414	AssertReleaseFailed();
3415	}
3416
3417
3418	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3419	PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src))
3420	{
3421	RT_NOREF(pFpuState, pu16FSW, pr80Dst, pr80Src);
3422	AssertReleaseFailed();
3423	}
3424
3425
3426	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3427	PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src))
3428	{
3429	RT_NOREF(pFpuState, pu16FSW, pd80Dst, pr80Src);
3430	AssertReleaseFailed();
3431	}
3432
3433
3434	IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3435	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
3436	{
3437	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
3438	AssertReleaseFailed();
3439	}
3440
3441
3442	IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3443	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
3444	{
3445	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
3446	AssertReleaseFailed();
3447	}
3448
3449
3450	IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3451	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3452	{
3453	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3454	AssertReleaseFailed();
3455	}
3456
3457
3458	IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3459	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
3460	{
3461	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
3462	AssertReleaseFailed();
3463	}
3464
3465
3466	IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3467	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
3468	{
3469	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
3470	AssertReleaseFailed();
3471	}
3472
3473
3474	IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3475	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3476	{
3477	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3478	AssertReleaseFailed();
3479	}
3480
3481
3482	IEM_DECL_IMPL_DEF(void, iemAImpl_ftst_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val))
3483	{
3484	RT_NOREF(pFpuState, pu16Fsw, pr80Val);
3485	AssertReleaseFailed();
3486	}
3487
3488
3489	IEM_DECL_IMPL_DEF(void, iemAImpl_fucom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
3490	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3491	{
3492	RT_NOREF(pFpuState, pFSW, pr80Val1, pr80Val2);
3493	AssertReleaseFailed();
3494	}
3495
3496
3497	IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fucomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
3498	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3499	{
3500	RT_NOREF(pFpuState, pu16Fsw, pr80Val1, pr80Val2);
3501	AssertReleaseFailed();
3502	return 0;
3503	}
3504
3505
3506	IEM_DECL_IMPL_DEF(void, iemAImpl_fxam_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val))
3507	{
3508	RT_NOREF(pFpuState, pu16Fsw, pr80Val);
3509	AssertReleaseFailed();
3510	}
3511
3512
3513	IEM_DECL_IMPL_DEF(void, iemAImpl_fxtract_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
3514	{
3515	RT_NOREF(pFpuState, pFpuResTwo, pr80Val);
3516	AssertReleaseFailed();
3517	}
3518
3519
3520	IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3521	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3522	{
3523	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3524	AssertReleaseFailed();
3525	}
3526
3527
3528	IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
3529	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
3530	{
3531	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
3532	AssertReleaseFailed();
3533	}
3534
3535	#endif /* IEM_WITHOUT_ASSEMBLY */
3536
3537
3538	/*********************************************************************************************************************************
3539	* MMX, SSE & AVX *
3540	*********************************************************************************************************************************/
3541
3542	IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
3543	{
3544	RT_NOREF(pFpuState);
3545	puDst->au32[0] = puSrc->au32[0];
3546	puDst->au32[1] = puSrc->au32[0];
3547	puDst->au32[2] = puSrc->au32[2];
3548	puDst->au32[3] = puSrc->au32[2];
3549	}
3550
3551	#ifdef IEM_WITH_VEX
3552
3553	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
3554	{
3555	pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[0];
3556	pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[0];
3557	pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[2];
3558	pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[2];
3559	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
3560	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
3561	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
3562	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
3563	}
3564
3565
3566	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
3567	{
3568	pXState->x87.aXMM[iYRegDst].au32[0] = pSrc->au32[0];
3569	pXState->x87.aXMM[iYRegDst].au32[1] = pSrc->au32[0];
3570	pXState->x87.aXMM[iYRegDst].au32[2] = pSrc->au32[2];
3571	pXState->x87.aXMM[iYRegDst].au32[3] = pSrc->au32[2];
3572	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[4];
3573	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[4];
3574	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[6];
3575	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[6];
3576	}
3577
3578	#endif /* IEM_WITH_VEX */
3579
3580
3581	IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
3582	{
3583	RT_NOREF(pFpuState);
3584	puDst->au32[0] = puSrc->au32[1];
3585	puDst->au32[1] = puSrc->au32[1];
3586	puDst->au32[2] = puSrc->au32[3];
3587	puDst->au32[3] = puSrc->au32[3];
3588	}
3589
3590
3591	IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, uint64_t uSrc))
3592	{
3593	RT_NOREF(pFpuState);
3594	puDst->au64[0] = uSrc;
3595	puDst->au64[1] = uSrc;
3596	}
3597
3598	#ifdef IEM_WITH_VEX
3599
3600	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
3601	{
3602	pXState->x87.aXMM[iYRegDst].au64[0] = pXState->x87.aXMM[iYRegSrc].au64[0];
3603	pXState->x87.aXMM[iYRegDst].au64[1] = pXState->x87.aXMM[iYRegSrc].au64[0];
3604	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
3605	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
3606	}
3607
3608	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
3609	{
3610	pXState->x87.aXMM[iYRegDst].au64[0] = pSrc->au64[0];
3611	pXState->x87.aXMM[iYRegDst].au64[1] = pSrc->au64[0];
3612	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pSrc->au64[2];
3613	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pSrc->au64[2];
3614	}
3615
3616	#endif /* IEM_WITH_VEX */
3617
3618	#ifdef IEM_WITHOUT_ASSEMBLY
3619
3620	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3621	{
3622	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3623	AssertReleaseFailed();
3624	}
3625
3626
3627	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3628	{
3629	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3630	AssertReleaseFailed();
3631	}
3632
3633
3634	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3635	{
3636	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3637	AssertReleaseFailed();
3638	}
3639
3640
3641	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3642	{
3643	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3644	AssertReleaseFailed();
3645	}
3646
3647
3648	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3649	{
3650	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3651	AssertReleaseFailed();
3652	}
3653
3654
3655	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3656	{
3657	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3658	AssertReleaseFailed();
3659	}
3660
3661
3662	IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3663	{
3664	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3665	AssertReleaseFailed();
3666	}
3667
3668
3669	IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3670	{
3671	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3672	AssertReleaseFailed();
3673	}
3674
3675
3676	IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3677	{
3678	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3679	AssertReleaseFailed();
3680
3681	}
3682
3683
3684	IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, PCRTUINT128U pu128Src))
3685	{
3686	RT_NOREF(pFpuState, pu64Dst, pu128Src);
3687	AssertReleaseFailed();
3688	}
3689
3690
3691	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src, uint8_t bEvil))
3692	{
3693	RT_NOREF(pFpuState, pu64Dst, pu64Src, bEvil);
3694	AssertReleaseFailed();
3695	}
3696
3697
3698	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufhw,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src, uint8_t bEvil))
3699	{
3700	RT_NOREF(pFpuState, pu128Dst, pu128Src, bEvil);
3701	AssertReleaseFailed();
3702	}
3703
3704
3705	IEM_DECL_IMPL_DEF(void, iemAImpl_pshuflw,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src, uint8_t bEvil))
3706	{
3707	RT_NOREF(pFpuState, pu128Dst, pu128Src, bEvil);
3708	AssertReleaseFailed();
3709	}
3710
3711
3712	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufd,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src, uint8_t bEvil))
3713	{
3714	RT_NOREF(pFpuState, pu128Dst, pu128Src, bEvil);
3715	AssertReleaseFailed();
3716	}
3717
3718	/* PUNPCKHxxx */
3719
3720	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3721	{
3722	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3723	AssertReleaseFailed();
3724	}
3725
3726
3727	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3728	{
3729	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3730	AssertReleaseFailed();
3731	}
3732
3733
3734	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3735	{
3736	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3737	AssertReleaseFailed();
3738	}
3739
3740
3741	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3742	{
3743	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3744	AssertReleaseFailed();
3745	}
3746
3747
3748	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
3749	{
3750	RT_NOREF(pFpuState, pu64Dst, pu64Src);
3751	AssertReleaseFailed();
3752	}
3753
3754
3755	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3756	{
3757	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3758	AssertReleaseFailed();
3759	}
3760
3761
3762	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhqdq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
3763	{
3764	RT_NOREF(pFpuState, pu128Dst, pu128Src);
3765	AssertReleaseFailed();
3766	}
3767
3768	/* PUNPCKLxxx */
3769
3770	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src))
3771	{
3772	RT_NOREF(pFpuState, pu64Dst, pu32Src);
3773	AssertReleaseFailed();
3774	}
3775
3776
3777	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
3778	{
3779	RT_NOREF(pFpuState, pu128Dst, pu64Src);
3780	AssertReleaseFailed();
3781	}
3782
3783
3784	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src))
3785	{
3786	RT_NOREF(pFpuState, pu64Dst, pu32Src);
3787	AssertReleaseFailed();
3788	}
3789
3790
3791	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
3792	{
3793	RT_NOREF(pFpuState, pu128Dst, pu64Src);
3794	AssertReleaseFailed();
3795	}
3796
3797
3798	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src))
3799	{
3800	RT_NOREF(pFpuState, pu64Dst, pu32Src);
3801	AssertReleaseFailed();
3802	}
3803
3804
3805	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
3806	{
3807	RT_NOREF(pFpuState, pu128Dst, pu64Src);
3808	AssertReleaseFailed();
3809	}
3810
3811
3812	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklqdq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
3813	{
3814	RT_NOREF(pFpuState, pu128Dst, pu64Src);
3815	AssertReleaseFailed();
3816	}
3817
3818	#endif /* IEM_WITHOUT_ASSEMBLY */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp@ 94224

Download in other formats: