IEMAllAImplC.cpp@ 94402

Last change on this file since 94402 was 94402, checked in by vboxsync, 3 years ago
tstIEMAImpl,VMM/IEM: Added tests for FPU instructions taking two floating point operands. bugref:9898
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 176.7 KB

Line
1	/* $Id: IEMAllAImplC.cpp 94402 2022-03-30 23:06:33Z vboxsync $ */
2	/** @file
3	* IEM - Instruction Implementation in Assembly, portable C variant.
4	*/
5
6	/*
7	* Copyright (C) 2011-2022 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*********************************************************************************************************************************
20	* Header Files *
21	*********************************************************************************************************************************/
22	#include "IEMInternal.h"
23	#include <VBox/vmm/vmcc.h>
24	#include <iprt/errcore.h>
25	#include <iprt/x86.h>
26	#include <iprt/uint128.h>
27
28
29	/*********************************************************************************************************************************
30	* Defined Constants And Macros *
31	*********************************************************************************************************************************/
32	/** @def IEM_WITHOUT_ASSEMBLY
33	* Enables all the code in this file.
34	*/
35	#if !defined(IEM_WITHOUT_ASSEMBLY)
36	# if defined(RT_ARCH_ARM32) \|\| defined(RT_ARCH_ARM64) \|\| defined(DOXYGEN_RUNNING)
37	# define IEM_WITHOUT_ASSEMBLY
38	# endif
39	#endif
40	/* IEM_WITH_ASSEMBLY trumps IEM_WITHOUT_ASSEMBLY for tstIEMAImplAsm purposes. */
41	#ifdef IEM_WITH_ASSEMBLY
42	# undef IEM_WITHOUT_ASSEMBLY
43	#endif
44
45	/**
46	* Calculates the signed flag value given a result and it's bit width.
47	*
48	* The signed flag (SF) is a duplication of the most significant bit in the
49	* result.
50	*
51	* @returns X86_EFL_SF or 0.
52	* @param a_uResult Unsigned result value.
53	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
54	*/
55	#define X86_EFL_CALC_SF(a_uResult, a_cBitsWidth) \
56	( (uint32_t)((a_uResult) >> ((a_cBitsWidth) - X86_EFL_SF_BIT - 1)) & X86_EFL_SF )
57
58	/**
59	* Calculates the zero flag value given a result.
60	*
61	* The zero flag (ZF) indicates whether the result is zero or not.
62	*
63	* @returns X86_EFL_ZF or 0.
64	* @param a_uResult Unsigned result value.
65	*/
66	#define X86_EFL_CALC_ZF(a_uResult) \
67	( (uint32_t)((a_uResult) == 0) << X86_EFL_ZF_BIT )
68
69	/**
70	* Extracts the OF flag from a OF calculation result.
71	*
72	* These are typically used by concating with a bitcount. The problem is that
73	* 8-bit values needs shifting in the other direction than the others.
74	*/
75	#define X86_EFL_GET_OF_8(a_uValue) (((uint32_t)(a_uValue) << (X86_EFL_OF_BIT - 8 + 1)) & X86_EFL_OF)
76	#define X86_EFL_GET_OF_16(a_uValue) ((uint32_t)((a_uValue) >> (16 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
77	#define X86_EFL_GET_OF_32(a_uValue) ((uint32_t)((a_uValue) >> (32 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
78	#define X86_EFL_GET_OF_64(a_uValue) ((uint32_t)((a_uValue) >> (64 - X86_EFL_OF_BIT - 1)) & X86_EFL_OF)
79
80	/**
81	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) after arithmetic op.
82	*
83	* @returns Status bits.
84	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
85	* @param a_uResult Unsigned result value.
86	* @param a_uSrc The source value (for AF calc).
87	* @param a_uDst The original destination value (for AF calc).
88	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
89	* @param a_CfExpr Bool expression for the carry flag (CF).
90	* @param a_uSrcOf The a_uSrc value to use for overflow calculation.
91	*/
92	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(a_pfEFlags, a_uResult, a_uDst, a_uSrc, a_cBitsWidth, a_CfExpr, a_uSrcOf) \
93	do { \
94	uint32_t fEflTmp = *(a_pfEFlags); \
95	fEflTmp &= ~X86_EFL_STATUS_BITS; \
96	fEflTmp \|= (a_CfExpr) << X86_EFL_CF_BIT; \
97	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
98	fEflTmp \|= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uSrc) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
99	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
100	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
101	\
102	/* Overflow during ADDition happens when both inputs have the same signed \
103	bit value and the result has a different sign bit value. \
104	\
105	Since subtraction can be rewritten as addition: 2 - 1 == 2 + -1, it \
106	follows that for SUBtraction the signed bit value must differ between \
107	the two inputs and the result's signed bit diff from the first input. \
108	Note! Must xor with sign bit to convert, not do (0 - a_uSrc). \
109	\
110	See also: http://teaching.idallen.com/dat2343/10f/notes/040_overflow.txt */ \
111	fEflTmp \|= X86_EFL_GET_OF_ ## a_cBitsWidth( ( ((uint ## a_cBitsWidth ## _t)~((a_uDst) ^ (a_uSrcOf))) \
112	& RT_BIT_64(a_cBitsWidth - 1)) \
113	& ((a_uResult) ^ (a_uDst)) ); \
114	*(a_pfEFlags) = fEflTmp; \
115	} while (0)
116
117	/**
118	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) after a logical op.
119	*
120	* CF and OF are defined to be 0 by logical operations. AF on the other hand is
121	* undefined. We do not set AF, as that seems to make the most sense (which
122	* probably makes it the most wrong in real life).
123	*
124	* @returns Status bits.
125	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
126	* @param a_uResult Unsigned result value.
127	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
128	* @param a_fExtra Additional bits to set.
129	*/
130	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(a_pfEFlags, a_uResult, a_cBitsWidth, a_fExtra) \
131	do { \
132	uint32_t fEflTmp = *(a_pfEFlags); \
133	fEflTmp &= ~X86_EFL_STATUS_BITS; \
134	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
135	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
136	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
137	fEflTmp \|= (a_fExtra); \
138	*(a_pfEFlags) = fEflTmp; \
139	} while (0)
140
141
142	/*********************************************************************************************************************************
143	* Global Variables *
144	*********************************************************************************************************************************/
145	/**
146	* Parity calculation table.
147	*
148	* This is also used by iemAllAImpl.asm.
149	*
150	* The generator code:
151	* @code
152	* #include <stdio.h>
153	*
154	* int main()
155	* {
156	* unsigned b;
157	* for (b = 0; b < 256; b++)
158	* {
159	* int cOnes = ( b & 1)
160	* + ((b >> 1) & 1)
161	* + ((b >> 2) & 1)
162	* + ((b >> 3) & 1)
163	* + ((b >> 4) & 1)
164	* + ((b >> 5) & 1)
165	* + ((b >> 6) & 1)
166	* + ((b >> 7) & 1);
167	* printf(" /" "* %#04x = %u%u%u%u%u%u%u%ub *" "/ %s,\n",
168	* b,
169	* (b >> 7) & 1,
170	* (b >> 6) & 1,
171	* (b >> 5) & 1,
172	* (b >> 4) & 1,
173	* (b >> 3) & 1,
174	* (b >> 2) & 1,
175	* (b >> 1) & 1,
176	* b & 1,
177	* cOnes & 1 ? "0" : "X86_EFL_PF");
178	* }
179	* return 0;
180	* }
181	* @endcode
182	*/
183	uint8_t const g_afParity[256] =
184	{
185	/* 0000 = 00000000b */ X86_EFL_PF,
186	/* 0x01 = 00000001b */ 0,
187	/* 0x02 = 00000010b */ 0,
188	/* 0x03 = 00000011b */ X86_EFL_PF,
189	/* 0x04 = 00000100b */ 0,
190	/* 0x05 = 00000101b */ X86_EFL_PF,
191	/* 0x06 = 00000110b */ X86_EFL_PF,
192	/* 0x07 = 00000111b */ 0,
193	/* 0x08 = 00001000b */ 0,
194	/* 0x09 = 00001001b */ X86_EFL_PF,
195	/* 0x0a = 00001010b */ X86_EFL_PF,
196	/* 0x0b = 00001011b */ 0,
197	/* 0x0c = 00001100b */ X86_EFL_PF,
198	/* 0x0d = 00001101b */ 0,
199	/* 0x0e = 00001110b */ 0,
200	/* 0x0f = 00001111b */ X86_EFL_PF,
201	/* 0x10 = 00010000b */ 0,
202	/* 0x11 = 00010001b */ X86_EFL_PF,
203	/* 0x12 = 00010010b */ X86_EFL_PF,
204	/* 0x13 = 00010011b */ 0,
205	/* 0x14 = 00010100b */ X86_EFL_PF,
206	/* 0x15 = 00010101b */ 0,
207	/* 0x16 = 00010110b */ 0,
208	/* 0x17 = 00010111b */ X86_EFL_PF,
209	/* 0x18 = 00011000b */ X86_EFL_PF,
210	/* 0x19 = 00011001b */ 0,
211	/* 0x1a = 00011010b */ 0,
212	/* 0x1b = 00011011b */ X86_EFL_PF,
213	/* 0x1c = 00011100b */ 0,
214	/* 0x1d = 00011101b */ X86_EFL_PF,
215	/* 0x1e = 00011110b */ X86_EFL_PF,
216	/* 0x1f = 00011111b */ 0,
217	/* 0x20 = 00100000b */ 0,
218	/* 0x21 = 00100001b */ X86_EFL_PF,
219	/* 0x22 = 00100010b */ X86_EFL_PF,
220	/* 0x23 = 00100011b */ 0,
221	/* 0x24 = 00100100b */ X86_EFL_PF,
222	/* 0x25 = 00100101b */ 0,
223	/* 0x26 = 00100110b */ 0,
224	/* 0x27 = 00100111b */ X86_EFL_PF,
225	/* 0x28 = 00101000b */ X86_EFL_PF,
226	/* 0x29 = 00101001b */ 0,
227	/* 0x2a = 00101010b */ 0,
228	/* 0x2b = 00101011b */ X86_EFL_PF,
229	/* 0x2c = 00101100b */ 0,
230	/* 0x2d = 00101101b */ X86_EFL_PF,
231	/* 0x2e = 00101110b */ X86_EFL_PF,
232	/* 0x2f = 00101111b */ 0,
233	/* 0x30 = 00110000b */ X86_EFL_PF,
234	/* 0x31 = 00110001b */ 0,
235	/* 0x32 = 00110010b */ 0,
236	/* 0x33 = 00110011b */ X86_EFL_PF,
237	/* 0x34 = 00110100b */ 0,
238	/* 0x35 = 00110101b */ X86_EFL_PF,
239	/* 0x36 = 00110110b */ X86_EFL_PF,
240	/* 0x37 = 00110111b */ 0,
241	/* 0x38 = 00111000b */ 0,
242	/* 0x39 = 00111001b */ X86_EFL_PF,
243	/* 0x3a = 00111010b */ X86_EFL_PF,
244	/* 0x3b = 00111011b */ 0,
245	/* 0x3c = 00111100b */ X86_EFL_PF,
246	/* 0x3d = 00111101b */ 0,
247	/* 0x3e = 00111110b */ 0,
248	/* 0x3f = 00111111b */ X86_EFL_PF,
249	/* 0x40 = 01000000b */ 0,
250	/* 0x41 = 01000001b */ X86_EFL_PF,
251	/* 0x42 = 01000010b */ X86_EFL_PF,
252	/* 0x43 = 01000011b */ 0,
253	/* 0x44 = 01000100b */ X86_EFL_PF,
254	/* 0x45 = 01000101b */ 0,
255	/* 0x46 = 01000110b */ 0,
256	/* 0x47 = 01000111b */ X86_EFL_PF,
257	/* 0x48 = 01001000b */ X86_EFL_PF,
258	/* 0x49 = 01001001b */ 0,
259	/* 0x4a = 01001010b */ 0,
260	/* 0x4b = 01001011b */ X86_EFL_PF,
261	/* 0x4c = 01001100b */ 0,
262	/* 0x4d = 01001101b */ X86_EFL_PF,
263	/* 0x4e = 01001110b */ X86_EFL_PF,
264	/* 0x4f = 01001111b */ 0,
265	/* 0x50 = 01010000b */ X86_EFL_PF,
266	/* 0x51 = 01010001b */ 0,
267	/* 0x52 = 01010010b */ 0,
268	/* 0x53 = 01010011b */ X86_EFL_PF,
269	/* 0x54 = 01010100b */ 0,
270	/* 0x55 = 01010101b */ X86_EFL_PF,
271	/* 0x56 = 01010110b */ X86_EFL_PF,
272	/* 0x57 = 01010111b */ 0,
273	/* 0x58 = 01011000b */ 0,
274	/* 0x59 = 01011001b */ X86_EFL_PF,
275	/* 0x5a = 01011010b */ X86_EFL_PF,
276	/* 0x5b = 01011011b */ 0,
277	/* 0x5c = 01011100b */ X86_EFL_PF,
278	/* 0x5d = 01011101b */ 0,
279	/* 0x5e = 01011110b */ 0,
280	/* 0x5f = 01011111b */ X86_EFL_PF,
281	/* 0x60 = 01100000b */ X86_EFL_PF,
282	/* 0x61 = 01100001b */ 0,
283	/* 0x62 = 01100010b */ 0,
284	/* 0x63 = 01100011b */ X86_EFL_PF,
285	/* 0x64 = 01100100b */ 0,
286	/* 0x65 = 01100101b */ X86_EFL_PF,
287	/* 0x66 = 01100110b */ X86_EFL_PF,
288	/* 0x67 = 01100111b */ 0,
289	/* 0x68 = 01101000b */ 0,
290	/* 0x69 = 01101001b */ X86_EFL_PF,
291	/* 0x6a = 01101010b */ X86_EFL_PF,
292	/* 0x6b = 01101011b */ 0,
293	/* 0x6c = 01101100b */ X86_EFL_PF,
294	/* 0x6d = 01101101b */ 0,
295	/* 0x6e = 01101110b */ 0,
296	/* 0x6f = 01101111b */ X86_EFL_PF,
297	/* 0x70 = 01110000b */ 0,
298	/* 0x71 = 01110001b */ X86_EFL_PF,
299	/* 0x72 = 01110010b */ X86_EFL_PF,
300	/* 0x73 = 01110011b */ 0,
301	/* 0x74 = 01110100b */ X86_EFL_PF,
302	/* 0x75 = 01110101b */ 0,
303	/* 0x76 = 01110110b */ 0,
304	/* 0x77 = 01110111b */ X86_EFL_PF,
305	/* 0x78 = 01111000b */ X86_EFL_PF,
306	/* 0x79 = 01111001b */ 0,
307	/* 0x7a = 01111010b */ 0,
308	/* 0x7b = 01111011b */ X86_EFL_PF,
309	/* 0x7c = 01111100b */ 0,
310	/* 0x7d = 01111101b */ X86_EFL_PF,
311	/* 0x7e = 01111110b */ X86_EFL_PF,
312	/* 0x7f = 01111111b */ 0,
313	/* 0x80 = 10000000b */ 0,
314	/* 0x81 = 10000001b */ X86_EFL_PF,
315	/* 0x82 = 10000010b */ X86_EFL_PF,
316	/* 0x83 = 10000011b */ 0,
317	/* 0x84 = 10000100b */ X86_EFL_PF,
318	/* 0x85 = 10000101b */ 0,
319	/* 0x86 = 10000110b */ 0,
320	/* 0x87 = 10000111b */ X86_EFL_PF,
321	/* 0x88 = 10001000b */ X86_EFL_PF,
322	/* 0x89 = 10001001b */ 0,
323	/* 0x8a = 10001010b */ 0,
324	/* 0x8b = 10001011b */ X86_EFL_PF,
325	/* 0x8c = 10001100b */ 0,
326	/* 0x8d = 10001101b */ X86_EFL_PF,
327	/* 0x8e = 10001110b */ X86_EFL_PF,
328	/* 0x8f = 10001111b */ 0,
329	/* 0x90 = 10010000b */ X86_EFL_PF,
330	/* 0x91 = 10010001b */ 0,
331	/* 0x92 = 10010010b */ 0,
332	/* 0x93 = 10010011b */ X86_EFL_PF,
333	/* 0x94 = 10010100b */ 0,
334	/* 0x95 = 10010101b */ X86_EFL_PF,
335	/* 0x96 = 10010110b */ X86_EFL_PF,
336	/* 0x97 = 10010111b */ 0,
337	/* 0x98 = 10011000b */ 0,
338	/* 0x99 = 10011001b */ X86_EFL_PF,
339	/* 0x9a = 10011010b */ X86_EFL_PF,
340	/* 0x9b = 10011011b */ 0,
341	/* 0x9c = 10011100b */ X86_EFL_PF,
342	/* 0x9d = 10011101b */ 0,
343	/* 0x9e = 10011110b */ 0,
344	/* 0x9f = 10011111b */ X86_EFL_PF,
345	/* 0xa0 = 10100000b */ X86_EFL_PF,
346	/* 0xa1 = 10100001b */ 0,
347	/* 0xa2 = 10100010b */ 0,
348	/* 0xa3 = 10100011b */ X86_EFL_PF,
349	/* 0xa4 = 10100100b */ 0,
350	/* 0xa5 = 10100101b */ X86_EFL_PF,
351	/* 0xa6 = 10100110b */ X86_EFL_PF,
352	/* 0xa7 = 10100111b */ 0,
353	/* 0xa8 = 10101000b */ 0,
354	/* 0xa9 = 10101001b */ X86_EFL_PF,
355	/* 0xaa = 10101010b */ X86_EFL_PF,
356	/* 0xab = 10101011b */ 0,
357	/* 0xac = 10101100b */ X86_EFL_PF,
358	/* 0xad = 10101101b */ 0,
359	/* 0xae = 10101110b */ 0,
360	/* 0xaf = 10101111b */ X86_EFL_PF,
361	/* 0xb0 = 10110000b */ 0,
362	/* 0xb1 = 10110001b */ X86_EFL_PF,
363	/* 0xb2 = 10110010b */ X86_EFL_PF,
364	/* 0xb3 = 10110011b */ 0,
365	/* 0xb4 = 10110100b */ X86_EFL_PF,
366	/* 0xb5 = 10110101b */ 0,
367	/* 0xb6 = 10110110b */ 0,
368	/* 0xb7 = 10110111b */ X86_EFL_PF,
369	/* 0xb8 = 10111000b */ X86_EFL_PF,
370	/* 0xb9 = 10111001b */ 0,
371	/* 0xba = 10111010b */ 0,
372	/* 0xbb = 10111011b */ X86_EFL_PF,
373	/* 0xbc = 10111100b */ 0,
374	/* 0xbd = 10111101b */ X86_EFL_PF,
375	/* 0xbe = 10111110b */ X86_EFL_PF,
376	/* 0xbf = 10111111b */ 0,
377	/* 0xc0 = 11000000b */ X86_EFL_PF,
378	/* 0xc1 = 11000001b */ 0,
379	/* 0xc2 = 11000010b */ 0,
380	/* 0xc3 = 11000011b */ X86_EFL_PF,
381	/* 0xc4 = 11000100b */ 0,
382	/* 0xc5 = 11000101b */ X86_EFL_PF,
383	/* 0xc6 = 11000110b */ X86_EFL_PF,
384	/* 0xc7 = 11000111b */ 0,
385	/* 0xc8 = 11001000b */ 0,
386	/* 0xc9 = 11001001b */ X86_EFL_PF,
387	/* 0xca = 11001010b */ X86_EFL_PF,
388	/* 0xcb = 11001011b */ 0,
389	/* 0xcc = 11001100b */ X86_EFL_PF,
390	/* 0xcd = 11001101b */ 0,
391	/* 0xce = 11001110b */ 0,
392	/* 0xcf = 11001111b */ X86_EFL_PF,
393	/* 0xd0 = 11010000b */ 0,
394	/* 0xd1 = 11010001b */ X86_EFL_PF,
395	/* 0xd2 = 11010010b */ X86_EFL_PF,
396	/* 0xd3 = 11010011b */ 0,
397	/* 0xd4 = 11010100b */ X86_EFL_PF,
398	/* 0xd5 = 11010101b */ 0,
399	/* 0xd6 = 11010110b */ 0,
400	/* 0xd7 = 11010111b */ X86_EFL_PF,
401	/* 0xd8 = 11011000b */ X86_EFL_PF,
402	/* 0xd9 = 11011001b */ 0,
403	/* 0xda = 11011010b */ 0,
404	/* 0xdb = 11011011b */ X86_EFL_PF,
405	/* 0xdc = 11011100b */ 0,
406	/* 0xdd = 11011101b */ X86_EFL_PF,
407	/* 0xde = 11011110b */ X86_EFL_PF,
408	/* 0xdf = 11011111b */ 0,
409	/* 0xe0 = 11100000b */ 0,
410	/* 0xe1 = 11100001b */ X86_EFL_PF,
411	/* 0xe2 = 11100010b */ X86_EFL_PF,
412	/* 0xe3 = 11100011b */ 0,
413	/* 0xe4 = 11100100b */ X86_EFL_PF,
414	/* 0xe5 = 11100101b */ 0,
415	/* 0xe6 = 11100110b */ 0,
416	/* 0xe7 = 11100111b */ X86_EFL_PF,
417	/* 0xe8 = 11101000b */ X86_EFL_PF,
418	/* 0xe9 = 11101001b */ 0,
419	/* 0xea = 11101010b */ 0,
420	/* 0xeb = 11101011b */ X86_EFL_PF,
421	/* 0xec = 11101100b */ 0,
422	/* 0xed = 11101101b */ X86_EFL_PF,
423	/* 0xee = 11101110b */ X86_EFL_PF,
424	/* 0xef = 11101111b */ 0,
425	/* 0xf0 = 11110000b */ X86_EFL_PF,
426	/* 0xf1 = 11110001b */ 0,
427	/* 0xf2 = 11110010b */ 0,
428	/* 0xf3 = 11110011b */ X86_EFL_PF,
429	/* 0xf4 = 11110100b */ 0,
430	/* 0xf5 = 11110101b */ X86_EFL_PF,
431	/* 0xf6 = 11110110b */ X86_EFL_PF,
432	/* 0xf7 = 11110111b */ 0,
433	/* 0xf8 = 11111000b */ 0,
434	/* 0xf9 = 11111001b */ X86_EFL_PF,
435	/* 0xfa = 11111010b */ X86_EFL_PF,
436	/* 0xfb = 11111011b */ 0,
437	/* 0xfc = 11111100b */ X86_EFL_PF,
438	/* 0xfd = 11111101b */ 0,
439	/* 0xfe = 11111110b */ 0,
440	/* 0xff = 11111111b */ X86_EFL_PF,
441	};
442
443
444	/*
445	* There are a few 64-bit on 32-bit things we'd rather do in C. Actually, doing
446	* it all in C is probably safer atm., optimize what's necessary later, maybe.
447	*/
448	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
449
450
451	/*********************************************************************************************************************************
452	* Binary Operations *
453	*********************************************************************************************************************************/
454
455	/*
456	* ADD
457	*/
458
459	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
460	{
461	uint64_t uDst = *puDst;
462	uint64_t uResult = uDst + uSrc;
463	*puDst = uResult;
464	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult < uDst, uSrc);
465	}
466
467	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
468
469	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
470	{
471	uint32_t uDst = *puDst;
472	uint32_t uResult = uDst + uSrc;
473	*puDst = uResult;
474	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult < uDst, uSrc);
475	}
476
477
478	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
479	{
480	uint16_t uDst = *puDst;
481	uint16_t uResult = uDst + uSrc;
482	*puDst = uResult;
483	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult < uDst, uSrc);
484	}
485
486
487	IEM_DECL_IMPL_DEF(void, iemAImpl_add_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
488	{
489	uint8_t uDst = *puDst;
490	uint8_t uResult = uDst + uSrc;
491	*puDst = uResult;
492	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult < uDst, uSrc);
493	}
494
495	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
496
497	/*
498	* ADC
499	*/
500
501	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
502	{
503	if (!(*pfEFlags & X86_EFL_CF))
504	iemAImpl_add_u64(puDst, uSrc, pfEFlags);
505	else
506	{
507	uint64_t uDst = *puDst;
508	uint64_t uResult = uDst + uSrc + 1;
509	*puDst = uResult;
510	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uResult <= uDst, uSrc);
511	}
512	}
513
514	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
515
516	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
517	{
518	if (!(*pfEFlags & X86_EFL_CF))
519	iemAImpl_add_u32(puDst, uSrc, pfEFlags);
520	else
521	{
522	uint32_t uDst = *puDst;
523	uint32_t uResult = uDst + uSrc + 1;
524	*puDst = uResult;
525	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uResult <= uDst, uSrc);
526	}
527	}
528
529
530	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
531	{
532	if (!(*pfEFlags & X86_EFL_CF))
533	iemAImpl_add_u16(puDst, uSrc, pfEFlags);
534	else
535	{
536	uint16_t uDst = *puDst;
537	uint16_t uResult = uDst + uSrc + 1;
538	*puDst = uResult;
539	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uResult <= uDst, uSrc);
540	}
541	}
542
543
544	IEM_DECL_IMPL_DEF(void, iemAImpl_adc_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
545	{
546	if (!(*pfEFlags & X86_EFL_CF))
547	iemAImpl_add_u8(puDst, uSrc, pfEFlags);
548	else
549	{
550	uint8_t uDst = *puDst;
551	uint8_t uResult = uDst + uSrc + 1;
552	*puDst = uResult;
553	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uResult <= uDst, uSrc);
554	}
555	}
556
557	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
558
559	/*
560	* SUB
561	*/
562
563	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
564	{
565	uint64_t uDst = *puDst;
566	uint64_t uResult = uDst - uSrc;
567	*puDst = uResult;
568	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst < uSrc, uSrc ^ RT_BIT_64(63));
569	}
570
571	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
572
573	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
574	{
575	uint32_t uDst = *puDst;
576	uint32_t uResult = uDst - uSrc;
577	*puDst = uResult;
578	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst < uSrc, uSrc ^ RT_BIT_32(31));
579	}
580
581
582	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
583	{
584	uint16_t uDst = *puDst;
585	uint16_t uResult = uDst - uSrc;
586	*puDst = uResult;
587	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst < uSrc, uSrc ^ (uint16_t)0x8000);
588	}
589
590
591	IEM_DECL_IMPL_DEF(void, iemAImpl_sub_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
592	{
593	uint8_t uDst = *puDst;
594	uint8_t uResult = uDst - uSrc;
595	*puDst = uResult;
596	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst < uSrc, uSrc ^ (uint8_t)0x80);
597	}
598
599	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
600
601	/*
602	* SBB
603	*/
604
605	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
606	{
607	if (!(*pfEFlags & X86_EFL_CF))
608	iemAImpl_sub_u64(puDst, uSrc, pfEFlags);
609	else
610	{
611	uint64_t uDst = *puDst;
612	uint64_t uResult = uDst - uSrc - 1;
613	*puDst = uResult;
614	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 64, uDst <= uSrc, uSrc ^ RT_BIT_64(63));
615	}
616	}
617
618	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
619
620	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
621	{
622	if (!(*pfEFlags & X86_EFL_CF))
623	iemAImpl_sub_u32(puDst, uSrc, pfEFlags);
624	else
625	{
626	uint32_t uDst = *puDst;
627	uint32_t uResult = uDst - uSrc - 1;
628	*puDst = uResult;
629	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 32, uDst <= uSrc, uSrc ^ RT_BIT_32(31));
630	}
631	}
632
633
634	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
635	{
636	if (!(*pfEFlags & X86_EFL_CF))
637	iemAImpl_sub_u16(puDst, uSrc, pfEFlags);
638	else
639	{
640	uint16_t uDst = *puDst;
641	uint16_t uResult = uDst - uSrc - 1;
642	*puDst = uResult;
643	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 16, uDst <= uSrc, uSrc ^ (uint16_t)0x8000);
644	}
645	}
646
647
648	IEM_DECL_IMPL_DEF(void, iemAImpl_sbb_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
649	{
650	if (!(*pfEFlags & X86_EFL_CF))
651	iemAImpl_sub_u8(puDst, uSrc, pfEFlags);
652	else
653	{
654	uint8_t uDst = *puDst;
655	uint8_t uResult = uDst - uSrc - 1;
656	*puDst = uResult;
657	IEM_EFL_UPDATE_STATUS_BITS_FOR_ARITHMETIC(pfEFlags, uResult, uDst, uSrc, 8, uDst <= uSrc, uSrc ^ (uint8_t)0x80);
658	}
659	}
660
661	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
662
663
664	/*
665	* OR
666	*/
667
668	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
669	{
670	uint64_t uResult = *puDst \| uSrc;
671	*puDst = uResult;
672	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
673	}
674
675	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
676
677	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
678	{
679	uint32_t uResult = *puDst \| uSrc;
680	*puDst = uResult;
681	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
682	}
683
684
685	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
686	{
687	uint16_t uResult = *puDst \| uSrc;
688	*puDst = uResult;
689	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
690	}
691
692
693	IEM_DECL_IMPL_DEF(void, iemAImpl_or_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
694	{
695	uint8_t uResult = *puDst \| uSrc;
696	*puDst = uResult;
697	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
698	}
699
700	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
701
702	/*
703	* XOR
704	*/
705
706	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
707	{
708	uint64_t uResult = *puDst ^ uSrc;
709	*puDst = uResult;
710	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
711	}
712
713	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
714
715	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
716	{
717	uint32_t uResult = *puDst ^ uSrc;
718	*puDst = uResult;
719	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
720	}
721
722
723	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
724	{
725	uint16_t uResult = *puDst ^ uSrc;
726	*puDst = uResult;
727	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
728	}
729
730
731	IEM_DECL_IMPL_DEF(void, iemAImpl_xor_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
732	{
733	uint8_t uResult = *puDst ^ uSrc;
734	*puDst = uResult;
735	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
736	}
737
738	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
739
740	/*
741	* AND
742	*/
743
744	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
745	{
746	uint64_t uResult = *puDst & uSrc;
747	*puDst = uResult;
748	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
749	}
750
751	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
752
753	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
754	{
755	uint32_t uResult = *puDst & uSrc;
756	*puDst = uResult;
757	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
758	}
759
760
761	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
762	{
763	uint16_t uResult = *puDst & uSrc;
764	*puDst = uResult;
765	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
766	}
767
768
769	IEM_DECL_IMPL_DEF(void, iemAImpl_and_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
770	{
771	uint8_t uResult = *puDst & uSrc;
772	*puDst = uResult;
773	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
774	}
775
776	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
777
778	/*
779	* CMP
780	*/
781
782	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
783	{
784	uint64_t uDstTmp = *puDst;
785	iemAImpl_sub_u64(&uDstTmp, uSrc, pfEFlags);
786	}
787
788	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
789
790	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
791	{
792	uint32_t uDstTmp = *puDst;
793	iemAImpl_sub_u32(&uDstTmp, uSrc, pfEFlags);
794	}
795
796
797	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
798	{
799	uint16_t uDstTmp = *puDst;
800	iemAImpl_sub_u16(&uDstTmp, uSrc, pfEFlags);
801	}
802
803
804	IEM_DECL_IMPL_DEF(void, iemAImpl_cmp_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
805	{
806	uint8_t uDstTmp = *puDst;
807	iemAImpl_sub_u8(&uDstTmp, uSrc, pfEFlags);
808	}
809
810	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
811
812	/*
813	* TEST
814	*/
815
816	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
817	{
818	uint64_t uResult = *puDst & uSrc;
819	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 64, 0);
820	}
821
822	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
823
824	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
825	{
826	uint32_t uResult = *puDst & uSrc;
827	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 32, 0);
828	}
829
830
831	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
832	{
833	uint16_t uResult = *puDst & uSrc;
834	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 16, 0);
835	}
836
837
838	IEM_DECL_IMPL_DEF(void, iemAImpl_test_u8,(uint8_t puDst, uint8_t uSrc, uint32_t pfEFlags))
839	{
840	uint8_t uResult = *puDst & uSrc;
841	IEM_EFL_UPDATE_STATUS_BITS_FOR_LOGIC(pfEFlags, uResult, 8, 0);
842	}
843
844	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
845
846
847	/*
848	* LOCK prefixed variants of the above
849	*/
850
851	/** 64-bit locked binary operand operation. */
852	# define DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \
853	do { \
854	uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
855	uint ## a_cBitsWidth ## _t uTmp; \
856	uint32_t fEflTmp; \
857	do \
858	{ \
859	uTmp = uOld; \
860	fEflTmp = *pfEFlags; \
861	iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, uSrc, &fEflTmp); \
862	} while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \
863	*pfEFlags = fEflTmp; \
864	} while (0)
865
866
867	#define EMIT_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth) \
868	IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \
869	uint ## a_cBitsWidth ## _t uSrc, \
870	uint32_t *pfEFlags)) \
871	{ \
872	DO_LOCKED_BIN_OP(a_Mnemonic, a_cBitsWidth); \
873	}
874
875	EMIT_LOCKED_BIN_OP(add, 64)
876	EMIT_LOCKED_BIN_OP(adc, 64)
877	EMIT_LOCKED_BIN_OP(sub, 64)
878	EMIT_LOCKED_BIN_OP(sbb, 64)
879	EMIT_LOCKED_BIN_OP(or, 64)
880	EMIT_LOCKED_BIN_OP(xor, 64)
881	EMIT_LOCKED_BIN_OP(and, 64)
882	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
883	EMIT_LOCKED_BIN_OP(add, 32)
884	EMIT_LOCKED_BIN_OP(adc, 32)
885	EMIT_LOCKED_BIN_OP(sub, 32)
886	EMIT_LOCKED_BIN_OP(sbb, 32)
887	EMIT_LOCKED_BIN_OP(or, 32)
888	EMIT_LOCKED_BIN_OP(xor, 32)
889	EMIT_LOCKED_BIN_OP(and, 32)
890
891	EMIT_LOCKED_BIN_OP(add, 16)
892	EMIT_LOCKED_BIN_OP(adc, 16)
893	EMIT_LOCKED_BIN_OP(sub, 16)
894	EMIT_LOCKED_BIN_OP(sbb, 16)
895	EMIT_LOCKED_BIN_OP(or, 16)
896	EMIT_LOCKED_BIN_OP(xor, 16)
897	EMIT_LOCKED_BIN_OP(and, 16)
898
899	EMIT_LOCKED_BIN_OP(add, 8)
900	EMIT_LOCKED_BIN_OP(adc, 8)
901	EMIT_LOCKED_BIN_OP(sub, 8)
902	EMIT_LOCKED_BIN_OP(sbb, 8)
903	EMIT_LOCKED_BIN_OP(or, 8)
904	EMIT_LOCKED_BIN_OP(xor, 8)
905	EMIT_LOCKED_BIN_OP(and, 8)
906	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
907
908
909	/*
910	* Bit operations (same signature as above).
911	*/
912
913	/*
914	* BT
915	*/
916
917	IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
918	{
919	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
920	not modified by either AMD (3990x) or Intel (i9-9980HK). */
921	Assert(uSrc < 64);
922	uint64_t uDst = *puDst;
923	if (uDst & RT_BIT_64(uSrc))
924	*pfEFlags \|= X86_EFL_CF;
925	else
926	*pfEFlags &= ~X86_EFL_CF;
927	}
928
929	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
930
931	IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
932	{
933	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
934	not modified by either AMD (3990x) or Intel (i9-9980HK). */
935	Assert(uSrc < 32);
936	uint32_t uDst = *puDst;
937	if (uDst & RT_BIT_32(uSrc))
938	*pfEFlags \|= X86_EFL_CF;
939	else
940	*pfEFlags &= ~X86_EFL_CF;
941	}
942
943	IEM_DECL_IMPL_DEF(void, iemAImpl_bt_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
944	{
945	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
946	not modified by either AMD (3990x) or Intel (i9-9980HK). */
947	Assert(uSrc < 16);
948	uint16_t uDst = *puDst;
949	if (uDst & RT_BIT_32(uSrc))
950	*pfEFlags \|= X86_EFL_CF;
951	else
952	*pfEFlags &= ~X86_EFL_CF;
953	}
954
955	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
956
957	/*
958	* BTC
959	*/
960
961	IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
962	{
963	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
964	not modified by either AMD (3990x) or Intel (i9-9980HK). */
965	Assert(uSrc < 64);
966	uint64_t fMask = RT_BIT_64(uSrc);
967	uint64_t uDst = *puDst;
968	if (uDst & fMask)
969	{
970	uDst &= ~fMask;
971	*puDst = uDst;
972	*pfEFlags \|= X86_EFL_CF;
973	}
974	else
975	{
976	uDst \|= fMask;
977	*puDst = uDst;
978	*pfEFlags &= ~X86_EFL_CF;
979	}
980	}
981
982	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
983
984	IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
985	{
986	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
987	not modified by either AMD (3990x) or Intel (i9-9980HK). */
988	Assert(uSrc < 32);
989	uint32_t fMask = RT_BIT_32(uSrc);
990	uint32_t uDst = *puDst;
991	if (uDst & fMask)
992	{
993	uDst &= ~fMask;
994	*puDst = uDst;
995	*pfEFlags \|= X86_EFL_CF;
996	}
997	else
998	{
999	uDst \|= fMask;
1000	*puDst = uDst;
1001	*pfEFlags &= ~X86_EFL_CF;
1002	}
1003	}
1004
1005
1006	IEM_DECL_IMPL_DEF(void, iemAImpl_btc_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1007	{
1008	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. However, it seems they're
1009	not modified by either AMD (3990x) or Intel (i9-9980HK). */
1010	Assert(uSrc < 16);
1011	uint16_t fMask = RT_BIT_32(uSrc);
1012	uint16_t uDst = *puDst;
1013	if (uDst & fMask)
1014	{
1015	uDst &= ~fMask;
1016	*puDst = uDst;
1017	*pfEFlags \|= X86_EFL_CF;
1018	}
1019	else
1020	{
1021	uDst \|= fMask;
1022	*puDst = uDst;
1023	*pfEFlags &= ~X86_EFL_CF;
1024	}
1025	}
1026
1027	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1028
1029	/*
1030	* BTR
1031	*/
1032
1033	IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1034	{
1035	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1036	logical operation (AND/OR/whatever). */
1037	Assert(uSrc < 64);
1038	uint64_t fMask = RT_BIT_64(uSrc);
1039	uint64_t uDst = *puDst;
1040	if (uDst & fMask)
1041	{
1042	uDst &= ~fMask;
1043	*puDst = uDst;
1044	*pfEFlags \|= X86_EFL_CF;
1045	}
1046	else
1047	*pfEFlags &= ~X86_EFL_CF;
1048	}
1049
1050	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1051
1052	IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1053	{
1054	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1055	logical operation (AND/OR/whatever). */
1056	Assert(uSrc < 32);
1057	uint32_t fMask = RT_BIT_32(uSrc);
1058	uint32_t uDst = *puDst;
1059	if (uDst & fMask)
1060	{
1061	uDst &= ~fMask;
1062	*puDst = uDst;
1063	*pfEFlags \|= X86_EFL_CF;
1064	}
1065	else
1066	*pfEFlags &= ~X86_EFL_CF;
1067	}
1068
1069
1070	IEM_DECL_IMPL_DEF(void, iemAImpl_btr_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1071	{
1072	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1073	logical operation (AND/OR/whatever). */
1074	Assert(uSrc < 16);
1075	uint16_t fMask = RT_BIT_32(uSrc);
1076	uint16_t uDst = *puDst;
1077	if (uDst & fMask)
1078	{
1079	uDst &= ~fMask;
1080	*puDst = uDst;
1081	*pfEFlags \|= X86_EFL_CF;
1082	}
1083	else
1084	*pfEFlags &= ~X86_EFL_CF;
1085	}
1086
1087	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1088
1089	/*
1090	* BTS
1091	*/
1092
1093	IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1094	{
1095	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1096	logical operation (AND/OR/whatever). */
1097	Assert(uSrc < 64);
1098	uint64_t fMask = RT_BIT_64(uSrc);
1099	uint64_t uDst = *puDst;
1100	if (uDst & fMask)
1101	*pfEFlags \|= X86_EFL_CF;
1102	else
1103	{
1104	uDst \|= fMask;
1105	*puDst = uDst;
1106	*pfEFlags &= ~X86_EFL_CF;
1107	}
1108	}
1109
1110	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1111
1112	IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1113	{
1114	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1115	logical operation (AND/OR/whatever). */
1116	Assert(uSrc < 32);
1117	uint32_t fMask = RT_BIT_32(uSrc);
1118	uint32_t uDst = *puDst;
1119	if (uDst & fMask)
1120	*pfEFlags \|= X86_EFL_CF;
1121	else
1122	{
1123	uDst \|= fMask;
1124	*puDst = uDst;
1125	*pfEFlags &= ~X86_EFL_CF;
1126	}
1127	}
1128
1129
1130	IEM_DECL_IMPL_DEF(void, iemAImpl_bts_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1131	{
1132	/* Note! "undefined" flags: OF, SF, ZF, AF, PF. We set them as after an
1133	logical operation (AND/OR/whatever). */
1134	Assert(uSrc < 16);
1135	uint16_t fMask = RT_BIT_32(uSrc);
1136	uint32_t uDst = *puDst;
1137	if (uDst & fMask)
1138	*pfEFlags \|= X86_EFL_CF;
1139	else
1140	{
1141	uDst \|= fMask;
1142	*puDst = uDst;
1143	*pfEFlags &= ~X86_EFL_CF;
1144	}
1145	}
1146
1147	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1148
1149
1150	EMIT_LOCKED_BIN_OP(btc, 64)
1151	EMIT_LOCKED_BIN_OP(btr, 64)
1152	EMIT_LOCKED_BIN_OP(bts, 64)
1153	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1154	EMIT_LOCKED_BIN_OP(btc, 32)
1155	EMIT_LOCKED_BIN_OP(btr, 32)
1156	EMIT_LOCKED_BIN_OP(bts, 32)
1157
1158	EMIT_LOCKED_BIN_OP(btc, 16)
1159	EMIT_LOCKED_BIN_OP(btr, 16)
1160	EMIT_LOCKED_BIN_OP(bts, 16)
1161	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1162
1163
1164	/*
1165	* Helpers for BSR and BSF.
1166	*
1167	* Note! "undefined" flags: OF, SF, AF, PF, CF.
1168	* Intel behavior modelled on 10980xe, AMD on 3990X. Other marchs may
1169	* produce different result (see https://www.sandpile.org/x86/flags.htm),
1170	* but we restrict ourselves to emulating these recent marchs.
1171	*/
1172	#define SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlag, a_iBit) do { \
1173	unsigned iBit = (a_iBit); \
1174	uint32_t fEfl = *pfEFlags & ~(X86_EFL_OF \| X86_EFL_SF \| X86_EFL_ZF \| X86_EFL_AF \| X86_EFL_PF \| X86_EFL_CF); \
1175	if (iBit) \
1176	{ \
1177	*puDst = --iBit; \
1178	fEfl \|= g_afParity[iBit]; \
1179	} \
1180	else \
1181	fEfl \|= X86_EFL_ZF \| X86_EFL_PF; \
1182	*pfEFlags = fEfl; \
1183	} while (0)
1184	#define SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlag, a_iBit) do { \
1185	unsigned const iBit = (a_iBit); \
1186	if (iBit) \
1187	{ \
1188	*puDst = iBit - 1; \
1189	*pfEFlags &= ~X86_EFL_ZF; \
1190	} \
1191	else \
1192	*pfEFlags \|= X86_EFL_ZF; \
1193	} while (0)
1194
1195
1196	/*
1197	* BSF - first (least significant) bit set
1198	*/
1199	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1200	{
1201	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
1202	}
1203
1204	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_intel,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1205	{
1206	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
1207	}
1208
1209	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u64_amd,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1210	{
1211	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU64(uSrc));
1212	}
1213
1214	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1215
1216	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1217	{
1218	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
1219	}
1220
1221	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_intel,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1222	{
1223	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
1224	}
1225
1226	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u32_amd,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1227	{
1228	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU32(uSrc));
1229	}
1230
1231
1232	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1233	{
1234	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
1235	}
1236
1237	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_intel,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1238	{
1239	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
1240	}
1241
1242	IEM_DECL_IMPL_DEF(void, iemAImpl_bsf_u16_amd,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1243	{
1244	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitFirstSetU16(uSrc));
1245	}
1246
1247	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1248
1249
1250	/*
1251	* BSR - last (most significant) bit set
1252	*/
1253	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1254	{
1255	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
1256	}
1257
1258	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_intel,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1259	{
1260	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
1261	}
1262
1263	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u64_amd,(uint64_t puDst, uint64_t uSrc, uint32_t pfEFlags))
1264	{
1265	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU64(uSrc));
1266	}
1267
1268	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1269
1270	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1271	{
1272	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
1273	}
1274
1275	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_intel,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1276	{
1277	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
1278	}
1279
1280	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u32_amd,(uint32_t puDst, uint32_t uSrc, uint32_t pfEFlags))
1281	{
1282	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU32(uSrc));
1283	}
1284
1285
1286	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1287	{
1288	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
1289	}
1290
1291	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_intel,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1292	{
1293	SET_BIT_SEARCH_RESULT_INTEL(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
1294	}
1295
1296	IEM_DECL_IMPL_DEF(void, iemAImpl_bsr_u16_amd,(uint16_t puDst, uint16_t uSrc, uint32_t pfEFlags))
1297	{
1298	SET_BIT_SEARCH_RESULT_AMD(puDst, pfEFlags, ASMBitLastSetU16(uSrc));
1299	}
1300
1301	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1302
1303
1304	/*
1305	* XCHG
1306	*/
1307
1308	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_locked,(uint64_t puMem, uint64_t puReg))
1309	{
1310	#if ARCH_BITS >= 64
1311	puReg = ASMAtomicXchgU64(puMem, puReg);
1312	#else
1313	uint64_t uOldMem = *puMem;
1314	while (!ASMAtomicCmpXchgExU64(puMem, *puReg, uOldMem, &uOldMem))
1315	ASMNopPause();
1316	*puReg = uOldMem;
1317	#endif
1318	}
1319
1320	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1321
1322	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_locked,(uint32_t puMem, uint32_t puReg))
1323	{
1324	puReg = ASMAtomicXchgU32(puMem, puReg);
1325	}
1326
1327
1328	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_locked,(uint16_t puMem, uint16_t puReg))
1329	{
1330	puReg = ASMAtomicXchgU16(puMem, puReg);
1331	}
1332
1333
1334	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_locked,(uint8_t puMem, uint8_t puReg))
1335	{
1336	puReg = ASMAtomicXchgU8(puMem, puReg);
1337	}
1338
1339	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1340
1341
1342	/* Unlocked variants for fDisregardLock mode: */
1343
1344	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u64_unlocked,(uint64_t puMem, uint64_t puReg))
1345	{
1346	uint64_t const uOld = *puMem;
1347	puMem = puReg;
1348	*puReg = uOld;
1349	}
1350
1351	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1352
1353	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u32_unlocked,(uint32_t puMem, uint32_t puReg))
1354	{
1355	uint32_t const uOld = *puMem;
1356	puMem = puReg;
1357	*puReg = uOld;
1358	}
1359
1360
1361	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u16_unlocked,(uint16_t puMem, uint16_t puReg))
1362	{
1363	uint16_t const uOld = *puMem;
1364	puMem = puReg;
1365	*puReg = uOld;
1366	}
1367
1368
1369	IEM_DECL_IMPL_DEF(void, iemAImpl_xchg_u8_unlocked,(uint8_t puMem, uint8_t puReg))
1370	{
1371	uint8_t const uOld = *puMem;
1372	puMem = puReg;
1373	*puReg = uOld;
1374	}
1375
1376	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1377
1378
1379	/*
1380	* XADD and LOCK XADD.
1381	*/
1382	#define EMIT_XADD(a_cBitsWidth, a_Type) \
1383	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth,(a_Type puDst, a_Type puReg, uint32_t *pfEFlags)) \
1384	{ \
1385	a_Type uDst = *puDst; \
1386	a_Type uResult = uDst; \
1387	iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, pfEFlags); \
1388	*puDst = uResult; \
1389	*puReg = uDst; \
1390	} \
1391	\
1392	IEM_DECL_IMPL_DEF(void, iemAImpl_xadd_u ## a_cBitsWidth ## _locked,(a_Type puDst, a_Type puReg, uint32_t *pfEFlags)) \
1393	{ \
1394	a_Type uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
1395	a_Type uResult; \
1396	uint32_t fEflTmp; \
1397	do \
1398	{ \
1399	uResult = uOld; \
1400	fEflTmp = *pfEFlags; \
1401	iemAImpl_add_u ## a_cBitsWidth(&uResult, *puReg, &fEflTmp); \
1402	} while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uResult, uOld, &uOld)); \
1403	*puReg = uOld; \
1404	*pfEFlags = fEflTmp; \
1405	}
1406	EMIT_XADD(64, uint64_t)
1407	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1408	EMIT_XADD(32, uint32_t)
1409	EMIT_XADD(16, uint16_t)
1410	EMIT_XADD(8, uint8_t)
1411	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1412
1413	#endif
1414
1415	/*
1416	* CMPXCHG, CMPXCHG8B, CMPXCHG16B
1417	*
1418	* Note! We don't have non-locking/atomic cmpxchg primitives, so all cmpxchg
1419	* instructions are emulated as locked.
1420	*/
1421	#if defined(IEM_WITHOUT_ASSEMBLY)
1422
1423	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8_locked, (uint8_t pu8Dst, uint8_t puAl, uint8_t uSrcReg, uint32_t *pEFlags))
1424	{
1425	uint8_t uOld = *puAl;
1426	if (ASMAtomicCmpXchgExU8(pu8Dst, uSrcReg, uOld, puAl))
1427	Assert(*puAl == uOld);
1428	iemAImpl_cmp_u8(&uOld, *puAl, pEFlags);
1429	}
1430
1431
1432	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16_locked,(uint16_t pu16Dst, uint16_t puAx, uint16_t uSrcReg, uint32_t *pEFlags))
1433	{
1434	uint16_t uOld = *puAx;
1435	if (ASMAtomicCmpXchgExU16(pu16Dst, uSrcReg, uOld, puAx))
1436	Assert(*puAx == uOld);
1437	iemAImpl_cmp_u16(&uOld, *puAx, pEFlags);
1438	}
1439
1440
1441	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32_locked,(uint32_t pu32Dst, uint32_t puEax, uint32_t uSrcReg, uint32_t *pEFlags))
1442	{
1443	uint32_t uOld = *puEax;
1444	if (ASMAtomicCmpXchgExU32(pu32Dst, uSrcReg, uOld, puEax))
1445	Assert(*puEax == uOld);
1446	iemAImpl_cmp_u32(&uOld, *puEax, pEFlags);
1447	}
1448
1449
1450	# if ARCH_BITS == 32
1451	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t pu64Dst, uint64_t puRax, uint64_t puSrcReg, uint32_t pEFlags))
1452	# else
1453	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64_locked,(uint64_t pu64Dst, uint64_t puRax, uint64_t uSrcReg, uint32_t *pEFlags))
1454	# endif
1455	{
1456	# if ARCH_BITS == 32
1457	uint64_t const uSrcReg = *puSrcReg;
1458	# endif
1459	uint64_t uOld = *puRax;
1460	if (ASMAtomicCmpXchgExU64(pu64Dst, uSrcReg, uOld, puRax))
1461	Assert(*puRax == uOld);
1462	iemAImpl_cmp_u64(&uOld, *puRax, pEFlags);
1463	}
1464
1465
1466	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b_locked,(uint64_t *pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx,
1467	uint32_t *pEFlags))
1468	{
1469	uint64_t const uNew = pu64EbxEcx->u;
1470	uint64_t const uOld = pu64EaxEdx->u;
1471	if (ASMAtomicCmpXchgExU64(pu64Dst, uNew, uOld, &pu64EaxEdx->u))
1472	{
1473	Assert(pu64EaxEdx->u == uOld);
1474	*pEFlags \|= X86_EFL_ZF;
1475	}
1476	else
1477	*pEFlags &= ~X86_EFL_ZF;
1478	}
1479
1480
1481	# if defined(RT_ARCH_AMD64) \|\| defined(RT_ARCH_ARM64)
1482	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_locked,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1483	uint32_t *pEFlags))
1484	{
1485	# ifdef VBOX_STRICT
1486	RTUINT128U const uOld = *pu128RaxRdx;
1487	# endif
1488	# if defined(RT_ARCH_AMD64)
1489	if (ASMAtomicCmpXchgU128v2(&pu128Dst->u, pu128RbxRcx->s.Hi, pu128RbxRcx->s.Lo, pu128RaxRdx->s.Hi, pu128RaxRdx->s.Lo,
1490	&pu128RaxRdx->u))
1491	# else
1492	if (ASMAtomicCmpXchgU128(&pu128Dst->u, pu128RbxRcx->u, pu128RaxRdx->u, &pu128RaxRdx->u))
1493	# endif
1494	{
1495	Assert(pu128RaxRdx->s.Lo == uOld.s.Lo && pu128RaxRdx->s.Hi == uOld.s.Hi);
1496	*pEFlags \|= X86_EFL_ZF;
1497	}
1498	else
1499	*pEFlags &= ~X86_EFL_ZF;
1500	}
1501	# endif
1502
1503	#endif /* defined(IEM_WITHOUT_ASSEMBLY) */
1504
1505	# if !defined(RT_ARCH_ARM64) /** @todo may need this for unaligned accesses... */
1506	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b_fallback,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx,
1507	PRTUINT128U pu128RbxRcx, uint32_t *pEFlags))
1508	{
1509	RTUINT128U u128Tmp = *pu128Dst;
1510	if ( u128Tmp.s.Lo == pu128RaxRdx->s.Lo
1511	&& u128Tmp.s.Hi == pu128RaxRdx->s.Hi)
1512	{
1513	pu128Dst = pu128RbxRcx;
1514	*pEFlags \|= X86_EFL_ZF;
1515	}
1516	else
1517	{
1518	*pu128RaxRdx = u128Tmp;
1519	*pEFlags &= ~X86_EFL_ZF;
1520	}
1521	}
1522	#endif /* !RT_ARCH_ARM64 */
1523
1524	#if defined(IEM_WITHOUT_ASSEMBLY)
1525
1526	/* Unlocked versions mapped to the locked ones: */
1527
1528	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u8, (uint8_t pu8Dst, uint8_t puAl, uint8_t uSrcReg, uint32_t *pEFlags))
1529	{
1530	iemAImpl_cmpxchg_u8_locked(pu8Dst, puAl, uSrcReg, pEFlags);
1531	}
1532
1533
1534	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u16, (uint16_t pu16Dst, uint16_t puAx, uint16_t uSrcReg, uint32_t *pEFlags))
1535	{
1536	iemAImpl_cmpxchg_u16_locked(pu16Dst, puAx, uSrcReg, pEFlags);
1537	}
1538
1539
1540	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u32, (uint32_t pu32Dst, uint32_t puEax, uint32_t uSrcReg, uint32_t *pEFlags))
1541	{
1542	iemAImpl_cmpxchg_u32_locked(pu32Dst, puEax, uSrcReg, pEFlags);
1543	}
1544
1545
1546	# if ARCH_BITS == 32
1547	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t pu64Dst, uint64_t puRax, uint64_t puSrcReg, uint32_t pEFlags))
1548	{
1549	iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, puSrcReg, pEFlags);
1550	}
1551	# else
1552	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg_u64, (uint64_t pu64Dst, uint64_t puRax, uint64_t uSrcReg, uint32_t *pEFlags))
1553	{
1554	iemAImpl_cmpxchg_u64_locked(pu64Dst, puRax, uSrcReg, pEFlags);
1555	}
1556	# endif
1557
1558
1559	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg8b,(uint64_t pu64Dst, PRTUINT64U pu64EaxEdx, PRTUINT64U pu64EbxEcx, uint32_t pEFlags))
1560	{
1561	iemAImpl_cmpxchg8b_locked(pu64Dst, pu64EaxEdx, pu64EbxEcx, pEFlags);
1562	}
1563
1564
1565	IEM_DECL_IMPL_DEF(void, iemAImpl_cmpxchg16b,(PRTUINT128U pu128Dst, PRTUINT128U pu128RaxRdx, PRTUINT128U pu128RbxRcx,
1566	uint32_t *pEFlags))
1567	{
1568	iemAImpl_cmpxchg16b_locked(pu128Dst, pu128RaxRdx, pu128RbxRcx, pEFlags);
1569	}
1570
1571	#endif /* defined(IEM_WITHOUT_ASSEMBLY) */
1572
1573	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1574
1575	/*
1576	* MUL, IMUL, DIV and IDIV helpers.
1577	*
1578	* - The U64 versions must use 128-bit intermediates, so we need to abstract the
1579	* division step so we can select between using C operators and
1580	* RTUInt128DivRem/RTUInt128MulU64ByU64.
1581	*
1582	* - The U8 versions work returns output in AL + AH instead of xDX + xAX, with the
1583	* IDIV/DIV taking all the input in AX too. This means we have to abstract some
1584	* input loads and the result storing.
1585	*/
1586
1587	DECLINLINE(void) RTUInt128DivRemByU64(PRTUINT128U pQuotient, PRTUINT128U pRemainder, PCRTUINT128U pDividend, uint64_t u64Divisor)
1588	{
1589	# ifdef __GNUC__ /* GCC maybe really annoying in function. */
1590	pQuotient->s.Lo = 0;
1591	pQuotient->s.Hi = 0;
1592	# endif
1593	RTUINT128U Divisor;
1594	Divisor.s.Lo = u64Divisor;
1595	Divisor.s.Hi = 0;
1596	RTUInt128DivRem(pQuotient, pRemainder, pDividend, &Divisor);
1597	}
1598
1599	# define DIV_LOAD(a_Dividend) \
1600	a_Dividend.s.Lo = puA, a_Dividend.s.Hi = puD
1601	# define DIV_LOAD_U8(a_Dividend) \
1602	a_Dividend.u = *puAX
1603
1604	# define DIV_STORE(a_Quotient, a_uReminder) puA = (a_Quotient), puD = (a_uReminder)
1605	# define DIV_STORE_U8(a_Quotient, a_uReminder) *puAX = (uint8_t)(a_Quotient) \| ((uint16_t)(a_uReminder) << 8)
1606
1607	# define MUL_LOAD_F1() *puA
1608	# define MUL_LOAD_F1_U8() ((uint8_t)*puAX)
1609
1610	# define MUL_STORE(a_Result) puA = (a_Result).s.Lo, puD = (a_Result).s.Hi
1611	# define MUL_STORE_U8(a_Result) *puAX = a_Result.u
1612
1613	# define MULDIV_NEG(a_Value, a_cBitsWidth2x) \
1614	(a_Value).u = UINT ## a_cBitsWidth2x ## _C(0) - (a_Value).u
1615	# define MULDIV_NEG_U128(a_Value, a_cBitsWidth2x) \
1616	RTUInt128AssignNeg(&(a_Value))
1617
1618	# define MULDIV_MUL(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \
1619	(a_Result).u = (uint ## a_cBitsWidth2x ## _t)(a_Factor1) * (a_Factor2)
1620	# define MULDIV_MUL_U128(a_Result, a_Factor1, a_Factor2, a_cBitsWidth2x) \
1621	RTUInt128MulU64ByU64(&(a_Result), a_Factor1, a_Factor2);
1622
1623	# define MULDIV_MODDIV(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \
1624	a_Quotient.u = (a_Dividend).u / (a_uDivisor), \
1625	a_Remainder.u = (a_Dividend).u % (a_uDivisor)
1626	# define MULDIV_MODDIV_U128(a_Quotient, a_Remainder, a_Dividend, a_uDivisor) \
1627	RTUInt128DivRemByU64(&a_Quotient, &a_Remainder, &a_Dividend, a_uDivisor)
1628
1629
1630	/*
1631	* MUL
1632	*/
1633	# define EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, a_Suffix, a_fIntelFlags) \
1634	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_mul_u,a_cBitsWidth,a_Suffix), a_Args) \
1635	{ \
1636	RTUINT ## a_cBitsWidth2x ## U Result; \
1637	a_fnMul(Result, a_fnLoadF1(), uFactor, a_cBitsWidth2x); \
1638	a_fnStore(Result); \
1639	\
1640	/* Calc EFLAGS: */ \
1641	uint32_t fEfl = *pfEFlags; \
1642	if (a_fIntelFlags) \
1643	{ /* Intel: 6700K and 10980XE behavior */ \
1644	fEfl &= ~(X86_EFL_SF \| X86_EFL_CF \| X86_EFL_OF \| X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_PF); \
1645	if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \
1646	fEfl \|= X86_EFL_SF; \
1647	fEfl \|= g_afParity[Result.s.Lo & 0xff]; \
1648	if (Result.s.Hi != 0) \
1649	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1650	} \
1651	else \
1652	{ /* AMD: 3990X */ \
1653	if (Result.s.Hi != 0) \
1654	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1655	else \
1656	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
1657	} \
1658	*pfEFlags = fEfl; \
1659	return 0; \
1660	} \
1661
1662	# define EMIT_MUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul) \
1663	EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, RT_NOTHING, 1) \
1664	EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _intel, 1) \
1665	EMIT_MUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnMul, _amd, 0) \
1666
1667	# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
1668	EMIT_MUL(64, 128, (uint64_t puA, uint64_t puD, uint64_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
1669	MUL_LOAD_F1, MUL_STORE, MULDIV_MUL_U128)
1670	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1671	EMIT_MUL(32, 64, (uint32_t puA, uint32_t puD, uint32_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
1672	MUL_LOAD_F1, MUL_STORE, MULDIV_MUL)
1673	EMIT_MUL(16, 32, (uint16_t puA, uint16_t puD, uint16_t uFactor, uint32_t *pfEFlags), (puA, puD, uFactor, pfEFlags),
1674	MUL_LOAD_F1, MUL_STORE, MULDIV_MUL)
1675	EMIT_MUL(8, 16, (uint16_t puAX, uint8_t uFactor, uint32_t pfEFlags), (puAX, uFactor, pfEFlags),
1676	MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_MUL)
1677	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1678	# endif /* !DOXYGEN_RUNNING */
1679
1680
1681	/*
1682	* IMUL
1683	*
1684	* The SF, ZF, AF and PF flags are "undefined". AMD (3990x) leaves these
1685	* flags as is. Whereas Intel skylake (6700K and 10980X (Cascade Lake)) always
1686	* clear AF and ZF and calculates SF and PF as per the lower half of the result.
1687	*/
1688	# define EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, \
1689	a_Suffix, a_fIntelFlags) \
1690	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_imul_u,a_cBitsWidth,a_Suffix),a_Args) \
1691	{ \
1692	RTUINT ## a_cBitsWidth2x ## U Result; \
1693	uint32_t fEfl = *pfEFlags & ~(X86_EFL_CF \| X86_EFL_OF); \
1694	\
1695	uint ## a_cBitsWidth ## _t const uFactor1 = a_fnLoadF1(); \
1696	if (!(uFactor1 & RT_BIT_64(a_cBitsWidth - 1))) \
1697	{ \
1698	if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \
1699	{ \
1700	a_fnMul(Result, uFactor1, uFactor2, a_cBitsWidth2x); \
1701	if (Result.s.Hi != 0 \|\| Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \
1702	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1703	} \
1704	else \
1705	{ \
1706	uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \
1707	a_fnMul(Result, uFactor1, uPositiveFactor2, a_cBitsWidth2x); \
1708	if (Result.s.Hi != 0 \|\| Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \
1709	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1710	a_fnNeg(Result, a_cBitsWidth2x); \
1711	} \
1712	} \
1713	else \
1714	{ \
1715	if (!(uFactor2 & RT_BIT_64(a_cBitsWidth - 1))) \
1716	{ \
1717	uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \
1718	a_fnMul(Result, uPositiveFactor1, uFactor2, a_cBitsWidth2x); \
1719	if (Result.s.Hi != 0 \|\| Result.s.Lo > RT_BIT_64(a_cBitsWidth - 1)) \
1720	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1721	a_fnNeg(Result, a_cBitsWidth2x); \
1722	} \
1723	else \
1724	{ \
1725	uint ## a_cBitsWidth ## _t const uPositiveFactor1 = UINT ## a_cBitsWidth ## _C(0) - uFactor1; \
1726	uint ## a_cBitsWidth ## _t const uPositiveFactor2 = UINT ## a_cBitsWidth ## _C(0) - uFactor2; \
1727	a_fnMul(Result, uPositiveFactor1, uPositiveFactor2, a_cBitsWidth2x); \
1728	if (Result.s.Hi != 0 \|\| Result.s.Lo >= RT_BIT_64(a_cBitsWidth - 1)) \
1729	fEfl \|= X86_EFL_CF \| X86_EFL_OF; \
1730	} \
1731	} \
1732	a_fnStore(Result); \
1733	\
1734	if (a_fIntelFlags) \
1735	{ \
1736	fEfl &= ~(X86_EFL_AF \| X86_EFL_ZF \| X86_EFL_SF \| X86_EFL_PF); \
1737	if (Result.s.Lo & RT_BIT_64(a_cBitsWidth - 1)) \
1738	fEfl \|= X86_EFL_SF; \
1739	fEfl \|= g_afParity[Result.s.Lo & 0xff]; \
1740	} \
1741	*pfEFlags = fEfl; \
1742	return 0; \
1743	}
1744	# define EMIT_IMUL(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul) \
1745	EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, RT_NOTHING, 1) \
1746	EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _intel, 1) \
1747	EMIT_IMUL_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoadF1, a_fnStore, a_fnNeg, a_fnMul, _amd, 0)
1748
1749	# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
1750	EMIT_IMUL(64, 128, (uint64_t puA, uint64_t puD, uint64_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
1751	MUL_LOAD_F1, MUL_STORE, MULDIV_NEG_U128, MULDIV_MUL_U128)
1752	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1753	EMIT_IMUL(32, 64, (uint32_t puA, uint32_t puD, uint32_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
1754	MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL)
1755	EMIT_IMUL(16, 32, (uint16_t puA, uint16_t puD, uint16_t uFactor2, uint32_t *pfEFlags), (puA, puD, uFactor2, pfEFlags),
1756	MUL_LOAD_F1, MUL_STORE, MULDIV_NEG, MULDIV_MUL)
1757	EMIT_IMUL(8, 16, (uint16_t puAX, uint8_t uFactor2, uint32_t pfEFlags), (puAX, uFactor2, pfEFlags),
1758	MUL_LOAD_F1_U8, MUL_STORE_U8, MULDIV_NEG, MULDIV_MUL)
1759	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1760	# endif /* !DOXYGEN_RUNNING */
1761
1762
1763	/*
1764	* IMUL with two operands are mapped onto the three operand variant, ignoring
1765	* the high part of the product.
1766	*/
1767	# define EMIT_IMUL_TWO(a_cBits, a_uType) \
1768	IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits,(a_uType puDst, a_uType uSrc, uint32_t pfEFlags)) \
1769	{ \
1770	a_uType uIgn; \
1771	iemAImpl_imul_u ## a_cBits(puDst, &uIgn, uSrc, pfEFlags); \
1772	} \
1773	\
1774	IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _intel,(a_uType puDst, a_uType uSrc, uint32_t pfEFlags)) \
1775	{ \
1776	a_uType uIgn; \
1777	iemAImpl_imul_u ## a_cBits ## _intel(puDst, &uIgn, uSrc, pfEFlags); \
1778	} \
1779	\
1780	IEM_DECL_IMPL_DEF(void, iemAImpl_imul_two_u ## a_cBits ## _amd,(a_uType puDst, a_uType uSrc, uint32_t pfEFlags)) \
1781	{ \
1782	a_uType uIgn; \
1783	iemAImpl_imul_u ## a_cBits ## _amd(puDst, &uIgn, uSrc, pfEFlags); \
1784	}
1785
1786	EMIT_IMUL_TWO(64, uint64_t)
1787	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1788	EMIT_IMUL_TWO(32, uint32_t)
1789	EMIT_IMUL_TWO(16, uint16_t)
1790	# endif
1791
1792
1793	/*
1794	* DIV
1795	*/
1796	# define EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, \
1797	a_Suffix, a_fIntelFlags) \
1798	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_div_u,a_cBitsWidth,a_Suffix),a_Args) \
1799	{ \
1800	RTUINT ## a_cBitsWidth2x ## U Dividend; \
1801	a_fnLoad(Dividend); \
1802	if ( uDivisor != 0 \
1803	&& Dividend.s.Hi < uDivisor) \
1804	{ \
1805	RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \
1806	a_fnDivRem(Quotient, Remainder, Dividend, uDivisor); \
1807	a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \
1808	\
1809	/* Calc EFLAGS: Intel 6700K and 10980XE leaves them alone. AMD 3990X sets AF and clears PF, ZF and SF. */ \
1810	if (!a_fIntelFlags) \
1811	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1812	return 0; \
1813	} \
1814	/* #DE */ \
1815	return -1; \
1816	}
1817	# define EMIT_DIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem) \
1818	EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, RT_NOTHING, 1) \
1819	EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _intel, 1) \
1820	EMIT_DIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnDivRem, _amd, 0)
1821
1822	# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
1823	EMIT_DIV(64,128,(uint64_t puA, uint64_t puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1824	DIV_LOAD, DIV_STORE, MULDIV_MODDIV_U128)
1825	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1826	EMIT_DIV(32,64, (uint32_t puA, uint32_t puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1827	DIV_LOAD, DIV_STORE, MULDIV_MODDIV)
1828	EMIT_DIV(16,32, (uint16_t puA, uint16_t puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1829	DIV_LOAD, DIV_STORE, MULDIV_MODDIV)
1830	EMIT_DIV(8,16, (uint16_t puAX, uint8_t uDivisor, uint32_t pfEFlags), (puAX, uDivisor, pfEFlags),
1831	DIV_LOAD_U8, DIV_STORE_U8, MULDIV_MODDIV)
1832	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1833	# endif /* !DOXYGEN_RUNNING */
1834
1835
1836	/*
1837	* IDIV
1838	*
1839	* EFLAGS are ignored and left as-is by Intel 6700K and 10980XE. AMD 3990X will
1840	* set AF and clear PF, ZF and SF just like it does for DIV.
1841	*
1842	*/
1843	# define EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, \
1844	a_Suffix, a_fIntelFlags) \
1845	IEM_DECL_IMPL_DEF(int, RT_CONCAT3(iemAImpl_idiv_u,a_cBitsWidth,a_Suffix),a_Args) \
1846	{ \
1847	/* Note! Skylake leaves all flags alone. */ \
1848	\
1849	/** @todo overflow checks */ \
1850	if (uDivisor != 0) \
1851	{ \
1852	/* \
1853	* Convert to unsigned division. \
1854	*/ \
1855	RTUINT ## a_cBitsWidth2x ## U Dividend; \
1856	a_fnLoad(Dividend); \
1857	bool const fSignedDividend = RT_BOOL(Dividend.s.Hi & RT_BIT_64(a_cBitsWidth - 1)); \
1858	if (fSignedDividend) \
1859	a_fnNeg(Dividend, a_cBitsWidth2x); \
1860	\
1861	uint ## a_cBitsWidth ## _t uDivisorPositive; \
1862	if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \
1863	uDivisorPositive = uDivisor; \
1864	else \
1865	uDivisorPositive = UINT ## a_cBitsWidth ## _C(0) - uDivisor; \
1866	\
1867	RTUINT ## a_cBitsWidth2x ## U Remainder, Quotient; \
1868	a_fnDivRem(Quotient, Remainder, Dividend, uDivisorPositive); \
1869	\
1870	/* \
1871	* Setup the result, checking for overflows. \
1872	*/ \
1873	if (!(uDivisor & RT_BIT_64(a_cBitsWidth - 1))) \
1874	{ \
1875	if (!fSignedDividend) \
1876	{ \
1877	/* Positive divisor, positive dividend => result positive. */ \
1878	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \
1879	{ \
1880	a_fnStore(Quotient.s.Lo, Remainder.s.Lo); \
1881	if (!a_fIntelFlags) \
1882	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1883	return 0; \
1884	} \
1885	} \
1886	else \
1887	{ \
1888	/* Positive divisor, negative dividend => result negative. */ \
1889	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \
1890	{ \
1891	a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \
1892	if (!a_fIntelFlags) \
1893	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1894	return 0; \
1895	} \
1896	} \
1897	} \
1898	else \
1899	{ \
1900	if (!fSignedDividend) \
1901	{ \
1902	/* Negative divisor, positive dividend => negative quotient, positive remainder. */ \
1903	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= RT_BIT_64(a_cBitsWidth - 1)) \
1904	{ \
1905	a_fnStore(UINT ## a_cBitsWidth ## _C(0) - Quotient.s.Lo, Remainder.s.Lo); \
1906	if (!a_fIntelFlags) \
1907	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1908	return 0; \
1909	} \
1910	} \
1911	else \
1912	{ \
1913	/* Negative divisor, negative dividend => positive quotient, negative remainder. */ \
1914	if (Quotient.s.Hi == 0 && Quotient.s.Lo <= (uint ## a_cBitsWidth ## _t)INT ## a_cBitsWidth ## _MAX) \
1915	{ \
1916	a_fnStore(Quotient.s.Lo, UINT ## a_cBitsWidth ## _C(0) - Remainder.s.Lo); \
1917	if (!a_fIntelFlags) \
1918	pfEFlags = (pfEFlags & ~(X86_EFL_PF \| X86_EFL_ZF \| X86_EFL_SF)) \| X86_EFL_AF; \
1919	return 0; \
1920	} \
1921	} \
1922	} \
1923	} \
1924	/* #DE */ \
1925	return -1; \
1926	}
1927	# define EMIT_IDIV(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem) \
1928	EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, RT_NOTHING, 1) \
1929	EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _intel, 1) \
1930	EMIT_IDIV_INNER(a_cBitsWidth, a_cBitsWidth2x, a_Args, a_CallArgs, a_fnLoad, a_fnStore, a_fnNeg, a_fnDivRem, _amd, 0)
1931
1932	# ifndef DOXYGEN_RUNNING /* this totally confuses doxygen for some reason */
1933	EMIT_IDIV(64,128,(uint64_t puA, uint64_t puD, uint64_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1934	DIV_LOAD, DIV_STORE, MULDIV_NEG_U128, MULDIV_MODDIV_U128)
1935	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1936	EMIT_IDIV(32,64,(uint32_t puA, uint32_t puD, uint32_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1937	DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV)
1938	EMIT_IDIV(16,32,(uint16_t puA, uint16_t puD, uint16_t uDivisor, uint32_t *pfEFlags), (puA, puD, uDivisor, pfEFlags),
1939	DIV_LOAD, DIV_STORE, MULDIV_NEG, MULDIV_MODDIV)
1940	EMIT_IDIV(8,16,(uint16_t puAX, uint8_t uDivisor, uint32_t pfEFlags), (puAX, uDivisor, pfEFlags),
1941	DIV_LOAD_U8, DIV_STORE_U8, MULDIV_NEG, MULDIV_MODDIV)
1942	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
1943	# endif /* !DOXYGEN_RUNNING */
1944
1945
1946	/*********************************************************************************************************************************
1947	* Unary operations. *
1948	*********************************************************************************************************************************/
1949
1950	/** @def IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC
1951	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an INC or DEC instruction.
1952	*
1953	* CF is NOT modified for hysterical raisins (allegedly for carrying and
1954	* borrowing in arithmetic loops on intel 8008).
1955	*
1956	* @returns Status bits.
1957	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
1958	* @param a_uResult Unsigned result value.
1959	* @param a_uDst The original destination value (for AF calc).
1960	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
1961	* @param a_OfMethod 0 for INC-style, 1 for DEC-style.
1962	*/
1963	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth, a_OfMethod) \
1964	do { \
1965	uint32_t fEflTmp = *(a_pfEFlags); \
1966	fEflTmp &= ~X86_EFL_STATUS_BITS \| X86_EFL_CF; \
1967	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
1968	fEflTmp \|= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
1969	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
1970	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
1971	fEflTmp \|= X86_EFL_GET_OF_ ## a_cBitsWidth(a_OfMethod == 0 ? (((a_uDst) ^ RT_BIT_64(a_cBitsWidth - 1)) & (a_uResult)) \
1972	: ((a_uDst) & ((a_uResult) ^ RT_BIT_64(a_cBitsWidth - 1))) ); \
1973	*(a_pfEFlags) = fEflTmp; \
1974	} while (0)
1975
1976	/*
1977	* INC
1978	*/
1979
1980	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u64,(uint64_t puDst, uint32_t pfEFlags))
1981	{
1982	uint64_t uDst = *puDst;
1983	uint64_t uResult = uDst + 1;
1984	*puDst = uResult;
1985	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 0 /INC/);
1986	}
1987
1988	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
1989
1990	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u32,(uint32_t puDst, uint32_t pfEFlags))
1991	{
1992	uint32_t uDst = *puDst;
1993	uint32_t uResult = uDst + 1;
1994	*puDst = uResult;
1995	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 0 /INC/);
1996	}
1997
1998
1999	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u16,(uint16_t puDst, uint32_t pfEFlags))
2000	{
2001	uint16_t uDst = *puDst;
2002	uint16_t uResult = uDst + 1;
2003	*puDst = uResult;
2004	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 0 /INC/);
2005	}
2006
2007	IEM_DECL_IMPL_DEF(void, iemAImpl_inc_u8,(uint8_t puDst, uint32_t pfEFlags))
2008	{
2009	uint8_t uDst = *puDst;
2010	uint8_t uResult = uDst + 1;
2011	*puDst = uResult;
2012	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 0 /INC/);
2013	}
2014
2015	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2016
2017
2018	/*
2019	* DEC
2020	*/
2021
2022	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u64,(uint64_t puDst, uint32_t pfEFlags))
2023	{
2024	uint64_t uDst = *puDst;
2025	uint64_t uResult = uDst - 1;
2026	*puDst = uResult;
2027	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 64, 1 /INC/);
2028	}
2029
2030	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2031
2032	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u32,(uint32_t puDst, uint32_t pfEFlags))
2033	{
2034	uint32_t uDst = *puDst;
2035	uint32_t uResult = uDst - 1;
2036	*puDst = uResult;
2037	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 32, 1 /INC/);
2038	}
2039
2040
2041	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u16,(uint16_t puDst, uint32_t pfEFlags))
2042	{
2043	uint16_t uDst = *puDst;
2044	uint16_t uResult = uDst - 1;
2045	*puDst = uResult;
2046	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 16, 1 /INC/);
2047	}
2048
2049
2050	IEM_DECL_IMPL_DEF(void, iemAImpl_dec_u8,(uint8_t puDst, uint32_t pfEFlags))
2051	{
2052	uint8_t uDst = *puDst;
2053	uint8_t uResult = uDst - 1;
2054	*puDst = uResult;
2055	IEM_EFL_UPDATE_STATUS_BITS_FOR_INC_DEC(pfEFlags, uResult, uDst, 8, 1 /INC/);
2056	}
2057
2058	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2059
2060
2061	/*
2062	* NOT
2063	*/
2064
2065	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u64,(uint64_t puDst, uint32_t pfEFlags))
2066	{
2067	uint64_t uDst = *puDst;
2068	uint64_t uResult = ~uDst;
2069	*puDst = uResult;
2070	/* EFLAGS are not modified. */
2071	RT_NOREF_PV(pfEFlags);
2072	}
2073
2074	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2075
2076	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u32,(uint32_t puDst, uint32_t pfEFlags))
2077	{
2078	uint32_t uDst = *puDst;
2079	uint32_t uResult = ~uDst;
2080	*puDst = uResult;
2081	/* EFLAGS are not modified. */
2082	RT_NOREF_PV(pfEFlags);
2083	}
2084
2085	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u16,(uint16_t puDst, uint32_t pfEFlags))
2086	{
2087	uint16_t uDst = *puDst;
2088	uint16_t uResult = ~uDst;
2089	*puDst = uResult;
2090	/* EFLAGS are not modified. */
2091	RT_NOREF_PV(pfEFlags);
2092	}
2093
2094	IEM_DECL_IMPL_DEF(void, iemAImpl_not_u8,(uint8_t puDst, uint32_t pfEFlags))
2095	{
2096	uint8_t uDst = *puDst;
2097	uint8_t uResult = ~uDst;
2098	*puDst = uResult;
2099	/* EFLAGS are not modified. */
2100	RT_NOREF_PV(pfEFlags);
2101	}
2102
2103	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2104
2105
2106	/*
2107	* NEG
2108	*/
2109
2110	/**
2111	* Updates the status bits (CF, PF, AF, ZF, SF, and OF) for an NEG instruction.
2112	*
2113	* @returns Status bits.
2114	* @param a_pfEFlags Pointer to the 32-bit EFLAGS value to update.
2115	* @param a_uResult Unsigned result value.
2116	* @param a_uDst The original destination value (for AF calc).
2117	* @param a_cBitsWidth The width of the result (8, 16, 32, 64).
2118	*/
2119	#define IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(a_pfEFlags, a_uResult, a_uDst, a_cBitsWidth) \
2120	do { \
2121	uint32_t fEflTmp = *(a_pfEFlags); \
2122	fEflTmp &= ~X86_EFL_STATUS_BITS & ~X86_EFL_CF; \
2123	fEflTmp \|= ((a_uDst) != 0) << X86_EFL_CF_BIT; \
2124	fEflTmp \|= g_afParity[(a_uResult) & 0xff]; \
2125	fEflTmp \|= ((uint32_t)(a_uResult) ^ (uint32_t)(a_uDst)) & X86_EFL_AF; \
2126	fEflTmp \|= X86_EFL_CALC_ZF(a_uResult); \
2127	fEflTmp \|= X86_EFL_CALC_SF(a_uResult, a_cBitsWidth); \
2128	fEflTmp \|= X86_EFL_GET_OF_ ## a_cBitsWidth((a_uDst) & (a_uResult)); \
2129	*(a_pfEFlags) = fEflTmp; \
2130	} while (0)
2131
2132	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u64,(uint64_t puDst, uint32_t pfEFlags))
2133	{
2134	uint64_t uDst = *puDst;
2135	uint64_t uResult = (uint64_t)0 - uDst;
2136	*puDst = uResult;
2137	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 64);
2138	}
2139
2140	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2141
2142	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u32,(uint32_t puDst, uint32_t pfEFlags))
2143	{
2144	uint32_t uDst = *puDst;
2145	uint32_t uResult = (uint32_t)0 - uDst;
2146	*puDst = uResult;
2147	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 32);
2148	}
2149
2150
2151	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u16,(uint16_t puDst, uint32_t pfEFlags))
2152	{
2153	uint16_t uDst = *puDst;
2154	uint16_t uResult = (uint16_t)0 - uDst;
2155	*puDst = uResult;
2156	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 16);
2157	}
2158
2159
2160	IEM_DECL_IMPL_DEF(void, iemAImpl_neg_u8,(uint8_t puDst, uint32_t pfEFlags))
2161	{
2162	uint8_t uDst = *puDst;
2163	uint8_t uResult = (uint8_t)0 - uDst;
2164	*puDst = uResult;
2165	IEM_EFL_UPDATE_STATUS_BITS_FOR_NEG(pfEFlags, uResult, uDst, 8);
2166	}
2167
2168	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2169
2170	/*
2171	* Locked variants.
2172	*/
2173
2174	/** Emit a function for doing a locked unary operand operation. */
2175	# define EMIT_LOCKED_UNARY_OP(a_Mnemonic, a_cBitsWidth) \
2176	IEM_DECL_IMPL_DEF(void, iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth ## _locked,(uint ## a_cBitsWidth ## _t *puDst, \
2177	uint32_t *pfEFlags)) \
2178	{ \
2179	uint ## a_cBitsWidth ## _t uOld = ASMAtomicUoReadU ## a_cBitsWidth(puDst); \
2180	uint ## a_cBitsWidth ## _t uTmp; \
2181	uint32_t fEflTmp; \
2182	do \
2183	{ \
2184	uTmp = uOld; \
2185	fEflTmp = *pfEFlags; \
2186	iemAImpl_ ## a_Mnemonic ## _u ## a_cBitsWidth(&uTmp, &fEflTmp); \
2187	} while (!ASMAtomicCmpXchgExU ## a_cBitsWidth(puDst, uTmp, uOld, &uOld)); \
2188	*pfEFlags = fEflTmp; \
2189	}
2190
2191	EMIT_LOCKED_UNARY_OP(inc, 64)
2192	EMIT_LOCKED_UNARY_OP(dec, 64)
2193	EMIT_LOCKED_UNARY_OP(not, 64)
2194	EMIT_LOCKED_UNARY_OP(neg, 64)
2195	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2196	EMIT_LOCKED_UNARY_OP(inc, 32)
2197	EMIT_LOCKED_UNARY_OP(dec, 32)
2198	EMIT_LOCKED_UNARY_OP(not, 32)
2199	EMIT_LOCKED_UNARY_OP(neg, 32)
2200
2201	EMIT_LOCKED_UNARY_OP(inc, 16)
2202	EMIT_LOCKED_UNARY_OP(dec, 16)
2203	EMIT_LOCKED_UNARY_OP(not, 16)
2204	EMIT_LOCKED_UNARY_OP(neg, 16)
2205
2206	EMIT_LOCKED_UNARY_OP(inc, 8)
2207	EMIT_LOCKED_UNARY_OP(dec, 8)
2208	EMIT_LOCKED_UNARY_OP(not, 8)
2209	EMIT_LOCKED_UNARY_OP(neg, 8)
2210	# endif
2211
2212	#endif /* !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2213
2214
2215	/*********************************************************************************************************************************
2216	* Shifting and Rotating *
2217	*********************************************************************************************************************************/
2218
2219	/*
2220	* ROL
2221	*/
2222	#define EMIT_ROL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \
2223	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rol_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2224	{ \
2225	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2226	if (cShift) \
2227	{ \
2228	if (a_cBitsWidth < 32) \
2229	cShift &= a_cBitsWidth - 1; \
2230	a_uType const uDst = *puDst; \
2231	a_uType const uResult = a_fnHlp(uDst, cShift); \
2232	*puDst = uResult; \
2233	\
2234	/* Calc EFLAGS. The OF bit is undefined if cShift > 1, we implement \
2235	it the same way as for 1 bit shifts. */ \
2236	AssertCompile(X86_EFL_CF_BIT == 0); \
2237	uint32_t fEfl = *pfEFlags; \
2238	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2239	uint32_t const fCarry = (uResult & X86_EFL_CF); \
2240	fEfl \|= fCarry; \
2241	if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
2242	fEfl \|= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; \
2243	else /* Intel 10980XE: According to the first sub-shift: */ \
2244	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
2245	*pfEFlags = fEfl; \
2246	} \
2247	}
2248
2249	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2250	EMIT_ROL(64, uint64_t, RT_NOTHING, 1, ASMRotateLeftU64)
2251	#endif
2252	EMIT_ROL(64, uint64_t, _intel, 1, ASMRotateLeftU64)
2253	EMIT_ROL(64, uint64_t, _amd, 0, ASMRotateLeftU64)
2254
2255	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2256	EMIT_ROL(32, uint32_t, RT_NOTHING, 1, ASMRotateLeftU32)
2257	#endif
2258	EMIT_ROL(32, uint32_t, _intel, 1, ASMRotateLeftU32)
2259	EMIT_ROL(32, uint32_t, _amd, 0, ASMRotateLeftU32)
2260
2261	DECL_FORCE_INLINE(uint16_t) iemAImpl_rol_u16_hlp(uint16_t uValue, uint8_t cShift)
2262	{
2263	return (uValue << cShift) \| (uValue >> (16 - cShift));
2264	}
2265	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2266	EMIT_ROL(16, uint16_t, RT_NOTHING, 1, iemAImpl_rol_u16_hlp)
2267	#endif
2268	EMIT_ROL(16, uint16_t, _intel, 1, iemAImpl_rol_u16_hlp)
2269	EMIT_ROL(16, uint16_t, _amd, 0, iemAImpl_rol_u16_hlp)
2270
2271	DECL_FORCE_INLINE(uint8_t) iemAImpl_rol_u8_hlp(uint8_t uValue, uint8_t cShift)
2272	{
2273	return (uValue << cShift) \| (uValue >> (8 - cShift));
2274	}
2275	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2276	EMIT_ROL(8, uint8_t, RT_NOTHING, 1, iemAImpl_rol_u8_hlp)
2277	#endif
2278	EMIT_ROL(8, uint8_t, _intel, 1, iemAImpl_rol_u8_hlp)
2279	EMIT_ROL(8, uint8_t, _amd, 0, iemAImpl_rol_u8_hlp)
2280
2281
2282	/*
2283	* ROR
2284	*/
2285	#define EMIT_ROR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags, a_fnHlp) \
2286	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_ror_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2287	{ \
2288	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2289	if (cShift) \
2290	{ \
2291	if (a_cBitsWidth < 32) \
2292	cShift &= a_cBitsWidth - 1; \
2293	a_uType const uDst = *puDst; \
2294	a_uType const uResult = a_fnHlp(uDst, cShift); \
2295	*puDst = uResult; \
2296	\
2297	/* Calc EFLAGS: */ \
2298	AssertCompile(X86_EFL_CF_BIT == 0); \
2299	uint32_t fEfl = *pfEFlags; \
2300	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2301	uint32_t const fCarry = (uResult >> ((a_cBitsWidth) - 1)) & X86_EFL_CF; \
2302	fEfl \|= fCarry; \
2303	if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
2304	fEfl \|= (((uResult >> ((a_cBitsWidth) - 2)) ^ fCarry) & 1) << X86_EFL_OF_BIT; \
2305	else /* Intel 10980XE: According to the first sub-shift: */ \
2306	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << (a_cBitsWidth - 1))); \
2307	*pfEFlags = fEfl; \
2308	} \
2309	}
2310
2311	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2312	EMIT_ROR(64, uint64_t, RT_NOTHING, 1, ASMRotateRightU64)
2313	#endif
2314	EMIT_ROR(64, uint64_t, _intel, 1, ASMRotateRightU64)
2315	EMIT_ROR(64, uint64_t, _amd, 0, ASMRotateRightU64)
2316
2317	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2318	EMIT_ROR(32, uint32_t, RT_NOTHING, 1, ASMRotateRightU32)
2319	#endif
2320	EMIT_ROR(32, uint32_t, _intel, 1, ASMRotateRightU32)
2321	EMIT_ROR(32, uint32_t, _amd, 0, ASMRotateRightU32)
2322
2323	DECL_FORCE_INLINE(uint16_t) iemAImpl_ror_u16_hlp(uint16_t uValue, uint8_t cShift)
2324	{
2325	return (uValue >> cShift) \| (uValue << (16 - cShift));
2326	}
2327	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2328	EMIT_ROR(16, uint16_t, RT_NOTHING, 1, iemAImpl_ror_u16_hlp)
2329	#endif
2330	EMIT_ROR(16, uint16_t, _intel, 1, iemAImpl_ror_u16_hlp)
2331	EMIT_ROR(16, uint16_t, _amd, 0, iemAImpl_ror_u16_hlp)
2332
2333	DECL_FORCE_INLINE(uint8_t) iemAImpl_ror_u8_hlp(uint8_t uValue, uint8_t cShift)
2334	{
2335	return (uValue >> cShift) \| (uValue << (8 - cShift));
2336	}
2337	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2338	EMIT_ROR(8, uint8_t, RT_NOTHING, 1, iemAImpl_ror_u8_hlp)
2339	#endif
2340	EMIT_ROR(8, uint8_t, _intel, 1, iemAImpl_ror_u8_hlp)
2341	EMIT_ROR(8, uint8_t, _amd, 0, iemAImpl_ror_u8_hlp)
2342
2343
2344	/*
2345	* RCL
2346	*/
2347	#define EMIT_RCL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2348	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcl_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2349	{ \
2350	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2351	if (a_cBitsWidth < 32 && a_fIntelFlags) \
2352	cShift %= a_cBitsWidth + 1; \
2353	if (cShift) \
2354	{ \
2355	if (a_cBitsWidth < 32 && !a_fIntelFlags) \
2356	cShift %= a_cBitsWidth + 1; \
2357	a_uType const uDst = *puDst; \
2358	a_uType uResult = uDst << cShift; \
2359	if (cShift > 1) \
2360	uResult \|= uDst >> (a_cBitsWidth + 1 - cShift); \
2361	\
2362	AssertCompile(X86_EFL_CF_BIT == 0); \
2363	uint32_t fEfl = *pfEFlags; \
2364	uint32_t fInCarry = fEfl & X86_EFL_CF; \
2365	uResult \|= (a_uType)fInCarry << (cShift - 1); \
2366	\
2367	*puDst = uResult; \
2368	\
2369	/* Calc EFLAGS. */ \
2370	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2371	uint32_t const fOutCarry = a_cBitsWidth >= 32 \|\| a_fIntelFlags \|\| cShift \
2372	? (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF : fInCarry; \
2373	fEfl \|= fOutCarry; \
2374	if (!a_fIntelFlags) /* AMD 3990X: According to the last sub-shift: */ \
2375	fEfl \|= ((uResult >> (a_cBitsWidth - 1)) ^ fOutCarry) << X86_EFL_OF_BIT; \
2376	else /* Intel 10980XE: According to the first sub-shift: */ \
2377	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
2378	*pfEFlags = fEfl; \
2379	} \
2380	}
2381
2382	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2383	EMIT_RCL(64, uint64_t, RT_NOTHING, 1)
2384	#endif
2385	EMIT_RCL(64, uint64_t, _intel, 1)
2386	EMIT_RCL(64, uint64_t, _amd, 0)
2387
2388	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2389	EMIT_RCL(32, uint32_t, RT_NOTHING, 1)
2390	#endif
2391	EMIT_RCL(32, uint32_t, _intel, 1)
2392	EMIT_RCL(32, uint32_t, _amd, 0)
2393
2394	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2395	EMIT_RCL(16, uint16_t, RT_NOTHING, 1)
2396	#endif
2397	EMIT_RCL(16, uint16_t, _intel, 1)
2398	EMIT_RCL(16, uint16_t, _amd, 0)
2399
2400	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2401	EMIT_RCL(8, uint8_t, RT_NOTHING, 1)
2402	#endif
2403	EMIT_RCL(8, uint8_t, _intel, 1)
2404	EMIT_RCL(8, uint8_t, _amd, 0)
2405
2406
2407	/*
2408	* RCR
2409	*/
2410	#define EMIT_RCR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2411	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_rcr_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2412	{ \
2413	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2414	if (a_cBitsWidth < 32 && a_fIntelFlags) \
2415	cShift %= a_cBitsWidth + 1; \
2416	if (cShift) \
2417	{ \
2418	if (a_cBitsWidth < 32 && !a_fIntelFlags) \
2419	cShift %= a_cBitsWidth + 1; \
2420	a_uType const uDst = *puDst; \
2421	a_uType uResult = uDst >> cShift; \
2422	if (cShift > 1) \
2423	uResult \|= uDst << (a_cBitsWidth + 1 - cShift); \
2424	\
2425	AssertCompile(X86_EFL_CF_BIT == 0); \
2426	uint32_t fEfl = *pfEFlags; \
2427	uint32_t fInCarry = fEfl & X86_EFL_CF; \
2428	uResult \|= (a_uType)fInCarry << (a_cBitsWidth - cShift); \
2429	*puDst = uResult; \
2430	\
2431	/* Calc EFLAGS. The OF bit is undefined if cShift > 1, we implement \
2432	it the same way as for 1 bit shifts. */ \
2433	fEfl &= ~(X86_EFL_CF \| X86_EFL_OF); \
2434	uint32_t const fOutCarry = a_cBitsWidth >= 32 \|\| a_fIntelFlags \|\| cShift \
2435	? (uDst >> (cShift - 1)) & X86_EFL_CF : fInCarry; \
2436	fEfl \|= fOutCarry; \
2437	if (!a_fIntelFlags) /* AMD 3990X: XOR two most signficant bits of the result: */ \
2438	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uResult ^ (uResult << 1)); \
2439	else /* Intel 10980XE: same as AMD, but only for the first sub-shift: */ \
2440	fEfl \|= (fInCarry ^ (uint32_t)(uDst >> (a_cBitsWidth - 1))) << X86_EFL_OF_BIT; \
2441	*pfEFlags = fEfl; \
2442	} \
2443	}
2444
2445	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2446	EMIT_RCR(64, uint64_t, RT_NOTHING, 1)
2447	#endif
2448	EMIT_RCR(64, uint64_t, _intel, 1)
2449	EMIT_RCR(64, uint64_t, _amd, 0)
2450
2451	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2452	EMIT_RCR(32, uint32_t, RT_NOTHING, 1)
2453	#endif
2454	EMIT_RCR(32, uint32_t, _intel, 1)
2455	EMIT_RCR(32, uint32_t, _amd, 0)
2456
2457	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2458	EMIT_RCR(16, uint16_t, RT_NOTHING, 1)
2459	#endif
2460	EMIT_RCR(16, uint16_t, _intel, 1)
2461	EMIT_RCR(16, uint16_t, _amd, 0)
2462
2463	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2464	EMIT_RCR(8, uint8_t, RT_NOTHING, 1)
2465	#endif
2466	EMIT_RCR(8, uint8_t, _intel, 1)
2467	EMIT_RCR(8, uint8_t, _amd, 0)
2468
2469
2470	/*
2471	* SHL
2472	*/
2473	#define EMIT_SHL(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2474	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shl_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2475	{ \
2476	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2477	if (cShift) \
2478	{ \
2479	a_uType const uDst = *puDst; \
2480	a_uType uResult = uDst << cShift; \
2481	*puDst = uResult; \
2482	\
2483	/* Calc EFLAGS. */ \
2484	AssertCompile(X86_EFL_CF_BIT == 0); \
2485	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2486	uint32_t fCarry = (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; \
2487	fEfl \|= fCarry; \
2488	if (!a_fIntelFlags) \
2489	fEfl \|= ((uResult >> (a_cBitsWidth - 1)) ^ fCarry) << X86_EFL_OF_BIT; /* AMD 3990X: Last shift result. */ \
2490	else \
2491	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); /* Intel 10980XE: First shift result. */ \
2492	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2493	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2494	fEfl \|= g_afParity[uResult & 0xff]; \
2495	if (!a_fIntelFlags) \
2496	fEfl \|= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
2497	*pfEFlags = fEfl; \
2498	} \
2499	}
2500
2501	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2502	EMIT_SHL(64, uint64_t, RT_NOTHING, 1)
2503	#endif
2504	EMIT_SHL(64, uint64_t, _intel, 1)
2505	EMIT_SHL(64, uint64_t, _amd, 0)
2506
2507	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2508	EMIT_SHL(32, uint32_t, RT_NOTHING, 1)
2509	#endif
2510	EMIT_SHL(32, uint32_t, _intel, 1)
2511	EMIT_SHL(32, uint32_t, _amd, 0)
2512
2513	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2514	EMIT_SHL(16, uint16_t, RT_NOTHING, 1)
2515	#endif
2516	EMIT_SHL(16, uint16_t, _intel, 1)
2517	EMIT_SHL(16, uint16_t, _amd, 0)
2518
2519	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2520	EMIT_SHL(8, uint8_t, RT_NOTHING, 1)
2521	#endif
2522	EMIT_SHL(8, uint8_t, _intel, 1)
2523	EMIT_SHL(8, uint8_t, _amd, 0)
2524
2525
2526	/*
2527	* SHR
2528	*/
2529	#define EMIT_SHR(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2530	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shr_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2531	{ \
2532	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2533	if (cShift) \
2534	{ \
2535	a_uType const uDst = *puDst; \
2536	a_uType uResult = uDst >> cShift; \
2537	*puDst = uResult; \
2538	\
2539	/* Calc EFLAGS. */ \
2540	AssertCompile(X86_EFL_CF_BIT == 0); \
2541	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2542	fEfl \|= (uDst >> (cShift - 1)) & X86_EFL_CF; \
2543	if (a_fIntelFlags \|\| cShift == 1) /* AMD 3990x does what intel documents; Intel 10980XE does this for all shift counts. */ \
2544	fEfl \|= (uDst >> (a_cBitsWidth - 1)) << X86_EFL_OF_BIT; \
2545	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2546	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2547	fEfl \|= g_afParity[uResult & 0xff]; \
2548	if (!a_fIntelFlags) \
2549	fEfl \|= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
2550	*pfEFlags = fEfl; \
2551	} \
2552	}
2553
2554	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2555	EMIT_SHR(64, uint64_t, RT_NOTHING, 1)
2556	#endif
2557	EMIT_SHR(64, uint64_t, _intel, 1)
2558	EMIT_SHR(64, uint64_t, _amd, 0)
2559
2560	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2561	EMIT_SHR(32, uint32_t, RT_NOTHING, 1)
2562	#endif
2563	EMIT_SHR(32, uint32_t, _intel, 1)
2564	EMIT_SHR(32, uint32_t, _amd, 0)
2565
2566	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2567	EMIT_SHR(16, uint16_t, RT_NOTHING, 1)
2568	#endif
2569	EMIT_SHR(16, uint16_t, _intel, 1)
2570	EMIT_SHR(16, uint16_t, _amd, 0)
2571
2572	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2573	EMIT_SHR(8, uint8_t, RT_NOTHING, 1)
2574	#endif
2575	EMIT_SHR(8, uint8_t, _intel, 1)
2576	EMIT_SHR(8, uint8_t, _amd, 0)
2577
2578
2579	/*
2580	* SAR
2581	*/
2582	#define EMIT_SAR(a_cBitsWidth, a_uType, a_iType, a_Suffix, a_fIntelFlags) \
2583	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_sar_u,a_cBitsWidth,a_Suffix),(a_uType puDst, uint8_t cShift, uint32_t pfEFlags)) \
2584	{ \
2585	cShift &= a_cBitsWidth >= 32 ? a_cBitsWidth - 1 : 31; \
2586	if (cShift) \
2587	{ \
2588	a_iType const iDst = (a_iType)*puDst; \
2589	a_uType uResult = iDst >> cShift; \
2590	*puDst = uResult; \
2591	\
2592	/* Calc EFLAGS. \
2593	Note! The OF flag is always zero because the result never differs from the input. */ \
2594	AssertCompile(X86_EFL_CF_BIT == 0); \
2595	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2596	fEfl \|= (iDst >> (cShift - 1)) & X86_EFL_CF; \
2597	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2598	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2599	fEfl \|= g_afParity[uResult & 0xff]; \
2600	if (!a_fIntelFlags) \
2601	fEfl \|= X86_EFL_AF; /* AMD 3990x sets it unconditionally, Intel 10980XE does the oposite */ \
2602	*pfEFlags = fEfl; \
2603	} \
2604	}
2605
2606	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2607	EMIT_SAR(64, uint64_t, int64_t, RT_NOTHING, 1)
2608	#endif
2609	EMIT_SAR(64, uint64_t, int64_t, _intel, 1)
2610	EMIT_SAR(64, uint64_t, int64_t, _amd, 0)
2611
2612	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2613	EMIT_SAR(32, uint32_t, int32_t, RT_NOTHING, 1)
2614	#endif
2615	EMIT_SAR(32, uint32_t, int32_t, _intel, 1)
2616	EMIT_SAR(32, uint32_t, int32_t, _amd, 0)
2617
2618	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2619	EMIT_SAR(16, uint16_t, int16_t, RT_NOTHING, 1)
2620	#endif
2621	EMIT_SAR(16, uint16_t, int16_t, _intel, 1)
2622	EMIT_SAR(16, uint16_t, int16_t, _amd, 0)
2623
2624	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2625	EMIT_SAR(8, uint8_t, int8_t, RT_NOTHING, 1)
2626	#endif
2627	EMIT_SAR(8, uint8_t, int8_t, _intel, 1)
2628	EMIT_SAR(8, uint8_t, int8_t, _amd, 0)
2629
2630
2631	/*
2632	* SHLD
2633	*
2634	* - CF is the last bit shifted out of puDst.
2635	* - AF is always cleared by Intel 10980XE.
2636	* - AF is always set by AMD 3990X.
2637	* - OF is set according to the first shift on Intel 10980XE, it seems.
2638	* - OF is set according to the last sub-shift on AMD 3990X.
2639	* - ZF, SF and PF are calculated according to the result by both vendors.
2640	*
2641	* For 16-bit shifts the count mask isn't 15, but 31, and the CPU will
2642	* pick either the source register or the destination register for input bits
2643	* when going beyond 16. According to https://www.sandpile.org/x86/flags.htm
2644	* intel has changed behaviour here several times. We implement what current
2645	* skylake based does for now, we can extend this later as needed.
2646	*/
2647	#define EMIT_SHLD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2648	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shld_u,a_cBitsWidth,a_Suffix),(a_uType *puDst, a_uType uSrc, uint8_t cShift, \
2649	uint32_t *pfEFlags)) \
2650	{ \
2651	cShift &= a_cBitsWidth - 1; \
2652	if (cShift) \
2653	{ \
2654	a_uType const uDst = *puDst; \
2655	a_uType uResult = uDst << cShift; \
2656	uResult \|= uSrc >> (a_cBitsWidth - cShift); \
2657	*puDst = uResult; \
2658	\
2659	/* CALC EFLAGS: */ \
2660	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2661	if (a_fIntelFlags) \
2662	/* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
2663	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uDst << 1)); \
2664	else \
2665	{ /* AMD 3990X: Set according to last shift. AF always set. */ \
2666	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth((uDst << (cShift - 1)) ^ uResult); \
2667	fEfl \|= X86_EFL_AF; \
2668	} \
2669	AssertCompile(X86_EFL_CF_BIT == 0); \
2670	fEfl \|= (uDst >> (a_cBitsWidth - cShift)) & X86_EFL_CF; /* CF = last bit shifted out */ \
2671	fEfl \|= g_afParity[uResult & 0xff]; \
2672	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2673	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2674	*pfEFlags = fEfl; \
2675	} \
2676	}
2677
2678	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2679	EMIT_SHLD(64, uint64_t, RT_NOTHING, 1)
2680	#endif
2681	EMIT_SHLD(64, uint64_t, _intel, 1)
2682	EMIT_SHLD(64, uint64_t, _amd, 0)
2683
2684	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2685	EMIT_SHLD(32, uint32_t, RT_NOTHING, 1)
2686	#endif
2687	EMIT_SHLD(32, uint32_t, _intel, 1)
2688	EMIT_SHLD(32, uint32_t, _amd, 0)
2689
2690	#define EMIT_SHLD_16(a_Suffix, a_fIntelFlags) \
2691	IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shld_u16,a_Suffix),(uint16_t puDst, uint16_t uSrc, uint8_t cShift, uint32_t pfEFlags)) \
2692	{ \
2693	cShift &= 31; \
2694	if (cShift) \
2695	{ \
2696	uint16_t const uDst = *puDst; \
2697	uint64_t const uTmp = a_fIntelFlags \
2698	? ((uint64_t)uDst << 32) \| ((uint32_t)uSrc << 16) \| uDst \
2699	: ((uint64_t)uDst << 32) \| ((uint32_t)uSrc << 16) \| uSrc; \
2700	uint16_t const uResult = (uint16_t)((uTmp << cShift) >> 32); \
2701	*puDst = uResult; \
2702	\
2703	/* CALC EFLAGS: */ \
2704	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2705	AssertCompile(X86_EFL_CF_BIT == 0); \
2706	if (a_fIntelFlags) \
2707	{ \
2708	fEfl \|= (uTmp >> (48 - cShift)) & X86_EFL_CF; /* CF = last bit shifted out of the combined operand */ \
2709	/* Intel 6700K & 10980XE: OF is et according to the first shift. AF always cleared. */ \
2710	fEfl \|= X86_EFL_GET_OF_16(uDst ^ (uDst << 1)); \
2711	} \
2712	else \
2713	{ \
2714	/* AMD 3990X: OF is set according to last shift, with some weirdness. AF always set. CF = last bit shifted out of uDst. */ \
2715	if (cShift < 16) \
2716	{ \
2717	fEfl \|= (uDst >> (16 - cShift)) & X86_EFL_CF; \
2718	fEfl \|= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ uResult); \
2719	} \
2720	else \
2721	{ \
2722	if (cShift == 16) \
2723	fEfl \|= uDst & X86_EFL_CF; \
2724	fEfl \|= X86_EFL_GET_OF_16((uDst << (cShift - 1)) ^ 0); \
2725	} \
2726	fEfl \|= X86_EFL_AF; \
2727	} \
2728	fEfl \|= g_afParity[uResult & 0xff]; \
2729	fEfl \|= X86_EFL_CALC_SF(uResult, 16); \
2730	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2731	*pfEFlags = fEfl; \
2732	} \
2733	}
2734
2735	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2736	EMIT_SHLD_16(RT_NOTHING, 1)
2737	#endif
2738	EMIT_SHLD_16(_intel, 1)
2739	EMIT_SHLD_16(_amd, 0)
2740
2741
2742	/*
2743	* SHRD
2744	*
2745	* EFLAGS behaviour seems to be the same as with SHLD:
2746	* - CF is the last bit shifted out of puDst.
2747	* - AF is always cleared by Intel 10980XE.
2748	* - AF is always set by AMD 3990X.
2749	* - OF is set according to the first shift on Intel 10980XE, it seems.
2750	* - OF is set according to the last sub-shift on AMD 3990X.
2751	* - ZF, SF and PF are calculated according to the result by both vendors.
2752	*
2753	* For 16-bit shifts the count mask isn't 15, but 31, and the CPU will
2754	* pick either the source register or the destination register for input bits
2755	* when going beyond 16. According to https://www.sandpile.org/x86/flags.htm
2756	* intel has changed behaviour here several times. We implement what current
2757	* skylake based does for now, we can extend this later as needed.
2758	*/
2759	#define EMIT_SHRD(a_cBitsWidth, a_uType, a_Suffix, a_fIntelFlags) \
2760	IEM_DECL_IMPL_DEF(void, RT_CONCAT3(iemAImpl_shrd_u,a_cBitsWidth,a_Suffix),(a_uType puDst, a_uType uSrc, uint8_t cShift, uint32_t pfEFlags)) \
2761	{ \
2762	cShift &= a_cBitsWidth - 1; \
2763	if (cShift) \
2764	{ \
2765	a_uType const uDst = *puDst; \
2766	a_uType uResult = uDst >> cShift; \
2767	uResult \|= uSrc << (a_cBitsWidth - cShift); \
2768	*puDst = uResult; \
2769	\
2770	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2771	AssertCompile(X86_EFL_CF_BIT == 0); \
2772	fEfl \|= (uDst >> (cShift - 1)) & X86_EFL_CF; \
2773	if (a_fIntelFlags) \
2774	/* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
2775	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ (uSrc << (a_cBitsWidth - 1))); \
2776	else \
2777	{ /* AMD 3990X: Set according to last shift. AF always set. */ \
2778	if (cShift > 1) /* Set according to last shift. */ \
2779	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth((uSrc << (a_cBitsWidth - cShift + 1)) ^ uResult); \
2780	else \
2781	fEfl \|= X86_EFL_GET_OF_ ## a_cBitsWidth(uDst ^ uResult); \
2782	fEfl \|= X86_EFL_AF; \
2783	} \
2784	fEfl \|= X86_EFL_CALC_SF(uResult, a_cBitsWidth); \
2785	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2786	fEfl \|= g_afParity[uResult & 0xff]; \
2787	*pfEFlags = fEfl; \
2788	} \
2789	}
2790
2791	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2792	EMIT_SHRD(64, uint64_t, RT_NOTHING, 1)
2793	#endif
2794	EMIT_SHRD(64, uint64_t, _intel, 1)
2795	EMIT_SHRD(64, uint64_t, _amd, 0)
2796
2797	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2798	EMIT_SHRD(32, uint32_t, RT_NOTHING, 1)
2799	#endif
2800	EMIT_SHRD(32, uint32_t, _intel, 1)
2801	EMIT_SHRD(32, uint32_t, _amd, 0)
2802
2803	#define EMIT_SHRD_16(a_Suffix, a_fIntelFlags) \
2804	IEM_DECL_IMPL_DEF(void, RT_CONCAT(iemAImpl_shrd_u16,a_Suffix),(uint16_t puDst, uint16_t uSrc, uint8_t cShift, uint32_t pfEFlags)) \
2805	{ \
2806	cShift &= 31; \
2807	if (cShift) \
2808	{ \
2809	uint16_t const uDst = *puDst; \
2810	uint64_t const uTmp = a_fIntelFlags \
2811	? uDst \| ((uint32_t)uSrc << 16) \| ((uint64_t)uDst << 32) \
2812	: uDst \| ((uint32_t)uSrc << 16) \| ((uint64_t)uSrc << 32); \
2813	uint16_t const uResult = (uint16_t)(uTmp >> cShift); \
2814	*puDst = uResult; \
2815	\
2816	uint32_t fEfl = *pfEFlags & ~X86_EFL_STATUS_BITS; \
2817	AssertCompile(X86_EFL_CF_BIT == 0); \
2818	if (a_fIntelFlags) \
2819	{ \
2820	/* Intel 10980XE: The CF is the last shifted out of the combined uTmp operand. */ \
2821	fEfl \|= (uTmp >> (cShift - 1)) & X86_EFL_CF; \
2822	/* Intel 6700K & 10980XE: Set according to the first shift. AF always cleared. */ \
2823	fEfl \|= X86_EFL_GET_OF_16(uDst ^ (uSrc << 15)); \
2824	} \
2825	else \
2826	{ \
2827	/* AMD 3990X: CF flag seems to be last bit shifted out of uDst, not the combined uSrc:uSrc:uDst operand. */ \
2828	fEfl \|= (uDst >> (cShift - 1)) & X86_EFL_CF; \
2829	/* AMD 3990X: Set according to last shift. AF always set. */ \
2830	if (cShift > 1) /* Set according to last shift. */ \
2831	fEfl \|= X86_EFL_GET_OF_16((uint16_t)(uTmp >> (cShift - 1)) ^ uResult); \
2832	else \
2833	fEfl \|= X86_EFL_GET_OF_16(uDst ^ uResult); \
2834	fEfl \|= X86_EFL_AF; \
2835	} \
2836	fEfl \|= X86_EFL_CALC_SF(uResult, 16); \
2837	fEfl \|= X86_EFL_CALC_ZF(uResult); \
2838	fEfl \|= g_afParity[uResult & 0xff]; \
2839	*pfEFlags = fEfl; \
2840	} \
2841	}
2842
2843	#if (!defined(RT_ARCH_X86) && !defined(RT_ARCH_AMD64)) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2844	EMIT_SHRD_16(RT_NOTHING, 1)
2845	#endif
2846	EMIT_SHRD_16(_intel, 1)
2847	EMIT_SHRD_16(_amd, 0)
2848
2849
2850	#if !defined(RT_ARCH_AMD64) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2851
2852	# if !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY)
2853	/*
2854	* BSWAP
2855	*/
2856
2857	IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u64,(uint64_t *puDst))
2858	{
2859	puDst = ASMByteSwapU64(puDst);
2860	}
2861
2862
2863	IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u32,(uint32_t *puDst))
2864	{
2865	puDst = ASMByteSwapU32(puDst);
2866	}
2867
2868
2869	/* Note! undocument, so 32-bit arg */
2870	IEM_DECL_IMPL_DEF(void, iemAImpl_bswap_u16,(uint32_t *puDst))
2871	{
2872	#if 0
2873	(uint16_t )puDst = ASMByteSwapU16((uint16_t )puDst);
2874	#else
2875	/* This is the behaviour AMD 3990x (64-bit mode): */
2876	(uint16_t )puDst = 0;
2877	#endif
2878	}
2879
2880	# endif /* !defined(RT_ARCH_X86) \|\| defined(IEM_WITHOUT_ASSEMBLY) */
2881
2882
2883
2884	# if defined(IEM_WITHOUT_ASSEMBLY)
2885
2886	/*
2887	* LFENCE, SFENCE & MFENCE.
2888	*/
2889
2890	IEM_DECL_IMPL_DEF(void, iemAImpl_lfence,(void))
2891	{
2892	ASMReadFence();
2893	}
2894
2895
2896	IEM_DECL_IMPL_DEF(void, iemAImpl_sfence,(void))
2897	{
2898	ASMWriteFence();
2899	}
2900
2901
2902	IEM_DECL_IMPL_DEF(void, iemAImpl_mfence,(void))
2903	{
2904	ASMMemoryFence();
2905	}
2906
2907
2908	# ifndef RT_ARCH_ARM64
2909	IEM_DECL_IMPL_DEF(void, iemAImpl_alt_mem_fence,(void))
2910	{
2911	ASMMemoryFence();
2912	}
2913	# endif
2914
2915	# endif
2916
2917	#endif /* !RT_ARCH_AMD64 \|\| IEM_WITHOUT_ASSEMBLY */
2918
2919
2920	IEM_DECL_IMPL_DEF(void, iemAImpl_arpl,(uint16_t pu16Dst, uint16_t u16Src, uint32_t pfEFlags))
2921	{
2922	if ((*pu16Dst & X86_SEL_RPL) < (u16Src & X86_SEL_RPL))
2923	{
2924	*pu16Dst &= X86_SEL_MASK_OFF_RPL;
2925	*pu16Dst \|= u16Src & X86_SEL_RPL;
2926
2927	*pfEFlags \|= X86_EFL_ZF;
2928	}
2929	else
2930	*pfEFlags &= ~X86_EFL_ZF;
2931	}
2932
2933
2934	#if defined(IEM_WITHOUT_ASSEMBLY)
2935
2936	/*********************************************************************************************************************************
2937	* x87 FPU Loads *
2938	*********************************************************************************************************************************/
2939
2940	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT32U pr32Val))
2941	{
2942	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
2943	if (RTFLOAT32U_IS_NORMAL(pr32Val))
2944	{
2945	pFpuRes->r80Result.sj64.fSign = pr32Val->s.fSign;
2946	pFpuRes->r80Result.sj64.fInteger = 1;
2947	pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction
2948	<< (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS);
2949	pFpuRes->r80Result.sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS;
2950	Assert(RTFLOAT80U_IS_NORMAL(&pFpuRes->r80Result));
2951	}
2952	else if (RTFLOAT32U_IS_ZERO(pr32Val))
2953	{
2954	pFpuRes->r80Result.s.fSign = pr32Val->s.fSign;
2955	pFpuRes->r80Result.s.uExponent = 0;
2956	pFpuRes->r80Result.s.uMantissa = 0;
2957	Assert(RTFLOAT80U_IS_ZERO(&pFpuRes->r80Result));
2958	}
2959	else if (RTFLOAT32U_IS_SUBNORMAL(pr32Val))
2960	{
2961	/* Subnormal values gets normalized. */
2962	pFpuRes->r80Result.sj64.fSign = pr32Val->s.fSign;
2963	pFpuRes->r80Result.sj64.fInteger = 1;
2964	unsigned const cExtraShift = RTFLOAT32U_FRACTION_BITS - ASMBitLastSetU32(pr32Val->s.uFraction);
2965	pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction
2966	<< (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS + cExtraShift + 1);
2967	pFpuRes->r80Result.sj64.uExponent = pr32Val->s.uExponent - RTFLOAT32U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift;
2968	pFpuRes->FSW \|= X86_FSW_DE;
2969	if (!(pFpuState->FCW & X86_FCW_DM))
2970	pFpuRes->FSW \|= X86_FSW_ES \| X86_FSW_B; /* The value is still pushed. */
2971	}
2972	else if (RTFLOAT32U_IS_INF(pr32Val))
2973	{
2974	pFpuRes->r80Result.s.fSign = pr32Val->s.fSign;
2975	pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_MAX;
2976	pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63);
2977	Assert(RTFLOAT80U_IS_INF(&pFpuRes->r80Result));
2978	}
2979	else
2980	{
2981	/* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */
2982	Assert(RTFLOAT32U_IS_NAN(pr32Val));
2983	pFpuRes->r80Result.sj64.fSign = pr32Val->s.fSign;
2984	pFpuRes->r80Result.sj64.uExponent = RTFLOAT80U_EXP_MAX;
2985	pFpuRes->r80Result.sj64.fInteger = 1;
2986	pFpuRes->r80Result.sj64.uFraction = (uint64_t)pr32Val->s.uFraction
2987	<< (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS);
2988	if (RTFLOAT32U_IS_SIGNALLING_NAN(pr32Val))
2989	{
2990	pFpuRes->r80Result.sj64.uFraction \|= RT_BIT_64(62); /* make quiet */
2991	Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
2992	pFpuRes->FSW \|= X86_FSW_IE;
2993
2994	if (!(pFpuState->FCW & X86_FCW_IM))
2995	{
2996	/* The value is not pushed. */
2997	pFpuRes->FSW &= ~X86_FSW_TOP_MASK;
2998	pFpuRes->FSW \|= X86_FSW_ES \| X86_FSW_B;
2999	pFpuRes->r80Result.au64[0] = 0;
3000	pFpuRes->r80Result.au16[4] = 0;
3001	}
3002	}
3003	else
3004	Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
3005	}
3006	}
3007
3008
3009	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT64U pr64Val))
3010	{
3011	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3012	if (RTFLOAT64U_IS_NORMAL(pr64Val))
3013	{
3014	pFpuRes->r80Result.sj64.fSign = pr64Val->s.fSign;
3015	pFpuRes->r80Result.sj64.fInteger = 1;
3016	pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
3017	pFpuRes->r80Result.sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS;
3018	Assert(RTFLOAT80U_IS_NORMAL(&pFpuRes->r80Result));
3019	}
3020	else if (RTFLOAT64U_IS_ZERO(pr64Val))
3021	{
3022	pFpuRes->r80Result.s.fSign = pr64Val->s.fSign;
3023	pFpuRes->r80Result.s.uExponent = 0;
3024	pFpuRes->r80Result.s.uMantissa = 0;
3025	Assert(RTFLOAT80U_IS_ZERO(&pFpuRes->r80Result));
3026	}
3027	else if (RTFLOAT64U_IS_SUBNORMAL(pr64Val))
3028	{
3029	/* Subnormal values gets normalized. */
3030	pFpuRes->r80Result.sj64.fSign = pr64Val->s.fSign;
3031	pFpuRes->r80Result.sj64.fInteger = 1;
3032	unsigned const cExtraShift = RTFLOAT64U_FRACTION_BITS - ASMBitLastSetU64(pr64Val->s64.uFraction);
3033	pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction
3034	<< (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS + cExtraShift + 1);
3035	pFpuRes->r80Result.sj64.uExponent = pr64Val->s.uExponent - RTFLOAT64U_EXP_BIAS + RTFLOAT80U_EXP_BIAS - cExtraShift;
3036	pFpuRes->FSW \|= X86_FSW_DE;
3037	if (!(pFpuState->FCW & X86_FCW_DM))
3038	pFpuRes->FSW \|= X86_FSW_ES \| X86_FSW_B; /* The value is still pushed. */
3039	}
3040	else if (RTFLOAT64U_IS_INF(pr64Val))
3041	{
3042	pFpuRes->r80Result.s.fSign = pr64Val->s.fSign;
3043	pFpuRes->r80Result.s.uExponent = RTFLOAT80U_EXP_MAX;
3044	pFpuRes->r80Result.s.uMantissa = RT_BIT_64(63);
3045	Assert(RTFLOAT80U_IS_INF(&pFpuRes->r80Result));
3046	}
3047	else
3048	{
3049	/* Signalling and quiet NaNs, both turn into quiet ones when loaded (weird). */
3050	Assert(RTFLOAT64U_IS_NAN(pr64Val));
3051	pFpuRes->r80Result.sj64.fSign = pr64Val->s.fSign;
3052	pFpuRes->r80Result.sj64.uExponent = RTFLOAT80U_EXP_MAX;
3053	pFpuRes->r80Result.sj64.fInteger = 1;
3054	pFpuRes->r80Result.sj64.uFraction = pr64Val->s64.uFraction << (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
3055	if (RTFLOAT64U_IS_SIGNALLING_NAN(pr64Val))
3056	{
3057	pFpuRes->r80Result.sj64.uFraction \|= RT_BIT_64(62); /* make quiet */
3058	Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
3059	pFpuRes->FSW \|= X86_FSW_IE;
3060
3061	if (!(pFpuState->FCW & X86_FCW_IM))
3062	{
3063	/* The value is not pushed. */
3064	pFpuRes->FSW &= ~X86_FSW_TOP_MASK;
3065	pFpuRes->FSW \|= X86_FSW_ES \| X86_FSW_B;
3066	pFpuRes->r80Result.au64[0] = 0;
3067	pFpuRes->r80Result.au16[4] = 0;
3068	}
3069	}
3070	else
3071	Assert(RTFLOAT80U_IS_QUIET_NAN(&pFpuRes->r80Result));
3072	}
3073	}
3074
3075
3076	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
3077	{
3078	pFpuRes->r80Result.au64[0] = pr80Val->au64[0];
3079	pFpuRes->r80Result.au16[4] = pr80Val->au16[4];
3080	/* Raises no exceptions. */
3081	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3082	}
3083
3084
3085	IEM_DECL_IMPL_DEF(void, iemAImpl_fld1,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3086	{
3087	pFpuRes->r80Result.sj64.fSign = 0;
3088	pFpuRes->r80Result.sj64.uExponent = 0 + 16383;
3089	pFpuRes->r80Result.sj64.fInteger = 1;
3090	pFpuRes->r80Result.sj64.uFraction = 0;
3091
3092	/*
3093	* FPU status word:
3094	* - TOP is irrelevant, but we must match x86 assembly version.
3095	* - C1 is always cleared as we don't have any stack overflows.
3096	* - C0, C2, and C3 are undefined and Intel 10980XE does not touch them.
3097	*/
3098	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3));
3099	}
3100
3101
3102	IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2e,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3103	{
3104	pFpuRes->r80Result.sj64.fSign = 0;
3105	pFpuRes->r80Result.sj64.uExponent = 0 + 16383;
3106	pFpuRes->r80Result.sj64.fInteger = 1;
3107	pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
3108	\|\| (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
3109	? UINT64_C(0x38aa3b295c17f0bc) : UINT64_C(0x38aa3b295c17f0bb);
3110	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3111	}
3112
3113
3114	IEM_DECL_IMPL_DEF(void, iemAImpl_fldl2t,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3115	{
3116	pFpuRes->r80Result.sj64.fSign = 0;
3117	pFpuRes->r80Result.sj64.uExponent = 1 + 16383;
3118	pFpuRes->r80Result.sj64.fInteger = 1;
3119	pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) != X86_FCW_RC_UP
3120	? UINT64_C(0x549a784bcd1b8afe) : UINT64_C(0x549a784bcd1b8aff);
3121	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3122	}
3123
3124
3125	IEM_DECL_IMPL_DEF(void, iemAImpl_fldlg2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3126	{
3127	pFpuRes->r80Result.sj64.fSign = 0;
3128	pFpuRes->r80Result.sj64.uExponent = -2 + 16383;
3129	pFpuRes->r80Result.sj64.fInteger = 1;
3130	pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
3131	\|\| (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
3132	? UINT64_C(0x1a209a84fbcff799) : UINT64_C(0x1a209a84fbcff798);
3133	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3134	}
3135
3136
3137	IEM_DECL_IMPL_DEF(void, iemAImpl_fldln2,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3138	{
3139	pFpuRes->r80Result.sj64.fSign = 0;
3140	pFpuRes->r80Result.sj64.uExponent = -1 + 16383;
3141	pFpuRes->r80Result.sj64.fInteger = 1;
3142	pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
3143	\|\| (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
3144	? UINT64_C(0x317217f7d1cf79ac) : UINT64_C(0x317217f7d1cf79ab);
3145	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3146	}
3147
3148
3149	IEM_DECL_IMPL_DEF(void, iemAImpl_fldpi,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3150	{
3151	pFpuRes->r80Result.sj64.fSign = 0;
3152	pFpuRes->r80Result.sj64.uExponent = 1 + 16383;
3153	pFpuRes->r80Result.sj64.fInteger = 1;
3154	pFpuRes->r80Result.sj64.uFraction = (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
3155	\|\| (pFpuState->FCW & X86_FCW_RC_MASK) == X86_FCW_RC_UP
3156	? UINT64_C(0x490fdaa22168c235) : UINT64_C(0x490fdaa22168c234);
3157	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3158	}
3159
3160
3161	IEM_DECL_IMPL_DEF(void, iemAImpl_fldz,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes))
3162	{
3163	pFpuRes->r80Result.s.fSign = 0;
3164	pFpuRes->r80Result.s.uExponent = 0;
3165	pFpuRes->r80Result.s.uMantissa = 0;
3166	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3167	}
3168
3169	#define EMIT_FILD(a_cBits) \
3170	IEM_DECL_IMPL_DEF(void, iemAImpl_fild_r80_from_i ## a_cBits,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, \
3171	int ## a_cBits ## _t const *piVal)) \
3172	{ \
3173	int ## a_cBits ## _t iVal = *piVal; \
3174	if (iVal == 0) \
3175	{ \
3176	pFpuRes->r80Result.s.fSign = 0; \
3177	pFpuRes->r80Result.s.uExponent = 0; \
3178	pFpuRes->r80Result.s.uMantissa = 0; \
3179	} \
3180	else \
3181	{ \
3182	if (iVal > 0) \
3183	pFpuRes->r80Result.s.fSign = 0; \
3184	else \
3185	{ \
3186	pFpuRes->r80Result.s.fSign = 1; \
3187	iVal = -iVal; \
3188	} \
3189	unsigned const cBits = ASMBitLastSetU ## a_cBits((uint ## a_cBits ## _t)iVal); \
3190	pFpuRes->r80Result.s.uExponent = cBits - 1 + RTFLOAT80U_EXP_BIAS; \
3191	pFpuRes->r80Result.s.uMantissa = (uint64_t)iVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits); \
3192	} \
3193	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */ \
3194	}
3195	EMIT_FILD(16)
3196	EMIT_FILD(32)
3197	EMIT_FILD(64)
3198
3199
3200	IEM_DECL_IMPL_DEF(void, iemAImpl_fld_r80_from_d80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTPBCD80U pd80Val))
3201	{
3202	pFpuRes->FSW = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); /* see iemAImpl_fld1 */
3203	if ( pd80Val->s.abPairs[0] == 0
3204	&& pd80Val->s.abPairs[1] == 0
3205	&& pd80Val->s.abPairs[2] == 0
3206	&& pd80Val->s.abPairs[3] == 0
3207	&& pd80Val->s.abPairs[4] == 0
3208	&& pd80Val->s.abPairs[5] == 0
3209	&& pd80Val->s.abPairs[6] == 0
3210	&& pd80Val->s.abPairs[7] == 0
3211	&& pd80Val->s.abPairs[8] == 0)
3212	{
3213	pFpuRes->r80Result.s.fSign = pd80Val->s.fSign;
3214	pFpuRes->r80Result.s.uExponent = 0;
3215	pFpuRes->r80Result.s.uMantissa = 0;
3216	}
3217	else
3218	{
3219	pFpuRes->r80Result.s.fSign = pd80Val->s.fSign;
3220
3221	size_t cPairs = RT_ELEMENTS(pd80Val->s.abPairs);
3222	while (cPairs > 0 && pd80Val->s.abPairs[cPairs - 1] == 0)
3223	cPairs--;
3224
3225	uint64_t uVal = 0;
3226	uint64_t uFactor = 1;
3227	for (size_t iPair = 0; iPair < cPairs; iPair++, uFactor *= 100)
3228	uVal += RTPBCD80U_LO_DIGIT(pd80Val->s.abPairs[iPair]) * uFactor
3229	+ RTPBCD80U_HI_DIGIT(pd80Val->s.abPairs[iPair]) * uFactor * 10;
3230
3231	unsigned const cBits = ASMBitLastSetU64(uVal);
3232	pFpuRes->r80Result.s.uExponent = cBits - 1 + RTFLOAT80U_EXP_BIAS;
3233	pFpuRes->r80Result.s.uMantissa = uVal << (RTFLOAT80U_FRACTION_BITS + 1 - cBits);
3234	}
3235	}
3236
3237
3238	/*********************************************************************************************************************************
3239	* x87 FPU Stores *
3240	*********************************************************************************************************************************/
3241
3242	/**
3243	* Helper for storing a deconstructed and normal R80 value as a 64-bit one.
3244	*
3245	* This uses the rounding rules indicated by fFcw and returns updated fFsw.
3246	*
3247	* @returns Updated FPU status word value.
3248	* @param fSignIn Incoming sign indicator.
3249	* @param uMantissaIn Incoming mantissa (dot between bit 63 and 62).
3250	* @param iExponentIn Unbiased exponent.
3251	* @param fFcw The FPU control word.
3252	* @param fFsw Prepped FPU status word, i.e. exceptions and C1 clear.
3253	* @param pr64Dst Where to return the output value, if one should be
3254	* returned.
3255	*
3256	* @note Tailored as a helper for iemAImpl_fst_r80_to_r32 right now.
3257	* @note Exact same logic as iemAImpl_StoreNormalR80AsR64.
3258	*/
3259	static uint16_t iemAImpl_StoreNormalR80AsR32(bool fSignIn, uint64_t uMantissaIn, int32_t iExponentIn,
3260	uint16_t fFcw, uint16_t fFsw, PRTFLOAT32U pr32Dst)
3261	{
3262	uint64_t const fRoundingOffMask = RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS) - 1; /* 0x7ff */
3263	uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
3264	? RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS - 1) /* 0x400 */
3265	: (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
3266	? fRoundingOffMask
3267	: 0;
3268	uint64_t fRoundedOff = uMantissaIn & fRoundingOffMask;
3269
3270	/*
3271	* Deal with potential overflows/underflows first, optimizing for none.
3272	* 0 and MAX are used for special values; MAX-1 may be rounded up to MAX.
3273	*/
3274	int32_t iExponentOut = (int32_t)iExponentIn + RTFLOAT32U_EXP_BIAS;
3275	if ((uint32_t)iExponentOut - 1 < (uint32_t)(RTFLOAT32U_EXP_MAX - 3))
3276	{ /* likely? */ }
3277	/*
3278	* Underflow if the exponent zero or negative. This is attempted mapped
3279	* to a subnormal number when possible, with some additional trickery ofc.
3280	*/
3281	else if (iExponentOut <= 0)
3282	{
3283	bool const fIsTiny = iExponentOut < 0
3284	\|\| UINT64_MAX - uMantissaIn > uRoundingAdd;
3285	if (!(fFcw & X86_FCW_UM) && fIsTiny)
3286	/* Note! 754-1985 sec 7.4 has something about bias adjust of 192 here, not in 2008 & 2019. Perhaps only 8087 & 287? */
3287	return fFsw \| X86_FSW_UE \| X86_FSW_ES \| X86_FSW_B;
3288
3289	if (iExponentOut <= 0)
3290	{
3291	uMantissaIn = iExponentOut <= -63
3292	? uMantissaIn != 0
3293	: (uMantissaIn >> (-iExponentOut + 1)) \| ((uMantissaIn & (RT_BIT_64(-iExponentOut + 1) - 1)) != 0);
3294	fRoundedOff = uMantissaIn & fRoundingOffMask;
3295	if (fRoundedOff && fIsTiny)
3296	fFsw \|= X86_FSW_UE;
3297	iExponentOut = 0;
3298	}
3299	}
3300	/*
3301	* Overflow if at or above max exponent value or if we will reach max
3302	* when rounding. Will return +/-zero or +/-max value depending on
3303	* whether we're rounding or not.
3304	*/
3305	else if ( iExponentOut >= RTFLOAT32U_EXP_MAX
3306	\|\| ( iExponentOut == RTFLOAT32U_EXP_MAX - 1
3307	&& UINT64_MAX - uMantissaIn <= uRoundingAdd))
3308	{
3309	fFsw \|= X86_FSW_OE;
3310	if (!(fFcw & X86_FCW_OM))
3311	return fFsw \| X86_FSW_ES \| X86_FSW_B;
3312	fFsw \|= X86_FSW_PE;
3313	if (uRoundingAdd)
3314	fFsw \|= X86_FSW_C1;
3315	if (!(fFcw & X86_FCW_PM))
3316	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3317
3318	pr32Dst->s.fSign = fSignIn;
3319	if (uRoundingAdd)
3320	{ /* Zero */
3321	pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX;
3322	pr32Dst->s.uFraction = 0;
3323	}
3324	else
3325	{ /* Max */
3326	pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX - 1;
3327	pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS) - 1;
3328	}
3329	return fFsw;
3330	}
3331
3332	/*
3333	* Normal or subnormal number.
3334	*/
3335	/* Do rounding - just truncate in near mode when midway on an even outcome. */
3336	uint64_t uMantissaOut = uMantissaIn;
3337	if ( (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST
3338	\|\| (uMantissaIn & RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS))
3339	\|\| fRoundedOff != uRoundingAdd)
3340	{
3341	uMantissaOut = uMantissaIn + uRoundingAdd;
3342	if (uMantissaOut >= uMantissaIn)
3343	{ /* likely */ }
3344	else
3345	{
3346	uMantissaOut >>= 1; /* (We don't need to add bit 63 here (the integer bit), as it will be chopped off below.) */
3347	iExponentOut++;
3348	Assert(iExponentOut < RTFLOAT32U_EXP_MAX); /* checked above */
3349	fFsw \|= X86_FSW_C1;
3350	}
3351	}
3352	else
3353	uMantissaOut = uMantissaIn;
3354
3355	/* Truncate the mantissa and set the return value. */
3356	uMantissaOut >>= RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS;
3357
3358	pr32Dst->s.uFraction = (uint32_t)uMantissaOut; /* Note! too big for bitfield if normal. */
3359	pr32Dst->s.uExponent = iExponentOut;
3360	pr32Dst->s.fSign = fSignIn;
3361
3362	/* Set status flags realted to rounding. */
3363	if (fRoundedOff)
3364	{
3365	fFsw \|= X86_FSW_PE;
3366	if (uMantissaOut > (uMantissaIn >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS)))
3367	fFsw \|= X86_FSW_C1;
3368	if (!(fFcw & X86_FCW_PM))
3369	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3370	}
3371
3372	return fFsw;
3373	}
3374
3375
3376	/**
3377	* @note Exact same logic as iemAImpl_fst_r80_to_r64.
3378	*/
3379	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r32,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3380	PRTFLOAT32U pr32Dst, PCRTFLOAT80U pr80Src))
3381	{
3382	uint16_t const fFcw = pFpuState->FCW;
3383	uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3));
3384	if (RTFLOAT80U_IS_NORMAL(pr80Src))
3385	fFsw = iemAImpl_StoreNormalR80AsR32(pr80Src->s.fSign, pr80Src->s.uMantissa,
3386	(int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS, fFcw, fFsw, pr32Dst);
3387	else if (RTFLOAT80U_IS_ZERO(pr80Src))
3388	{
3389	pr32Dst->s.fSign = pr80Src->s.fSign;
3390	pr32Dst->s.uExponent = 0;
3391	pr32Dst->s.uFraction = 0;
3392	Assert(RTFLOAT32U_IS_ZERO(pr32Dst));
3393	}
3394	else if (RTFLOAT80U_IS_INF(pr80Src))
3395	{
3396	pr32Dst->s.fSign = pr80Src->s.fSign;
3397	pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX;
3398	pr32Dst->s.uFraction = 0;
3399	Assert(RTFLOAT32U_IS_INF(pr32Dst));
3400	}
3401	else if (RTFLOAT80U_IS_INDEFINITE(pr80Src))
3402	{
3403	/* Mapped to +/-QNaN */
3404	pr32Dst->s.fSign = pr80Src->s.fSign;
3405	pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX;
3406	pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
3407	}
3408	else if (RTFLOAT80U_IS_PSEUDO_INF(pr80Src) \|\| RTFLOAT80U_IS_UNNORMAL(pr80Src) \|\| RTFLOAT80U_IS_PSEUDO_NAN(pr80Src))
3409	{
3410	/* Pseudo-Inf / Pseudo-Nan / Unnormal -> QNaN (during load, probably) */
3411	if (fFcw & X86_FCW_IM)
3412	{
3413	pr32Dst->s.fSign = 1;
3414	pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX;
3415	pr32Dst->s.uFraction = RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
3416	fFsw \|= X86_FSW_IE;
3417	}
3418	else
3419	fFsw \|= X86_FSW_IE \| X86_FSW_ES \| X86_FSW_B;;
3420	}
3421	else if (RTFLOAT80U_IS_NAN(pr80Src))
3422	{
3423	/* IM applies to signalled NaN input only. Everything is converted to quiet NaN. */
3424	if ((fFcw & X86_FCW_IM) \|\| !RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
3425	{
3426	pr32Dst->s.fSign = pr80Src->s.fSign;
3427	pr32Dst->s.uExponent = RTFLOAT32U_EXP_MAX;
3428	pr32Dst->s.uFraction = (uint32_t)(pr80Src->sj64.uFraction >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT32U_FRACTION_BITS));
3429	pr32Dst->s.uFraction \|= RT_BIT_32(RTFLOAT32U_FRACTION_BITS - 1);
3430	if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
3431	fFsw \|= X86_FSW_IE;
3432	}
3433	else
3434	fFsw \|= X86_FSW_IE \| X86_FSW_ES \| X86_FSW_B;
3435	}
3436	else
3437	{
3438	/* Denormal values causes both an underflow and precision exception. */
3439	Assert(RTFLOAT80U_IS_DENORMAL(pr80Src) \|\| RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src));
3440	if (fFcw & X86_FCW_UM)
3441	{
3442	pr32Dst->s.fSign = pr80Src->s.fSign;
3443	pr32Dst->s.uExponent = 0;
3444	if ((fFcw & X86_FCW_RC_MASK) == (!pr80Src->s.fSign ? X86_FCW_RC_UP : X86_FCW_RC_DOWN))
3445	{
3446	pr32Dst->s.uFraction = 1;
3447	fFsw \|= X86_FSW_UE \| X86_FSW_PE \| X86_FSW_C1;
3448	if (!(fFcw & X86_FCW_PM))
3449	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3450	}
3451	else
3452	{
3453	pr32Dst->s.uFraction = 0;
3454	fFsw \|= X86_FSW_UE \| X86_FSW_PE;
3455	if (!(fFcw & X86_FCW_PM))
3456	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3457	}
3458	}
3459	else
3460	fFsw \|= X86_FSW_UE \| X86_FSW_ES \| X86_FSW_B;
3461	}
3462	*pu16FSW = fFsw;
3463	}
3464
3465
3466	/**
3467	* Helper for storing a deconstructed and normal R80 value as a 64-bit one.
3468	*
3469	* This uses the rounding rules indicated by fFcw and returns updated fFsw.
3470	*
3471	* @returns Updated FPU status word value.
3472	* @param fSignIn Incoming sign indicator.
3473	* @param uMantissaIn Incoming mantissa (dot between bit 63 and 62).
3474	* @param iExponentIn Unbiased exponent.
3475	* @param fFcw The FPU control word.
3476	* @param fFsw Prepped FPU status word, i.e. exceptions and C1 clear.
3477	* @param pr64Dst Where to return the output value, if one should be
3478	* returned.
3479	*
3480	* @note Tailored as a helper for iemAImpl_fst_r80_to_r64 right now.
3481	* @note Exact same logic as iemAImpl_StoreNormalR80AsR32.
3482	*/
3483	static uint16_t iemAImpl_StoreNormalR80AsR64(bool fSignIn, uint64_t uMantissaIn, int32_t iExponentIn,
3484	uint16_t fFcw, uint16_t fFsw, PRTFLOAT64U pr64Dst)
3485	{
3486	uint64_t const fRoundingOffMask = RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS) - 1; /* 0x7ff */
3487	uint32_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
3488	? RT_BIT_64(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS - 1) /* 0x400 */
3489	: (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
3490	? fRoundingOffMask
3491	: 0;
3492	uint32_t fRoundedOff = uMantissaIn & fRoundingOffMask;
3493
3494	/*
3495	* Deal with potential overflows/underflows first, optimizing for none.
3496	* 0 and MAX are used for special values; MAX-1 may be rounded up to MAX.
3497	*/
3498	int32_t iExponentOut = (int32_t)iExponentIn + RTFLOAT64U_EXP_BIAS;
3499	if ((uint32_t)iExponentOut - 1 < (uint32_t)(RTFLOAT64U_EXP_MAX - 3))
3500	{ /* likely? */ }
3501	/*
3502	* Underflow if the exponent zero or negative. This is attempted mapped
3503	* to a subnormal number when possible, with some additional trickery ofc.
3504	*/
3505	else if (iExponentOut <= 0)
3506	{
3507	bool const fIsTiny = iExponentOut < 0
3508	\|\| UINT64_MAX - uMantissaIn > uRoundingAdd;
3509	if (!(fFcw & X86_FCW_UM) && fIsTiny)
3510	/* Note! 754-1985 sec 7.4 has something about bias adjust of 1536 here, not in 2008 & 2019. Perhaps only 8087 & 287? */
3511	return fFsw \| X86_FSW_UE \| X86_FSW_ES \| X86_FSW_B;
3512
3513	if (iExponentOut <= 0)
3514	{
3515	uMantissaIn = iExponentOut <= -63
3516	? uMantissaIn != 0
3517	: (uMantissaIn >> (-iExponentOut + 1)) \| ((uMantissaIn & (RT_BIT_64(-iExponentOut + 1) - 1)) != 0);
3518	fRoundedOff = uMantissaIn & fRoundingOffMask;
3519	if (fRoundedOff && fIsTiny)
3520	fFsw \|= X86_FSW_UE;
3521	iExponentOut = 0;
3522	}
3523	}
3524	/*
3525	* Overflow if at or above max exponent value or if we will reach max
3526	* when rounding. Will return +/-zero or +/-max value depending on
3527	* whether we're rounding or not.
3528	*/
3529	else if ( iExponentOut >= RTFLOAT64U_EXP_MAX
3530	\|\| ( iExponentOut == RTFLOAT64U_EXP_MAX - 1
3531	&& UINT64_MAX - uMantissaIn <= uRoundingAdd))
3532	{
3533	fFsw \|= X86_FSW_OE;
3534	if (!(fFcw & X86_FCW_OM))
3535	return fFsw \| X86_FSW_ES \| X86_FSW_B;
3536	fFsw \|= X86_FSW_PE;
3537	if (uRoundingAdd)
3538	fFsw \|= X86_FSW_C1;
3539	if (!(fFcw & X86_FCW_PM))
3540	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3541
3542	pr64Dst->s64.fSign = fSignIn;
3543	if (uRoundingAdd)
3544	{ /* Zero */
3545	pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX;
3546	pr64Dst->s64.uFraction = 0;
3547	}
3548	else
3549	{ /* Max */
3550	pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX - 1;
3551	pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS) - 1;
3552	}
3553	return fFsw;
3554	}
3555
3556	/*
3557	* Normal or subnormal number.
3558	*/
3559	/* Do rounding - just truncate in near mode when midway on an even outcome. */
3560	uint64_t uMantissaOut = uMantissaIn;
3561	if ( (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_NEAREST
3562	\|\| (uMantissaIn & RT_BIT_32(RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS))
3563	\|\| fRoundedOff != uRoundingAdd)
3564	{
3565	uMantissaOut = uMantissaIn + uRoundingAdd;
3566	if (uMantissaOut >= uMantissaIn)
3567	{ /* likely */ }
3568	else
3569	{
3570	uMantissaOut >>= 1; /* (We don't need to add bit 63 here (the integer bit), as it will be chopped off below.) */
3571	iExponentOut++;
3572	Assert(iExponentOut < RTFLOAT64U_EXP_MAX); /* checked above */
3573	fFsw \|= X86_FSW_C1;
3574	}
3575	}
3576	else
3577	uMantissaOut = uMantissaIn;
3578
3579	/* Truncate the mantissa and set the return value. */
3580	uMantissaOut >>= RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS;
3581
3582	pr64Dst->s64.uFraction = uMantissaOut; /* Note! too big for bitfield if normal. */
3583	pr64Dst->s64.uExponent = iExponentOut;
3584	pr64Dst->s64.fSign = fSignIn;
3585
3586	/* Set status flags realted to rounding. */
3587	if (fRoundedOff)
3588	{
3589	fFsw \|= X86_FSW_PE;
3590	if (uMantissaOut > (uMantissaIn >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS)))
3591	fFsw \|= X86_FSW_C1;
3592	if (!(fFcw & X86_FCW_PM))
3593	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3594	}
3595
3596	return fFsw;
3597	}
3598
3599
3600	/**
3601	* @note Exact same logic as iemAImpl_fst_r80_to_r32.
3602	*/
3603	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r64,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3604	PRTFLOAT64U pr64Dst, PCRTFLOAT80U pr80Src))
3605	{
3606	uint16_t const fFcw = pFpuState->FCW;
3607	uint16_t fFsw = (7 << X86_FSW_TOP_SHIFT) \| (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3));
3608	if (RTFLOAT80U_IS_NORMAL(pr80Src))
3609	fFsw = iemAImpl_StoreNormalR80AsR64(pr80Src->s.fSign, pr80Src->s.uMantissa,
3610	(int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS, fFcw, fFsw, pr64Dst);
3611	else if (RTFLOAT80U_IS_ZERO(pr80Src))
3612	{
3613	pr64Dst->s64.fSign = pr80Src->s.fSign;
3614	pr64Dst->s64.uExponent = 0;
3615	pr64Dst->s64.uFraction = 0;
3616	Assert(RTFLOAT64U_IS_ZERO(pr64Dst));
3617	}
3618	else if (RTFLOAT80U_IS_INF(pr80Src))
3619	{
3620	pr64Dst->s64.fSign = pr80Src->s.fSign;
3621	pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX;
3622	pr64Dst->s64.uFraction = 0;
3623	Assert(RTFLOAT64U_IS_INF(pr64Dst));
3624	}
3625	else if (RTFLOAT80U_IS_INDEFINITE(pr80Src))
3626	{
3627	/* Mapped to +/-QNaN */
3628	pr64Dst->s64.fSign = pr80Src->s.fSign;
3629	pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX;
3630	pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
3631	}
3632	else if (RTFLOAT80U_IS_PSEUDO_INF(pr80Src) \|\| RTFLOAT80U_IS_UNNORMAL(pr80Src) \|\| RTFLOAT80U_IS_PSEUDO_NAN(pr80Src))
3633	{
3634	/* Pseudo-Inf / Pseudo-Nan / Unnormal -> QNaN (during load, probably) */
3635	if (fFcw & X86_FCW_IM)
3636	{
3637	pr64Dst->s64.fSign = 1;
3638	pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX;
3639	pr64Dst->s64.uFraction = RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
3640	fFsw \|= X86_FSW_IE;
3641	}
3642	else
3643	fFsw \|= X86_FSW_IE \| X86_FSW_ES \| X86_FSW_B;;
3644	}
3645	else if (RTFLOAT80U_IS_NAN(pr80Src))
3646	{
3647	/* IM applies to signalled NaN input only. Everything is converted to quiet NaN. */
3648	if ((fFcw & X86_FCW_IM) \|\| !RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
3649	{
3650	pr64Dst->s64.fSign = pr80Src->s.fSign;
3651	pr64Dst->s64.uExponent = RTFLOAT64U_EXP_MAX;
3652	pr64Dst->s64.uFraction = pr80Src->sj64.uFraction >> (RTFLOAT80U_FRACTION_BITS - RTFLOAT64U_FRACTION_BITS);
3653	pr64Dst->s64.uFraction \|= RT_BIT_64(RTFLOAT64U_FRACTION_BITS - 1);
3654	if (RTFLOAT80U_IS_SIGNALLING_NAN(pr80Src))
3655	fFsw \|= X86_FSW_IE;
3656	}
3657	else
3658	fFsw \|= X86_FSW_IE \| X86_FSW_ES \| X86_FSW_B;
3659	}
3660	else
3661	{
3662	/* Denormal values causes both an underflow and precision exception. */
3663	Assert(RTFLOAT80U_IS_DENORMAL(pr80Src) \|\| RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src));
3664	if (fFcw & X86_FCW_UM)
3665	{
3666	pr64Dst->s64.fSign = pr80Src->s.fSign;
3667	pr64Dst->s64.uExponent = 0;
3668	if ((fFcw & X86_FCW_RC_MASK) == (!pr80Src->s.fSign ? X86_FCW_RC_UP : X86_FCW_RC_DOWN))
3669	{
3670	pr64Dst->s64.uFraction = 1;
3671	fFsw \|= X86_FSW_UE \| X86_FSW_PE \| X86_FSW_C1;
3672	if (!(fFcw & X86_FCW_PM))
3673	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3674	}
3675	else
3676	{
3677	pr64Dst->s64.uFraction = 0;
3678	fFsw \|= X86_FSW_UE \| X86_FSW_PE;
3679	if (!(fFcw & X86_FCW_PM))
3680	fFsw \|= X86_FSW_ES \| X86_FSW_B;
3681	}
3682	}
3683	else
3684	fFsw \|= X86_FSW_UE \| X86_FSW_ES \| X86_FSW_B;
3685	}
3686	*pu16FSW = fFsw;
3687	}
3688
3689
3690	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
3691	PRTFLOAT80U pr80Dst, PCRTFLOAT80U pr80Src))
3692	{
3693	/*
3694	* FPU status word:
3695	* - TOP is irrelevant, but we must match x86 assembly version (0).
3696	* - C1 is always cleared as we don't have any stack overflows.
3697	* - C0, C2, and C3 are undefined and Intel 10980XE does not touch them.
3698	*/
3699	pu16FSW = pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3); / see iemAImpl_fld1 */
3700	pr80Dst = pr80Src;
3701	}
3702
3703
3704	/*
3705	*
3706	* Mantissa:
3707	* 63 56 48 40 32 24 16 8 0
3708	* v v v v v v v v v
3709	* 1[.]111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000 1111 0000
3710	* \ \ \ \ \ \ \ \ \ \ \ \ \ \ \ \
3711	* Exp: 0 4 8 12 16 20 24 28 32 36 40 44 48 52 56 60
3712	*
3713	* int64_t has the same width, only bit 63 is the sign bit. So, the max we can map over
3714	* are bits 1 thru 63, dropping off bit 0, with an exponent of 62. The number of bits we
3715	* drop off from the mantissa increases with decreasing exponent, till an exponent of 0
3716	* where we'll drop off all but bit 63.
3717	*/
3718	#define EMIT_FIST(a_cBits, a_iType, a_iTypeMin, a_iTypeIndefinite) \
3719	IEM_DECL_IMPL_DEF(void, iemAImpl_fist_r80_to_i ## a_cBits,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, \
3720	a_iType *piDst, PCRTFLOAT80U pr80Val)) \
3721	{ \
3722	uint16_t const fFcw = pFpuState->FCW; \
3723	uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); \
3724	bool const fSignIn = pr80Val->s.fSign; \
3725	\
3726	/* \
3727	* Deal with normal numbers first. \
3728	*/ \
3729	if (RTFLOAT80U_IS_NORMAL(pr80Val)) \
3730	{ \
3731	uint64_t uMantissa = pr80Val->s.uMantissa; \
3732	int32_t iExponent = (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS; \
3733	\
3734	if ((uint32_t)iExponent <= a_cBits - 2) \
3735	{ \
3736	unsigned const cShiftOff = 63 - iExponent; \
3737	uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; \
3738	uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST \
3739	? RT_BIT_64(cShiftOff - 1) \
3740	: (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP) \
3741	? fRoundingOffMask \
3742	: 0; \
3743	uint64_t fRoundedOff = uMantissa & fRoundingOffMask; \
3744	\
3745	uMantissa >>= cShiftOff; \
3746	uint64_t const uRounding = (fRoundedOff + uRoundingAdd) >> cShiftOff; \
3747	uMantissa += uRounding; \
3748	if (!(uMantissa & RT_BIT_64(a_cBits - 1))) \
3749	{ \
3750	if (fRoundedOff) \
3751	{ \
3752	if ((uMantissa & 1) && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST && fRoundedOff == uRoundingAdd) \
3753	uMantissa &= ~(uint64_t)1; /* round to even number if equal distance between up/down. */ \
3754	else if (uRounding) \
3755	fFsw \|= X86_FSW_C1; \
3756	fFsw \|= X86_FSW_PE; \
3757	if (!(fFcw & X86_FCW_PM)) \
3758	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3759	} \
3760	\
3761	if (!fSignIn) \
3762	*piDst = (a_iType)uMantissa; \
3763	else \
3764	*piDst = -(a_iType)uMantissa; \
3765	} \
3766	else \
3767	{ \
3768	/* overflowed after rounding. */ \
3769	AssertMsg(iExponent == a_cBits - 2 && uMantissa == RT_BIT_64(a_cBits - 1), \
3770	("e=%d m=%#RX64 (org %#RX64) s=%d; shift=%d ro=%#RX64 rm=%#RX64 ra=%#RX64\n", iExponent, uMantissa, \
3771	pr80Val->s.uMantissa, fSignIn, cShiftOff, fRoundedOff, fRoundingOffMask, uRoundingAdd)); \
3772	\
3773	/* Special case for the integer minimum value. */ \
3774	if (fSignIn) \
3775	{ \
3776	*piDst = a_iTypeMin; \
3777	fFsw \|= X86_FSW_PE \| X86_FSW_C1; \
3778	if (!(fFcw & X86_FCW_PM)) \
3779	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3780	} \
3781	else \
3782	{ \
3783	fFsw \|= X86_FSW_IE; \
3784	if (fFcw & X86_FCW_IM) \
3785	*piDst = a_iTypeMin; \
3786	else \
3787	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT); \
3788	} \
3789	} \
3790	} \
3791	/* \
3792	* Tiny sub-zero numbers. \
3793	*/ \
3794	else if (iExponent < 0) \
3795	{ \
3796	if (!fSignIn) \
3797	{ \
3798	if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP \
3799	\|\| (iExponent == -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) \
3800	{ \
3801	*piDst = 1; \
3802	fFsw \|= X86_FSW_C1; \
3803	} \
3804	else \
3805	*piDst = 0; \
3806	} \
3807	else \
3808	{ \
3809	if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP \
3810	\|\| (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_ZERO \
3811	\|\| (iExponent < -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST)) \
3812	*piDst = 0; \
3813	else \
3814	{ \
3815	*piDst = -1; \
3816	fFsw \|= X86_FSW_C1; \
3817	} \
3818	} \
3819	fFsw \|= X86_FSW_PE; \
3820	if (!(fFcw & X86_FCW_PM)) \
3821	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3822	} \
3823	/* \
3824	* Special MIN case. \
3825	*/ \
3826	else if ( fSignIn && iExponent == a_cBits - 1 \
3827	&& ( a_cBits < 64 && (fFcw & X86_FCW_RC_MASK) != X86_FCW_RC_DOWN \
3828	? uMantissa < (RT_BIT_64(63) \| RT_BIT_64(65 - a_cBits)) \
3829	: uMantissa == RT_BIT_64(63))) \
3830	{ \
3831	*piDst = a_iTypeMin; \
3832	if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \
3833	{ \
3834	fFsw \|= X86_FSW_PE; \
3835	if (!(fFcw & X86_FCW_PM)) \
3836	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3837	} \
3838	} \
3839	/* \
3840	* Too large/small number outside the target integer range. \
3841	*/ \
3842	else \
3843	{ \
3844	fFsw \|= X86_FSW_IE; \
3845	if (fFcw & X86_FCW_IM) \
3846	*piDst = a_iTypeIndefinite; \
3847	else \
3848	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT); \
3849	} \
3850	} \
3851	/* \
3852	* Map both +0 and -0 to integer zero (signless/+). \
3853	*/ \
3854	else if (RTFLOAT80U_IS_ZERO(pr80Val)) \
3855	*piDst = 0; \
3856	/* \
3857	* Denormals are just really tiny sub-zero numbers that are either rounded \
3858	* to zero, 1 or -1 depending on sign and rounding control. \
3859	*/ \
3860	else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) \|\| RTFLOAT80U_IS_DENORMAL(pr80Val)) \
3861	{ \
3862	if ((fFcw & X86_FCW_RC_MASK) != (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)) \
3863	*piDst = 0; \
3864	else \
3865	{ \
3866	*piDst = fSignIn ? -1 : 1; \
3867	fFsw \|= X86_FSW_C1; \
3868	} \
3869	fFsw \|= X86_FSW_PE; \
3870	if (!(fFcw & X86_FCW_PM)) \
3871	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3872	} \
3873	/* \
3874	* All other special values are considered invalid arguments and result \
3875	* in an IE exception and indefinite value if masked. \
3876	*/ \
3877	else \
3878	{ \
3879	fFsw \|= X86_FSW_IE; \
3880	if (fFcw & X86_FCW_IM) \
3881	*piDst = a_iTypeIndefinite; \
3882	else \
3883	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT); \
3884	} \
3885	*pu16FSW = fFsw; \
3886	}
3887	EMIT_FIST(64, int64_t, INT64_MIN, X86_FPU_INT64_INDEFINITE)
3888	EMIT_FIST(32, int32_t, INT32_MIN, X86_FPU_INT32_INDEFINITE)
3889	EMIT_FIST(16, int16_t, INT16_MIN, X86_FPU_INT16_INDEFINITE)
3890
3891
3892	/*
3893	* The FISTT instruction was added with SSE3 and are a lot simpler than FIST.
3894	*
3895	* The 16-bit version is a bit peculiar, though, as it seems to be raising IE
3896	* as if it was the 32-bit version (i.e. starting with exp 31 instead of 15),
3897	* thus the @a a_cBitsIn.
3898	*/
3899	#define EMIT_FISTT(a_cBits, a_cBitsIn, a_iType, a_iTypeMin, a_iTypeMax, a_iTypeIndefinite) \
3900	IEM_DECL_IMPL_DEF(void, iemAImpl_fistt_r80_to_i ## a_cBits,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW, \
3901	a_iType *piDst, PCRTFLOAT80U pr80Val)) \
3902	{ \
3903	uint16_t const fFcw = pFpuState->FCW; \
3904	uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3)); \
3905	bool const fSignIn = pr80Val->s.fSign; \
3906	\
3907	/* \
3908	* Deal with normal numbers first. \
3909	*/ \
3910	if (RTFLOAT80U_IS_NORMAL(pr80Val)) \
3911	{ \
3912	uint64_t uMantissa = pr80Val->s.uMantissa; \
3913	int32_t iExponent = (int32_t)pr80Val->s.uExponent - RTFLOAT80U_EXP_BIAS; \
3914	\
3915	if ((uint32_t)iExponent <= a_cBitsIn - 2) \
3916	{ \
3917	unsigned const cShiftOff = 63 - iExponent; \
3918	uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1; \
3919	uint64_t const fRoundedOff = uMantissa & fRoundingOffMask; \
3920	uMantissa >>= cShiftOff; \
3921	/Assert(!(uMantissa & RT_BIT_64(a_cBits - 1)));/ \
3922	if (!fSignIn) \
3923	*piDst = (a_iType)uMantissa; \
3924	else \
3925	*piDst = -(a_iType)uMantissa; \
3926	\
3927	if (fRoundedOff) \
3928	{ \
3929	fFsw \|= X86_FSW_PE; \
3930	if (!(fFcw & X86_FCW_PM)) \
3931	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3932	} \
3933	} \
3934	/* \
3935	* Tiny sub-zero numbers. \
3936	*/ \
3937	else if (iExponent < 0) \
3938	{ \
3939	*piDst = 0; \
3940	fFsw \|= X86_FSW_PE; \
3941	if (!(fFcw & X86_FCW_PM)) \
3942	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3943	} \
3944	/* \
3945	* Special MIN case. \
3946	*/ \
3947	else if ( fSignIn && iExponent == a_cBits - 1 \
3948	&& (a_cBits < 64 \
3949	? uMantissa < (RT_BIT_64(63) \| RT_BIT_64(65 - a_cBits)) \
3950	: uMantissa == RT_BIT_64(63)) ) \
3951	{ \
3952	*piDst = a_iTypeMin; \
3953	if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \
3954	{ \
3955	fFsw \|= X86_FSW_PE; \
3956	if (!(fFcw & X86_FCW_PM)) \
3957	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3958	} \
3959	} \
3960	/* \
3961	* Figure this weirdness. \
3962	*/ \
3963	else if (a_cBits == 16 && fSignIn && iExponent == 31 && uMantissa < UINT64_C(0x8000100000000000) ) \
3964	{ \
3965	*piDst = 0; \
3966	if (uMantissa & (RT_BIT_64(64 - a_cBits + 1) - 1)) \
3967	{ \
3968	fFsw \|= X86_FSW_PE; \
3969	if (!(fFcw & X86_FCW_PM)) \
3970	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3971	} \
3972	} \
3973	/* \
3974	* Too large/small number outside the target integer range. \
3975	*/ \
3976	else \
3977	{ \
3978	fFsw \|= X86_FSW_IE; \
3979	if (fFcw & X86_FCW_IM) \
3980	*piDst = a_iTypeIndefinite; \
3981	else \
3982	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT); \
3983	} \
3984	} \
3985	/* \
3986	* Map both +0 and -0 to integer zero (signless/+). \
3987	*/ \
3988	else if (RTFLOAT80U_IS_ZERO(pr80Val)) \
3989	*piDst = 0; \
3990	/* \
3991	* Denormals are just really tiny sub-zero numbers that are trucated to zero. \
3992	*/ \
3993	else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Val) \|\| RTFLOAT80U_IS_DENORMAL(pr80Val)) \
3994	{ \
3995	*piDst = 0; \
3996	fFsw \|= X86_FSW_PE; \
3997	if (!(fFcw & X86_FCW_PM)) \
3998	fFsw \|= X86_FSW_ES \| X86_FSW_B; \
3999	} \
4000	/* \
4001	* All other special values are considered invalid arguments and result \
4002	* in an IE exception and indefinite value if masked. \
4003	*/ \
4004	else \
4005	{ \
4006	fFsw \|= X86_FSW_IE; \
4007	if (fFcw & X86_FCW_IM) \
4008	*piDst = a_iTypeIndefinite; \
4009	else \
4010	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT); \
4011	} \
4012	*pu16FSW = fFsw; \
4013	}
4014	EMIT_FISTT(64, 64, int64_t, INT64_MIN, INT64_MAX, X86_FPU_INT64_INDEFINITE)
4015	EMIT_FISTT(32, 32, int32_t, INT32_MIN, INT32_MAX, X86_FPU_INT32_INDEFINITE)
4016	EMIT_FISTT(16, 32, int16_t, INT16_MIN, INT16_MAX, 0 /* X86_FPU_INT16_INDEFINITE - weird weird weird! */)
4017
4018
4019	IEM_DECL_IMPL_DEF(void, iemAImpl_fst_r80_to_d80,(PCX86FXSTATE pFpuState, uint16_t *pu16FSW,
4020	PRTPBCD80U pd80Dst, PCRTFLOAT80U pr80Src))
4021	{
4022	/static RTPBCD80U const s_ad80MaxMin[2] = { RTPBCD80U_INIT_MAX(), RTPBCD80U_INIT_MIN() };/
4023	static RTPBCD80U const s_ad80Zeros[2] = { RTPBCD80U_INIT_ZERO(0), RTPBCD80U_INIT_ZERO(1) };
4024	static RTPBCD80U const s_ad80One[2] = { RTPBCD80U_INIT_C(0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1),
4025	RTPBCD80U_INIT_C(1, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,1) };
4026	static RTPBCD80U const s_d80Indefinite = RTPBCD80U_INIT_INDEFINITE();
4027
4028	uint16_t const fFcw = pFpuState->FCW;
4029	uint16_t fFsw = (pFpuState->FSW & (X86_FSW_C0 \| X86_FSW_C2 \| X86_FSW_C3));
4030	bool const fSignIn = pr80Src->s.fSign;
4031
4032	/*
4033	* Deal with normal numbers first.
4034	*/
4035	if (RTFLOAT80U_IS_NORMAL(pr80Src))
4036	{
4037	uint64_t uMantissa = pr80Src->s.uMantissa;
4038	int32_t iExponent = (int32_t)pr80Src->s.uExponent - RTFLOAT80U_EXP_BIAS;
4039	if ( (uint32_t)iExponent <= 58
4040	\|\| ((uint32_t)iExponent == 59 && uMantissa <= UINT64_C(0xde0b6b3a763fffff)) )
4041	{
4042	unsigned const cShiftOff = 63 - iExponent;
4043	uint64_t const fRoundingOffMask = RT_BIT_64(cShiftOff) - 1;
4044	uint64_t const uRoundingAdd = (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST
4045	? RT_BIT_64(cShiftOff - 1)
4046	: (fFcw & X86_FCW_RC_MASK) == (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP)
4047	? fRoundingOffMask
4048	: 0;
4049	uint64_t fRoundedOff = uMantissa & fRoundingOffMask;
4050
4051	uMantissa >>= cShiftOff;
4052	uint64_t const uRounding = (fRoundedOff + uRoundingAdd) >> cShiftOff;
4053	uMantissa += uRounding;
4054	if (uMantissa <= (uint64_t)RTPBCD80U_MAX)
4055	{
4056	if (fRoundedOff)
4057	{
4058	if ((uMantissa & 1) && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST && fRoundedOff == uRoundingAdd)
4059	uMantissa &= ~(uint64_t)1; /* round to even number if equal distance between up/down. */
4060	else if (uRounding)
4061	fFsw \|= X86_FSW_C1;
4062	fFsw \|= X86_FSW_PE;
4063	if (!(fFcw & X86_FCW_PM))
4064	fFsw \|= X86_FSW_ES \| X86_FSW_B;
4065	}
4066
4067	pd80Dst->s.fSign = fSignIn;
4068	pd80Dst->s.uPad = 0;
4069	for (size_t iPair = 0; iPair < RT_ELEMENTS(pd80Dst->s.abPairs); iPair++)
4070	{
4071	unsigned const uDigits = uMantissa % 100;
4072	uMantissa /= 100;
4073	uint8_t const bLo = uDigits % 10;
4074	uint8_t const bHi = uDigits / 10;
4075	pd80Dst->s.abPairs[iPair] = RTPBCD80U_MAKE_PAIR(bHi, bLo);
4076	}
4077	}
4078	else
4079	{
4080	/* overflowed after rounding. */
4081	fFsw \|= X86_FSW_IE;
4082	if (fFcw & X86_FCW_IM)
4083	*pd80Dst = s_d80Indefinite;
4084	else
4085	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT);
4086	}
4087	}
4088	/*
4089	* Tiny sub-zero numbers.
4090	*/
4091	else if (iExponent < 0)
4092	{
4093	if (!fSignIn)
4094	{
4095	if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP
4096	\|\| (iExponent == -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST))
4097	{
4098	*pd80Dst = s_ad80One[fSignIn];
4099	fFsw \|= X86_FSW_C1;
4100	}
4101	else
4102	*pd80Dst = s_ad80Zeros[fSignIn];
4103	}
4104	else
4105	{
4106	if ( (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_UP
4107	\|\| (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_ZERO
4108	\|\| (iExponent < -1 && (fFcw & X86_FCW_RC_MASK) == X86_FCW_RC_NEAREST))
4109	*pd80Dst = s_ad80Zeros[fSignIn];
4110	else
4111	{
4112	*pd80Dst = s_ad80One[fSignIn];
4113	fFsw \|= X86_FSW_C1;
4114	}
4115	}
4116	fFsw \|= X86_FSW_PE;
4117	if (!(fFcw & X86_FCW_PM))
4118	fFsw \|= X86_FSW_ES \| X86_FSW_B;
4119	}
4120	/*
4121	* Too large/small number outside the target integer range.
4122	*/
4123	else
4124	{
4125	fFsw \|= X86_FSW_IE;
4126	if (fFcw & X86_FCW_IM)
4127	*pd80Dst = s_d80Indefinite;
4128	else
4129	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT);
4130	}
4131	}
4132	/*
4133	* Map both +0 and -0 to integer zero (signless/+).
4134	*/
4135	else if (RTFLOAT80U_IS_ZERO(pr80Src))
4136	*pd80Dst = s_ad80Zeros[fSignIn];
4137	/*
4138	* Denormals are just really tiny sub-zero numbers that are either rounded
4139	* to zero, 1 or -1 depending on sign and rounding control.
4140	*/
4141	else if (RTFLOAT80U_IS_PSEUDO_DENORMAL(pr80Src) \|\| RTFLOAT80U_IS_DENORMAL(pr80Src))
4142	{
4143	if ((fFcw & X86_FCW_RC_MASK) != (fSignIn ? X86_FCW_RC_DOWN : X86_FCW_RC_UP))
4144	*pd80Dst = s_ad80Zeros[fSignIn];
4145	else
4146	{
4147	*pd80Dst = s_ad80One[fSignIn];
4148	fFsw \|= X86_FSW_C1;
4149	}
4150	fFsw \|= X86_FSW_PE;
4151	if (!(fFcw & X86_FCW_PM))
4152	fFsw \|= X86_FSW_ES \| X86_FSW_B;
4153	}
4154	/*
4155	* All other special values are considered invalid arguments and result
4156	* in an IE exception and indefinite value if masked.
4157	*/
4158	else
4159	{
4160	fFsw \|= X86_FSW_IE;
4161	if (fFcw & X86_FCW_IM)
4162	*pd80Dst = s_d80Indefinite;
4163	else
4164	fFsw \|= X86_FSW_ES \| X86_FSW_B \| (7 << X86_FSW_TOP_SHIFT);
4165	}
4166	*pu16FSW = fFsw;
4167	}
4168
4169
4170	/*********************************************************************************************************************************
4171	* x86 FPU Division Operations *
4172	*********************************************************************************************************************************/
4173
4174	IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4175	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4176	{
4177	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
4178	AssertReleaseFailed();
4179	}
4180
4181
4182	IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4183	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4184	{
4185	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
4186	AssertReleaseFailed();
4187	}
4188
4189
4190	IEM_DECL_IMPL_DEF(void, iemAImpl_fdiv_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4191	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4192	{
4193	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4194	AssertReleaseFailed();
4195	}
4196
4197
4198	IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4199	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4200	{
4201	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
4202	AssertReleaseFailed();
4203	}
4204
4205
4206	IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4207	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4208	{
4209	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
4210	AssertReleaseFailed();
4211	}
4212
4213
4214	IEM_DECL_IMPL_DEF(void, iemAImpl_fdivr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4215	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4216	{
4217	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4218	AssertReleaseFailed();
4219	}
4220
4221
4222	IEM_DECL_IMPL_DEF(void, iemAImpl_fidiv_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4223	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4224	{
4225	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
4226	AssertReleaseFailed();
4227	}
4228
4229
4230	IEM_DECL_IMPL_DEF(void, iemAImpl_fidiv_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4231	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4232	{
4233	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
4234	AssertReleaseFailed();
4235	}
4236
4237
4238	IEM_DECL_IMPL_DEF(void, iemAImpl_fidivr_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4239	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4240	{
4241	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
4242	AssertReleaseFailed();
4243	}
4244
4245
4246	IEM_DECL_IMPL_DEF(void, iemAImpl_fidivr_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4247	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4248	{
4249	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
4250	AssertReleaseFailed();
4251	}
4252
4253
4254	IEM_DECL_IMPL_DEF(void, iemAImpl_fprem_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4255	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4256	{
4257	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4258	AssertReleaseFailed();
4259	}
4260
4261
4262	IEM_DECL_IMPL_DEF(void, iemAImpl_fprem1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4263	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4264	{
4265	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4266	AssertReleaseFailed();
4267	}
4268
4269
4270	/*********************************************************************************************************************************
4271	* x87 FPU Multiplication Operations *
4272	*********************************************************************************************************************************/
4273
4274	IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4275	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4276	{
4277	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
4278	AssertReleaseFailed();
4279	}
4280
4281
4282	IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4283	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4284	{
4285	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
4286	AssertReleaseFailed();
4287	}
4288
4289
4290	IEM_DECL_IMPL_DEF(void, iemAImpl_fmul_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4291	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4292	{
4293	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4294	AssertReleaseFailed();
4295	}
4296
4297
4298	IEM_DECL_IMPL_DEF(void, iemAImpl_fimul_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4299	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4300	{
4301	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
4302	AssertReleaseFailed();
4303	}
4304
4305
4306	IEM_DECL_IMPL_DEF(void, iemAImpl_fimul_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4307	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4308	{
4309	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
4310	AssertReleaseFailed();
4311	}
4312
4313
4314	/*********************************************************************************************************************************
4315	* x87 FPU Addition and Subtraction *
4316	*********************************************************************************************************************************/
4317
4318	IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4319	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4320	{
4321	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
4322	AssertReleaseFailed();
4323	}
4324
4325
4326	IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4327	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4328	{
4329	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
4330	AssertReleaseFailed();
4331	}
4332
4333
4334	IEM_DECL_IMPL_DEF(void, iemAImpl_fadd_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4335	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4336	{
4337	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4338	AssertReleaseFailed();
4339	}
4340
4341
4342	IEM_DECL_IMPL_DEF(void, iemAImpl_fiadd_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4343	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4344	{
4345	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
4346	AssertReleaseFailed();
4347	}
4348
4349
4350	IEM_DECL_IMPL_DEF(void, iemAImpl_fiadd_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4351	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4352	{
4353	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
4354	AssertReleaseFailed();
4355	}
4356
4357
4358	IEM_DECL_IMPL_DEF(void, iemAImpl_fisub_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4359	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4360	{
4361	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
4362	AssertReleaseFailed();
4363	}
4364
4365
4366	IEM_DECL_IMPL_DEF(void, iemAImpl_fisub_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4367	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4368	{
4369	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
4370	AssertReleaseFailed();
4371	}
4372
4373
4374	IEM_DECL_IMPL_DEF(void, iemAImpl_fisubr_r80_by_i16,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4375	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4376	{
4377	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi16Val2);
4378	AssertReleaseFailed();
4379	}
4380
4381
4382	IEM_DECL_IMPL_DEF(void, iemAImpl_fisubr_r80_by_i32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4383	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4384	{
4385	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pi32Val2);
4386	AssertReleaseFailed();
4387	}
4388
4389
4390	IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4391	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4392	{
4393	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
4394	AssertReleaseFailed();
4395	}
4396
4397
4398	IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4399	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4400	{
4401	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
4402	AssertReleaseFailed();
4403	}
4404
4405
4406	IEM_DECL_IMPL_DEF(void, iemAImpl_fsub_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4407	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4408	{
4409	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4410	AssertReleaseFailed();
4411	}
4412
4413
4414	IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r32,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4415	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4416	{
4417	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr32Val2);
4418	AssertReleaseFailed();
4419	}
4420
4421
4422	IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r64,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4423	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4424	{
4425	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr64Val2);
4426	AssertReleaseFailed();
4427	}
4428
4429
4430	IEM_DECL_IMPL_DEF(void, iemAImpl_fsubr_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4431	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4432	{
4433	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4434	AssertReleaseFailed();
4435	}
4436
4437
4438	/*********************************************************************************************************************************
4439	* x87 FPU Trigometric Operations *
4440	*********************************************************************************************************************************/
4441
4442	IEM_DECL_IMPL_DEF(void, iemAImpl_fpatan_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4443	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4444	{
4445	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4446	AssertReleaseFailed();
4447	}
4448
4449
4450	IEM_DECL_IMPL_DEF(void, iemAImpl_fptan_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
4451	{
4452	RT_NOREF(pFpuState, pFpuResTwo, pr80Val);
4453	AssertReleaseFailed();
4454	}
4455
4456
4457	IEM_DECL_IMPL_DEF(void, iemAImpl_fsin_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4458	{
4459	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4460	AssertReleaseFailed();
4461	}
4462
4463
4464	IEM_DECL_IMPL_DEF(void, iemAImpl_fsincos_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
4465	{
4466	RT_NOREF(pFpuState, pFpuResTwo, pr80Val);
4467	AssertReleaseFailed();
4468	}
4469
4470
4471	IEM_DECL_IMPL_DEF(void, iemAImpl_fcos_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4472	{
4473	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4474	AssertReleaseFailed();
4475	}
4476
4477
4478	/*********************************************************************************************************************************
4479	* x87 FPU Compare and Testing Operations *
4480	*********************************************************************************************************************************/
4481
4482	IEM_DECL_IMPL_DEF(void, iemAImpl_ftst_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val))
4483	{
4484	RT_NOREF(pFpuState, pu16Fsw, pr80Val);
4485	AssertReleaseFailed();
4486	}
4487
4488
4489	IEM_DECL_IMPL_DEF(void, iemAImpl_fxam_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw, PCRTFLOAT80U pr80Val))
4490	{
4491	RT_NOREF(pFpuState, pu16Fsw, pr80Val);
4492	AssertReleaseFailed();
4493	}
4494
4495
4496	IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r32,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
4497	PCRTFLOAT80U pr80Val1, PCRTFLOAT32U pr32Val2))
4498	{
4499	RT_NOREF(pFpuState, pFSW, pr80Val1, pr32Val2);
4500	AssertReleaseFailed();
4501	}
4502
4503
4504	IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r64,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
4505	PCRTFLOAT80U pr80Val1, PCRTFLOAT64U pr64Val2))
4506	{
4507	RT_NOREF(pFpuState, pFSW, pr80Val1, pr64Val2);
4508	AssertReleaseFailed();
4509	}
4510
4511
4512	IEM_DECL_IMPL_DEF(void, iemAImpl_fcom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
4513	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4514	{
4515	RT_NOREF(pFpuState, pFSW, pr80Val1, pr80Val2);
4516	AssertReleaseFailed();
4517	}
4518
4519
4520	IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fcomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
4521	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4522	{
4523	RT_NOREF(pFpuState, pFSW, pr80Val1, pr80Val2);
4524	AssertReleaseFailed();
4525	return 0;
4526	}
4527
4528
4529	IEM_DECL_IMPL_DEF(void, iemAImpl_fucom_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pFSW,
4530	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4531	{
4532	RT_NOREF(pFpuState, pFSW, pr80Val1, pr80Val2);
4533	AssertReleaseFailed();
4534	}
4535
4536
4537	IEM_DECL_IMPL_DEF(uint32_t, iemAImpl_fucomi_r80_by_r80,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
4538	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4539	{
4540	RT_NOREF(pFpuState, pu16Fsw, pr80Val1, pr80Val2);
4541	AssertReleaseFailed();
4542	return 0;
4543	}
4544
4545
4546	IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i16,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
4547	PCRTFLOAT80U pr80Val1, int16_t const *pi16Val2))
4548	{
4549	RT_NOREF(pFpuState, pu16Fsw, pr80Val1, pi16Val2);
4550	AssertReleaseFailed();
4551	}
4552
4553
4554	IEM_DECL_IMPL_DEF(void, iemAImpl_ficom_r80_by_i32,(PCX86FXSTATE pFpuState, uint16_t *pu16Fsw,
4555	PCRTFLOAT80U pr80Val1, int32_t const *pi32Val2))
4556	{
4557	RT_NOREF(pFpuState, pu16Fsw, pr80Val1, pi32Val2);
4558	AssertReleaseFailed();
4559	}
4560
4561
4562	/*********************************************************************************************************************************
4563	* x87 FPU Other Operations *
4564	*********************************************************************************************************************************/
4565
4566
4567	IEM_DECL_IMPL_DEF(void, iemAImpl_frndint_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4568	{
4569	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4570	AssertReleaseFailed();
4571	}
4572
4573
4574	IEM_DECL_IMPL_DEF(void, iemAImpl_fscale_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4575	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4576	{
4577	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4578	AssertReleaseFailed();
4579	}
4580
4581
4582	IEM_DECL_IMPL_DEF(void, iemAImpl_fsqrt_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4583	{
4584	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4585	AssertReleaseFailed();
4586	}
4587
4588
4589	IEM_DECL_IMPL_DEF(void, iemAImpl_f2xm1_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4590	{
4591	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4592	AssertReleaseFailed();
4593	}
4594
4595
4596	IEM_DECL_IMPL_DEF(void, iemAImpl_fabs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4597	{
4598	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4599	AssertReleaseFailed();
4600	}
4601
4602
4603	IEM_DECL_IMPL_DEF(void, iemAImpl_fchs_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes, PCRTFLOAT80U pr80Val))
4604	{
4605	RT_NOREF(pFpuState, pFpuRes, pr80Val);
4606	AssertReleaseFailed();
4607	}
4608
4609
4610	IEM_DECL_IMPL_DEF(void, iemAImpl_fxtract_r80_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULTTWO pFpuResTwo, PCRTFLOAT80U pr80Val))
4611	{
4612	RT_NOREF(pFpuState, pFpuResTwo, pr80Val);
4613	AssertReleaseFailed();
4614	}
4615
4616
4617	IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2x_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4618	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4619	{
4620	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4621	AssertReleaseFailed();
4622	}
4623
4624
4625	IEM_DECL_IMPL_DEF(void, iemAImpl_fyl2xp1_r80_by_r80,(PCX86FXSTATE pFpuState, PIEMFPURESULT pFpuRes,
4626	PCRTFLOAT80U pr80Val1, PCRTFLOAT80U pr80Val2))
4627	{
4628	RT_NOREF(pFpuState, pFpuRes, pr80Val1, pr80Val2);
4629	AssertReleaseFailed();
4630	}
4631
4632	#endif /* IEM_WITHOUT_ASSEMBLY */
4633
4634
4635	/*********************************************************************************************************************************
4636	* MMX, SSE & AVX *
4637	*********************************************************************************************************************************/
4638
4639	IEM_DECL_IMPL_DEF(void, iemAImpl_movsldup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
4640	{
4641	RT_NOREF(pFpuState);
4642	puDst->au32[0] = puSrc->au32[0];
4643	puDst->au32[1] = puSrc->au32[0];
4644	puDst->au32[2] = puSrc->au32[2];
4645	puDst->au32[3] = puSrc->au32[2];
4646	}
4647
4648	#ifdef IEM_WITH_VEX
4649
4650	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
4651	{
4652	pXState->x87.aXMM[iYRegDst].au32[0] = pXState->x87.aXMM[iYRegSrc].au32[0];
4653	pXState->x87.aXMM[iYRegDst].au32[1] = pXState->x87.aXMM[iYRegSrc].au32[0];
4654	pXState->x87.aXMM[iYRegDst].au32[2] = pXState->x87.aXMM[iYRegSrc].au32[2];
4655	pXState->x87.aXMM[iYRegDst].au32[3] = pXState->x87.aXMM[iYRegSrc].au32[2];
4656	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
4657	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[0];
4658	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
4659	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au32[2];
4660	}
4661
4662
4663	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovsldup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
4664	{
4665	pXState->x87.aXMM[iYRegDst].au32[0] = pSrc->au32[0];
4666	pXState->x87.aXMM[iYRegDst].au32[1] = pSrc->au32[0];
4667	pXState->x87.aXMM[iYRegDst].au32[2] = pSrc->au32[2];
4668	pXState->x87.aXMM[iYRegDst].au32[3] = pSrc->au32[2];
4669	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[0] = pSrc->au32[4];
4670	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[1] = pSrc->au32[4];
4671	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[2] = pSrc->au32[6];
4672	pXState->u.YmmHi.aYmmHi[iYRegDst].au32[3] = pSrc->au32[6];
4673	}
4674
4675	#endif /* IEM_WITH_VEX */
4676
4677
4678	IEM_DECL_IMPL_DEF(void, iemAImpl_movshdup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, PCRTUINT128U puSrc))
4679	{
4680	RT_NOREF(pFpuState);
4681	puDst->au32[0] = puSrc->au32[1];
4682	puDst->au32[1] = puSrc->au32[1];
4683	puDst->au32[2] = puSrc->au32[3];
4684	puDst->au32[3] = puSrc->au32[3];
4685	}
4686
4687
4688	IEM_DECL_IMPL_DEF(void, iemAImpl_movddup,(PCX86FXSTATE pFpuState, PRTUINT128U puDst, uint64_t uSrc))
4689	{
4690	RT_NOREF(pFpuState);
4691	puDst->au64[0] = uSrc;
4692	puDst->au64[1] = uSrc;
4693	}
4694
4695	#ifdef IEM_WITH_VEX
4696
4697	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rr,(PX86XSAVEAREA pXState, uint8_t iYRegDst, uint8_t iYRegSrc))
4698	{
4699	pXState->x87.aXMM[iYRegDst].au64[0] = pXState->x87.aXMM[iYRegSrc].au64[0];
4700	pXState->x87.aXMM[iYRegDst].au64[1] = pXState->x87.aXMM[iYRegSrc].au64[0];
4701	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
4702	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pXState->u.YmmHi.aYmmHi[iYRegSrc].au64[0];
4703	}
4704
4705	IEM_DECL_IMPL_DEF(void, iemAImpl_vmovddup_256_rm,(PX86XSAVEAREA pXState, uint8_t iYRegDst, PCRTUINT256U pSrc))
4706	{
4707	pXState->x87.aXMM[iYRegDst].au64[0] = pSrc->au64[0];
4708	pXState->x87.aXMM[iYRegDst].au64[1] = pSrc->au64[0];
4709	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[0] = pSrc->au64[2];
4710	pXState->u.YmmHi.aYmmHi[iYRegDst].au64[1] = pSrc->au64[2];
4711	}
4712
4713	#endif /* IEM_WITH_VEX */
4714
4715	#ifdef IEM_WITHOUT_ASSEMBLY
4716
4717	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4718	{
4719	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4720	AssertReleaseFailed();
4721	}
4722
4723
4724	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqb_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4725	{
4726	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4727	AssertReleaseFailed();
4728	}
4729
4730
4731	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4732	{
4733	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4734	AssertReleaseFailed();
4735	}
4736
4737
4738	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4739	{
4740	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4741	AssertReleaseFailed();
4742	}
4743
4744
4745	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4746	{
4747	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4748	AssertReleaseFailed();
4749	}
4750
4751
4752	IEM_DECL_IMPL_DEF(void, iemAImpl_pcmpeqd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4753	{
4754	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4755	AssertReleaseFailed();
4756	}
4757
4758
4759	IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4760	{
4761	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4762	AssertReleaseFailed();
4763	}
4764
4765
4766	IEM_DECL_IMPL_DEF(void, iemAImpl_pxor_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4767	{
4768	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4769	AssertReleaseFailed();
4770	}
4771
4772
4773	IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4774	{
4775	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4776	AssertReleaseFailed();
4777
4778	}
4779
4780
4781	IEM_DECL_IMPL_DEF(void, iemAImpl_pmovmskb_u128,(PCX86FXSTATE pFpuState, uint64_t *pu64Dst, PCRTUINT128U pu128Src))
4782	{
4783	RT_NOREF(pFpuState, pu64Dst, pu128Src);
4784	AssertReleaseFailed();
4785	}
4786
4787
4788	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufw,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src, uint8_t bEvil))
4789	{
4790	RT_NOREF(pFpuState, pu64Dst, pu64Src, bEvil);
4791	AssertReleaseFailed();
4792	}
4793
4794
4795	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufhw,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src, uint8_t bEvil))
4796	{
4797	RT_NOREF(pFpuState, pu128Dst, pu128Src, bEvil);
4798	AssertReleaseFailed();
4799	}
4800
4801
4802	IEM_DECL_IMPL_DEF(void, iemAImpl_pshuflw,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src, uint8_t bEvil))
4803	{
4804	RT_NOREF(pFpuState, pu128Dst, pu128Src, bEvil);
4805	AssertReleaseFailed();
4806	}
4807
4808
4809	IEM_DECL_IMPL_DEF(void, iemAImpl_pshufd,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src, uint8_t bEvil))
4810	{
4811	RT_NOREF(pFpuState, pu128Dst, pu128Src, bEvil);
4812	AssertReleaseFailed();
4813	}
4814
4815	/* PUNPCKHxxx */
4816
4817	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4818	{
4819	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4820	AssertReleaseFailed();
4821	}
4822
4823
4824	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhbw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4825	{
4826	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4827	AssertReleaseFailed();
4828	}
4829
4830
4831	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4832	{
4833	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4834	AssertReleaseFailed();
4835	}
4836
4837
4838	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhwd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4839	{
4840	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4841	AssertReleaseFailed();
4842	}
4843
4844
4845	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint64_t const pu64Src))
4846	{
4847	RT_NOREF(pFpuState, pu64Dst, pu64Src);
4848	AssertReleaseFailed();
4849	}
4850
4851
4852	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhdq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4853	{
4854	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4855	AssertReleaseFailed();
4856	}
4857
4858
4859	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckhqdq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, PCRTUINT128U pu128Src))
4860	{
4861	RT_NOREF(pFpuState, pu128Dst, pu128Src);
4862	AssertReleaseFailed();
4863	}
4864
4865	/* PUNPCKLxxx */
4866
4867	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src))
4868	{
4869	RT_NOREF(pFpuState, pu64Dst, pu32Src);
4870	AssertReleaseFailed();
4871	}
4872
4873
4874	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklbw_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
4875	{
4876	RT_NOREF(pFpuState, pu128Dst, pu64Src);
4877	AssertReleaseFailed();
4878	}
4879
4880
4881	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src))
4882	{
4883	RT_NOREF(pFpuState, pu64Dst, pu32Src);
4884	AssertReleaseFailed();
4885	}
4886
4887
4888	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklwd_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
4889	{
4890	RT_NOREF(pFpuState, pu128Dst, pu64Src);
4891	AssertReleaseFailed();
4892	}
4893
4894
4895	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u64,(PCX86FXSTATE pFpuState, uint64_t pu64Dst, uint32_t const pu32Src))
4896	{
4897	RT_NOREF(pFpuState, pu64Dst, pu32Src);
4898	AssertReleaseFailed();
4899	}
4900
4901
4902	IEM_DECL_IMPL_DEF(void, iemAImpl_punpckldq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
4903	{
4904	RT_NOREF(pFpuState, pu128Dst, pu64Src);
4905	AssertReleaseFailed();
4906	}
4907
4908
4909	IEM_DECL_IMPL_DEF(void, iemAImpl_punpcklqdq_u128,(PCX86FXSTATE pFpuState, PRTUINT128U pu128Dst, uint64_t const *pu64Src))
4910	{
4911	RT_NOREF(pFpuState, pu128Dst, pu64Src);
4912	AssertReleaseFailed();
4913	}
4914
4915	#endif /* IEM_WITHOUT_ASSEMBLY */

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMAll/IEMAllAImplC.cpp@ 94402

Download in other formats: