stack-vcc.asm@ 96420

Last change on this file since 96420 was 96420, checked in by vboxsync, 3 years ago
IPRT/nocrt: Implemented GSHandlerCheck so we can avoid overrunning the stack due to the int3 in the stub. bugref:10261
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 13.9 KB

Line
1	; $Id: stack-vcc.asm 96420 2022-08-23 02:14:54Z vboxsync $
2	;; @file
3	; IPRT - Stack related Visual C++ support routines.
4	;
5
6	;
7	; Copyright (C) 2022 Oracle and/or its affiliates.
8	;
9	; This file is part of VirtualBox base platform packages, as
10	; available from https://www.virtualbox.org.
11	;
12	; This program is free software; you can redistribute it and/or
13	; modify it under the terms of the GNU General Public License
14	; as published by the Free Software Foundation, in version 3 of the
15	; License.
16	;
17	; This program is distributed in the hope that it will be useful, but
18	; WITHOUT ANY WARRANTY; without even the implied warranty of
19	; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	; General Public License for more details.
21	;
22	; You should have received a copy of the GNU General Public License
23	; along with this program; if not, see <https://www.gnu.org/licenses>.
24	;
25	; The contents of this file may alternatively be used under the terms
26	; of the Common Development and Distribution License Version 1.0
27	; (CDDL), a copy of it is provided in the "COPYING.CDDL" file included
28	; in the VirtualBox distribution, in which case the provisions of the
29	; CDDL are applicable instead of those of the GPL.
30	;
31	; You may elect to license modified versions of this file under the
32	; terms and conditions of either the GPL or the CDDL or both.
33	;
34	; SPDX-License-Identifier: GPL-3.0-only OR CDDL-1.0
35	;
36
37
38
39	;*********************************************************************************************************************************
40	;* Header Files *
41	;*********************************************************************************************************************************
42	%if 0 ; YASM's builtin SEH64 support doesn't cope well with code alignment, so use our own.
43	%define RT_ASM_WITH_SEH64
44	%else
45	%define RT_ASM_WITH_SEH64_ALT
46	%endif
47	%include "iprt/asmdefs.mac"
48	%include "iprt/x86.mac"
49
50
51	;*********************************************************************************************************************************
52	;* Structures and Typedefs *
53	;*********************************************************************************************************************************
54
55	;; Variable descriptor.
56	struc RTC_VAR_DESC_T
57	.offFrame resd 1
58	.cbVar resd 1
59	alignb RTCCPTR_CB
60	.pszName RTCCPTR_RES 1
61	endstruc
62
63	;; Frame descriptor.
64	struc RTC_FRAME_DESC_T
65	.cVars resd 1
66	alignb RTCCPTR_CB
67	.paVars RTCCPTR_RES 1 ; Array of RTC_VAR_DESC_T.
68	endstruc
69
70	;; An alloca allocation.
71	struc RTC_ALLOCA_ENTRY_T
72	.uGuard1 resd 1
73	.pNext RTCCPTR_RES 1 ; Misaligned.
74	%if ARCH_BITS == 32
75	.pNextPad resd 1
76	%endif
77	.cb RTCCPTR_RES 1 ; Misaligned.
78	%if ARCH_BITS == 32
79	.cbPad resd 1
80	%endif
81	.auGuard2 resd 3
82	endstruc
83
84	%ifdef RT_ARCH_X86
85	%define FASTCALL_NAME(a_Name, a_cbArgs) $@ %+ a_Name %+ @ %+ a_cbArgs
86	%else
87	%define FASTCALL_NAME(a_Name, a_cbArgs) NAME(a_Name)
88	%endif
89
90
91	;*********************************************************************************************************************************
92	;* Defined Constants And Macros *
93	;*********************************************************************************************************************************
94	%define VARIABLE_MARKER_PRE 0xcccccccc
95	%define VARIABLE_MARKER_POST 0xcccccccc
96
97	%define ALLOCA_FILLER_BYTE 0xcc
98	%define ALLOCA_FILLER_32 0xcccccccc
99
100
101	;*********************************************************************************************************************************
102	;* Global Variables *
103	;*********************************************************************************************************************************
104	BEGINDATA
105	GLOBALNAME __security_cookie
106	dd 0xdeadbeef
107	dd 0x0c00ffe0
108
109
110	;*********************************************************************************************************************************
111	;* External Symbols *
112	;*********************************************************************************************************************************
113	BEGINCODE
114	extern NAME(_RTC_StackVarCorrupted)
115	extern NAME(_RTC_SecurityCookieMismatch)
116	%ifdef RT_ARCH_X86
117	extern NAME(_RTC_CheckEspFailed)
118	%endif
119
120
121
122	;;
123	; Probe stack to trigger guard faults, and for x86 to allocate stack space.
124	;
125	; @param xAX Frame size.
126	; @uses AMD64: Nothing (because we don't quite now the convention).
127	; x86: ESP = ESP - EAX; nothing else
128	;
129	ALIGNCODE(64)
130	GLOBALNAME_RAW __alloca_probe, __alloca_probe, function
131	BEGINPROC_RAW __chkstk
132	push xBP
133	SEH64_PUSH_xBP
134	mov xBP, xSP
135	SEH64_SET_FRAME_xBP 0
136	push xAX
137	SEH64_PUSH_GREG xAX
138	push xBX
139	SEH64_PUSH_GREG xBX
140	SEH64_END_PROLOGUE
141
142	;
143	; Adjust eax so we can use xBP for stack addressing.
144	;
145	sub xAX, xCB*2
146	jle .touch_loop_done
147
148	;
149	; Subtract what's left of the current page from eax and only engage
150	; the touch loop if (int)xAX > 0.
151	;
152	mov ebx, PAGE_SIZE - 1
153	and ebx, ebp
154	sub xAX, xBX
155	jnl .touch_loop
156
157	.touch_loop_done:
158	pop xBX
159	pop xAX
160	leave
161	%ifndef RT_ARCH_X86
162	ret
163	%else
164	;
165	; Do the stack space allocation and jump to the return location.
166	;
167	sub esp, eax
168	add esp, 4
169	jmp dword [esp + eax - 4]
170	%endif
171
172	;
173	; The touch loop.
174	;
175	.touch_loop:
176	sub xBX, PAGE_SIZE
177	mov [xBP + xBX], bl
178	sub xAX, PAGE_SIZE
179	jnl .touch_loop
180	jmp .touch_loop_done
181	ENDPROC_RAW __chkstk
182
183
184	%ifdef RT_ARCH_X86
185	;;
186	; 8 and 16 byte aligned alloca w/ probing.
187	;
188	; This routine adjusts the allocation size so __chkstk will return a
189	; correctly aligned allocation.
190	;
191	; @param xAX Unaligned allocation size.
192	;
193	%macro __alloc_probe_xxx 1
194	ALIGNCODE(16)
195	BEGINPROC_RAW __alloca_probe_ %+ %1
196	push ecx
197
198	;
199	; Calc the ESP address after the allocation and adjust EAX so that it
200	; will be aligned as desired.
201	;
202	lea ecx, [esp + 8]
203	sub ecx, eax
204	and ecx, %1 - 1
205	add eax, ecx
206	jc .bad_alloc_size
207	.continue:
208
209	pop ecx
210	jmp __alloca_probe
211
212	.bad_alloc_size:
213	%ifdef RT_STRICT
214	int3
215	%endif
216	or eax, 0xfffffff0
217	jmp .continue
218	ENDPROC_RAW __alloca_probe_ %+ %1
219	%endmacro
220
221	__alloc_probe_xxx 16
222	__alloc_probe_xxx 8
223	%endif ; RT_ARCH_X86
224
225
226	;;
227	; This just initializes a global and calls _RTC_SetErrorFuncW to NULL, and
228	; since we don't have either of those we have nothing to do here.
229	BEGINPROC _RTC_InitBase
230	SEH64_END_PROLOGUE
231	ret
232	ENDPROC _RTC_InitBase
233
234
235	;;
236	; Nothing to do here.
237	BEGINPROC _RTC_Shutdown
238	SEH64_END_PROLOGUE
239	ret
240	ENDPROC _RTC_Shutdown
241
242
243
244
245	;;
246	; Checks stack variable markers.
247	;
248	; This seems to be a regular C function in the CRT, but x86 is conveniently
249	; using the fastcall convention which makes it very similar to amd64.
250	;
251	; We try make this as sleek as possible, leaving all the trouble for when we
252	; find a corrupted stack variable and need to call a C function to complain.
253	;
254	; @param pStackFrame The caller RSP/ESP. [RCX/ECX]
255	; @param pFrameDesc Frame descriptor. [RDX/EDX]
256	;
257	ALIGNCODE(64)
258	BEGINPROC_RAW FASTCALL_NAME(_RTC_CheckStackVars, 8)
259	push xBP
260	SEH64_PUSH_xBP
261	SEH64_END_PROLOGUE
262
263	;
264	; Load the variable count into eax and check that it's not zero.
265	;
266	mov eax, [xDX + RTC_FRAME_DESC_T.cVars]
267	test eax, eax
268	jz .return
269
270	;
271	; Make edx/rdx point to the current variable and xBP be the frame pointer.
272	; The latter frees up xCX for scratch use and incidentally make stack access
273	; go via SS instead of DS (mostly irrlevant in 64-bit and 32-bit mode).
274	;
275	mov xDX, [xDX + RTC_FRAME_DESC_T.paVars]
276	mov xBP, xCX
277
278	;
279	; Loop thru the variables and check that their markers/fences haven't be
280	; trampled over.
281	;
282	.next_var:
283	; Marker before the variable.
284	%if ARCH_BITS == 64
285	movsxd rcx, dword [xDX + RTC_VAR_DESC_T.offFrame]
286	%else
287	mov xCX, dword [xDX + RTC_VAR_DESC_T.offFrame]
288	%endif
289	cmp dword [xBP + xCX - 4], VARIABLE_MARKER_PRE
290	jne .corrupted
291
292	; Marker after the variable.
293	add ecx, dword [xDX + RTC_VAR_DESC_T.cbVar]
294	%if ARCH_BITS == 64
295	movsxd rcx, ecx
296	%endif
297	cmp dword [xBP + xCX], VARIABLE_MARKER_POST
298	jne .corrupted
299
300	;
301	; Advance to the next variable.
302	;
303	.advance:
304	add xDX, RTC_VAR_DESC_T_size
305	dec eax
306	jnz .next_var
307
308	;
309	; Return.
310	;
311	.return:
312	pop xBP
313	ret
314
315	;
316	; Complain about corrupt variable.
317	;
318	.corrupted:
319	push xAX
320	push xDX
321	%ifdef RT_ARCH_AMD64
322	sub xSP, 28h
323	mov xCX, xBP ; frame pointer + variable descriptor.
324	%else
325	push xBP ; save EBP
326	push xDX ; parameter 2 - variable descriptor
327	push xBP ; parameter 1 - frame pointer.
328	lea xBP, [xSP + 3*xCB] ; turn it into a frame pointer during the call for better unwind.
329	%endif
330
331	call NAME(_RTC_StackVarCorrupted)
332
333	%ifdef RT_ARCH_AMD64
334	add xSP, 28h
335	%else
336	add xSP, xCB * 4 ; parameters
337	pop xBP
338	%endif
339	pop xDX
340	pop xAX
341	jmp .advance
342	ENDPROC_RAW FASTCALL_NAME(_RTC_CheckStackVars, 8)
343
344
345	%ifdef RT_ARCH_X86
346	;;
347	; Called to follow up on a 'CMP ESP, EBP' kind of instruction,
348	; expected to report failure if the compare failed.
349	;
350	ALIGNCODE(16)
351	BEGINPROC _RTC_CheckEsp
352	jne .unexpected_esp
353	ret
354
355	.unexpected_esp:
356	push ebp
357	mov ebp, esp
358	push eax
359	push ecx
360	push edx
361
362	; DECLASM(void) _RTC_CheckEspFailed(uintptr_t uEip, uintptr_t uEsp, uintptr_t uEbp)
363	push dword [ebp]
364	lea edx, [ebp + 8]
365	push edx
366	mov ecx, [ebp + 8]
367	push ecx
368	call NAME(_RTC_CheckEspFailed)
369
370	pop edx
371	pop ecx
372	pop eax
373	leave
374	ret
375	ENDPROC _RTC_CheckEsp
376	%endif ; RT_ARCH_X86
377
378
379
380	;;
381	; Initialize an alloca allocation list entry and add it to it.
382	;
383	; When this is call, presumably _RTC_CheckStackVars2 is used to verify the frame.
384	;
385	; @param pNewEntry Pointer to the new entry. [RCX/ECX]
386	; @param cbEntry The entry size, including header. [RDX/EDX]
387	; @param ppHead Pointer to the list head pointer. [R8/stack]
388	;
389	ALIGNCODE(64)
390	BEGINPROC_RAW FASTCALL_NAME(_RTC_AllocaHelper, 12)
391	SEH64_END_PROLOGUE
392
393	;
394	; Check that input isn't NULL or the size isn't zero.
395	;
396	test xCX, xCX
397	jz .return
398	test xDX, xDX
399	jz .return
400	%if ARCH_BITS == 64
401	test r8, r8
402	%else
403	cmp dword [xSP + xCB], 0
404	%endif
405	jz .return
406
407	;
408	; Memset the memory to ALLOCA_FILLER
409	;
410	%if ARCH_BITS == 64
411	mov r10, rdi ; save rdi
412	mov r11, rcx ; save pNewEntry
413	%else
414	push xDI
415	push xCX
416	cld ; paranoia
417	%endif
418
419	mov al, ALLOCA_FILLER_BYTE
420	mov xDI, xCX ; entry pointer
421	mov xCX, xDX ; entry size (in bytes)
422	rep stosb
423
424	%if ARCH_BITS == 64
425	mov rdi, r10
426	%else
427	pop xCX
428	pop xDI
429	%endif
430
431	;
432	; Fill in the entry and link it as onto the head of the chain.
433	;
434	%if ARCH_BITS == 64
435	mov [r11 + RTC_ALLOCA_ENTRY_T.cb], xDX
436	mov xAX, [r8]
437	mov [r11 + RTC_ALLOCA_ENTRY_T.pNext], xAX
438	mov [r8], r11
439	%else
440	mov [xCX + RTC_ALLOCA_ENTRY_T.cb], xDX
441	mov xAX, [xSP + xCB] ; ppHead
442	mov xDX, [xAX]
443	mov [xCX + RTC_ALLOCA_ENTRY_T.pNext], xDX
444	mov [xAX], xCX
445	%endif
446
447	.return:
448	%if ARCH_BITS == 64
449	ret
450	%else
451	ret 4
452	%endif
453	ENDPROC_RAW FASTCALL_NAME(_RTC_AllocaHelper, 12)
454
455
456	;;
457	; Checks if the secuity cookie ok, complaining and terminating if it isn't.
458	;
459	ALIGNCODE(16)
460	BEGINPROC_RAW FASTCALL_NAME(__security_check_cookie, 4)
461	SEH64_END_PROLOGUE
462	cmp xCX, [NAME(__security_cookie) xWrtRIP]
463	jne .corrupted
464	;; amd64 version checks if the top 16 bits are zero, we skip that for now.
465	ret
466
467	.corrupted:
468	%ifdef RT_ARCH_AMD64
469	jmp NAME(_RTC_SecurityCookieMismatch)
470	%else
471	push ebp
472	mov ebp, esp
473	push ecx
474	call NAME(_RTC_SecurityCookieMismatch)
475	pop ecx
476	leave
477	ret
478	%endif
479	ENDPROC_RAW FASTCALL_NAME(__security_check_cookie, 4)
480
481
482
483	; Not stack related stubs.
484	BEGINPROC __C_specific_handler
485	SEH64_END_PROLOGUE
486	int3
487	ENDPROC __C_specific_handler
488
489
490	BEGINPROC __report_rangecheckfailure
491	SEH64_END_PROLOGUE
492	int3
493	ENDPROC __report_rangecheckfailure
494

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/Runtime/common/compiler/vcc/stack-vcc.asm@ 96420

Download in other formats: