VirtualBox

source: vbox/trunk/src/VBox/Runtime/common/fuzz/fuzz-observer.cpp@ 72895

Last change on this file since 72895 was 72650, checked in by vboxsync, 7 years ago

Runtime/fuzzing: Add some simple statistics [build fix]

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 38.5 KB
Line 
1/* $Id: fuzz-observer.cpp 72650 2018-06-22 07:42:39Z vboxsync $ */
2/** @file
3 * IPRT - Fuzzing framework API, observer.
4 */
5
6/*
7 * Copyright (C) 2018 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 *
17 * The contents of this file may alternatively be used under the terms
18 * of the Common Development and Distribution License Version 1.0
19 * (CDDL) only, as it comes in the "COPYING.CDDL" file of the
20 * VirtualBox OSE distribution, in which case the provisions of the
21 * CDDL are applicable instead of those of the GPL.
22 *
23 * You may elect to license modified versions of this file under the
24 * terms and conditions of either the GPL or the CDDL or both.
25 */
26
27
28/*********************************************************************************************************************************
29* Header Files *
30*********************************************************************************************************************************/
31#include <iprt/fuzz.h>
32#include "internal/iprt.h"
33
34#include <iprt/asm.h>
35#include <iprt/assert.h>
36#include <iprt/ctype.h>
37#include <iprt/dir.h>
38#include <iprt/err.h>
39#include <iprt/env.h>
40#include <iprt/file.h>
41#include <iprt/md5.h>
42#include <iprt/mem.h>
43#include <iprt/mp.h>
44#include <iprt/path.h>
45#include <iprt/pipe.h>
46#include <iprt/poll.h>
47#include <iprt/process.h>
48#include <iprt/semaphore.h>
49#include <iprt/stream.h>
50#include <iprt/string.h>
51#include <iprt/time.h>
52#include <iprt/thread.h>
53
54
55/** Poll ID for the reading end of the stdout pipe from the client process. */
56#define RTFUZZOBS_EXEC_CTX_POLL_ID_STDOUT 0
57/** Poll ID for the reading end of the stderr pipe from the client process. */
58#define RTFUZZOBS_EXEC_CTX_POLL_ID_STDERR 1
59/** Poll ID for the writing end of the stdin pipe to the client process. */
60#define RTFUZZOBS_EXEC_CTX_POLL_ID_STDIN 2
61
62
63/*********************************************************************************************************************************
64* Structures and Typedefs *
65*********************************************************************************************************************************/
66/** Pointer to the internal fuzzing observer state. */
67typedef struct RTFUZZOBSINT *PRTFUZZOBSINT;
68
69
70/**
71 * Observer thread state for one process.
72 */
73typedef struct RTFUZZOBSTHRD
74{
75 /** The thread handle. */
76 RTTHREAD hThread;
77 /** The observer ID. */
78 uint32_t idObs;
79 /** Flag whether to shutdown. */
80 volatile bool fShutdown;
81 /** Pointer to te global observer state. */
82 PRTFUZZOBSINT pFuzzObs;
83 /** Current fuzzer input. */
84 RTFUZZINPUT hFuzzInput;
85 /** Flag whether to keep the input. */
86 bool fKeepInput;
87 /** Flag whether a new input is waiting. */
88 volatile bool fNewInput;
89} RTFUZZOBSTHRD;
90/** Pointer to an observer thread state. */
91typedef RTFUZZOBSTHRD *PRTFUZZOBSTHRD;
92
93
94/**
95 * Internal fuzzing observer state.
96 */
97typedef struct RTFUZZOBSINT
98{
99 /** The fuzzing context used for this observer. */
100 RTFUZZCTX hFuzzCtx;
101 /** Temp directory for input files. */
102 char *pszTmpDir;
103 /** Results directory. */
104 char *pszResultsDir;
105 /** The binary to run. */
106 char *pszBinary;
107 /** Arguments to run the binary with, terminated by a NULL entry. */
108 char **papszArgs;
109 /** Number of arguments. */
110 uint32_t cArgs;
111 /** Maximum time to wait for the client to terminate until it is considered hung and killed. */
112 RTMSINTERVAL msWaitMax;
113 /** Flags controlling how the binary is executed. */
114 uint32_t fFlags;
115 /** Flag whether to shutdown the master and all workers. */
116 volatile bool fShutdown;
117 /** Global observer thread handle. */
118 RTTHREAD hThreadGlobal;
119 /** The event semaphore handle for the global observer thread. */
120 RTSEMEVENT hEvtGlobal;
121 /** Notification event bitmap. */
122 volatile uint64_t bmEvt;
123 /** Number of threads created - one for each process. */
124 uint32_t cThreads;
125 /** Pointer to the array of observer thread states. */
126 PRTFUZZOBSTHRD paObsThreads;
127 /** Timestamp of the last stats query. */
128 uint64_t tsLastStats;
129 /** Last number of fuzzed inputs per second if we didn't gather enough data in between
130 * statistic queries. */
131 uint32_t cFuzzedInputsPerSecLast;
132 /** Fuzzing statistics. */
133 RTFUZZOBSSTATS Stats;
134} RTFUZZOBSINT;
135
136
137/**
138 * Stdout/Stderr buffer.
139 */
140typedef struct RTFUZZOBSSTDOUTERRBUF
141{
142 /** Current amount buffered. */
143 size_t cbBuf;
144 /** Maxmium amount to buffer. */
145 size_t cbBufMax;
146 /** Base pointer to the data buffer. */
147 uint8_t *pbBase;
148} RTFUZZOBSSTDOUTERRBUF;
149/** Pointer to a stdout/stderr buffer. */
150typedef RTFUZZOBSSTDOUTERRBUF *PRTFUZZOBSSTDOUTERRBUF;
151
152
153/**
154 * Worker execution context.
155 */
156typedef struct RTFUZZOBSEXECCTX
157{
158 /** The stdout pipe handle - reading end. */
159 RTPIPE hPipeStdoutR;
160 /** The stdout pipe handle - writing end. */
161 RTPIPE hPipeStdoutW;
162 /** The stderr pipe handle - reading end. */
163 RTPIPE hPipeStderrR;
164 /** The stderr pipe handle - writing end. */
165 RTPIPE hPipeStderrW;
166 /** The stdin pipe handle - reading end. */
167 RTPIPE hPipeStdinR;
168 /** The stind pipe handle - writing end. */
169 RTPIPE hPipeStdinW;
170 /** The stdout handle. */
171 RTHANDLE StdoutHandle;
172 /** The stderr handle. */
173 RTHANDLE StderrHandle;
174 /** The stdin handle. */
175 RTHANDLE StdinHandle;
176 /** The pollset to monitor. */
177 RTPOLLSET hPollSet;
178 /** The process to monitor. */
179 RTPROCESS hProc;
180 /** Execution time of the process. */
181 RTMSINTERVAL msExec;
182 /** Current input data pointer. */
183 uint8_t *pbInputCur;
184 /** Number of bytes left for the input. */
185 size_t cbInputLeft;
186 /** The stdout data buffer. */
187 RTFUZZOBSSTDOUTERRBUF StdOutBuf;
188 /** The stderr data buffer. */
189 RTFUZZOBSSTDOUTERRBUF StdErrBuf;
190 /** Modified arguments vector - variable in size. */
191 char *apszArgs[1];
192} RTFUZZOBSEXECCTX;
193/** Pointer to an execution context. */
194typedef RTFUZZOBSEXECCTX *PRTFUZZOBSEXECCTX;
195/** Pointer to an execution context pointer. */
196typedef PRTFUZZOBSEXECCTX *PPRTFUZZOBSEXECCTX;
197
198
199/**
200 * A variable descriptor.
201 */
202typedef struct RTFUZZOBSVARIABLE
203{
204 /** The variable. */
205 const char *pszVar;
206 /** Length of the variable in characters - excluding the terminator. */
207 uint32_t cchVar;
208 /** The replacement value. */
209 const char *pszVal;
210} RTFUZZOBSVARIABLE;
211/** Pointer to a variable descriptor. */
212typedef RTFUZZOBSVARIABLE *PRTFUZZOBSVARIABLE;
213
214
215/**
216 * Initializes the given stdout/stderr buffer.
217 *
218 * @returns nothing.
219 * @param pBuf The buffer to initialize.
220 */
221static void rtFuzzObsStdOutErrBufInit(PRTFUZZOBSSTDOUTERRBUF pBuf)
222{
223 pBuf->cbBuf = 0;
224 pBuf->cbBufMax = 0;
225 pBuf->pbBase = NULL;
226}
227
228
229/**
230 * Frees all allocated resources in the given stdout/stderr buffer.
231 *
232 * @returns nothing.
233 * @param pBuf The buffer to free.
234 */
235static void rtFuzzObsStdOutErrBufFree(PRTFUZZOBSSTDOUTERRBUF pBuf)
236{
237 if (pBuf->pbBase)
238 RTMemFree(pBuf->pbBase);
239}
240
241
242/**
243 * Clears the given stdout/stderr buffer.
244 *
245 * @returns nothing.
246 * @param pBuf The buffer to clear.
247 */
248static void rtFuzzObsStdOutErrBufClear(PRTFUZZOBSSTDOUTERRBUF pBuf)
249{
250 pBuf->cbBuf = 0;
251}
252
253
254/**
255 * Fills the given stdout/stderr buffer from the given pipe.
256 *
257 * @returns IPRT status code.
258 * @param pBuf The buffer to fill.
259 * @param hPipeRead The pipe to read from.
260 */
261static int rtFuzzObsStdOutErrBufFill(PRTFUZZOBSSTDOUTERRBUF pBuf, RTPIPE hPipeRead)
262{
263 int rc = VINF_SUCCESS;
264
265 size_t cbRead = 0;
266 size_t cbThisRead = 0;
267 do
268 {
269 cbThisRead = pBuf->cbBufMax - pBuf->cbBuf;
270 if (!cbThisRead)
271 {
272 /* Try to increase the buffer. */
273 uint8_t *pbNew = (uint8_t *)RTMemRealloc(pBuf->pbBase, pBuf->cbBufMax + _4K);
274 if (RT_LIKELY(pbNew))
275 {
276 pBuf->cbBufMax += _4K;
277 pBuf->pbBase = pbNew;
278 }
279 cbThisRead = pBuf->cbBufMax - pBuf->cbBuf;
280 }
281
282 if (cbThisRead)
283 {
284 rc = RTPipeRead(hPipeRead, pBuf->pbBase + pBuf->cbBuf, cbThisRead, &cbRead);
285 if (RT_SUCCESS(rc))
286 pBuf->cbBuf += cbRead;
287 }
288 else
289 rc = VERR_NO_MEMORY;
290 } while ( RT_SUCCESS(rc)
291 && cbRead == cbThisRead);
292
293 return rc;
294}
295
296
297/**
298 * Writes the given stdout/stderr buffer to the given filename.
299 *
300 * @returns IPRT status code.
301 * @param pBuf The buffer to write.
302 * @param pszFilename The filename to write the buffer to.
303 */
304static int rtFuzzStdOutErrBufWriteToFile(PRTFUZZOBSSTDOUTERRBUF pBuf, const char *pszFilename)
305{
306 RTFILE hFile;
307 int rc = RTFileOpen(&hFile, pszFilename, RTFILE_O_CREATE | RTFILE_O_WRITE | RTFILE_O_DENY_NONE);
308 if (RT_SUCCESS(rc))
309 {
310 rc = RTFileWrite(hFile, pBuf->pbBase, pBuf->cbBuf, NULL);
311 AssertRC(rc);
312 RTFileClose(hFile);
313
314 if (RT_FAILURE(rc))
315 RTFileDelete(pszFilename);
316 }
317
318 return rc;
319}
320
321
322/**
323 * Replaces a variable with its value.
324 *
325 * @returns VINF_SUCCESS or VERR_NO_STR_MEMORY.
326 * @param ppszNew In/Out.
327 * @param pcchNew In/Out. (Messed up on failure.)
328 * @param offVar Variable offset.
329 * @param cchVar Variable length.
330 * @param pszValue The value.
331 * @param cchValue Value length.
332 */
333static int rtFuzzObsReplaceStringVariable(char **ppszNew, size_t *pcchNew, size_t offVar, size_t cchVar,
334 const char *pszValue, size_t cchValue)
335{
336 size_t const cchAfter = *pcchNew - offVar - cchVar;
337 if (cchVar < cchValue)
338 {
339 *pcchNew += cchValue - cchVar;
340 int rc = RTStrRealloc(ppszNew, *pcchNew + 1);
341 if (RT_FAILURE(rc))
342 return rc;
343 }
344
345 char *pszNew = *ppszNew;
346 memmove(&pszNew[offVar + cchValue], &pszNew[offVar + cchVar], cchAfter + 1);
347 memcpy(&pszNew[offVar], pszValue, cchValue);
348 return VINF_SUCCESS;
349}
350
351
352/**
353 * Replace the variables found in the source string, returning a new string that
354 * lives on the string heap.
355 *
356 * @returns IPRT status code.
357 * @param pszSrc The source string.
358 * @param paVars Pointer to the array of known variables.
359 * @param ppszNew Where to return the new string.
360 */
361static int rtFuzzObsReplaceStringVariables(const char *pszSrc, PRTFUZZOBSVARIABLE paVars, char **ppszNew)
362{
363 /* Lazy approach that employs memmove. */
364 int rc = VINF_SUCCESS;
365 size_t cchNew = strlen(pszSrc);
366 char *pszNew = RTStrDup(pszSrc);
367
368 if (paVars)
369 {
370 char *pszDollar = pszNew;
371 while ((pszDollar = strchr(pszDollar, '$')) != NULL)
372 {
373 if (pszDollar[1] == '{')
374 {
375 const char *pszEnd = strchr(&pszDollar[2], '}');
376 if (pszEnd)
377 {
378 size_t const cchVar = pszEnd - pszDollar + 1; /* includes "${}" */
379 size_t offDollar = pszDollar - pszNew;
380 PRTFUZZOBSVARIABLE pVar = paVars;
381 while (pVar->pszVar != NULL)
382 {
383 if ( cchVar == pVar->cchVar
384 && !memcmp(pszDollar, pVar->pszVar, cchVar))
385 {
386 size_t const cchValue = strlen(pVar->pszVal);
387 rc = rtFuzzObsReplaceStringVariable(&pszNew, &cchNew, offDollar,
388 cchVar, pVar->pszVal, cchValue);
389 offDollar += cchValue;
390 break;
391 }
392
393 pVar++;
394 }
395
396 pszDollar = &pszNew[offDollar];
397
398 if (RT_FAILURE(rc))
399 {
400 RTStrFree(pszNew);
401 *ppszNew = NULL;
402 return rc;
403 }
404 }
405 }
406 }
407 }
408
409 *ppszNew = pszNew;
410 return rc;
411}
412
413/**
414 * Prepares the argument vector for the child process.
415 *
416 * @returns IPRT status code.
417 * @param pThis The internal fuzzing observer state.
418 * @param pExecCtx The execution context to prepare the argument vector for.
419 * @param paVars Pointer to the array of known variables.
420 */
421static int rtFuzzObsExecCtxArgvPrepare(PRTFUZZOBSINT pThis, PRTFUZZOBSEXECCTX pExecCtx, PRTFUZZOBSVARIABLE paVars)
422{
423 int rc = VINF_SUCCESS;
424 for (unsigned i = 0; i < pThis->cArgs && RT_SUCCESS(rc); i++)
425 rc = rtFuzzObsReplaceStringVariables(pThis->papszArgs[i], paVars, &pExecCtx->apszArgs[i]);
426
427 return rc;
428}
429
430
431/**
432 * Creates a new execution context.
433 *
434 * @returns IPRT status code.
435 * @param ppExecCtx Where to store the pointer to the execution context on success.
436 * @param pThis The internal fuzzing observer state.
437 */
438static int rtFuzzObsExecCtxCreate(PPRTFUZZOBSEXECCTX ppExecCtx, PRTFUZZOBSINT pThis)
439{
440 int rc = VINF_SUCCESS;
441 PRTFUZZOBSEXECCTX pExecCtx = (PRTFUZZOBSEXECCTX)RTMemAllocZ(RT_OFFSETOF(RTFUZZOBSEXECCTX, apszArgs[pThis->cArgs + 1]));
442 if (RT_LIKELY(pExecCtx))
443 {
444 pExecCtx->hPipeStdoutR = NIL_RTPIPE;
445 pExecCtx->hPipeStdoutW = NIL_RTPIPE;
446 pExecCtx->hPipeStderrR = NIL_RTPIPE;
447 pExecCtx->hPipeStderrW = NIL_RTPIPE;
448 pExecCtx->hPipeStdinR = NIL_RTPIPE;
449 pExecCtx->hPipeStdinW = NIL_RTPIPE;
450 pExecCtx->hPollSet = NIL_RTPOLLSET;
451 pExecCtx->hProc = NIL_RTPROCESS;
452 pExecCtx->msExec = 0;
453 rtFuzzObsStdOutErrBufInit(&pExecCtx->StdOutBuf);
454 rtFuzzObsStdOutErrBufInit(&pExecCtx->StdErrBuf);
455
456 rc = RTPollSetCreate(&pExecCtx->hPollSet);
457 if (RT_SUCCESS(rc))
458 {
459 rc = RTPipeCreate(&pExecCtx->hPipeStdoutR, &pExecCtx->hPipeStdoutW, RTPIPE_C_INHERIT_WRITE);
460 if (RT_SUCCESS(rc))
461 {
462 RTHANDLE Handle;
463 Handle.enmType = RTHANDLETYPE_PIPE;
464 Handle.u.hPipe = pExecCtx->hPipeStdoutR;
465 rc = RTPollSetAdd(pExecCtx->hPollSet, &Handle, RTPOLL_EVT_READ, RTFUZZOBS_EXEC_CTX_POLL_ID_STDOUT);
466 AssertRC(rc);
467
468 rc = RTPipeCreate(&pExecCtx->hPipeStderrR, &pExecCtx->hPipeStderrW, RTPIPE_C_INHERIT_WRITE);
469 if (RT_SUCCESS(rc))
470 {
471 Handle.u.hPipe = pExecCtx->hPipeStderrR;
472 rc = RTPollSetAdd(pExecCtx->hPollSet, &Handle, RTPOLL_EVT_READ, RTFUZZOBS_EXEC_CTX_POLL_ID_STDERR);
473 AssertRC(rc);
474
475 /* Create the stdin pipe handles if not a file input. */
476 if (!(pThis->fFlags & RTFUZZ_OBS_BINARY_F_INPUT_FILE))
477 {
478 rc = RTPipeCreate(&pExecCtx->hPipeStdinR, &pExecCtx->hPipeStdinW, RTPIPE_C_INHERIT_READ);
479 if (RT_SUCCESS(rc))
480 {
481 pExecCtx->StdinHandle.enmType = RTHANDLETYPE_PIPE;
482 pExecCtx->StdinHandle.u.hPipe = pExecCtx->hPipeStdinR;
483
484 Handle.u.hPipe = pExecCtx->hPipeStdinW;
485 rc = RTPollSetAdd(pExecCtx->hPollSet, &Handle, RTPOLL_EVT_WRITE, RTFUZZOBS_EXEC_CTX_POLL_ID_STDIN);
486 AssertRC(rc);
487 }
488 }
489 else
490 {
491 pExecCtx->StdinHandle.enmType = RTHANDLETYPE_PIPE;
492 pExecCtx->StdinHandle.u.hPipe = NIL_RTPIPE;
493 }
494
495 if (RT_SUCCESS(rc))
496 {
497 pExecCtx->StdoutHandle.enmType = RTHANDLETYPE_PIPE;
498 pExecCtx->StdoutHandle.u.hPipe = pExecCtx->hPipeStdoutW;
499 pExecCtx->StderrHandle.enmType = RTHANDLETYPE_PIPE;
500 pExecCtx->StderrHandle.u.hPipe = pExecCtx->hPipeStderrW;
501 *ppExecCtx = pExecCtx;
502 return VINF_SUCCESS;
503 }
504
505 RTPipeClose(pExecCtx->hPipeStderrR);
506 RTPipeClose(pExecCtx->hPipeStderrW);
507 }
508
509 RTPipeClose(pExecCtx->hPipeStdoutR);
510 RTPipeClose(pExecCtx->hPipeStdoutW);
511 }
512
513 RTPollSetDestroy(pExecCtx->hPollSet);
514 }
515
516 RTMemFree(pExecCtx);
517 }
518 else
519 rc = VERR_NO_MEMORY;
520
521 return rc;
522}
523
524
525/**
526 * Destroys the given execution context.
527 *
528 * @returns nothing.
529 * @param pThis The internal fuzzing observer state.
530 * @param pExecCtx The execution context to destroy.
531 */
532static void rtFuzzObsExecCtxDestroy(PRTFUZZOBSINT pThis, PRTFUZZOBSEXECCTX pExecCtx)
533{
534 RTPipeClose(pExecCtx->hPipeStdoutR);
535 RTPipeClose(pExecCtx->hPipeStdoutW);
536 RTPipeClose(pExecCtx->hPipeStderrR);
537 RTPipeClose(pExecCtx->hPipeStderrW);
538
539 if (!(pThis->fFlags & RTFUZZ_OBS_BINARY_F_INPUT_FILE))
540 {
541 RTPipeClose(pExecCtx->hPipeStdinR);
542 RTPipeClose(pExecCtx->hPipeStdinW);
543 }
544
545 RTPollSetDestroy(pExecCtx->hPollSet);
546 char **ppszArg = &pExecCtx->apszArgs[0];
547 while (*ppszArg != NULL)
548 {
549 RTStrFree(*ppszArg);
550 ppszArg++;
551 }
552
553 rtFuzzObsStdOutErrBufFree(&pExecCtx->StdOutBuf);
554 rtFuzzObsStdOutErrBufFree(&pExecCtx->StdErrBuf);
555 RTMemFree(pExecCtx);
556}
557
558
559/**
560 * Runs the client binary pumping all data back and forth waiting for the client to finish.
561 *
562 * @returns IPRT status code.
563 * @retval VERR_TIMEOUT if the client didn't finish in the given deadline and was killed.
564 * @param pThis The internal fuzzing observer state.
565 * @param pExecCtx The execution context.
566 * @param pProcStat Where to store the process exit status on success.
567 */
568static int rtFuzzObsExecCtxClientRun(PRTFUZZOBSINT pThis, PRTFUZZOBSEXECCTX pExecCtx, PRTPROCSTATUS pProcStat)
569{
570 rtFuzzObsStdOutErrBufClear(&pExecCtx->StdOutBuf);
571 rtFuzzObsStdOutErrBufClear(&pExecCtx->StdErrBuf);
572
573 int rc = RTProcCreateEx(pThis->pszBinary, &pExecCtx->apszArgs[0], RTENV_DEFAULT, 0 /*fFlags*/, &pExecCtx->StdinHandle,
574 &pExecCtx->StdoutHandle, &pExecCtx->StderrHandle, NULL, NULL, &pExecCtx->hProc);
575 if (RT_SUCCESS(rc))
576 {
577 uint64_t tsMilliesStart = RTTimeSystemMilliTS();
578 for (;;)
579 {
580 /* Wait a bit for something to happen on one of the pipes. */
581 uint32_t fEvtsRecv = 0;
582 uint32_t idEvt = 0;
583 rc = RTPoll(pExecCtx->hPollSet, 10 /*cMillies*/, &fEvtsRecv, &idEvt);
584 if (RT_SUCCESS(rc))
585 {
586 if (idEvt == RTFUZZOBS_EXEC_CTX_POLL_ID_STDOUT)
587 {
588 Assert(fEvtsRecv & RTPOLL_EVT_READ);
589 rc = rtFuzzObsStdOutErrBufFill(&pExecCtx->StdOutBuf, pExecCtx->hPipeStdoutR);
590 AssertRC(rc);
591 }
592 else if (idEvt == RTFUZZOBS_EXEC_CTX_POLL_ID_STDERR)
593 {
594 Assert(fEvtsRecv & RTPOLL_EVT_READ);
595 rc = rtFuzzObsStdOutErrBufFill(&pExecCtx->StdErrBuf, pExecCtx->hPipeStderrR);
596 AssertRC(rc);
597 }
598 else if (idEvt == RTFUZZOBS_EXEC_CTX_POLL_ID_STDIN)
599 {
600 /* Feed the next input. */
601 Assert(fEvtsRecv & RTPOLL_EVT_WRITE);
602 size_t cbWritten = 0;
603 rc = RTPipeWrite(pExecCtx->hPipeStdinW, pExecCtx->pbInputCur, pExecCtx->cbInputLeft, &cbWritten);
604 if (RT_SUCCESS(rc))
605 {
606 pExecCtx->cbInputLeft -= cbWritten;
607 if (!pExecCtx->cbInputLeft)
608 {
609 /* Close stdin pipe. */
610 rc = RTPollSetRemove(pExecCtx->hPollSet, RTFUZZOBS_EXEC_CTX_POLL_ID_STDIN);
611 AssertRC(rc);
612 RTPipeClose(pExecCtx->hPipeStdinW);
613 }
614 }
615 }
616 else
617 AssertMsgFailed(("Invalid poll ID returned: %u!\n", idEvt));
618 }
619 else
620 Assert(rc == VERR_TIMEOUT);
621
622 /* Check the process status. */
623 rc = RTProcWait(pExecCtx->hProc, RTPROCWAIT_FLAGS_NOBLOCK, pProcStat);
624 if (RT_SUCCESS(rc))
625 break;
626 else
627 {
628 Assert(rc == VERR_PROCESS_RUNNING);
629 /* Check whether we reached the limit. */
630 if (RTTimeSystemMilliTS() - tsMilliesStart > pThis->msWaitMax)
631 {
632 rc = VERR_TIMEOUT;
633 break;
634 }
635 }
636 } /* for (;;) */
637
638 /* Kill the process on a timeout. */
639 if (rc == VERR_TIMEOUT)
640 {
641 int rc2 = RTProcTerminate(pExecCtx->hProc);
642 AssertRC(rc2);
643 }
644 }
645
646 return rc;
647}
648
649
650/**
651 * Adds the input to the results directory.
652 *
653 * @returns IPRT status code.
654 * @param pThis The internal fuzzing observer state.
655 * @param hFuzzInput Fuzzing input handle to write.
656 * @param pExecCtx Execution context.
657 */
658static int rtFuzzObsAddInputToResults(PRTFUZZOBSINT pThis, RTFUZZINPUT hFuzzInput, PRTFUZZOBSEXECCTX pExecCtx)
659{
660 char aszDigest[RTMD5_STRING_LEN + 1];
661 int rc = RTFuzzInputQueryDigestString(hFuzzInput, &aszDigest[0], sizeof(aszDigest));
662 if (RT_SUCCESS(rc))
663 {
664 /* Create a directory. */
665 char szPath[RTPATH_MAX];
666 rc = RTPathJoin(szPath, sizeof(szPath), pThis->pszResultsDir, &aszDigest[0]);
667 AssertRC(rc);
668
669 rc = RTDirCreate(&szPath[0], 0700, 0 /*fCreate*/);
670 if (RT_SUCCESS(rc))
671 {
672 /* Write the input. */
673 char szTmp[RTPATH_MAX];
674 rc = RTPathJoin(szTmp, sizeof(szTmp), &szPath[0], "input");
675 AssertRC(rc);
676
677 rc = RTFuzzInputWriteToFile(hFuzzInput, &szTmp[0]);
678 if (RT_SUCCESS(rc))
679 {
680 /* Stdout and Stderr. */
681 rc = RTPathJoin(szTmp, sizeof(szTmp), &szPath[0], "stdout");
682 AssertRC(rc);
683 rc = rtFuzzStdOutErrBufWriteToFile(&pExecCtx->StdOutBuf, &szTmp[0]);
684 if (RT_SUCCESS(rc))
685 {
686 rc = RTPathJoin(szTmp, sizeof(szTmp), &szPath[0], "stderr");
687 AssertRC(rc);
688 rc = rtFuzzStdOutErrBufWriteToFile(&pExecCtx->StdOutBuf, &szTmp[0]);
689 }
690 }
691 }
692 }
693
694 return rc;
695}
696
697
698/**
699 * Fuzzing observer worker loop.
700 *
701 * @returns IPRT status code.
702 * @param hThrd The thread handle.
703 * @param pvUser Opaque user data.
704 */
705static DECLCALLBACK(int) rtFuzzObsWorkerLoop(RTTHREAD hThrd, void *pvUser)
706{
707 PRTFUZZOBSTHRD pObsThrd = (PRTFUZZOBSTHRD)pvUser;
708 PRTFUZZOBSINT pThis = pObsThrd->pFuzzObs;
709 PRTFUZZOBSEXECCTX pExecCtx = NULL;
710
711 int rc = rtFuzzObsExecCtxCreate(&pExecCtx, pThis);
712 if (RT_FAILURE(rc))
713 return rc;
714
715 while (!pObsThrd->fShutdown)
716 {
717 char szInput[RTPATH_MAX];
718
719 /* Wait for work. */
720 rc = RTThreadUserWait(hThrd, RT_INDEFINITE_WAIT);
721 AssertRC(rc);
722
723 if (pObsThrd->fShutdown)
724 break;
725
726 if (!ASMAtomicXchgBool(&pObsThrd->fNewInput, false))
727 continue;
728
729 AssertPtr(pObsThrd->hFuzzInput);
730
731 if (pThis->fFlags & RTFUZZ_OBS_BINARY_F_INPUT_FILE)
732 {
733 char szFilename[32];
734
735 ssize_t cbBuf = RTStrPrintf2(&szFilename[0], sizeof(szFilename), "%u", pObsThrd->idObs);
736 Assert(cbBuf > 0); RT_NOREF(cbBuf);
737
738 RT_ZERO(szInput);
739 rc = RTPathJoin(szInput, sizeof(szInput), pThis->pszTmpDir, &szFilename[0]);
740 AssertRC(rc);
741
742 rc = RTFuzzInputWriteToFile(pObsThrd->hFuzzInput, &szInput[0]);
743 if (RT_SUCCESS(rc))
744 {
745 RTFUZZOBSVARIABLE aVar[2] = {
746 { "${INPUT}", sizeof("${INPUT}") - 1, &szInput[0] },
747 { NULL, 0, NULL }
748 };
749 rc = rtFuzzObsExecCtxArgvPrepare(pThis, pExecCtx, &aVar[0]);
750 }
751 }
752 else
753 {
754 rc = RTFuzzInputQueryData(pObsThrd->hFuzzInput, (void **)&pExecCtx->pbInputCur, &pExecCtx->cbInputLeft);
755 if (RT_SUCCESS(rc))
756 rc = rtFuzzObsExecCtxArgvPrepare(pThis, pExecCtx, NULL);
757 }
758
759 if (RT_SUCCESS(rc))
760 {
761 RTPROCSTATUS ProcSts;
762 rc = rtFuzzObsExecCtxClientRun(pThis, pExecCtx, &ProcSts);
763 ASMAtomicIncU32(&pThis->Stats.cFuzzedInputs);
764 ASMAtomicIncU32(&pThis->Stats.cFuzzedInputsPerSec);
765
766 if (RT_SUCCESS(rc))
767 {
768 if (ProcSts.enmReason != RTPROCEXITREASON_NORMAL)
769 {
770 ASMAtomicIncU32(&pThis->Stats.cFuzzedInputsCrash);
771 rc = rtFuzzObsAddInputToResults(pThis, pObsThrd->hFuzzInput, pExecCtx);
772 }
773 }
774 else if (rc == VERR_TIMEOUT)
775 {
776 ASMAtomicIncU32(&pThis->Stats.cFuzzedInputsHang);
777 rc = rtFuzzObsAddInputToResults(pThis, pObsThrd->hFuzzInput, pExecCtx);
778 }
779 else
780 AssertFailed();
781
782 if (pThis->fFlags & RTFUZZ_OBS_BINARY_F_INPUT_FILE)
783 RTFileDelete(&szInput[0]);
784 }
785
786 ASMAtomicBitSet(&pThis->bmEvt, pObsThrd->idObs);
787 RTSemEventSignal(pThis->hEvtGlobal);
788 }
789
790 rtFuzzObsExecCtxDestroy(pThis, pExecCtx);
791 return VINF_SUCCESS;
792}
793
794
795/**
796 * Fuzzing observer master worker loop.
797 *
798 * @returns IPRT status code.
799 * @param hThread The thread handle.
800 * @param pvUser Opaque user data.
801 */
802static DECLCALLBACK(int) rtFuzzObsMasterLoop(RTTHREAD hThread, void *pvUser)
803{
804 RT_NOREF(hThread);
805 int rc = VINF_SUCCESS;
806 PRTFUZZOBSINT pThis = (PRTFUZZOBSINT)pvUser;
807
808 RTThreadUserSignal(hThread);
809
810 while ( !pThis->fShutdown
811 && RT_SUCCESS(rc))
812 {
813 uint64_t bmEvt = ASMAtomicXchgU64(&pThis->bmEvt, 0);
814 uint32_t idxObs = 0;
815 while (bmEvt != 0)
816 {
817 if (bmEvt & 0x1)
818 {
819 /* Create a new input for this observer and kick it. */
820 PRTFUZZOBSTHRD pObsThrd = &pThis->paObsThreads[idxObs];
821
822 /* Release the old input. */
823 if (pObsThrd->hFuzzInput)
824 {
825 if (pObsThrd->fKeepInput)
826 {
827 int rc2 = RTFuzzInputAddToCtxCorpus(pObsThrd->hFuzzInput);
828 Assert(RT_SUCCESS(rc2) || rc2 == VERR_ALREADY_EXISTS); RT_NOREF(rc2);
829 pObsThrd->fKeepInput= false;
830 }
831 RTFuzzInputRelease(pObsThrd->hFuzzInput);
832 }
833
834 rc = RTFuzzCtxInputGenerate(pThis->hFuzzCtx, &pObsThrd->hFuzzInput);
835 if (RT_SUCCESS(rc))
836 {
837 ASMAtomicWriteBool(&pObsThrd->fNewInput, true);
838 RTThreadUserSignal(pObsThrd->hThread);
839 }
840 }
841
842 idxObs++;
843 bmEvt >>= 1;
844 }
845
846 rc = RTSemEventWait(pThis->hEvtGlobal, RT_INDEFINITE_WAIT);
847 }
848
849 return VINF_SUCCESS;
850}
851
852
853/**
854 * Initializes the given worker thread structure.
855 *
856 * @returns IPRT status code.
857 * @param pThis The internal fuzzing observer state.
858 * @param iObs Observer ID.
859 * @param pObsThrd The observer thread structure.
860 */
861static int rtFuzzObsWorkerThreadInit(PRTFUZZOBSINT pThis, uint32_t idObs, PRTFUZZOBSTHRD pObsThrd)
862{
863 pObsThrd->pFuzzObs = pThis;
864 pObsThrd->hFuzzInput = NULL;
865 pObsThrd->idObs = idObs;
866 pObsThrd->fShutdown = false;
867
868 ASMAtomicBitSet(&pThis->bmEvt, idObs);
869 return RTThreadCreate(&pObsThrd->hThread, rtFuzzObsWorkerLoop, pObsThrd, 0, RTTHREADTYPE_IO,
870 RTTHREADFLAGS_WAITABLE, "Fuzz-Worker");
871}
872
873
874/**
875 * Creates the given amount of worker threads and puts them into waiting state.
876 *
877 * @returns IPRT status code.
878 * @param pThis The internal fuzzing observer state.
879 * @param cThreads Number of worker threads to create.
880 */
881static int rtFuzzObsWorkersCreate(PRTFUZZOBSINT pThis, uint32_t cThreads)
882{
883 int rc = VINF_SUCCESS;
884 PRTFUZZOBSTHRD paObsThreads = (PRTFUZZOBSTHRD)RTMemAllocZ(cThreads * sizeof(RTFUZZOBSTHRD));
885 if (RT_LIKELY(paObsThreads))
886 {
887 for (unsigned i = 0; i < cThreads && RT_SUCCESS(rc); i++)
888 {
889 rc = rtFuzzObsWorkerThreadInit(pThis, i, &paObsThreads[i]);
890 if (RT_FAILURE(rc))
891 {
892 /* Rollback. */
893
894 }
895 }
896
897 if (RT_SUCCESS(rc))
898 {
899 pThis->paObsThreads = paObsThreads;
900 pThis->cThreads = cThreads;
901 }
902 else
903 RTMemFree(paObsThreads);
904 }
905
906 return rc;
907}
908
909
910/**
911 * Creates the global worker thread managing the input creation and other worker threads.
912 *
913 * @returns IPRT status code.
914 * @param pThis The internal fuzzing observer state.
915 */
916static int rtFuzzObsMasterCreate(PRTFUZZOBSINT pThis)
917{
918 pThis->fShutdown = false;
919
920 int rc = RTSemEventCreate(&pThis->hEvtGlobal);
921 if (RT_SUCCESS(rc))
922 {
923 rc = RTThreadCreate(&pThis->hThreadGlobal, rtFuzzObsMasterLoop, pThis, 0, RTTHREADTYPE_IO,
924 RTTHREADFLAGS_WAITABLE, "Fuzz-Master");
925 if (RT_SUCCESS(rc))
926 {
927 RTThreadUserWait(pThis->hThreadGlobal, RT_INDEFINITE_WAIT);
928 }
929 else
930 {
931 RTSemEventDestroy(pThis->hEvtGlobal);
932 pThis->hEvtGlobal = NIL_RTSEMEVENT;
933 }
934 }
935
936 return rc;
937}
938
939
940RTDECL(int) RTFuzzObsCreate(PRTFUZZOBS phFuzzObs)
941{
942 AssertPtrReturn(phFuzzObs, VERR_INVALID_POINTER);
943
944 int rc = VINF_SUCCESS;
945 PRTFUZZOBSINT pThis = (PRTFUZZOBSINT)RTMemAllocZ(sizeof(*pThis));
946 if (RT_LIKELY(pThis))
947 {
948 pThis->pszBinary = NULL;
949 pThis->papszArgs = NULL;
950 pThis->fFlags = 0;
951 pThis->msWaitMax = 1000;
952 pThis->hThreadGlobal = NIL_RTTHREAD;
953 pThis->hEvtGlobal = NIL_RTSEMEVENT;
954 pThis->bmEvt = 0;
955 pThis->cThreads = 0;
956 pThis->paObsThreads = NULL;
957 pThis->tsLastStats = RTTimeMilliTS();
958 pThis->Stats.cFuzzedInputsPerSec = 0;
959 pThis->Stats.cFuzzedInputs = 0;
960 pThis->Stats.cFuzzedInputsHang = 0;
961 pThis->Stats.cFuzzedInputsCrash = 0;
962 rc = RTFuzzCtxCreate(&pThis->hFuzzCtx);
963 if (RT_SUCCESS(rc))
964 {
965 *phFuzzObs = pThis;
966 return VINF_SUCCESS;
967 }
968
969 RTMemFree(pThis);
970 }
971 else
972 rc = VERR_NO_MEMORY;
973
974 return rc;
975}
976
977
978RTDECL(int) RTFuzzObsDestroy(RTFUZZOBS hFuzzObs)
979{
980 PRTFUZZOBSINT pThis = hFuzzObs;
981 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
982
983 RTFuzzObsExecStop(hFuzzObs);
984
985 /* Clean up all acquired resources. */
986 for (unsigned i = 0; i < pThis->cArgs; i++)
987 RTStrFree(pThis->papszArgs[i]);
988
989 RTMemFree(pThis->papszArgs);
990
991 if (pThis->hEvtGlobal != NIL_RTSEMEVENT)
992 RTSemEventDestroy(pThis->hEvtGlobal);
993
994 if (pThis->pszResultsDir)
995 RTStrFree(pThis->pszResultsDir);
996 if (pThis->pszTmpDir)
997 RTStrFree(pThis->pszTmpDir);
998 if (pThis->pszBinary)
999 RTStrFree(pThis->pszBinary);
1000 RTFuzzCtxRelease(pThis->hFuzzCtx);
1001 RTMemFree(pThis);
1002 return VINF_SUCCESS;
1003}
1004
1005
1006RTDECL(int) RTFuzzObsQueryCtx(RTFUZZOBS hFuzzObs, PRTFUZZCTX phFuzzCtx)
1007{
1008 PRTFUZZOBSINT pThis = hFuzzObs;
1009 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1010 AssertPtrReturn(phFuzzCtx, VERR_INVALID_POINTER);
1011
1012 RTFuzzCtxRetain(pThis->hFuzzCtx);
1013 *phFuzzCtx = pThis->hFuzzCtx;
1014 return VINF_SUCCESS;
1015}
1016
1017
1018RTDECL(int) RTFuzzObsQueryStats(RTFUZZOBS hFuzzObs, PRTFUZZOBSSTATS pStats)
1019{
1020 PRTFUZZOBSINT pThis = hFuzzObs;
1021 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1022 AssertPtrReturn(pStats, VERR_INVALID_POINTER);
1023
1024 uint64_t tsStatsQuery = RTTimeMilliTS();
1025 uint32_t cFuzzedInputsPerSec = ASMAtomicXchgU32(&pThis->Stats.cFuzzedInputsPerSec, 0);
1026
1027 pStats->cFuzzedInputsCrash = ASMAtomicReadU32(&pThis->Stats.cFuzzedInputsCrash);
1028 pStats->cFuzzedInputsHang = ASMAtomicReadU32(&pThis->Stats.cFuzzedInputsHang);
1029 pStats->cFuzzedInputs = ASMAtomicReadU32(&pThis->Stats.cFuzzedInputs);
1030 uint64_t cPeriodSec = (tsStatsQuery - pThis->tsLastStats) / 1000;
1031 if (cPeriodSec)
1032 {
1033 pStats->cFuzzedInputsPerSec = cFuzzedInputsPerSec / cPeriodSec;
1034 pThis->cFuzzedInputsPerSecLast = pStats->cFuzzedInputsPerSec;
1035 pThis->tsLastStats = tsStatsQuery;
1036 }
1037 else
1038 pStats->cFuzzedInputsPerSec = pThis->cFuzzedInputsPerSecLast;
1039 return VINF_SUCCESS;
1040}
1041
1042
1043RTDECL(int) RTFuzzObsSetTmpDirectory(RTFUZZOBS hFuzzObs, const char *pszTmp)
1044{
1045 PRTFUZZOBSINT pThis = hFuzzObs;
1046 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1047 AssertPtrReturn(pszTmp, VERR_INVALID_POINTER);
1048
1049 int rc = VINF_SUCCESS;
1050 pThis->pszTmpDir = RTStrDup(pszTmp);
1051 if (!pThis->pszTmpDir)
1052 rc = VERR_NO_STR_MEMORY;
1053 return rc;
1054}
1055
1056
1057RTDECL(int) RTFuzzObsSetResultDirectory(RTFUZZOBS hFuzzObs, const char *pszResults)
1058{
1059 PRTFUZZOBSINT pThis = hFuzzObs;
1060 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1061 AssertPtrReturn(pszResults, VERR_INVALID_POINTER);
1062
1063 int rc = VINF_SUCCESS;
1064 pThis->pszResultsDir = RTStrDup(pszResults);
1065 if (!pThis->pszResultsDir)
1066 rc = VERR_NO_STR_MEMORY;
1067 return rc;
1068}
1069
1070
1071RTDECL(int) RTFuzzObsSetTestBinary(RTFUZZOBS hFuzzObs, const char *pszBinary, uint32_t fFlags)
1072{
1073 PRTFUZZOBSINT pThis = hFuzzObs;
1074 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1075 AssertPtrReturn(pszBinary, VERR_INVALID_POINTER);
1076
1077 int rc = VINF_SUCCESS;
1078 pThis->fFlags = fFlags;
1079 pThis->pszBinary = RTStrDup(pszBinary);
1080 if (RT_UNLIKELY(!pThis->pszBinary))
1081 rc = VERR_NO_STR_MEMORY;
1082 return rc;
1083}
1084
1085
1086RTDECL(int) RTFuzzObsSetTestBinaryArgs(RTFUZZOBS hFuzzObs, const char * const *papszArgs, unsigned cArgs)
1087{
1088 PRTFUZZOBSINT pThis = hFuzzObs;
1089 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1090
1091 int rc = VINF_SUCCESS;
1092 char **papszArgsOld = pThis->papszArgs;
1093 if (papszArgs)
1094 {
1095 pThis->papszArgs = (char **)RTMemAllocZ(sizeof(char **) * (cArgs + 1));
1096 if (RT_LIKELY(pThis->papszArgs))
1097 {
1098 for (unsigned i = 0; i < cArgs; i++)
1099 {
1100 pThis->papszArgs[i] = RTStrDup(papszArgs[i]);
1101 if (RT_UNLIKELY(!pThis->papszArgs[i]))
1102 {
1103 while (i > 0)
1104 {
1105 i--;
1106 RTStrFree(pThis->papszArgs[i]);
1107 }
1108 break;
1109 }
1110 }
1111
1112 if (RT_FAILURE(rc))
1113 RTMemFree(pThis->papszArgs);
1114 }
1115 else
1116 rc = VERR_NO_MEMORY;
1117
1118 if (RT_FAILURE(rc))
1119 pThis->papszArgs = papszArgsOld;
1120 else
1121 pThis->cArgs = cArgs;
1122 }
1123 else
1124 {
1125 pThis->papszArgs = NULL;
1126 pThis->cArgs = 0;
1127 if (papszArgsOld)
1128 {
1129 char **ppsz = papszArgsOld;
1130 while (*ppsz != NULL)
1131 {
1132 RTStrFree(*ppsz);
1133 ppsz++;
1134 }
1135 RTMemFree(papszArgsOld);
1136 }
1137 }
1138
1139 return rc;
1140}
1141
1142
1143RTDECL(int) RTFuzzObsExecStart(RTFUZZOBS hFuzzObs, uint32_t cProcs)
1144{
1145 PRTFUZZOBSINT pThis = hFuzzObs;
1146 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1147 AssertReturn(cProcs <= sizeof(uint64_t) * 8, VERR_INVALID_PARAMETER);
1148 AssertReturn( (pThis->fFlags & RTFUZZ_OBS_BINARY_F_INPUT_FILE)
1149 || pThis->pszTmpDir != NULL,
1150 VERR_INVALID_STATE);
1151
1152 int rc = VINF_SUCCESS;
1153 if (!cProcs)
1154 cProcs = RT_MIN(RTMpGetPresentCoreCount(), sizeof(uint64_t) * 8);
1155
1156 /* Spin up the worker threads first. */
1157 rc = rtFuzzObsWorkersCreate(pThis, cProcs);
1158 if (RT_SUCCESS(rc))
1159 {
1160 /* Spin up the global thread. */
1161 rc = rtFuzzObsMasterCreate(pThis);
1162 }
1163
1164 return rc;
1165}
1166
1167
1168RTDECL(int) RTFuzzObsExecStop(RTFUZZOBS hFuzzObs)
1169{
1170 PRTFUZZOBSINT pThis = hFuzzObs;
1171 AssertPtrReturn(pThis, VERR_INVALID_HANDLE);
1172
1173 /* Wait for the master thread to terminate. */
1174 if (pThis->hThreadGlobal != NIL_RTTHREAD)
1175 {
1176 ASMAtomicXchgBool(&pThis->fShutdown, true);
1177 RTSemEventSignal(pThis->hEvtGlobal);
1178 RTThreadWait(pThis->hThreadGlobal, RT_INDEFINITE_WAIT, NULL);
1179 pThis->hThreadGlobal = NIL_RTTHREAD;
1180 }
1181
1182 /* Destroy the workers. */
1183 if (pThis->paObsThreads)
1184 {
1185 for (unsigned i = 0; i < pThis->cThreads; i++)
1186 {
1187 PRTFUZZOBSTHRD pThrd = &pThis->paObsThreads[i];
1188 ASMAtomicXchgBool(&pThrd->fShutdown, true);
1189 RTThreadUserSignal(pThrd->hThread);
1190 RTThreadWait(pThrd->hThread, RT_INDEFINITE_WAIT, NULL);
1191 if (pThrd->hFuzzInput)
1192 RTFuzzInputRelease(pThrd->hFuzzInput);
1193 }
1194
1195 RTMemFree(pThis->paObsThreads);
1196 pThis->paObsThreads = NULL;
1197 pThis->cThreads = 0;
1198 }
1199
1200 RTSemEventDestroy(pThis->hEvtGlobal);
1201 pThis->hEvtGlobal = NIL_RTSEMEVENT;
1202 return VINF_SUCCESS;
1203}
1204
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette