VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR3/FTM.cpp@ 57009

Last change on this file since 57009 was 56287, checked in by vboxsync, 10 years ago

VMM: Updated (C) year.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 42.3 KB
Line 
1/* $Id: FTM.cpp 56287 2015-06-09 11:15:22Z vboxsync $ */
2/** @file
3 * FTM - Fault Tolerance Manager
4 */
5
6/*
7 * Copyright (C) 2010-2015 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*******************************************************************************
20* Header Files *
21*******************************************************************************/
22#define LOG_GROUP LOG_GROUP_FTM
23#include <VBox/vmm/ftm.h>
24#include <VBox/vmm/em.h>
25#include <VBox/vmm/pdm.h>
26#include <VBox/vmm/pgm.h>
27#include <VBox/vmm/ssm.h>
28#include <VBox/vmm/vmm.h>
29#include "FTMInternal.h"
30#include <VBox/vmm/vm.h>
31#include <VBox/vmm/uvm.h>
32#include <VBox/err.h>
33#include <VBox/param.h>
34#include <VBox/log.h>
35
36#include <iprt/assert.h>
37#include <iprt/thread.h>
38#include <iprt/string.h>
39#include <iprt/mem.h>
40#include <iprt/tcp.h>
41#include <iprt/socket.h>
42#include <iprt/semaphore.h>
43#include <iprt/asm.h>
44
45#include "internal/pgm.h"
46
47
48/*******************************************************************************
49 * Structures and Typedefs *
50 *******************************************************************************/
51
52/**
53 * TCP stream header.
54 *
55 * This is an extra layer for fixing the problem with figuring out when the SSM
56 * stream ends.
57 */
58typedef struct FTMTCPHDR
59{
60 /** Magic value. */
61 uint32_t u32Magic;
62 /** The size of the data block following this header.
63 * 0 indicates the end of the stream, while UINT32_MAX indicates
64 * cancelation. */
65 uint32_t cb;
66} FTMTCPHDR;
67/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
68#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
69/** The max block size. */
70#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
71
72/**
73 * TCP stream header.
74 *
75 * This is an extra layer for fixing the problem with figuring out when the SSM
76 * stream ends.
77 */
78typedef struct FTMTCPHDRMEM
79{
80 /** Magic value. */
81 uint32_t u32Magic;
82 /** Size (Uncompressed) of the pages following the header. */
83 uint32_t cbPageRange;
84 /** GC Physical address of the page(s) to sync. */
85 RTGCPHYS GCPhys;
86 /** The size of the data block following this header.
87 * 0 indicates the end of the stream, while UINT32_MAX indicates
88 * cancelation. */
89 uint32_t cb;
90} FTMTCPHDRMEM;
91
92/*******************************************************************************
93* Global Variables *
94*******************************************************************************/
95static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
96
97static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser);
98
99/**
100 * Initializes the FTM.
101 *
102 * @returns VBox status code.
103 * @param pVM Pointer to the VM.
104 */
105VMMR3_INT_DECL(int) FTMR3Init(PVM pVM)
106{
107 /*
108 * Assert alignment and sizes.
109 */
110 AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
111 AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
112
113 /** @todo saved state for master nodes! */
114 pVM->ftm.s.pszAddress = NULL;
115 pVM->ftm.s.pszPassword = NULL;
116 pVM->fFaultTolerantMaster = false;
117 pVM->ftm.s.fIsStandbyNode = false;
118 pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
119 pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
120 pVM->ftm.s.hSocket = NIL_RTSOCKET;
121
122 /*
123 * Initialize the PGM critical section.
124 */
125 int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
126 AssertRCReturn(rc, rc);
127
128 /*
129 * Register statistics.
130 */
131 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
132 STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
133 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
134 STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
135 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
136 STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
137 STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
138 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
139 STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
140#ifdef VBOX_WITH_STATISTICS
141 STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
142 STAM_REG(pVM, &pVM->ftm.s.StatCheckpointPause, STAMTYPE_PROFILE, "/FT/Checkpoint/Pause", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
143 STAM_REG(pVM, &pVM->ftm.s.StatCheckpointResume, STAMTYPE_PROFILE, "/FT/Checkpoint/Resume", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
144 STAM_REG(pVM, &pVM->ftm.s.StatSentMemRAM, STAMTYPE_COUNTER, "/FT/Sent/Mem/RAM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
145 STAM_REG(pVM, &pVM->ftm.s.StatSentMemMMIO2, STAMTYPE_COUNTER, "/FT/Sent/Mem/MMIO2", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
146 STAM_REG(pVM, &pVM->ftm.s.StatSentMemShwROM, STAMTYPE_COUNTER, "/FT/Sent/Mem/ShwROM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
147 STAM_REG(pVM, &pVM->ftm.s.StatSentStateWrite, STAMTYPE_COUNTER, "/FT/Sent/State/Writes", STAMUNIT_BYTES, "The nr of write calls.");
148#endif
149 return VINF_SUCCESS;
150}
151
152/**
153 * Terminates the FTM.
154 *
155 * Termination means cleaning up and freeing all resources,
156 * the VM itself is at this point powered off or suspended.
157 *
158 * @returns VBox status code.
159 * @param pVM Pointer to the VM.
160 */
161VMMR3_INT_DECL(int) FTMR3Term(PVM pVM)
162{
163 if (pVM->ftm.s.hShutdownEvent != NIL_RTSEMEVENT)
164 {
165 RTSemEventDestroy(pVM->ftm.s.hShutdownEvent);
166 pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
167 }
168 if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
169 {
170 RTTcpClientClose(pVM->ftm.s.hSocket);
171 pVM->ftm.s.hSocket = NIL_RTSOCKET;
172 }
173 if (pVM->ftm.s.standby.hServer)
174 {
175 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
176 pVM->ftm.s.standby.hServer = NULL;
177 }
178 if (pVM->ftm.s.pszAddress)
179 RTMemFree(pVM->ftm.s.pszAddress);
180 if (pVM->ftm.s.pszPassword)
181 RTMemFree(pVM->ftm.s.pszPassword);
182
183 /* Remove all pending memory updates. */
184 if (pVM->ftm.s.standby.pPhysPageTree)
185 {
186 RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, NULL);
187 pVM->ftm.s.standby.pPhysPageTree = NULL;
188 }
189
190 pVM->ftm.s.pszAddress = NULL;
191 pVM->ftm.s.pszPassword = NULL;
192
193 PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
194 return VINF_SUCCESS;
195}
196
197
198static int ftmR3TcpWriteACK(PVM pVM)
199{
200 int rc = RTTcpWrite(pVM->ftm.s.hSocket, RT_STR_TUPLE("ACK\n"));
201 if (RT_FAILURE(rc))
202 {
203 LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
204 }
205 return rc;
206}
207
208
209static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
210{
211 char szMsg[256];
212 size_t cch;
213 if (pszMsgText && *pszMsgText)
214 {
215 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
216 for (size_t off = 6; off + 1 < cch; off++)
217 if (szMsg[off] == '\n')
218 szMsg[off] = '\r';
219 }
220 else
221 cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
222 int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
223 if (RT_FAILURE(rc))
224 LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
225 return rc;
226}
227
228/**
229 * Reads a string from the socket.
230 *
231 * @returns VBox status code.
232 *
233 * @param pState The teleporter state structure.
234 * @param pszBuf The output buffer.
235 * @param cchBuf The size of the output buffer.
236 *
237 */
238static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
239{
240 char *pszStart = pszBuf;
241 RTSOCKET Sock = pVM->ftm.s.hSocket;
242
243 AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
244 *pszBuf = '\0';
245
246 /* dead simple approach. */
247 for (;;)
248 {
249 char ch;
250 int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
251 if (RT_FAILURE(rc))
252 {
253 LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
254 return rc;
255 }
256 if ( ch == '\n'
257 || ch == '\0')
258 return VINF_SUCCESS;
259 if (cchBuf <= 1)
260 {
261 LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
262 return VERR_BUFFER_OVERFLOW;
263 }
264 *pszBuf++ = ch;
265 *pszBuf = '\0';
266 cchBuf--;
267 }
268}
269
270/**
271 * Reads an ACK or NACK.
272 *
273 * @returns VBox status code.
274 * @param pVM Pointer to the VM.
275 * @param pszWhich Which ACK is this this?
276 * @param pszNAckMsg Optional NACK message.
277 */
278static int ftmR3TcpReadACK(PVM pVM, const char *pszWhich, const char *pszNAckMsg = NULL)
279{
280 char szMsg[256];
281 int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
282 if (RT_FAILURE(rc))
283 return rc;
284
285 if (!strcmp(szMsg, "ACK"))
286 return VINF_SUCCESS;
287
288 if (!strncmp(szMsg, RT_STR_TUPLE("NACK=")))
289 {
290 char *pszMsgText = strchr(szMsg, ';');
291 if (pszMsgText)
292 *pszMsgText++ = '\0';
293
294 int32_t vrc2;
295 rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
296 if (rc == VINF_SUCCESS)
297 {
298 /*
299 * Well formed NACK, transform it into an error.
300 */
301 if (pszNAckMsg)
302 {
303 LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
304 return VERR_INTERNAL_ERROR;
305 }
306
307 if (pszMsgText)
308 {
309 pszMsgText = RTStrStrip(pszMsgText);
310 for (size_t off = 0; pszMsgText[off]; off++)
311 if (pszMsgText[off] == '\r')
312 pszMsgText[off] = '\n';
313
314 LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
315 }
316 return VERR_INTERNAL_ERROR_2;
317 }
318
319 if (pszMsgText)
320 pszMsgText[-1] = ';';
321 }
322 return VERR_INTERNAL_ERROR_3;
323}
324
325/**
326 * Submitts a command to the destination and waits for the ACK.
327 *
328 * @returns VBox status code.
329 *
330 * @param pVM Pointer to the VM.
331 * @param pszCommand The command.
332 * @param fWaitForAck Whether to wait for the ACK.
333 */
334static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
335{
336 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), RT_STR_TUPLE("\n"));
337 if (RT_FAILURE(rc))
338 return rc;
339 if (!fWaitForAck)
340 return VINF_SUCCESS;
341 return ftmR3TcpReadACK(pVM, pszCommand);
342}
343
344/**
345 * @copydoc SSMSTRMOPS::pfnWrite
346 */
347static DECLCALLBACK(int) ftmR3TcpOpWrite(void *pvUser, uint64_t offStream, const void *pvBuf, size_t cbToWrite)
348{
349 PVM pVM = (PVM)pvUser;
350 NOREF(offStream);
351
352 AssertReturn(cbToWrite > 0, VINF_SUCCESS);
353 AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
354 AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
355
356 STAM_COUNTER_INC(&pVM->ftm.s.StatSentStateWrite);
357 for (;;)
358 {
359 FTMTCPHDR Hdr;
360 Hdr.u32Magic = FTMTCPHDR_MAGIC;
361 Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
362 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
363 if (RT_FAILURE(rc))
364 {
365 LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
366 return rc;
367 }
368 pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
369 pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
370 if (Hdr.cb == cbToWrite)
371 return VINF_SUCCESS;
372
373 /* advance */
374 cbToWrite -= Hdr.cb;
375 pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
376 }
377}
378
379
380/**
381 * Selects and poll for close condition.
382 *
383 * We can use a relatively high poll timeout here since it's only used to get
384 * us out of error paths. In the normal cause of events, we'll get a
385 * end-of-stream header.
386 *
387 * @returns VBox status code.
388 *
389 * @param pState The teleporter state data.
390 */
391static int ftmR3TcpReadSelect(PVM pVM)
392{
393 int rc;
394 do
395 {
396 rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
397 if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
398 {
399 pVM->ftm.s.syncstate.fIOError = true;
400 LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
401 break;
402 }
403 if (pVM->ftm.s.syncstate.fStopReading)
404 {
405 rc = VERR_EOF;
406 break;
407 }
408 } while (rc == VERR_TIMEOUT);
409 return rc;
410}
411
412
413/**
414 * @copydoc SSMSTRMOPS::pfnRead
415 */
416static DECLCALLBACK(int) ftmR3TcpOpRead(void *pvUser, uint64_t offStream, void *pvBuf, size_t cbToRead, size_t *pcbRead)
417{
418 PVM pVM = (PVM)pvUser;
419 AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
420 NOREF(offStream);
421
422 for (;;)
423 {
424 int rc;
425
426 /*
427 * Check for various conditions and may have been signalled.
428 */
429 if (pVM->ftm.s.syncstate.fEndOfStream)
430 return VERR_EOF;
431 if (pVM->ftm.s.syncstate.fStopReading)
432 return VERR_EOF;
433 if (pVM->ftm.s.syncstate.fIOError)
434 return VERR_IO_GEN_FAILURE;
435
436 /*
437 * If there is no more data in the current block, read the next
438 * block header.
439 */
440 if (!pVM->ftm.s.syncstate.cbReadBlock)
441 {
442 rc = ftmR3TcpReadSelect(pVM);
443 if (RT_FAILURE(rc))
444 return rc;
445 FTMTCPHDR Hdr;
446 rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
447 if (RT_FAILURE(rc))
448 {
449 pVM->ftm.s.syncstate.fIOError = true;
450 LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
451 return rc;
452 }
453 pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
454
455 if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
456 || Hdr.cb > FTMTCPHDR_MAX_SIZE
457 || Hdr.cb == 0))
458 {
459 if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
460 && ( Hdr.cb == 0
461 || Hdr.cb == UINT32_MAX)
462 )
463 {
464 pVM->ftm.s.syncstate.fEndOfStream = true;
465 pVM->ftm.s.syncstate.cbReadBlock = 0;
466 return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
467 }
468 pVM->ftm.s.syncstate.fIOError = true;
469 LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
470 return VERR_IO_GEN_FAILURE;
471 }
472
473 pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
474 if (pVM->ftm.s.syncstate.fStopReading)
475 return VERR_EOF;
476 }
477
478 /*
479 * Read more data.
480 */
481 rc = ftmR3TcpReadSelect(pVM);
482 if (RT_FAILURE(rc))
483 return rc;
484
485 uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
486 rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
487 if (RT_FAILURE(rc))
488 {
489 pVM->ftm.s.syncstate.fIOError = true;
490 LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
491 return rc;
492 }
493 if (pcbRead)
494 {
495 cb = (uint32_t)*pcbRead;
496 pVM->ftm.s.StatReceivedState.c += cb;
497 pVM->ftm.s.syncstate.uOffStream += cb;
498 pVM->ftm.s.syncstate.cbReadBlock -= cb;
499 return VINF_SUCCESS;
500 }
501 pVM->ftm.s.StatReceivedState.c += cb;
502 pVM->ftm.s.syncstate.uOffStream += cb;
503 pVM->ftm.s.syncstate.cbReadBlock -= cb;
504 if (cbToRead == cb)
505 return VINF_SUCCESS;
506
507 /* Advance to the next block. */
508 cbToRead -= cb;
509 pvBuf = (uint8_t *)pvBuf + cb;
510 }
511}
512
513
514/**
515 * @copydoc SSMSTRMOPS::pfnSeek
516 */
517static DECLCALLBACK(int) ftmR3TcpOpSeek(void *pvUser, int64_t offSeek, unsigned uMethod, uint64_t *poffActual)
518{
519 NOREF(pvUser); NOREF(offSeek); NOREF(uMethod); NOREF(poffActual);
520 return VERR_NOT_SUPPORTED;
521}
522
523
524/**
525 * @copydoc SSMSTRMOPS::pfnTell
526 */
527static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
528{
529 PVM pVM = (PVM)pvUser;
530 return pVM->ftm.s.syncstate.uOffStream;
531}
532
533
534/**
535 * @copydoc SSMSTRMOPS::pfnSize
536 */
537static DECLCALLBACK(int) ftmR3TcpOpSize(void *pvUser, uint64_t *pcb)
538{
539 NOREF(pvUser); NOREF(pcb);
540 return VERR_NOT_SUPPORTED;
541}
542
543
544/**
545 * @copydoc SSMSTRMOPS::pfnIsOk
546 */
547static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
548{
549 PVM pVM = (PVM)pvUser;
550
551 if (pVM->fFaultTolerantMaster)
552 {
553 /* Poll for incoming NACKs and errors from the other side */
554 int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
555 if (rc != VERR_TIMEOUT)
556 {
557 if (RT_SUCCESS(rc))
558 {
559 LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
560 rc = VERR_SSM_CANCELLED;
561 }
562 else
563 LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
564 return rc;
565 }
566 }
567
568 return VINF_SUCCESS;
569}
570
571
572/**
573 * @copydoc SSMSTRMOPS::pfnClose
574 */
575static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
576{
577 PVM pVM = (PVM)pvUser;
578
579 if (pVM->fFaultTolerantMaster)
580 {
581 FTMTCPHDR EofHdr;
582 EofHdr.u32Magic = FTMTCPHDR_MAGIC;
583 EofHdr.cb = fCanceled ? UINT32_MAX : 0;
584 int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
585 if (RT_FAILURE(rc))
586 {
587 LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
588 return rc;
589 }
590 }
591 else
592 {
593 ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
594 }
595
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Method table for a TCP based stream.
602 */
603static SSMSTRMOPS const g_ftmR3TcpOps =
604{
605 SSMSTRMOPS_VERSION,
606 ftmR3TcpOpWrite,
607 ftmR3TcpOpRead,
608 ftmR3TcpOpSeek,
609 ftmR3TcpOpTell,
610 ftmR3TcpOpSize,
611 ftmR3TcpOpIsOk,
612 ftmR3TcpOpClose,
613 SSMSTRMOPS_VERSION
614};
615
616
617/**
618 * VMR3ReqCallWait callback
619 *
620 * @param pVM Pointer to the VM.
621 *
622 */
623static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
624{
625 int rc = PGMR3PhysWriteProtectRAM(pVM);
626 AssertRC(rc);
627}
628
629
630/**
631 * Sync the VM state
632 *
633 * @returns VBox status code.
634 * @param pVM Pointer to the VM.
635 */
636static int ftmR3PerformFullSync(PVM pVM)
637{
638 bool fSuspended = false;
639
640 int rc = VMR3Suspend(pVM->pUVM, VMSUSPENDREASON_FTM_SYNC);
641 AssertRCReturn(rc, rc);
642
643 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
644
645 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
646
647 /* Reset the sync state. */
648 pVM->ftm.s.syncstate.uOffStream = 0;
649 pVM->ftm.s.syncstate.cbReadBlock = 0;
650 pVM->ftm.s.syncstate.fStopReading = false;
651 pVM->ftm.s.syncstate.fIOError = false;
652 pVM->ftm.s.syncstate.fEndOfStream = false;
653
654 rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
655 AssertRC(rc);
656
657 pVM->ftm.s.fDeltaLoadSaveActive = false;
658 rc = VMR3SaveFT(pVM->pUVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
659 AssertRC(rc);
660
661 rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
662 AssertRC(rc);
663
664 RTSocketRelease(pVM->ftm.s.hSocket);
665
666 /* Write protect all memory. */
667 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
668 AssertRCReturn(rc, rc);
669
670 rc = VMR3Resume(pVM->pUVM, VMRESUMEREASON_FTM_SYNC);
671 AssertRC(rc);
672
673 return rc;
674}
675
676
677/**
678 * PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
679 *
680 * @param pVM Pointer to the VM.
681 * @param GCPhys GC physical address
682 * @param pRange HC virtual address of the page(s)
683 * @param cbRange Size of the dirty range in bytes.
684 * @param pvUser User argument
685 */
686static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t *pRange, unsigned cbRange, void *pvUser)
687{
688 NOREF(pvUser);
689 FTMTCPHDRMEM Hdr;
690 Hdr.u32Magic = FTMTCPHDR_MAGIC;
691 Hdr.GCPhys = GCPhys;
692 Hdr.cbPageRange = cbRange;
693 Hdr.cb = cbRange;
694 /** @todo compress page(s). */
695 int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
696 if (RT_FAILURE(rc))
697 {
698 LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
699 return rc;
700 }
701 pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
702
703#ifdef VBOX_WITH_STATISTICS
704 switch (PGMPhysGetPageType(pVM, GCPhys))
705 {
706 case PGMPAGETYPE_RAM:
707 pVM->ftm.s.StatSentMemRAM.c += Hdr.cb + sizeof(Hdr);
708 break;
709
710 case PGMPAGETYPE_MMIO2:
711 pVM->ftm.s.StatSentMemMMIO2.c += Hdr.cb + sizeof(Hdr);
712 break;
713
714 case PGMPAGETYPE_ROM_SHADOW:
715 pVM->ftm.s.StatSentMemShwROM.c += Hdr.cb + sizeof(Hdr);
716 break;
717
718 case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
719 case PGMPAGETYPE_SPECIAL_ALIAS_MMIO:
720 AssertFailed();
721 break;
722
723 default:
724 AssertFailed();
725 break;
726 }
727#endif
728
729 return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
730}
731
732/**
733 * Thread function which starts syncing process for this master VM
734 *
735 * @param hThread The thread handle.
736 * @param pvUser Pointer to the VM.
737 * @return VINF_SUCCESS (ignored).
738 *
739 */
740static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD hThread, void *pvUser)
741{
742 int rc = VINF_SUCCESS;
743 PVM pVM = (PVM)pvUser;
744 NOREF(hThread);
745
746 for (;;)
747 {
748 /*
749 * Try connect to the standby machine.
750 */
751 Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
752 rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
753 if (RT_SUCCESS(rc))
754 {
755 Log(("ftmR3MasterThread: CONNECTED\n"));
756
757 /* Disable Nagle. */
758 rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /*fEnable*/);
759 AssertRC(rc);
760
761 /* Read and check the welcome message. */
762 char szLine[RT_MAX(128, sizeof(g_szWelcome))];
763 RT_ZERO(szLine);
764 rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
765 if ( RT_SUCCESS(rc)
766 && !strcmp(szLine, g_szWelcome))
767 {
768 /* password */
769 if (pVM->ftm.s.pszPassword)
770 rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
771
772 if (RT_SUCCESS(rc))
773 {
774 /* ACK */
775 rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
776 if (RT_SUCCESS(rc))
777 {
778 /** todo: verify VM config. */
779 break;
780 }
781 }
782 }
783 /* Failed, so don't bother anymore. */
784 return VINF_SUCCESS;
785 }
786 rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, 1000 /* 1 second */);
787 if (rc != VERR_TIMEOUT)
788 return VINF_SUCCESS; /* told to quit */
789 }
790
791 /* Successfully initialized the connection to the standby node.
792 * Start the sync process.
793 */
794
795 /* First sync all memory and write protect everything so
796 * we can send changed pages later on.
797 */
798
799 rc = ftmR3PerformFullSync(pVM);
800
801 for (;;)
802 {
803 rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
804 if (rc != VERR_TIMEOUT)
805 break; /* told to quit */
806
807 if (!pVM->ftm.s.fCheckpointingActive)
808 {
809 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
810 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
811
812 rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
813 AssertRC(rc);
814
815 /* sync the changed memory with the standby node. */
816 /* Write protect all memory. */
817 if (!pVM->ftm.s.fCheckpointingActive)
818 {
819 rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
820 AssertRC(rc);
821 }
822
823 /* Enumerate all dirty pages and send them to the standby VM. */
824 if (!pVM->ftm.s.fCheckpointingActive)
825 {
826 rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
827 Assert(rc == VINF_SUCCESS || rc == VERR_INTERRUPTED);
828 }
829
830 /* Send last memory header to signal the end. */
831 FTMTCPHDRMEM Hdr;
832 Hdr.u32Magic = FTMTCPHDR_MAGIC;
833 Hdr.GCPhys = 0;
834 Hdr.cbPageRange = 0;
835 Hdr.cb = 0;
836 rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
837 if (RT_FAILURE(rc))
838 LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
839
840 rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
841 AssertRC(rc);
842
843 PDMCritSectLeave(&pVM->ftm.s.CritSect);
844 }
845 }
846 return rc;
847}
848
849/**
850 * Syncs memory from the master VM
851 *
852 * @returns VBox status code.
853 * @param pVM Pointer to the VM.
854 */
855static int ftmR3SyncMem(PVM pVM)
856{
857 while (true)
858 {
859 FTMTCPHDRMEM Hdr;
860 RTGCPHYS GCPhys;
861
862 /* Read memory header. */
863 int rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
864 if (RT_FAILURE(rc))
865 {
866 Log(("RTTcpRead failed with %Rrc\n", rc));
867 break;
868 }
869 pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
870
871 if (Hdr.cb == 0)
872 break; /* end of sync. */
873
874 Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
875 GCPhys = Hdr.GCPhys;
876
877 /* Must be a multiple of PAGE_SIZE. */
878 Assert((Hdr.cbPageRange & 0xfff) == 0);
879
880 while (Hdr.cbPageRange)
881 {
882 PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)RTAvlGCPhysGet(&pVM->ftm.s.standby.pPhysPageTree, GCPhys);
883 if (!pNode)
884 {
885 /* Allocate memory for the node and page. */
886 pNode = (PFTMPHYSPAGETREENODE)RTMemAllocZ(sizeof(*pNode) + PAGE_SIZE);
887 AssertBreak(pNode);
888
889 /* Insert the node into the tree. */
890 pNode->Core.Key = GCPhys;
891 pNode->pPage = (void *)(pNode + 1);
892 bool fRet = RTAvlGCPhysInsert(&pVM->ftm.s.standby.pPhysPageTree, &pNode->Core);
893 Assert(fRet); NOREF(fRet);
894 }
895
896 /* Fetch the page. */
897 rc = RTTcpRead(pVM->ftm.s.hSocket, pNode->pPage, PAGE_SIZE, NULL);
898 if (RT_FAILURE(rc))
899 {
900 Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
901 break;
902 }
903 pVM->ftm.s.StatReceivedMem.c += PAGE_SIZE;
904 Hdr.cbPageRange -= PAGE_SIZE;
905 GCPhys += PAGE_SIZE;
906 }
907 }
908 return VINF_SUCCESS;
909}
910
911
912/**
913 * Callback handler for RTAvlGCPhysDestroy
914 *
915 * @returns 0 to continue, otherwise stop
916 * @param pBaseNode Node to destroy
917 * @param pvUser Pointer to the VM.
918 */
919static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser)
920{
921 PVM pVM = (PVM)pvUser;
922 PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)pBaseNode;
923
924 if (pVM) /* NULL when the VM is destroyed. */
925 {
926 /* Update the guest memory of the standby VM. */
927 int rc = PGMR3PhysWriteExternal(pVM, pNode->Core.Key, pNode->pPage, PAGE_SIZE, PGMACCESSORIGIN_FTM);
928 AssertRC(rc);
929 }
930 RTMemFree(pNode);
931 return 0;
932}
933
934/**
935 * Thread function which monitors the health of the master VM
936 *
937 * @param hThread The thread handle.
938 * @param pvUser Pointer to the VM.
939 * @return VINF_SUCCESS (ignored).
940 *
941 */
942static DECLCALLBACK(int) ftmR3StandbyThread(RTTHREAD hThread, void *pvUser)
943{
944 PVM pVM = (PVM)pvUser;
945 NOREF(hThread);
946
947 for (;;)
948 {
949 uint64_t u64TimeNow;
950
951 int rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
952 if (rc != VERR_TIMEOUT)
953 break; /* told to quit */
954
955 if (pVM->ftm.s.standby.u64LastHeartbeat)
956 {
957 u64TimeNow = RTTimeMilliTS();
958
959 if (u64TimeNow > pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 4)
960 {
961 /* Timeout; prepare to fallover. */
962 LogRel(("FTSync: TIMEOUT (%RX64 vs %RX64 ms): activate standby VM!\n", u64TimeNow, pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 2));
963
964 pVM->ftm.s.fActivateStandby = true;
965 /** todo: prevent split-brain. */
966 break;
967 }
968 }
969 }
970
971 return VINF_SUCCESS;
972}
973
974
975/**
976 * Listen for incoming traffic destined for the standby VM.
977 *
978 * @copydoc FNRTTCPSERVE
979 *
980 * @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
981 */
982static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
983{
984 PVM pVM = (PVM)pvUser;
985
986 pVM->ftm.s.hSocket = Sock;
987
988 /*
989 * Disable Nagle.
990 */
991 int rc = RTTcpSetSendCoalescing(Sock, false /*fEnable*/);
992 AssertRC(rc);
993
994 /* Send the welcome message to the master node. */
995 rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
996 if (RT_FAILURE(rc))
997 {
998 LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
999 return VINF_SUCCESS;
1000 }
1001
1002 /*
1003 * Password.
1004 */
1005 const char *pszPassword = pVM->ftm.s.pszPassword;
1006 if (pszPassword)
1007 {
1008 unsigned off = 0;
1009 while (pszPassword[off])
1010 {
1011 char ch;
1012 rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
1013 if ( RT_FAILURE(rc)
1014 || pszPassword[off] != ch)
1015 {
1016 if (RT_FAILURE(rc))
1017 LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
1018 else
1019 LogRel(("FTSync: Invalid password (off=%u)\n", off));
1020 ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
1021 return VINF_SUCCESS;
1022 }
1023 off++;
1024 }
1025 }
1026 rc = ftmR3TcpWriteACK(pVM);
1027 if (RT_FAILURE(rc))
1028 return VINF_SUCCESS;
1029
1030 /** @todo verify VM config. */
1031
1032 /*
1033 * Stop the server.
1034 *
1035 * Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
1036 * to it we must not return that value!
1037 */
1038 RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1039
1040 /*
1041 * Command processing loop.
1042 */
1043 //bool fDone = false;
1044 for (;;)
1045 {
1046 bool fFullSync = false;
1047 char szCmd[128];
1048
1049 rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
1050 if (RT_FAILURE(rc))
1051 break;
1052
1053 pVM->ftm.s.standby.u64LastHeartbeat = RTTimeMilliTS();
1054 if (!strcmp(szCmd, "mem-sync"))
1055 {
1056 rc = ftmR3TcpWriteACK(pVM);
1057 AssertRC(rc);
1058 if (RT_FAILURE(rc))
1059 continue;
1060
1061 rc = ftmR3SyncMem(pVM);
1062 AssertRC(rc);
1063
1064 rc = ftmR3TcpWriteACK(pVM);
1065 AssertRC(rc);
1066 }
1067 else
1068 if ( !strcmp(szCmd, "checkpoint")
1069 || !strcmp(szCmd, "full-sync")
1070 || (fFullSync = true)) /* intended assignment */
1071 {
1072 rc = ftmR3TcpWriteACK(pVM);
1073 AssertRC(rc);
1074 if (RT_FAILURE(rc))
1075 continue;
1076
1077 /* Flush all pending memory updates. */
1078 if (pVM->ftm.s.standby.pPhysPageTree)
1079 {
1080 RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, pVM);
1081 pVM->ftm.s.standby.pPhysPageTree = NULL;
1082 }
1083
1084 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1085
1086 /* Reset the sync state. */
1087 pVM->ftm.s.syncstate.uOffStream = 0;
1088 pVM->ftm.s.syncstate.cbReadBlock = 0;
1089 pVM->ftm.s.syncstate.fStopReading = false;
1090 pVM->ftm.s.syncstate.fIOError = false;
1091 pVM->ftm.s.syncstate.fEndOfStream = false;
1092
1093 pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
1094 rc = VMR3LoadFromStreamFT(pVM->pUVM, &g_ftmR3TcpOps, pVM);
1095 pVM->ftm.s.fDeltaLoadSaveActive = false;
1096 RTSocketRelease(pVM->ftm.s.hSocket);
1097 AssertRC(rc);
1098 if (RT_FAILURE(rc))
1099 {
1100 LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
1101 ftmR3TcpWriteNACK(pVM, rc);
1102 continue;
1103 }
1104
1105 /* The EOS might not have been read, make sure it is. */
1106 pVM->ftm.s.syncstate.fStopReading = false;
1107 size_t cbRead;
1108 rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
1109 if (rc != VERR_EOF)
1110 {
1111 LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
1112 ftmR3TcpWriteNACK(pVM, rc);
1113 continue;
1114 }
1115
1116 rc = ftmR3TcpWriteACK(pVM);
1117 AssertRC(rc);
1118 }
1119 }
1120 LogFlowFunc(("returns mRc=%Rrc\n", rc));
1121 return VERR_TCP_SERVER_STOP;
1122}
1123
1124/**
1125 * Powers on the fault tolerant virtual machine.
1126 *
1127 * @returns VBox status code.
1128 *
1129 * @param pUVM The user mode VM handle.
1130 * @param fMaster FT master or standby
1131 * @param uInterval FT sync interval
1132 * @param pszAddress Standby VM address
1133 * @param uPort Standby VM port
1134 * @param pszPassword FT password (NULL for none)
1135 *
1136 * @thread Any thread.
1137 * @vmstate Created
1138 * @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
1139 */
1140VMMR3DECL(int) FTMR3PowerOn(PUVM pUVM, bool fMaster, unsigned uInterval,
1141 const char *pszAddress, unsigned uPort, const char *pszPassword)
1142{
1143 UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1144 PVM pVM = pUVM->pVM;
1145 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1146
1147 VMSTATE enmVMState = VMR3GetState(pVM);
1148 AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1149 ("%s\n", VMR3GetStateName(enmVMState)),
1150 VERR_INTERNAL_ERROR_4);
1151 AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1152
1153 if (pVM->ftm.s.uInterval)
1154 pVM->ftm.s.uInterval = uInterval;
1155 else
1156 pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1157
1158 pVM->ftm.s.uPort = uPort;
1159 pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1160 if (pszPassword)
1161 pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1162
1163 int rc = RTSemEventCreate(&pVM->ftm.s.hShutdownEvent);
1164 if (RT_FAILURE(rc))
1165 return rc;
1166
1167 if (fMaster)
1168 {
1169 rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1170 0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1171 if (RT_FAILURE(rc))
1172 return rc;
1173
1174 pVM->fFaultTolerantMaster = true;
1175 if (PGMIsUsingLargePages(pVM))
1176 {
1177 /* Must disable large page usage as 2 MB pages are too big to write monitor. */
1178 LogRel(("FTSync: disabling large page usage.\n"));
1179 PGMSetLargePageUsage(pVM, false);
1180 }
1181 /** @todo might need to disable page fusion as well */
1182
1183 return VMR3PowerOn(pVM->pUVM);
1184 }
1185
1186
1187 /* standby */
1188 rc = RTThreadCreate(NULL, ftmR3StandbyThread, pVM,
1189 0, RTTHREADTYPE_DEFAULT, 0, "ftmStandby");
1190 if (RT_FAILURE(rc))
1191 return rc;
1192
1193 rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1194 if (RT_FAILURE(rc))
1195 return rc;
1196 pVM->ftm.s.fIsStandbyNode = true;
1197
1198 rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1199 /** @todo deal with the exit code to check if we should activate this standby VM. */
1200 if (pVM->ftm.s.fActivateStandby)
1201 {
1202 /** @todo fallover. */
1203 }
1204
1205 if (pVM->ftm.s.standby.hServer)
1206 {
1207 RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1208 pVM->ftm.s.standby.hServer = NULL;
1209 }
1210 if (rc == VERR_TCP_SERVER_SHUTDOWN)
1211 rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1212 return rc;
1213}
1214
1215/**
1216 * Powers off the fault tolerant virtual machine (standby).
1217 *
1218 * @returns VBox status code.
1219 *
1220 * @param pUVM The user mode VM handle.
1221 */
1222VMMR3DECL(int) FTMR3CancelStandby(PUVM pUVM)
1223{
1224 UVM_ASSERT_VALID_EXT_RETURN(pUVM, VERR_INVALID_VM_HANDLE);
1225 PVM pVM = pUVM->pVM;
1226 VM_ASSERT_VALID_EXT_RETURN(pVM, VERR_INVALID_VM_HANDLE);
1227 AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1228 Assert(pVM->ftm.s.standby.hServer);
1229
1230 return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1231}
1232
1233/**
1234 * Rendezvous callback used by FTMR3SetCheckpoint
1235 * Sync state + changed memory with the standby node.
1236 *
1237 * This is only called on one of the EMTs while the other ones are waiting for
1238 * it to complete this function.
1239 *
1240 * @returns VINF_SUCCESS (VBox strict status code).
1241 * @param pVM Pointer to the VM.
1242 * @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1243 * @param pvUser Not used.
1244 */
1245static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1246{
1247 int rc = VINF_SUCCESS;
1248 bool fSuspended = false;
1249 NOREF(pVCpu);
1250 NOREF(pvUser);
1251
1252 /* We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1253 * is only a short suspend.
1254 */
1255 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointPause, a);
1256 PDMR3Suspend(pVM);
1257
1258 /* Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1259 EMR3NotifySuspend(pVM);
1260 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointPause, a);
1261
1262 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1263
1264 RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1265
1266 /* Reset the sync state. */
1267 pVM->ftm.s.syncstate.uOffStream = 0;
1268 pVM->ftm.s.syncstate.cbReadBlock = 0;
1269 pVM->ftm.s.syncstate.fStopReading = false;
1270 pVM->ftm.s.syncstate.fIOError = false;
1271 pVM->ftm.s.syncstate.fEndOfStream = false;
1272
1273 rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1274 AssertRC(rc);
1275
1276 pVM->ftm.s.fDeltaLoadSaveActive = true;
1277 rc = VMR3SaveFT(pVM->pUVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1278 pVM->ftm.s.fDeltaLoadSaveActive = false;
1279 AssertRC(rc);
1280
1281 rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1282 AssertRC(rc);
1283
1284 RTSocketRelease(pVM->ftm.s.hSocket);
1285
1286 /* Write protect all memory. */
1287 rc = PGMR3PhysWriteProtectRAM(pVM);
1288 AssertRC(rc);
1289
1290 /* We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1291 * is only a short suspend.
1292 */
1293 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointResume, b);
1294 PGMR3ResetNoMorePhysWritesFlag(pVM);
1295 PDMR3Resume(pVM);
1296
1297 /* Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1298 EMR3NotifyResume(pVM);
1299 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointResume, b);
1300
1301 return rc;
1302}
1303
1304/**
1305 * Performs a full sync to the standby node
1306 *
1307 * @returns VBox status code.
1308 *
1309 * @param pVM Pointer to the VM.
1310 * @param enmCheckpoint Checkpoint type
1311 */
1312VMMR3_INT_DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1313{
1314 int rc;
1315
1316 if (!pVM->fFaultTolerantMaster)
1317 return VINF_SUCCESS;
1318
1319 switch (enmCheckpoint)
1320 {
1321 case FTMCHECKPOINTTYPE_NETWORK:
1322 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1323 break;
1324
1325 case FTMCHECKPOINTTYPE_STORAGE:
1326 STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1327 break;
1328
1329 default:
1330 AssertMsgFailedReturn(("%d\n", enmCheckpoint), VERR_INVALID_PARAMETER);
1331 }
1332
1333 pVM->ftm.s.fCheckpointingActive = true;
1334 if (VM_IS_EMT(pVM))
1335 {
1336 PVMCPU pVCpu = VMMGetCpu(pVM);
1337
1338 /* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1339 while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1340 {
1341 if (VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1342 {
1343 rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1344 AssertRC(rc);
1345 }
1346
1347 if (VM_FF_IS_PENDING(pVM, VM_FF_REQUEST))
1348 {
1349 rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY, true /*fPriorityOnly*/);
1350 AssertRC(rc);
1351 }
1352 }
1353 }
1354 else
1355 rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1356
1357 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1358
1359 STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1360
1361 rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1362
1363 STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1364
1365 PDMCritSectLeave(&pVM->ftm.s.CritSect);
1366 pVM->ftm.s.fCheckpointingActive = false;
1367
1368 return rc;
1369}
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette