FTM.cpp@ 36847

Last change on this file since 36847 was 35346, checked in by vboxsync, 14 years ago
VMM reorg: Moving the public include files from include/VBox to include/VBox/vmm.
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 41.5 KB

Line
1	/* $Id: FTM.cpp 35346 2010-12-27 16:13:13Z vboxsync $ */
2	/** @file
3	* FTM - Fault Tolerance Manager
4	*/
5
6	/*
7	* Copyright (C) 2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_FTM
23	#include "FTMInternal.h"
24	#include <VBox/vmm/vm.h>
25	#include <VBox/vmm/vmm.h>
26	#include <VBox/err.h>
27	#include <VBox/param.h>
28	#include <VBox/vmm/ssm.h>
29	#include <VBox/log.h>
30	#include <VBox/vmm/pgm.h>
31	#include <VBox/vmm/pdm.h>
32
33	#include <iprt/assert.h>
34	#include <iprt/thread.h>
35	#include <iprt/string.h>
36	#include <iprt/mem.h>
37	#include <iprt/tcp.h>
38	#include <iprt/socket.h>
39	#include <iprt/semaphore.h>
40	#include <iprt/asm.h>
41
42	#include "internal/vm.h"
43	#include "internal/em.h"
44	#include "internal/pgm.h"
45
46	/*******************************************************************************
47	* Structures and Typedefs *
48	*******************************************************************************/
49
50	/**
51	* TCP stream header.
52	*
53	* This is an extra layer for fixing the problem with figuring out when the SSM
54	* stream ends.
55	*/
56	typedef struct FTMTCPHDR
57	{
58	/** Magic value. */
59	uint32_t u32Magic;
60	/** The size of the data block following this header.
61	* 0 indicates the end of the stream, while UINT32_MAX indicates
62	* cancelation. */
63	uint32_t cb;
64	} FTMTCPHDR;
65	/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
66	#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
67	/** The max block size. */
68	#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
69
70	/**
71	* TCP stream header.
72	*
73	* This is an extra layer for fixing the problem with figuring out when the SSM
74	* stream ends.
75	*/
76	typedef struct FTMTCPHDRMEM
77	{
78	/** Magic value. */
79	uint32_t u32Magic;
80	/** Size (Uncompressed) of the pages following the header. */
81	uint32_t cbPageRange;
82	/** GC Physical address of the page(s) to sync. */
83	RTGCPHYS GCPhys;
84	/** The size of the data block following this header.
85	* 0 indicates the end of the stream, while UINT32_MAX indicates
86	* cancelation. */
87	uint32_t cb;
88	} FTMTCPHDRMEM;
89
90	/*******************************************************************************
91	* Global Variables *
92	*******************************************************************************/
93	static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
94
95	static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser);
96
97	/**
98	* Initializes the FTM.
99	*
100	* @returns VBox status code.
101	* @param pVM The VM to operate on.
102	*/
103	VMMR3DECL(int) FTMR3Init(PVM pVM)
104	{
105	/*
106	* Assert alignment and sizes.
107	*/
108	AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
109	AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
110
111	/** @todo saved state for master nodes! */
112	pVM->ftm.s.pszAddress = NULL;
113	pVM->ftm.s.pszPassword = NULL;
114	pVM->fFaultTolerantMaster = false;
115	pVM->ftm.s.fIsStandbyNode = false;
116	pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
117	pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
118	pVM->ftm.s.hSocket = NIL_RTSOCKET;
119
120	/*
121	* Initialize the PGM critical section.
122	*/
123	int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
124	AssertRCReturn(rc, rc);
125
126	/*
127	* Register statistics.
128	*/
129	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
130	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
131	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
132	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
133	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
134	STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
135	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
136	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
137	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
138	#ifdef VBOX_WITH_STATISTICS
139	STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
140	STAM_REG(pVM, &pVM->ftm.s.StatCheckpointPause, STAMTYPE_PROFILE, "/FT/Checkpoint/Pause", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
141	STAM_REG(pVM, &pVM->ftm.s.StatCheckpointResume, STAMTYPE_PROFILE, "/FT/Checkpoint/Resume", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
142	STAM_REG(pVM, &pVM->ftm.s.StatSentMemRAM, STAMTYPE_COUNTER, "/FT/Sent/Mem/RAM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
143	STAM_REG(pVM, &pVM->ftm.s.StatSentMemMMIO2, STAMTYPE_COUNTER, "/FT/Sent/Mem/MMIO2", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
144	STAM_REG(pVM, &pVM->ftm.s.StatSentMemShwROM, STAMTYPE_COUNTER, "/FT/Sent/Mem/ShwROM", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
145	STAM_REG(pVM, &pVM->ftm.s.StatSentStateWrite, STAMTYPE_COUNTER, "/FT/Sent/State/Writes", STAMUNIT_BYTES, "The nr of write calls.");
146	#endif
147	return VINF_SUCCESS;
148	}
149
150	/**
151	* Terminates the FTM.
152	*
153	* Termination means cleaning up and freeing all resources,
154	* the VM itself is at this point powered off or suspended.
155	*
156	* @returns VBox status code.
157	* @param pVM The VM to operate on.
158	*/
159	VMMR3DECL(int) FTMR3Term(PVM pVM)
160	{
161	if (pVM->ftm.s.hShutdownEvent != NIL_RTSEMEVENT)
162	{
163	RTSemEventDestroy(pVM->ftm.s.hShutdownEvent);
164	pVM->ftm.s.hShutdownEvent = NIL_RTSEMEVENT;
165	}
166	if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
167	{
168	RTTcpClientClose(pVM->ftm.s.hSocket);
169	pVM->ftm.s.hSocket = NIL_RTSOCKET;
170	}
171	if (pVM->ftm.s.standby.hServer)
172	{
173	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
174	pVM->ftm.s.standby.hServer = NULL;
175	}
176	if (pVM->ftm.s.pszAddress)
177	RTMemFree(pVM->ftm.s.pszAddress);
178	if (pVM->ftm.s.pszPassword)
179	RTMemFree(pVM->ftm.s.pszPassword);
180
181	/* Remove all pending memory updates. */
182	if (pVM->ftm.s.standby.pPhysPageTree)
183	{
184	RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, NULL);
185	pVM->ftm.s.standby.pPhysPageTree = NULL;
186	}
187
188	pVM->ftm.s.pszAddress = NULL;
189	pVM->ftm.s.pszPassword = NULL;
190
191	PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
192	return VINF_SUCCESS;
193	}
194
195
196	static int ftmR3TcpWriteACK(PVM pVM)
197	{
198	int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
199	if (RT_FAILURE(rc))
200	{
201	LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
202	}
203	return rc;
204	}
205
206
207	static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
208	{
209	char szMsg[256];
210	size_t cch;
211	if (pszMsgText && *pszMsgText)
212	{
213	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
214	for (size_t off = 6; off + 1 < cch; off++)
215	if (szMsg[off] == '\n')
216	szMsg[off] = '\r';
217	}
218	else
219	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
220	int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
221	if (RT_FAILURE(rc))
222	LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
223	return rc;
224	}
225
226	/**
227	* Reads a string from the socket.
228	*
229	* @returns VBox status code.
230	*
231	* @param pState The teleporter state structure.
232	* @param pszBuf The output buffer.
233	* @param cchBuf The size of the output buffer.
234	*
235	*/
236	static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
237	{
238	char *pszStart = pszBuf;
239	RTSOCKET Sock = pVM->ftm.s.hSocket;
240
241	AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
242	*pszBuf = '\0';
243
244	/* dead simple approach. */
245	for (;;)
246	{
247	char ch;
248	int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
249	if (RT_FAILURE(rc))
250	{
251	LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
252	return rc;
253	}
254	if ( ch == '\n'
255	\|\| ch == '\0')
256	return VINF_SUCCESS;
257	if (cchBuf <= 1)
258	{
259	LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
260	return VERR_BUFFER_OVERFLOW;
261	}
262	*pszBuf++ = ch;
263	*pszBuf = '\0';
264	cchBuf--;
265	}
266	}
267
268	/**
269	* Reads an ACK or NACK.
270	*
271	* @returns VBox status code.
272	* @param pVM The VM to operate on.
273	* @param pszWhich Which ACK is this this?
274	* @param pszNAckMsg Optional NACK message.
275	*/
276	static int ftmR3TcpReadACK(PVM pVM, const char pszWhich, const char pszNAckMsg = NULL)
277	{
278	char szMsg[256];
279	int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
280	if (RT_FAILURE(rc))
281	return rc;
282
283	if (!strcmp(szMsg, "ACK"))
284	return VINF_SUCCESS;
285
286	if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
287	{
288	char *pszMsgText = strchr(szMsg, ';');
289	if (pszMsgText)
290	*pszMsgText++ = '\0';
291
292	int32_t vrc2;
293	rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
294	if (rc == VINF_SUCCESS)
295	{
296	/*
297	* Well formed NACK, transform it into an error.
298	*/
299	if (pszNAckMsg)
300	{
301	LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
302	return VERR_INTERNAL_ERROR;
303	}
304
305	if (pszMsgText)
306	{
307	pszMsgText = RTStrStrip(pszMsgText);
308	for (size_t off = 0; pszMsgText[off]; off++)
309	if (pszMsgText[off] == '\r')
310	pszMsgText[off] = '\n';
311
312	LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
313	}
314	return VERR_INTERNAL_ERROR_2;
315	}
316
317	if (pszMsgText)
318	pszMsgText[-1] = ';';
319	}
320	return VERR_INTERNAL_ERROR_3;
321	}
322
323	/**
324	* Submitts a command to the destination and waits for the ACK.
325	*
326	* @returns VBox status code.
327	*
328	* @param pVM The VM to operate on.
329	* @param pszCommand The command.
330	* @param fWaitForAck Whether to wait for the ACK.
331	*/
332	static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
333	{
334	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
335	if (RT_FAILURE(rc))
336	return rc;
337	if (!fWaitForAck)
338	return VINF_SUCCESS;
339	return ftmR3TcpReadACK(pVM, pszCommand);
340	}
341
342	/**
343	* @copydoc SSMSTRMOPS::pfnWrite
344	*/
345	static DECLCALLBACK(int) ftmR3TcpOpWrite(void pvUser, uint64_t offStream, const void pvBuf, size_t cbToWrite)
346	{
347	PVM pVM = (PVM)pvUser;
348
349	AssertReturn(cbToWrite > 0, VINF_SUCCESS);
350	AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
351	AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
352
353	STAM_COUNTER_INC(&pVM->ftm.s.StatSentStateWrite);
354	for (;;)
355	{
356	FTMTCPHDR Hdr;
357	Hdr.u32Magic = FTMTCPHDR_MAGIC;
358	Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
359	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
360	if (RT_FAILURE(rc))
361	{
362	LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
363	return rc;
364	}
365	pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
366	pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
367	if (Hdr.cb == cbToWrite)
368	return VINF_SUCCESS;
369
370	/* advance */
371	cbToWrite -= Hdr.cb;
372	pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
373	}
374	}
375
376
377	/**
378	* Selects and poll for close condition.
379	*
380	* We can use a relatively high poll timeout here since it's only used to get
381	* us out of error paths. In the normal cause of events, we'll get a
382	* end-of-stream header.
383	*
384	* @returns VBox status code.
385	*
386	* @param pState The teleporter state data.
387	*/
388	static int ftmR3TcpReadSelect(PVM pVM)
389	{
390	int rc;
391	do
392	{
393	rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
394	if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
395	{
396	pVM->ftm.s.syncstate.fIOError = true;
397	LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
398	break;
399	}
400	if (pVM->ftm.s.syncstate.fStopReading)
401	{
402	rc = VERR_EOF;
403	break;
404	}
405	} while (rc == VERR_TIMEOUT);
406	return rc;
407	}
408
409
410	/**
411	* @copydoc SSMSTRMOPS::pfnRead
412	*/
413	static DECLCALLBACK(int) ftmR3TcpOpRead(void pvUser, uint64_t offStream, void pvBuf, size_t cbToRead, size_t *pcbRead)
414	{
415	PVM pVM = (PVM)pvUser;
416	AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
417
418	for (;;)
419	{
420	int rc;
421
422	/*
423	* Check for various conditions and may have been signalled.
424	*/
425	if (pVM->ftm.s.syncstate.fEndOfStream)
426	return VERR_EOF;
427	if (pVM->ftm.s.syncstate.fStopReading)
428	return VERR_EOF;
429	if (pVM->ftm.s.syncstate.fIOError)
430	return VERR_IO_GEN_FAILURE;
431
432	/*
433	* If there is no more data in the current block, read the next
434	* block header.
435	*/
436	if (!pVM->ftm.s.syncstate.cbReadBlock)
437	{
438	rc = ftmR3TcpReadSelect(pVM);
439	if (RT_FAILURE(rc))
440	return rc;
441	FTMTCPHDR Hdr;
442	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
443	if (RT_FAILURE(rc))
444	{
445	pVM->ftm.s.syncstate.fIOError = true;
446	LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
447	return rc;
448	}
449	pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
450
451	if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
452	\|\| Hdr.cb > FTMTCPHDR_MAX_SIZE
453	\|\| Hdr.cb == 0))
454	{
455	if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
456	&& ( Hdr.cb == 0
457	\|\| Hdr.cb == UINT32_MAX)
458	)
459	{
460	pVM->ftm.s.syncstate.fEndOfStream = true;
461	pVM->ftm.s.syncstate.cbReadBlock = 0;
462	return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
463	}
464	pVM->ftm.s.syncstate.fIOError = true;
465	LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
466	return VERR_IO_GEN_FAILURE;
467	}
468
469	pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
470	if (pVM->ftm.s.syncstate.fStopReading)
471	return VERR_EOF;
472	}
473
474	/*
475	* Read more data.
476	*/
477	rc = ftmR3TcpReadSelect(pVM);
478	if (RT_FAILURE(rc))
479	return rc;
480
481	uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
482	rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
483	if (RT_FAILURE(rc))
484	{
485	pVM->ftm.s.syncstate.fIOError = true;
486	LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
487	return rc;
488	}
489	if (pcbRead)
490	{
491	cb = (uint32_t)*pcbRead;
492	pVM->ftm.s.StatReceivedState.c += cb;
493	pVM->ftm.s.syncstate.uOffStream += cb;
494	pVM->ftm.s.syncstate.cbReadBlock -= cb;
495	return VINF_SUCCESS;
496	}
497	pVM->ftm.s.StatReceivedState.c += cb;
498	pVM->ftm.s.syncstate.uOffStream += cb;
499	pVM->ftm.s.syncstate.cbReadBlock -= cb;
500	if (cbToRead == cb)
501	return VINF_SUCCESS;
502
503	/* Advance to the next block. */
504	cbToRead -= cb;
505	pvBuf = (uint8_t *)pvBuf + cb;
506	}
507	}
508
509
510	/**
511	* @copydoc SSMSTRMOPS::pfnSeek
512	*/
513	static DECLCALLBACK(int) ftmR3TcpOpSeek(void pvUser, int64_t offSeek, unsigned uMethod, uint64_t poffActual)
514	{
515	return VERR_NOT_SUPPORTED;
516	}
517
518
519	/**
520	* @copydoc SSMSTRMOPS::pfnTell
521	*/
522	static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
523	{
524	PVM pVM = (PVM)pvUser;
525	return pVM->ftm.s.syncstate.uOffStream;
526	}
527
528
529	/**
530	* @copydoc SSMSTRMOPS::pfnSize
531	*/
532	static DECLCALLBACK(int) ftmR3TcpOpSize(void pvUser, uint64_t pcb)
533	{
534	return VERR_NOT_SUPPORTED;
535	}
536
537
538	/**
539	* @copydoc SSMSTRMOPS::pfnIsOk
540	*/
541	static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
542	{
543	PVM pVM = (PVM)pvUser;
544
545	if (pVM->fFaultTolerantMaster)
546	{
547	/* Poll for incoming NACKs and errors from the other side */
548	int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
549	if (rc != VERR_TIMEOUT)
550	{
551	if (RT_SUCCESS(rc))
552	{
553	LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
554	rc = VERR_SSM_CANCELLED;
555	}
556	else
557	LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
558	return rc;
559	}
560	}
561
562	return VINF_SUCCESS;
563	}
564
565
566	/**
567	* @copydoc SSMSTRMOPS::pfnClose
568	*/
569	static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
570	{
571	PVM pVM = (PVM)pvUser;
572
573	if (pVM->fFaultTolerantMaster)
574	{
575	FTMTCPHDR EofHdr;
576	EofHdr.u32Magic = FTMTCPHDR_MAGIC;
577	EofHdr.cb = fCanceled ? UINT32_MAX : 0;
578	int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
579	if (RT_FAILURE(rc))
580	{
581	LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
582	return rc;
583	}
584	}
585	else
586	{
587	ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
588	}
589
590	return VINF_SUCCESS;
591	}
592
593
594	/**
595	* Method table for a TCP based stream.
596	*/
597	static SSMSTRMOPS const g_ftmR3TcpOps =
598	{
599	SSMSTRMOPS_VERSION,
600	ftmR3TcpOpWrite,
601	ftmR3TcpOpRead,
602	ftmR3TcpOpSeek,
603	ftmR3TcpOpTell,
604	ftmR3TcpOpSize,
605	ftmR3TcpOpIsOk,
606	ftmR3TcpOpClose,
607	SSMSTRMOPS_VERSION
608	};
609
610
611	/**
612	* VMR3ReqCallWait callback
613	*
614	* @param pVM The VM handle.
615	*
616	*/
617	static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
618	{
619	int rc = PGMR3PhysWriteProtectRAM(pVM);
620	AssertRC(rc);
621	}
622
623
624	/**
625	* Sync the VM state
626	*
627	* @returns VBox status code.
628	* @param pVM The VM handle.
629	*/
630	static int ftmR3PerformFullSync(PVM pVM)
631	{
632	bool fSuspended = false;
633
634	int rc = VMR3Suspend(pVM);
635	AssertRCReturn(rc, rc);
636
637	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
638
639	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
640
641	/* Reset the sync state. */
642	pVM->ftm.s.syncstate.uOffStream = 0;
643	pVM->ftm.s.syncstate.cbReadBlock = 0;
644	pVM->ftm.s.syncstate.fStopReading = false;
645	pVM->ftm.s.syncstate.fIOError = false;
646	pVM->ftm.s.syncstate.fEndOfStream = false;
647
648	rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
649	AssertRC(rc);
650
651	pVM->ftm.s.fDeltaLoadSaveActive = false;
652	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
653	AssertRC(rc);
654
655	rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
656	AssertRC(rc);
657
658	RTSocketRelease(pVM->ftm.s.hSocket);
659
660	/* Write protect all memory. */
661	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
662	AssertRCReturn(rc, rc);
663
664	rc = VMR3Resume(pVM);
665	AssertRC(rc);
666
667	return rc;
668	}
669
670
671	/**
672	* PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
673	*
674	* @param pVM VM Handle.
675	* @param GCPhys GC physical address
676	* @param pRange HC virtual address of the page(s)
677	* @param cbRange Size of the dirty range in bytes.
678	* @param pvUser User argument
679	*/
680	static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t pRange, unsigned cbRange, void pvUser)
681	{
682	FTMTCPHDRMEM Hdr;
683	Hdr.u32Magic = FTMTCPHDR_MAGIC;
684	Hdr.GCPhys = GCPhys;
685	Hdr.cbPageRange = cbRange;
686	Hdr.cb = cbRange;
687	/** @todo compress page(s). */
688	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
689	if (RT_FAILURE(rc))
690	{
691	LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
692	return rc;
693	}
694	pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
695
696	#ifdef VBOX_WITH_STATISTICS
697	switch (PGMPhysGetPageType(pVM, GCPhys))
698	{
699	case PGMPAGETYPE_RAM:
700	pVM->ftm.s.StatSentMemRAM.c += Hdr.cb + sizeof(Hdr);
701	break;
702
703	case PGMPAGETYPE_MMIO2:
704	pVM->ftm.s.StatSentMemMMIO2.c += Hdr.cb + sizeof(Hdr);
705	break;
706
707	case PGMPAGETYPE_ROM_SHADOW:
708	pVM->ftm.s.StatSentMemShwROM.c += Hdr.cb + sizeof(Hdr);
709	break;
710
711	case PGMPAGETYPE_MMIO2_ALIAS_MMIO:
712	AssertFailed();
713	break;
714
715	default:
716	AssertFailed();
717	break;
718	}
719	#endif
720
721	return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
722	}
723
724	/**
725	* Thread function which starts syncing process for this master VM
726	*
727	* @param Thread The thread id.
728	* @param pvUser Not used
729	* @return VINF_SUCCESS (ignored).
730	*
731	*/
732	static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
733	{
734	int rc = VINF_SUCCESS;
735	PVM pVM = (PVM)pvUser;
736
737	for (;;)
738	{
739	/*
740	* Try connect to the standby machine.
741	*/
742	Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
743	rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
744	if (RT_SUCCESS(rc))
745	{
746	Log(("ftmR3MasterThread: CONNECTED\n"));
747
748	/* Disable Nagle. */
749	rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /fEnable/);
750	AssertRC(rc);
751
752	/* Read and check the welcome message. */
753	char szLine[RT_MAX(128, sizeof(g_szWelcome))];
754	RT_ZERO(szLine);
755	rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
756	if ( RT_SUCCESS(rc)
757	&& !strcmp(szLine, g_szWelcome))
758	{
759	/* password */
760	if (pVM->ftm.s.pszPassword)
761	rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
762
763	if (RT_SUCCESS(rc))
764	{
765	/* ACK */
766	rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
767	if (RT_SUCCESS(rc))
768	{
769	/** todo: verify VM config. */
770	break;
771	}
772	}
773	}
774	/* Failed, so don't bother anymore. */
775	return VINF_SUCCESS;
776	}
777	rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, 1000 /* 1 second */);
778	if (rc != VERR_TIMEOUT)
779	return VINF_SUCCESS; /* told to quit */
780	}
781
782	/* Successfully initialized the connection to the standby node.
783	* Start the sync process.
784	*/
785
786	/* First sync all memory and write protect everything so
787	* we can send changed pages later on.
788	*/
789
790	rc = ftmR3PerformFullSync(pVM);
791
792	for (;;)
793	{
794	rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
795	if (rc != VERR_TIMEOUT)
796	break; /* told to quit */
797
798	if (!pVM->ftm.s.fCheckpointingActive)
799	{
800	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
801	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
802
803	rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
804	AssertRC(rc);
805
806	/* sync the changed memory with the standby node. */
807	/* Write protect all memory. */
808	if (!pVM->ftm.s.fCheckpointingActive)
809	{
810	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
811	AssertRC(rc);
812	}
813
814	/* Enumerate all dirty pages and send them to the standby VM. */
815	if (!pVM->ftm.s.fCheckpointingActive)
816	{
817	rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
818	Assert(rc == VINF_SUCCESS \|\| rc == VERR_INTERRUPTED);
819	}
820
821	/* Send last memory header to signal the end. */
822	FTMTCPHDRMEM Hdr;
823	Hdr.u32Magic = FTMTCPHDR_MAGIC;
824	Hdr.GCPhys = 0;
825	Hdr.cbPageRange = 0;
826	Hdr.cb = 0;
827	rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
828	if (RT_FAILURE(rc))
829	LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
830
831	rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
832	AssertRC(rc);
833
834	PDMCritSectLeave(&pVM->ftm.s.CritSect);
835	}
836	}
837	return rc;
838	}
839
840	/**
841	* Syncs memory from the master VM
842	*
843	* @returns VBox status code.
844	* @param pVM VM Handle.
845	*/
846	static int ftmR3SyncMem(PVM pVM)
847	{
848	while (true)
849	{
850	FTMTCPHDRMEM Hdr;
851	RTGCPHYS GCPhys;
852
853	/* Read memory header. */
854	int rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
855	if (RT_FAILURE(rc))
856	{
857	Log(("RTTcpRead failed with %Rrc\n", rc));
858	break;
859	}
860	pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
861
862	if (Hdr.cb == 0)
863	break; /* end of sync. */
864
865	Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
866	GCPhys = Hdr.GCPhys;
867
868	/* Must be a multiple of PAGE_SIZE. */
869	Assert((Hdr.cbPageRange & 0xfff) == 0);
870
871	while (Hdr.cbPageRange)
872	{
873	PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)RTAvlGCPhysGet(&pVM->ftm.s.standby.pPhysPageTree, GCPhys);
874	if (!pNode)
875	{
876	/* Allocate memory for the node and page. */
877	pNode = (PFTMPHYSPAGETREENODE)RTMemAllocZ(sizeof(*pNode) + PAGE_SIZE);
878	AssertBreak(pNode);
879
880	/* Insert the node into the tree. */
881	pNode->Core.Key = GCPhys;
882	pNode->pPage = (void *)(pNode + 1);
883	bool fRet = RTAvlGCPhysInsert(&pVM->ftm.s.standby.pPhysPageTree, &pNode->Core);
884	Assert(fRet);
885	}
886
887	/* Fetch the page. */
888	rc = RTTcpRead(pVM->ftm.s.hSocket, pNode->pPage, PAGE_SIZE, NULL);
889	if (RT_FAILURE(rc))
890	{
891	Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
892	break;
893	}
894	pVM->ftm.s.StatReceivedMem.c += PAGE_SIZE;
895	Hdr.cbPageRange -= PAGE_SIZE;
896	GCPhys += PAGE_SIZE;
897	}
898	}
899	return VINF_SUCCESS;
900	}
901
902
903	/**
904	* Callback handler for RTAvlGCPhysDestroy
905	*
906	* @returns 0 to continue, otherwise stop
907	* @param pBaseNode Node to destroy
908	* @param pvUser User parameter
909	*/
910	static DECLCALLBACK(int) ftmR3PageTreeDestroyCallback(PAVLGCPHYSNODECORE pBaseNode, void *pvUser)
911	{
912	PVM pVM = (PVM)pvUser;
913	PFTMPHYSPAGETREENODE pNode = (PFTMPHYSPAGETREENODE)pBaseNode;
914
915	if (pVM) /* NULL when the VM is destroyed. */
916	{
917	/* Update the guest memory of the standby VM. */
918	int rc = PGMR3PhysWriteExternal(pVM, pNode->Core.Key, pNode->pPage, PAGE_SIZE, "FTMemSync");
919	AssertRC(rc);
920	}
921	RTMemFree(pNode);
922	return 0;
923	}
924
925	/**
926	* Thread function which monitors the health of the master VM
927	*
928	* @param Thread The thread id.
929	* @param pvUser Not used
930	* @return VINF_SUCCESS (ignored).
931	*
932	*/
933	static DECLCALLBACK(int) ftmR3StandbyThread(RTTHREAD Thread, void *pvUser)
934	{
935	PVM pVM = (PVM)pvUser;
936
937	for (;;)
938	{
939	uint64_t u64TimeNow;
940
941	int rc = RTSemEventWait(pVM->ftm.s.hShutdownEvent, pVM->ftm.s.uInterval);
942	if (rc != VERR_TIMEOUT)
943	break; /* told to quit */
944
945	if (pVM->ftm.s.standby.u64LastHeartbeat)
946	{
947	u64TimeNow = RTTimeMilliTS();
948
949	if (u64TimeNow > pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 4)
950	{
951	/* Timeout; prepare to fallover. */
952	LogRel(("FTSync: TIMEOUT (%RX64 vs %RX64 ms): activate standby VM!\n", u64TimeNow, pVM->ftm.s.standby.u64LastHeartbeat + pVM->ftm.s.uInterval * 2));
953
954	pVM->ftm.s.fActivateStandby = true;
955	/** todo: prevent split-brain. */
956	break;
957	}
958	}
959	}
960
961	return VINF_SUCCESS;
962	}
963
964
965	/**
966	* Listen for incoming traffic destined for the standby VM.
967	*
968	* @copydoc FNRTTCPSERVE
969	*
970	* @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
971	*/
972	static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
973	{
974	PVM pVM = (PVM)pvUser;
975
976	pVM->ftm.s.hSocket = Sock;
977
978	/*
979	* Disable Nagle.
980	*/
981	int rc = RTTcpSetSendCoalescing(Sock, false /fEnable/);
982	AssertRC(rc);
983
984	/* Send the welcome message to the master node. */
985	rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
986	if (RT_FAILURE(rc))
987	{
988	LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
989	return VINF_SUCCESS;
990	}
991
992	/*
993	* Password.
994	*/
995	const char *pszPassword = pVM->ftm.s.pszPassword;
996	if (pszPassword)
997	{
998	unsigned off = 0;
999	while (pszPassword[off])
1000	{
1001	char ch;
1002	rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
1003	if ( RT_FAILURE(rc)
1004	\|\| pszPassword[off] != ch)
1005	{
1006	if (RT_FAILURE(rc))
1007	LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
1008	else
1009	LogRel(("FTSync: Invalid password (off=%u)\n", off));
1010	ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
1011	return VINF_SUCCESS;
1012	}
1013	off++;
1014	}
1015	}
1016	rc = ftmR3TcpWriteACK(pVM);
1017	if (RT_FAILURE(rc))
1018	return VINF_SUCCESS;
1019
1020	/** todo: verify VM config. */
1021
1022	/*
1023	* Stop the server.
1024	*
1025	* Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
1026	* to it we must not return that value!
1027	*/
1028	RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1029
1030	/*
1031	* Command processing loop.
1032	*/
1033	bool fDone = false;
1034	for (;;)
1035	{
1036	bool fFullSync = false;
1037	char szCmd[128];
1038
1039	rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
1040	if (RT_FAILURE(rc))
1041	break;
1042
1043	pVM->ftm.s.standby.u64LastHeartbeat = RTTimeMilliTS();
1044	if (!strcmp(szCmd, "mem-sync"))
1045	{
1046	rc = ftmR3TcpWriteACK(pVM);
1047	AssertRC(rc);
1048	if (RT_FAILURE(rc))
1049	continue;
1050
1051	rc = ftmR3SyncMem(pVM);
1052	AssertRC(rc);
1053
1054	rc = ftmR3TcpWriteACK(pVM);
1055	AssertRC(rc);
1056	}
1057	else
1058	if ( !strcmp(szCmd, "checkpoint")
1059	\|\| !strcmp(szCmd, "full-sync")
1060	\|\| (fFullSync = true)) /* intended assignment */
1061	{
1062	rc = ftmR3TcpWriteACK(pVM);
1063	AssertRC(rc);
1064	if (RT_FAILURE(rc))
1065	continue;
1066
1067	/* Flush all pending memory updates. */
1068	if (pVM->ftm.s.standby.pPhysPageTree)
1069	{
1070	RTAvlGCPhysDestroy(&pVM->ftm.s.standby.pPhysPageTree, ftmR3PageTreeDestroyCallback, pVM);
1071	pVM->ftm.s.standby.pPhysPageTree = NULL;
1072	}
1073
1074	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1075
1076	/* Reset the sync state. */
1077	pVM->ftm.s.syncstate.uOffStream = 0;
1078	pVM->ftm.s.syncstate.cbReadBlock = 0;
1079	pVM->ftm.s.syncstate.fStopReading = false;
1080	pVM->ftm.s.syncstate.fIOError = false;
1081	pVM->ftm.s.syncstate.fEndOfStream = false;
1082
1083	pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
1084	rc = VMR3LoadFromStreamFT(pVM, &g_ftmR3TcpOps, pVM);
1085	pVM->ftm.s.fDeltaLoadSaveActive = false;
1086	RTSocketRelease(pVM->ftm.s.hSocket);
1087	AssertRC(rc);
1088	if (RT_FAILURE(rc))
1089	{
1090	LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
1091	ftmR3TcpWriteNACK(pVM, rc);
1092	continue;
1093	}
1094
1095	/* The EOS might not have been read, make sure it is. */
1096	pVM->ftm.s.syncstate.fStopReading = false;
1097	size_t cbRead;
1098	rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
1099	if (rc != VERR_EOF)
1100	{
1101	LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
1102	ftmR3TcpWriteNACK(pVM, rc);
1103	continue;
1104	}
1105
1106	rc = ftmR3TcpWriteACK(pVM);
1107	AssertRC(rc);
1108	}
1109	}
1110	LogFlowFunc(("returns mRc=%Rrc\n", rc));
1111	return VERR_TCP_SERVER_STOP;
1112	}
1113
1114	/**
1115	* Powers on the fault tolerant virtual machine.
1116	*
1117	* @returns VBox status code.
1118	*
1119	* @param pVM The VM to operate on.
1120	* @param fMaster FT master or standby
1121	* @param uInterval FT sync interval
1122	* @param pszAddress Standby VM address
1123	* @param uPort Standby VM port
1124	* @param pszPassword FT password (NULL for none)
1125	*
1126	* @thread Any thread.
1127	* @vmstate Created
1128	* @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
1129	*/
1130	VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char pszAddress, unsigned uPort, const char pszPassword)
1131	{
1132	int rc = VINF_SUCCESS;
1133
1134	VMSTATE enmVMState = VMR3GetState(pVM);
1135	AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1136	("%s\n", VMR3GetStateName(enmVMState)),
1137	VERR_INTERNAL_ERROR_4);
1138	AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1139
1140	if (pVM->ftm.s.uInterval)
1141	pVM->ftm.s.uInterval = uInterval;
1142	else
1143	pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1144
1145	pVM->ftm.s.uPort = uPort;
1146	pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1147	if (pszPassword)
1148	pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1149
1150	rc = RTSemEventCreate(&pVM->ftm.s.hShutdownEvent);
1151	if (RT_FAILURE(rc))
1152	return rc;
1153
1154	if (fMaster)
1155	{
1156	rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1157	0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1158	if (RT_FAILURE(rc))
1159	return rc;
1160
1161	pVM->fFaultTolerantMaster = true;
1162	if (PGMIsUsingLargePages(pVM))
1163	{
1164	/* Must disable large page usage as 2 MB pages are too big to write monitor. */
1165	LogRel(("FTSync: disabling large page usage.\n"));
1166	PGMSetLargePageUsage(pVM, false);
1167	}
1168	/** @todo might need to disable page fusion as well */
1169
1170	return VMR3PowerOn(pVM);
1171	}
1172	else
1173	{
1174	/* standby */
1175	rc = RTThreadCreate(NULL, ftmR3StandbyThread, pVM,
1176	0, RTTHREADTYPE_DEFAULT, 0, "ftmStandby");
1177	if (RT_FAILURE(rc))
1178	return rc;
1179
1180	rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1181	if (RT_FAILURE(rc))
1182	return rc;
1183	pVM->ftm.s.fIsStandbyNode = true;
1184
1185	rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1186	/** @todo deal with the exit code to check if we should activate this standby VM. */
1187	if (pVM->ftm.s.fActivateStandby)
1188	{
1189	/** @todo fallover. */
1190	}
1191
1192	if (pVM->ftm.s.standby.hServer)
1193	{
1194	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1195	pVM->ftm.s.standby.hServer = NULL;
1196	}
1197	if (rc == VERR_TCP_SERVER_SHUTDOWN)
1198	rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1199	}
1200	return rc;
1201	}
1202
1203	/**
1204	* Powers off the fault tolerant virtual machine (standby).
1205	*
1206	* @returns VBox status code.
1207	*
1208	* @param pVM The VM to operate on.
1209	*/
1210	VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1211	{
1212	AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1213	Assert(pVM->ftm.s.standby.hServer);
1214
1215	return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1216	}
1217
1218	/**
1219	* Rendezvous callback used by FTMR3SetCheckpoint
1220	* Sync state + changed memory with the standby node.
1221	*
1222	* This is only called on one of the EMTs while the other ones are waiting for
1223	* it to complete this function.
1224	*
1225	* @returns VINF_SUCCESS (VBox strict status code).
1226	* @param pVM The VM handle.
1227	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1228	* @param pvUser User parameter
1229	*/
1230	static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1231	{
1232	int rc = VINF_SUCCESS;
1233	bool fSuspended = false;
1234
1235	/** We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1236	* is only a short suspend.
1237	*/
1238	STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointPause, a);
1239	PDMR3Suspend(pVM);
1240
1241	/** Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1242	EMR3NotifySuspend(pVM);
1243	STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointPause, a);
1244
1245	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1246
1247	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1248
1249	/* Reset the sync state. */
1250	pVM->ftm.s.syncstate.uOffStream = 0;
1251	pVM->ftm.s.syncstate.cbReadBlock = 0;
1252	pVM->ftm.s.syncstate.fStopReading = false;
1253	pVM->ftm.s.syncstate.fIOError = false;
1254	pVM->ftm.s.syncstate.fEndOfStream = false;
1255
1256	rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1257	AssertRC(rc);
1258
1259	pVM->ftm.s.fDeltaLoadSaveActive = true;
1260	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1261	pVM->ftm.s.fDeltaLoadSaveActive = false;
1262	AssertRC(rc);
1263
1264	rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1265	AssertRC(rc);
1266
1267	RTSocketRelease(pVM->ftm.s.hSocket);
1268
1269	/* Write protect all memory. */
1270	rc = PGMR3PhysWriteProtectRAM(pVM);
1271	AssertRC(rc);
1272
1273	/** We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1274	* is only a short suspend.
1275	*/
1276	STAM_PROFILE_START(&pVM->ftm.s.StatCheckpointResume, b);
1277	PGMR3ResetNoMorePhysWritesFlag(pVM);
1278	PDMR3Resume(pVM);
1279
1280	/** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1281	EMR3NotifyResume(pVM);
1282	STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpointResume, b);
1283
1284	return rc;
1285	}
1286
1287	/**
1288	* Performs a full sync to the standby node
1289	*
1290	* @returns VBox status code.
1291	*
1292	* @param pVM The VM to operate on.
1293	* @param enmCheckpoint Checkpoint type
1294	*/
1295	VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1296	{
1297	int rc;
1298
1299	if (!pVM->fFaultTolerantMaster)
1300	return VINF_SUCCESS;
1301
1302	switch (enmCheckpoint)
1303	{
1304	case FTMCHECKPOINTTYPE_NETWORK:
1305	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1306	break;
1307
1308	case FTMCHECKPOINTTYPE_STORAGE:
1309	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1310	break;
1311
1312	default:
1313	break;
1314	}
1315	pVM->ftm.s.fCheckpointingActive = true;
1316	if (VM_IS_EMT(pVM))
1317	{
1318	PVMCPU pVCpu = VMMGetCpu(pVM);
1319
1320	/* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1321	while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1322	{
1323	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1324	{
1325	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1326	AssertRC(rc);
1327	}
1328
1329	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1330	{
1331	rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
1332	AssertRC(rc);
1333	}
1334	}
1335	}
1336	else
1337	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1338
1339	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1340
1341	STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1342
1343	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1344
1345	STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1346
1347	PDMCritSectLeave(&pVM->ftm.s.CritSect);
1348	pVM->ftm.s.fCheckpointingActive = false;
1349
1350	return rc;
1351	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/VMMR3/FTM.cpp@ 36847

Download in other formats: