FTM.cpp@ 32215

Last change on this file since 32215 was 32206, checked in by vboxsync, 14 years ago
FT: no state changes for loading either
Property svn:eol-style set to `native` Property svn:keywords set to `Author Date Id Revision`
File size: 36.4 KB

Line
1	/* $Id: FTM.cpp 32206 2010-09-02 14:13:13Z vboxsync $ */
2	/** @file
3	* FTM - Fault Tolerance Manager
4	*/
5
6	/*
7	* Copyright (C) 2010 Oracle Corporation
8	*
9	* This file is part of VirtualBox Open Source Edition (OSE), as
10	* available from http://www.virtualbox.org. This file is free software;
11	* you can redistribute it and/or modify it under the terms of the GNU
12	* General Public License (GPL) as published by the Free Software
13	* Foundation, in version 2 as it comes in the "COPYING" file of the
14	* VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15	* hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16	*/
17
18
19	/*******************************************************************************
20	* Header Files *
21	*******************************************************************************/
22	#define LOG_GROUP LOG_GROUP_FTM
23	#include "FTMInternal.h"
24	#include <VBox/vm.h>
25	#include <VBox/vmm.h>
26	#include <VBox/err.h>
27	#include <VBox/param.h>
28	#include <VBox/ssm.h>
29	#include <VBox/log.h>
30	#include <VBox/pgm.h>
31	#include <VBox/pdm.h>
32
33	#include <iprt/assert.h>
34	#include <iprt/thread.h>
35	#include <iprt/string.h>
36	#include <iprt/mem.h>
37	#include <iprt/tcp.h>
38	#include <iprt/socket.h>
39	#include <iprt/semaphore.h>
40	#include <iprt/asm.h>
41
42	#include <include/internal/vm.h>
43	#include <include/internal/em.h>
44
45	/*******************************************************************************
46	* Structures and Typedefs *
47	*******************************************************************************/
48
49	/**
50	* TCP stream header.
51	*
52	* This is an extra layer for fixing the problem with figuring out when the SSM
53	* stream ends.
54	*/
55	typedef struct FTMTCPHDR
56	{
57	/** Magic value. */
58	uint32_t u32Magic;
59	/** The size of the data block following this header.
60	* 0 indicates the end of the stream, while UINT32_MAX indicates
61	* cancelation. */
62	uint32_t cb;
63	} FTMTCPHDR;
64	/** Magic value for FTMTCPHDR::u32Magic. (Egberto Gismonti Amin) */
65	#define FTMTCPHDR_MAGIC UINT32_C(0x19471205)
66	/** The max block size. */
67	#define FTMTCPHDR_MAX_SIZE UINT32_C(0x00fffff8)
68
69	/**
70	* TCP stream header.
71	*
72	* This is an extra layer for fixing the problem with figuring out when the SSM
73	* stream ends.
74	*/
75	typedef struct FTMTCPHDRMEM
76	{
77	/** Magic value. */
78	uint32_t u32Magic;
79	/** Size (Uncompressed) of the pages following the header. */
80	uint32_t cbPageRange;
81	/** GC Physical address of the page(s) to sync. */
82	RTGCPHYS GCPhys;
83	/** The size of the data block following this header.
84	* 0 indicates the end of the stream, while UINT32_MAX indicates
85	* cancelation. */
86	uint32_t cb;
87	} FTMTCPHDRMEM;
88
89	/*******************************************************************************
90	* Global Variables *
91	*******************************************************************************/
92	static const char g_szWelcome[] = "VirtualBox-Fault-Tolerance-Sync-1.0\n";
93
94	/**
95	* Initializes the FTM.
96	*
97	* @returns VBox status code.
98	* @param pVM The VM to operate on.
99	*/
100	VMMR3DECL(int) FTMR3Init(PVM pVM)
101	{
102	/*
103	* Assert alignment and sizes.
104	*/
105	AssertCompile(sizeof(pVM->ftm.s) <= sizeof(pVM->ftm.padding));
106	AssertCompileMemberAlignment(FTM, CritSect, sizeof(uintptr_t));
107
108	/** @todo saved state for master nodes! */
109	pVM->ftm.s.pszAddress = NULL;
110	pVM->ftm.s.pszPassword = NULL;
111	pVM->fFaultTolerantMaster = false;
112	pVM->ftm.s.fIsStandbyNode = false;
113	pVM->ftm.s.standby.hServer = NIL_RTTCPSERVER;
114	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
115	pVM->ftm.s.hSocket = NIL_RTSOCKET;
116
117	/*
118	* Initialize the PGM critical section.
119	*/
120	int rc = PDMR3CritSectInit(pVM, &pVM->ftm.s.CritSect, RT_SRC_POS, "FTM");
121	AssertRCReturn(rc, rc);
122
123	/*
124	* Register statistics.
125	*/
126	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedMem, STAMTYPE_COUNTER, "/FT/Received/Mem", STAMUNIT_BYTES, "The amount of memory pages that was received.");
127	STAM_REL_REG(pVM, &pVM->ftm.s.StatReceivedState, STAMTYPE_COUNTER, "/FT/Received/State", STAMUNIT_BYTES, "The amount of state information that was received.");
128	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentMem, STAMTYPE_COUNTER, "/FT/Sent/Mem", STAMUNIT_BYTES, "The amount of memory pages that was sent.");
129	STAM_REL_REG(pVM, &pVM->ftm.s.StatSentState, STAMTYPE_COUNTER, "/FT/Sent/State", STAMUNIT_BYTES, "The amount of state information that was sent.");
130	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaVM, STAMTYPE_COUNTER, "/FT/Sync/DeltaVM", STAMUNIT_OCCURENCES, "Number of delta vm syncs.");
131	STAM_REL_REG(pVM, &pVM->ftm.s.StatFullSync, STAMTYPE_COUNTER, "/FT/Sync/Full", STAMUNIT_OCCURENCES, "Number of full vm syncs.");
132	STAM_REL_REG(pVM, &pVM->ftm.s.StatDeltaMem, STAMTYPE_COUNTER, "/FT/Sync/DeltaMem", STAMUNIT_OCCURENCES, "Number of delta mem syncs.");
133	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointStorage, STAMTYPE_COUNTER, "/FT/Checkpoint/Storage", STAMUNIT_OCCURENCES, "Number of storage checkpoints.");
134	STAM_REL_REG(pVM, &pVM->ftm.s.StatCheckpointNetwork, STAMTYPE_COUNTER, "/FT/Checkpoint/Network", STAMUNIT_OCCURENCES, "Number of network checkpoints.");
135	#ifdef VBOX_WITH_STATISTICS
136	STAM_REG(pVM, &pVM->ftm.s.StatCheckpoint, STAMTYPE_PROFILE, "/FT/Checkpoint", STAMUNIT_TICKS_PER_CALL, "Profiling of FTMR3SetCheckpoint.");
137	#endif
138	return VINF_SUCCESS;
139	}
140
141	/**
142	* Terminates the FTM.
143	*
144	* Termination means cleaning up and freeing all resources,
145	* the VM itself is at this point powered off or suspended.
146	*
147	* @returns VBox status code.
148	* @param pVM The VM to operate on.
149	*/
150	VMMR3DECL(int) FTMR3Term(PVM pVM)
151	{
152	if (pVM->ftm.s.master.hShutdownEvent != NIL_RTSEMEVENT)
153	{
154	RTSemEventDestroy(pVM->ftm.s.master.hShutdownEvent);
155	pVM->ftm.s.master.hShutdownEvent = NIL_RTSEMEVENT;
156	}
157	if (pVM->ftm.s.hSocket != NIL_RTSOCKET)
158	{
159	RTTcpClientClose(pVM->ftm.s.hSocket);
160	pVM->ftm.s.hSocket = NIL_RTSOCKET;
161	}
162	if (pVM->ftm.s.standby.hServer)
163	{
164	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
165	pVM->ftm.s.standby.hServer = NULL;
166	}
167	if (pVM->ftm.s.pszAddress)
168	RTMemFree(pVM->ftm.s.pszAddress);
169	if (pVM->ftm.s.pszPassword)
170	RTMemFree(pVM->ftm.s.pszPassword);
171
172	pVM->ftm.s.pszAddress = NULL;
173	pVM->ftm.s.pszPassword = NULL;
174
175	PDMR3CritSectDelete(&pVM->ftm.s.CritSect);
176	return VINF_SUCCESS;
177	}
178
179
180	static int ftmR3TcpWriteACK(PVM pVM)
181	{
182	int rc = RTTcpWrite(pVM->ftm.s.hSocket, "ACK\n", sizeof("ACK\n") - 1);
183	if (RT_FAILURE(rc))
184	{
185	LogRel(("FTSync: RTTcpWrite(,ACK,) -> %Rrc\n", rc));
186	}
187	return rc;
188	}
189
190
191	static int ftmR3TcpWriteNACK(PVM pVM, int32_t rc2, const char *pszMsgText = NULL)
192	{
193	char szMsg[256];
194	size_t cch;
195	if (pszMsgText && *pszMsgText)
196	{
197	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d;%s\n", rc2, pszMsgText);
198	for (size_t off = 6; off + 1 < cch; off++)
199	if (szMsg[off] == '\n')
200	szMsg[off] = '\r';
201	}
202	else
203	cch = RTStrPrintf(szMsg, sizeof(szMsg), "NACK=%d\n", rc2);
204	int rc = RTTcpWrite(pVM->ftm.s.hSocket, szMsg, cch);
205	if (RT_FAILURE(rc))
206	LogRel(("FTSync: RTTcpWrite(,%s,%zu) -> %Rrc\n", szMsg, cch, rc));
207	return rc;
208	}
209
210	/**
211	* Reads a string from the socket.
212	*
213	* @returns VBox status code.
214	*
215	* @param pState The teleporter state structure.
216	* @param pszBuf The output buffer.
217	* @param cchBuf The size of the output buffer.
218	*
219	*/
220	static int ftmR3TcpReadLine(PVM pVM, char *pszBuf, size_t cchBuf)
221	{
222	char *pszStart = pszBuf;
223	RTSOCKET Sock = pVM->ftm.s.hSocket;
224
225	AssertReturn(cchBuf > 1, VERR_INTERNAL_ERROR);
226	*pszBuf = '\0';
227
228	/* dead simple approach. */
229	for (;;)
230	{
231	char ch;
232	int rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
233	if (RT_FAILURE(rc))
234	{
235	LogRel(("FTSync: RTTcpRead -> %Rrc while reading string ('%s')\n", rc, pszStart));
236	return rc;
237	}
238	if ( ch == '\n'
239	\|\| ch == '\0')
240	return VINF_SUCCESS;
241	if (cchBuf <= 1)
242	{
243	LogRel(("FTSync: String buffer overflow: '%s'\n", pszStart));
244	return VERR_BUFFER_OVERFLOW;
245	}
246	*pszBuf++ = ch;
247	*pszBuf = '\0';
248	cchBuf--;
249	}
250	}
251
252	/**
253	* Reads an ACK or NACK.
254	*
255	* @returns VBox status code.
256	* @param pVM The VM to operate on.
257	* @param pszWhich Which ACK is this this?
258	* @param pszNAckMsg Optional NACK message.
259	*/
260	static int ftmR3TcpReadACK(PVM pVM, const char pszWhich, const char pszNAckMsg = NULL)
261	{
262	char szMsg[256];
263	int rc = ftmR3TcpReadLine(pVM, szMsg, sizeof(szMsg));
264	if (RT_FAILURE(rc))
265	return rc;
266
267	if (!strcmp(szMsg, "ACK"))
268	return VINF_SUCCESS;
269
270	if (!strncmp(szMsg, "NACK=", sizeof("NACK=") - 1))
271	{
272	char *pszMsgText = strchr(szMsg, ';');
273	if (pszMsgText)
274	*pszMsgText++ = '\0';
275
276	int32_t vrc2;
277	rc = RTStrToInt32Full(&szMsg[sizeof("NACK=") - 1], 10, &vrc2);
278	if (rc == VINF_SUCCESS)
279	{
280	/*
281	* Well formed NACK, transform it into an error.
282	*/
283	if (pszNAckMsg)
284	{
285	LogRel(("FTSync: %s: NACK=%Rrc (%d)\n", pszWhich, vrc2, vrc2));
286	return VERR_INTERNAL_ERROR;
287	}
288
289	if (pszMsgText)
290	{
291	pszMsgText = RTStrStrip(pszMsgText);
292	for (size_t off = 0; pszMsgText[off]; off++)
293	if (pszMsgText[off] == '\r')
294	pszMsgText[off] = '\n';
295
296	LogRel(("FTSync: %s: NACK=%Rrc (%d) - '%s'\n", pszWhich, vrc2, vrc2, pszMsgText));
297	}
298	return VERR_INTERNAL_ERROR_2;
299	}
300
301	if (pszMsgText)
302	pszMsgText[-1] = ';';
303	}
304	return VERR_INTERNAL_ERROR_3;
305	}
306
307	/**
308	* Submitts a command to the destination and waits for the ACK.
309	*
310	* @returns VBox status code.
311	*
312	* @param pVM The VM to operate on.
313	* @param pszCommand The command.
314	* @param fWaitForAck Whether to wait for the ACK.
315	*/
316	static int ftmR3TcpSubmitCommand(PVM pVM, const char *pszCommand, bool fWaitForAck = true)
317	{
318	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, pszCommand, strlen(pszCommand), "\n", sizeof("\n") - 1);
319	if (RT_FAILURE(rc))
320	return rc;
321	if (!fWaitForAck)
322	return VINF_SUCCESS;
323	return ftmR3TcpReadACK(pVM, pszCommand);
324	}
325
326	/**
327	* @copydoc SSMSTRMOPS::pfnWrite
328	*/
329	static DECLCALLBACK(int) ftmR3TcpOpWrite(void pvUser, uint64_t offStream, const void pvBuf, size_t cbToWrite)
330	{
331	PVM pVM = (PVM)pvUser;
332
333	AssertReturn(cbToWrite > 0, VINF_SUCCESS);
334	AssertReturn(cbToWrite < UINT32_MAX, VERR_OUT_OF_RANGE);
335	AssertReturn(pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
336
337	for (;;)
338	{
339	FTMTCPHDR Hdr;
340	Hdr.u32Magic = FTMTCPHDR_MAGIC;
341	Hdr.cb = RT_MIN((uint32_t)cbToWrite, FTMTCPHDR_MAX_SIZE);
342	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pvBuf, (size_t)Hdr.cb);
343	if (RT_FAILURE(rc))
344	{
345	LogRel(("FTSync/TCP: Write error: %Rrc (cb=%#x)\n", rc, Hdr.cb));
346	return rc;
347	}
348	pVM->ftm.s.StatSentState.c += Hdr.cb + sizeof(Hdr);
349	pVM->ftm.s.syncstate.uOffStream += Hdr.cb;
350	if (Hdr.cb == cbToWrite)
351	return VINF_SUCCESS;
352
353	/* advance */
354	cbToWrite -= Hdr.cb;
355	pvBuf = (uint8_t const *)pvBuf + Hdr.cb;
356	}
357	}
358
359
360	/**
361	* Selects and poll for close condition.
362	*
363	* We can use a relatively high poll timeout here since it's only used to get
364	* us out of error paths. In the normal cause of events, we'll get a
365	* end-of-stream header.
366	*
367	* @returns VBox status code.
368	*
369	* @param pState The teleporter state data.
370	*/
371	static int ftmR3TcpReadSelect(PVM pVM)
372	{
373	int rc;
374	do
375	{
376	rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 1000);
377	if (RT_FAILURE(rc) && rc != VERR_TIMEOUT)
378	{
379	pVM->ftm.s.syncstate.fIOError = true;
380	LogRel(("FTSync/TCP: Header select error: %Rrc\n", rc));
381	break;
382	}
383	if (pVM->ftm.s.syncstate.fStopReading)
384	{
385	rc = VERR_EOF;
386	break;
387	}
388	} while (rc == VERR_TIMEOUT);
389	return rc;
390	}
391
392
393	/**
394	* @copydoc SSMSTRMOPS::pfnRead
395	*/
396	static DECLCALLBACK(int) ftmR3TcpOpRead(void pvUser, uint64_t offStream, void pvBuf, size_t cbToRead, size_t *pcbRead)
397	{
398	PVM pVM = (PVM)pvUser;
399	AssertReturn(!pVM->fFaultTolerantMaster, VERR_INVALID_HANDLE);
400
401	for (;;)
402	{
403	int rc;
404
405	/*
406	* Check for various conditions and may have been signalled.
407	*/
408	if (pVM->ftm.s.syncstate.fEndOfStream)
409	return VERR_EOF;
410	if (pVM->ftm.s.syncstate.fStopReading)
411	return VERR_EOF;
412	if (pVM->ftm.s.syncstate.fIOError)
413	return VERR_IO_GEN_FAILURE;
414
415	/*
416	* If there is no more data in the current block, read the next
417	* block header.
418	*/
419	if (!pVM->ftm.s.syncstate.cbReadBlock)
420	{
421	rc = ftmR3TcpReadSelect(pVM);
422	if (RT_FAILURE(rc))
423	return rc;
424	FTMTCPHDR Hdr;
425	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
426	if (RT_FAILURE(rc))
427	{
428	pVM->ftm.s.syncstate.fIOError = true;
429	LogRel(("FTSync/TCP: Header read error: %Rrc\n", rc));
430	return rc;
431	}
432	pVM->ftm.s.StatReceivedState.c += sizeof(Hdr);
433
434	if (RT_UNLIKELY( Hdr.u32Magic != FTMTCPHDR_MAGIC
435	\|\| Hdr.cb > FTMTCPHDR_MAX_SIZE
436	\|\| Hdr.cb == 0))
437	{
438	if ( Hdr.u32Magic == FTMTCPHDR_MAGIC
439	&& ( Hdr.cb == 0
440	\|\| Hdr.cb == UINT32_MAX)
441	)
442	{
443	pVM->ftm.s.syncstate.fEndOfStream = true;
444	pVM->ftm.s.syncstate.cbReadBlock = 0;
445	return Hdr.cb ? VERR_SSM_CANCELLED : VERR_EOF;
446	}
447	pVM->ftm.s.syncstate.fIOError = true;
448	LogRel(("FTSync/TCP: Invalid block: u32Magic=%#x cb=%#x\n", Hdr.u32Magic, Hdr.cb));
449	return VERR_IO_GEN_FAILURE;
450	}
451
452	pVM->ftm.s.syncstate.cbReadBlock = Hdr.cb;
453	if (pVM->ftm.s.syncstate.fStopReading)
454	return VERR_EOF;
455	}
456
457	/*
458	* Read more data.
459	*/
460	rc = ftmR3TcpReadSelect(pVM);
461	if (RT_FAILURE(rc))
462	return rc;
463
464	uint32_t cb = (uint32_t)RT_MIN(pVM->ftm.s.syncstate.cbReadBlock, cbToRead);
465	rc = RTTcpRead(pVM->ftm.s.hSocket, pvBuf, cb, pcbRead);
466	if (RT_FAILURE(rc))
467	{
468	pVM->ftm.s.syncstate.fIOError = true;
469	LogRel(("FTSync/TCP: Data read error: %Rrc (cb=%#x)\n", rc, cb));
470	return rc;
471	}
472	if (pcbRead)
473	{
474	cb = (uint32_t)*pcbRead;
475	pVM->ftm.s.StatReceivedState.c += cb;
476	pVM->ftm.s.syncstate.uOffStream += cb;
477	pVM->ftm.s.syncstate.cbReadBlock -= cb;
478	return VINF_SUCCESS;
479	}
480	pVM->ftm.s.StatReceivedState.c += cb;
481	pVM->ftm.s.syncstate.uOffStream += cb;
482	pVM->ftm.s.syncstate.cbReadBlock -= cb;
483	if (cbToRead == cb)
484	return VINF_SUCCESS;
485
486	/* Advance to the next block. */
487	cbToRead -= cb;
488	pvBuf = (uint8_t *)pvBuf + cb;
489	}
490	}
491
492
493	/**
494	* @copydoc SSMSTRMOPS::pfnSeek
495	*/
496	static DECLCALLBACK(int) ftmR3TcpOpSeek(void pvUser, int64_t offSeek, unsigned uMethod, uint64_t poffActual)
497	{
498	return VERR_NOT_SUPPORTED;
499	}
500
501
502	/**
503	* @copydoc SSMSTRMOPS::pfnTell
504	*/
505	static DECLCALLBACK(uint64_t) ftmR3TcpOpTell(void *pvUser)
506	{
507	PVM pVM = (PVM)pvUser;
508	return pVM->ftm.s.syncstate.uOffStream;
509	}
510
511
512	/**
513	* @copydoc SSMSTRMOPS::pfnSize
514	*/
515	static DECLCALLBACK(int) ftmR3TcpOpSize(void pvUser, uint64_t pcb)
516	{
517	return VERR_NOT_SUPPORTED;
518	}
519
520
521	/**
522	* @copydoc SSMSTRMOPS::pfnIsOk
523	*/
524	static DECLCALLBACK(int) ftmR3TcpOpIsOk(void *pvUser)
525	{
526	PVM pVM = (PVM)pvUser;
527
528	if (pVM->fFaultTolerantMaster)
529	{
530	/* Poll for incoming NACKs and errors from the other side */
531	int rc = RTTcpSelectOne(pVM->ftm.s.hSocket, 0);
532	if (rc != VERR_TIMEOUT)
533	{
534	if (RT_SUCCESS(rc))
535	{
536	LogRel(("FTSync/TCP: Incoming data detect by IsOk, assuming it is a cancellation NACK.\n"));
537	rc = VERR_SSM_CANCELLED;
538	}
539	else
540	LogRel(("FTSync/TCP: RTTcpSelectOne -> %Rrc (IsOk).\n", rc));
541	return rc;
542	}
543	}
544
545	return VINF_SUCCESS;
546	}
547
548
549	/**
550	* @copydoc SSMSTRMOPS::pfnClose
551	*/
552	static DECLCALLBACK(int) ftmR3TcpOpClose(void *pvUser, bool fCanceled)
553	{
554	PVM pVM = (PVM)pvUser;
555
556	if (pVM->fFaultTolerantMaster)
557	{
558	FTMTCPHDR EofHdr;
559	EofHdr.u32Magic = FTMTCPHDR_MAGIC;
560	EofHdr.cb = fCanceled ? UINT32_MAX : 0;
561	int rc = RTTcpWrite(pVM->ftm.s.hSocket, &EofHdr, sizeof(EofHdr));
562	if (RT_FAILURE(rc))
563	{
564	LogRel(("FTSync/TCP: EOF Header write error: %Rrc\n", rc));
565	return rc;
566	}
567	}
568	else
569	{
570	ASMAtomicWriteBool(&pVM->ftm.s.syncstate.fStopReading, true);
571	}
572
573	return VINF_SUCCESS;
574	}
575
576
577	/**
578	* Method table for a TCP based stream.
579	*/
580	static SSMSTRMOPS const g_ftmR3TcpOps =
581	{
582	SSMSTRMOPS_VERSION,
583	ftmR3TcpOpWrite,
584	ftmR3TcpOpRead,
585	ftmR3TcpOpSeek,
586	ftmR3TcpOpTell,
587	ftmR3TcpOpSize,
588	ftmR3TcpOpIsOk,
589	ftmR3TcpOpClose,
590	SSMSTRMOPS_VERSION
591	};
592
593
594	/**
595	* VMR3ReqCallWait callback
596	*
597	* @param pVM The VM handle.
598	*
599	*/
600	static DECLCALLBACK(void) ftmR3WriteProtectMemory(PVM pVM)
601	{
602	int rc = PGMR3PhysWriteProtectRAM(pVM);
603	AssertRC(rc);
604	}
605
606
607	/**
608	* Sync the VM state
609	*
610	* @returns VBox status code.
611	* @param pVM The VM handle.
612	*/
613	static int ftmR3PerformFullSync(PVM pVM)
614	{
615	bool fSuspended = false;
616
617	int rc = VMR3Suspend(pVM);
618	AssertRCReturn(rc, rc);
619
620	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatFullSync);
621
622	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
623
624	/* Reset the sync state. */
625	pVM->ftm.s.syncstate.uOffStream = 0;
626	pVM->ftm.s.syncstate.cbReadBlock = 0;
627	pVM->ftm.s.syncstate.fStopReading = false;
628	pVM->ftm.s.syncstate.fIOError = false;
629	pVM->ftm.s.syncstate.fEndOfStream = false;
630
631	rc = ftmR3TcpSubmitCommand(pVM, "full-sync");
632	AssertRC(rc);
633
634	pVM->ftm.s.fDeltaLoadSaveActive = false;
635	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, false /* fSkipStateChanges */);
636	AssertRC(rc);
637
638	rc = ftmR3TcpReadACK(pVM, "full-sync-complete");
639	AssertRC(rc);
640
641	RTSocketRelease(pVM->ftm.s.hSocket);
642
643	/* Write protect all memory. */
644	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
645	AssertRCReturn(rc, rc);
646
647	rc = VMR3Resume(pVM);
648	AssertRC(rc);
649
650	return rc;
651	}
652
653
654	/**
655	* PGMR3PhysEnumDirtyFTPages callback for syncing dirty physical pages
656	*
657	* @param pVM VM Handle.
658	* @param GCPhys GC physical address
659	* @param pRange HC virtual address of the page(s)
660	* @param cbRange Size of the dirty range in bytes.
661	* @param pvUser User argument
662	*/
663	static DECLCALLBACK(int) ftmR3SyncDirtyPage(PVM pVM, RTGCPHYS GCPhys, uint8_t pRange, unsigned cbRange, void pvUser)
664	{
665	FTMTCPHDRMEM Hdr;
666	Hdr.u32Magic = FTMTCPHDR_MAGIC;
667	Hdr.GCPhys = GCPhys;
668	Hdr.cbPageRange = cbRange;
669	Hdr.cb = cbRange;
670	/** @todo compress page(s). */
671	int rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 2, &Hdr, sizeof(Hdr), pRange, (size_t)Hdr.cb);
672	if (RT_FAILURE(rc))
673	{
674	LogRel(("FTSync/TCP: Write error (ftmR3SyncDirtyPage): %Rrc (cb=%#x)\n", rc, Hdr.cb));
675	return rc;
676	}
677	pVM->ftm.s.StatSentMem.c += Hdr.cb + sizeof(Hdr);
678	return (pVM->ftm.s.fCheckpointingActive) ? VERR_INTERRUPTED : VINF_SUCCESS;
679	}
680
681	/**
682	* Thread function which starts syncing process for this master VM
683	*
684	* @param Thread The thread id.
685	* @param pvUser Not used
686	* @return VINF_SUCCESS (ignored).
687	*
688	*/
689	static DECLCALLBACK(int) ftmR3MasterThread(RTTHREAD Thread, void *pvUser)
690	{
691	int rc = VINF_SUCCESS;
692	PVM pVM = (PVM)pvUser;
693
694	for (;;)
695	{
696	/*
697	* Try connect to the standby machine.
698	*/
699	Log(("ftmR3MasterThread: client connect to %s %d\n", pVM->ftm.s.pszAddress, pVM->ftm.s.uPort));
700	rc = RTTcpClientConnect(pVM->ftm.s.pszAddress, pVM->ftm.s.uPort, &pVM->ftm.s.hSocket);
701	if (RT_SUCCESS(rc))
702	{
703	Log(("ftmR3MasterThread: CONNECTED\n"));
704
705	/* Disable Nagle. */
706	rc = RTTcpSetSendCoalescing(pVM->ftm.s.hSocket, false /fEnable/);
707	AssertRC(rc);
708
709	/* Read and check the welcome message. */
710	char szLine[RT_MAX(128, sizeof(g_szWelcome))];
711	RT_ZERO(szLine);
712	rc = RTTcpRead(pVM->ftm.s.hSocket, szLine, sizeof(g_szWelcome) - 1, NULL);
713	if ( RT_SUCCESS(rc)
714	&& !strcmp(szLine, g_szWelcome))
715	{
716	/* password */
717	if (pVM->ftm.s.pszPassword)
718	rc = RTTcpWrite(pVM->ftm.s.hSocket, pVM->ftm.s.pszPassword, strlen(pVM->ftm.s.pszPassword));
719
720	if (RT_SUCCESS(rc))
721	{
722	/* ACK */
723	rc = ftmR3TcpReadACK(pVM, "password", "Invalid password");
724	if (RT_SUCCESS(rc))
725	{
726	/** todo: verify VM config. */
727	break;
728	}
729	}
730	}
731	/* Failed, so don't bother anymore. */
732	return VINF_SUCCESS;
733	}
734	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, 1000 /* 1 second */);
735	if (rc != VERR_TIMEOUT)
736	return VINF_SUCCESS; /* told to quit */
737	}
738
739	/* Successfully initialized the connection to the standby node.
740	* Start the sync process.
741	*/
742
743	/* First sync all memory and write protect everything so
744	* we can send changed pages later on.
745	*/
746
747	rc = ftmR3PerformFullSync(pVM);
748
749	for (;;)
750	{
751	rc = RTSemEventWait(pVM->ftm.s.master.hShutdownEvent, pVM->ftm.s.uInterval);
752	if (rc != VERR_TIMEOUT)
753	break; /* told to quit */
754
755	if (!pVM->ftm.s.fCheckpointingActive)
756	{
757	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
758	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
759
760	rc = ftmR3TcpSubmitCommand(pVM, "mem-sync");
761	AssertRC(rc);
762
763	/* sync the changed memory with the standby node. */
764	/* Write protect all memory. */
765	if (!pVM->ftm.s.fCheckpointingActive)
766	{
767	rc = VMR3ReqCallWait(pVM, VMCPUID_ANY, (PFNRT)ftmR3WriteProtectMemory, 1, pVM);
768	AssertRC(rc);
769	}
770
771	/* Enumerate all dirty pages and send them to the standby VM. */
772	if (!pVM->ftm.s.fCheckpointingActive)
773	{
774	rc = PGMR3PhysEnumDirtyFTPages(pVM, ftmR3SyncDirtyPage, NULL /* pvUser */);
775	Assert(rc == VINF_SUCCESS \|\| rc == VERR_INTERRUPTED);
776	}
777
778	/* Send last memory header to signal the end. */
779	FTMTCPHDRMEM Hdr;
780	Hdr.u32Magic = FTMTCPHDR_MAGIC;
781	Hdr.GCPhys = 0;
782	Hdr.cbPageRange = 0;
783	Hdr.cb = 0;
784	rc = RTTcpSgWriteL(pVM->ftm.s.hSocket, 1, &Hdr, sizeof(Hdr));
785	if (RT_FAILURE(rc))
786	LogRel(("FTSync/TCP: Write error (ftmR3MasterThread): %Rrc (cb=%#x)\n", rc, Hdr.cb));
787
788	rc = ftmR3TcpReadACK(pVM, "mem-sync-complete");
789	AssertRC(rc);
790
791	PDMCritSectLeave(&pVM->ftm.s.CritSect);
792	}
793	}
794	return rc;
795	}
796
797	/**
798	* Listen for incoming traffic destined for the standby VM.
799	*
800	* @copydoc FNRTTCPSERVE
801	*
802	* @returns VINF_SUCCESS or VERR_TCP_SERVER_STOP.
803	*/
804	static DECLCALLBACK(int) ftmR3StandbyServeConnection(RTSOCKET Sock, void *pvUser)
805	{
806	PVM pVM = (PVM)pvUser;
807
808	pVM->ftm.s.hSocket = Sock;
809
810	/*
811	* Disable Nagle.
812	*/
813	int rc = RTTcpSetSendCoalescing(Sock, false /fEnable/);
814	AssertRC(rc);
815
816	/* Send the welcome message to the master node. */
817	rc = RTTcpWrite(Sock, g_szWelcome, sizeof(g_szWelcome) - 1);
818	if (RT_FAILURE(rc))
819	{
820	LogRel(("Teleporter: Failed to write welcome message: %Rrc\n", rc));
821	return VINF_SUCCESS;
822	}
823
824	/*
825	* Password.
826	*/
827	const char *pszPassword = pVM->ftm.s.pszPassword;
828	if (pszPassword)
829	{
830	unsigned off = 0;
831	while (pszPassword[off])
832	{
833	char ch;
834	rc = RTTcpRead(Sock, &ch, sizeof(ch), NULL);
835	if ( RT_FAILURE(rc)
836	\|\| pszPassword[off] != ch)
837	{
838	if (RT_FAILURE(rc))
839	LogRel(("FTSync: Password read failure (off=%u): %Rrc\n", off, rc));
840	else
841	LogRel(("FTSync: Invalid password (off=%u)\n", off));
842	ftmR3TcpWriteNACK(pVM, VERR_AUTHENTICATION_FAILURE);
843	return VINF_SUCCESS;
844	}
845	off++;
846	}
847	}
848	rc = ftmR3TcpWriteACK(pVM);
849	if (RT_FAILURE(rc))
850	return VINF_SUCCESS;
851
852	/** todo: verify VM config. */
853
854	/*
855	* Stop the server.
856	*
857	* Note! After this point we must return VERR_TCP_SERVER_STOP, while prior
858	* to it we must not return that value!
859	*/
860	RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
861
862	/*
863	* Command processing loop.
864	*/
865	bool fDone = false;
866	for (;;)
867	{
868	bool fFullSync = false;
869	char szCmd[128];
870
871	rc = ftmR3TcpReadLine(pVM, szCmd, sizeof(szCmd));
872	if (RT_FAILURE(rc))
873	break;
874
875	if (!strcmp(szCmd, "mem-sync"))
876	{
877	rc = ftmR3TcpWriteACK(pVM);
878	AssertRC(rc);
879	if (RT_FAILURE(rc))
880	continue;
881
882	while (true)
883	{
884	FTMTCPHDRMEM Hdr;
885	void *pPage;
886
887	/* Read memory header. */
888	rc = RTTcpRead(pVM->ftm.s.hSocket, &Hdr, sizeof(Hdr), NULL);
889	if (RT_FAILURE(rc))
890	{
891	Log(("RTTcpRead failed with %Rrc\n", rc));
892	break;
893	}
894	pVM->ftm.s.StatReceivedMem.c += sizeof(Hdr);
895
896	if (Hdr.cb == 0)
897	break; /* end of sync. */
898
899	Assert(Hdr.cb == Hdr.cbPageRange); /** @todo uncompress */
900
901	/* Allocate memory to hold the page(s). */
902	pPage = RTMemAlloc(Hdr.cbPageRange);
903	AssertBreak(pPage);
904
905	/* Fetch the page(s). */
906	rc = RTTcpRead(pVM->ftm.s.hSocket, pPage, Hdr.cb, NULL);
907	if (RT_FAILURE(rc))
908	{
909	Log(("RTTcpRead page data (%d bytes) failed with %Rrc\n", Hdr.cb, rc));
910	break;
911	}
912	pVM->ftm.s.StatReceivedMem.c += Hdr.cb;
913
914	/* Update the guest memory of the standby VM. */
915	#if 1
916	rc = PGMR3PhysWriteExternal(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange, "FTMemSync");
917	#else
918	rc = PGMPhysWrite(pVM, Hdr.GCPhys, pPage, Hdr.cbPageRange);
919	#endif
920	AssertRC(rc);
921
922	RTMemFree(pPage);
923	}
924
925	rc = ftmR3TcpWriteACK(pVM);
926	AssertRC(rc);
927	}
928	else
929	if ( !strcmp(szCmd, "checkpoint")
930	\|\| !strcmp(szCmd, "full-sync")
931	\|\| (fFullSync = true)) /* intended assignment */
932	{
933	rc = ftmR3TcpWriteACK(pVM);
934	AssertRC(rc);
935	if (RT_FAILURE(rc))
936	continue;
937
938	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
939
940	/* Reset the sync state. */
941	pVM->ftm.s.syncstate.uOffStream = 0;
942	pVM->ftm.s.syncstate.cbReadBlock = 0;
943	pVM->ftm.s.syncstate.fStopReading = false;
944	pVM->ftm.s.syncstate.fIOError = false;
945	pVM->ftm.s.syncstate.fEndOfStream = false;
946
947	pVM->ftm.s.fDeltaLoadSaveActive = (fFullSync == false);
948	rc = VMR3LoadFromStreamFT(pVM, &g_ftmR3TcpOps, pVM);
949	pVM->ftm.s.fDeltaLoadSaveActive = false;
950	RTSocketRelease(pVM->ftm.s.hSocket);
951	AssertRC(rc);
952	if (RT_FAILURE(rc))
953	{
954	LogRel(("FTSync: VMR3LoadFromStream -> %Rrc\n", rc));
955	ftmR3TcpWriteNACK(pVM, rc);
956	continue;
957	}
958
959	/* The EOS might not have been read, make sure it is. */
960	pVM->ftm.s.syncstate.fStopReading = false;
961	size_t cbRead;
962	rc = ftmR3TcpOpRead(pVM, pVM->ftm.s.syncstate.uOffStream, szCmd, 1, &cbRead);
963	if (rc != VERR_EOF)
964	{
965	LogRel(("FTSync: Draining teleporterTcpOpRead -> %Rrc\n", rc));
966	ftmR3TcpWriteNACK(pVM, rc);
967	continue;
968	}
969
970	rc = ftmR3TcpWriteACK(pVM);
971	AssertRC(rc);
972	}
973	}
974	LogFlowFunc(("returns mRc=%Rrc\n", rc));
975	return VERR_TCP_SERVER_STOP;
976	}
977
978	/**
979	* Powers on the fault tolerant virtual machine.
980	*
981	* @returns VBox status code.
982	*
983	* @param pVM The VM to operate on.
984	* @param fMaster FT master or standby
985	* @param uInterval FT sync interval
986	* @param pszAddress Standby VM address
987	* @param uPort Standby VM port
988	* @param pszPassword FT password (NULL for none)
989	*
990	* @thread Any thread.
991	* @vmstate Created
992	* @vmstateto PoweringOn+Running (master), PoweringOn+Running_FT (standby)
993	*/
994	VMMR3DECL(int) FTMR3PowerOn(PVM pVM, bool fMaster, unsigned uInterval, const char pszAddress, unsigned uPort, const char pszPassword)
995	{
996	int rc = VINF_SUCCESS;
997
998	VMSTATE enmVMState = VMR3GetState(pVM);
999	AssertMsgReturn(enmVMState == VMSTATE_CREATED,
1000	("%s\n", VMR3GetStateName(enmVMState)),
1001	VERR_INTERNAL_ERROR_4);
1002	AssertReturn(pszAddress, VERR_INVALID_PARAMETER);
1003
1004	if (pVM->ftm.s.uInterval)
1005	pVM->ftm.s.uInterval = uInterval;
1006	else
1007	pVM->ftm.s.uInterval = 50; /* standard sync interval of 50ms */
1008
1009	pVM->ftm.s.uPort = uPort;
1010	pVM->ftm.s.pszAddress = RTStrDup(pszAddress);
1011	if (pszPassword)
1012	pVM->ftm.s.pszPassword = RTStrDup(pszPassword);
1013	if (fMaster)
1014	{
1015	rc = RTSemEventCreate(&pVM->ftm.s.master.hShutdownEvent);
1016	if (RT_FAILURE(rc))
1017	return rc;
1018
1019	rc = RTThreadCreate(NULL, ftmR3MasterThread, pVM,
1020	0, RTTHREADTYPE_IO /* higher than normal priority */, 0, "ftmMaster");
1021	if (RT_FAILURE(rc))
1022	return rc;
1023
1024	pVM->fFaultTolerantMaster = true;
1025	if (PGMIsUsingLargePages(pVM))
1026	{
1027	/* Must disable large page usage as 2 MB pages are too big to write monitor. */
1028	LogRel(("FTSync: disabling large page usage.\n"));
1029	PGMSetLargePageUsage(pVM, false);
1030	}
1031	/** @todo might need to disable page fusion as well */
1032
1033	return VMR3PowerOn(pVM);
1034	}
1035	else
1036	{
1037	/* standby */
1038	rc = RTTcpServerCreateEx(pszAddress, uPort, &pVM->ftm.s.standby.hServer);
1039	if (RT_FAILURE(rc))
1040	return rc;
1041	pVM->ftm.s.fIsStandbyNode = true;
1042
1043	rc = RTTcpServerListen(pVM->ftm.s.standby.hServer, ftmR3StandbyServeConnection, pVM);
1044	/** @todo deal with the exit code to check if we should activate this standby VM. */
1045
1046	if (pVM->ftm.s.standby.hServer)
1047	{
1048	RTTcpServerDestroy(pVM->ftm.s.standby.hServer);
1049	pVM->ftm.s.standby.hServer = NULL;
1050	}
1051	if (rc == VERR_TCP_SERVER_SHUTDOWN)
1052	rc = VINF_SUCCESS; /* ignore this error; the standby process was cancelled. */
1053	}
1054	return rc;
1055	}
1056
1057	/**
1058	* Powers off the fault tolerant virtual machine (standby).
1059	*
1060	* @returns VBox status code.
1061	*
1062	* @param pVM The VM to operate on.
1063	*/
1064	VMMR3DECL(int) FTMR3CancelStandby(PVM pVM)
1065	{
1066	AssertReturn(!pVM->fFaultTolerantMaster, VERR_NOT_SUPPORTED);
1067	Assert(pVM->ftm.s.standby.hServer);
1068
1069	return RTTcpServerShutdown(pVM->ftm.s.standby.hServer);
1070	}
1071
1072	/**
1073	* Rendezvous callback used by FTMR3SetCheckpoint
1074	* Sync state + changed memory with the standby node.
1075	*
1076	* This is only called on one of the EMTs while the other ones are waiting for
1077	* it to complete this function.
1078	*
1079	* @returns VINF_SUCCESS (VBox strict status code).
1080	* @param pVM The VM handle.
1081	* @param pVCpu The VMCPU for the EMT we're being called on. Unused.
1082	* @param pvUser User parameter
1083	*/
1084	static DECLCALLBACK(VBOXSTRICTRC) ftmR3SetCheckpointRendezvous(PVM pVM, PVMCPU pVCpu, void *pvUser)
1085	{
1086	int rc = VINF_SUCCESS;
1087	bool fSuspended = false;
1088
1089	/** We don't call VMR3Suspend here to avoid the overhead of state changes and notifications. This
1090	* is only a short suspend.
1091	*/
1092	PDMR3Suspend(pVM);
1093
1094	/** Hack alert: as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1095	EMR3NotifySuspend(pVM);
1096
1097	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatDeltaVM);
1098
1099	RTSocketRetain(pVM->ftm.s.hSocket); /* For concurrent access by I/O thread and EMT. */
1100
1101	/* Reset the sync state. */
1102	pVM->ftm.s.syncstate.uOffStream = 0;
1103	pVM->ftm.s.syncstate.cbReadBlock = 0;
1104	pVM->ftm.s.syncstate.fStopReading = false;
1105	pVM->ftm.s.syncstate.fIOError = false;
1106	pVM->ftm.s.syncstate.fEndOfStream = false;
1107
1108	rc = ftmR3TcpSubmitCommand(pVM, "checkpoint");
1109	AssertRC(rc);
1110
1111	pVM->ftm.s.fDeltaLoadSaveActive = true;
1112	rc = VMR3SaveFT(pVM, &g_ftmR3TcpOps, pVM, &fSuspended, true /* fSkipStateChanges */);
1113	pVM->ftm.s.fDeltaLoadSaveActive = false;
1114	AssertRC(rc);
1115
1116	rc = ftmR3TcpReadACK(pVM, "checkpoint-complete");
1117	AssertRC(rc);
1118
1119	RTSocketRelease(pVM->ftm.s.hSocket);
1120
1121	/* Write protect all memory. */
1122	rc = PGMR3PhysWriteProtectRAM(pVM);
1123	AssertRC(rc);
1124
1125	/** We don't call VMR3Resume here to avoid the overhead of state changes and notifications. This
1126	* is only a short suspend.
1127	*/
1128	PDMR3Resume(pVM);
1129
1130	/** Hack alert as EM is responsible for dealing with the suspend state. We must do this here ourselves, but only for this EMT.*/
1131	EMR3NotifyResume(pVM);
1132	return rc;
1133	}
1134
1135	/**
1136	* Performs a full sync to the standby node
1137	*
1138	* @returns VBox status code.
1139	*
1140	* @param pVM The VM to operate on.
1141	* @param enmCheckpoint Checkpoint type
1142	*/
1143	VMMR3DECL(int) FTMR3SetCheckpoint(PVM pVM, FTMCHECKPOINTTYPE enmCheckpoint)
1144	{
1145	int rc;
1146
1147	if (!pVM->fFaultTolerantMaster)
1148	return VINF_SUCCESS;
1149
1150	switch (enmCheckpoint)
1151	{
1152	case FTMCHECKPOINTTYPE_NETWORK:
1153	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointNetwork);
1154	break;
1155
1156	case FTMCHECKPOINTTYPE_STORAGE:
1157	STAM_REL_COUNTER_INC(&pVM->ftm.s.StatCheckpointStorage);
1158	break;
1159
1160	default:
1161	break;
1162	}
1163	pVM->ftm.s.fCheckpointingActive = true;
1164	if (VM_IS_EMT(pVM))
1165	{
1166	PVMCPU pVCpu = VMMGetCpu(pVM);
1167
1168	/* We must take special care here as the memory sync is competing with us and requires a responsive EMT. */
1169	while ((rc = PDMCritSectTryEnter(&pVM->ftm.s.CritSect)) == VERR_SEM_BUSY)
1170	{
1171	if (VM_FF_ISPENDING(pVM, VM_FF_EMT_RENDEZVOUS))
1172	{
1173	rc = VMMR3EmtRendezvousFF(pVM, pVCpu);
1174	AssertRC(rc);
1175	}
1176
1177	if (VM_FF_ISPENDING(pVM, VM_FF_REQUEST))
1178	{
1179	rc = VMR3ReqProcessU(pVM->pUVM, VMCPUID_ANY);
1180	AssertRC(rc);
1181	}
1182	}
1183	}
1184	else
1185	rc = PDMCritSectEnter(&pVM->ftm.s.CritSect, VERR_SEM_BUSY);
1186
1187	AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc));
1188
1189	STAM_PROFILE_START(&pVM->ftm.s.StatCheckpoint, a);
1190
1191	rc = VMMR3EmtRendezvous(pVM, VMMEMTRENDEZVOUS_FLAGS_TYPE_ONCE, ftmR3SetCheckpointRendezvous, NULL);
1192
1193	STAM_PROFILE_STOP(&pVM->ftm.s.StatCheckpoint, a);
1194
1195	PDMCritSectLeave(&pVM->ftm.s.CritSect);
1196	pVM->ftm.s.fCheckpointingActive = false;
1197
1198	return rc;
1199	}

Note: See TracBrowser for help on using the repository browser.

source: vbox/trunk/src/VBox/VMM/FTM.cpp@ 32215

Download in other formats: