VirtualBox

source: vbox/trunk/src/VBox/Additions/linux/sharedfolders/regops.c@ 95210

Last change on this file since 95210 was 95210, checked in by vboxsync, 2 years ago

Additions: Linux: vboxsf: Initial support for kernel 5.19(-rc1), bugref:10237.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 149.0 KB
Line 
1/* $Id: regops.c 95210 2022-06-06 19:34:35Z vboxsync $ */
2/** @file
3 * vboxsf - VBox Linux Shared Folders VFS, regular file inode and file operations.
4 */
5
6/*
7 * Copyright (C) 2006-2022 Oracle Corporation
8 *
9 * Permission is hereby granted, free of charge, to any person
10 * obtaining a copy of this software and associated documentation
11 * files (the "Software"), to deal in the Software without
12 * restriction, including without limitation the rights to use,
13 * copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the
15 * Software is furnished to do so, subject to the following
16 * conditions:
17 *
18 * The above copyright notice and this permission notice shall be
19 * included in all copies or substantial portions of the Software.
20 *
21 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
22 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
23 * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
24 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
25 * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
26 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
27 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 * OTHER DEALINGS IN THE SOFTWARE.
29 */
30
31
32/*********************************************************************************************************************************
33* Header Files *
34*********************************************************************************************************************************/
35#include "vfsmod.h"
36#include <linux/uio.h>
37#if RTLNX_VER_MIN(2,5,32)
38# include <linux/aio.h> /* struct kiocb before 4.1 */
39#endif
40#if RTLNX_VER_MIN(2,5,12)
41# include <linux/buffer_head.h>
42#endif
43#if RTLNX_VER_RANGE(2,5,12, 2,6,31)
44# include <linux/writeback.h>
45#endif
46#if RTLNX_VER_RANGE(2,6,23, 3,16,0)
47# include <linux/splice.h>
48#endif
49#if RTLNX_VER_RANGE(2,6,17, 2,6,23)
50# include <linux/pipe_fs_i.h>
51#endif
52#if RTLNX_VER_MIN(2,4,10)
53# include <linux/swap.h> /* for mark_page_accessed */
54#endif
55#include <iprt/err.h>
56
57#if RTLNX_VER_MAX(2,6,18)
58# define SEEK_END 2
59#endif
60
61#if RTLNX_VER_MAX(3,16,0)
62# define iter_is_iovec(a_pIter) ( !((a_pIter)->type & ITER_KVEC) )
63#elif RTLNX_VER_MAX(3,19,0)
64# define iter_is_iovec(a_pIter) ( !((a_pIter)->type & (ITER_KVEC | ITER_BVEC)) )
65#endif
66
67#if RTLNX_VER_MAX(4,17,0)
68# define vm_fault_t int
69#endif
70
71#if RTLNX_VER_MAX(2,5,20)
72# define pgoff_t unsigned long
73#endif
74
75#if RTLNX_VER_MAX(2,5,12)
76# define PageUptodate(a_pPage) Page_Uptodate(a_pPage)
77#endif
78
79
80/*********************************************************************************************************************************
81* Defined Constants And Macros *
82*********************************************************************************************************************************/
83/** @def VBSF_GET_ITER_TYPE
84 * Accessor for getting iov iter type member which changed name in 5.14. */
85#if RTLNX_VER_MIN(5,14,0)
86# define VBSF_GET_ITER_TYPE(a_pIter) ((a_pIter)->iter_type)
87#else
88# define VBSF_GET_ITER_TYPE(a_pIter) ((a_pIter)->type)
89#endif
90
91
92/*********************************************************************************************************************************
93* Structures and Typedefs *
94*********************************************************************************************************************************/
95#if RTLNX_VER_MAX(3,16,0)
96struct vbsf_iov_iter {
97 unsigned int type;
98 unsigned int v_write : 1;
99 size_t iov_offset;
100 size_t nr_segs;
101 struct iovec const *iov;
102# ifdef VBOX_STRICT
103 struct iovec const *iov_org;
104 size_t nr_segs_org;
105# endif
106};
107# ifdef VBOX_STRICT
108# define VBSF_IOV_ITER_INITIALIZER(a_cSegs, a_pIov, a_fWrite) \
109 { vbsf_iov_iter_detect_type(a_pIov, a_cSegs), a_fWrite, 0, a_cSegs, a_pIov, a_pIov, a_cSegs }
110# else
111# define VBSF_IOV_ITER_INITIALIZER(a_cSegs, a_pIov, a_fWrite) \
112 { vbsf_iov_iter_detect_type(a_pIov, a_cSegs), a_fWrite, 0, a_cSegs, a_pIov }
113# endif
114# define ITER_KVEC 1
115# define iov_iter vbsf_iov_iter
116#endif
117
118#if RTLNX_VER_MIN(2,6,19)
119/** Used by vbsf_iter_lock_pages() to keep the first page of the next segment. */
120struct vbsf_iter_stash {
121 struct page *pPage;
122 size_t off;
123 size_t cb;
124# if RTLNX_VER_MAX(4,11,0)
125 size_t offFromEnd;
126 struct iov_iter Copy;
127# endif
128};
129#endif /* >= 3.16.0 */
130/** Initializer for struct vbsf_iter_stash. */
131#if RTLNX_VER_MIN(4,11,0)
132# define VBSF_ITER_STASH_INITIALIZER { NULL, 0 }
133#else
134# define VBSF_ITER_STASH_INITIALIZER { NULL, 0, ~(size_t)0 }
135#endif
136
137
138/*********************************************************************************************************************************
139* Internal Functions *
140*********************************************************************************************************************************/
141DECLINLINE(void) vbsf_put_page(struct page *pPage);
142static void vbsf_unlock_user_pages(struct page **papPages, size_t cPages, bool fSetDirty, bool fLockPgHack);
143static void vbsf_reg_write_sync_page_cache(struct address_space *mapping, loff_t offFile, uint32_t cbRange,
144 uint8_t const *pbSrcBuf, struct page **papSrcPages,
145 uint32_t offSrcPage, size_t cSrcPages);
146
147
148/*********************************************************************************************************************************
149* Provide more recent uio.h functionality to older kernels. *
150*********************************************************************************************************************************/
151#if RTLNX_VER_RANGE(2,6,19, 3,16,0)
152
153/**
154 * Detects the vector type.
155 */
156static int vbsf_iov_iter_detect_type(struct iovec const *paIov, size_t cSegs)
157{
158 /* Check the first segment with a non-zero length. */
159 while (cSegs-- > 0) {
160 if (paIov->iov_len > 0) {
161 if (access_ok(VERIFY_READ, paIov->iov_base, paIov->iov_len))
162#if RTLNX_VER_MIN(5,10,0)
163 return (uintptr_t)paIov->iov_base >= TASK_SIZE_MAX ? ITER_KVEC : 0;
164#else
165 return (uintptr_t)paIov->iov_base >= USER_DS.seg ? ITER_KVEC : 0;
166#endif
167 AssertMsgFailed(("%p LB %#zx\n", paIov->iov_base, paIov->iov_len));
168 break;
169 }
170 paIov++;
171 }
172 return 0;
173}
174
175
176# undef iov_iter_count
177# define iov_iter_count(a_pIter) vbsf_iov_iter_count(a_pIter)
178static size_t vbsf_iov_iter_count(struct vbsf_iov_iter const *iter)
179{
180 size_t cbRet = 0;
181 size_t cLeft = iter->nr_segs;
182 struct iovec const *iov = iter->iov;
183 while (cLeft-- > 0) {
184 cbRet += iov->iov_len;
185 iov++;
186 }
187 return cbRet - iter->iov_offset;
188}
189
190
191# undef iov_iter_single_seg_count
192# define iov_iter_single_seg_count(a_pIter) vbsf_iov_iter_single_seg_count(a_pIter)
193static size_t vbsf_iov_iter_single_seg_count(struct vbsf_iov_iter const *iter)
194{
195 if (iter->nr_segs > 0)
196 return iter->iov->iov_len - iter->iov_offset;
197 return 0;
198}
199
200
201# undef iov_iter_advance
202# define iov_iter_advance(a_pIter, a_cbSkip) vbsf_iov_iter_advance(a_pIter, a_cbSkip)
203static void vbsf_iov_iter_advance(struct vbsf_iov_iter *iter, size_t cbSkip)
204{
205 SFLOG2(("vbsf_iov_iter_advance: cbSkip=%#zx\n", cbSkip));
206 if (iter->nr_segs > 0) {
207 size_t const cbLeftCur = iter->iov->iov_len - iter->iov_offset;
208 Assert(iter->iov_offset <= iter->iov->iov_len);
209 if (cbLeftCur > cbSkip) {
210 iter->iov_offset += cbSkip;
211 } else {
212 cbSkip -= cbLeftCur;
213 iter->iov_offset = 0;
214 iter->iov++;
215 iter->nr_segs--;
216 while (iter->nr_segs > 0) {
217 size_t const cbSeg = iter->iov->iov_len;
218 if (cbSeg > cbSkip) {
219 iter->iov_offset = cbSkip;
220 break;
221 }
222 cbSkip -= cbSeg;
223 iter->iov++;
224 iter->nr_segs--;
225 }
226 }
227 }
228}
229
230
231# undef iov_iter_get_pages
232# define iov_iter_get_pages(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0) \
233 vbsf_iov_iter_get_pages(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0)
234static ssize_t vbsf_iov_iter_get_pages(struct vbsf_iov_iter *iter, struct page **papPages,
235 size_t cbMax, unsigned cMaxPages, size_t *poffPg0)
236{
237 while (iter->nr_segs > 0) {
238 size_t const cbLeft = iter->iov->iov_len - iter->iov_offset;
239 Assert(iter->iov->iov_len >= iter->iov_offset);
240 if (cbLeft > 0) {
241 uintptr_t uPtrFrom = (uintptr_t)iter->iov->iov_base + iter->iov_offset;
242 size_t offPg0 = *poffPg0 = uPtrFrom & PAGE_OFFSET_MASK;
243 size_t cPagesLeft = RT_ALIGN_Z(offPg0 + cbLeft, PAGE_SIZE) >> PAGE_SHIFT;
244 size_t cPages = RT_MIN(cPagesLeft, cMaxPages);
245 struct task_struct *pTask = current;
246 size_t cPagesLocked;
247
248 down_read(&pTask->mm->mmap_sem);
249 cPagesLocked = get_user_pages(pTask, pTask->mm, uPtrFrom, cPages, iter->v_write, 1 /*force*/, papPages, NULL);
250 up_read(&pTask->mm->mmap_sem);
251 if (cPagesLocked == cPages) {
252 size_t cbRet = (cPages << PAGE_SHIFT) - offPg0;
253 if (cPages == cPagesLeft) {
254 size_t offLastPg = (uPtrFrom + cbLeft) & PAGE_OFFSET_MASK;
255 if (offLastPg)
256 cbRet -= PAGE_SIZE - offLastPg;
257 }
258 Assert(cbRet <= cbLeft);
259 return cbRet;
260 }
261 if (cPagesLocked > 0)
262 vbsf_unlock_user_pages(papPages, cPagesLocked, false /*fSetDirty*/, false /*fLockPgHack*/);
263 return -EFAULT;
264 }
265 iter->iov_offset = 0;
266 iter->iov++;
267 iter->nr_segs--;
268 }
269 AssertFailed();
270 return 0;
271}
272
273
274# undef iov_iter_truncate
275# define iov_iter_truncate(iter, cbNew) vbsf_iov_iter_truncate(iter, cbNew)
276static void vbsf_iov_iter_truncate(struct vbsf_iov_iter *iter, size_t cbNew)
277{
278 /* we have no counter or stuff, so it's a no-op. */
279 RT_NOREF(iter, cbNew);
280}
281
282
283# undef iov_iter_revert
284# define iov_iter_revert(a_pIter, a_cbRewind) vbsf_iov_iter_revert(a_pIter, a_cbRewind)
285void vbsf_iov_iter_revert(struct vbsf_iov_iter *iter, size_t cbRewind)
286{
287 SFLOG2(("vbsf_iov_iter_revert: cbRewind=%#zx\n", cbRewind));
288 if (iter->iov_offset > 0) {
289 if (cbRewind <= iter->iov_offset) {
290 iter->iov_offset -= cbRewind;
291 return;
292 }
293 cbRewind -= iter->iov_offset;
294 iter->iov_offset = 0;
295 }
296
297 while (cbRewind > 0) {
298 struct iovec const *pIov = --iter->iov;
299 size_t const cbSeg = pIov->iov_len;
300 iter->nr_segs++;
301
302 Assert((uintptr_t)pIov >= (uintptr_t)iter->iov_org);
303 Assert(iter->nr_segs <= iter->nr_segs_org);
304
305 if (cbRewind <= cbSeg) {
306 iter->iov_offset = cbSeg - cbRewind;
307 break;
308 }
309 cbRewind -= cbSeg;
310 }
311}
312
313#endif /* 2.6.19 <= linux < 3.16.0 */
314#if RTLNX_VER_RANGE(3,16,0, 3,16,35)
315
316/** This is for implementing cMaxPage on 3.16 which doesn't have it. */
317static ssize_t vbsf_iov_iter_get_pages_3_16(struct iov_iter *iter, struct page **papPages,
318 size_t cbMax, unsigned cMaxPages, size_t *poffPg0)
319{
320 if (!(iter->type & ITER_BVEC)) {
321 size_t const offPg0 = iter->iov_offset & PAGE_OFFSET_MASK;
322 size_t const cbMaxPages = ((size_t)cMaxPages << PAGE_SHIFT) - offPg0;
323 if (cbMax > cbMaxPages)
324 cbMax = cbMaxPages;
325 }
326 /* else: BVEC works a page at a time and shouldn't have much of a problem here. */
327 return iov_iter_get_pages(iter, papPages, cbMax, poffPg0);
328}
329# undef iov_iter_get_pages
330# define iov_iter_get_pages(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0) \
331 vbsf_iov_iter_get_pages_3_16(a_pIter, a_papPages, a_cbMax, a_cMaxPages, a_poffPg0)
332
333#endif /* 3.16.0-3.16.34 */
334#if RTLNX_VER_RANGE(2,6,19, 3,18,0)
335
336static size_t copy_from_iter(uint8_t *pbDst, size_t cbToCopy, struct iov_iter *pSrcIter)
337{
338 size_t const cbTotal = cbToCopy;
339 Assert(iov_iter_count(pSrcIter) >= cbToCopy);
340# if RTLNX_VER_MIN(3,16,0)
341 if (pSrcIter->type & ITER_BVEC) {
342 while (cbToCopy > 0) {
343 size_t const offPage = (uintptr_t)pbDst & PAGE_OFFSET_MASK;
344 size_t const cbThisCopy = RT_MIN(PAGE_SIZE - offPage, cbToCopy);
345 struct page *pPage = rtR0MemObjLinuxVirtToPage(pbDst);
346 size_t cbCopied = copy_page_from_iter(pPage, offPage, cbThisCopy, pSrcIter);
347 AssertStmt(cbCopied <= cbThisCopy, cbCopied = cbThisCopy);
348 pbDst += cbCopied;
349 cbToCopy -= cbCopied;
350 if (cbCopied != cbToCopy)
351 break;
352 }
353 } else
354# endif
355 {
356 while (cbToCopy > 0) {
357 size_t cbThisCopy = iov_iter_single_seg_count(pSrcIter);
358 if (cbThisCopy > 0) {
359 if (cbThisCopy > cbToCopy)
360 cbThisCopy = cbToCopy;
361 if (pSrcIter->type & ITER_KVEC)
362 memcpy(pbDst, (void *)pSrcIter->iov->iov_base + pSrcIter->iov_offset, cbThisCopy);
363 else if (copy_from_user(pbDst, pSrcIter->iov->iov_base + pSrcIter->iov_offset, cbThisCopy) != 0)
364 break;
365 pbDst += cbThisCopy;
366 cbToCopy -= cbThisCopy;
367 }
368 iov_iter_advance(pSrcIter, cbThisCopy);
369 }
370 }
371 return cbTotal - cbToCopy;
372}
373
374
375static size_t copy_to_iter(uint8_t const *pbSrc, size_t cbToCopy, struct iov_iter *pDstIter)
376{
377 size_t const cbTotal = cbToCopy;
378 Assert(iov_iter_count(pDstIter) >= cbToCopy);
379# if RTLNX_VER_MIN(3,16,0)
380 if (pDstIter->type & ITER_BVEC) {
381 while (cbToCopy > 0) {
382 size_t const offPage = (uintptr_t)pbSrc & PAGE_OFFSET_MASK;
383 size_t const cbThisCopy = RT_MIN(PAGE_SIZE - offPage, cbToCopy);
384 struct page *pPage = rtR0MemObjLinuxVirtToPage((void *)pbSrc);
385 size_t cbCopied = copy_page_to_iter(pPage, offPage, cbThisCopy, pDstIter);
386 AssertStmt(cbCopied <= cbThisCopy, cbCopied = cbThisCopy);
387 pbSrc += cbCopied;
388 cbToCopy -= cbCopied;
389 if (cbCopied != cbToCopy)
390 break;
391 }
392 } else
393# endif
394 {
395 while (cbToCopy > 0) {
396 size_t cbThisCopy = iov_iter_single_seg_count(pDstIter);
397 if (cbThisCopy > 0) {
398 if (cbThisCopy > cbToCopy)
399 cbThisCopy = cbToCopy;
400 if (pDstIter->type & ITER_KVEC)
401 memcpy((void *)pDstIter->iov->iov_base + pDstIter->iov_offset, pbSrc, cbThisCopy);
402 else if (copy_to_user(pDstIter->iov->iov_base + pDstIter->iov_offset, pbSrc, cbThisCopy) != 0) {
403 break;
404 }
405 pbSrc += cbThisCopy;
406 cbToCopy -= cbThisCopy;
407 }
408 iov_iter_advance(pDstIter, cbThisCopy);
409 }
410 }
411 return cbTotal - cbToCopy;
412}
413
414#endif /* 3.16.0 <= linux < 3.18.0 */
415
416
417
418/*********************************************************************************************************************************
419* Handle management *
420*********************************************************************************************************************************/
421
422/**
423 * Called when an inode is released to unlink all handles that might impossibly
424 * still be associated with it.
425 *
426 * @param pInodeInfo The inode which handles to drop.
427 */
428void vbsf_handle_drop_chain(struct vbsf_inode_info *pInodeInfo)
429{
430 struct vbsf_handle *pCur, *pNext;
431 unsigned long fSavedFlags;
432 SFLOGFLOW(("vbsf_handle_drop_chain: %p\n", pInodeInfo));
433 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
434
435 RTListForEachSafe(&pInodeInfo->HandleList, pCur, pNext, struct vbsf_handle, Entry) {
436 AssertMsg( (pCur->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST))
437 == (VBSF_HANDLE_F_MAGIC | VBSF_HANDLE_F_ON_LIST), ("%p %#x\n", pCur, pCur->fFlags));
438 pCur->fFlags |= VBSF_HANDLE_F_ON_LIST;
439 RTListNodeRemove(&pCur->Entry);
440 }
441
442 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
443}
444
445
446/**
447 * Locates a handle that matches all the flags in @a fFlags.
448 *
449 * @returns Pointer to handle on success (retained), use vbsf_handle_release() to
450 * release it. NULL if no suitable handle was found.
451 * @param pInodeInfo The inode info to search.
452 * @param fFlagsSet The flags that must be set.
453 * @param fFlagsClear The flags that must be clear.
454 */
455struct vbsf_handle *vbsf_handle_find(struct vbsf_inode_info *pInodeInfo, uint32_t fFlagsSet, uint32_t fFlagsClear)
456{
457 struct vbsf_handle *pCur;
458 unsigned long fSavedFlags;
459 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
460
461 RTListForEach(&pInodeInfo->HandleList, pCur, struct vbsf_handle, Entry) {
462 AssertMsg( (pCur->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST))
463 == (VBSF_HANDLE_F_MAGIC | VBSF_HANDLE_F_ON_LIST), ("%p %#x\n", pCur, pCur->fFlags));
464 if ((pCur->fFlags & (fFlagsSet | fFlagsClear)) == fFlagsSet) {
465 uint32_t cRefs = ASMAtomicIncU32(&pCur->cRefs);
466 if (cRefs > 1) {
467 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
468 SFLOGFLOW(("vbsf_handle_find: returns %p\n", pCur));
469 return pCur;
470 }
471 /* Oops, already being closed (safe as it's only ever increased here). */
472 ASMAtomicDecU32(&pCur->cRefs);
473 }
474 }
475
476 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
477 SFLOGFLOW(("vbsf_handle_find: returns NULL!\n"));
478 return NULL;
479}
480
481
482/**
483 * Slow worker for vbsf_handle_release() that does the freeing.
484 *
485 * @returns 0 (ref count).
486 * @param pHandle The handle to release.
487 * @param pSuperInfo The info structure for the shared folder associated with
488 * the handle.
489 * @param pszCaller The caller name (for logging failures).
490 */
491uint32_t vbsf_handle_release_slow(struct vbsf_handle *pHandle, struct vbsf_super_info *pSuperInfo, const char *pszCaller)
492{
493 int rc;
494 unsigned long fSavedFlags;
495
496 SFLOGFLOW(("vbsf_handle_release_slow: %p (%s)\n", pHandle, pszCaller));
497
498 /*
499 * Remove from the list.
500 */
501 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
502
503 AssertMsg((pHandle->fFlags & VBSF_HANDLE_F_MAGIC_MASK) == VBSF_HANDLE_F_MAGIC, ("%p %#x\n", pHandle, pHandle->fFlags));
504 Assert(pHandle->pInodeInfo);
505 Assert(pHandle->pInodeInfo && pHandle->pInodeInfo->u32Magic == SF_INODE_INFO_MAGIC);
506
507 if (pHandle->fFlags & VBSF_HANDLE_F_ON_LIST) {
508 pHandle->fFlags &= ~VBSF_HANDLE_F_ON_LIST;
509 RTListNodeRemove(&pHandle->Entry);
510 }
511
512 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
513
514 /*
515 * Actually destroy it.
516 */
517 rc = VbglR0SfHostReqCloseSimple(pSuperInfo->map.root, pHandle->hHost);
518 if (RT_FAILURE(rc))
519 LogFunc(("Caller %s: VbglR0SfHostReqCloseSimple %#RX64 failed with rc=%Rrc\n", pszCaller, pHandle->hHost, rc));
520 pHandle->hHost = SHFL_HANDLE_NIL;
521 pHandle->fFlags = VBSF_HANDLE_F_MAGIC_DEAD;
522 kfree(pHandle);
523 return 0;
524}
525
526
527/**
528 * Appends a handle to a handle list.
529 *
530 * @param pInodeInfo The inode to add it to.
531 * @param pHandle The handle to add.
532 */
533void vbsf_handle_append(struct vbsf_inode_info *pInodeInfo, struct vbsf_handle *pHandle)
534{
535#ifdef VBOX_STRICT
536 struct vbsf_handle *pCur;
537#endif
538 unsigned long fSavedFlags;
539
540 SFLOGFLOW(("vbsf_handle_append: %p (to %p)\n", pHandle, pInodeInfo));
541 AssertMsg((pHandle->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST)) == VBSF_HANDLE_F_MAGIC,
542 ("%p %#x\n", pHandle, pHandle->fFlags));
543 Assert(pInodeInfo->u32Magic == SF_INODE_INFO_MAGIC);
544
545 spin_lock_irqsave(&g_SfHandleLock, fSavedFlags);
546
547 AssertMsg((pHandle->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST)) == VBSF_HANDLE_F_MAGIC,
548 ("%p %#x\n", pHandle, pHandle->fFlags));
549#ifdef VBOX_STRICT
550 RTListForEach(&pInodeInfo->HandleList, pCur, struct vbsf_handle, Entry) {
551 Assert(pCur != pHandle);
552 AssertMsg( (pCur->fFlags & (VBSF_HANDLE_F_MAGIC_MASK | VBSF_HANDLE_F_ON_LIST))
553 == (VBSF_HANDLE_F_MAGIC | VBSF_HANDLE_F_ON_LIST), ("%p %#x\n", pCur, pCur->fFlags));
554 }
555 pHandle->pInodeInfo = pInodeInfo;
556#endif
557
558 pHandle->fFlags |= VBSF_HANDLE_F_ON_LIST;
559 RTListAppend(&pInodeInfo->HandleList, &pHandle->Entry);
560
561 spin_unlock_irqrestore(&g_SfHandleLock, fSavedFlags);
562}
563
564
565
566/*********************************************************************************************************************************
567* Misc *
568*********************************************************************************************************************************/
569
570#if RTLNX_VER_MAX(2,6,6)
571/** Any writable mappings? */
572DECLINLINE(bool) mapping_writably_mapped(struct address_space const *mapping)
573{
574# if RTLNX_VER_MIN(2,5,6)
575 return !list_empty(&mapping->i_mmap_shared);
576# else
577 return mapping->i_mmap_shared != NULL;
578# endif
579}
580#endif
581
582
583#if RTLNX_VER_MAX(2,5,12)
584/** Missing in 2.4.x, so just stub it for now. */
585DECLINLINE(bool) PageWriteback(struct page const *page)
586{
587 return false;
588}
589#endif
590
591
592/**
593 * Helper for deciding wheter we should do a read via the page cache or not.
594 *
595 * By default we will only use the page cache if there is a writable memory
596 * mapping of the file with a chance that it may have modified any of the pages
597 * already.
598 */
599DECLINLINE(bool) vbsf_should_use_cached_read(struct file *file, struct address_space *mapping, struct vbsf_super_info *pSuperInfo)
600{
601 if ( (file->f_flags & O_DIRECT)
602 || pSuperInfo->enmCacheMode == kVbsfCacheMode_None)
603 return false;
604 if ( pSuperInfo->enmCacheMode == kVbsfCacheMode_Read
605 || pSuperInfo->enmCacheMode == kVbsfCacheMode_ReadWrite)
606 return true;
607 Assert(pSuperInfo->enmCacheMode == kVbsfCacheMode_Strict);
608 return mapping
609 && mapping->nrpages > 0
610 && mapping_writably_mapped(mapping);
611}
612
613
614
615/*********************************************************************************************************************************
616* Pipe / splice stuff mainly for 2.6.17 >= linux < 2.6.31 (where no fallbacks were available) *
617*********************************************************************************************************************************/
618
619#if RTLNX_VER_RANGE(2,6,17, 3,16,0)
620
621# if RTLNX_VER_MAX(2,6,30)
622# define LOCK_PIPE(a_pPipe) do { if ((a_pPipe)->inode) mutex_lock(&(a_pPipe)->inode->i_mutex); } while (0)
623# define UNLOCK_PIPE(a_pPipe) do { if ((a_pPipe)->inode) mutex_unlock(&(a_pPipe)->inode->i_mutex); } while (0)
624# else
625# define LOCK_PIPE(a_pPipe) pipe_lock(a_pPipe)
626# define UNLOCK_PIPE(a_pPipe) pipe_unlock(a_pPipe)
627# endif
628
629
630/** Waits for the pipe buffer status to change. */
631static void vbsf_wait_pipe(struct pipe_inode_info *pPipe)
632{
633 DEFINE_WAIT(WaitStuff);
634# ifdef TASK_NONINTERACTIVE
635 prepare_to_wait(&pPipe->wait, &WaitStuff, TASK_INTERRUPTIBLE | TASK_NONINTERACTIVE);
636# else
637 prepare_to_wait(&pPipe->wait, &WaitStuff, TASK_INTERRUPTIBLE);
638# endif
639 UNLOCK_PIPE(pPipe);
640
641 schedule();
642
643 finish_wait(&pPipe->wait, &WaitStuff);
644 LOCK_PIPE(pPipe);
645}
646
647
648/** Worker for vbsf_feed_pages_to_pipe that wakes up readers. */
649static void vbsf_wake_up_pipe(struct pipe_inode_info *pPipe, bool fReaders)
650{
651 smp_mb();
652 if (waitqueue_active(&pPipe->wait))
653 wake_up_interruptible_sync(&pPipe->wait);
654 if (fReaders)
655 kill_fasync(&pPipe->fasync_readers, SIGIO, POLL_IN);
656 else
657 kill_fasync(&pPipe->fasync_writers, SIGIO, POLL_OUT);
658}
659
660#endif
661#if RTLNX_VER_RANGE(2,6,17, 2,6,31)
662
663/** Verify pipe buffer content (needed for page-cache to ensure idle page). */
664static int vbsf_pipe_buf_confirm(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
665{
666 /*SFLOG3(("vbsf_pipe_buf_confirm: %p\n", pPipeBuf));*/
667 return 0;
668}
669
670
671/** Maps the buffer page. */
672static void *vbsf_pipe_buf_map(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf, int atomic)
673{
674 void *pvRet;
675 if (!atomic)
676 pvRet = kmap(pPipeBuf->page);
677 else {
678 pPipeBuf->flags |= PIPE_BUF_FLAG_ATOMIC;
679 pvRet = kmap_atomic(pPipeBuf->page, KM_USER0);
680 }
681 /*SFLOG3(("vbsf_pipe_buf_map: %p -> %p\n", pPipeBuf, pvRet));*/
682 return pvRet;
683}
684
685
686/** Unmaps the buffer page. */
687static void vbsf_pipe_buf_unmap(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf, void *pvMapping)
688{
689 /*SFLOG3(("vbsf_pipe_buf_unmap: %p/%p\n", pPipeBuf, pvMapping)); */
690 if (!(pPipeBuf->flags & PIPE_BUF_FLAG_ATOMIC))
691 kunmap(pPipeBuf->page);
692 else {
693 pPipeBuf->flags &= ~PIPE_BUF_FLAG_ATOMIC;
694 kunmap_atomic(pvMapping, KM_USER0);
695 }
696}
697
698
699/** Gets a reference to the page. */
700static void vbsf_pipe_buf_get(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
701{
702 page_cache_get(pPipeBuf->page);
703 /*SFLOG3(("vbsf_pipe_buf_get: %p (return count=%d)\n", pPipeBuf, page_count(pPipeBuf->page)));*/
704}
705
706
707/** Release the buffer page (counter to vbsf_pipe_buf_get). */
708static void vbsf_pipe_buf_release(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
709{
710 /*SFLOG3(("vbsf_pipe_buf_release: %p (incoming count=%d)\n", pPipeBuf, page_count(pPipeBuf->page)));*/
711 page_cache_release(pPipeBuf->page);
712}
713
714
715/** Attempt to steal the page.
716 * @returns 0 success, 1 on failure. */
717static int vbsf_pipe_buf_steal(struct pipe_inode_info *pPipe, struct pipe_buffer *pPipeBuf)
718{
719 if (page_count(pPipeBuf->page) == 1) {
720 lock_page(pPipeBuf->page);
721 SFLOG3(("vbsf_pipe_buf_steal: %p -> 0\n", pPipeBuf));
722 return 0;
723 }
724 SFLOG3(("vbsf_pipe_buf_steal: %p -> 1\n", pPipeBuf));
725 return 1;
726}
727
728
729/**
730 * Pipe buffer operations for used by vbsf_feed_pages_to_pipe.
731 */
732static struct pipe_buf_operations vbsf_pipe_buf_ops = {
733 .can_merge = 0,
734# if RTLNX_VER_MIN(2,6,23)
735 .confirm = vbsf_pipe_buf_confirm,
736# else
737 .pin = vbsf_pipe_buf_confirm,
738# endif
739 .map = vbsf_pipe_buf_map,
740 .unmap = vbsf_pipe_buf_unmap,
741 .get = vbsf_pipe_buf_get,
742 .release = vbsf_pipe_buf_release,
743 .steal = vbsf_pipe_buf_steal,
744};
745
746
747/**
748 * Feeds the pages to the pipe.
749 *
750 * Pages given to the pipe are set to NULL in papPages.
751 */
752static ssize_t vbsf_feed_pages_to_pipe(struct pipe_inode_info *pPipe, struct page **papPages, size_t cPages, uint32_t offPg0,
753 uint32_t cbActual, unsigned fFlags)
754{
755 ssize_t cbRet = 0;
756 size_t iPage = 0;
757 bool fNeedWakeUp = false;
758
759 LOCK_PIPE(pPipe);
760 for (;;) {
761 if ( pPipe->readers > 0
762 && pPipe->nrbufs < PIPE_BUFFERS) {
763 struct pipe_buffer *pPipeBuf = &pPipe->bufs[(pPipe->curbuf + pPipe->nrbufs) % PIPE_BUFFERS];
764 uint32_t const cbThisPage = RT_MIN(cbActual, PAGE_SIZE - offPg0);
765 pPipeBuf->len = cbThisPage;
766 pPipeBuf->offset = offPg0;
767# if RTLNX_VER_MIN(2,6,23)
768 pPipeBuf->private = 0;
769# endif
770 pPipeBuf->ops = &vbsf_pipe_buf_ops;
771 pPipeBuf->flags = fFlags & SPLICE_F_GIFT ? PIPE_BUF_FLAG_GIFT : 0;
772 pPipeBuf->page = papPages[iPage];
773
774 papPages[iPage++] = NULL;
775 pPipe->nrbufs++;
776 fNeedWakeUp |= pPipe->inode != NULL;
777 offPg0 = 0;
778 cbRet += cbThisPage;
779
780 /* done? */
781 cbActual -= cbThisPage;
782 if (!cbActual)
783 break;
784 } else if (pPipe->readers == 0) {
785 SFLOGFLOW(("vbsf_feed_pages_to_pipe: no readers!\n"));
786 send_sig(SIGPIPE, current, 0);
787 if (cbRet == 0)
788 cbRet = -EPIPE;
789 break;
790 } else if (fFlags & SPLICE_F_NONBLOCK) {
791 if (cbRet == 0)
792 cbRet = -EAGAIN;
793 break;
794 } else if (signal_pending(current)) {
795 if (cbRet == 0)
796 cbRet = -ERESTARTSYS;
797 SFLOGFLOW(("vbsf_feed_pages_to_pipe: pending signal! (%zd)\n", cbRet));
798 break;
799 } else {
800 if (fNeedWakeUp) {
801 vbsf_wake_up_pipe(pPipe, true /*fReaders*/);
802 fNeedWakeUp = 0;
803 }
804 pPipe->waiting_writers++;
805 vbsf_wait_pipe(pPipe);
806 pPipe->waiting_writers--;
807 }
808 }
809 UNLOCK_PIPE(pPipe);
810
811 if (fNeedWakeUp)
812 vbsf_wake_up_pipe(pPipe, true /*fReaders*/);
813
814 return cbRet;
815}
816
817
818/**
819 * For splicing from a file to a pipe.
820 */
821static ssize_t vbsf_splice_read(struct file *file, loff_t *poffset, struct pipe_inode_info *pipe, size_t len, unsigned int flags)
822{
823 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
824 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
825 ssize_t cbRet;
826
827 SFLOGFLOW(("vbsf_splice_read: file=%p poffset=%p{%#RX64} pipe=%p len=%#zx flags=%#x\n", file, poffset, *poffset, pipe, len, flags));
828 if (vbsf_should_use_cached_read(file, inode->i_mapping, pSuperInfo)) {
829 cbRet = generic_file_splice_read(file, poffset, pipe, len, flags);
830 } else {
831 /*
832 * Create a read request.
833 */
834 loff_t offFile = *poffset;
835 size_t cPages = RT_MIN(RT_ALIGN_Z((offFile & ~PAGE_CACHE_MASK) + len, PAGE_CACHE_SIZE) >> PAGE_CACHE_SHIFT,
836 PIPE_BUFFERS);
837 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ,
838 PgLst.aPages[cPages]));
839 if (pReq) {
840 /*
841 * Allocate pages.
842 */
843 struct page *apPages[PIPE_BUFFERS];
844 size_t i;
845 pReq->PgLst.offFirstPage = (uint16_t)offFile & (uint16_t)PAGE_OFFSET_MASK;
846 cbRet = 0;
847 for (i = 0; i < cPages; i++) {
848 struct page *pPage;
849 apPages[i] = pPage = alloc_page(GFP_USER);
850 if (pPage) {
851 pReq->PgLst.aPages[i] = page_to_phys(pPage);
852# ifdef VBOX_STRICT
853 ASMMemFill32(kmap(pPage), PAGE_SIZE, UINT32_C(0xdeadbeef));
854 kunmap(pPage);
855# endif
856 } else {
857 cbRet = -ENOMEM;
858 break;
859 }
860 }
861 if (cbRet == 0) {
862 /*
863 * Do the reading.
864 */
865 uint32_t const cbToRead = RT_MIN((cPages << PAGE_SHIFT) - (offFile & PAGE_OFFSET_MASK), len);
866 struct vbsf_reg_info *sf_r = (struct vbsf_reg_info *)file->private_data;
867 int vrc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbToRead, cPages);
868 if (RT_SUCCESS(vrc)) {
869 /*
870 * Get the number of bytes read, jettison the request
871 * and, in case of EOF, any unnecessary pages.
872 */
873 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
874 AssertStmt(cbActual <= cbToRead, cbActual = cbToRead);
875 SFLOG2(("vbsf_splice_read: read -> %#x bytes @ %#RX64\n", cbActual, offFile));
876
877 VbglR0PhysHeapFree(pReq);
878 pReq = NULL;
879
880 /*
881 * Now, feed it to the pipe thingy.
882 * This will take ownership of the all pages no matter what happens.
883 */
884 cbRet = vbsf_feed_pages_to_pipe(pipe, apPages, cPages, offFile & PAGE_OFFSET_MASK, cbActual, flags);
885 if (cbRet > 0)
886 *poffset = offFile + cbRet;
887 } else {
888 cbRet = -RTErrConvertToErrno(vrc);
889 SFLOGFLOW(("vbsf_splice_read: Read failed: %Rrc -> %zd\n", vrc, cbRet));
890 }
891 i = cPages;
892 }
893
894 while (i-- > 0)
895 if (apPages[i])
896 __free_pages(apPages[i], 0);
897 if (pReq)
898 VbglR0PhysHeapFree(pReq);
899 } else {
900 cbRet = -ENOMEM;
901 }
902 }
903 SFLOGFLOW(("vbsf_splice_read: returns %zd (%#zx), *poffset=%#RX64\n", cbRet, cbRet, *poffset));
904 return cbRet;
905}
906
907#endif /* 2.6.17 <= LINUX_VERSION_CODE < 2.6.31 */
908#if RTLNX_VER_RANGE(2,6,17, 3,16,0)
909
910/**
911 * For splicing from a pipe to a file.
912 *
913 * Since we can combine buffers and request allocations, this should be faster
914 * than the default implementation.
915 */
916static ssize_t vbsf_splice_write(struct pipe_inode_info *pPipe, struct file *file, loff_t *poffset, size_t len, unsigned int flags)
917{
918 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
919 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
920 ssize_t cbRet;
921
922 SFLOGFLOW(("vbsf_splice_write: pPipe=%p file=%p poffset=%p{%#RX64} len=%#zx flags=%#x\n", pPipe, file, poffset, *poffset, len, flags));
923 /** @todo later if (false) {
924 cbRet = generic_file_splice_write(pPipe, file, poffset, len, flags);
925 } else */ {
926 /*
927 * Prepare a write request.
928 */
929# ifdef PIPE_BUFFERS
930 uint32_t const cMaxPages = RT_MIN(PIPE_BUFFERS, RT_ALIGN_Z(len, PAGE_SIZE) >> PAGE_SHIFT);
931# else
932 uint32_t const cMaxPages = RT_MIN(RT_MAX(RT_MIN(pPipe->buffers, 256), PIPE_DEF_BUFFERS),
933 RT_ALIGN_Z(len, PAGE_SIZE) >> PAGE_SHIFT);
934# endif
935 VBOXSFWRITEPGLSTREQ *pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ,
936 PgLst.aPages[cMaxPages]));
937 if (pReq) {
938 /*
939 * Feed from the pipe.
940 */
941 struct vbsf_reg_info *sf_r = (struct vbsf_reg_info *)file->private_data;
942 struct address_space *mapping = inode->i_mapping;
943 loff_t offFile = *poffset;
944 bool fNeedWakeUp = false;
945 cbRet = 0;
946
947 LOCK_PIPE(pPipe);
948
949 for (;;) {
950 unsigned cBufs = pPipe->nrbufs;
951 /*SFLOG2(("vbsf_splice_write: nrbufs=%#x curbuf=%#x\n", cBufs, pPipe->curbuf));*/
952 if (cBufs) {
953 /*
954 * There is data available. Write it to the file.
955 */
956 int vrc;
957 struct pipe_buffer *pPipeBuf = &pPipe->bufs[pPipe->curbuf];
958 uint32_t cPagesToWrite = 1;
959 uint32_t cbToWrite = pPipeBuf->len;
960
961 Assert(pPipeBuf->offset < PAGE_SIZE);
962 Assert(pPipeBuf->offset + pPipeBuf->len <= PAGE_SIZE);
963
964 pReq->PgLst.offFirstPage = pPipeBuf->offset & PAGE_OFFSET;
965 pReq->PgLst.aPages[0] = page_to_phys(pPipeBuf->page);
966
967 /* Add any adjacent page buffers: */
968 while ( cPagesToWrite < cBufs
969 && cPagesToWrite < cMaxPages
970 && ((pReq->PgLst.offFirstPage + cbToWrite) & PAGE_OFFSET_MASK) == 0) {
971# ifdef PIPE_BUFFERS
972 struct pipe_buffer *pPipeBuf2 = &pPipe->bufs[(pPipe->curbuf + cPagesToWrite) % PIPE_BUFFERS];
973# else
974 struct pipe_buffer *pPipeBuf2 = &pPipe->bufs[(pPipe->curbuf + cPagesToWrite) % pPipe->buffers];
975# endif
976 Assert(pPipeBuf2->len <= PAGE_SIZE);
977 Assert(pPipeBuf2->offset < PAGE_SIZE);
978 if (pPipeBuf2->offset != 0)
979 break;
980 pReq->PgLst.aPages[cPagesToWrite] = page_to_phys(pPipeBuf2->page);
981 cbToWrite += pPipeBuf2->len;
982 cPagesToWrite += 1;
983 }
984
985 /* Check that we don't have signals pending before we issue the write, as
986 we'll only end up having to cancel the HGCM request 99% of the time: */
987 if (!signal_pending(current)) {
988 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
989 vrc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile,
990 cbToWrite, cPagesToWrite);
991 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
992 } else
993 vrc = VERR_INTERRUPTED;
994 if (RT_SUCCESS(vrc)) {
995 /*
996 * Get the number of bytes actually written, update file position
997 * and return value, and advance the pipe buffer.
998 */
999 uint32_t cbActual = pReq->Parms.cb32Write.u.value32;
1000 AssertStmt(cbActual <= cbToWrite, cbActual = cbToWrite);
1001 SFLOG2(("vbsf_splice_write: write -> %#x bytes @ %#RX64\n", cbActual, offFile));
1002
1003 cbRet += cbActual;
1004
1005 while (cbActual > 0) {
1006 uint32_t cbAdvance = RT_MIN(pPipeBuf->len, cbActual);
1007
1008 vbsf_reg_write_sync_page_cache(mapping, offFile, cbAdvance, NULL,
1009 &pPipeBuf->page, pPipeBuf->offset, 1);
1010
1011 offFile += cbAdvance;
1012 cbActual -= cbAdvance;
1013 pPipeBuf->offset += cbAdvance;
1014 pPipeBuf->len -= cbAdvance;
1015
1016 if (!pPipeBuf->len) {
1017 struct pipe_buf_operations const *pOps = pPipeBuf->ops;
1018 pPipeBuf->ops = NULL;
1019 pOps->release(pPipe, pPipeBuf);
1020
1021# ifdef PIPE_BUFFERS
1022 pPipe->curbuf = (pPipe->curbuf + 1) % PIPE_BUFFERS;
1023# else
1024 pPipe->curbuf = (pPipe->curbuf + 1) % pPipe->buffers;
1025# endif
1026 pPipe->nrbufs -= 1;
1027 pPipeBuf = &pPipe->bufs[pPipe->curbuf];
1028
1029# if RTLNX_VER_MAX(2,6,30)
1030 fNeedWakeUp |= pPipe->inode != NULL;
1031# else
1032 fNeedWakeUp = true;
1033# endif
1034 } else {
1035 Assert(cbActual == 0);
1036 break;
1037 }
1038 }
1039
1040 *poffset = offFile;
1041 } else {
1042 if (cbRet == 0)
1043 cbRet = vrc == VERR_INTERRUPTED ? -ERESTARTSYS : -RTErrConvertToErrno(vrc);
1044 SFLOGFLOW(("vbsf_splice_write: Write failed: %Rrc -> %zd (cbRet=%#zx)\n",
1045 vrc, -RTErrConvertToErrno(vrc), cbRet));
1046 break;
1047 }
1048 } else {
1049 /*
1050 * Wait for data to become available, if there is chance that'll happen.
1051 */
1052 /* Quit if there are no writers (think EOF): */
1053 if (pPipe->writers == 0) {
1054 SFLOGFLOW(("vbsf_splice_write: No buffers. No writers. The show is done!\n"));
1055 break;
1056 }
1057
1058 /* Quit if if we've written some and no writers waiting on the lock: */
1059 if (cbRet > 0 && pPipe->waiting_writers == 0) {
1060 SFLOGFLOW(("vbsf_splice_write: No waiting writers, returning what we've got.\n"));
1061 break;
1062 }
1063
1064 /* Quit with EAGAIN if non-blocking: */
1065 if (flags & SPLICE_F_NONBLOCK) {
1066 if (cbRet == 0)
1067 cbRet = -EAGAIN;
1068 break;
1069 }
1070
1071 /* Quit if we've got pending signals: */
1072 if (signal_pending(current)) {
1073 if (cbRet == 0)
1074 cbRet = -ERESTARTSYS;
1075 SFLOGFLOW(("vbsf_splice_write: pending signal! (%zd)\n", cbRet));
1076 break;
1077 }
1078
1079 /* Wake up writers before we start waiting: */
1080 if (fNeedWakeUp) {
1081 vbsf_wake_up_pipe(pPipe, false /*fReaders*/);
1082 fNeedWakeUp = false;
1083 }
1084 vbsf_wait_pipe(pPipe);
1085 }
1086 } /* feed loop */
1087
1088 if (fNeedWakeUp)
1089 vbsf_wake_up_pipe(pPipe, false /*fReaders*/);
1090
1091 UNLOCK_PIPE(pPipe);
1092
1093 VbglR0PhysHeapFree(pReq);
1094 } else {
1095 cbRet = -ENOMEM;
1096 }
1097 }
1098 SFLOGFLOW(("vbsf_splice_write: returns %zd (%#zx), *poffset=%#RX64\n", cbRet, cbRet, *poffset));
1099 return cbRet;
1100}
1101
1102#endif /* 2.6.17 <= LINUX_VERSION_CODE < 3.16.0 */
1103
1104#if RTLNX_VER_RANGE(2,5,30, 2,6,23)
1105/**
1106 * Our own senfile implementation that does not go via the page cache like
1107 * generic_file_sendfile() does.
1108 */
1109static ssize_t vbsf_reg_sendfile(struct file *pFile, loff_t *poffFile, size_t cbToSend, read_actor_t pfnActor,
1110# if RTLNX_VER_MIN(2,6,8)
1111 void *pvUser
1112# else
1113 void __user *pvUser
1114# endif
1115 )
1116{
1117 struct inode *inode = VBSF_GET_F_DENTRY(pFile)->d_inode;
1118 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
1119 ssize_t cbRet;
1120 SFLOGFLOW(("vbsf_reg_sendfile: pFile=%p poffFile=%p{%#RX64} cbToSend=%#zx pfnActor=%p pvUser=%p\n",
1121 pFile, poffFile, poffFile ? *poffFile : 0, cbToSend, pfnActor, pvUser));
1122 Assert(pSuperInfo);
1123
1124 /*
1125 * Return immediately if asked to send nothing.
1126 */
1127 if (cbToSend == 0)
1128 return 0;
1129
1130 /*
1131 * Like for vbsf_reg_read() and vbsf_reg_read_iter(), we allow going via
1132 * the page cache in some cases or configs.
1133 */
1134 if (vbsf_should_use_cached_read(pFile, inode->i_mapping, pSuperInfo)) {
1135 cbRet = generic_file_sendfile(pFile, poffFile, cbToSend, pfnActor, pvUser);
1136 SFLOGFLOW(("vbsf_reg_sendfile: returns %#zx *poffFile=%#RX64 [generic_file_sendfile]\n", cbRet, poffFile ? *poffFile : UINT64_MAX));
1137 } else {
1138 /*
1139 * Allocate a request and a bunch of pages for reading from the file.
1140 */
1141 struct page *apPages[16];
1142 loff_t offFile = poffFile ? *poffFile : 0;
1143 size_t const cPages = cbToSend + ((size_t)offFile & PAGE_OFFSET_MASK) >= RT_ELEMENTS(apPages) * PAGE_SIZE
1144 ? RT_ELEMENTS(apPages)
1145 : RT_ALIGN_Z(cbToSend + ((size_t)offFile & PAGE_OFFSET_MASK), PAGE_SIZE) >> PAGE_SHIFT;
1146 size_t iPage;
1147 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ,
1148 PgLst.aPages[cPages]));
1149 if (pReq) {
1150 Assert(cPages > 0);
1151 cbRet = 0;
1152 for (iPage = 0; iPage < cPages; iPage++) {
1153 struct page *pPage;
1154 apPages[iPage] = pPage = alloc_page(GFP_USER);
1155 if (pPage) {
1156 Assert(page_count(pPage) == 1);
1157 pReq->PgLst.aPages[iPage] = page_to_phys(pPage);
1158 } else {
1159 while (iPage-- > 0)
1160 vbsf_put_page(apPages[iPage]);
1161 cbRet = -ENOMEM;
1162 break;
1163 }
1164 }
1165 if (cbRet == 0) {
1166 /*
1167 * Do the job.
1168 */
1169 struct vbsf_reg_info *sf_r = (struct vbsf_reg_info *)pFile->private_data;
1170 read_descriptor_t RdDesc;
1171 RdDesc.count = cbToSend;
1172# if RTLNX_VER_MIN(2,6,8)
1173 RdDesc.arg.data = pvUser;
1174# else
1175 RdDesc.buf = pvUser;
1176# endif
1177 RdDesc.written = 0;
1178 RdDesc.error = 0;
1179
1180 Assert(sf_r);
1181 Assert((sf_r->Handle.fFlags & VBSF_HANDLE_F_MAGIC_MASK) == VBSF_HANDLE_F_MAGIC);
1182
1183 while (cbToSend > 0) {
1184 /*
1185 * Read another chunk. For paranoid reasons, we keep data where the page cache
1186 * would keep it, i.e. page offset bits corresponds to the file offset bits.
1187 */
1188 uint32_t const offPg0 = (uint32_t)offFile & (uint32_t)PAGE_OFFSET_MASK;
1189 uint32_t const cbToRead = RT_MIN((cPages << PAGE_SHIFT) - offPg0, cbToSend);
1190 uint32_t const cPagesToRead = RT_ALIGN_Z(cbToRead + offPg0, PAGE_SIZE) >> PAGE_SHIFT;
1191 int vrc;
1192 pReq->PgLst.offFirstPage = (uint16_t)offPg0;
1193 if (!signal_pending(current))
1194 vrc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile,
1195 cbToRead, cPagesToRead);
1196 else
1197 vrc = VERR_INTERRUPTED;
1198 if (RT_SUCCESS(vrc)) {
1199 /*
1200 * Pass what we read to the actor.
1201 */
1202 uint32_t off = offPg0;
1203 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
1204 bool const fIsEof = cbActual < cbToRead;
1205 AssertStmt(cbActual <= cbToRead, cbActual = cbToRead);
1206 SFLOG3(("vbsf_reg_sendfile: Read %#x bytes (offPg0=%#x), wanted %#x ...\n", cbActual, offPg0, cbToRead));
1207
1208 iPage = 0;
1209 while (cbActual > 0) {
1210 uint32_t const cbPage = RT_MIN(cbActual, PAGE_SIZE - off);
1211 int const cbRetActor = pfnActor(&RdDesc, apPages[iPage], off, cbPage);
1212 Assert(cbRetActor >= 0); /* Returns zero on failure, with RdDesc.error holding the status code. */
1213
1214 AssertMsg(iPage < cPages && iPage < cPagesToRead, ("iPage=%#x cPages=%#x cPagesToRead=%#x\n", iPage, cPages, cPagesToRead));
1215
1216 offFile += cbRetActor;
1217 if ((uint32_t)cbRetActor == cbPage && RdDesc.count > 0) {
1218 cbActual -= cbPage;
1219 cbToSend -= cbPage;
1220 iPage++;
1221 } else {
1222 SFLOG3(("vbsf_reg_sendfile: cbRetActor=%#x (%d) cbPage=%#x RdDesc{count=%#lx error=%d} iPage=%#x/%#x/%#x cbToSend=%#zx\n",
1223 cbRetActor, cbRetActor, cbPage, RdDesc.count, RdDesc.error, iPage, cPagesToRead, cPages, cbToSend));
1224 vrc = VERR_CALLBACK_RETURN;
1225 break;
1226 }
1227 off = 0;
1228 }
1229
1230 /*
1231 * Are we done yet?
1232 */
1233 if (RT_FAILURE_NP(vrc) || cbToSend == 0 || RdDesc.error != 0 || fIsEof) {
1234 break;
1235 }
1236
1237 /*
1238 * Replace pages held by the actor.
1239 */
1240 vrc = VINF_SUCCESS;
1241 for (iPage = 0; iPage < cPages; iPage++) {
1242 struct page *pPage = apPages[iPage];
1243 if (page_count(pPage) != 1) {
1244 struct page *pNewPage = alloc_page(GFP_USER);
1245 if (pNewPage) {
1246 SFLOGFLOW(("vbsf_reg_sendfile: Replacing page #%x: %p -> %p\n", iPage, pPage, pNewPage));
1247 vbsf_put_page(pPage);
1248 apPages[iPage] = pNewPage;
1249 } else {
1250 SFLOGFLOW(("vbsf_reg_sendfile: Failed to allocate a replacement page.\n"));
1251 vrc = VERR_NO_MEMORY;
1252 break;
1253 }
1254 }
1255 }
1256 if (RT_FAILURE(vrc))
1257 break; /* RdDesc.written should be non-zero, so don't bother with setting error. */
1258 } else {
1259 RdDesc.error = vrc == VERR_INTERRUPTED ? -ERESTARTSYS : -RTErrConvertToErrno(vrc);
1260 SFLOGFLOW(("vbsf_reg_sendfile: Read failed: %Rrc -> %zd (RdDesc.error=%#d)\n",
1261 vrc, -RTErrConvertToErrno(vrc), RdDesc.error));
1262 break;
1263 }
1264 }
1265
1266 /*
1267 * Free memory.
1268 */
1269 for (iPage = 0; iPage < cPages; iPage++)
1270 vbsf_put_page(apPages[iPage]);
1271
1272 /*
1273 * Set the return values.
1274 */
1275 if (RdDesc.written) {
1276 cbRet = RdDesc.written;
1277 if (poffFile)
1278 *poffFile = offFile;
1279 } else {
1280 cbRet = RdDesc.error;
1281 }
1282 }
1283 VbglR0PhysHeapFree(pReq);
1284 } else {
1285 cbRet = -ENOMEM;
1286 }
1287 SFLOGFLOW(("vbsf_reg_sendfile: returns %#zx offFile=%#RX64\n", cbRet, offFile));
1288 }
1289 return cbRet;
1290}
1291#endif /* 2.5.30 <= LINUX_VERSION_CODE < 2.6.23 */
1292
1293
1294/*********************************************************************************************************************************
1295* File operations on regular files *
1296*********************************************************************************************************************************/
1297
1298/** Wrapper around put_page / page_cache_release. */
1299DECLINLINE(void) vbsf_put_page(struct page *pPage)
1300{
1301#if RTLNX_VER_MIN(4,6,0)
1302 put_page(pPage);
1303#else
1304 page_cache_release(pPage);
1305#endif
1306}
1307
1308
1309/** Wrapper around get_page / page_cache_get. */
1310DECLINLINE(void) vbsf_get_page(struct page *pPage)
1311{
1312#if RTLNX_VER_MIN(4,6,0)
1313 get_page(pPage);
1314#else
1315 page_cache_get(pPage);
1316#endif
1317}
1318
1319
1320/** Companion to vbsf_lock_user_pages(). */
1321static void vbsf_unlock_user_pages(struct page **papPages, size_t cPages, bool fSetDirty, bool fLockPgHack)
1322{
1323 /* We don't mark kernel pages dirty: */
1324 if (fLockPgHack)
1325 fSetDirty = false;
1326
1327 while (cPages-- > 0)
1328 {
1329 struct page *pPage = papPages[cPages];
1330 Assert((ssize_t)cPages >= 0);
1331 if (fSetDirty && !PageReserved(pPage))
1332 set_page_dirty(pPage);
1333 vbsf_put_page(pPage);
1334 }
1335}
1336
1337
1338/**
1339 * Worker for vbsf_lock_user_pages_failed_check_kernel() and
1340 * vbsf_iter_lock_pages().
1341 */
1342static int vbsf_lock_kernel_pages(uint8_t *pbStart, bool fWrite, size_t cPages, struct page **papPages)
1343{
1344 uintptr_t const uPtrFrom = (uintptr_t)pbStart;
1345 uintptr_t const uPtrLast = (uPtrFrom & ~(uintptr_t)PAGE_OFFSET_MASK) + (cPages << PAGE_SHIFT) - 1;
1346 uint8_t *pbPage = (uint8_t *)uPtrLast;
1347 size_t iPage = cPages;
1348
1349 /*
1350 * Touch the pages first (paranoia^2).
1351 */
1352 if (fWrite) {
1353 uint8_t volatile *pbProbe = (uint8_t volatile *)uPtrFrom;
1354 while (iPage-- > 0) {
1355 *pbProbe = *pbProbe;
1356 pbProbe += PAGE_SIZE;
1357 }
1358 } else {
1359 uint8_t const *pbProbe = (uint8_t const *)uPtrFrom;
1360 while (iPage-- > 0) {
1361 ASMProbeReadByte(pbProbe);
1362 pbProbe += PAGE_SIZE;
1363 }
1364 }
1365
1366 /*
1367 * Get the pages.
1368 * Note! Fixes here probably applies to rtR0MemObjNativeLockKernel as well.
1369 */
1370 iPage = cPages;
1371 if ( uPtrFrom >= (unsigned long)__va(0)
1372 && uPtrLast < (unsigned long)high_memory) {
1373 /* The physical page mapping area: */
1374 while (iPage-- > 0) {
1375 struct page *pPage = papPages[iPage] = virt_to_page(pbPage);
1376 vbsf_get_page(pPage);
1377 pbPage -= PAGE_SIZE;
1378 }
1379 } else {
1380 /* This is vmalloc or some such thing, so go thru page tables: */
1381 while (iPage-- > 0) {
1382 struct page *pPage = rtR0MemObjLinuxVirtToPage(pbPage);
1383 if (pPage) {
1384 papPages[iPage] = pPage;
1385 vbsf_get_page(pPage);
1386 pbPage -= PAGE_SIZE;
1387 } else {
1388 while (++iPage < cPages) {
1389 pPage = papPages[iPage];
1390 vbsf_put_page(pPage);
1391 }
1392 return -EFAULT;
1393 }
1394 }
1395 }
1396 return 0;
1397}
1398
1399
1400/**
1401 * Catches kernel_read() and kernel_write() calls and works around them.
1402 *
1403 * The file_operations::read and file_operations::write callbacks supposedly
1404 * hands us the user buffers to read into and write out of. To allow the kernel
1405 * to read and write without allocating buffers in userland, they kernel_read()
1406 * and kernel_write() increases the user space address limit before calling us
1407 * so that copyin/copyout won't reject it. Our problem is that get_user_pages()
1408 * works on the userspace address space structures and will not be fooled by an
1409 * increased addr_limit.
1410 *
1411 * This code tries to detect this situation and fake get_user_lock() for the
1412 * kernel buffer.
1413 */
1414static int vbsf_lock_user_pages_failed_check_kernel(uintptr_t uPtrFrom, size_t cPages, bool fWrite, int rcFailed,
1415 struct page **papPages, bool *pfLockPgHack)
1416{
1417 /*
1418 * Check that this is valid user memory that is actually in the kernel range.
1419 */
1420#if RTLNX_VER_MIN(5,10,0)
1421 if ( access_ok((void *)uPtrFrom, cPages << PAGE_SHIFT)
1422 && uPtrFrom >= TASK_SIZE_MAX)
1423#elif RTLNX_VER_MIN(5,0,0) || RTLNX_RHEL_MIN(8,1)
1424 if ( access_ok((void *)uPtrFrom, cPages << PAGE_SHIFT)
1425 && uPtrFrom >= USER_DS.seg)
1426#else
1427 if ( access_ok(fWrite ? VERIFY_WRITE : VERIFY_READ, (void *)uPtrFrom, cPages << PAGE_SHIFT)
1428 && uPtrFrom >= USER_DS.seg)
1429#endif
1430 {
1431 int rc = vbsf_lock_kernel_pages((uint8_t *)uPtrFrom, fWrite, cPages, papPages);
1432 if (rc == 0) {
1433 *pfLockPgHack = true;
1434 return 0;
1435 }
1436 }
1437
1438 return rcFailed;
1439}
1440
1441
1442/** Wrapper around get_user_pages. */
1443DECLINLINE(int) vbsf_lock_user_pages(uintptr_t uPtrFrom, size_t cPages, bool fWrite, struct page **papPages, bool *pfLockPgHack)
1444{
1445# if RTLNX_VER_MIN(4,9,0) \
1446 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_RANGE(4,4,73, 4,4,74) /** @todo Figure out when & what exactly. */) \
1447 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_RANGE(4,4,75, 4,4,90) /** @todo Figure out when & what exactly. */) \
1448 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_RANGE(4,4,92, 4,5,0) /** @todo Figure out when & what exactly. */)
1449 ssize_t cPagesLocked = get_user_pages_unlocked(uPtrFrom, cPages, papPages,
1450 fWrite ? FOLL_WRITE | FOLL_FORCE : FOLL_FORCE);
1451# elif RTLNX_VER_MIN(4,6,0)
1452 ssize_t cPagesLocked = get_user_pages_unlocked(uPtrFrom, cPages, fWrite, 1 /*force*/, papPages);
1453# elif RTLNX_VER_RANGE(4,4,168, 4,5,0)
1454 ssize_t cPagesLocked = get_user_pages_unlocked(current, current->mm, uPtrFrom, cPages, papPages,
1455 fWrite ? FOLL_WRITE | FOLL_FORCE : FOLL_FORCE);
1456# elif RTLNX_VER_MIN(4,0,0)
1457 ssize_t cPagesLocked = get_user_pages_unlocked(current, current->mm, uPtrFrom, cPages, fWrite, 1 /*force*/, papPages);
1458# else
1459 struct task_struct *pTask = current;
1460 ssize_t cPagesLocked;
1461 down_read(&pTask->mm->mmap_sem);
1462 cPagesLocked = get_user_pages(pTask, pTask->mm, uPtrFrom, cPages, fWrite, 1 /*force*/, papPages, NULL);
1463 up_read(&pTask->mm->mmap_sem);
1464# endif
1465 *pfLockPgHack = false;
1466 if (cPagesLocked == cPages)
1467 return 0;
1468
1469 /*
1470 * It failed.
1471 */
1472 if (cPagesLocked < 0)
1473 return vbsf_lock_user_pages_failed_check_kernel(uPtrFrom, cPages, fWrite, (int)cPagesLocked, papPages, pfLockPgHack);
1474
1475 vbsf_unlock_user_pages(papPages, cPagesLocked, false /*fSetDirty*/, false /*fLockPgHack*/);
1476
1477 /* We could use uPtrFrom + cPagesLocked to get the correct status here... */
1478 return -EFAULT;
1479}
1480
1481#if RTLNX_VER_MAX(5,10,0) /* No regular .read/.write for 5.10, only .read_iter/.write_iter or in-kernel reads/writes fail. */
1482
1483/**
1484 * Read function used when accessing files that are memory mapped.
1485 *
1486 * We read from the page cache here to present the a cohertent picture of the
1487 * the file content.
1488 */
1489static ssize_t vbsf_reg_read_mapped(struct file *file, char /*__user*/ *buf, size_t size, loff_t *off)
1490{
1491# if RTLNX_VER_MIN(3,16,0)
1492 struct iovec iov = { .iov_base = buf, .iov_len = size };
1493 struct iov_iter iter;
1494 struct kiocb kiocb;
1495 ssize_t cbRet;
1496
1497 init_sync_kiocb(&kiocb, file);
1498 kiocb.ki_pos = *off;
1499 iov_iter_init(&iter, READ, &iov, 1, size);
1500
1501 cbRet = generic_file_read_iter(&kiocb, &iter);
1502
1503 *off = kiocb.ki_pos;
1504 return cbRet;
1505
1506# elif RTLNX_VER_MIN(2,6,19)
1507 struct iovec iov = { .iov_base = buf, .iov_len = size };
1508 struct kiocb kiocb;
1509 ssize_t cbRet;
1510
1511 init_sync_kiocb(&kiocb, file);
1512 kiocb.ki_pos = *off;
1513
1514 cbRet = generic_file_aio_read(&kiocb, &iov, 1, *off);
1515 if (cbRet == -EIOCBQUEUED)
1516 cbRet = wait_on_sync_kiocb(&kiocb);
1517
1518 *off = kiocb.ki_pos;
1519 return cbRet;
1520
1521# else /* 2.6.18 or earlier: */
1522 return generic_file_read(file, buf, size, off);
1523# endif
1524}
1525
1526
1527/**
1528 * Fallback case of vbsf_reg_read() that locks the user buffers and let the host
1529 * write directly to them.
1530 */
1531static ssize_t vbsf_reg_read_locking(struct file *file, char /*__user*/ *buf, size_t size, loff_t *off,
1532 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r)
1533{
1534 /*
1535 * Lock pages and execute the read, taking care not to pass the host
1536 * more than it can handle in one go or more than we care to allocate
1537 * page arrays for. The latter limit is set at just short of 32KB due
1538 * to how the physical heap works.
1539 */
1540 struct page *apPagesStack[16];
1541 struct page **papPages = &apPagesStack[0];
1542 struct page **papPagesFree = NULL;
1543 VBOXSFREADPGLSTREQ *pReq;
1544 loff_t offFile = *off;
1545 ssize_t cbRet = -ENOMEM;
1546 size_t cPages = (((uintptr_t)buf & PAGE_OFFSET_MASK) + size + PAGE_OFFSET_MASK) >> PAGE_SHIFT;
1547 size_t cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 1), cPages);
1548 bool fLockPgHack;
1549
1550 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
1551 while (!pReq && cMaxPages > 4) {
1552 cMaxPages /= 2;
1553 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
1554 }
1555 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
1556 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
1557 if (pReq && papPages) {
1558 cbRet = 0;
1559 for (;;) {
1560 /*
1561 * Figure out how much to process now and lock the user pages.
1562 */
1563 int rc;
1564 size_t cbChunk = (uintptr_t)buf & PAGE_OFFSET_MASK;
1565 pReq->PgLst.offFirstPage = (uint16_t)cbChunk;
1566 cPages = RT_ALIGN_Z(cbChunk + size, PAGE_SIZE) >> PAGE_SHIFT;
1567 if (cPages <= cMaxPages)
1568 cbChunk = size;
1569 else {
1570 cPages = cMaxPages;
1571 cbChunk = (cMaxPages << PAGE_SHIFT) - cbChunk;
1572 }
1573
1574 rc = vbsf_lock_user_pages((uintptr_t)buf, cPages, true /*fWrite*/, papPages, &fLockPgHack);
1575 if (rc == 0) {
1576 size_t iPage = cPages;
1577 while (iPage-- > 0)
1578 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
1579 } else {
1580 cbRet = rc;
1581 break;
1582 }
1583
1584 /*
1585 * Issue the request and unlock the pages.
1586 */
1587 rc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbChunk, cPages);
1588
1589 Assert(cPages <= cMaxPages);
1590 vbsf_unlock_user_pages(papPages, cPages, true /*fSetDirty*/, fLockPgHack);
1591
1592 if (RT_SUCCESS(rc)) {
1593 /*
1594 * Success, advance position and buffer.
1595 */
1596 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
1597 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
1598 cbRet += cbActual;
1599 offFile += cbActual;
1600 buf = (uint8_t *)buf + cbActual;
1601 size -= cbActual;
1602
1603 /*
1604 * Are we done already? If so commit the new file offset.
1605 */
1606 if (!size || cbActual < cbChunk) {
1607 *off = offFile;
1608 break;
1609 }
1610 } else if (rc == VERR_NO_MEMORY && cMaxPages > 4) {
1611 /*
1612 * The host probably doesn't have enough heap to handle the
1613 * request, reduce the page count and retry.
1614 */
1615 cMaxPages /= 4;
1616 Assert(cMaxPages > 0);
1617 } else {
1618 /*
1619 * If we've successfully read stuff, return it rather than
1620 * the error. (Not sure if this is such a great idea...)
1621 */
1622 if (cbRet > 0) {
1623 SFLOGFLOW(("vbsf_reg_read: read at %#RX64 -> %Rrc; got cbRet=%#zx already\n", offFile, rc, cbRet));
1624 *off = offFile;
1625 } else {
1626 SFLOGFLOW(("vbsf_reg_read: read at %#RX64 -> %Rrc\n", offFile, rc));
1627 cbRet = -EPROTO;
1628 }
1629 break;
1630 }
1631 }
1632 }
1633 if (papPagesFree)
1634 kfree(papPages);
1635 if (pReq)
1636 VbglR0PhysHeapFree(pReq);
1637 SFLOGFLOW(("vbsf_reg_read: returns %zd (%#zx), *off=%RX64 [lock]\n", cbRet, cbRet, *off));
1638 return cbRet;
1639}
1640
1641
1642/**
1643 * Read from a regular file.
1644 *
1645 * @param file the file
1646 * @param buf the buffer
1647 * @param size length of the buffer
1648 * @param off offset within the file (in/out).
1649 * @returns the number of read bytes on success, Linux error code otherwise
1650 */
1651static ssize_t vbsf_reg_read(struct file *file, char /*__user*/ *buf, size_t size, loff_t *off)
1652{
1653 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
1654 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
1655 struct vbsf_reg_info *sf_r = file->private_data;
1656 struct address_space *mapping = inode->i_mapping;
1657
1658 SFLOGFLOW(("vbsf_reg_read: inode=%p file=%p buf=%p size=%#zx off=%#llx\n", inode, file, buf, size, *off));
1659
1660 if (!S_ISREG(inode->i_mode)) {
1661 LogFunc(("read from non regular file %d\n", inode->i_mode));
1662 return -EINVAL;
1663 }
1664
1665 /** @todo XXX Check read permission according to inode->i_mode! */
1666
1667 if (!size)
1668 return 0;
1669
1670 /*
1671 * If there is a mapping and O_DIRECT isn't in effect, we must at a
1672 * heed dirty pages in the mapping and read from them. For simplicity
1673 * though, we just do page cache reading when there are writable
1674 * mappings around with any kind of pages loaded.
1675 */
1676 if (vbsf_should_use_cached_read(file, mapping, pSuperInfo))
1677 return vbsf_reg_read_mapped(file, buf, size, off);
1678
1679 /*
1680 * For small requests, try use an embedded buffer provided we get a heap block
1681 * that does not cross page boundraries (see host code).
1682 */
1683 if (size <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) /* see allocator */) {
1684 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) + size;
1685 VBOXSFREADEMBEDDEDREQ *pReq = (VBOXSFREADEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
1686 if (pReq) {
1687 if ((PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
1688 ssize_t cbRet;
1689 int vrc = VbglR0SfHostReqReadEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, *off, (uint32_t)size);
1690 if (RT_SUCCESS(vrc)) {
1691 cbRet = pReq->Parms.cb32Read.u.value32;
1692 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
1693 if (copy_to_user(buf, pReq->abData, cbRet) == 0)
1694 *off += cbRet;
1695 else
1696 cbRet = -EFAULT;
1697 } else
1698 cbRet = -EPROTO;
1699 VbglR0PhysHeapFree(pReq);
1700 SFLOGFLOW(("vbsf_reg_read: returns %zd (%#zx), *off=%RX64 [embed]\n", cbRet, cbRet, *off));
1701 return cbRet;
1702 }
1703 VbglR0PhysHeapFree(pReq);
1704 }
1705 }
1706
1707# if 0 /* Turns out this is slightly slower than locking the pages even for 4KB reads (4.19/amd64). */
1708 /*
1709 * For medium sized requests try use a bounce buffer.
1710 */
1711 if (size <= _64K /** @todo make this configurable? */) {
1712 void *pvBounce = kmalloc(size, GFP_KERNEL);
1713 if (pvBounce) {
1714 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
1715 if (pReq) {
1716 ssize_t cbRet;
1717 int vrc = VbglR0SfHostReqReadContig(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, *off,
1718 (uint32_t)size, pvBounce, virt_to_phys(pvBounce));
1719 if (RT_SUCCESS(vrc)) {
1720 cbRet = pReq->Parms.cb32Read.u.value32;
1721 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
1722 if (copy_to_user(buf, pvBounce, cbRet) == 0)
1723 *off += cbRet;
1724 else
1725 cbRet = -EFAULT;
1726 } else
1727 cbRet = -EPROTO;
1728 VbglR0PhysHeapFree(pReq);
1729 kfree(pvBounce);
1730 SFLOGFLOW(("vbsf_reg_read: returns %zd (%#zx), *off=%RX64 [bounce]\n", cbRet, cbRet, *off));
1731 return cbRet;
1732 }
1733 kfree(pvBounce);
1734 }
1735 }
1736# endif
1737
1738 return vbsf_reg_read_locking(file, buf, size, off, pSuperInfo, sf_r);
1739}
1740
1741#endif /* < 5.10.0 */
1742
1743/**
1744 * Helper the synchronizes the page cache content with something we just wrote
1745 * to the host.
1746 */
1747static void vbsf_reg_write_sync_page_cache(struct address_space *mapping, loff_t offFile, uint32_t cbRange,
1748 uint8_t const *pbSrcBuf, struct page **papSrcPages,
1749 uint32_t offSrcPage, size_t cSrcPages)
1750{
1751 Assert(offSrcPage < PAGE_SIZE);
1752 if (mapping && mapping->nrpages > 0) {
1753 /*
1754 * Work the pages in the write range.
1755 */
1756 while (cbRange > 0) {
1757 /*
1758 * Lookup the page at offFile. We're fine if there aren't
1759 * any there. We're skip if it's dirty or is being written
1760 * back, at least for now.
1761 */
1762 size_t const offDstPage = offFile & PAGE_OFFSET_MASK;
1763 size_t const cbToCopy = RT_MIN(PAGE_SIZE - offDstPage, cbRange);
1764 pgoff_t const idxPage = offFile >> PAGE_SHIFT;
1765 struct page *pDstPage = find_lock_page(mapping, idxPage);
1766 if (pDstPage) {
1767 if ( pDstPage->mapping == mapping /* ignore if re-purposed (paranoia) */
1768 && pDstPage->index == idxPage
1769 && !PageDirty(pDstPage) /* ignore if dirty */
1770 && !PageWriteback(pDstPage) /* ignore if being written back */ ) {
1771 /*
1772 * Map the page and do the copying.
1773 */
1774 uint8_t *pbDst = (uint8_t *)kmap(pDstPage);
1775 if (pbSrcBuf)
1776 memcpy(&pbDst[offDstPage], pbSrcBuf, cbToCopy);
1777 else {
1778 uint32_t const cbSrc0 = PAGE_SIZE - offSrcPage;
1779 uint8_t const *pbSrc = (uint8_t const *)kmap(papSrcPages[0]);
1780 AssertMsg(cSrcPages >= 1, ("offFile=%#llx cbRange=%#zx cbToCopy=%#zx\n", offFile, cbRange, cbToCopy));
1781 memcpy(&pbDst[offDstPage], &pbSrc[offSrcPage], RT_MIN(cbToCopy, cbSrc0));
1782 kunmap(papSrcPages[0]);
1783 if (cbToCopy > cbSrc0) {
1784 AssertMsg(cSrcPages >= 2, ("offFile=%#llx cbRange=%#zx cbToCopy=%#zx\n", offFile, cbRange, cbToCopy));
1785 pbSrc = (uint8_t const *)kmap(papSrcPages[1]);
1786 memcpy(&pbDst[offDstPage + cbSrc0], pbSrc, cbToCopy - cbSrc0);
1787 kunmap(papSrcPages[1]);
1788 }
1789 }
1790 kunmap(pDstPage);
1791 flush_dcache_page(pDstPage);
1792 if (cbToCopy == PAGE_SIZE)
1793 SetPageUptodate(pDstPage);
1794# if RTLNX_VER_MIN(2,4,10)
1795 mark_page_accessed(pDstPage);
1796# endif
1797 } else
1798 SFLOGFLOW(("vbsf_reg_write_sync_page_cache: Skipping page %p: mapping=%p (vs %p) writeback=%d offset=%#lx (vs%#lx)\n",
1799 pDstPage, pDstPage->mapping, mapping, PageWriteback(pDstPage), pDstPage->index, idxPage));
1800 unlock_page(pDstPage);
1801 vbsf_put_page(pDstPage);
1802 }
1803
1804 /*
1805 * Advance.
1806 */
1807 if (pbSrcBuf)
1808 pbSrcBuf += cbToCopy;
1809 else
1810 {
1811 offSrcPage += cbToCopy;
1812 Assert(offSrcPage < PAGE_SIZE * 2);
1813 if (offSrcPage >= PAGE_SIZE) {
1814 offSrcPage &= PAGE_OFFSET_MASK;
1815 papSrcPages++;
1816# ifdef VBOX_STRICT
1817 Assert(cSrcPages > 0);
1818 cSrcPages--;
1819# endif
1820 }
1821 }
1822 offFile += cbToCopy;
1823 cbRange -= cbToCopy;
1824 }
1825 }
1826 RT_NOREF(cSrcPages);
1827}
1828
1829#if RTLNX_VER_MAX(5,10,0) /* No regular .read/.write for 5.10, only .read_iter/.write_iter or in-kernel reads/writes fail. */
1830
1831/**
1832 * Fallback case of vbsf_reg_write() that locks the user buffers and let the host
1833 * write directly to them.
1834 */
1835static ssize_t vbsf_reg_write_locking(struct file *file, const char /*__user*/ *buf, size_t size, loff_t *off, loff_t offFile,
1836 struct inode *inode, struct vbsf_inode_info *sf_i,
1837 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r)
1838{
1839 /*
1840 * Lock pages and execute the write, taking care not to pass the host
1841 * more than it can handle in one go or more than we care to allocate
1842 * page arrays for. The latter limit is set at just short of 32KB due
1843 * to how the physical heap works.
1844 */
1845 struct page *apPagesStack[16];
1846 struct page **papPages = &apPagesStack[0];
1847 struct page **papPagesFree = NULL;
1848 VBOXSFWRITEPGLSTREQ *pReq;
1849 ssize_t cbRet = -ENOMEM;
1850 size_t cPages = (((uintptr_t)buf & PAGE_OFFSET_MASK) + size + PAGE_OFFSET_MASK) >> PAGE_SHIFT;
1851 size_t cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 1), cPages);
1852 bool fLockPgHack;
1853
1854 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
1855 while (!pReq && cMaxPages > 4) {
1856 cMaxPages /= 2;
1857 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
1858 }
1859 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
1860 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
1861 if (pReq && papPages) {
1862 cbRet = 0;
1863 for (;;) {
1864 /*
1865 * Figure out how much to process now and lock the user pages.
1866 */
1867 int rc;
1868 size_t cbChunk = (uintptr_t)buf & PAGE_OFFSET_MASK;
1869 pReq->PgLst.offFirstPage = (uint16_t)cbChunk;
1870 cPages = RT_ALIGN_Z(cbChunk + size, PAGE_SIZE) >> PAGE_SHIFT;
1871 if (cPages <= cMaxPages)
1872 cbChunk = size;
1873 else {
1874 cPages = cMaxPages;
1875 cbChunk = (cMaxPages << PAGE_SHIFT) - cbChunk;
1876 }
1877
1878 rc = vbsf_lock_user_pages((uintptr_t)buf, cPages, false /*fWrite*/, papPages, &fLockPgHack);
1879 if (rc == 0) {
1880 size_t iPage = cPages;
1881 while (iPage-- > 0)
1882 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
1883 } else {
1884 cbRet = rc;
1885 break;
1886 }
1887
1888 /*
1889 * Issue the request and unlock the pages.
1890 */
1891 rc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbChunk, cPages);
1892 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
1893 if (RT_SUCCESS(rc)) {
1894 /*
1895 * Success, advance position and buffer.
1896 */
1897 uint32_t cbActual = pReq->Parms.cb32Write.u.value32;
1898 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
1899
1900 vbsf_reg_write_sync_page_cache(inode->i_mapping, offFile, cbActual, NULL /*pbKrnlBuf*/,
1901 papPages, (uintptr_t)buf & PAGE_OFFSET_MASK, cPages);
1902 Assert(cPages <= cMaxPages);
1903 vbsf_unlock_user_pages(papPages, cPages, false /*fSetDirty*/, fLockPgHack);
1904
1905 cbRet += cbActual;
1906 buf = (uint8_t *)buf + cbActual;
1907 size -= cbActual;
1908
1909 offFile += cbActual;
1910 if ((file->f_flags & O_APPEND) && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
1911 offFile = pReq->Parms.off64Write.u.value64;
1912 if (offFile > i_size_read(inode))
1913 i_size_write(inode, offFile);
1914
1915 sf_i->force_restat = 1; /* mtime (and size) may have changed */
1916
1917 /*
1918 * Are we done already? If so commit the new file offset.
1919 */
1920 if (!size || cbActual < cbChunk) {
1921 *off = offFile;
1922 break;
1923 }
1924 } else {
1925 vbsf_unlock_user_pages(papPages, cPages, false /*fSetDirty*/, fLockPgHack);
1926 if (rc == VERR_NO_MEMORY && cMaxPages > 4) {
1927 /*
1928 * The host probably doesn't have enough heap to handle the
1929 * request, reduce the page count and retry.
1930 */
1931 cMaxPages /= 4;
1932 Assert(cMaxPages > 0);
1933 } else {
1934 /*
1935 * If we've successfully written stuff, return it rather than
1936 * the error. (Not sure if this is such a great idea...)
1937 */
1938 if (cbRet > 0) {
1939 SFLOGFLOW(("vbsf_reg_write: write at %#RX64 -> %Rrc; got cbRet=%#zx already\n", offFile, rc, cbRet));
1940 *off = offFile;
1941 } else {
1942 SFLOGFLOW(("vbsf_reg_write: write at %#RX64 -> %Rrc\n", offFile, rc));
1943 cbRet = -EPROTO;
1944 }
1945 break;
1946 }
1947 }
1948 }
1949 }
1950 if (papPagesFree)
1951 kfree(papPages);
1952 if (pReq)
1953 VbglR0PhysHeapFree(pReq);
1954 SFLOGFLOW(("vbsf_reg_write: returns %zd (%#zx), *off=%RX64 [lock]\n", cbRet, cbRet, *off));
1955 return cbRet;
1956}
1957
1958
1959/**
1960 * Write to a regular file.
1961 *
1962 * @param file the file
1963 * @param buf the buffer
1964 * @param size length of the buffer
1965 * @param off offset within the file
1966 * @returns the number of written bytes on success, Linux error code otherwise
1967 */
1968static ssize_t vbsf_reg_write(struct file *file, const char *buf, size_t size, loff_t * off)
1969{
1970 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
1971 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
1972 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
1973 struct vbsf_reg_info *sf_r = file->private_data;
1974 struct address_space *mapping = inode->i_mapping;
1975 loff_t pos;
1976
1977 SFLOGFLOW(("vbsf_reg_write: inode=%p file=%p buf=%p size=%#zx off=%#llx\n", inode, file, buf, size, *off));
1978 Assert(sf_i);
1979 Assert(pSuperInfo);
1980 Assert(sf_r);
1981 AssertReturn(S_ISREG(inode->i_mode), -EINVAL);
1982
1983 pos = *off;
1984 if (file->f_flags & O_APPEND)
1985 pos = i_size_read(inode);
1986
1987 /** @todo XXX Check write permission according to inode->i_mode! */
1988
1989 if (!size) {
1990 if (file->f_flags & O_APPEND) /** @todo check if this is the consensus behavior... */
1991 *off = pos;
1992 return 0;
1993 }
1994
1995 /** @todo Implement the read-write caching mode. */
1996
1997 /*
1998 * If there are active writable mappings, coordinate with any
1999 * pending writes via those.
2000 */
2001 if ( mapping
2002 && mapping->nrpages > 0
2003 && mapping_writably_mapped(mapping)) {
2004# if RTLNX_VER_MIN(2,6,32)
2005 int err = filemap_fdatawait_range(mapping, pos, pos + size - 1);
2006 if (err)
2007 return err;
2008# else
2009 /** @todo ... */
2010# endif
2011 }
2012
2013 /*
2014 * For small requests, try use an embedded buffer provided we get a heap block
2015 * that does not cross page boundraries (see host code).
2016 */
2017 if (size <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) /* see allocator */) {
2018 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) + size;
2019 VBOXSFWRITEEMBEDDEDREQ *pReq = (VBOXSFWRITEEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
2020 if ( pReq
2021 && (PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
2022 ssize_t cbRet;
2023 if (copy_from_user(pReq->abData, buf, size) == 0) {
2024 int vrc = VbglR0SfHostReqWriteEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost,
2025 pos, (uint32_t)size);
2026 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2027 if (RT_SUCCESS(vrc)) {
2028 cbRet = pReq->Parms.cb32Write.u.value32;
2029 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
2030 vbsf_reg_write_sync_page_cache(mapping, pos, (uint32_t)cbRet, pReq->abData,
2031 NULL /*papSrcPages*/, 0 /*offSrcPage0*/, 0 /*cSrcPages*/);
2032 pos += cbRet;
2033 if ((file->f_flags & O_APPEND) && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
2034 pos = pReq->Parms.off64Write.u.value64;
2035 *off = pos;
2036 if (pos > i_size_read(inode))
2037 i_size_write(inode, pos);
2038 } else
2039 cbRet = -EPROTO;
2040 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2041 } else
2042 cbRet = -EFAULT;
2043
2044 VbglR0PhysHeapFree(pReq);
2045 SFLOGFLOW(("vbsf_reg_write: returns %zd (%#zx), *off=%RX64 [embed]\n", cbRet, cbRet, *off));
2046 return cbRet;
2047 }
2048 if (pReq)
2049 VbglR0PhysHeapFree(pReq);
2050 }
2051
2052# if 0 /* Turns out this is slightly slower than locking the pages even for 4KB reads (4.19/amd64). */
2053 /*
2054 * For medium sized requests try use a bounce buffer.
2055 */
2056 if (size <= _64K /** @todo make this configurable? */) {
2057 void *pvBounce = kmalloc(size, GFP_KERNEL);
2058 if (pvBounce) {
2059 if (copy_from_user(pvBounce, buf, size) == 0) {
2060 VBOXSFWRITEPGLSTREQ *pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
2061 if (pReq) {
2062 ssize_t cbRet;
2063 int vrc = VbglR0SfHostReqWriteContig(pSuperInfo->map.root, pReq, sf_r->handle, pos,
2064 (uint32_t)size, pvBounce, virt_to_phys(pvBounce));
2065 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2066 if (RT_SUCCESS(vrc)) {
2067 cbRet = pReq->Parms.cb32Write.u.value32;
2068 AssertStmt(cbRet <= (ssize_t)size, cbRet = size);
2069 vbsf_reg_write_sync_page_cache(mapping, pos, (uint32_t)cbRet, (uint8_t const *)pvBounce,
2070 NULL /*papSrcPages*/, 0 /*offSrcPage0*/, 0 /*cSrcPages*/);
2071 pos += cbRet;
2072 *off = pos;
2073 if (pos > i_size_read(inode))
2074 i_size_write(inode, pos);
2075 } else
2076 cbRet = -EPROTO;
2077 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2078 VbglR0PhysHeapFree(pReq);
2079 kfree(pvBounce);
2080 SFLOGFLOW(("vbsf_reg_write: returns %zd (%#zx), *off=%RX64 [bounce]\n", cbRet, cbRet, *off));
2081 return cbRet;
2082 }
2083 kfree(pvBounce);
2084 } else {
2085 kfree(pvBounce);
2086 SFLOGFLOW(("vbsf_reg_write: returns -EFAULT, *off=%RX64 [bounce]\n", *off));
2087 return -EFAULT;
2088 }
2089 }
2090 }
2091# endif
2092
2093 return vbsf_reg_write_locking(file, buf, size, off, pos, inode, sf_i, pSuperInfo, sf_r);
2094}
2095
2096#endif /* < 5.10.0 */
2097#if RTLNX_VER_MIN(2,6,19)
2098
2099/**
2100 * Companion to vbsf_iter_lock_pages().
2101 */
2102DECLINLINE(void) vbsf_iter_unlock_pages(struct iov_iter *iter, struct page **papPages, size_t cPages, bool fSetDirty)
2103{
2104 /* We don't mark kernel pages dirty (KVECs, BVECs, PIPEs): */
2105 if (!iter_is_iovec(iter))
2106 fSetDirty = false;
2107
2108 while (cPages-- > 0)
2109 {
2110 struct page *pPage = papPages[cPages];
2111 if (fSetDirty && !PageReserved(pPage))
2112 set_page_dirty(pPage);
2113 vbsf_put_page(pPage);
2114 }
2115}
2116
2117
2118/**
2119 * Locks up to @a cMaxPages from the I/O vector iterator, advancing the
2120 * iterator.
2121 *
2122 * @returns 0 on success, negative errno value on failure.
2123 * @param iter The iterator to lock pages from.
2124 * @param fWrite Whether to write (true) or read (false) lock the pages.
2125 * @param pStash Where we stash peek results.
2126 * @param cMaxPages The maximum number of pages to get.
2127 * @param papPages Where to return the locked pages.
2128 * @param pcPages Where to return the number of pages.
2129 * @param poffPage0 Where to return the offset into the first page.
2130 * @param pcbChunk Where to return the number of bytes covered.
2131 */
2132static int vbsf_iter_lock_pages(struct iov_iter *iter, bool fWrite, struct vbsf_iter_stash *pStash, size_t cMaxPages,
2133 struct page **papPages, size_t *pcPages, size_t *poffPage0, size_t *pcbChunk)
2134{
2135 size_t cbChunk = 0;
2136 size_t cPages = 0;
2137 size_t offPage0 = 0;
2138 int rc = 0;
2139
2140 Assert(iov_iter_count(iter) + pStash->cb > 0);
2141 if (!(VBSF_GET_ITER_TYPE(iter) & ITER_KVEC)) {
2142 /*
2143 * Do we have a stashed page?
2144 */
2145 if (pStash->pPage) {
2146 papPages[0] = pStash->pPage;
2147 offPage0 = pStash->off;
2148 cbChunk = pStash->cb;
2149 cPages = 1;
2150 pStash->pPage = NULL;
2151 pStash->off = 0;
2152 pStash->cb = 0;
2153 if ( offPage0 + cbChunk < PAGE_SIZE
2154 || iov_iter_count(iter) == 0) {
2155 *poffPage0 = offPage0;
2156 *pcbChunk = cbChunk;
2157 *pcPages = cPages;
2158 SFLOGFLOW(("vbsf_iter_lock_pages: returns %d - cPages=%#zx offPage0=%#zx cbChunk=%zx (stashed)\n",
2159 rc, cPages, offPage0, cbChunk));
2160 return 0;
2161 }
2162 cMaxPages -= 1;
2163 SFLOG3(("vbsf_iter_lock_pages: Picked up stashed page: %#zx LB %#zx\n", offPage0, cbChunk));
2164 } else {
2165# if RTLNX_VER_MAX(4,11,0)
2166 /*
2167 * Copy out our starting point to assist rewinding.
2168 */
2169 pStash->offFromEnd = iov_iter_count(iter);
2170 pStash->Copy = *iter;
2171# endif
2172 }
2173
2174 /*
2175 * Get pages segment by segment.
2176 */
2177 do {
2178 /*
2179 * Make a special case of the first time thru here, since that's
2180 * the most typical scenario.
2181 */
2182 ssize_t cbSegRet;
2183 if (cPages == 0) {
2184# if RTLNX_VER_MAX(3,19,0)
2185 while (!iov_iter_single_seg_count(iter)) /* Old code didn't skip empty segments which caused EFAULTs. */
2186 iov_iter_advance(iter, 0);
2187# endif
2188 cbSegRet = iov_iter_get_pages(iter, papPages, iov_iter_count(iter), cMaxPages, &offPage0);
2189 if (cbSegRet > 0) {
2190 iov_iter_advance(iter, cbSegRet);
2191 cbChunk = (size_t)cbSegRet;
2192 cPages = RT_ALIGN_Z(offPage0 + cbSegRet, PAGE_SIZE) >> PAGE_SHIFT;
2193 cMaxPages -= cPages;
2194 SFLOG3(("vbsf_iter_lock_pages: iov_iter_get_pages -> %#zx @ %#zx; %#zx pages [first]\n", cbSegRet, offPage0, cPages));
2195 if ( cMaxPages == 0
2196 || ((offPage0 + (size_t)cbSegRet) & PAGE_OFFSET_MASK))
2197 break;
2198 } else {
2199 AssertStmt(cbSegRet < 0, cbSegRet = -EFAULT);
2200 rc = (int)cbSegRet;
2201 break;
2202 }
2203 } else {
2204 /*
2205 * Probe first page of new segment to check that we've got a zero offset and
2206 * can continue on the current chunk. Stash the page if the offset isn't zero.
2207 */
2208 size_t offPgProbe;
2209 size_t cbSeg = iov_iter_single_seg_count(iter);
2210 while (!cbSeg) {
2211 iov_iter_advance(iter, 0);
2212 cbSeg = iov_iter_single_seg_count(iter);
2213 }
2214 cbSegRet = iov_iter_get_pages(iter, &papPages[cPages], iov_iter_count(iter), 1, &offPgProbe);
2215 if (cbSegRet > 0) {
2216 iov_iter_advance(iter, cbSegRet); /** @todo maybe not do this if we stash the page? */
2217 Assert(offPgProbe + cbSegRet <= PAGE_SIZE);
2218 if (offPgProbe == 0) {
2219 cbChunk += cbSegRet;
2220 cPages += 1;
2221 cMaxPages -= 1;
2222 SFLOG3(("vbsf_iter_lock_pages: iov_iter_get_pages(1) -> %#zx @ %#zx\n", cbSegRet, offPgProbe));
2223 if ( cMaxPages == 0
2224 || cbSegRet != PAGE_SIZE)
2225 break;
2226
2227 /*
2228 * Get the rest of the segment (if anything remaining).
2229 */
2230 cbSeg -= cbSegRet;
2231 if (cbSeg > 0) {
2232 cbSegRet = iov_iter_get_pages(iter, &papPages[cPages], iov_iter_count(iter), cMaxPages, &offPgProbe);
2233 if (cbSegRet > 0) {
2234 size_t const cPgRet = RT_ALIGN_Z((size_t)cbSegRet, PAGE_SIZE) >> PAGE_SHIFT;
2235 Assert(offPgProbe == 0);
2236 iov_iter_advance(iter, cbSegRet);
2237 SFLOG3(("vbsf_iter_lock_pages: iov_iter_get_pages() -> %#zx; %#zx pages\n", cbSegRet, cPgRet));
2238 cPages += cPgRet;
2239 cMaxPages -= cPgRet;
2240 cbChunk += cbSegRet;
2241 if ( cMaxPages == 0
2242 || ((size_t)cbSegRet & PAGE_OFFSET_MASK))
2243 break;
2244 } else {
2245 AssertStmt(cbSegRet < 0, cbSegRet = -EFAULT);
2246 rc = (int)cbSegRet;
2247 break;
2248 }
2249 }
2250 } else {
2251 /* The segment didn't start at a page boundrary, so stash it for
2252 the next round: */
2253 SFLOGFLOW(("vbsf_iter_lock_pages: iov_iter_get_pages(1) -> %#zx @ %#zx; stashed\n", cbSegRet, offPgProbe));
2254 Assert(papPages[cPages]);
2255 pStash->pPage = papPages[cPages];
2256 pStash->off = offPgProbe;
2257 pStash->cb = cbSegRet;
2258 break;
2259 }
2260 } else {
2261 AssertStmt(cbSegRet < 0, cbSegRet = -EFAULT);
2262 rc = (int)cbSegRet;
2263 break;
2264 }
2265 }
2266 Assert(cMaxPages > 0);
2267 } while (iov_iter_count(iter) > 0);
2268
2269 } else {
2270 /*
2271 * The silly iov_iter_get_pages_alloc() function doesn't handle KVECs,
2272 * so everyone needs to do that by themselves.
2273 *
2274 * Note! Fixes here may apply to rtR0MemObjNativeLockKernel()
2275 * and vbsf_lock_user_pages_failed_check_kernel() as well.
2276 */
2277# if RTLNX_VER_MAX(4,11,0)
2278 pStash->offFromEnd = iov_iter_count(iter);
2279 pStash->Copy = *iter;
2280# endif
2281 do {
2282 uint8_t *pbBuf;
2283 size_t offStart;
2284 size_t cPgSeg;
2285
2286 size_t cbSeg = iov_iter_single_seg_count(iter);
2287 while (!cbSeg) {
2288 iov_iter_advance(iter, 0);
2289 cbSeg = iov_iter_single_seg_count(iter);
2290 }
2291
2292# if RTLNX_VER_MIN(3,19,0)
2293 pbBuf = iter->kvec->iov_base + iter->iov_offset;
2294# else
2295 pbBuf = iter->iov->iov_base + iter->iov_offset;
2296# endif
2297 offStart = (uintptr_t)pbBuf & PAGE_OFFSET_MASK;
2298 if (!cPages)
2299 offPage0 = offStart;
2300 else if (offStart)
2301 break;
2302
2303 cPgSeg = RT_ALIGN_Z(cbSeg, PAGE_SIZE) >> PAGE_SHIFT;
2304 if (cPgSeg > cMaxPages) {
2305 cPgSeg = cMaxPages;
2306 cbSeg = (cPgSeg << PAGE_SHIFT) - offStart;
2307 }
2308
2309 rc = vbsf_lock_kernel_pages(pbBuf, fWrite, cPgSeg, &papPages[cPages]);
2310 if (rc == 0) {
2311 iov_iter_advance(iter, cbSeg);
2312 cbChunk += cbSeg;
2313 cPages += cPgSeg;
2314 cMaxPages -= cPgSeg;
2315 if ( cMaxPages == 0
2316 || ((offStart + cbSeg) & PAGE_OFFSET_MASK) != 0)
2317 break;
2318 } else
2319 break;
2320 } while (iov_iter_count(iter) > 0);
2321 }
2322
2323 /*
2324 * Clean up if we failed; set return values.
2325 */
2326 if (rc == 0) {
2327 /* likely */
2328 } else {
2329 if (cPages > 0)
2330 vbsf_iter_unlock_pages(iter, papPages, cPages, false /*fSetDirty*/);
2331 offPage0 = cbChunk = cPages = 0;
2332 }
2333 *poffPage0 = offPage0;
2334 *pcbChunk = cbChunk;
2335 *pcPages = cPages;
2336 SFLOGFLOW(("vbsf_iter_lock_pages: returns %d - cPages=%#zx offPage0=%#zx cbChunk=%zx\n", rc, cPages, offPage0, cbChunk));
2337 return rc;
2338}
2339
2340
2341/**
2342 * Rewinds the I/O vector.
2343 */
2344static bool vbsf_iter_rewind(struct iov_iter *iter, struct vbsf_iter_stash *pStash, size_t cbToRewind, size_t cbChunk)
2345{
2346 size_t cbExtra;
2347 if (!pStash->pPage) {
2348 cbExtra = 0;
2349 } else {
2350 cbExtra = pStash->cb;
2351 vbsf_put_page(pStash->pPage);
2352 pStash->pPage = NULL;
2353 pStash->cb = 0;
2354 pStash->off = 0;
2355 }
2356
2357# if RTLNX_VER_MIN(4,11,0) || RTLNX_VER_MAX(3,16,0)
2358 iov_iter_revert(iter, cbToRewind + cbExtra);
2359 return true;
2360# else
2361 /** @todo impl this */
2362 return false;
2363# endif
2364}
2365
2366
2367/**
2368 * Cleans up the page locking stash.
2369 */
2370DECLINLINE(void) vbsf_iter_cleanup_stash(struct iov_iter *iter, struct vbsf_iter_stash *pStash)
2371{
2372 if (pStash->pPage)
2373 vbsf_iter_rewind(iter, pStash, 0, 0);
2374}
2375
2376
2377/**
2378 * Calculates the longest span of pages we could transfer to the host in a
2379 * single request.
2380 *
2381 * @returns Page count, non-zero.
2382 * @param iter The I/O vector iterator to inspect.
2383 */
2384static size_t vbsf_iter_max_span_of_pages(struct iov_iter *iter)
2385{
2386 size_t cPages;
2387# if RTLNX_VER_MIN(3,16,0)
2388 if (iter_is_iovec(iter) || (VBSF_GET_ITER_TYPE(iter) & ITER_KVEC)) {
2389# endif
2390 const struct iovec *pCurIov = iter->iov;
2391 size_t cLeft = iter->nr_segs;
2392 size_t cPagesSpan = 0;
2393
2394 /* iovect and kvec are identical, except for the __user tagging of iov_base. */
2395 AssertCompileMembersSameSizeAndOffset(struct iovec, iov_base, struct kvec, iov_base);
2396 AssertCompileMembersSameSizeAndOffset(struct iovec, iov_len, struct kvec, iov_len);
2397 AssertCompile(sizeof(struct iovec) == sizeof(struct kvec));
2398
2399 cPages = 1;
2400 AssertReturn(cLeft > 0, cPages);
2401
2402 /* Special case: segment offset. */
2403 if (iter->iov_offset > 0) {
2404 if (iter->iov_offset < pCurIov->iov_len) {
2405 size_t const cbSegLeft = pCurIov->iov_len - iter->iov_offset;
2406 size_t const offPage0 = ((uintptr_t)pCurIov->iov_base + iter->iov_offset) & PAGE_OFFSET_MASK;
2407 cPages = cPagesSpan = RT_ALIGN_Z(offPage0 + cbSegLeft, PAGE_SIZE) >> PAGE_SHIFT;
2408 if ((offPage0 + cbSegLeft) & PAGE_OFFSET_MASK)
2409 cPagesSpan = 0;
2410 }
2411 SFLOGFLOW(("vbsf_iter: seg[0]= %p LB %#zx\n", pCurIov->iov_base, pCurIov->iov_len));
2412 pCurIov++;
2413 cLeft--;
2414 }
2415
2416 /* Full segments. */
2417 while (cLeft-- > 0) {
2418 if (pCurIov->iov_len > 0) {
2419 size_t const offPage0 = (uintptr_t)pCurIov->iov_base & PAGE_OFFSET_MASK;
2420 if (offPage0 == 0) {
2421 if (!(pCurIov->iov_len & PAGE_OFFSET_MASK)) {
2422 cPagesSpan += pCurIov->iov_len >> PAGE_SHIFT;
2423 } else {
2424 cPagesSpan += RT_ALIGN_Z(pCurIov->iov_len, PAGE_SIZE) >> PAGE_SHIFT;
2425 if (cPagesSpan > cPages)
2426 cPages = cPagesSpan;
2427 cPagesSpan = 0;
2428 }
2429 } else {
2430 if (cPagesSpan > cPages)
2431 cPages = cPagesSpan;
2432 if (!((offPage0 + pCurIov->iov_len) & PAGE_OFFSET_MASK)) {
2433 cPagesSpan = pCurIov->iov_len >> PAGE_SHIFT;
2434 } else {
2435 cPagesSpan += RT_ALIGN_Z(offPage0 + pCurIov->iov_len, PAGE_SIZE) >> PAGE_SHIFT;
2436 if (cPagesSpan > cPages)
2437 cPages = cPagesSpan;
2438 cPagesSpan = 0;
2439 }
2440 }
2441 }
2442 SFLOGFLOW(("vbsf_iter: seg[%u]= %p LB %#zx\n", iter->nr_segs - cLeft, pCurIov->iov_base, pCurIov->iov_len));
2443 pCurIov++;
2444 }
2445 if (cPagesSpan > cPages)
2446 cPages = cPagesSpan;
2447# if RTLNX_VER_MIN(3,16,0)
2448 } else {
2449 /* Won't bother with accurate counts for the next two types, just make
2450 some rough estimates (does pipes have segments?): */
2451 size_t cSegs = VBSF_GET_ITER_TYPE(iter) & ITER_BVEC ? RT_MAX(1, iter->nr_segs) : 1;
2452 cPages = (iov_iter_count(iter) + (PAGE_SIZE * 2 - 2) * cSegs) >> PAGE_SHIFT;
2453 }
2454# endif
2455 SFLOGFLOW(("vbsf_iter_max_span_of_pages: returns %#zx\n", cPages));
2456 return cPages;
2457}
2458
2459
2460/**
2461 * Worker for vbsf_reg_read_iter() that deals with larger reads using page
2462 * locking.
2463 */
2464static ssize_t vbsf_reg_read_iter_locking(struct kiocb *kio, struct iov_iter *iter, size_t cbToRead,
2465 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r)
2466{
2467 /*
2468 * Estimate how many pages we may possible submit in a single request so
2469 * that we can allocate matching request buffer and page array.
2470 */
2471 struct page *apPagesStack[16];
2472 struct page **papPages = &apPagesStack[0];
2473 struct page **papPagesFree = NULL;
2474 VBOXSFREADPGLSTREQ *pReq;
2475 ssize_t cbRet = 0;
2476 size_t cMaxPages = vbsf_iter_max_span_of_pages(iter);
2477 cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 2), cMaxPages);
2478
2479 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
2480 while (!pReq && cMaxPages > 4) {
2481 cMaxPages /= 2;
2482 pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFREADPGLSTREQ, PgLst.aPages[cMaxPages]));
2483 }
2484 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
2485 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
2486 if (pReq && papPages) {
2487
2488 /*
2489 * The read loop.
2490 */
2491 struct vbsf_iter_stash Stash = VBSF_ITER_STASH_INITIALIZER;
2492 do {
2493 /*
2494 * Grab as many pages as we can. This means that if adjacent
2495 * segments both starts and ends at a page boundrary, we can
2496 * do them both in the same transfer from the host.
2497 */
2498 size_t cPages = 0;
2499 size_t cbChunk = 0;
2500 size_t offPage0 = 0;
2501 int rc = vbsf_iter_lock_pages(iter, true /*fWrite*/, &Stash, cMaxPages, papPages, &cPages, &offPage0, &cbChunk);
2502 if (rc == 0) {
2503 size_t iPage = cPages;
2504 while (iPage-- > 0)
2505 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
2506 pReq->PgLst.offFirstPage = (uint16_t)offPage0;
2507 AssertStmt(cbChunk <= cbToRead, cbChunk = cbToRead);
2508 } else {
2509 cbRet = rc;
2510 break;
2511 }
2512
2513 /*
2514 * Issue the request and unlock the pages.
2515 */
2516 rc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, kio->ki_pos, cbChunk, cPages);
2517 SFLOGFLOW(("vbsf_reg_read_iter_locking: VbglR0SfHostReqReadPgLst -> %d (cbActual=%#x cbChunk=%#zx of %#zx cPages=%#zx offPage0=%#x\n",
2518 rc, pReq->Parms.cb32Read.u.value32, cbChunk, cbToRead, cPages, offPage0));
2519
2520 vbsf_iter_unlock_pages(iter, papPages, cPages, true /*fSetDirty*/);
2521
2522 if (RT_SUCCESS(rc)) {
2523 /*
2524 * Success, advance position and buffer.
2525 */
2526 uint32_t cbActual = pReq->Parms.cb32Read.u.value32;
2527 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
2528 cbRet += cbActual;
2529 kio->ki_pos += cbActual;
2530 cbToRead -= cbActual;
2531
2532 /*
2533 * Are we done already?
2534 */
2535 if (!cbToRead)
2536 break;
2537 if (cbActual < cbChunk) { /* We ASSUME end-of-file here. */
2538 if (vbsf_iter_rewind(iter, &Stash, cbChunk - cbActual, cbActual))
2539 iov_iter_truncate(iter, 0);
2540 break;
2541 }
2542 } else {
2543 /*
2544 * Try rewind the iter structure.
2545 */
2546 bool const fRewindOkay = vbsf_iter_rewind(iter, &Stash, cbChunk, cbChunk);
2547 if (rc == VERR_NO_MEMORY && cMaxPages > 4 && fRewindOkay) {
2548 /*
2549 * The host probably doesn't have enough heap to handle the
2550 * request, reduce the page count and retry.
2551 */
2552 cMaxPages /= 4;
2553 Assert(cMaxPages > 0);
2554 } else {
2555 /*
2556 * If we've successfully read stuff, return it rather than
2557 * the error. (Not sure if this is such a great idea...)
2558 */
2559 if (cbRet <= 0)
2560 cbRet = -EPROTO;
2561 break;
2562 }
2563 }
2564 } while (cbToRead > 0);
2565
2566 vbsf_iter_cleanup_stash(iter, &Stash);
2567 }
2568 else
2569 cbRet = -ENOMEM;
2570 if (papPagesFree)
2571 kfree(papPages);
2572 if (pReq)
2573 VbglR0PhysHeapFree(pReq);
2574 SFLOGFLOW(("vbsf_reg_read_iter_locking: returns %#zx (%zd)\n", cbRet, cbRet));
2575 return cbRet;
2576}
2577
2578
2579/**
2580 * Read into I/O vector iterator.
2581 *
2582 * @returns Number of bytes read on success, negative errno on error.
2583 * @param kio The kernel I/O control block (or something like that).
2584 * @param iter The I/O vector iterator describing the buffer.
2585 */
2586# if RTLNX_VER_MIN(3,16,0)
2587static ssize_t vbsf_reg_read_iter(struct kiocb *kio, struct iov_iter *iter)
2588# else
2589static ssize_t vbsf_reg_aio_read(struct kiocb *kio, const struct iovec *iov, unsigned long cSegs, loff_t offFile)
2590# endif
2591{
2592# if RTLNX_VER_MAX(3,16,0)
2593 struct vbsf_iov_iter fake_iter = VBSF_IOV_ITER_INITIALIZER(cSegs, iov, 0 /*write*/);
2594 struct vbsf_iov_iter *iter = &fake_iter;
2595# endif
2596 size_t cbToRead = iov_iter_count(iter);
2597 struct inode *inode = VBSF_GET_F_DENTRY(kio->ki_filp)->d_inode;
2598 struct address_space *mapping = inode->i_mapping;
2599
2600 struct vbsf_reg_info *sf_r = kio->ki_filp->private_data;
2601 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
2602
2603 SFLOGFLOW(("vbsf_reg_read_iter: inode=%p file=%p size=%#zx off=%#llx type=%#x\n",
2604 inode, kio->ki_filp, cbToRead, kio->ki_pos, VBSF_GET_ITER_TYPE(iter) ));
2605 AssertReturn(S_ISREG(inode->i_mode), -EINVAL);
2606
2607 /*
2608 * Do we have anything at all to do here?
2609 */
2610 if (!cbToRead)
2611 return 0;
2612
2613 /*
2614 * If there is a mapping and O_DIRECT isn't in effect, we must at a
2615 * heed dirty pages in the mapping and read from them. For simplicity
2616 * though, we just do page cache reading when there are writable
2617 * mappings around with any kind of pages loaded.
2618 */
2619 if (vbsf_should_use_cached_read(kio->ki_filp, mapping, pSuperInfo)) {
2620# if RTLNX_VER_MIN(3,16,0)
2621 return generic_file_read_iter(kio, iter);
2622# else
2623 return generic_file_aio_read(kio, iov, cSegs, offFile);
2624# endif
2625 }
2626
2627 /*
2628 * Now now we reject async I/O requests.
2629 */
2630 if (!is_sync_kiocb(kio)) {
2631 SFLOGFLOW(("vbsf_reg_read_iter: async I/O not yet supported\n")); /** @todo extend FsPerf with AIO tests. */
2632 return -EOPNOTSUPP;
2633 }
2634
2635 /*
2636 * For small requests, try use an embedded buffer provided we get a heap block
2637 * that does not cross page boundraries (see host code).
2638 */
2639 if (cbToRead <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) /* see allocator */) {
2640 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFREADEMBEDDEDREQ, abData[0]) + cbToRead;
2641 VBOXSFREADEMBEDDEDREQ *pReq = (VBOXSFREADEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
2642 if (pReq) {
2643 if ((PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
2644 ssize_t cbRet;
2645 int vrc = VbglR0SfHostReqReadEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost,
2646 kio->ki_pos, (uint32_t)cbToRead);
2647 if (RT_SUCCESS(vrc)) {
2648 cbRet = pReq->Parms.cb32Read.u.value32;
2649 AssertStmt(cbRet <= (ssize_t)cbToRead, cbRet = cbToRead);
2650 if (copy_to_iter(pReq->abData, cbRet, iter) == cbRet) {
2651 kio->ki_pos += cbRet;
2652 if (cbRet < cbToRead)
2653 iov_iter_truncate(iter, 0);
2654 } else
2655 cbRet = -EFAULT;
2656 } else
2657 cbRet = -EPROTO;
2658 VbglR0PhysHeapFree(pReq);
2659 SFLOGFLOW(("vbsf_reg_read_iter: returns %#zx (%zd)\n", cbRet, cbRet));
2660 return cbRet;
2661 }
2662 VbglR0PhysHeapFree(pReq);
2663 }
2664 }
2665
2666 /*
2667 * Otherwise do the page locking thing.
2668 */
2669 return vbsf_reg_read_iter_locking(kio, iter, cbToRead, pSuperInfo, sf_r);
2670}
2671
2672
2673/**
2674 * Worker for vbsf_reg_write_iter() that deals with larger writes using page
2675 * locking.
2676 */
2677static ssize_t vbsf_reg_write_iter_locking(struct kiocb *kio, struct iov_iter *iter, size_t cbToWrite, loff_t offFile,
2678 struct vbsf_super_info *pSuperInfo, struct vbsf_reg_info *sf_r, struct inode *inode,
2679 struct vbsf_inode_info *sf_i, struct address_space *mapping, bool fAppend)
2680{
2681 /*
2682 * Estimate how many pages we may possible submit in a single request so
2683 * that we can allocate matching request buffer and page array.
2684 */
2685 struct page *apPagesStack[16];
2686 struct page **papPages = &apPagesStack[0];
2687 struct page **papPagesFree = NULL;
2688 VBOXSFWRITEPGLSTREQ *pReq;
2689 ssize_t cbRet = 0;
2690 size_t cMaxPages = vbsf_iter_max_span_of_pages(iter);
2691 cMaxPages = RT_MIN(RT_MAX(pSuperInfo->cMaxIoPages, 2), cMaxPages);
2692
2693 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
2694 while (!pReq && cMaxPages > 4) {
2695 cMaxPages /= 2;
2696 pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(RT_UOFFSETOF_DYN(VBOXSFWRITEPGLSTREQ, PgLst.aPages[cMaxPages]));
2697 }
2698 if (pReq && cMaxPages > RT_ELEMENTS(apPagesStack))
2699 papPagesFree = papPages = kmalloc(cMaxPages * sizeof(sizeof(papPages[0])), GFP_KERNEL);
2700 if (pReq && papPages) {
2701
2702 /*
2703 * The write loop.
2704 */
2705 struct vbsf_iter_stash Stash = VBSF_ITER_STASH_INITIALIZER;
2706 do {
2707 /*
2708 * Grab as many pages as we can. This means that if adjacent
2709 * segments both starts and ends at a page boundrary, we can
2710 * do them both in the same transfer from the host.
2711 */
2712 size_t cPages = 0;
2713 size_t cbChunk = 0;
2714 size_t offPage0 = 0;
2715 int rc = vbsf_iter_lock_pages(iter, false /*fWrite*/, &Stash, cMaxPages, papPages, &cPages, &offPage0, &cbChunk);
2716 if (rc == 0) {
2717 size_t iPage = cPages;
2718 while (iPage-- > 0)
2719 pReq->PgLst.aPages[iPage] = page_to_phys(papPages[iPage]);
2720 pReq->PgLst.offFirstPage = (uint16_t)offPage0;
2721 AssertStmt(cbChunk <= cbToWrite, cbChunk = cbToWrite);
2722 } else {
2723 cbRet = rc;
2724 break;
2725 }
2726
2727 /*
2728 * Issue the request and unlock the pages.
2729 */
2730 rc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root, pReq, sf_r->Handle.hHost, offFile, cbChunk, cPages);
2731 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2732 SFLOGFLOW(("vbsf_reg_write_iter_locking: VbglR0SfHostReqWritePgLst -> %d (cbActual=%#x cbChunk=%#zx of %#zx cPages=%#zx offPage0=%#x\n",
2733 rc, pReq->Parms.cb32Write.u.value32, cbChunk, cbToWrite, cPages, offPage0));
2734 if (RT_SUCCESS(rc)) {
2735 /*
2736 * Success, advance position and buffer.
2737 */
2738 uint32_t cbActual = pReq->Parms.cb32Write.u.value32;
2739 AssertStmt(cbActual <= cbChunk, cbActual = cbChunk);
2740
2741 vbsf_reg_write_sync_page_cache(mapping, offFile, cbActual, NULL /*pbSrcBuf*/, papPages, offPage0, cPages);
2742 vbsf_iter_unlock_pages(iter, papPages, cPages, false /*fSetDirty*/);
2743
2744 cbRet += cbActual;
2745 cbToWrite -= cbActual;
2746
2747 offFile += cbActual;
2748 if (fAppend && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
2749 offFile = pReq->Parms.off64Write.u.value64;
2750 kio->ki_pos = offFile;
2751 if (offFile > i_size_read(inode))
2752 i_size_write(inode, offFile);
2753
2754 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2755
2756 /*
2757 * Are we done already?
2758 */
2759 if (!cbToWrite)
2760 break;
2761 if (cbActual < cbChunk) { /* We ASSUME end-of-file here. */
2762 if (vbsf_iter_rewind(iter, &Stash, cbChunk - cbActual, cbActual))
2763 iov_iter_truncate(iter, 0);
2764 break;
2765 }
2766 } else {
2767 /*
2768 * Try rewind the iter structure.
2769 */
2770 bool fRewindOkay;
2771 vbsf_iter_unlock_pages(iter, papPages, cPages, false /*fSetDirty*/);
2772 fRewindOkay = vbsf_iter_rewind(iter, &Stash, cbChunk, cbChunk);
2773 if (rc == VERR_NO_MEMORY && cMaxPages > 4 && fRewindOkay) {
2774 /*
2775 * The host probably doesn't have enough heap to handle the
2776 * request, reduce the page count and retry.
2777 */
2778 cMaxPages /= 4;
2779 Assert(cMaxPages > 0);
2780 } else {
2781 /*
2782 * If we've successfully written stuff, return it rather than
2783 * the error. (Not sure if this is such a great idea...)
2784 */
2785 if (cbRet <= 0)
2786 cbRet = -EPROTO;
2787 break;
2788 }
2789 }
2790 } while (cbToWrite > 0);
2791
2792 vbsf_iter_cleanup_stash(iter, &Stash);
2793 }
2794 else
2795 cbRet = -ENOMEM;
2796 if (papPagesFree)
2797 kfree(papPages);
2798 if (pReq)
2799 VbglR0PhysHeapFree(pReq);
2800 SFLOGFLOW(("vbsf_reg_write_iter_locking: returns %#zx (%zd)\n", cbRet, cbRet));
2801 return cbRet;
2802}
2803
2804
2805/**
2806 * Write from I/O vector iterator.
2807 *
2808 * @returns Number of bytes written on success, negative errno on error.
2809 * @param kio The kernel I/O control block (or something like that).
2810 * @param iter The I/O vector iterator describing the buffer.
2811 */
2812# if RTLNX_VER_MIN(3,16,0)
2813static ssize_t vbsf_reg_write_iter(struct kiocb *kio, struct iov_iter *iter)
2814# else
2815static ssize_t vbsf_reg_aio_write(struct kiocb *kio, const struct iovec *iov, unsigned long cSegs, loff_t offFile)
2816# endif
2817{
2818# if RTLNX_VER_MAX(3,16,0)
2819 struct vbsf_iov_iter fake_iter = VBSF_IOV_ITER_INITIALIZER(cSegs, iov, 1 /*write*/);
2820 struct vbsf_iov_iter *iter = &fake_iter;
2821# endif
2822 size_t cbToWrite = iov_iter_count(iter);
2823 struct inode *inode = VBSF_GET_F_DENTRY(kio->ki_filp)->d_inode;
2824 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
2825 struct address_space *mapping = inode->i_mapping;
2826
2827 struct vbsf_reg_info *sf_r = kio->ki_filp->private_data;
2828 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
2829# if RTLNX_VER_MIN(3,16,0)
2830 loff_t offFile = kio->ki_pos;
2831# endif
2832# if RTLNX_VER_MIN(4,1,0)
2833 bool const fAppend = RT_BOOL(kio->ki_flags & IOCB_APPEND);
2834# else
2835 bool const fAppend = RT_BOOL(kio->ki_filp->f_flags & O_APPEND);
2836# endif
2837
2838
2839 SFLOGFLOW(("vbsf_reg_write_iter: inode=%p file=%p size=%#zx off=%#llx type=%#x\n",
2840 inode, kio->ki_filp, cbToWrite, offFile, VBSF_GET_ITER_TYPE(iter) ));
2841 AssertReturn(S_ISREG(inode->i_mode), -EINVAL);
2842
2843 /*
2844 * Enforce APPEND flag (more later).
2845 */
2846 if (fAppend)
2847 kio->ki_pos = offFile = i_size_read(inode);
2848
2849 /*
2850 * Do we have anything at all to do here?
2851 */
2852 if (!cbToWrite)
2853 return 0;
2854
2855 /** @todo Implement the read-write caching mode. */
2856
2857 /*
2858 * Now now we reject async I/O requests.
2859 */
2860 if (!is_sync_kiocb(kio)) {
2861 SFLOGFLOW(("vbsf_reg_write_iter: async I/O not yet supported\n")); /** @todo extend FsPerf with AIO tests. */
2862 return -EOPNOTSUPP;
2863 }
2864
2865 /*
2866 * If there are active writable mappings, coordinate with any
2867 * pending writes via those.
2868 */
2869 if ( mapping
2870 && mapping->nrpages > 0
2871 && mapping_writably_mapped(mapping)) {
2872# if RTLNX_VER_MIN(2,6,32)
2873 int err = filemap_fdatawait_range(mapping, offFile, offFile + cbToWrite - 1);
2874 if (err)
2875 return err;
2876# else
2877 /** @todo ... */
2878# endif
2879 }
2880
2881 /*
2882 * For small requests, try use an embedded buffer provided we get a heap block
2883 * that does not cross page boundraries (see host code).
2884 */
2885 if (cbToWrite <= PAGE_SIZE / 4 * 3 - RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) /* see allocator */) {
2886 uint32_t const cbReq = RT_UOFFSETOF(VBOXSFWRITEEMBEDDEDREQ, abData[0]) + cbToWrite;
2887 VBOXSFWRITEEMBEDDEDREQ *pReq = (VBOXSFWRITEEMBEDDEDREQ *)VbglR0PhysHeapAlloc(cbReq);
2888 if (pReq) {
2889 if ((PAGE_SIZE - ((uintptr_t)pReq & PAGE_OFFSET_MASK)) >= cbReq) {
2890 ssize_t cbRet;
2891 if (copy_from_iter(pReq->abData, cbToWrite, iter) == cbToWrite) {
2892 int vrc = VbglR0SfHostReqWriteEmbedded(pSuperInfo->map.root, pReq, sf_r->Handle.hHost,
2893 offFile, (uint32_t)cbToWrite);
2894 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
2895 if (RT_SUCCESS(vrc)) {
2896 cbRet = pReq->Parms.cb32Write.u.value32;
2897 AssertStmt(cbRet <= (ssize_t)cbToWrite, cbRet = cbToWrite);
2898 vbsf_reg_write_sync_page_cache(mapping, offFile, (uint32_t)cbRet, pReq->abData,
2899 NULL /*papSrcPages*/, 0 /*offSrcPage0*/, 0 /*cSrcPages*/);
2900
2901 offFile += cbRet;
2902 if (fAppend && (g_fSfFeatures & SHFL_FEATURE_WRITE_UPDATES_OFFSET))
2903 offFile = pReq->Parms.off64Write.u.value64;
2904 kio->ki_pos = offFile;
2905 if (offFile > i_size_read(inode))
2906 i_size_write(inode, offFile);
2907
2908# if RTLNX_VER_MIN(4,11,0)
2909 if ((size_t)cbRet < cbToWrite)
2910 iov_iter_revert(iter, cbToWrite - cbRet);
2911# endif
2912 } else
2913 cbRet = -EPROTO;
2914 sf_i->force_restat = 1; /* mtime (and size) may have changed */
2915 } else
2916 cbRet = -EFAULT;
2917 VbglR0PhysHeapFree(pReq);
2918 SFLOGFLOW(("vbsf_reg_write_iter: returns %#zx (%zd)\n", cbRet, cbRet));
2919 return cbRet;
2920 }
2921 VbglR0PhysHeapFree(pReq);
2922 }
2923 }
2924
2925 /*
2926 * Otherwise do the page locking thing.
2927 */
2928 return vbsf_reg_write_iter_locking(kio, iter, cbToWrite, offFile, pSuperInfo, sf_r, inode, sf_i, mapping, fAppend);
2929}
2930
2931#endif /* >= 2.6.19 */
2932
2933/**
2934 * Used by vbsf_reg_open() and vbsf_inode_atomic_open() to
2935 *
2936 * @returns shared folders create flags.
2937 * @param fLnxOpen The linux O_XXX flags to convert.
2938 * @param pfHandle Pointer to vbsf_handle::fFlags.
2939 * @param pszCaller Caller, for logging purposes.
2940 */
2941uint32_t vbsf_linux_oflags_to_vbox(unsigned fLnxOpen, uint32_t *pfHandle, const char *pszCaller)
2942{
2943 uint32_t fVBoxFlags = SHFL_CF_ACCESS_DENYNONE;
2944
2945 /*
2946 * Disposition.
2947 */
2948 if (fLnxOpen & O_CREAT) {
2949 Log(("%s: O_CREAT set\n", pszCaller));
2950 fVBoxFlags |= SHFL_CF_ACT_CREATE_IF_NEW;
2951 if (fLnxOpen & O_EXCL) {
2952 Log(("%s: O_EXCL set\n", pszCaller));
2953 fVBoxFlags |= SHFL_CF_ACT_FAIL_IF_EXISTS;
2954 } else if (fLnxOpen & O_TRUNC) {
2955 Log(("%s: O_TRUNC set\n", pszCaller));
2956 fVBoxFlags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS;
2957 } else
2958 fVBoxFlags |= SHFL_CF_ACT_OPEN_IF_EXISTS;
2959 } else {
2960 fVBoxFlags |= SHFL_CF_ACT_FAIL_IF_NEW;
2961 if (fLnxOpen & O_TRUNC) {
2962 Log(("%s: O_TRUNC set\n", pszCaller));
2963 fVBoxFlags |= SHFL_CF_ACT_OVERWRITE_IF_EXISTS;
2964 }
2965 }
2966
2967 /*
2968 * Access.
2969 */
2970 switch (fLnxOpen & O_ACCMODE) {
2971 case O_RDONLY:
2972 fVBoxFlags |= SHFL_CF_ACCESS_READ;
2973 *pfHandle |= VBSF_HANDLE_F_READ;
2974 break;
2975
2976 case O_WRONLY:
2977 fVBoxFlags |= SHFL_CF_ACCESS_WRITE;
2978 *pfHandle |= VBSF_HANDLE_F_WRITE;
2979 break;
2980
2981 case O_RDWR:
2982 fVBoxFlags |= SHFL_CF_ACCESS_READWRITE;
2983 *pfHandle |= VBSF_HANDLE_F_READ | VBSF_HANDLE_F_WRITE;
2984 break;
2985
2986 default:
2987 BUG();
2988 }
2989
2990 if (fLnxOpen & O_APPEND) {
2991 Log(("%s: O_APPEND set\n", pszCaller));
2992 fVBoxFlags |= SHFL_CF_ACCESS_APPEND;
2993 *pfHandle |= VBSF_HANDLE_F_APPEND;
2994 }
2995
2996 /*
2997 * Only directories?
2998 */
2999 if (fLnxOpen & O_DIRECTORY) {
3000 Log(("%s: O_DIRECTORY set\n", pszCaller));
3001 fVBoxFlags |= SHFL_CF_DIRECTORY;
3002 }
3003
3004 return fVBoxFlags;
3005}
3006
3007
3008/**
3009 * Open a regular file.
3010 *
3011 * @param inode the inode
3012 * @param file the file
3013 * @returns 0 on success, Linux error code otherwise
3014 */
3015static int vbsf_reg_open(struct inode *inode, struct file *file)
3016{
3017 int rc, rc_linux = 0;
3018 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3019 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
3020 struct dentry *dentry = VBSF_GET_F_DENTRY(file);
3021 struct vbsf_reg_info *sf_r;
3022 VBOXSFCREATEREQ *pReq;
3023
3024 SFLOGFLOW(("vbsf_reg_open: inode=%p file=%p flags=%#x %s\n", inode, file, file->f_flags, sf_i ? sf_i->path->String.ach : NULL));
3025 Assert(pSuperInfo);
3026 Assert(sf_i);
3027
3028 sf_r = kmalloc(sizeof(*sf_r), GFP_KERNEL);
3029 if (!sf_r) {
3030 LogRelFunc(("could not allocate reg info\n"));
3031 return -ENOMEM;
3032 }
3033
3034 RTListInit(&sf_r->Handle.Entry);
3035 sf_r->Handle.cRefs = 1;
3036 sf_r->Handle.fFlags = VBSF_HANDLE_F_FILE | VBSF_HANDLE_F_MAGIC;
3037 sf_r->Handle.hHost = SHFL_HANDLE_NIL;
3038
3039 /* Already open? */
3040 if (sf_i->handle != SHFL_HANDLE_NIL) {
3041 /*
3042 * This inode was created with vbsf_create_worker(). Check the CreateFlags:
3043 * O_CREAT, O_TRUNC: inherent true (file was just created). Not sure
3044 * about the access flags (SHFL_CF_ACCESS_*).
3045 */
3046 sf_i->force_restat = 1;
3047 sf_r->Handle.hHost = sf_i->handle;
3048 sf_i->handle = SHFL_HANDLE_NIL;
3049 file->private_data = sf_r;
3050
3051 sf_r->Handle.fFlags |= VBSF_HANDLE_F_READ | VBSF_HANDLE_F_WRITE; /** @todo fix */
3052 vbsf_handle_append(sf_i, &sf_r->Handle);
3053 SFLOGFLOW(("vbsf_reg_open: returns 0 (#1) - sf_i=%p hHost=%#llx\n", sf_i, sf_r->Handle.hHost));
3054 return 0;
3055 }
3056
3057 pReq = (VBOXSFCREATEREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq) + sf_i->path->u16Size);
3058 if (!pReq) {
3059 kfree(sf_r);
3060 LogRelFunc(("Failed to allocate a VBOXSFCREATEREQ buffer!\n"));
3061 return -ENOMEM;
3062 }
3063 memcpy(&pReq->StrPath, sf_i->path, SHFLSTRING_HEADER_SIZE + sf_i->path->u16Size);
3064 RT_ZERO(pReq->CreateParms);
3065 pReq->CreateParms.Handle = SHFL_HANDLE_NIL;
3066
3067 /* We check the value of pReq->CreateParms.Handle afterwards to
3068 * find out if the call succeeded or failed, as the API does not seem
3069 * to cleanly distinguish error and informational messages.
3070 *
3071 * Furthermore, we must set pReq->CreateParms.Handle to SHFL_HANDLE_NIL
3072 * to make the shared folders host service use our fMode parameter */
3073
3074 /* We ignore O_EXCL, as the Linux kernel seems to call create
3075 beforehand itself, so O_EXCL should always fail. */
3076 pReq->CreateParms.CreateFlags = vbsf_linux_oflags_to_vbox(file->f_flags & ~O_EXCL, &sf_r->Handle.fFlags, __FUNCTION__);
3077 pReq->CreateParms.Info.Attr.fMode = inode->i_mode;
3078 LogFunc(("vbsf_reg_open: calling VbglR0SfHostReqCreate, file %s, flags=%#x, %#x\n",
3079 sf_i->path->String.utf8, file->f_flags, pReq->CreateParms.CreateFlags));
3080 rc = VbglR0SfHostReqCreate(pSuperInfo->map.root, pReq);
3081 if (RT_FAILURE(rc)) {
3082 LogFunc(("VbglR0SfHostReqCreate failed flags=%d,%#x rc=%Rrc\n", file->f_flags, pReq->CreateParms.CreateFlags, rc));
3083 kfree(sf_r);
3084 VbglR0PhysHeapFree(pReq);
3085 return -RTErrConvertToErrno(rc);
3086 }
3087
3088 if (pReq->CreateParms.Handle != SHFL_HANDLE_NIL) {
3089 vbsf_dentry_chain_increase_ttl(dentry);
3090 vbsf_update_inode(inode, sf_i, &pReq->CreateParms.Info, pSuperInfo, false /*fInodeLocked*/, 0 /*fSetAttrs*/);
3091 rc_linux = 0;
3092 } else {
3093 switch (pReq->CreateParms.Result) {
3094 case SHFL_PATH_NOT_FOUND:
3095 vbsf_dentry_invalidate_ttl(dentry);
3096 rc_linux = -ENOENT;
3097 break;
3098 case SHFL_FILE_NOT_FOUND:
3099 vbsf_dentry_invalidate_ttl(dentry);
3100 /** @todo sf_dentry_increase_parent_ttl(file->f_dentry); if we can trust it. */
3101 rc_linux = -ENOENT;
3102 break;
3103 case SHFL_FILE_EXISTS:
3104 vbsf_dentry_chain_increase_ttl(dentry);
3105 vbsf_update_inode(inode, sf_i, &pReq->CreateParms.Info, pSuperInfo, false /*fInodeLocked*/, 0 /*fSetAttrs*/);
3106 rc_linux = -EEXIST;
3107 break;
3108 default:
3109 vbsf_dentry_chain_increase_parent_ttl(dentry);
3110 rc_linux = 0;
3111 break;
3112 }
3113 }
3114
3115 sf_r->Handle.hHost = pReq->CreateParms.Handle;
3116 file->private_data = sf_r;
3117 vbsf_handle_append(sf_i, &sf_r->Handle);
3118 VbglR0PhysHeapFree(pReq);
3119 SFLOGFLOW(("vbsf_reg_open: returns 0 (#2) - sf_i=%p hHost=%#llx\n", sf_i, sf_r->Handle.hHost));
3120 return rc_linux;
3121}
3122
3123
3124/**
3125 * Close a regular file.
3126 *
3127 * @param inode the inode
3128 * @param file the file
3129 * @returns 0 on success, Linux error code otherwise
3130 */
3131static int vbsf_reg_release(struct inode *inode, struct file *file)
3132{
3133 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
3134 struct vbsf_reg_info *sf_r = file->private_data;
3135
3136 SFLOGFLOW(("vbsf_reg_release: inode=%p file=%p\n", inode, file));
3137 if (sf_r) {
3138 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3139 struct address_space *mapping = inode->i_mapping;
3140 Assert(pSuperInfo);
3141
3142 /* If we're closing the last handle for this inode, make sure the flush
3143 the mapping or we'll end up in vbsf_writepage without a handle. */
3144 if ( mapping
3145 && mapping->nrpages > 0
3146 /** @todo && last writable handle */ ) {
3147#if RTLNX_VER_MIN(2,4,25)
3148 if (filemap_fdatawrite(mapping) != -EIO)
3149#else
3150 if ( filemap_fdatasync(mapping) == 0
3151 && fsync_inode_data_buffers(inode) == 0)
3152#endif
3153 filemap_fdatawait(inode->i_mapping);
3154 }
3155
3156 /* Release sf_r, closing the handle if we're the last user. */
3157 file->private_data = NULL;
3158 vbsf_handle_release(&sf_r->Handle, pSuperInfo, "vbsf_reg_release");
3159
3160 sf_i->handle = SHFL_HANDLE_NIL;
3161 }
3162 return 0;
3163}
3164
3165
3166/**
3167 * Wrapper around generic/default seek function that ensures that we've got
3168 * the up-to-date file size when doing anything relative to EOF.
3169 *
3170 * The issue is that the host may extend the file while we weren't looking and
3171 * if the caller wishes to append data, it may end up overwriting existing data
3172 * if we operate with a stale size. So, we always retrieve the file size on EOF
3173 * relative seeks.
3174 */
3175static loff_t vbsf_reg_llseek(struct file *file, loff_t off, int whence)
3176{
3177 SFLOGFLOW(("vbsf_reg_llseek: file=%p off=%lld whence=%d\n", file, off, whence));
3178
3179 switch (whence) {
3180#ifdef SEEK_HOLE
3181 case SEEK_HOLE:
3182 case SEEK_DATA:
3183#endif
3184 case SEEK_END: {
3185 struct vbsf_reg_info *sf_r = file->private_data;
3186 int rc = vbsf_inode_revalidate_with_handle(VBSF_GET_F_DENTRY(file), sf_r->Handle.hHost,
3187 true /*fForce*/, false /*fInodeLocked*/);
3188 if (rc == 0)
3189 break;
3190 return rc;
3191 }
3192 }
3193
3194#if RTLNX_VER_MIN(2,4,8)
3195 return generic_file_llseek(file, off, whence);
3196#else
3197 return default_llseek(file, off, whence);
3198#endif
3199}
3200
3201
3202/**
3203 * Flush region of file - chiefly mmap/msync.
3204 *
3205 * We cannot use the noop_fsync / simple_sync_file here as that means
3206 * msync(,,MS_SYNC) will return before the data hits the host, thereby
3207 * causing coherency issues with O_DIRECT access to the same file as
3208 * well as any host interaction with the file.
3209 */
3210#if RTLNX_VER_MIN(3,1,0) \
3211 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_MIN(3,0,101) /** @todo figure when exactly */)
3212static int vbsf_reg_fsync(struct file *file, loff_t start, loff_t end, int datasync)
3213{
3214# if RTLNX_VER_MIN(3,16,0)
3215 return __generic_file_fsync(file, start, end, datasync);
3216# else
3217 return generic_file_fsync(file, start, end, datasync);
3218# endif
3219}
3220#elif RTLNX_VER_MIN(2,6,35)
3221static int vbsf_reg_fsync(struct file *file, int datasync)
3222{
3223 return generic_file_fsync(file, datasync);
3224}
3225#else /* < 2.6.35 */
3226static int vbsf_reg_fsync(struct file *file, struct dentry *dentry, int datasync)
3227{
3228# if RTLNX_VER_MIN(2,6,31)
3229 return simple_fsync(file, dentry, datasync);
3230# else
3231 int rc;
3232 struct inode *inode = dentry->d_inode;
3233 AssertReturn(inode, -EINVAL);
3234
3235 /** @todo What about file_fsync()? (<= 2.5.11) */
3236
3237# if RTLNX_VER_MIN(2,5,12)
3238 rc = sync_mapping_buffers(inode->i_mapping);
3239 if ( rc == 0
3240 && (inode->i_state & I_DIRTY)
3241 && ((inode->i_state & I_DIRTY_DATASYNC) || !datasync)
3242 ) {
3243 struct writeback_control wbc = {
3244 .sync_mode = WB_SYNC_ALL,
3245 .nr_to_write = 0
3246 };
3247 rc = sync_inode(inode, &wbc);
3248 }
3249# else /* < 2.5.12 */
3250 /** @todo
3251 * Somethings is buggy here or in the 2.4.21-27.EL kernel I'm testing on.
3252 *
3253 * In theory we shouldn't need to do anything here, since msync will call
3254 * writepage() on each dirty page and we write them out synchronously. So, the
3255 * problem is elsewhere... Doesn't happen all the time either. Sigh.
3256 */
3257 rc = fsync_inode_buffers(inode);
3258# if RTLNX_VER_MIN(2,4,10)
3259 if (rc == 0 && datasync)
3260 rc = fsync_inode_data_buffers(inode);
3261# endif
3262
3263# endif /* < 2.5.12 */
3264 return rc;
3265# endif
3266}
3267#endif /* < 2.6.35 */
3268
3269
3270#if RTLNX_VER_MIN(4,5,0)
3271/**
3272 * Copy a datablock from one file to another on the host side.
3273 */
3274static ssize_t vbsf_reg_copy_file_range(struct file *pFileSrc, loff_t offSrc, struct file *pFileDst, loff_t offDst,
3275 size_t cbRange, unsigned int fFlags)
3276{
3277 ssize_t cbRet;
3278 if (g_uSfLastFunction >= SHFL_FN_COPY_FILE_PART) {
3279 struct inode *pInodeSrc = pFileSrc->f_inode;
3280 struct vbsf_inode_info *pInodeInfoSrc = VBSF_GET_INODE_INFO(pInodeSrc);
3281 struct vbsf_super_info *pSuperInfoSrc = VBSF_GET_SUPER_INFO(pInodeSrc->i_sb);
3282 struct vbsf_reg_info *pFileInfoSrc = (struct vbsf_reg_info *)pFileSrc->private_data;
3283 struct inode *pInodeDst = pInodeSrc;
3284 struct vbsf_inode_info *pInodeInfoDst = VBSF_GET_INODE_INFO(pInodeDst);
3285 struct vbsf_super_info *pSuperInfoDst = VBSF_GET_SUPER_INFO(pInodeDst->i_sb);
3286 struct vbsf_reg_info *pFileInfoDst = (struct vbsf_reg_info *)pFileDst->private_data;
3287 VBOXSFCOPYFILEPARTREQ *pReq;
3288
3289 /*
3290 * Some extra validation.
3291 */
3292 AssertPtrReturn(pInodeInfoSrc, -EOPNOTSUPP);
3293 Assert(pInodeInfoSrc->u32Magic == SF_INODE_INFO_MAGIC);
3294 AssertPtrReturn(pInodeInfoDst, -EOPNOTSUPP);
3295 Assert(pInodeInfoDst->u32Magic == SF_INODE_INFO_MAGIC);
3296
3297# if RTLNX_VER_MAX(4,11,0)
3298 if (!S_ISREG(pInodeSrc->i_mode) || !S_ISREG(pInodeDst->i_mode))
3299 return S_ISDIR(pInodeSrc->i_mode) || S_ISDIR(pInodeDst->i_mode) ? -EISDIR : -EINVAL;
3300# endif
3301
3302 /*
3303 * Allocate the request and issue it.
3304 */
3305 pReq = (VBOXSFCOPYFILEPARTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
3306 if (pReq) {
3307 int vrc = VbglR0SfHostReqCopyFilePart(pSuperInfoSrc->map.root, pFileInfoSrc->Handle.hHost, offSrc,
3308 pSuperInfoDst->map.root, pFileInfoDst->Handle.hHost, offDst,
3309 cbRange, 0 /*fFlags*/, pReq);
3310 if (RT_SUCCESS(vrc))
3311 cbRet = pReq->Parms.cb64ToCopy.u.value64;
3312 else if (vrc == VERR_NOT_IMPLEMENTED)
3313 cbRet = -EOPNOTSUPP;
3314 else
3315 cbRet = -RTErrConvertToErrno(vrc);
3316
3317 VbglR0PhysHeapFree(pReq);
3318 } else
3319 cbRet = -ENOMEM;
3320 } else {
3321 cbRet = -EOPNOTSUPP;
3322 }
3323 SFLOGFLOW(("vbsf_reg_copy_file_range: returns %zd\n", cbRet));
3324 return cbRet;
3325}
3326#endif /* > 4.5 */
3327
3328
3329#ifdef SFLOG_ENABLED
3330/*
3331 * This is just for logging page faults and such.
3332 */
3333
3334/** Pointer to the ops generic_file_mmap returns the first time it's called. */
3335static struct vm_operations_struct const *g_pGenericFileVmOps = NULL;
3336/** Merge of g_LoggingVmOpsTemplate and g_pGenericFileVmOps. */
3337static struct vm_operations_struct g_LoggingVmOps;
3338
3339
3340/* Generic page fault callback: */
3341# if RTLNX_VER_MIN(4,11,0)
3342static vm_fault_t vbsf_vmlog_fault(struct vm_fault *vmf)
3343{
3344 vm_fault_t rc;
3345 SFLOGFLOW(("vbsf_vmlog_fault: vmf=%p flags=%#x addr=%p\n", vmf, vmf->flags, vmf->address));
3346 rc = g_pGenericFileVmOps->fault(vmf);
3347 SFLOGFLOW(("vbsf_vmlog_fault: returns %d\n", rc));
3348 return rc;
3349}
3350# elif RTLNX_VER_MIN(2,6,23)
3351static int vbsf_vmlog_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
3352{
3353 int rc;
3354# if RTLNX_VER_MIN(4,10,0)
3355 SFLOGFLOW(("vbsf_vmlog_fault: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->address));
3356# else
3357 SFLOGFLOW(("vbsf_vmlog_fault: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->virtual_address));
3358# endif
3359 rc = g_pGenericFileVmOps->fault(vma, vmf);
3360 SFLOGFLOW(("vbsf_vmlog_fault: returns %d\n", rc));
3361 return rc;
3362}
3363# endif
3364
3365
3366/* Special/generic page fault handler: */
3367# if RTLNX_VER_MIN(2,6,26)
3368# elif RTLNX_VER_MIN(2,6,1)
3369static struct page *vbsf_vmlog_nopage(struct vm_area_struct *vma, unsigned long address, int *type)
3370{
3371 struct page *page;
3372 SFLOGFLOW(("vbsf_vmlog_nopage: vma=%p address=%p type=%p:{%#x}\n", vma, address, type, type ? *type : 0));
3373 page = g_pGenericFileVmOps->nopage(vma, address, type);
3374 SFLOGFLOW(("vbsf_vmlog_nopage: returns %p\n", page));
3375 return page;
3376}
3377# else
3378static struct page *vbsf_vmlog_nopage(struct vm_area_struct *vma, unsigned long address, int write_access_or_unused)
3379{
3380 struct page *page;
3381 SFLOGFLOW(("vbsf_vmlog_nopage: vma=%p address=%p wau=%d\n", vma, address, write_access_or_unused));
3382 page = g_pGenericFileVmOps->nopage(vma, address, write_access_or_unused);
3383 SFLOGFLOW(("vbsf_vmlog_nopage: returns %p\n", page));
3384 return page;
3385}
3386# endif /* < 2.6.26 */
3387
3388
3389/* Special page fault callback for making something writable: */
3390# if RTLNX_VER_MIN(4,11,0)
3391static vm_fault_t vbsf_vmlog_page_mkwrite(struct vm_fault *vmf)
3392{
3393 vm_fault_t rc;
3394 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vmf=%p flags=%#x addr=%p\n", vmf, vmf->flags, vmf->address));
3395 rc = g_pGenericFileVmOps->page_mkwrite(vmf);
3396 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: returns %d\n", rc));
3397 return rc;
3398}
3399# elif RTLNX_VER_MIN(2,6,30)
3400static int vbsf_vmlog_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
3401{
3402 int rc;
3403# if RTLNX_VER_MIN(4,10,0)
3404 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->address));
3405# else
3406 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vma=%p vmf=%p flags=%#x addr=%p\n", vma, vmf, vmf->flags, vmf->virtual_address));
3407# endif
3408 rc = g_pGenericFileVmOps->page_mkwrite(vma, vmf);
3409 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: returns %d\n", rc));
3410 return rc;
3411}
3412# elif RTLNX_VER_MIN(2,6,18)
3413static int vbsf_vmlog_page_mkwrite(struct vm_area_struct *vma, struct page *page)
3414{
3415 int rc;
3416 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: vma=%p page=%p\n", vma, page));
3417 rc = g_pGenericFileVmOps->page_mkwrite(vma, page);
3418 SFLOGFLOW(("vbsf_vmlog_page_mkwrite: returns %d\n", rc));
3419 return rc;
3420}
3421# endif
3422
3423
3424/* Special page fault callback for mapping pages: */
3425# if RTLNX_VER_MIN(5,12,0)
3426static vm_fault_t vbsf_vmlog_map_pages(struct vm_fault *vmf, pgoff_t start, pgoff_t end)
3427{
3428 vm_fault_t rc;
3429 SFLOGFLOW(("vbsf_vmlog_map_pages: vmf=%p (flags=%#x addr=%p) start=%p end=%p\n", vmf, vmf->flags, vmf->address, start, end));
3430 rc = g_pGenericFileVmOps->map_pages(vmf, start, end);
3431 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3432 return rc;
3433}
3434# elif RTLNX_VER_MIN(4,10,0)
3435static void vbsf_vmlog_map_pages(struct vm_fault *vmf, pgoff_t start, pgoff_t end)
3436{
3437 SFLOGFLOW(("vbsf_vmlog_map_pages: vmf=%p (flags=%#x addr=%p) start=%p end=%p\n", vmf, vmf->flags, vmf->address, start, end));
3438 g_pGenericFileVmOps->map_pages(vmf, start, end);
3439 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3440}
3441# elif RTLNX_VER_MIN(4,8,0)
3442static void vbsf_vmlog_map_pages(struct fault_env *fenv, pgoff_t start, pgoff_t end)
3443{
3444 SFLOGFLOW(("vbsf_vmlog_map_pages: fenv=%p (flags=%#x addr=%p) start=%p end=%p\n", fenv, fenv->flags, fenv->address, start, end));
3445 g_pGenericFileVmOps->map_pages(fenv, start, end);
3446 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3447}
3448# elif RTLNX_VER_MIN(3,15,0)
3449static void vbsf_vmlog_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf)
3450{
3451 SFLOGFLOW(("vbsf_vmlog_map_pages: vma=%p vmf=%p (flags=%#x addr=%p)\n", vma, vmf, vmf->flags, vmf->virtual_address));
3452 g_pGenericFileVmOps->map_pages(vma, vmf);
3453 SFLOGFLOW(("vbsf_vmlog_map_pages: returns\n"));
3454}
3455# endif
3456
3457
3458/** Overload template. */
3459static struct vm_operations_struct const g_LoggingVmOpsTemplate = {
3460# if RTLNX_VER_MIN(2,6,23)
3461 .fault = vbsf_vmlog_fault,
3462# endif
3463# if RTLNX_VER_MAX(2,6,26)
3464 .nopage = vbsf_vmlog_nopage,
3465# endif
3466# if RTLNX_VER_MIN(2,6,18)
3467 .page_mkwrite = vbsf_vmlog_page_mkwrite,
3468# endif
3469# if RTLNX_VER_MIN(3,15,0)
3470 .map_pages = vbsf_vmlog_map_pages,
3471# endif
3472};
3473
3474/** file_operations::mmap wrapper for logging purposes. */
3475extern int vbsf_reg_mmap(struct file *file, struct vm_area_struct *vma)
3476{
3477 int rc;
3478 SFLOGFLOW(("vbsf_reg_mmap: file=%p vma=%p\n", file, vma));
3479 rc = generic_file_mmap(file, vma);
3480 if (rc == 0) {
3481 /* Merge the ops and template the first time thru (there's a race here). */
3482 if (g_pGenericFileVmOps == NULL) {
3483 uintptr_t const *puSrc1 = (uintptr_t *)vma->vm_ops;
3484 uintptr_t const *puSrc2 = (uintptr_t *)&g_LoggingVmOpsTemplate;
3485 uintptr_t volatile *puDst = (uintptr_t *)&g_LoggingVmOps;
3486 size_t cbLeft = sizeof(g_LoggingVmOps) / sizeof(*puDst);
3487 while (cbLeft-- > 0) {
3488 *puDst = *puSrc2 && *puSrc1 ? *puSrc2 : *puSrc1;
3489 puSrc1++;
3490 puSrc2++;
3491 puDst++;
3492 }
3493 g_pGenericFileVmOps = vma->vm_ops;
3494 vma->vm_ops = &g_LoggingVmOps;
3495 } else if (g_pGenericFileVmOps == vma->vm_ops)
3496 vma->vm_ops = &g_LoggingVmOps;
3497 else
3498 SFLOGFLOW(("vbsf_reg_mmap: Warning: vm_ops=%p, expected %p!\n", vma->vm_ops, g_pGenericFileVmOps));
3499 }
3500 SFLOGFLOW(("vbsf_reg_mmap: returns %d\n", rc));
3501 return rc;
3502}
3503
3504#endif /* SFLOG_ENABLED */
3505
3506
3507/**
3508 * File operations for regular files.
3509 *
3510 * Note on splice_read/splice_write/sendfile:
3511 * - Splice was introduced in 2.6.17. The generic_file_splice_read/write
3512 * methods go thru the page cache, which is undesirable and is why we
3513 * need to cook our own versions of the code as long as we cannot track
3514 * host-side writes and correctly invalidate the guest page-cache.
3515 * - Sendfile reimplemented using splice in 2.6.23.
3516 * - The default_file_splice_read/write no-page-cache fallback functions,
3517 * were introduced in 2.6.31. The write one work in page units.
3518 * - Since linux 3.16 there is iter_file_splice_write that uses iter_write.
3519 * - Since linux 4.9 the generic_file_splice_read function started using
3520 * read_iter.
3521 */
3522struct file_operations vbsf_reg_fops = {
3523 .open = vbsf_reg_open,
3524#if RTLNX_VER_MAX(5,10,0) /* No regular .read/.write for 5.10, only .read_iter/.write_iter or in-kernel reads/writes fail. */
3525 .read = vbsf_reg_read,
3526 .write = vbsf_reg_write,
3527#endif
3528#if RTLNX_VER_MIN(3,16,0)
3529 .read_iter = vbsf_reg_read_iter,
3530 .write_iter = vbsf_reg_write_iter,
3531#elif RTLNX_VER_MIN(2,6,19)
3532 .aio_read = vbsf_reg_aio_read,
3533 .aio_write = vbsf_reg_aio_write,
3534#endif
3535 .release = vbsf_reg_release,
3536#ifdef SFLOG_ENABLED
3537 .mmap = vbsf_reg_mmap,
3538#else
3539 .mmap = generic_file_mmap,
3540#endif
3541#if RTLNX_VER_RANGE(2,6,17, 2,6,31)
3542 .splice_read = vbsf_splice_read,
3543#endif
3544#if RTLNX_VER_MIN(3,16,0)
3545 .splice_write = iter_file_splice_write,
3546#elif RTLNX_VER_MIN(2,6,17)
3547 .splice_write = vbsf_splice_write,
3548#endif
3549#if RTLNX_VER_RANGE(2,5,30, 2,6,23)
3550 .sendfile = vbsf_reg_sendfile,
3551#endif
3552 .llseek = vbsf_reg_llseek,
3553 .fsync = vbsf_reg_fsync,
3554#if RTLNX_VER_MIN(4,5,0)
3555 .copy_file_range = vbsf_reg_copy_file_range,
3556#endif
3557};
3558
3559
3560/**
3561 * Inodes operations for regular files.
3562 */
3563struct inode_operations vbsf_reg_iops = {
3564#if RTLNX_VER_MIN(2,5,18)
3565 .getattr = vbsf_inode_getattr,
3566#else
3567 .revalidate = vbsf_inode_revalidate,
3568#endif
3569 .setattr = vbsf_inode_setattr,
3570};
3571
3572
3573
3574/*********************************************************************************************************************************
3575* Address Space Operations on Regular Files (for mmap, sendfile, direct I/O) *
3576*********************************************************************************************************************************/
3577
3578/**
3579 * Used to read the content of a page into the page cache.
3580 *
3581 * Needed for mmap and reads+writes when the file is mmapped in a
3582 * shared+writeable fashion.
3583 */
3584#if RTLNX_VER_MIN(5,19,0)
3585static int vbsf_read_folio(struct file *file, struct folio *folio)
3586{
3587 struct page *page = &folio->page;
3588#else
3589static int vbsf_readpage(struct file *file, struct page *page)
3590{
3591#endif
3592 struct inode *inode = VBSF_GET_F_DENTRY(file)->d_inode;
3593 int err;
3594
3595 SFLOGFLOW(("vbsf_readpage: inode=%p file=%p page=%p off=%#llx\n", inode, file, page, (uint64_t)page->index << PAGE_SHIFT));
3596 Assert(PageLocked(page));
3597
3598 if (PageUptodate(page)) {
3599 unlock_page(page);
3600 return 0;
3601 }
3602
3603 if (!is_bad_inode(inode)) {
3604 VBOXSFREADPGLSTREQ *pReq = (VBOXSFREADPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
3605 if (pReq) {
3606 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3607 struct vbsf_reg_info *sf_r = file->private_data;
3608 uint32_t cbRead;
3609 int vrc;
3610
3611 pReq->PgLst.offFirstPage = 0;
3612 pReq->PgLst.aPages[0] = page_to_phys(page);
3613 vrc = VbglR0SfHostReqReadPgLst(pSuperInfo->map.root,
3614 pReq,
3615 sf_r->Handle.hHost,
3616 (uint64_t)page->index << PAGE_SHIFT,
3617 PAGE_SIZE,
3618 1 /*cPages*/);
3619
3620 cbRead = pReq->Parms.cb32Read.u.value32;
3621 AssertStmt(cbRead <= PAGE_SIZE, cbRead = PAGE_SIZE);
3622 VbglR0PhysHeapFree(pReq);
3623
3624 if (RT_SUCCESS(vrc)) {
3625 if (cbRead == PAGE_SIZE) {
3626 /* likely */
3627 } else {
3628 uint8_t *pbMapped = (uint8_t *)kmap(page);
3629 RT_BZERO(&pbMapped[cbRead], PAGE_SIZE - cbRead);
3630 kunmap(page);
3631 /** @todo truncate the inode file size? */
3632 }
3633
3634 flush_dcache_page(page);
3635 SetPageUptodate(page);
3636 unlock_page(page);
3637 return 0;
3638 }
3639 err = -RTErrConvertToErrno(vrc);
3640 } else
3641 err = -ENOMEM;
3642 } else
3643 err = -EIO;
3644 SetPageError(page);
3645 unlock_page(page);
3646 return err;
3647}
3648
3649
3650/**
3651 * Used to write out the content of a dirty page cache page to the host file.
3652 *
3653 * Needed for mmap and writes when the file is mmapped in a shared+writeable
3654 * fashion.
3655 */
3656#if RTLNX_VER_MIN(2,5,52)
3657static int vbsf_writepage(struct page *page, struct writeback_control *wbc)
3658#else
3659static int vbsf_writepage(struct page *page)
3660#endif
3661{
3662 struct address_space *mapping = page->mapping;
3663 struct inode *inode = mapping->host;
3664 struct vbsf_inode_info *sf_i = VBSF_GET_INODE_INFO(inode);
3665 struct vbsf_handle *pHandle = vbsf_handle_find(sf_i, VBSF_HANDLE_F_WRITE, VBSF_HANDLE_F_APPEND);
3666 int err;
3667
3668 SFLOGFLOW(("vbsf_writepage: inode=%p page=%p off=%#llx pHandle=%p (%#llx)\n",
3669 inode, page, (uint64_t)page->index << PAGE_SHIFT, pHandle, pHandle ? pHandle->hHost : 0));
3670
3671 if (pHandle) {
3672 struct vbsf_super_info *pSuperInfo = VBSF_GET_SUPER_INFO(inode->i_sb);
3673 VBOXSFWRITEPGLSTREQ *pReq = (VBOXSFWRITEPGLSTREQ *)VbglR0PhysHeapAlloc(sizeof(*pReq));
3674 if (pReq) {
3675 uint64_t const cbFile = i_size_read(inode);
3676 uint64_t const offInFile = (uint64_t)page->index << PAGE_SHIFT;
3677 uint32_t const cbToWrite = page->index != (cbFile >> PAGE_SHIFT) ? PAGE_SIZE
3678 : (uint32_t)cbFile & (uint32_t)PAGE_OFFSET_MASK;
3679 int vrc;
3680
3681 pReq->PgLst.offFirstPage = 0;
3682 pReq->PgLst.aPages[0] = page_to_phys(page);
3683 vrc = VbglR0SfHostReqWritePgLst(pSuperInfo->map.root,
3684 pReq,
3685 pHandle->hHost,
3686 offInFile,
3687 cbToWrite,
3688 1 /*cPages*/);
3689 sf_i->ModificationTimeAtOurLastWrite = sf_i->ModificationTime;
3690 AssertMsgStmt(pReq->Parms.cb32Write.u.value32 == cbToWrite || RT_FAILURE(vrc), /* lazy bird */
3691 ("%#x vs %#x\n", pReq->Parms.cb32Write, cbToWrite),
3692 vrc = VERR_WRITE_ERROR);
3693 VbglR0PhysHeapFree(pReq);
3694
3695 if (RT_SUCCESS(vrc)) {
3696 /* Update the inode if we've extended the file. */
3697 /** @todo is this necessary given the cbToWrite calc above? */
3698 uint64_t const offEndOfWrite = offInFile + cbToWrite;
3699 if ( offEndOfWrite > cbFile
3700 && offEndOfWrite > i_size_read(inode))
3701 i_size_write(inode, offEndOfWrite);
3702
3703 /* Update and unlock the page. */
3704 if (PageError(page))
3705 ClearPageError(page);
3706 SetPageUptodate(page);
3707 unlock_page(page);
3708
3709 vbsf_handle_release(pHandle, pSuperInfo, "vbsf_writepage");
3710 return 0;
3711 }
3712
3713 /*
3714 * We failed.
3715 */
3716 err = -EIO;
3717 } else
3718 err = -ENOMEM;
3719 vbsf_handle_release(pHandle, pSuperInfo, "vbsf_writepage");
3720 } else {
3721 /** @todo we could re-open the file here and deal with this... */
3722 static uint64_t volatile s_cCalls = 0;
3723 if (s_cCalls++ < 16)
3724 printk("vbsf_writepage: no writable handle for %s..\n", sf_i->path->String.ach);
3725 err = -EIO;
3726 }
3727 SetPageError(page);
3728 unlock_page(page);
3729 return err;
3730}
3731
3732
3733#if RTLNX_VER_MIN(2,6,24)
3734/**
3735 * Called when writing thru the page cache (which we shouldn't be doing).
3736 */
3737static inline void vbsf_write_begin_warn(loff_t pos, unsigned len, unsigned flags)
3738{
3739 /** @todo r=bird: We shouldn't ever get here, should we? Because we don't use
3740 * the page cache for any writes AFAIK. We could just as well use
3741 * simple_write_begin & simple_write_end here if we think we really
3742 * need to have non-NULL function pointers in the table... */
3743 static uint64_t volatile s_cCalls = 0;
3744 if (s_cCalls++ < 16) {
3745 printk("vboxsf: Unexpected call to vbsf_write_begin(pos=%#llx len=%#x flags=%#x)! Please report.\n",
3746 (unsigned long long)pos, len, flags);
3747 RTLogBackdoorPrintf("vboxsf: Unexpected call to vbsf_write_begin(pos=%#llx len=%#x flags=%#x)! Please report.\n",
3748 (unsigned long long)pos, len, flags);
3749# ifdef WARN_ON
3750 WARN_ON(1);
3751# endif
3752 }
3753}
3754
3755# if RTLNX_VER_MIN(5,19,0)
3756int vbsf_write_begin(struct file *file, struct address_space *mapping, loff_t pos,
3757 unsigned len, struct page **pagep, void **fsdata)
3758{
3759 vbsf_write_begin_warn(pos, len, 0);
3760 return simple_write_begin(file, mapping, pos, len, pagep, fsdata);
3761}
3762# else
3763int vbsf_write_begin(struct file *file, struct address_space *mapping, loff_t pos,
3764 unsigned len, unsigned flags, struct page **pagep, void **fsdata)
3765{
3766 vbsf_write_begin_warn(pos, len, flags);
3767 return simple_write_begin(file, mapping, pos, len, flags, pagep, fsdata);
3768}
3769# endif
3770
3771#endif /* KERNEL_VERSION >= 2.6.24 */
3772
3773#if RTLNX_VER_MIN(5,14,0)
3774/**
3775 * Companion to vbsf_write_begin (i.e. shouldn't be called).
3776 */
3777static int vbsf_write_end(struct file *file, struct address_space *mapping,
3778 loff_t pos, unsigned int len, unsigned int copied,
3779 struct page *page, void *fsdata)
3780{
3781 static uint64_t volatile s_cCalls = 0;
3782 if (s_cCalls++ < 16)
3783 {
3784 printk("vboxsf: Unexpected call to vbsf_write_end(pos=%#llx len=%#x)! Please report.\n",
3785 (unsigned long long)pos, len);
3786 RTLogBackdoorPrintf("vboxsf: Unexpected call to vbsf_write_end(pos=%#llx len=%#x)! Please report.\n",
3787 (unsigned long long)pos, len);
3788# ifdef WARN_ON
3789 WARN_ON(1);
3790# endif
3791 }
3792 return -ENOTSUPP;
3793}
3794#endif /* KERNEL_VERSION >= 5.14.0 */
3795
3796
3797#if RTLNX_VER_MIN(2,4,10)
3798
3799# ifdef VBOX_UEK
3800# undef iov_iter /* HACK ALERT! Don't put anything needing vbsf_iov_iter after this fun! */
3801# endif
3802
3803/**
3804 * This is needed to make open accept O_DIRECT as well as dealing with direct
3805 * I/O requests if we don't intercept them earlier.
3806 */
3807# if RTLNX_VER_MIN(4, 7, 0) \
3808 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_RANGE(4,4,73, 4,4,74) /** @todo Figure out when exactly. */) \
3809 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_RANGE(4,4,75, 4,4,90) /** @todo Figure out when exactly. */) \
3810 || (defined(CONFIG_SUSE_KERNEL) && RTLNX_VER_RANGE(4,4,92, 4,5,0) /** @todo Figure out when exactly. */)
3811static ssize_t vbsf_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3812# elif RTLNX_VER_MIN(4, 1, 0)
3813static ssize_t vbsf_direct_IO(struct kiocb *iocb, struct iov_iter *iter, loff_t offset)
3814# elif RTLNX_VER_MIN(3, 16, 0) || defined(VBOX_UEK)
3815static ssize_t vbsf_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, loff_t offset)
3816# elif RTLNX_VER_MIN(2, 6, 6)
3817static ssize_t vbsf_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3818# elif RTLNX_VER_MIN(2, 5, 55)
3819static int vbsf_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3820# elif RTLNX_VER_MIN(2, 5, 41)
3821static int vbsf_direct_IO(int rw, struct file *file, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3822# elif RTLNX_VER_MIN(2, 5, 35)
3823static int vbsf_direct_IO(int rw, struct inode *inode, const struct iovec *iov, loff_t offset, unsigned long nr_segs)
3824# elif RTLNX_VER_MIN(2, 5, 26)
3825static int vbsf_direct_IO(int rw, struct inode *inode, char *buf, loff_t offset, size_t count)
3826# elif LINUX_VERSION_CODE == KERNEL_VERSION(2, 4, 21) && defined(I_NEW) /* RHEL3 Frankenkernel. */
3827static int vbsf_direct_IO(int rw, struct file *file, struct kiobuf *buf, unsigned long whatever1, int whatever2)
3828# else
3829static int vbsf_direct_IO(int rw, struct inode *inode, struct kiobuf *buf, unsigned long whatever1, int whatever2)
3830# endif
3831{
3832 TRACE();
3833 return -EINVAL;
3834}
3835
3836#endif
3837
3838/**
3839 * Address space (for the page cache) operations for regular files.
3840 *
3841 * @todo the FsPerf touch/flush (mmap) test fails on 4.4.0 (ubuntu 16.04 lts).
3842 */
3843struct address_space_operations vbsf_reg_aops = {
3844#if RTLNX_VER_MIN(5,19,0)
3845 .read_folio = vbsf_read_folio,
3846#else
3847 .readpage = vbsf_readpage,
3848#endif
3849 .writepage = vbsf_writepage,
3850 /** @todo Need .writepages if we want msync performance... */
3851#if RTLNX_VER_MIN(5,18,0)
3852 .dirty_folio = filemap_dirty_folio,
3853#elif RTLNX_VER_MIN(2,5,12)
3854 .set_page_dirty = __set_page_dirty_buffers,
3855#endif
3856#if RTLNX_VER_MIN(5,14,0)
3857 .write_begin = vbsf_write_begin,
3858 .write_end = vbsf_write_end,
3859#elif RTLNX_VER_MIN(2,6,24)
3860 .write_begin = vbsf_write_begin,
3861 .write_end = simple_write_end,
3862#elif RTLNX_VER_MIN(2,5,45)
3863 .prepare_write = simple_prepare_write,
3864 .commit_write = simple_commit_write,
3865#endif
3866#if RTLNX_VER_MIN(2,4,10)
3867 .direct_IO = vbsf_direct_IO,
3868#endif
3869};
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette