kern_sendfile.c (8f0a223c78964c347946fe2573b6f4e4ab4d2e06) kern_sendfile.c (59e1ac9d79ff3ca4bedb1f792060254867f43604)
1/*-
2 * Copyright (c) 2013-2015 Gleb Smirnoff <glebius@FreeBSD.org>
3 * Copyright (c) 1998, David Greenman. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 242 unchanged lines hidden (view full) ---

251
252 /* Rest of pages are full sized. */
253 *space -= (old - new) * PAGE_SIZE;
254
255 KASSERT(*space >= 0, ("%s: space went backwards", __func__));
256}
257
258/*
1/*-
2 * Copyright (c) 2013-2015 Gleb Smirnoff <glebius@FreeBSD.org>
3 * Copyright (c) 1998, David Greenman. All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 242 unchanged lines hidden (view full) ---

251
252 /* Rest of pages are full sized. */
253 *space -= (old - new) * PAGE_SIZE;
254
255 KASSERT(*space >= 0, ("%s: space went backwards", __func__));
256}
257
258/*
259 * Wait for all in-flight ios to complete, we must not unwire pages
260 * under them.
261 */
262static void
263sendfile_iowait(struct sf_io *sfio, const char *wmesg)
264{
265 while (atomic_load_int(&sfio->nios) != 1)
266 pause(wmesg, 1);
267}
268
269/*
259 * I/O completion callback.
260 */
261static void
262sendfile_iodone(void *arg, vm_page_t *pg, int count, int error)
263{
264 struct sf_io *sfio = arg;
265 struct socket *so;
266

--- 165 unchanged lines hidden (view full) ---

432 pa[j] = bogus_page;
433 }
434
435 refcount_acquire(&sfio->nios);
436 rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,
437 i + count == npages ? &rhpages : NULL,
438 &sendfile_iodone, sfio);
439 if (__predict_false(rv != VM_PAGER_OK)) {
270 * I/O completion callback.
271 */
272static void
273sendfile_iodone(void *arg, vm_page_t *pg, int count, int error)
274{
275 struct sf_io *sfio = arg;
276 struct socket *so;
277

--- 165 unchanged lines hidden (view full) ---

443 pa[j] = bogus_page;
444 }
445
446 refcount_acquire(&sfio->nios);
447 rv = vm_pager_get_pages_async(obj, pa + i, count, NULL,
448 i + count == npages ? &rhpages : NULL,
449 &sendfile_iodone, sfio);
450 if (__predict_false(rv != VM_PAGER_OK)) {
451 sendfile_iowait(sfio, "sferrio");
452
440 /*
441 * Perform full pages recovery before returning EIO.
442 * Pages from 0 to npages are wired.
443 * Pages from i to npages are also busied.
444 * Pages from (i + 1) to (i + count - 1) may be
445 * substituted to bogus page, and not busied.
446 */
447 for (j = 0; j < npages; j++) {

--- 514 unchanged lines hidden (view full) ---

962 * buffers, do not sleep, since several
963 * threads might exhaust the buffers and then
964 * deadlock.
965 */
966 sf = sf_buf_alloc(pa[i],
967 m != NULL ? SFB_NOWAIT : SFB_CATCH);
968 if (sf == NULL) {
969 SFSTAT_INC(sf_allocfail);
453 /*
454 * Perform full pages recovery before returning EIO.
455 * Pages from 0 to npages are wired.
456 * Pages from i to npages are also busied.
457 * Pages from (i + 1) to (i + count - 1) may be
458 * substituted to bogus page, and not busied.
459 */
460 for (j = 0; j < npages; j++) {

--- 514 unchanged lines hidden (view full) ---

975 * buffers, do not sleep, since several
976 * threads might exhaust the buffers and then
977 * deadlock.
978 */
979 sf = sf_buf_alloc(pa[i],
980 m != NULL ? SFB_NOWAIT : SFB_CATCH);
981 if (sf == NULL) {
982 SFSTAT_INC(sf_allocfail);
983 sendfile_iowait(sfio, "sfnosf");
970 for (int j = i; j < npages; j++)
971 vm_page_unwire(pa[j], PQ_INACTIVE);
972 if (m == NULL)
973 softerr = ENOBUFS;
974 fixspace(npages, i, off, &space);
975 npages = i;
976 break;
977 }

--- 282 unchanged lines hidden --- 		984 for (int j = i; j < npages; j++)
985 vm_page_unwire(pa[j], PQ_INACTIVE);
986 if (m == NULL)
987 softerr = ENOBUFS;
988 fixspace(npages, i, off, &space);
989 npages = i;
990 break;
991 }

--- 282 unchanged lines hidden ---