vm_pageout.c (e9f995d82479767d01dc846b3834dd6d11f2ce70) vm_pageout.c (222d01951f8677015e3e96c6950e809c0d983c09)
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005 Yahoo! Technologies Norway AS

--- 524 unchanged lines hidden (view full) ---

533 p = TAILQ_FIRST(&object->memq);
534 vm_page_lock_queues();
535 while (p && (rcount-- > 0)) {
536 if (pmap_resident_count(pmap) <= desired) {
537 vm_page_unlock_queues();
538 goto unlock_return;
539 }
540 next = TAILQ_NEXT(p, listq);
1/*-
2 * Copyright (c) 1991 Regents of the University of California.
3 * All rights reserved.
4 * Copyright (c) 1994 John S. Dyson
5 * All rights reserved.
6 * Copyright (c) 1994 David Greenman
7 * All rights reserved.
8 * Copyright (c) 2005 Yahoo! Technologies Norway AS

--- 524 unchanged lines hidden (view full) ---

533 p = TAILQ_FIRST(&object->memq);
534 vm_page_lock_queues();
535 while (p && (rcount-- > 0)) {
536 if (pmap_resident_count(pmap) <= desired) {
537 vm_page_unlock_queues();
538 goto unlock_return;
539 }
540 next = TAILQ_NEXT(p, listq);
541 cnt.v_pdpages++;
541 VMCNT_ADD(pdpages, 1);
542 if (p->wire_count != 0 ||
543 p->hold_count != 0 ||
544 p->busy != 0 ||
545 (p->oflags & VPO_BUSY) ||
546 (p->flags & PG_UNMANAGED) ||
547 !pmap_page_exists_quick(pmap, p)) {
548 p = next;
549 continue;

--- 184 unchanged lines hidden (view full) ---

734 */
735 if ((maxlaunder = vm_max_launder) <= 1)
736 maxlaunder = 1;
737 if (pass)
738 maxlaunder = 10000;
739 vm_page_lock_queues();
740rescan0:
741 addl_page_shortage = addl_page_shortage_init;
542 if (p->wire_count != 0 ||
543 p->hold_count != 0 ||
544 p->busy != 0 ||
545 (p->oflags & VPO_BUSY) ||
546 (p->flags & PG_UNMANAGED) ||
547 !pmap_page_exists_quick(pmap, p)) {
548 p = next;
549 continue;

--- 184 unchanged lines hidden (view full) ---

734 */
735 if ((maxlaunder = vm_max_launder) <= 1)
736 maxlaunder = 1;
737 if (pass)
738 maxlaunder = 10000;
739 vm_page_lock_queues();
740rescan0:
741 addl_page_shortage = addl_page_shortage_init;
742 maxscan = cnt.v_inactive_count;
742 maxscan = VMCNT_GET(inactive_count);
743
744 for (m = TAILQ_FIRST(&vm_page_queues[PQ_INACTIVE].pl);
745 m != NULL && maxscan-- > 0 && page_shortage > 0;
746 m = next) {
747
743
744 for (m = TAILQ_FIRST(&vm_page_queues[PQ_INACTIVE].pl);
745 m != NULL && maxscan-- > 0 && page_shortage > 0;
746 m = next) {
747
748 cnt.v_pdpages++;
748 VMCNT_ADD(pdpages, 1);
749
750 if (VM_PAGE_GETQUEUE(m) != PQ_INACTIVE) {
751 goto rescan0;
752 }
753
754 next = TAILQ_NEXT(m, pageq);
755 object = m->object;
756

--- 94 unchanged lines hidden (view full) ---

851 vm_page_dirty(m);
852 }
853
854 if (m->valid == 0) {
855 /*
856 * Invalid pages can be easily freed
857 */
858 vm_page_free(m);
749
750 if (VM_PAGE_GETQUEUE(m) != PQ_INACTIVE) {
751 goto rescan0;
752 }
753
754 next = TAILQ_NEXT(m, pageq);
755 object = m->object;
756

--- 94 unchanged lines hidden (view full) ---

851 vm_page_dirty(m);
852 }
853
854 if (m->valid == 0) {
855 /*
856 * Invalid pages can be easily freed
857 */
858 vm_page_free(m);
859 cnt.v_dfree++;
859 VMCNT_ADD(dfree, 1);
860 --page_shortage;
861 } else if (m->dirty == 0) {
862 /*
863 * Clean pages can be placed onto the cache queue.
864 * This effectively frees them.
865 */
866 vm_page_cache(m);
867 --page_shortage;

--- 170 unchanged lines hidden (view full) ---

1038 }
1039 VM_OBJECT_UNLOCK(object);
1040 }
1041
1042 /*
1043 * Compute the number of pages we want to try to move from the
1044 * active queue to the inactive queue.
1045 */
860 --page_shortage;
861 } else if (m->dirty == 0) {
862 /*
863 * Clean pages can be placed onto the cache queue.
864 * This effectively frees them.
865 */
866 vm_page_cache(m);
867 --page_shortage;

--- 170 unchanged lines hidden (view full) ---

1038 }
1039 VM_OBJECT_UNLOCK(object);
1040 }
1041
1042 /*
1043 * Compute the number of pages we want to try to move from the
1044 * active queue to the inactive queue.
1045 */
1046 page_shortage = vm_paging_target() +
1047 cnt.v_inactive_target - cnt.v_inactive_count;
1046 page_shortage = vm_paging_target() + VMCNT_GET(inactive_target) -
1047 VMCNT_GET(inactive_count);
1048 page_shortage += addl_page_shortage;
1049
1050 /*
1051 * Scan the active queue for things we can deactivate. We nominally
1052 * track the per-page activity counter and use it to locate
1053 * deactivation candidates.
1054 */
1048 page_shortage += addl_page_shortage;
1049
1050 /*
1051 * Scan the active queue for things we can deactivate. We nominally
1052 * track the per-page activity counter and use it to locate
1053 * deactivation candidates.
1054 */
1055 pcount = cnt.v_active_count;
1055 pcount = VMCNT_GET(active_count);
1056 m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1057
1058 while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) {
1059
1060 KASSERT(VM_PAGE_INQUEUE2(m, PQ_ACTIVE),
1061 ("vm_pageout_scan: page %p isn't active", m));
1062
1063 next = TAILQ_NEXT(m, pageq);

--- 20 unchanged lines hidden (view full) ---

1084 m = next;
1085 continue;
1086 }
1087
1088 /*
1089 * The count for pagedaemon pages is done after checking the
1090 * page for eligibility...
1091 */
1056 m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1057
1058 while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) {
1059
1060 KASSERT(VM_PAGE_INQUEUE2(m, PQ_ACTIVE),
1061 ("vm_pageout_scan: page %p isn't active", m));
1062
1063 next = TAILQ_NEXT(m, pageq);

--- 20 unchanged lines hidden (view full) ---

1084 m = next;
1085 continue;
1086 }
1087
1088 /*
1089 * The count for pagedaemon pages is done after checking the
1090 * page for eligibility...
1091 */
1092 cnt.v_pdpages++;
1092 VMCNT_ADD(pdpages, 1);
1093
1094 /*
1095 * Check to see "how much" the page has been used.
1096 */
1097 actcount = 0;
1098 if (object->ref_count != 0) {
1099 if (m->flags & PG_REFERENCED) {
1100 actcount += 1;

--- 43 unchanged lines hidden (view full) ---

1144 /*
1145 * We try to maintain some *really* free pages, this allows interrupt
1146 * code to be guaranteed space. Since both cache and free queues
1147 * are considered basically 'free', moving pages from cache to free
1148 * does not effect other calculations.
1149 */
1150 cache_cur = cache_last_free;
1151 cache_first_failure = -1;
1093
1094 /*
1095 * Check to see "how much" the page has been used.
1096 */
1097 actcount = 0;
1098 if (object->ref_count != 0) {
1099 if (m->flags & PG_REFERENCED) {
1100 actcount += 1;

--- 43 unchanged lines hidden (view full) ---

1144 /*
1145 * We try to maintain some *really* free pages, this allows interrupt
1146 * code to be guaranteed space. Since both cache and free queues
1147 * are considered basically 'free', moving pages from cache to free
1148 * does not effect other calculations.
1149 */
1150 cache_cur = cache_last_free;
1151 cache_first_failure = -1;
1152 while (cnt.v_free_count < cnt.v_free_reserved && (cache_cur =
1153 (cache_cur + PQ_PRIME2) & PQ_COLORMASK) != cache_first_failure) {
1152 while (VMCNT_GET(free_count) < VMCNT_GET(free_reserved) &&
1153 (cache_cur = (cache_cur + PQ_PRIME2) & PQ_COLORMASK) !=
1154 cache_first_failure) {
1154 TAILQ_FOREACH(m, &vm_page_queues[PQ_CACHE + cache_cur].pl,
1155 pageq) {
1156 KASSERT(m->dirty == 0,
1157 ("Found dirty cache page %p", m));
1158 KASSERT(!pmap_page_is_mapped(m),
1159 ("Found mapped cache page %p", m));
1160 KASSERT((m->flags & PG_UNMANAGED) == 0,
1161 ("Found unmanaged cache page %p", m));
1162 KASSERT(m->wire_count == 0,
1163 ("Found wired cache page %p", m));
1164 if (m->hold_count == 0 && VM_OBJECT_TRYLOCK(object =
1165 m->object)) {
1166 KASSERT((m->oflags & VPO_BUSY) == 0 &&
1167 m->busy == 0, ("Found busy cache page %p",
1168 m));
1169 vm_page_free(m);
1170 VM_OBJECT_UNLOCK(object);
1155 TAILQ_FOREACH(m, &vm_page_queues[PQ_CACHE + cache_cur].pl,
1156 pageq) {
1157 KASSERT(m->dirty == 0,
1158 ("Found dirty cache page %p", m));
1159 KASSERT(!pmap_page_is_mapped(m),
1160 ("Found mapped cache page %p", m));
1161 KASSERT((m->flags & PG_UNMANAGED) == 0,
1162 ("Found unmanaged cache page %p", m));
1163 KASSERT(m->wire_count == 0,
1164 ("Found wired cache page %p", m));
1165 if (m->hold_count == 0 && VM_OBJECT_TRYLOCK(object =
1166 m->object)) {
1167 KASSERT((m->oflags & VPO_BUSY) == 0 &&
1168 m->busy == 0, ("Found busy cache page %p",
1169 m));
1170 vm_page_free(m);
1171 VM_OBJECT_UNLOCK(object);
1171 cnt.v_dfree++;
1172 VMCNT_ADD(dfree, 1);
1172 cache_last_free = cache_cur;
1173 cache_first_failure = -1;
1174 break;
1175 }
1176 }
1177 if (m == NULL && cache_first_failure == -1)
1178 cache_first_failure = cache_cur;
1179 }

--- 106 unchanged lines hidden (view full) ---

1286 }
1287 sx_sunlock(&allproc_lock);
1288 if (bigproc != NULL) {
1289 killproc(bigproc, "out of swap space");
1290 mtx_lock_spin(&sched_lock);
1291 sched_nice(bigproc, PRIO_MIN);
1292 mtx_unlock_spin(&sched_lock);
1293 PROC_UNLOCK(bigproc);
1173 cache_last_free = cache_cur;
1174 cache_first_failure = -1;
1175 break;
1176 }
1177 }
1178 if (m == NULL && cache_first_failure == -1)
1179 cache_first_failure = cache_cur;
1180 }

--- 106 unchanged lines hidden (view full) ---

1287 }
1288 sx_sunlock(&allproc_lock);
1289 if (bigproc != NULL) {
1290 killproc(bigproc, "out of swap space");
1291 mtx_lock_spin(&sched_lock);
1292 sched_nice(bigproc, PRIO_MIN);
1293 mtx_unlock_spin(&sched_lock);
1294 PROC_UNLOCK(bigproc);
1294 wakeup(&cnt.v_free_count);
1295 wakeup(VMCNT_PTR(free_count));
1295 }
1296 }
1297 mtx_unlock(&Giant);
1298}
1299
1300/*
1301 * This routine tries to maintain the pseudo LRU active queue,
1302 * so that during long periods of time where there is no paging,

--- 6 unchanged lines hidden (view full) ---

1309 vm_object_t object;
1310 vm_page_t m,next;
1311 int pcount,tpcount; /* Number of pages to check */
1312 static int fullintervalcount = 0;
1313 int page_shortage;
1314
1315 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1316 page_shortage =
1296 }
1297 }
1298 mtx_unlock(&Giant);
1299}
1300
1301/*
1302 * This routine tries to maintain the pseudo LRU active queue,
1303 * so that during long periods of time where there is no paging,

--- 6 unchanged lines hidden (view full) ---

1310 vm_object_t object;
1311 vm_page_t m,next;
1312 int pcount,tpcount; /* Number of pages to check */
1313 static int fullintervalcount = 0;
1314 int page_shortage;
1315
1316 mtx_assert(&vm_page_queue_mtx, MA_OWNED);
1317 page_shortage =
1317 (cnt.v_inactive_target + cnt.v_cache_max + cnt.v_free_min) -
1318 (cnt.v_free_count + cnt.v_inactive_count + cnt.v_cache_count);
1318 (VMCNT_GET(inactive_target) + VMCNT_GET(cache_max) +
1319 VMCNT_GET(free_min)) - (VMCNT_GET(free_count) +
1320 VMCNT_GET(inactive_count) + VMCNT_GET(cache_count));
1319
1320 if (page_shortage <= 0)
1321 return;
1322
1321
1322 if (page_shortage <= 0)
1323 return;
1324
1323 pcount = cnt.v_active_count;
1325 pcount = VMCNT_GET(active_count);
1324 fullintervalcount += vm_pageout_stats_interval;
1325 if (fullintervalcount < vm_pageout_full_stats_interval) {
1326 fullintervalcount += vm_pageout_stats_interval;
1327 if (fullintervalcount < vm_pageout_full_stats_interval) {
1326 tpcount = (vm_pageout_stats_max * cnt.v_active_count) / cnt.v_page_count;
1328 tpcount = (vm_pageout_stats_max * VMCNT_GET(active_count)) /
1329 VMCNT_GET(page_count);
1327 if (pcount > tpcount)
1328 pcount = tpcount;
1329 } else {
1330 fullintervalcount = 0;
1331 }
1332
1333 m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1334 while ((m != NULL) && (pcount-- > 0)) {

--- 69 unchanged lines hidden (view full) ---

1404static void
1405vm_pageout()
1406{
1407 int error, pass;
1408
1409 /*
1410 * Initialize some paging parameters.
1411 */
1330 if (pcount > tpcount)
1331 pcount = tpcount;
1332 } else {
1333 fullintervalcount = 0;
1334 }
1335
1336 m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
1337 while ((m != NULL) && (pcount-- > 0)) {

--- 69 unchanged lines hidden (view full) ---

1407static void
1408vm_pageout()
1409{
1410 int error, pass;
1411
1412 /*
1413 * Initialize some paging parameters.
1414 */
1412 cnt.v_interrupt_free_min = 2;
1413 if (cnt.v_page_count < 2000)
1415 VMCNT_SET(interrupt_free_min, 2);
1416 if (VMCNT_GET(page_count) < 2000)
1414 vm_pageout_page_count = 8;
1415
1416 /*
1417 * v_free_reserved needs to include enough for the largest
1418 * swap pager structures plus enough for any pv_entry structs
1419 * when paging.
1420 */
1417 vm_pageout_page_count = 8;
1418
1419 /*
1420 * v_free_reserved needs to include enough for the largest
1421 * swap pager structures plus enough for any pv_entry structs
1422 * when paging.
1423 */
1421 if (cnt.v_page_count > 1024)
1422 cnt.v_free_min = 4 + (cnt.v_page_count - 1024) / 200;
1423 else
1424 cnt.v_free_min = 4;
1425 cnt.v_pageout_free_min = (2*MAXBSIZE)/PAGE_SIZE +
1426 cnt.v_interrupt_free_min;
1427 cnt.v_free_reserved = vm_pageout_page_count +
1428 cnt.v_pageout_free_min + (cnt.v_page_count / 768) + PQ_NUMCOLORS;
1429 cnt.v_free_severe = cnt.v_free_min / 2;
1430 cnt.v_free_min += cnt.v_free_reserved;
1431 cnt.v_free_severe += cnt.v_free_reserved;
1424 VMCNT_SET(free_min, (VMCNT_GET(page_count) > 1024) ? (4 +
1425 (VMCNT_GET(page_count) - 1024) / 200) : 4);
1426 VMCNT_SET(pageout_free_min, (2 * MAXBSIZE) / PAGE_SIZE +
1427 VMCNT_GET(interrupt_free_min));
1428 VMCNT_SET(free_reserved, vm_pageout_page_count +
1429 VMCNT_GET(pageout_free_min) + (VMCNT_GET(page_count) / 768) +
1430 PQ_NUMCOLORS);
1431 VMCNT_SET(free_severe, VMCNT_GET(free_min) / 2);
1432 VMCNT_ADD(free_min, VMCNT_GET(free_reserved));
1433 VMCNT_ADD(free_severe, VMCNT_GET(free_reserved));
1432
1433 /*
1434 * v_free_target and v_cache_min control pageout hysteresis. Note
1435 * that these are more a measure of the VM cache queue hysteresis
1436 * then the VM free queue. Specifically, v_free_target is the
1437 * high water mark (free+cache pages).
1438 *
1439 * v_free_reserved + v_cache_min (mostly means v_cache_min) is the
1440 * low water mark, while v_free_min is the stop. v_cache_min must
1441 * be big enough to handle memory needs while the pageout daemon
1442 * is signalled and run to free more pages.
1443 */
1434
1435 /*
1436 * v_free_target and v_cache_min control pageout hysteresis. Note
1437 * that these are more a measure of the VM cache queue hysteresis
1438 * then the VM free queue. Specifically, v_free_target is the
1439 * high water mark (free+cache pages).
1440 *
1441 * v_free_reserved + v_cache_min (mostly means v_cache_min) is the
1442 * low water mark, while v_free_min is the stop. v_cache_min must
1443 * be big enough to handle memory needs while the pageout daemon
1444 * is signalled and run to free more pages.
1445 */
1444 if (cnt.v_free_count > 6144)
1445 cnt.v_free_target = 4 * cnt.v_free_min + cnt.v_free_reserved;
1446 else
1447 cnt.v_free_target = 2 * cnt.v_free_min + cnt.v_free_reserved;
1446 VMCNT_SET(free_target, ((VMCNT_GET(free_count) > 6144) ? 4 : 2) *
1447 VMCNT_GET(free_min) + VMCNT_GET(free_reserved));
1448
1448
1449 if (cnt.v_free_count > 2048) {
1450 cnt.v_cache_min = cnt.v_free_target;
1451 cnt.v_cache_max = 2 * cnt.v_cache_min;
1452 cnt.v_inactive_target = (3 * cnt.v_free_target) / 2;
1449 if (VMCNT_GET(free_count) > 2048) {
1450 VMCNT_SET(cache_min, VMCNT_GET(free_target));
1451 VMCNT_SET(cache_max, 2 * VMCNT_GET(cache_min));
1452 VMCNT_SET(inactive_target, (3 * VMCNT_GET(free_target) / 2));
1453 } else {
1453 } else {
1454 cnt.v_cache_min = 0;
1455 cnt.v_cache_max = 0;
1456 cnt.v_inactive_target = cnt.v_free_count / 4;
1454 VMCNT_SET(cache_min, 0);
1455 VMCNT_SET(cache_max, 0);
1456 VMCNT_SET(inactive_target, VMCNT_GET(free_count) / 4);
1457 }
1457 }
1458 if (cnt.v_inactive_target > cnt.v_free_count / 3)
1459 cnt.v_inactive_target = cnt.v_free_count / 3;
1458 if (VMCNT_GET(inactive_target) > VMCNT_GET(free_count) / 3)
1459 VMCNT_SET(inactive_target, VMCNT_GET(free_count) / 3);
1460
1461 /* XXX does not really belong here */
1462 if (vm_page_max_wired == 0)
1460
1461 /* XXX does not really belong here */
1462 if (vm_page_max_wired == 0)
1463 vm_page_max_wired = cnt.v_free_count / 3;
1463 vm_page_max_wired = VMCNT_GET(free_count) / 3;
1464
1465 if (vm_pageout_stats_max == 0)
1464
1465 if (vm_pageout_stats_max == 0)
1466 vm_pageout_stats_max = cnt.v_free_target;
1466 vm_pageout_stats_max = VMCNT_GET(free_target);
1467
1468 /*
1469 * Set interval in seconds for stats scan.
1470 */
1471 if (vm_pageout_stats_interval == 0)
1472 vm_pageout_stats_interval = 5;
1473 if (vm_pageout_full_stats_interval == 0)
1474 vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;

--- 9 unchanged lines hidden (view full) ---

1484 * not clear vm_pages_needed until we reach our target,
1485 * otherwise we may be woken up over and over again and
1486 * waste a lot of cpu.
1487 */
1488 mtx_lock(&vm_page_queue_free_mtx);
1489 if (vm_pages_needed && !vm_page_count_min()) {
1490 if (!vm_paging_needed())
1491 vm_pages_needed = 0;
1467
1468 /*
1469 * Set interval in seconds for stats scan.
1470 */
1471 if (vm_pageout_stats_interval == 0)
1472 vm_pageout_stats_interval = 5;
1473 if (vm_pageout_full_stats_interval == 0)
1474 vm_pageout_full_stats_interval = vm_pageout_stats_interval * 4;

--- 9 unchanged lines hidden (view full) ---

1484 * not clear vm_pages_needed until we reach our target,
1485 * otherwise we may be woken up over and over again and
1486 * waste a lot of cpu.
1487 */
1488 mtx_lock(&vm_page_queue_free_mtx);
1489 if (vm_pages_needed && !vm_page_count_min()) {
1490 if (!vm_paging_needed())
1491 vm_pages_needed = 0;
1492 wakeup(&cnt.v_free_count);
1492 wakeup(VMCNT_PTR(free_count));
1493 }
1494 if (vm_pages_needed) {
1495 /*
1496 * Still not done, take a second pass without waiting
1497 * (unlimited dirty cleaning), otherwise sleep a bit
1498 * and try again.
1499 */
1500 ++pass;

--- 18 unchanged lines hidden (view full) ---

1519 pass = 0;
1520 vm_page_lock_queues();
1521 vm_pageout_page_stats();
1522 vm_page_unlock_queues();
1523 continue;
1524 }
1525 }
1526 if (vm_pages_needed)
1493 }
1494 if (vm_pages_needed) {
1495 /*
1496 * Still not done, take a second pass without waiting
1497 * (unlimited dirty cleaning), otherwise sleep a bit
1498 * and try again.
1499 */
1500 ++pass;

--- 18 unchanged lines hidden (view full) ---

1519 pass = 0;
1520 vm_page_lock_queues();
1521 vm_pageout_page_stats();
1522 vm_page_unlock_queues();
1523 continue;
1524 }
1525 }
1526 if (vm_pages_needed)
1527 cnt.v_pdwakeups++;
1527 VMCNT_ADD(pdwakeups, 1);
1528 mtx_unlock(&vm_page_queue_free_mtx);
1529 vm_pageout_scan(pass);
1530 }
1531}
1532
1533/*
1534 * Unless the free page queue lock is held by the caller, this function
1535 * should be regarded as advisory. Specifically, the caller should

--- 102 unchanged lines hidden --- 		1528 mtx_unlock(&vm_page_queue_free_mtx);
1529 vm_pageout_scan(pass);
1530 }
1531}
1532
1533/*
1534 * Unless the free page queue lock is held by the caller, this function
1535 * should be regarded as advisory. Specifically, the caller should

--- 102 unchanged lines hidden ---