truncate.c (8a549bea51138be2126a2cc6aabe8f17ef66b79b) truncate.c (b85e0effd3dcbf9118b896232f59526ab1a39a74)
1/*
2 * mm/truncate.c - code for taking down pages from address_spaces
3 *
4 * Copyright (C) 2002, Linus Torvalds
5 *
6 * 10Sep2002 Andrew Morton
7 * Initial version.
8 */

--- 185 unchanged lines hidden (view full) ---

194 * (if lstart is not page aligned)).
195 *
196 * Truncate takes two passes - the first pass is nonblocking. It will not
197 * block on page locks and it will not block on writeback. The second pass
198 * will wait. This is to prevent as much IO as possible in the affected region.
199 * The first pass will remove most pages, so the search cost of the second pass
200 * is low.
201 *
1/*
2 * mm/truncate.c - code for taking down pages from address_spaces
3 *
4 * Copyright (C) 2002, Linus Torvalds
5 *
6 * 10Sep2002 Andrew Morton
7 * Initial version.
8 */

--- 185 unchanged lines hidden (view full) ---

194 * (if lstart is not page aligned)).
195 *
196 * Truncate takes two passes - the first pass is nonblocking. It will not
197 * block on page locks and it will not block on writeback. The second pass
198 * will wait. This is to prevent as much IO as possible in the affected region.
199 * The first pass will remove most pages, so the search cost of the second pass
200 * is low.
201 *
202 * When looking at page->index outside the page lock we need to be careful to
203 * copy it into a local to avoid races (it could change at any time).
204 *
205 * We pass down the cache-hot hint to the page freeing code. Even if the
206 * mapping is large, it is probably the case that the final pages are the most
207 * recently touched, and freeing happens in ascending file offset order.
208 */
209void truncate_inode_pages_range(struct address_space *mapping,
210 loff_t lstart, loff_t lend)
211{
212 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
202 * We pass down the cache-hot hint to the page freeing code. Even if the
203 * mapping is large, it is probably the case that the final pages are the most
204 * recently touched, and freeing happens in ascending file offset order.
205 */
206void truncate_inode_pages_range(struct address_space *mapping,
207 loff_t lstart, loff_t lend)
208{
209 const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
213 pgoff_t end;
214 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
215 struct pagevec pvec;
210 const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
211 struct pagevec pvec;
216 pgoff_t next;
212 pgoff_t index;
213 pgoff_t end;
217 int i;
218
219 cleancache_flush_inode(mapping);
220 if (mapping->nrpages == 0)
221 return;
222
223 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
224 end = (lend >> PAGE_CACHE_SHIFT);
225
226 pagevec_init(&pvec, 0);
214 int i;
215
216 cleancache_flush_inode(mapping);
217 if (mapping->nrpages == 0)
218 return;
219
220 BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
221 end = (lend >> PAGE_CACHE_SHIFT);
222
223 pagevec_init(&pvec, 0);
227 next = start;
228 while (next <= end &&
229 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
224 index = start;
225 while (index <= end && pagevec_lookup(&pvec, mapping, index,
226 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
230 mem_cgroup_uncharge_start();
231 for (i = 0; i < pagevec_count(&pvec); i++) {
232 struct page *page = pvec.pages[i];
227 mem_cgroup_uncharge_start();
228 for (i = 0; i < pagevec_count(&pvec); i++) {
229 struct page *page = pvec.pages[i];
233 pgoff_t page_index = page->index;
234
230
235 if (page_index > end) {
236 next = page_index;
231 /* We rely upon deletion not changing page->index */
232 index = page->index;
233 if (index > end)
237 break;
234 break;
238 }
239
235
240 if (page_index > next)
241 next = page_index;
242 next++;
243 if (!trylock_page(page))
244 continue;
236 if (!trylock_page(page))
237 continue;
238 WARN_ON(page->index != index);
245 if (PageWriteback(page)) {
246 unlock_page(page);
247 continue;
248 }
249 truncate_inode_page(mapping, page);
250 unlock_page(page);
251 }
252 pagevec_release(&pvec);
253 mem_cgroup_uncharge_end();
254 cond_resched();
239 if (PageWriteback(page)) {
240 unlock_page(page);
241 continue;
242 }
243 truncate_inode_page(mapping, page);
244 unlock_page(page);
245 }
246 pagevec_release(&pvec);
247 mem_cgroup_uncharge_end();
248 cond_resched();
249 index++;
255 }
256
257 if (partial) {
258 struct page *page = find_lock_page(mapping, start - 1);
259 if (page) {
260 wait_on_page_writeback(page);
261 truncate_partial_page(page, partial);
262 unlock_page(page);
263 page_cache_release(page);
264 }
265 }
266
250 }
251
252 if (partial) {
253 struct page *page = find_lock_page(mapping, start - 1);
254 if (page) {
255 wait_on_page_writeback(page);
256 truncate_partial_page(page, partial);
257 unlock_page(page);
258 page_cache_release(page);
259 }
260 }
261
267 next = start;
262 index = start;
268 for ( ; ; ) {
269 cond_resched();
263 for ( ; ; ) {
264 cond_resched();
270 if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
271 if (next == start)
265 if (!pagevec_lookup(&pvec, mapping, index,
266 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
267 if (index == start)
272 break;
268 break;
273 next = start;
269 index = start;
274 continue;
275 }
276 if (pvec.pages[0]->index > end) {
277 pagevec_release(&pvec);
278 break;
279 }
280 mem_cgroup_uncharge_start();
281 for (i = 0; i < pagevec_count(&pvec); i++) {
282 struct page *page = pvec.pages[i];
283
270 continue;
271 }
272 if (pvec.pages[0]->index > end) {
273 pagevec_release(&pvec);
274 break;
275 }
276 mem_cgroup_uncharge_start();
277 for (i = 0; i < pagevec_count(&pvec); i++) {
278 struct page *page = pvec.pages[i];
279
284 if (page->index > end)
280 /* We rely upon deletion not changing page->index */
281 index = page->index;
282 if (index > end)
285 break;
283 break;
284
286 lock_page(page);
285 lock_page(page);
286 WARN_ON(page->index != index);
287 wait_on_page_writeback(page);
288 truncate_inode_page(mapping, page);
287 wait_on_page_writeback(page);
288 truncate_inode_page(mapping, page);
289 if (page->index > next)
290 next = page->index;
291 next++;
292 unlock_page(page);
293 }
294 pagevec_release(&pvec);
295 mem_cgroup_uncharge_end();
289 unlock_page(page);
290 }
291 pagevec_release(&pvec);
292 mem_cgroup_uncharge_end();
293 index++;
296 }
297 cleancache_flush_inode(mapping);
298}
299EXPORT_SYMBOL(truncate_inode_pages_range);
300
301/**
302 * truncate_inode_pages - truncate *all* the pages from an offset
303 * @mapping: mapping to truncate

--- 24 unchanged lines hidden (view full) ---

328 * invalidate_mapping_pages() will not block on IO activity. It will not
329 * invalidate pages which are dirty, locked, under writeback or mapped into
330 * pagetables.
331 */
332unsigned long invalidate_mapping_pages(struct address_space *mapping,
333 pgoff_t start, pgoff_t end)
334{
335 struct pagevec pvec;
294 }
295 cleancache_flush_inode(mapping);
296}
297EXPORT_SYMBOL(truncate_inode_pages_range);
298
299/**
300 * truncate_inode_pages - truncate *all* the pages from an offset
301 * @mapping: mapping to truncate

--- 24 unchanged lines hidden (view full) ---

326 * invalidate_mapping_pages() will not block on IO activity. It will not
327 * invalidate pages which are dirty, locked, under writeback or mapped into
328 * pagetables.
329 */
330unsigned long invalidate_mapping_pages(struct address_space *mapping,
331 pgoff_t start, pgoff_t end)
332{
333 struct pagevec pvec;
336 pgoff_t next = start;
334 pgoff_t index = start;
337 unsigned long ret;
338 unsigned long count = 0;
339 int i;
340
341 pagevec_init(&pvec, 0);
335 unsigned long ret;
336 unsigned long count = 0;
337 int i;
338
339 pagevec_init(&pvec, 0);
342 while (next <= end &&
343 pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
340 while (index <= end && pagevec_lookup(&pvec, mapping, index,
341 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
344 mem_cgroup_uncharge_start();
345 for (i = 0; i < pagevec_count(&pvec); i++) {
346 struct page *page = pvec.pages[i];
342 mem_cgroup_uncharge_start();
343 for (i = 0; i < pagevec_count(&pvec); i++) {
344 struct page *page = pvec.pages[i];
347 pgoff_t index;
348 int lock_failed;
349
345
350 lock_failed = !trylock_page(page);
351
352 /*
353 * We really shouldn't be looking at the ->index of an
354 * unlocked page. But we're not allowed to lock these
355 * pages. So we rely upon nobody altering the ->index
356 * of this (pinned-by-us) page.
357 */
346 /* We rely upon deletion not changing page->index */
358 index = page->index;
347 index = page->index;
359 if (index > next)
360 next = index;
361 next++;
362 if (lock_failed)
363 continue;
348 if (index > end)
349 break;
364
350
351 if (!trylock_page(page))
352 continue;
353 WARN_ON(page->index != index);
365 ret = invalidate_inode_page(page);
366 unlock_page(page);
367 /*
368 * Invalidation is a hint that the page is no longer
369 * of interest and try to speed up its reclaim.
370 */
371 if (!ret)
372 deactivate_page(page);
373 count += ret;
354 ret = invalidate_inode_page(page);
355 unlock_page(page);
356 /*
357 * Invalidation is a hint that the page is no longer
358 * of interest and try to speed up its reclaim.
359 */
360 if (!ret)
361 deactivate_page(page);
362 count += ret;
374 if (next > end)
375 break;
376 }
377 pagevec_release(&pvec);
378 mem_cgroup_uncharge_end();
379 cond_resched();
363 }
364 pagevec_release(&pvec);
365 mem_cgroup_uncharge_end();
366 cond_resched();
367 index++;
380 }
381 return count;
382}
383EXPORT_SYMBOL(invalidate_mapping_pages);
384
385/*
386 * This is like invalidate_complete_page(), except it ignores the page's
387 * refcount. We do this because invalidate_inode_pages2() needs stronger

--- 49 unchanged lines hidden (view full) ---

437 * invalidation.
438 *
439 * Returns -EBUSY if any pages could not be invalidated.
440 */
441int invalidate_inode_pages2_range(struct address_space *mapping,
442 pgoff_t start, pgoff_t end)
443{
444 struct pagevec pvec;
368 }
369 return count;
370}
371EXPORT_SYMBOL(invalidate_mapping_pages);
372
373/*
374 * This is like invalidate_complete_page(), except it ignores the page's
375 * refcount. We do this because invalidate_inode_pages2() needs stronger

--- 49 unchanged lines hidden (view full) ---

425 * invalidation.
426 *
427 * Returns -EBUSY if any pages could not be invalidated.
428 */
429int invalidate_inode_pages2_range(struct address_space *mapping,
430 pgoff_t start, pgoff_t end)
431{
432 struct pagevec pvec;
445 pgoff_t next;
433 pgoff_t index;
446 int i;
447 int ret = 0;
448 int ret2 = 0;
449 int did_range_unmap = 0;
434 int i;
435 int ret = 0;
436 int ret2 = 0;
437 int did_range_unmap = 0;
450 int wrapped = 0;
451
452 cleancache_flush_inode(mapping);
453 pagevec_init(&pvec, 0);
438
439 cleancache_flush_inode(mapping);
440 pagevec_init(&pvec, 0);
454 next = start;
455 while (next <= end && !wrapped &&
456 pagevec_lookup(&pvec, mapping, next,
457 min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
441 index = start;
442 while (index <= end && pagevec_lookup(&pvec, mapping, index,
443 min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
458 mem_cgroup_uncharge_start();
459 for (i = 0; i < pagevec_count(&pvec); i++) {
460 struct page *page = pvec.pages[i];
444 mem_cgroup_uncharge_start();
445 for (i = 0; i < pagevec_count(&pvec); i++) {
446 struct page *page = pvec.pages[i];
461 pgoff_t page_index;
462
447
448 /* We rely upon deletion not changing page->index */
449 index = page->index;
450 if (index > end)
451 break;
452
463 lock_page(page);
453 lock_page(page);
454 WARN_ON(page->index != index);
464 if (page->mapping != mapping) {
465 unlock_page(page);
466 continue;
467 }
455 if (page->mapping != mapping) {
456 unlock_page(page);
457 continue;
458 }
468 page_index = page->index;
469 next = page_index + 1;
470 if (next == 0)
471 wrapped = 1;
472 if (page_index > end) {
473 unlock_page(page);
474 break;
475 }
476 wait_on_page_writeback(page);
477 if (page_mapped(page)) {
478 if (!did_range_unmap) {
479 /*
480 * Zap the rest of the file in one hit.
481 */
482 unmap_mapping_range(mapping,
459 wait_on_page_writeback(page);
460 if (page_mapped(page)) {
461 if (!did_range_unmap) {
462 /*
463 * Zap the rest of the file in one hit.
464 */
465 unmap_mapping_range(mapping,
483 (loff_t)page_index<<PAGE_CACHE_SHIFT,
484 (loff_t)(end - page_index + 1)
485 << PAGE_CACHE_SHIFT,
466 (loff_t)index << PAGE_CACHE_SHIFT,
467 (loff_t)(1 + end - index)
468 << PAGE_CACHE_SHIFT,
486 0);
487 did_range_unmap = 1;
488 } else {
489 /*
490 * Just zap this page
491 */
492 unmap_mapping_range(mapping,
469 0);
470 did_range_unmap = 1;
471 } else {
472 /*
473 * Just zap this page
474 */
475 unmap_mapping_range(mapping,
493 (loff_t)page_index<<PAGE_CACHE_SHIFT,
494 PAGE_CACHE_SIZE, 0);
476 (loff_t)index << PAGE_CACHE_SHIFT,
477 PAGE_CACHE_SIZE, 0);
495 }
496 }
497 BUG_ON(page_mapped(page));
498 ret2 = do_launder_page(mapping, page);
499 if (ret2 == 0) {
500 if (!invalidate_complete_page2(mapping, page))
501 ret2 = -EBUSY;
502 }
503 if (ret2 < 0)
504 ret = ret2;
505 unlock_page(page);
506 }
507 pagevec_release(&pvec);
508 mem_cgroup_uncharge_end();
509 cond_resched();
478 }
479 }
480 BUG_ON(page_mapped(page));
481 ret2 = do_launder_page(mapping, page);
482 if (ret2 == 0) {
483 if (!invalidate_complete_page2(mapping, page))
484 ret2 = -EBUSY;
485 }
486 if (ret2 < 0)
487 ret = ret2;
488 unlock_page(page);
489 }
490 pagevec_release(&pvec);
491 mem_cgroup_uncharge_end();
492 cond_resched();
493 index++;
510 }
511 cleancache_flush_inode(mapping);
512 return ret;
513}
514EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
515
516/**
517 * invalidate_inode_pages2 - remove all pages from an address_space

--- 119 unchanged lines hidden --- 		494 }
495 cleancache_flush_inode(mapping);
496 return ret;
497}
498EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
499
500/**
501 * invalidate_inode_pages2 - remove all pages from an address_space

--- 119 unchanged lines hidden ---