1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Copyright (C) International Business Machines Corp., 2000-2005
4 * Portions Copyright (C) Christoph Hellwig, 2001-2002
5 */
6
7 #include <linux/blkdev.h>
8 #include <linux/fs.h>
9 #include <linux/mm.h>
10 #include <linux/module.h>
11 #include <linux/bio.h>
12 #include <linux/slab.h>
13 #include <linux/init.h>
14 #include <linux/buffer_head.h>
15 #include <linux/mempool.h>
16 #include <linux/seq_file.h>
17 #include <linux/writeback.h>
18 #include <linux/migrate.h>
19 #include "jfs_incore.h"
20 #include "jfs_superblock.h"
21 #include "jfs_filsys.h"
22 #include "jfs_metapage.h"
23 #include "jfs_txnmgr.h"
24 #include "jfs_debug.h"
25
26 #ifdef CONFIG_JFS_STATISTICS
27 static struct {
28 uint pagealloc; /* # of page allocations */
29 uint pagefree; /* # of page frees */
30 uint lockwait; /* # of sleeping lock_metapage() calls */
31 } mpStat;
32 #endif
33
34 #define metapage_locked(mp) test_bit(META_locked, &(mp)->flag)
35 #define trylock_metapage(mp) test_and_set_bit_lock(META_locked, &(mp)->flag)
36
unlock_metapage(struct metapage * mp)37 static inline void unlock_metapage(struct metapage *mp)
38 {
39 clear_bit_unlock(META_locked, &mp->flag);
40 wake_up(&mp->wait);
41 }
42
__lock_metapage(struct metapage * mp)43 static inline void __lock_metapage(struct metapage *mp)
44 {
45 DECLARE_WAITQUEUE(wait, current);
46 INCREMENT(mpStat.lockwait);
47 add_wait_queue_exclusive(&mp->wait, &wait);
48 do {
49 set_current_state(TASK_UNINTERRUPTIBLE);
50 if (metapage_locked(mp)) {
51 folio_unlock(mp->folio);
52 io_schedule();
53 folio_lock(mp->folio);
54 }
55 } while (trylock_metapage(mp));
56 __set_current_state(TASK_RUNNING);
57 remove_wait_queue(&mp->wait, &wait);
58 }
59
60 /*
61 * Must have mp->folio locked
62 */
lock_metapage(struct metapage * mp)63 static inline void lock_metapage(struct metapage *mp)
64 {
65 if (trylock_metapage(mp))
66 __lock_metapage(mp);
67 }
68
69 #define METAPOOL_MIN_PAGES 32
70 static struct kmem_cache *metapage_cache;
71 static mempool_t *metapage_mempool;
72
73 #define MPS_PER_PAGE (PAGE_SIZE >> L2PSIZE)
74
75 #if MPS_PER_PAGE > 1
76
77 struct meta_anchor {
78 int mp_count;
79 atomic_t io_count;
80 blk_status_t status;
81 struct metapage *mp[MPS_PER_PAGE];
82 };
83
folio_to_mp(struct folio * folio,int offset)84 static inline struct metapage *folio_to_mp(struct folio *folio, int offset)
85 {
86 struct meta_anchor *anchor = folio->private;
87
88 if (!anchor)
89 return NULL;
90 return anchor->mp[offset >> L2PSIZE];
91 }
92
insert_metapage(struct folio * folio,struct metapage * mp)93 static inline int insert_metapage(struct folio *folio, struct metapage *mp)
94 {
95 struct meta_anchor *a;
96 int index;
97 int l2mp_blocks; /* log2 blocks per metapage */
98
99 a = folio->private;
100 if (!a) {
101 a = kzalloc(sizeof(struct meta_anchor), GFP_NOFS);
102 if (!a)
103 return -ENOMEM;
104 folio_attach_private(folio, a);
105 kmap(&folio->page);
106 }
107
108 if (mp) {
109 l2mp_blocks = L2PSIZE - folio->mapping->host->i_blkbits;
110 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1);
111 a->mp_count++;
112 a->mp[index] = mp;
113 }
114
115 return 0;
116 }
117
remove_metapage(struct folio * folio,struct metapage * mp)118 static inline void remove_metapage(struct folio *folio, struct metapage *mp)
119 {
120 struct meta_anchor *a = folio->private;
121 int l2mp_blocks = L2PSIZE - folio->mapping->host->i_blkbits;
122 int index;
123
124 index = (mp->index >> l2mp_blocks) & (MPS_PER_PAGE - 1);
125
126 BUG_ON(a->mp[index] != mp);
127
128 a->mp[index] = NULL;
129 if (--a->mp_count == 0) {
130 kfree(a);
131 folio_detach_private(folio);
132 kunmap(&folio->page);
133 }
134 }
135
inc_io(struct folio * folio)136 static inline void inc_io(struct folio *folio)
137 {
138 struct meta_anchor *anchor = folio->private;
139
140 atomic_inc(&anchor->io_count);
141 }
142
dec_io(struct folio * folio,blk_status_t status,void (* handler)(struct folio *,blk_status_t))143 static inline void dec_io(struct folio *folio, blk_status_t status,
144 void (*handler)(struct folio *, blk_status_t))
145 {
146 struct meta_anchor *anchor = folio->private;
147
148 if (anchor->status == BLK_STS_OK)
149 anchor->status = status;
150
151 if (atomic_dec_and_test(&anchor->io_count))
152 handler(folio, anchor->status);
153 }
154
155 #ifdef CONFIG_MIGRATION
__metapage_migrate_folio(struct address_space * mapping,struct folio * dst,struct folio * src,enum migrate_mode mode)156 static int __metapage_migrate_folio(struct address_space *mapping,
157 struct folio *dst, struct folio *src,
158 enum migrate_mode mode)
159 {
160 struct meta_anchor *src_anchor = src->private;
161 struct metapage *mps[MPS_PER_PAGE] = {0};
162 struct metapage *mp;
163 int i, rc;
164
165 for (i = 0; i < MPS_PER_PAGE; i++) {
166 mp = src_anchor->mp[i];
167 if (mp && metapage_locked(mp))
168 return -EAGAIN;
169 }
170
171 rc = filemap_migrate_folio(mapping, dst, src, mode);
172 if (rc != MIGRATEPAGE_SUCCESS)
173 return rc;
174
175 for (i = 0; i < MPS_PER_PAGE; i++) {
176 mp = src_anchor->mp[i];
177 if (!mp)
178 continue;
179 if (unlikely(insert_metapage(dst, mp))) {
180 /* If error, roll-back previosly inserted pages */
181 for (int j = 0 ; j < i; j++) {
182 if (mps[j])
183 remove_metapage(dst, mps[j]);
184 }
185 return -EAGAIN;
186 }
187 mps[i] = mp;
188 }
189
190 /* Update the metapage and remove it from src */
191 for (i = 0; i < MPS_PER_PAGE; i++) {
192 mp = mps[i];
193 if (mp) {
194 int page_offset = mp->data - folio_address(src);
195
196 mp->data = folio_address(dst) + page_offset;
197 mp->folio = dst;
198 remove_metapage(src, mp);
199 }
200 }
201
202 return MIGRATEPAGE_SUCCESS;
203 }
204 #endif /* CONFIG_MIGRATION */
205
206 #else
207
folio_to_mp(struct folio * folio,int offset)208 static inline struct metapage *folio_to_mp(struct folio *folio, int offset)
209 {
210 return folio->private;
211 }
212
insert_metapage(struct folio * folio,struct metapage * mp)213 static inline int insert_metapage(struct folio *folio, struct metapage *mp)
214 {
215 if (mp) {
216 folio_attach_private(folio, mp);
217 kmap(&folio->page);
218 }
219 return 0;
220 }
221
remove_metapage(struct folio * folio,struct metapage * mp)222 static inline void remove_metapage(struct folio *folio, struct metapage *mp)
223 {
224 folio_detach_private(folio);
225 kunmap(&folio->page);
226 }
227
228 #define inc_io(folio) do {} while(0)
229 #define dec_io(folio, status, handler) handler(folio, status)
230
231 #ifdef CONFIG_MIGRATION
__metapage_migrate_folio(struct address_space * mapping,struct folio * dst,struct folio * src,enum migrate_mode mode)232 static int __metapage_migrate_folio(struct address_space *mapping,
233 struct folio *dst, struct folio *src,
234 enum migrate_mode mode)
235 {
236 struct metapage *mp;
237 int page_offset;
238 int rc;
239
240 mp = folio_to_mp(src, 0);
241 if (metapage_locked(mp))
242 return -EAGAIN;
243
244 rc = filemap_migrate_folio(mapping, dst, src, mode);
245 if (rc != MIGRATEPAGE_SUCCESS)
246 return rc;
247
248 if (unlikely(insert_metapage(dst, mp)))
249 return -EAGAIN;
250
251 page_offset = mp->data - folio_address(src);
252 mp->data = folio_address(dst) + page_offset;
253 mp->folio = dst;
254 remove_metapage(src, mp);
255
256 return MIGRATEPAGE_SUCCESS;
257 }
258 #endif /* CONFIG_MIGRATION */
259
260 #endif
261
alloc_metapage(gfp_t gfp_mask)262 static inline struct metapage *alloc_metapage(gfp_t gfp_mask)
263 {
264 struct metapage *mp = mempool_alloc(metapage_mempool, gfp_mask);
265
266 if (mp) {
267 mp->lid = 0;
268 mp->lsn = 0;
269 mp->data = NULL;
270 mp->clsn = 0;
271 mp->log = NULL;
272 init_waitqueue_head(&mp->wait);
273 }
274 return mp;
275 }
276
free_metapage(struct metapage * mp)277 static inline void free_metapage(struct metapage *mp)
278 {
279 mempool_free(mp, metapage_mempool);
280 }
281
metapage_init(void)282 int __init metapage_init(void)
283 {
284 /*
285 * Allocate the metapage structures
286 */
287 metapage_cache = kmem_cache_create("jfs_mp", sizeof(struct metapage),
288 0, 0, NULL);
289 if (metapage_cache == NULL)
290 return -ENOMEM;
291
292 metapage_mempool = mempool_create_slab_pool(METAPOOL_MIN_PAGES,
293 metapage_cache);
294
295 if (metapage_mempool == NULL) {
296 kmem_cache_destroy(metapage_cache);
297 return -ENOMEM;
298 }
299
300 return 0;
301 }
302
metapage_exit(void)303 void metapage_exit(void)
304 {
305 mempool_destroy(metapage_mempool);
306 kmem_cache_destroy(metapage_cache);
307 }
308
drop_metapage(struct folio * folio,struct metapage * mp)309 static inline void drop_metapage(struct folio *folio, struct metapage *mp)
310 {
311 if (mp->count || mp->nohomeok || test_bit(META_dirty, &mp->flag) ||
312 test_bit(META_io, &mp->flag))
313 return;
314 remove_metapage(folio, mp);
315 INCREMENT(mpStat.pagefree);
316 free_metapage(mp);
317 }
318
319 /*
320 * Metapage address space operations
321 */
322
metapage_get_blocks(struct inode * inode,sector_t lblock,int * len)323 static sector_t metapage_get_blocks(struct inode *inode, sector_t lblock,
324 int *len)
325 {
326 int rc = 0;
327 int xflag;
328 s64 xaddr;
329 sector_t file_blocks = (inode->i_size + inode->i_sb->s_blocksize - 1) >>
330 inode->i_blkbits;
331
332 if (lblock >= file_blocks)
333 return 0;
334 if (lblock + *len > file_blocks)
335 *len = file_blocks - lblock;
336
337 if (inode->i_ino) {
338 rc = xtLookup(inode, (s64)lblock, *len, &xflag, &xaddr, len, 0);
339 if ((rc == 0) && *len)
340 lblock = (sector_t)xaddr;
341 else
342 lblock = 0;
343 } /* else no mapping */
344
345 return lblock;
346 }
347
last_read_complete(struct folio * folio,blk_status_t status)348 static void last_read_complete(struct folio *folio, blk_status_t status)
349 {
350 if (status)
351 printk(KERN_ERR "Read error %d at %#llx\n", status,
352 folio_pos(folio));
353
354 folio_end_read(folio, status == 0);
355 }
356
metapage_read_end_io(struct bio * bio)357 static void metapage_read_end_io(struct bio *bio)
358 {
359 struct folio *folio = bio->bi_private;
360
361 dec_io(folio, bio->bi_status, last_read_complete);
362 bio_put(bio);
363 }
364
remove_from_logsync(struct metapage * mp)365 static void remove_from_logsync(struct metapage *mp)
366 {
367 struct jfs_log *log = mp->log;
368 unsigned long flags;
369 /*
370 * This can race. Recheck that log hasn't been set to null, and after
371 * acquiring logsync lock, recheck lsn
372 */
373 if (!log)
374 return;
375
376 LOGSYNC_LOCK(log, flags);
377 if (mp->lsn) {
378 mp->log = NULL;
379 mp->lsn = 0;
380 mp->clsn = 0;
381 log->count--;
382 list_del(&mp->synclist);
383 }
384 LOGSYNC_UNLOCK(log, flags);
385 }
386
last_write_complete(struct folio * folio,blk_status_t status)387 static void last_write_complete(struct folio *folio, blk_status_t status)
388 {
389 struct metapage *mp;
390 unsigned int offset;
391
392 if (status) {
393 int err = blk_status_to_errno(status);
394 printk(KERN_ERR "metapage_write_end_io: I/O error\n");
395 mapping_set_error(folio->mapping, err);
396 }
397
398 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
399 mp = folio_to_mp(folio, offset);
400 if (mp && test_bit(META_io, &mp->flag)) {
401 if (mp->lsn)
402 remove_from_logsync(mp);
403 clear_bit(META_io, &mp->flag);
404 }
405 /*
406 * I'd like to call drop_metapage here, but I don't think it's
407 * safe unless I have the page locked
408 */
409 }
410 folio_end_writeback(folio);
411 }
412
metapage_write_end_io(struct bio * bio)413 static void metapage_write_end_io(struct bio *bio)
414 {
415 struct folio *folio = bio->bi_private;
416
417 BUG_ON(!folio->private);
418
419 dec_io(folio, bio->bi_status, last_write_complete);
420 bio_put(bio);
421 }
422
metapage_write_folio(struct folio * folio,struct writeback_control * wbc,void * unused)423 static int metapage_write_folio(struct folio *folio,
424 struct writeback_control *wbc, void *unused)
425 {
426 struct bio *bio = NULL;
427 int block_offset; /* block offset of mp within page */
428 struct inode *inode = folio->mapping->host;
429 int blocks_per_mp = JFS_SBI(inode->i_sb)->nbperpage;
430 int len;
431 int xlen;
432 struct metapage *mp;
433 int redirty = 0;
434 sector_t lblock;
435 int nr_underway = 0;
436 sector_t pblock;
437 sector_t next_block = 0;
438 sector_t page_start;
439 unsigned long bio_bytes = 0;
440 unsigned long bio_offset = 0;
441 int offset;
442 int bad_blocks = 0;
443
444 page_start = folio_pos(folio) >> inode->i_blkbits;
445 BUG_ON(!folio_test_locked(folio));
446 BUG_ON(folio_test_writeback(folio));
447 folio_start_writeback(folio);
448
449 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
450 mp = folio_to_mp(folio, offset);
451
452 if (!mp || !test_bit(META_dirty, &mp->flag))
453 continue;
454
455 if (mp->nohomeok && !test_bit(META_forcewrite, &mp->flag)) {
456 redirty = 1;
457 /*
458 * Make sure this page isn't blocked indefinitely.
459 * If the journal isn't undergoing I/O, push it
460 */
461 if (mp->log && !(mp->log->cflag & logGC_PAGEOUT))
462 jfs_flush_journal(mp->log, 0);
463 continue;
464 }
465
466 clear_bit(META_dirty, &mp->flag);
467 set_bit(META_io, &mp->flag);
468 block_offset = offset >> inode->i_blkbits;
469 lblock = page_start + block_offset;
470 if (bio) {
471 if (xlen && lblock == next_block) {
472 /* Contiguous, in memory & on disk */
473 len = min(xlen, blocks_per_mp);
474 xlen -= len;
475 bio_bytes += len << inode->i_blkbits;
476 continue;
477 }
478 /* Not contiguous */
479 bio_add_folio_nofail(bio, folio, bio_bytes, bio_offset);
480 /*
481 * Increment counter before submitting i/o to keep
482 * count from hitting zero before we're through
483 */
484 inc_io(folio);
485 if (!bio->bi_iter.bi_size)
486 goto dump_bio;
487 submit_bio(bio);
488 nr_underway++;
489 bio = NULL;
490 } else
491 inc_io(folio);
492 xlen = (folio_size(folio) - offset) >> inode->i_blkbits;
493 pblock = metapage_get_blocks(inode, lblock, &xlen);
494 if (!pblock) {
495 printk(KERN_ERR "JFS: metapage_get_blocks failed\n");
496 /*
497 * We already called inc_io(), but can't cancel it
498 * with dec_io() until we're done with the page
499 */
500 bad_blocks++;
501 continue;
502 }
503 len = min(xlen, (int)JFS_SBI(inode->i_sb)->nbperpage);
504
505 bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_WRITE, GFP_NOFS);
506 bio->bi_iter.bi_sector = pblock << (inode->i_blkbits - 9);
507 bio->bi_end_io = metapage_write_end_io;
508 bio->bi_private = folio;
509
510 /* Don't call bio_add_page yet, we may add to this vec */
511 bio_offset = offset;
512 bio_bytes = len << inode->i_blkbits;
513
514 xlen -= len;
515 next_block = lblock + len;
516 }
517 if (bio) {
518 bio_add_folio_nofail(bio, folio, bio_bytes, bio_offset);
519 if (!bio->bi_iter.bi_size)
520 goto dump_bio;
521
522 submit_bio(bio);
523 nr_underway++;
524 }
525 if (redirty)
526 folio_redirty_for_writepage(wbc, folio);
527
528 folio_unlock(folio);
529
530 if (bad_blocks)
531 goto err_out;
532
533 if (nr_underway == 0)
534 folio_end_writeback(folio);
535
536 return 0;
537 dump_bio:
538 print_hex_dump(KERN_ERR, "JFS: dump of bio: ", DUMP_PREFIX_ADDRESS, 16,
539 4, bio, sizeof(*bio), 0);
540 bio_put(bio);
541 folio_unlock(folio);
542 dec_io(folio, BLK_STS_OK, last_write_complete);
543 err_out:
544 while (bad_blocks--)
545 dec_io(folio, BLK_STS_OK, last_write_complete);
546 return -EIO;
547 }
548
metapage_writepages(struct address_space * mapping,struct writeback_control * wbc)549 static int metapage_writepages(struct address_space *mapping,
550 struct writeback_control *wbc)
551 {
552 struct blk_plug plug;
553 int err;
554
555 blk_start_plug(&plug);
556 err = write_cache_pages(mapping, wbc, metapage_write_folio, NULL);
557 blk_finish_plug(&plug);
558
559 return err;
560 }
561
metapage_read_folio(struct file * fp,struct folio * folio)562 static int metapage_read_folio(struct file *fp, struct folio *folio)
563 {
564 struct inode *inode = folio->mapping->host;
565 struct bio *bio = NULL;
566 int block_offset;
567 int blocks_per_page = i_blocks_per_folio(inode, folio);
568 sector_t page_start; /* address of page in fs blocks */
569 sector_t pblock;
570 int xlen;
571 unsigned int len;
572 int offset;
573
574 BUG_ON(!folio_test_locked(folio));
575 page_start = folio_pos(folio) >> inode->i_blkbits;
576
577 block_offset = 0;
578 while (block_offset < blocks_per_page) {
579 xlen = blocks_per_page - block_offset;
580 pblock = metapage_get_blocks(inode, page_start + block_offset,
581 &xlen);
582 if (pblock) {
583 if (!folio->private)
584 insert_metapage(folio, NULL);
585 inc_io(folio);
586 if (bio)
587 submit_bio(bio);
588
589 bio = bio_alloc(inode->i_sb->s_bdev, 1, REQ_OP_READ,
590 GFP_NOFS);
591 bio->bi_iter.bi_sector =
592 pblock << (inode->i_blkbits - 9);
593 bio->bi_end_io = metapage_read_end_io;
594 bio->bi_private = folio;
595 len = xlen << inode->i_blkbits;
596 offset = block_offset << inode->i_blkbits;
597 bio_add_folio_nofail(bio, folio, len, offset);
598 block_offset += xlen;
599 } else
600 block_offset++;
601 }
602 if (bio)
603 submit_bio(bio);
604 else
605 folio_unlock(folio);
606
607 return 0;
608 }
609
metapage_release_folio(struct folio * folio,gfp_t gfp_mask)610 static bool metapage_release_folio(struct folio *folio, gfp_t gfp_mask)
611 {
612 struct metapage *mp;
613 bool ret = true;
614 int offset;
615
616 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
617 mp = folio_to_mp(folio, offset);
618
619 if (!mp)
620 continue;
621
622 jfs_info("metapage_release_folio: mp = 0x%p", mp);
623 if (mp->count || mp->nohomeok ||
624 test_bit(META_dirty, &mp->flag)) {
625 jfs_info("count = %ld, nohomeok = %d", mp->count,
626 mp->nohomeok);
627 ret = false;
628 continue;
629 }
630 if (mp->lsn)
631 remove_from_logsync(mp);
632 remove_metapage(folio, mp);
633 INCREMENT(mpStat.pagefree);
634 free_metapage(mp);
635 }
636 return ret;
637 }
638
639 #ifdef CONFIG_MIGRATION
640 /*
641 * metapage_migrate_folio - Migration function for JFS metapages
642 */
metapage_migrate_folio(struct address_space * mapping,struct folio * dst,struct folio * src,enum migrate_mode mode)643 static int metapage_migrate_folio(struct address_space *mapping,
644 struct folio *dst, struct folio *src,
645 enum migrate_mode mode)
646 {
647 int expected_count;
648
649 if (!src->private)
650 return filemap_migrate_folio(mapping, dst, src, mode);
651
652 /* Check whether page does not have extra refs before we do more work */
653 expected_count = folio_expected_ref_count(src) + 1;
654 if (folio_ref_count(src) != expected_count)
655 return -EAGAIN;
656 return __metapage_migrate_folio(mapping, dst, src, mode);
657 }
658 #else
659 #define metapage_migrate_folio NULL
660 #endif /* CONFIG_MIGRATION */
661
metapage_invalidate_folio(struct folio * folio,size_t offset,size_t length)662 static void metapage_invalidate_folio(struct folio *folio, size_t offset,
663 size_t length)
664 {
665 BUG_ON(offset || length < folio_size(folio));
666
667 BUG_ON(folio_test_writeback(folio));
668
669 metapage_release_folio(folio, 0);
670 }
671
672 const struct address_space_operations jfs_metapage_aops = {
673 .read_folio = metapage_read_folio,
674 .writepages = metapage_writepages,
675 .release_folio = metapage_release_folio,
676 .invalidate_folio = metapage_invalidate_folio,
677 .dirty_folio = filemap_dirty_folio,
678 .migrate_folio = metapage_migrate_folio,
679 };
680
__get_metapage(struct inode * inode,unsigned long lblock,unsigned int size,int absolute,unsigned long new)681 struct metapage *__get_metapage(struct inode *inode, unsigned long lblock,
682 unsigned int size, int absolute,
683 unsigned long new)
684 {
685 int l2BlocksPerPage;
686 int l2bsize;
687 struct address_space *mapping;
688 struct metapage *mp = NULL;
689 struct folio *folio;
690 unsigned long page_index;
691 unsigned long page_offset;
692
693 jfs_info("__get_metapage: ino = %ld, lblock = 0x%lx, abs=%d",
694 inode->i_ino, lblock, absolute);
695
696 l2bsize = inode->i_blkbits;
697 l2BlocksPerPage = PAGE_SHIFT - l2bsize;
698 page_index = lblock >> l2BlocksPerPage;
699 page_offset = (lblock - (page_index << l2BlocksPerPage)) << l2bsize;
700 if ((page_offset + size) > PAGE_SIZE) {
701 jfs_err("MetaData crosses page boundary!!");
702 jfs_err("lblock = %lx, size = %d", lblock, size);
703 dump_stack();
704 return NULL;
705 }
706 if (absolute)
707 mapping = JFS_SBI(inode->i_sb)->direct_inode->i_mapping;
708 else {
709 /*
710 * If an nfs client tries to read an inode that is larger
711 * than any existing inodes, we may try to read past the
712 * end of the inode map
713 */
714 if ((lblock << inode->i_blkbits) >= inode->i_size)
715 return NULL;
716 mapping = inode->i_mapping;
717 }
718
719 if (new && (PSIZE == PAGE_SIZE)) {
720 folio = filemap_grab_folio(mapping, page_index);
721 if (IS_ERR(folio)) {
722 jfs_err("filemap_grab_folio failed!");
723 return NULL;
724 }
725 folio_mark_uptodate(folio);
726 } else {
727 folio = read_mapping_folio(mapping, page_index, NULL);
728 if (IS_ERR(folio)) {
729 jfs_err("read_mapping_page failed!");
730 return NULL;
731 }
732 folio_lock(folio);
733 }
734
735 mp = folio_to_mp(folio, page_offset);
736 if (mp) {
737 if (mp->logical_size != size) {
738 jfs_error(inode->i_sb,
739 "get_mp->logical_size != size\n");
740 jfs_err("logical_size = %d, size = %d",
741 mp->logical_size, size);
742 dump_stack();
743 goto unlock;
744 }
745 mp->count++;
746 lock_metapage(mp);
747 if (test_bit(META_discard, &mp->flag)) {
748 if (!new) {
749 jfs_error(inode->i_sb,
750 "using a discarded metapage\n");
751 discard_metapage(mp);
752 goto unlock;
753 }
754 clear_bit(META_discard, &mp->flag);
755 }
756 } else {
757 INCREMENT(mpStat.pagealloc);
758 mp = alloc_metapage(GFP_NOFS);
759 if (!mp)
760 goto unlock;
761 mp->folio = folio;
762 mp->sb = inode->i_sb;
763 mp->flag = 0;
764 mp->xflag = COMMIT_PAGE;
765 mp->count = 1;
766 mp->nohomeok = 0;
767 mp->logical_size = size;
768 mp->data = folio_address(folio) + page_offset;
769 mp->index = lblock;
770 if (unlikely(insert_metapage(folio, mp))) {
771 free_metapage(mp);
772 goto unlock;
773 }
774 lock_metapage(mp);
775 }
776
777 if (new) {
778 jfs_info("zeroing mp = 0x%p", mp);
779 memset(mp->data, 0, PSIZE);
780 }
781
782 folio_unlock(folio);
783 jfs_info("__get_metapage: returning = 0x%p data = 0x%p", mp, mp->data);
784 return mp;
785
786 unlock:
787 folio_unlock(folio);
788 return NULL;
789 }
790
grab_metapage(struct metapage * mp)791 void grab_metapage(struct metapage * mp)
792 {
793 jfs_info("grab_metapage: mp = 0x%p", mp);
794 folio_get(mp->folio);
795 folio_lock(mp->folio);
796 mp->count++;
797 lock_metapage(mp);
798 folio_unlock(mp->folio);
799 }
800
metapage_write_one(struct folio * folio)801 static int metapage_write_one(struct folio *folio)
802 {
803 struct address_space *mapping = folio->mapping;
804 struct writeback_control wbc = {
805 .sync_mode = WB_SYNC_ALL,
806 .nr_to_write = folio_nr_pages(folio),
807 };
808 int ret = 0;
809
810 BUG_ON(!folio_test_locked(folio));
811
812 folio_wait_writeback(folio);
813
814 if (folio_clear_dirty_for_io(folio)) {
815 folio_get(folio);
816 ret = metapage_write_folio(folio, &wbc, NULL);
817 if (ret == 0)
818 folio_wait_writeback(folio);
819 folio_put(folio);
820 } else {
821 folio_unlock(folio);
822 }
823
824 if (!ret)
825 ret = filemap_check_errors(mapping);
826 return ret;
827 }
828
force_metapage(struct metapage * mp)829 void force_metapage(struct metapage *mp)
830 {
831 struct folio *folio = mp->folio;
832 jfs_info("force_metapage: mp = 0x%p", mp);
833 set_bit(META_forcewrite, &mp->flag);
834 clear_bit(META_sync, &mp->flag);
835 folio_get(folio);
836 folio_lock(folio);
837 folio_mark_dirty(folio);
838 if (metapage_write_one(folio))
839 jfs_error(mp->sb, "metapage_write_one() failed\n");
840 clear_bit(META_forcewrite, &mp->flag);
841 folio_put(folio);
842 }
843
hold_metapage(struct metapage * mp)844 void hold_metapage(struct metapage *mp)
845 {
846 folio_lock(mp->folio);
847 }
848
put_metapage(struct metapage * mp)849 void put_metapage(struct metapage *mp)
850 {
851 if (mp->count || mp->nohomeok) {
852 /* Someone else will release this */
853 folio_unlock(mp->folio);
854 return;
855 }
856 folio_get(mp->folio);
857 mp->count++;
858 lock_metapage(mp);
859 folio_unlock(mp->folio);
860 release_metapage(mp);
861 }
862
release_metapage(struct metapage * mp)863 void release_metapage(struct metapage * mp)
864 {
865 struct folio *folio = mp->folio;
866 jfs_info("release_metapage: mp = 0x%p, flag = 0x%lx", mp, mp->flag);
867
868 folio_lock(folio);
869 unlock_metapage(mp);
870
871 assert(mp->count);
872 if (--mp->count || mp->nohomeok) {
873 folio_unlock(folio);
874 folio_put(folio);
875 return;
876 }
877
878 if (test_bit(META_dirty, &mp->flag)) {
879 folio_mark_dirty(folio);
880 if (test_bit(META_sync, &mp->flag)) {
881 clear_bit(META_sync, &mp->flag);
882 if (metapage_write_one(folio))
883 jfs_error(mp->sb, "metapage_write_one() failed\n");
884 folio_lock(folio);
885 }
886 } else if (mp->lsn) /* discard_metapage doesn't remove it */
887 remove_from_logsync(mp);
888
889 /* Try to keep metapages from using up too much memory */
890 drop_metapage(folio, mp);
891
892 folio_unlock(folio);
893 folio_put(folio);
894 }
895
__invalidate_metapages(struct inode * ip,s64 addr,int len)896 void __invalidate_metapages(struct inode *ip, s64 addr, int len)
897 {
898 sector_t lblock;
899 int l2BlocksPerPage = PAGE_SHIFT - ip->i_blkbits;
900 int BlocksPerPage = 1 << l2BlocksPerPage;
901 /* All callers are interested in block device's mapping */
902 struct address_space *mapping =
903 JFS_SBI(ip->i_sb)->direct_inode->i_mapping;
904 struct metapage *mp;
905 unsigned int offset;
906
907 /*
908 * Mark metapages to discard. They will eventually be
909 * released, but should not be written.
910 */
911 for (lblock = addr & ~(BlocksPerPage - 1); lblock < addr + len;
912 lblock += BlocksPerPage) {
913 struct folio *folio = filemap_lock_folio(mapping,
914 lblock >> l2BlocksPerPage);
915 if (IS_ERR(folio))
916 continue;
917 for (offset = 0; offset < PAGE_SIZE; offset += PSIZE) {
918 mp = folio_to_mp(folio, offset);
919 if (!mp)
920 continue;
921 if (mp->index < addr)
922 continue;
923 if (mp->index >= addr + len)
924 break;
925
926 clear_bit(META_dirty, &mp->flag);
927 set_bit(META_discard, &mp->flag);
928 if (mp->lsn)
929 remove_from_logsync(mp);
930 }
931 folio_unlock(folio);
932 folio_put(folio);
933 }
934 }
935
936 #ifdef CONFIG_JFS_STATISTICS
jfs_mpstat_proc_show(struct seq_file * m,void * v)937 int jfs_mpstat_proc_show(struct seq_file *m, void *v)
938 {
939 seq_printf(m,
940 "JFS Metapage statistics\n"
941 "=======================\n"
942 "page allocations = %d\n"
943 "page frees = %d\n"
944 "lock waits = %d\n",
945 mpStat.pagealloc,
946 mpStat.pagefree,
947 mpStat.lockwait);
948 return 0;
949 }
950 #endif
951