direct-io.c (fc30eea1542dd787c6aa46e970014e97e390c5b2) | direct-io.c (fcb14cb1bdacec5b4374fe161e83fb8208164a85) |
---|---|
1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * fs/direct-io.c 4 * 5 * Copyright (C) 2002, Linus Torvalds. 6 * 7 * O_DIRECT 8 * --- 103 unchanged lines hidden (view full) --- 112 unsigned head; /* next page to process */ 113 unsigned tail; /* last valid page + 1 */ 114 size_t from, to; 115}; 116 117/* dio_state communicated between submission path and end_io */ 118struct dio { 119 int flags; /* doesn't change */ | 1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * fs/direct-io.c 4 * 5 * Copyright (C) 2002, Linus Torvalds. 6 * 7 * O_DIRECT 8 * --- 103 unchanged lines hidden (view full) --- 112 unsigned head; /* next page to process */ 113 unsigned tail; /* last valid page + 1 */ 114 size_t from, to; 115}; 116 117/* dio_state communicated between submission path and end_io */ 118struct dio { 119 int flags; /* doesn't change */ |
120 int op; 121 int op_flags; | 120 blk_opf_t opf; /* request operation type and flags */ |
122 struct gendisk *bio_disk; 123 struct inode *inode; 124 loff_t i_size; /* i_size when submitted */ 125 dio_iodone_t *end_io; /* IO completion function */ 126 127 void *private; /* copy from map_bh.b_private */ 128 129 /* BIO completion state */ --- 32 unchanged lines hidden (view full) --- 162 return sdio->tail - sdio->head; 163} 164 165/* 166 * Go grab and pin some userspace pages. Typically we'll get 64 at a time. 167 */ 168static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) 169{ | 121 struct gendisk *bio_disk; 122 struct inode *inode; 123 loff_t i_size; /* i_size when submitted */ 124 dio_iodone_t *end_io; /* IO completion function */ 125 126 void *private; /* copy from map_bh.b_private */ 127 128 /* BIO completion state */ --- 32 unchanged lines hidden (view full) --- 161 return sdio->tail - sdio->head; 162} 163 164/* 165 * Go grab and pin some userspace pages. Typically we'll get 64 at a time. 166 */ 167static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) 168{ |
169 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
170 ssize_t ret; 171 172 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, 173 &sdio->from); 174 | 170 ssize_t ret; 171 172 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, 173 &sdio->from); 174 |
175 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { | 175 if (ret < 0 && sdio->blocks_available && dio_op == REQ_OP_WRITE) { |
176 struct page *page = ZERO_PAGE(0); 177 /* 178 * A memory fault, but the filesystem has some outstanding 179 * mapped blocks. We need to use those blocks up to avoid 180 * leaking stale data in the file. 181 */ 182 if (dio->page_errors == 0) 183 dio->page_errors = ret; --- 45 unchanged lines hidden (view full) --- 229 * 230 * It lets the filesystem know if it registered an interest earlier via 231 * get_block. Pass the private field of the map buffer_head so that 232 * filesystems can use it to hold additional state between get_block calls and 233 * dio_complete. 234 */ 235static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) 236{ | 176 struct page *page = ZERO_PAGE(0); 177 /* 178 * A memory fault, but the filesystem has some outstanding 179 * mapped blocks. We need to use those blocks up to avoid 180 * leaking stale data in the file. 181 */ 182 if (dio->page_errors == 0) 183 dio->page_errors = ret; --- 45 unchanged lines hidden (view full) --- 229 * 230 * It lets the filesystem know if it registered an interest earlier via 231 * get_block. Pass the private field of the map buffer_head so that 232 * filesystems can use it to hold additional state between get_block calls and 233 * dio_complete. 234 */ 235static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) 236{ |
237 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
237 loff_t offset = dio->iocb->ki_pos; 238 ssize_t transferred = 0; 239 int err; 240 241 /* 242 * AIO submission can race with bio completion to get here while 243 * expecting to have the last io completed by bio completion. 244 * In that case -EIOCBQUEUED is in fact not an error we want 245 * to preserve through this call. 246 */ 247 if (ret == -EIOCBQUEUED) 248 ret = 0; 249 250 if (dio->result) { 251 transferred = dio->result; 252 253 /* Check for short read case */ | 238 loff_t offset = dio->iocb->ki_pos; 239 ssize_t transferred = 0; 240 int err; 241 242 /* 243 * AIO submission can race with bio completion to get here while 244 * expecting to have the last io completed by bio completion. 245 * In that case -EIOCBQUEUED is in fact not an error we want 246 * to preserve through this call. 247 */ 248 if (ret == -EIOCBQUEUED) 249 ret = 0; 250 251 if (dio->result) { 252 transferred = dio->result; 253 254 /* Check for short read case */ |
254 if ((dio->op == REQ_OP_READ) && | 255 if (dio_op == REQ_OP_READ && |
255 ((offset + transferred) > dio->i_size)) 256 transferred = dio->i_size - offset; 257 /* ignore EFAULT if some IO has been done */ 258 if (unlikely(ret == -EFAULT) && transferred) 259 ret = 0; 260 } 261 262 if (ret == 0) --- 18 unchanged lines hidden (view full) --- 281 * this invalidation fails, tough, the write still worked... 282 * 283 * And this page cache invalidation has to be after dio->end_io(), as 284 * some filesystems convert unwritten extents to real allocations in 285 * end_io() when necessary, otherwise a racing buffer read would cache 286 * zeros from unwritten extents. 287 */ 288 if (flags & DIO_COMPLETE_INVALIDATE && | 256 ((offset + transferred) > dio->i_size)) 257 transferred = dio->i_size - offset; 258 /* ignore EFAULT if some IO has been done */ 259 if (unlikely(ret == -EFAULT) && transferred) 260 ret = 0; 261 } 262 263 if (ret == 0) --- 18 unchanged lines hidden (view full) --- 282 * this invalidation fails, tough, the write still worked... 283 * 284 * And this page cache invalidation has to be after dio->end_io(), as 285 * some filesystems convert unwritten extents to real allocations in 286 * end_io() when necessary, otherwise a racing buffer read would cache 287 * zeros from unwritten extents. 288 */ 289 if (flags & DIO_COMPLETE_INVALIDATE && |
289 ret > 0 && dio->op == REQ_OP_WRITE && | 290 ret > 0 && dio_op == REQ_OP_WRITE && |
290 dio->inode->i_mapping->nrpages) { 291 err = invalidate_inode_pages2_range(dio->inode->i_mapping, 292 offset >> PAGE_SHIFT, 293 (offset + ret - 1) >> PAGE_SHIFT); 294 if (err) 295 dio_warn_stale_pagecache(dio->iocb->ki_filp); 296 } 297 298 inode_dio_end(dio->inode); 299 300 if (flags & DIO_COMPLETE_ASYNC) { 301 /* 302 * generic_write_sync expects ki_pos to have been updated 303 * already, but the submission path only does this for 304 * synchronous I/O. 305 */ 306 dio->iocb->ki_pos += transferred; 307 | 291 dio->inode->i_mapping->nrpages) { 292 err = invalidate_inode_pages2_range(dio->inode->i_mapping, 293 offset >> PAGE_SHIFT, 294 (offset + ret - 1) >> PAGE_SHIFT); 295 if (err) 296 dio_warn_stale_pagecache(dio->iocb->ki_filp); 297 } 298 299 inode_dio_end(dio->inode); 300 301 if (flags & DIO_COMPLETE_ASYNC) { 302 /* 303 * generic_write_sync expects ki_pos to have been updated 304 * already, but the submission path only does this for 305 * synchronous I/O. 306 */ 307 dio->iocb->ki_pos += transferred; 308 |
308 if (ret > 0 && dio->op == REQ_OP_WRITE) | 309 if (ret > 0 && dio_op == REQ_OP_WRITE) |
309 ret = generic_write_sync(dio->iocb, ret); 310 dio->iocb->ki_complete(dio->iocb, ret); 311 } 312 313 kmem_cache_free(dio_cache, dio); 314 return ret; 315} 316 --- 7 unchanged lines hidden (view full) --- 324static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio); 325 326/* 327 * Asynchronous IO callback. 328 */ 329static void dio_bio_end_aio(struct bio *bio) 330{ 331 struct dio *dio = bio->bi_private; | 310 ret = generic_write_sync(dio->iocb, ret); 311 dio->iocb->ki_complete(dio->iocb, ret); 312 } 313 314 kmem_cache_free(dio_cache, dio); 315 return ret; 316} 317 --- 7 unchanged lines hidden (view full) --- 325static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio); 326 327/* 328 * Asynchronous IO callback. 329 */ 330static void dio_bio_end_aio(struct bio *bio) 331{ 332 struct dio *dio = bio->bi_private; |
333 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
332 unsigned long remaining; 333 unsigned long flags; 334 bool defer_completion = false; 335 336 /* cleanup the bio */ 337 dio_bio_complete(dio, bio); 338 339 spin_lock_irqsave(&dio->bio_lock, flags); --- 8 unchanged lines hidden (view full) --- 348 * when the inode has pages mapped and this is AIO write. 349 * We need to invalidate those pages because there is a 350 * chance they contain stale data in the case buffered IO 351 * went in between AIO submission and completion into the 352 * same region. 353 */ 354 if (dio->result) 355 defer_completion = dio->defer_completion || | 334 unsigned long remaining; 335 unsigned long flags; 336 bool defer_completion = false; 337 338 /* cleanup the bio */ 339 dio_bio_complete(dio, bio); 340 341 spin_lock_irqsave(&dio->bio_lock, flags); --- 8 unchanged lines hidden (view full) --- 350 * when the inode has pages mapped and this is AIO write. 351 * We need to invalidate those pages because there is a 352 * chance they contain stale data in the case buffered IO 353 * went in between AIO submission and completion into the 354 * same region. 355 */ 356 if (dio->result) 357 defer_completion = dio->defer_completion || |
356 (dio->op == REQ_OP_WRITE && | 358 (dio_op == REQ_OP_WRITE && |
357 dio->inode->i_mapping->nrpages); 358 if (defer_completion) { 359 INIT_WORK(&dio->complete_work, dio_aio_complete_work); 360 queue_work(dio->inode->i_sb->s_dio_done_wq, 361 &dio->complete_work); 362 } else { 363 dio_complete(dio, 0, DIO_COMPLETE_ASYNC); 364 } --- 26 unchanged lines hidden (view full) --- 391 sector_t first_sector, int nr_vecs) 392{ 393 struct bio *bio; 394 395 /* 396 * bio_alloc() is guaranteed to return a bio when allowed to sleep and 397 * we request a valid number of vectors. 398 */ | 359 dio->inode->i_mapping->nrpages); 360 if (defer_completion) { 361 INIT_WORK(&dio->complete_work, dio_aio_complete_work); 362 queue_work(dio->inode->i_sb->s_dio_done_wq, 363 &dio->complete_work); 364 } else { 365 dio_complete(dio, 0, DIO_COMPLETE_ASYNC); 366 } --- 26 unchanged lines hidden (view full) --- 393 sector_t first_sector, int nr_vecs) 394{ 395 struct bio *bio; 396 397 /* 398 * bio_alloc() is guaranteed to return a bio when allowed to sleep and 399 * we request a valid number of vectors. 400 */ |
399 bio = bio_alloc(bdev, nr_vecs, dio->op | dio->op_flags, GFP_KERNEL); | 401 bio = bio_alloc(bdev, nr_vecs, dio->opf, GFP_KERNEL); |
400 bio->bi_iter.bi_sector = first_sector; 401 if (dio->is_async) 402 bio->bi_end_io = dio_bio_end_aio; 403 else 404 bio->bi_end_io = dio_bio_end_io; 405 sdio->bio = bio; 406 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; 407} 408 409/* 410 * In the AIO read case we speculatively dirty the pages before starting IO. 411 * During IO completion, any of these pages which happen to have been written 412 * back will be redirtied by bio_check_pages_dirty(). 413 * 414 * bios hold a dio reference between submit_bio and ->end_io. 415 */ 416static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) 417{ | 402 bio->bi_iter.bi_sector = first_sector; 403 if (dio->is_async) 404 bio->bi_end_io = dio_bio_end_aio; 405 else 406 bio->bi_end_io = dio_bio_end_io; 407 sdio->bio = bio; 408 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; 409} 410 411/* 412 * In the AIO read case we speculatively dirty the pages before starting IO. 413 * During IO completion, any of these pages which happen to have been written 414 * back will be redirtied by bio_check_pages_dirty(). 415 * 416 * bios hold a dio reference between submit_bio and ->end_io. 417 */ 418static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) 419{ |
420 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
418 struct bio *bio = sdio->bio; 419 unsigned long flags; 420 421 bio->bi_private = dio; 422 /* don't account direct I/O as memory stall */ 423 bio_clear_flag(bio, BIO_WORKINGSET); 424 425 spin_lock_irqsave(&dio->bio_lock, flags); 426 dio->refcount++; 427 spin_unlock_irqrestore(&dio->bio_lock, flags); 428 | 421 struct bio *bio = sdio->bio; 422 unsigned long flags; 423 424 bio->bi_private = dio; 425 /* don't account direct I/O as memory stall */ 426 bio_clear_flag(bio, BIO_WORKINGSET); 427 428 spin_lock_irqsave(&dio->bio_lock, flags); 429 dio->refcount++; 430 spin_unlock_irqrestore(&dio->bio_lock, flags); 431 |
429 if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) | 432 if (dio->is_async && dio_op == REQ_OP_READ && dio->should_dirty) |
430 bio_set_pages_dirty(bio); 431 432 dio->bio_disk = bio->bi_bdev->bd_disk; 433 434 if (sdio->submit_io) 435 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); 436 else 437 submit_bio(bio); --- 49 unchanged lines hidden (view full) --- 487} 488 489/* 490 * Process one completed BIO. No locks are held. 491 */ 492static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) 493{ 494 blk_status_t err = bio->bi_status; | 433 bio_set_pages_dirty(bio); 434 435 dio->bio_disk = bio->bi_bdev->bd_disk; 436 437 if (sdio->submit_io) 438 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); 439 else 440 submit_bio(bio); --- 49 unchanged lines hidden (view full) --- 490} 491 492/* 493 * Process one completed BIO. No locks are held. 494 */ 495static blk_status_t dio_bio_complete(struct dio *dio, struct bio *bio) 496{ 497 blk_status_t err = bio->bi_status; |
495 bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty; | 498 const enum req_op dio_op = dio->opf & REQ_OP_MASK; 499 bool should_dirty = dio_op == REQ_OP_READ && dio->should_dirty; |
496 497 if (err) { 498 if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT)) 499 dio->io_error = -EAGAIN; 500 else 501 dio->io_error = -EIO; 502 } 503 --- 110 unchanged lines hidden (view full) --- 614 * In the case of filesystem holes: the fs may return an arbitrarily-large 615 * hole by returning an appropriate value in b_size and by clearing 616 * buffer_mapped(). However the direct-io code will only process holes one 617 * block at a time - it will repeatedly call get_block() as it walks the hole. 618 */ 619static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, 620 struct buffer_head *map_bh) 621{ | 500 501 if (err) { 502 if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT)) 503 dio->io_error = -EAGAIN; 504 else 505 dio->io_error = -EIO; 506 } 507 --- 110 unchanged lines hidden (view full) --- 618 * In the case of filesystem holes: the fs may return an arbitrarily-large 619 * hole by returning an appropriate value in b_size and by clearing 620 * buffer_mapped(). However the direct-io code will only process holes one 621 * block at a time - it will repeatedly call get_block() as it walks the hole. 622 */ 623static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, 624 struct buffer_head *map_bh) 625{ |
626 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
622 int ret; 623 sector_t fs_startblk; /* Into file, in filesystem-sized blocks */ 624 sector_t fs_endblk; /* Into file, in filesystem-sized blocks */ 625 unsigned long fs_count; /* Number of filesystem-sized blocks */ 626 int create; 627 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; 628 loff_t i_size; 629 --- 18 unchanged lines hidden (view full) --- 648 * overwrites are permitted. We will return early to the caller 649 * once we see an unmapped buffer head returned, and the caller 650 * will fall back to buffered I/O. 651 * 652 * Otherwise the decision is left to the get_blocks method, 653 * which may decide to handle it or also return an unmapped 654 * buffer head. 655 */ | 627 int ret; 628 sector_t fs_startblk; /* Into file, in filesystem-sized blocks */ 629 sector_t fs_endblk; /* Into file, in filesystem-sized blocks */ 630 unsigned long fs_count; /* Number of filesystem-sized blocks */ 631 int create; 632 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; 633 loff_t i_size; 634 --- 18 unchanged lines hidden (view full) --- 653 * overwrites are permitted. We will return early to the caller 654 * once we see an unmapped buffer head returned, and the caller 655 * will fall back to buffered I/O. 656 * 657 * Otherwise the decision is left to the get_blocks method, 658 * which may decide to handle it or also return an unmapped 659 * buffer head. 660 */ |
656 create = dio->op == REQ_OP_WRITE; | 661 create = dio_op == REQ_OP_WRITE; |
657 if (dio->flags & DIO_SKIP_HOLES) { 658 i_size = i_size_read(dio->inode); 659 if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits) 660 create = 0; 661 } 662 663 ret = (*sdio->get_block)(dio->inode, fs_startblk, 664 map_bh, create); --- 131 unchanged lines hidden (view full) --- 796 * If that doesn't work out then we put the old page into the bio and add this 797 * page to the dio instead. 798 */ 799static inline int 800submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, 801 unsigned offset, unsigned len, sector_t blocknr, 802 struct buffer_head *map_bh) 803{ | 662 if (dio->flags & DIO_SKIP_HOLES) { 663 i_size = i_size_read(dio->inode); 664 if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits) 665 create = 0; 666 } 667 668 ret = (*sdio->get_block)(dio->inode, fs_startblk, 669 map_bh, create); --- 131 unchanged lines hidden (view full) --- 801 * If that doesn't work out then we put the old page into the bio and add this 802 * page to the dio instead. 803 */ 804static inline int 805submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, 806 unsigned offset, unsigned len, sector_t blocknr, 807 struct buffer_head *map_bh) 808{ |
809 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
804 int ret = 0; 805 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ 806 | 810 int ret = 0; 811 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */ 812 |
807 if (dio->op == REQ_OP_WRITE) { | 813 if (dio_op == REQ_OP_WRITE) { |
808 /* 809 * Read accounting is performed in submit_bio() 810 */ 811 task_io_account_write(len); 812 } 813 814 /* 815 * Can we just grow the current page's presence in the dio? --- 96 unchanged lines hidden (view full) --- 912 * 913 * For best results, the blockdev should be set up with 512-byte i_blkbits and 914 * it should set b_size to PAGE_SIZE or more inside get_block(). This gives 915 * fine alignment but still allows this function to work in PAGE_SIZE units. 916 */ 917static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, 918 struct buffer_head *map_bh) 919{ | 814 /* 815 * Read accounting is performed in submit_bio() 816 */ 817 task_io_account_write(len); 818 } 819 820 /* 821 * Can we just grow the current page's presence in the dio? --- 96 unchanged lines hidden (view full) --- 918 * 919 * For best results, the blockdev should be set up with 512-byte i_blkbits and 920 * it should set b_size to PAGE_SIZE or more inside get_block(). This gives 921 * fine alignment but still allows this function to work in PAGE_SIZE units. 922 */ 923static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, 924 struct buffer_head *map_bh) 925{ |
926 const enum req_op dio_op = dio->opf & REQ_OP_MASK; |
|
920 const unsigned blkbits = sdio->blkbits; 921 const unsigned i_blkbits = blkbits + sdio->blkfactor; 922 int ret = 0; 923 924 while (sdio->block_in_file < sdio->final_block_in_request) { 925 struct page *page; 926 size_t from, to; 927 --- 59 unchanged lines hidden (view full) --- 987 sdio->blocks_available -= dio_remainder; 988 } 989do_holes: 990 /* Handle holes */ 991 if (!buffer_mapped(map_bh)) { 992 loff_t i_size_aligned; 993 994 /* AKPM: eargh, -ENOTBLK is a hack */ | 927 const unsigned blkbits = sdio->blkbits; 928 const unsigned i_blkbits = blkbits + sdio->blkfactor; 929 int ret = 0; 930 931 while (sdio->block_in_file < sdio->final_block_in_request) { 932 struct page *page; 933 size_t from, to; 934 --- 59 unchanged lines hidden (view full) --- 994 sdio->blocks_available -= dio_remainder; 995 } 996do_holes: 997 /* Handle holes */ 998 if (!buffer_mapped(map_bh)) { 999 loff_t i_size_aligned; 1000 1001 /* AKPM: eargh, -ENOTBLK is a hack */ |
995 if (dio->op == REQ_OP_WRITE) { | 1002 if (dio_op == REQ_OP_WRITE) { |
996 put_page(page); 997 return -ENOTBLK; 998 } 999 1000 /* 1001 * Be sure to account for a partial block as the 1002 * last block in the file 1003 */ --- 187 unchanged lines hidden (view full) --- 1191 dio->is_async = false; 1192 else if (iov_iter_rw(iter) == WRITE && end > i_size_read(inode)) 1193 dio->is_async = false; 1194 else 1195 dio->is_async = true; 1196 1197 dio->inode = inode; 1198 if (iov_iter_rw(iter) == WRITE) { | 1003 put_page(page); 1004 return -ENOTBLK; 1005 } 1006 1007 /* 1008 * Be sure to account for a partial block as the 1009 * last block in the file 1010 */ --- 187 unchanged lines hidden (view full) --- 1198 dio->is_async = false; 1199 else if (iov_iter_rw(iter) == WRITE && end > i_size_read(inode)) 1200 dio->is_async = false; 1201 else 1202 dio->is_async = true; 1203 1204 dio->inode = inode; 1205 if (iov_iter_rw(iter) == WRITE) { |
1199 dio->op = REQ_OP_WRITE; 1200 dio->op_flags = REQ_SYNC | REQ_IDLE; | 1206 dio->opf = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE; |
1201 if (iocb->ki_flags & IOCB_NOWAIT) | 1207 if (iocb->ki_flags & IOCB_NOWAIT) |
1202 dio->op_flags |= REQ_NOWAIT; | 1208 dio->opf |= REQ_NOWAIT; |
1203 } else { | 1209 } else { |
1204 dio->op = REQ_OP_READ; | 1210 dio->opf = REQ_OP_READ; |
1205 } 1206 1207 /* 1208 * For AIO O_(D)SYNC writes we need to defer completions to a workqueue 1209 * so that we can call ->fsync. 1210 */ 1211 if (dio->is_async && iov_iter_rw(iter) == WRITE) { 1212 retval = 0; | 1211 } 1212 1213 /* 1214 * For AIO O_(D)SYNC writes we need to defer completions to a workqueue 1215 * so that we can call ->fsync. 1216 */ 1217 if (dio->is_async && iov_iter_rw(iter) == WRITE) { 1218 retval = 0; |
1213 if (iocb->ki_flags & IOCB_DSYNC) | 1219 if (iocb_is_dsync(iocb)) |
1214 retval = dio_set_defer_completion(dio); 1215 else if (!dio->inode->i_sb->s_dio_done_wq) { 1216 /* 1217 * In case of AIO write racing with buffered read we 1218 * need to defer completion. We can't decide this now, 1219 * however the workqueue needs to be initialized here. 1220 */ 1221 retval = sb_init_dio_done_wq(dio->inode->i_sb); --- 18 unchanged lines hidden (view full) --- 1240 sdio.final_block_in_bio = -1; 1241 sdio.next_block_for_io = -1; 1242 1243 dio->iocb = iocb; 1244 1245 spin_lock_init(&dio->bio_lock); 1246 dio->refcount = 1; 1247 | 1220 retval = dio_set_defer_completion(dio); 1221 else if (!dio->inode->i_sb->s_dio_done_wq) { 1222 /* 1223 * In case of AIO write racing with buffered read we 1224 * need to defer completion. We can't decide this now, 1225 * however the workqueue needs to be initialized here. 1226 */ 1227 retval = sb_init_dio_done_wq(dio->inode->i_sb); --- 18 unchanged lines hidden (view full) --- 1246 sdio.final_block_in_bio = -1; 1247 sdio.next_block_for_io = -1; 1248 1249 dio->iocb = iocb; 1250 1251 spin_lock_init(&dio->bio_lock); 1252 dio->refcount = 1; 1253 |
1248 dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ; | 1254 dio->should_dirty = user_backed_iter(iter) && iov_iter_rw(iter) == READ; |
1249 sdio.iter = iter; 1250 sdio.final_block_in_request = end >> blkbits; 1251 1252 /* 1253 * In case of non-aligned buffers, we may need 2 more 1254 * pages since we need to zero out first and last block. 1255 */ 1256 if (unlikely(sdio.blkfactor)) --- 87 unchanged lines hidden --- | 1255 sdio.iter = iter; 1256 sdio.final_block_in_request = end >> blkbits; 1257 1258 /* 1259 * In case of non-aligned buffers, we may need 2 more 1260 * pages since we need to zero out first and last block. 1261 */ 1262 if (unlikely(sdio.blkfactor)) --- 87 unchanged lines hidden --- |