30 static unsigned int cache_mode(struct cached_dev *dc)  in cache_mode()  argument
32 	return BDEV_CACHE_MODE(&dc->sb);  in cache_mode()
35 static bool verify(struct cached_dev *dc)  in verify()  argument
37 	return dc->verify;  in verify()
358 static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k)  in iohash()  argument
360 	return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];  in iohash()
363 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)  in check_should_bypass()  argument
365 	struct cache_set *c = dc->disk.c;  in check_should_bypass()
366 	unsigned int mode = cache_mode(dc);  in check_should_bypass()
371 	if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||  in check_should_bypass()
383 		if (BDEV_STATE(&dc->sb) == BDEV_STATE_CLEAN &&  in check_should_bypass()
407 		    (dc->cache_readahead_policy != BCH_CACHE_READA_ALL))  in check_should_bypass()
417 	if (bypass_torture_test(dc)) {  in check_should_bypass()
425 	if (!congested && !dc->sequential_cutoff)  in check_should_bypass()
428 	spin_lock(&dc->io_lock);  in check_should_bypass()
430 	hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)  in check_should_bypass()
435 	i = list_first_entry(&dc->io_lru, struct io, lru);  in check_should_bypass()
448 	hlist_add_head(&i->hash, iohash(dc, i->last));  in check_should_bypass()
449 	list_move_tail(&i->lru, &dc->io_lru);  in check_should_bypass()
451 	spin_unlock(&dc->io_lock);  in check_should_bypass()
456 	if (dc->sequential_cutoff &&  in check_should_bypass()
457 	    sectors >= dc->sequential_cutoff >> 9) {  in check_should_bypass()
471 	bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));  in check_should_bypass()
596 	struct cached_dev *dc;  in CLOSURE_CALLBACK()  local
622 			dc = container_of(s->d, struct cached_dev, disk);  in CLOSURE_CALLBACK()
623 			if (dc && atomic_read(&dc->has_dirty))  in CLOSURE_CALLBACK()
657 		struct cached_dev *dc = container_of(s->d,  in backing_request_endio()  local
669 				dc->bdev, bio->bi_status);  in backing_request_endio()
676 		bch_count_backing_io_errors(dc, bio);  in backing_request_endio()
769 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);  in CLOSURE_CALLBACK()  local
771 	cached_dev_put(dc);  in CLOSURE_CALLBACK()
836 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);  in CLOSURE_CALLBACK()  local
860 	if (verify(dc) && s->recoverable && !s->read_dirty_data)  in CLOSURE_CALLBACK()
861 		bch_data_verify(dc, s->orig_bio);  in CLOSURE_CALLBACK()
863 	closure_get(&dc->disk.cl);  in CLOSURE_CALLBACK()
878 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);  in CLOSURE_CALLBACK()  local
886 	else if (s->iop.bio || verify(dc))  in CLOSURE_CALLBACK()
896 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);  in cached_dev_cache_miss()  local
931 			0, GFP_NOWAIT, &dc->disk.bio_split);  in cached_dev_cache_miss()
962 static void cached_dev_read(struct cached_dev *dc, struct search *s)  in cached_dev_read()  argument
975 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);  in CLOSURE_CALLBACK()  local
977 	up_read_non_owner(&dc->writeback_lock);  in CLOSURE_CALLBACK()
981 static void cached_dev_write(struct cached_dev *dc, struct search *s)  in cached_dev_write()  argument
985 	struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);  in cached_dev_write()
986 	struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);  in cached_dev_write()
990 	down_read_non_owner(&dc->writeback_lock);  in cached_dev_write()
991 	if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) {  in cached_dev_write()
1010 	if (should_writeback(dc, s->orig_bio,  in cached_dev_write()
1011 			     cache_mode(dc),  in cached_dev_write()
1022 		    !bdev_max_discard_sectors(dc->bdev))  in cached_dev_write()
1030 		bch_writeback_add(dc);  in cached_dev_write()
1042 						 GFP_NOIO, &dc->disk.bio_split);  in cached_dev_write()
1054 					     &dc->disk.bio_split);  in cached_dev_write()
1090 		struct cached_dev *dc = bio->bi_private;  in detached_dev_end_io()  local
1093 		bch_count_backing_io_errors(dc, bio);  in detached_dev_end_io()
1105 	struct cached_dev *dc = container_of(d, struct cached_dev, disk);  in detached_dev_do_request()  local
1109 	    !bdev_max_discard_sectors(dc->bdev)) {  in detached_dev_do_request()
1115 	clone_bio = bio_alloc_clone(dc->bdev, orig_bio, GFP_NOIO,  in detached_dev_do_request()
1125 	clone_bio->bi_private = dc;  in detached_dev_do_request()
1135 	struct cached_dev *dc;  in quit_max_writeback_rate()  local
1155 			dc = container_of(d, struct cached_dev, disk);  in quit_max_writeback_rate()
1161 			atomic_long_set(&dc->writeback_rate.rate, 1);  in quit_max_writeback_rate()
1175 	struct cached_dev *dc = container_of(d, struct cached_dev, disk);  in cached_dev_submit_bio()  local
1180 		     dc->io_disable)) {  in cached_dev_submit_bio()
1197 			quit_max_writeback_rate(d->c, dc);  in cached_dev_submit_bio()
1203 	bio->bi_iter.bi_sector += dc->sb.data_offset;  in cached_dev_submit_bio()
1205 	if (cached_dev_get(dc)) {  in cached_dev_submit_bio()
1206 		bio_set_dev(bio, dc->bdev);  in cached_dev_submit_bio()
1219 			s->iop.bypass = check_should_bypass(dc, bio);  in cached_dev_submit_bio()
1222 				cached_dev_write(dc, s);  in cached_dev_submit_bio()
1224 				cached_dev_read(dc, s);  in cached_dev_submit_bio()
1235 	struct cached_dev *dc = container_of(d, struct cached_dev, disk);  in cached_dev_ioctl()  local
1237 	if (dc->io_disable)  in cached_dev_ioctl()
1239 	if (!dc->bdev->bd_disk->fops->ioctl)  in cached_dev_ioctl()
1241 	return dc->bdev->bd_disk->fops->ioctl(dc->bdev, mode, cmd, arg);  in cached_dev_ioctl()
1244 void bch_cached_dev_request_init(struct cached_dev *dc)  in bch_cached_dev_request_init()  argument
1246 	dc->disk.cache_miss			= cached_dev_cache_miss;  in bch_cached_dev_request_init()
1247 	dc->disk.ioctl				= cached_dev_ioctl;  in bch_cached_dev_request_init()