30 static uint64_t __calc_target_rate(struct cached_dev *dc)  in __calc_target_rate()  argument
32 	struct cache_set *c = dc->disk.c;  in __calc_target_rate()
48 		div64_u64(bdev_nr_sectors(dc->bdev) << WRITEBACK_SHARE_SHIFT,  in __calc_target_rate()
52 		div_u64(cache_sectors * dc->writeback_percent, 100);  in __calc_target_rate()
61 static void __update_writeback_rate(struct cached_dev *dc)  in __update_writeback_rate()  argument
83 	int64_t target = __calc_target_rate(dc);  in __update_writeback_rate()
84 	int64_t dirty = bcache_dev_sectors_dirty(&dc->disk);  in __update_writeback_rate()
87 		div_s64(error, dc->writeback_rate_p_term_inverse);  in __update_writeback_rate()
101 	struct cache_set *c = dc->disk.c;  in __update_writeback_rate()
105 	if (dc->writeback_consider_fragment &&  in __update_writeback_rate()
113 			fp_term = (int64_t)dc->writeback_rate_fp_term_low *  in __update_writeback_rate()
116 			fp_term = (int64_t)dc->writeback_rate_fp_term_mid *  in __update_writeback_rate()
119 			fp_term = (int64_t)dc->writeback_rate_fp_term_high *  in __update_writeback_rate()
129 	if ((error < 0 && dc->writeback_rate_integral > 0) ||  in __update_writeback_rate()
131 			 dc->writeback_rate.next + NSEC_PER_MSEC))) {  in __update_writeback_rate()
142 		dc->writeback_rate_integral += error *  in __update_writeback_rate()
143 			dc->writeback_rate_update_seconds;  in __update_writeback_rate()
146 	integral_scaled = div_s64(dc->writeback_rate_integral,  in __update_writeback_rate()
147 			dc->writeback_rate_i_term_inverse);  in __update_writeback_rate()
150 			dc->writeback_rate_minimum, NSEC_PER_SEC);  in __update_writeback_rate()
152 	dc->writeback_rate_proportional = proportional_scaled;  in __update_writeback_rate()
153 	dc->writeback_rate_integral_scaled = integral_scaled;  in __update_writeback_rate()
154 	dc->writeback_rate_change = new_rate -  in __update_writeback_rate()
155 			atomic_long_read(&dc->writeback_rate.rate);  in __update_writeback_rate()
156 	atomic_long_set(&dc->writeback_rate.rate, new_rate);  in __update_writeback_rate()
157 	dc->writeback_rate_target = target;  in __update_writeback_rate()
208 				       struct cached_dev *dc)  in set_at_max_writeback_rate()  argument
224 	atomic_long_set(&dc->writeback_rate.rate, INT_MAX);  in set_at_max_writeback_rate()
227 	dc->writeback_rate_proportional = 0;  in set_at_max_writeback_rate()
228 	dc->writeback_rate_integral_scaled = 0;  in set_at_max_writeback_rate()
229 	dc->writeback_rate_change = 0;  in set_at_max_writeback_rate()
244 	struct cached_dev *dc = container_of(to_delayed_work(work),  in update_writeback_rate()  local
247 	struct cache_set *c = dc->disk.c;  in update_writeback_rate()
253 	set_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);  in update_writeback_rate()
261 	if (!test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) ||  in update_writeback_rate()
263 		clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);  in update_writeback_rate()
275 	if (atomic_read(&dc->has_dirty) && dc->writeback_percent &&  in update_writeback_rate()
276 	    !set_at_max_writeback_rate(c, dc)) {  in update_writeback_rate()
278 			if (!down_read_trylock((&dc->writeback_lock))) {  in update_writeback_rate()
279 				dc->rate_update_retry++;  in update_writeback_rate()
280 				if (dc->rate_update_retry <=  in update_writeback_rate()
283 				down_read(&dc->writeback_lock);  in update_writeback_rate()
284 				dc->rate_update_retry = 0;  in update_writeback_rate()
286 			__update_writeback_rate(dc);  in update_writeback_rate()
288 			up_read(&dc->writeback_lock);  in update_writeback_rate()
297 	if (test_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags) &&  in update_writeback_rate()
299 		schedule_delayed_work(&dc->writeback_rate_update,  in update_writeback_rate()
300 			      dc->writeback_rate_update_seconds * HZ);  in update_writeback_rate()
307 	clear_bit(BCACHE_DEV_RATE_DW_RUNNING, &dc->disk.flags);  in update_writeback_rate()
312 static unsigned int writeback_delay(struct cached_dev *dc,  in writeback_delay()  argument
315 	if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||  in writeback_delay()
316 	    !dc->writeback_percent)  in writeback_delay()
319 	return bch_next_delay(&dc->writeback_rate, sectors);  in writeback_delay()
324 	struct cached_dev	*dc;  member
336 	if (!io->dc->writeback_percent)  in dirty_init()
355 	struct cached_dev *dc = io->dc;  in CLOSURE_CALLBACK()  local
372 			atomic_inc(&PTR_BUCKET(dc->disk.c, &w->key, i)->pin);  in CLOSURE_CALLBACK()
374 		ret = bch_btree_insert(dc->disk.c, &keys, NULL, &w->key);  in CLOSURE_CALLBACK()
380 				? &dc->disk.c->writeback_keys_failed  in CLOSURE_CALLBACK()
381 				: &dc->disk.c->writeback_keys_done);  in CLOSURE_CALLBACK()
384 	bch_keybuf_del(&dc->writeback_keys, w);  in CLOSURE_CALLBACK()
385 	up(&dc->in_flight);  in CLOSURE_CALLBACK()
397 		bch_count_backing_io_errors(io->dc, bio);  in dirty_endio()
407 	struct cached_dev *dc = io->dc;  in CLOSURE_CALLBACK()  local
411 	if (atomic_read(&dc->writeback_sequence_next) != io->sequence) {  in CLOSURE_CALLBACK()
413 		closure_wait(&dc->writeback_ordering_wait, cl);  in CLOSURE_CALLBACK()
415 		if (atomic_read(&dc->writeback_sequence_next) == io->sequence) {  in CLOSURE_CALLBACK()
420 			closure_wake_up(&dc->writeback_ordering_wait);  in CLOSURE_CALLBACK()
423 		continue_at(cl, write_dirty, io->dc->writeback_write_wq);  in CLOSURE_CALLBACK()
439 		bio_set_dev(&io->bio, io->dc->bdev);  in CLOSURE_CALLBACK()
443 		closure_bio_submit(io->dc->disk.c, &io->bio, cl);  in CLOSURE_CALLBACK()
446 	atomic_set(&dc->writeback_sequence_next, next_sequence);  in CLOSURE_CALLBACK()
447 	closure_wake_up(&dc->writeback_ordering_wait);  in CLOSURE_CALLBACK()
449 	continue_at(cl, write_dirty_finish, io->dc->writeback_write_wq);  in CLOSURE_CALLBACK()
458 	bch_count_io_errors(io->dc->disk.c->cache,  in read_dirty_endio()
469 	closure_bio_submit(io->dc->disk.c, &io->bio, cl);  in CLOSURE_CALLBACK()
471 	continue_at(cl, write_dirty, io->dc->writeback_write_wq);  in CLOSURE_CALLBACK()
474 static void read_dirty(struct cached_dev *dc)  in read_dirty()  argument
484 	BUG_ON(!llist_empty(&dc->writeback_ordering_wait.list));  in read_dirty()
485 	atomic_set(&dc->writeback_sequence_next, sequence);  in read_dirty()
493 	next = bch_keybuf_next(&dc->writeback_keys);  in read_dirty()
496 	       !test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) &&  in read_dirty()
502 			BUG_ON(ptr_stale(dc->disk.c, &next->key, 0));  in read_dirty()
533 		} while ((next = bch_keybuf_next(&dc->writeback_keys)));  in read_dirty()
546 			io->dc		= dc;  in read_dirty()
552 			bio_set_dev(&io->bio, dc->disk.c->cache->bdev);  in read_dirty()
560 			down(&dc->in_flight);  in read_dirty()
570 		delay = writeback_delay(dc, size);  in read_dirty()
573 		       !test_bit(CACHE_SET_IO_DISABLE, &dc->disk.c->flags) &&  in read_dirty()
576 			delay = writeback_delay(dc, 0);  in read_dirty()
584 		bch_keybuf_del(&dc->writeback_keys, w);  in read_dirty()
643 	struct cached_dev *dc = container_of(buf,  in dirty_pred()  local
647 	BUG_ON(KEY_INODE(k) != dc->disk.id);  in dirty_pred()
652 static void refill_full_stripes(struct cached_dev *dc)  in refill_full_stripes()  argument
654 	struct keybuf *buf = &dc->writeback_keys;  in refill_full_stripes()
659 	stripe = offset_to_stripe(&dc->disk, KEY_OFFSET(&buf->last_scanned));  in refill_full_stripes()
666 		stripe = find_next_bit(dc->disk.full_dirty_stripes,  in refill_full_stripes()
667 				       dc->disk.nr_stripes, stripe);  in refill_full_stripes()
669 		if (stripe == dc->disk.nr_stripes)  in refill_full_stripes()
672 		next_stripe = find_next_zero_bit(dc->disk.full_dirty_stripes,  in refill_full_stripes()
673 						 dc->disk.nr_stripes, stripe);  in refill_full_stripes()
675 		buf->last_scanned = KEY(dc->disk.id,  in refill_full_stripes()
676 					stripe * dc->disk.stripe_size, 0);  in refill_full_stripes()
678 		bch_refill_keybuf(dc->disk.c, buf,  in refill_full_stripes()
679 				  &KEY(dc->disk.id,  in refill_full_stripes()
680 				       next_stripe * dc->disk.stripe_size, 0),  in refill_full_stripes()
691 		if (stripe == dc->disk.nr_stripes) {  in refill_full_stripes()
701 static bool refill_dirty(struct cached_dev *dc)  in refill_dirty()  argument
703 	struct keybuf *buf = &dc->writeback_keys;  in refill_dirty()
704 	struct bkey start = KEY(dc->disk.id, 0, 0);  in refill_dirty()
705 	struct bkey end = KEY(dc->disk.id, MAX_KEY_OFFSET, 0);  in refill_dirty()
717 	if (dc->partial_stripes_expensive) {  in refill_dirty()
718 		refill_full_stripes(dc);  in refill_dirty()
724 	bch_refill_keybuf(dc->disk.c, buf, &end, dirty_pred);  in refill_dirty()
734 	bch_refill_keybuf(dc->disk.c, buf, &start_pos, dirty_pred);  in refill_dirty()
741 	struct cached_dev *dc = arg;  in bch_writeback_thread()  local
742 	struct cache_set *c = dc->disk.c;  in bch_writeback_thread()
745 	bch_ratelimit_reset(&dc->writeback_rate);  in bch_writeback_thread()
749 		down_write(&dc->writeback_lock);  in bch_writeback_thread()
758 		if (!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) &&  in bch_writeback_thread()
759 		    (!atomic_read(&dc->has_dirty) || !dc->writeback_running)) {  in bch_writeback_thread()
760 			up_write(&dc->writeback_lock);  in bch_writeback_thread()
773 		searched_full_index = refill_dirty(dc);  in bch_writeback_thread()
776 		    RB_EMPTY_ROOT(&dc->writeback_keys.keys)) {  in bch_writeback_thread()
777 			atomic_set(&dc->has_dirty, 0);  in bch_writeback_thread()
778 			SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);  in bch_writeback_thread()
779 			bch_write_bdev_super(dc, NULL);  in bch_writeback_thread()
786 			if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags)) {  in bch_writeback_thread()
790 				memset(&dc->sb.set_uuid, 0, 16);  in bch_writeback_thread()
791 				SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);  in bch_writeback_thread()
793 				bch_write_bdev_super(dc, &cl);  in bch_writeback_thread()
796 				up_write(&dc->writeback_lock);  in bch_writeback_thread()
818 		up_write(&dc->writeback_lock);  in bch_writeback_thread()
820 		read_dirty(dc);  in bch_writeback_thread()
823 			unsigned int delay = dc->writeback_delay * HZ;  in bch_writeback_thread()
828 			       !test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))  in bch_writeback_thread()
831 			bch_ratelimit_reset(&dc->writeback_rate);  in bch_writeback_thread()
835 	if (dc->writeback_write_wq)  in bch_writeback_thread()
836 		destroy_workqueue(dc->writeback_write_wq);  in bch_writeback_thread()
838 	cached_dev_put(dc);  in bch_writeback_thread()
1051 void bch_cached_dev_writeback_init(struct cached_dev *dc)  in bch_cached_dev_writeback_init()  argument
1053 	sema_init(&dc->in_flight, 64);  in bch_cached_dev_writeback_init()
1054 	init_rwsem(&dc->writeback_lock);  in bch_cached_dev_writeback_init()
1055 	bch_keybuf_init(&dc->writeback_keys);  in bch_cached_dev_writeback_init()
1057 	dc->writeback_metadata		= true;  in bch_cached_dev_writeback_init()
1058 	dc->writeback_running		= false;  in bch_cached_dev_writeback_init()
1059 	dc->writeback_consider_fragment = true;  in bch_cached_dev_writeback_init()
1060 	dc->writeback_percent		= 10;  in bch_cached_dev_writeback_init()
1061 	dc->writeback_delay		= 30;  in bch_cached_dev_writeback_init()
1062 	atomic_long_set(&dc->writeback_rate.rate, 1024);  in bch_cached_dev_writeback_init()
1063 	dc->writeback_rate_minimum	= 8;  in bch_cached_dev_writeback_init()
1065 	dc->writeback_rate_update_seconds = WRITEBACK_RATE_UPDATE_SECS_DEFAULT;  in bch_cached_dev_writeback_init()
1066 	dc->writeback_rate_p_term_inverse = 40;  in bch_cached_dev_writeback_init()
1067 	dc->writeback_rate_fp_term_low = 1;  in bch_cached_dev_writeback_init()
1068 	dc->writeback_rate_fp_term_mid = 10;  in bch_cached_dev_writeback_init()
1069 	dc->writeback_rate_fp_term_high = 1000;  in bch_cached_dev_writeback_init()
1070 	dc->writeback_rate_i_term_inverse = 10000;  in bch_cached_dev_writeback_init()
1073 	dc->rate_update_retry = 0;  in bch_cached_dev_writeback_init()
1075 	WARN_ON(test_and_clear_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags));  in bch_cached_dev_writeback_init()
1076 	INIT_DELAYED_WORK(&dc->writeback_rate_update, update_writeback_rate);  in bch_cached_dev_writeback_init()
1079 int bch_cached_dev_writeback_start(struct cached_dev *dc)  in bch_cached_dev_writeback_start()  argument
1081 	dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq",  in bch_cached_dev_writeback_start()
1083 	if (!dc->writeback_write_wq)  in bch_cached_dev_writeback_start()
1086 	cached_dev_get(dc);  in bch_cached_dev_writeback_start()
1087 	dc->writeback_thread = kthread_create(bch_writeback_thread, dc,  in bch_cached_dev_writeback_start()
1089 	if (IS_ERR(dc->writeback_thread)) {  in bch_cached_dev_writeback_start()
1090 		cached_dev_put(dc);  in bch_cached_dev_writeback_start()
1091 		destroy_workqueue(dc->writeback_write_wq);  in bch_cached_dev_writeback_start()
1092 		return PTR_ERR(dc->writeback_thread);  in bch_cached_dev_writeback_start()
1094 	dc->writeback_running = true;  in bch_cached_dev_writeback_start()
1096 	WARN_ON(test_and_set_bit(BCACHE_DEV_WB_RUNNING, &dc->disk.flags));  in bch_cached_dev_writeback_start()
1097 	schedule_delayed_work(&dc->writeback_rate_update,  in bch_cached_dev_writeback_start()
1098 			      dc->writeback_rate_update_seconds * HZ);  in bch_cached_dev_writeback_start()
1100 	bch_writeback_queue(dc);  in bch_cached_dev_writeback_start()