39 unsigned bch2_journal_dev_buckets_available(struct journal *j,  in bch2_journal_dev_buckets_available()  argument
59 void bch2_journal_set_watermark(struct journal *j)  in bch2_journal_set_watermark()  argument
61 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_set_watermark()
62 	bool low_on_space = j->space[journal_space_clean].total * 4 <=  in bch2_journal_set_watermark()
63 		j->space[journal_space_total].total;  in bch2_journal_set_watermark()
64 	bool low_on_pin = fifo_free(&j->pin) < j->pin.size / 4;  in bch2_journal_set_watermark()
75 	mod_bit(JOURNAL_space_low, &j->flags, low_on_space || low_on_pin);  in bch2_journal_set_watermark()
77 	swap(watermark, j->watermark);  in bch2_journal_set_watermark()
78 	if (watermark > j->watermark)  in bch2_journal_set_watermark()
79 		journal_wake(j);  in bch2_journal_set_watermark()
83 journal_dev_space_available(struct journal *j, struct bch_dev *ca,  in journal_dev_space_available()  argument
86 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in journal_dev_space_available()
98 	buckets = bch2_journal_dev_buckets_available(j, ja, from);  in journal_dev_space_available()
105 	for (seq = journal_last_unwritten_seq(j);  in journal_dev_space_available()
106 	     seq <= journal_cur_seq(j);  in journal_dev_space_available()
108 		unwritten = j->buf[seq & JOURNAL_BUF_MASK].sectors;  in journal_dev_space_available()
141 static struct journal_space __journal_space_available(struct journal *j, unsigned nr_devs_want,  in __journal_space_available()  argument
144 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in __journal_space_available()
158 		space = journal_dev_space_available(j, ca, from);  in __journal_space_available()
187 void bch2_journal_space_available(struct journal *j)  in bch2_journal_space_available()  argument
189 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_space_available()
191 	unsigned max_entry_size	 = min(j->buf[0].buf_size >> 9,  in bch2_journal_space_available()
192 				       j->buf[1].buf_size >> 9);  in bch2_journal_space_available()
197 	lockdep_assert_held(&j->lock);  in bch2_journal_space_available()
207 		       ja->bucket_seq[ja->dirty_idx] < journal_last_seq(j))  in bch2_journal_space_available()
211 		       ja->bucket_seq[ja->dirty_idx_ondisk] < j->last_seq_ondisk)  in bch2_journal_space_available()
214 		can_discard |= __should_discard_bucket(j, ja);  in bch2_journal_space_available()
220 	j->can_discard = can_discard;  in bch2_journal_space_available()
242 		j->space[i] = __journal_space_available(j, nr_devs_want, i);  in bch2_journal_space_available()
244 	clean_ondisk	= j->space[journal_space_clean_ondisk].total;  in bch2_journal_space_available()
245 	clean		= j->space[journal_space_clean].total;  in bch2_journal_space_available()
246 	total		= j->space[journal_space_total].total;  in bch2_journal_space_available()
248 	if (!j->space[journal_space_discarded].next_entry)  in bch2_journal_space_available()
251 	if ((j->space[journal_space_clean_ondisk].next_entry <  in bch2_journal_space_available()
252 	     j->space[journal_space_clean_ondisk].total) &&  in bch2_journal_space_available()
255 		set_bit(JOURNAL_may_skip_flush, &j->flags);  in bch2_journal_space_available()
257 		clear_bit(JOURNAL_may_skip_flush, &j->flags);  in bch2_journal_space_available()
259 	bch2_journal_set_watermark(j);  in bch2_journal_space_available()
261 	j->cur_entry_sectors	= !ret  in bch2_journal_space_available()
262 		? j->space[journal_space_discarded].next_entry  in bch2_journal_space_available()
264 	j->cur_entry_error	= ret;  in bch2_journal_space_available()
267 		journal_wake(j);  in bch2_journal_space_available()
272 static bool __should_discard_bucket(struct journal *j, struct journal_device *ja)  in __should_discard_bucket()  argument
276 	return bch2_journal_dev_buckets_available(j, ja, journal_space_discarded) <  in __should_discard_bucket()
281 static bool should_discard_bucket(struct journal *j, struct journal_device *ja)  in should_discard_bucket()  argument
283 	spin_lock(&j->lock);  in should_discard_bucket()
284 	bool ret = __should_discard_bucket(j, ja);  in should_discard_bucket()
285 	spin_unlock(&j->lock);  in should_discard_bucket()
294 void bch2_journal_do_discards(struct journal *j)  in bch2_journal_do_discards()  argument
296 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_do_discards()
298 	mutex_lock(&j->discard_lock);  in bch2_journal_do_discards()
303 		while (should_discard_bucket(j, ja)) {  in bch2_journal_do_discards()
312 			spin_lock(&j->lock);  in bch2_journal_do_discards()
315 			bch2_journal_space_available(j);  in bch2_journal_do_discards()
316 			spin_unlock(&j->lock);  in bch2_journal_do_discards()
320 	mutex_unlock(&j->discard_lock);  in bch2_journal_do_discards()
328 void bch2_journal_reclaim_fast(struct journal *j)  in bch2_journal_reclaim_fast()  argument
332 	lockdep_assert_held(&j->lock);  in bch2_journal_reclaim_fast()
338 	while (!fifo_empty(&j->pin) &&  in bch2_journal_reclaim_fast()
339 	       j->pin.front <= j->seq_ondisk &&  in bch2_journal_reclaim_fast()
340 	       !atomic_read(&fifo_peek_front(&j->pin).count)) {  in bch2_journal_reclaim_fast()
341 		j->pin.front++;  in bch2_journal_reclaim_fast()
346 		bch2_journal_space_available(j);  in bch2_journal_reclaim_fast()
347 		__closure_wake_up(&j->reclaim_flush_wait);  in bch2_journal_reclaim_fast()
351 bool __bch2_journal_pin_put(struct journal *j, u64 seq)  in __bch2_journal_pin_put()  argument
353 	struct journal_entry_pin_list *pin_list = journal_seq_pin(j, seq);  in __bch2_journal_pin_put()
358 void bch2_journal_pin_put(struct journal *j, u64 seq)  in bch2_journal_pin_put()  argument
360 	if (__bch2_journal_pin_put(j, seq)) {  in bch2_journal_pin_put()
361 		spin_lock(&j->lock);  in bch2_journal_pin_put()
362 		bch2_journal_reclaim_fast(j);  in bch2_journal_pin_put()
363 		spin_unlock(&j->lock);  in bch2_journal_pin_put()
367 static inline bool __journal_pin_drop(struct journal *j,  in __journal_pin_drop()  argument
375 	if (j->flush_in_progress == pin)  in __journal_pin_drop()
376 		j->flush_in_progress_dropped = true;  in __journal_pin_drop()
378 	pin_list = journal_seq_pin(j, pin->seq);  in __journal_pin_drop()
382 	if (j->reclaim_flush_wait.list.first)  in __journal_pin_drop()
383 		__closure_wake_up(&j->reclaim_flush_wait);  in __journal_pin_drop()
390 		pin_list == &fifo_peek_front(&j->pin);  in __journal_pin_drop()
393 void bch2_journal_pin_drop(struct journal *j,  in bch2_journal_pin_drop()  argument
396 	spin_lock(&j->lock);  in bch2_journal_pin_drop()
397 	if (__journal_pin_drop(j, pin))  in bch2_journal_pin_drop()
398 		bch2_journal_reclaim_fast(j);  in bch2_journal_pin_drop()
399 	spin_unlock(&j->lock);  in bch2_journal_pin_drop()
417 static inline void bch2_journal_pin_set_locked(struct journal *j, u64 seq,  in bch2_journal_pin_set_locked()  argument
422 	struct journal_entry_pin_list *pin_list = journal_seq_pin(j, seq);  in bch2_journal_pin_set_locked()
435 	    j->reclaim_flush_wait.list.first)  in bch2_journal_pin_set_locked()
436 		__closure_wake_up(&j->reclaim_flush_wait);  in bch2_journal_pin_set_locked()
441 void bch2_journal_pin_copy(struct journal *j,  in bch2_journal_pin_copy()  argument
446 	spin_lock(&j->lock);  in bch2_journal_pin_copy()
450 	if (seq < journal_last_seq(j)) {  in bch2_journal_pin_copy()
457 		spin_unlock(&j->lock);  in bch2_journal_pin_copy()
461 	bool reclaim = __journal_pin_drop(j, dst);  in bch2_journal_pin_copy()
463 	bch2_journal_pin_set_locked(j, seq, dst, flush_fn, journal_pin_type(dst, flush_fn));  in bch2_journal_pin_copy()
466 		bch2_journal_reclaim_fast(j);  in bch2_journal_pin_copy()
472 	if (seq == journal_last_seq(j))  in bch2_journal_pin_copy()
473 		journal_wake(j);  in bch2_journal_pin_copy()
474 	spin_unlock(&j->lock);  in bch2_journal_pin_copy()
477 void bch2_journal_pin_set(struct journal *j, u64 seq,  in bch2_journal_pin_set()  argument
481 	spin_lock(&j->lock);  in bch2_journal_pin_set()
483 	BUG_ON(seq < journal_last_seq(j));  in bch2_journal_pin_set()
485 	bool reclaim = __journal_pin_drop(j, pin);  in bch2_journal_pin_set()
487 	bch2_journal_pin_set_locked(j, seq, pin, flush_fn, journal_pin_type(pin, flush_fn));  in bch2_journal_pin_set()
490 		bch2_journal_reclaim_fast(j);  in bch2_journal_pin_set()
495 	if (seq == journal_last_seq(j))  in bch2_journal_pin_set()
496 		journal_wake(j);  in bch2_journal_pin_set()
498 	spin_unlock(&j->lock);  in bch2_journal_pin_set()
506 void bch2_journal_pin_flush(struct journal *j, struct journal_entry_pin *pin)  in bch2_journal_pin_flush()  argument
510 	wait_event(j->pin_flush_wait, j->flush_in_progress != pin);  in bch2_journal_pin_flush()
523 journal_get_next_pin(struct journal *j,  in journal_get_next_pin()  argument
532 	fifo_for_each_entry_ptr(pin_list, &j->pin, *seq) {  in journal_get_next_pin()
550 static size_t journal_flush_pins(struct journal *j,  in journal_flush_pins()  argument
563 	lockdep_assert_held(&j->reclaim_lock);  in journal_flush_pins()
581 		j->last_flushed = jiffies;  in journal_flush_pins()
583 		spin_lock(&j->lock);  in journal_flush_pins()
584 		pin = journal_get_next_pin(j, seq_to_flush,  in journal_flush_pins()
588 			BUG_ON(j->flush_in_progress);  in journal_flush_pins()
589 			j->flush_in_progress = pin;  in journal_flush_pins()
590 			j->flush_in_progress_dropped = false;  in journal_flush_pins()
593 		spin_unlock(&j->lock);  in journal_flush_pins()
604 		err = flush_fn(j, pin, seq);  in journal_flush_pins()
606 		spin_lock(&j->lock);  in journal_flush_pins()
608 		if (likely(!err && !j->flush_in_progress_dropped))  in journal_flush_pins()
609 			list_move(&pin->list, &journal_seq_pin(j, seq)->flushed[journal_pin_type(pin, flush_fn)]);  in journal_flush_pins()
610 		j->flush_in_progress = NULL;  in journal_flush_pins()
611 		j->flush_in_progress_dropped = false;  in journal_flush_pins()
612 		spin_unlock(&j->lock);  in journal_flush_pins()
614 		wake_up(&j->pin_flush_wait);  in journal_flush_pins()
625 static u64 journal_seq_to_flush(struct journal *j)  in journal_seq_to_flush()  argument
627 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in journal_seq_to_flush()
630 	guard(spinlock)(&j->lock);  in journal_seq_to_flush()
650 		     (s64) journal_cur_seq(j) -  in journal_seq_to_flush()
651 		     (j->pin.size >> 1));  in journal_seq_to_flush()
675 static int __bch2_journal_reclaim(struct journal *j, bool direct, bool kicked)  in __bch2_journal_reclaim()  argument
677 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in __bch2_journal_reclaim()
691 	lockdep_assert_held(&j->reclaim_lock);  in __bch2_journal_reclaim()
698 		ret = bch2_journal_error(j);  in __bch2_journal_reclaim()
703 		bch2_journal_do_discards(j);  in __bch2_journal_reclaim()
705 		seq_to_flush = journal_seq_to_flush(j);  in __bch2_journal_reclaim()
712 		if (time_after(jiffies, j->last_flushed +  in __bch2_journal_reclaim()
716 		if (j->watermark != BCH_WATERMARK_stripe)  in __bch2_journal_reclaim()
732 		nr_flushed = journal_flush_pins(j, seq_to_flush,  in __bch2_journal_reclaim()
737 			j->nr_direct_reclaim += nr_flushed;  in __bch2_journal_reclaim()
739 			j->nr_background_reclaim += nr_flushed;  in __bch2_journal_reclaim()
743 			wake_up(&j->reclaim_wait);  in __bch2_journal_reclaim()
751 int bch2_journal_reclaim(struct journal *j)  in bch2_journal_reclaim()  argument
753 	return __bch2_journal_reclaim(j, true, true);  in bch2_journal_reclaim()
758 	struct journal *j = arg;  in bch2_journal_reclaim_thread()  local
759 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_reclaim_thread()
766 	j->last_flushed = jiffies;  in bch2_journal_reclaim_thread()
769 		bool kicked = j->reclaim_kicked;  in bch2_journal_reclaim_thread()
771 		j->reclaim_kicked = false;  in bch2_journal_reclaim_thread()
773 		mutex_lock(&j->reclaim_lock);  in bch2_journal_reclaim_thread()
774 		ret = __bch2_journal_reclaim(j, false, kicked);  in bch2_journal_reclaim_thread()
775 		mutex_unlock(&j->reclaim_lock);  in bch2_journal_reclaim_thread()
779 		j->next_reclaim = j->last_flushed + delay;  in bch2_journal_reclaim_thread()
781 		if (!time_in_range(j->next_reclaim, now, now + delay))  in bch2_journal_reclaim_thread()
782 			j->next_reclaim = now + delay;  in bch2_journal_reclaim_thread()
788 			if (j->reclaim_kicked)  in bch2_journal_reclaim_thread()
791 			spin_lock(&j->lock);  in bch2_journal_reclaim_thread()
792 			journal_empty = fifo_empty(&j->pin);  in bch2_journal_reclaim_thread()
793 			spin_unlock(&j->lock);  in bch2_journal_reclaim_thread()
795 			long timeout = j->next_reclaim - jiffies;  in bch2_journal_reclaim_thread()
810 void bch2_journal_reclaim_stop(struct journal *j)  in bch2_journal_reclaim_stop()  argument
812 	struct task_struct *p = j->reclaim_thread;  in bch2_journal_reclaim_stop()
814 	j->reclaim_thread = NULL;  in bch2_journal_reclaim_stop()
822 int bch2_journal_reclaim_start(struct journal *j)  in bch2_journal_reclaim_start()  argument
824 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_reclaim_start()
828 	if (j->reclaim_thread)  in bch2_journal_reclaim_start()
831 	p = kthread_create(bch2_journal_reclaim_thread, j,  in bch2_journal_reclaim_start()
839 	j->reclaim_thread = p;  in bch2_journal_reclaim_start()
844 static bool journal_pins_still_flushing(struct journal *j, u64 seq_to_flush,  in journal_pins_still_flushing()  argument
850 	spin_lock(&j->lock);  in journal_pins_still_flushing()
851 	fifo_for_each_entry_ptr(pin_list, &j->pin, seq) {  in journal_pins_still_flushing()
859 				spin_unlock(&j->lock);  in journal_pins_still_flushing()
863 	spin_unlock(&j->lock);  in journal_pins_still_flushing()
868 static bool journal_flush_pins_or_still_flushing(struct journal *j, u64 seq_to_flush,  in journal_flush_pins_or_still_flushing()  argument
871 	return  journal_flush_pins(j, seq_to_flush, types, 0, 0, 0) ||  in journal_flush_pins_or_still_flushing()
872 		journal_pins_still_flushing(j, seq_to_flush, types);  in journal_flush_pins_or_still_flushing()
875 static int journal_flush_done(struct journal *j, u64 seq_to_flush,  in journal_flush_done()  argument
880 	ret = bch2_journal_error(j);  in journal_flush_done()
884 	mutex_lock(&j->reclaim_lock);  in journal_flush_done()
889 		if (journal_flush_pins_or_still_flushing(j, seq_to_flush, BIT(type))) {  in journal_flush_done()
894 	if (seq_to_flush > journal_cur_seq(j))  in journal_flush_done()
895 		bch2_journal_entry_close(j);  in journal_flush_done()
897 	spin_lock(&j->lock);  in journal_flush_done()
902 	ret = !test_bit(JOURNAL_replay_done, &j->flags) ||  in journal_flush_done()
903 		journal_last_seq(j) > seq_to_flush ||  in journal_flush_done()
904 		!fifo_used(&j->pin);  in journal_flush_done()
906 	spin_unlock(&j->lock);  in journal_flush_done()
908 	mutex_unlock(&j->reclaim_lock);  in journal_flush_done()
913 bool bch2_journal_flush_pins(struct journal *j, u64 seq_to_flush)  in bch2_journal_flush_pins()  argument
918 	if (!test_bit(JOURNAL_running, &j->flags))  in bch2_journal_flush_pins()
921 	closure_wait_event(&j->reclaim_flush_wait,  in bch2_journal_flush_pins()
922 		journal_flush_done(j, seq_to_flush, &did_work));  in bch2_journal_flush_pins()
927 int bch2_journal_flush_device_pins(struct journal *j, int dev_idx)  in bch2_journal_flush_device_pins()  argument
929 	struct bch_fs *c = container_of(j, struct bch_fs, journal);  in bch2_journal_flush_device_pins()
934 	spin_lock(&j->lock);  in bch2_journal_flush_device_pins()
935 	fifo_for_each_entry_ptr(p, &j->pin, iter)  in bch2_journal_flush_device_pins()
940 	spin_unlock(&j->lock);  in bch2_journal_flush_device_pins()
942 	bch2_journal_flush_pins(j, seq);  in bch2_journal_flush_device_pins()
944 	ret = bch2_journal_error(j);  in bch2_journal_flush_device_pins()
963 	spin_lock(&j->lock);  in bch2_journal_flush_device_pins()
967 		seq = max(seq, journal_last_seq(j));  in bch2_journal_flush_device_pins()
968 		if (seq >= j->pin.back)  in bch2_journal_flush_device_pins()
971 					 journal_seq_pin(j, seq)->devs);  in bch2_journal_flush_device_pins()
975 			spin_unlock(&j->lock);  in bch2_journal_flush_device_pins()
977 			spin_lock(&j->lock);  in bch2_journal_flush_device_pins()
980 	spin_unlock(&j->lock);  in bch2_journal_flush_device_pins()
988 bool bch2_journal_seq_pins_to_text(struct printbuf *out, struct journal *j, u64 *seq)  in bch2_journal_seq_pins_to_text()  argument
993 	spin_lock(&j->lock);  in bch2_journal_seq_pins_to_text()
994 	if (!test_bit(JOURNAL_running, &j->flags)) {  in bch2_journal_seq_pins_to_text()
995 		spin_unlock(&j->lock);  in bch2_journal_seq_pins_to_text()
999 	*seq = max(*seq, j->pin.front);  in bch2_journal_seq_pins_to_text()
1001 	if (*seq >= j->pin.back) {  in bch2_journal_seq_pins_to_text()
1002 		spin_unlock(&j->lock);  in bch2_journal_seq_pins_to_text()
1008 	pin_list = journal_seq_pin(j, *seq);  in bch2_journal_seq_pins_to_text()
1026 	spin_unlock(&j->lock);  in bch2_journal_seq_pins_to_text()
1031 void bch2_journal_pins_to_text(struct printbuf *out, struct journal *j)  in bch2_journal_pins_to_text()  argument
1035 	while (!bch2_journal_seq_pins_to_text(out, j, &seq))  in bch2_journal_pins_to_text()