146 	EBUG_ON(!trans->c->btree_write_buffer.flushing.pin.seq);  in wb_flush_one()
147 	EBUG_ON(trans->c->btree_write_buffer.flushing.pin.seq > wb->journal_seq);  in wb_flush_one()
228 	bch2_journal_pin_add(j, wb->inc.keys.data[0].journal_seq, &wb->flushing.pin,  in move_keys_from_inc_to_flushing()
231 	darray_resize(&wb->flushing.keys, min_t(size_t, 1U << 20, wb->flushing.keys.nr + wb->inc.keys.nr));  in move_keys_from_inc_to_flushing()
232 	darray_resize(&wb->sorted, wb->flushing.keys.size);  in move_keys_from_inc_to_flushing()
234 	if (!wb->flushing.keys.nr && wb->sorted.size >= wb->inc.keys.nr) {  in move_keys_from_inc_to_flushing()
235 		swap(wb->flushing.keys, wb->inc.keys);  in move_keys_from_inc_to_flushing()
239 	size_t nr = min(darray_room(wb->flushing.keys),  in move_keys_from_inc_to_flushing()
240 			wb->sorted.size - wb->flushing.keys.nr);  in move_keys_from_inc_to_flushing()
243 	memcpy(&darray_top(wb->flushing.keys),  in move_keys_from_inc_to_flushing()
251 	wb->flushing.keys.nr	+= nr;  in move_keys_from_inc_to_flushing()
266 	BUG_ON(wb->sorted.size < wb->flushing.keys.nr);  in move_keys_from_inc_to_flushing()
287 	for (size_t i = 0; i < wb->flushing.keys.nr; i++) {  in bch2_btree_write_buffer_flush_locked()
289 		wb->sorted.data[i].btree = wb->flushing.keys.data[i].btree;  in bch2_btree_write_buffer_flush_locked()
290 		memcpy(&wb->sorted.data[i].pos, &wb->flushing.keys.data[i].k.k.p, sizeof(struct bpos));  in bch2_btree_write_buffer_flush_locked()
292 	wb->sorted.nr = wb->flushing.keys.nr;  in bch2_btree_write_buffer_flush_locked()
299 	 * However, since we're not flushing in the order they appear in the  in bch2_btree_write_buffer_flush_locked()
311 		struct btree_write_buffered_key *k = &wb->flushing.keys.data[i->idx];  in bch2_btree_write_buffer_flush_locked()
314 			prefetch(&wb->flushing.keys.data[n->idx]);  in bch2_btree_write_buffer_flush_locked()
326 			struct btree_write_buffered_key *n = &wb->flushing.keys.data[i[1].idx];  in bch2_btree_write_buffer_flush_locked()
406 		trace_and_count(c, write_buffer_flush_slowpath, trans, slowpath, wb->flushing.keys.nr);  in bch2_btree_write_buffer_flush_locked()
408 		sort(wb->flushing.keys.data,  in bch2_btree_write_buffer_flush_locked()
409 		     wb->flushing.keys.nr,  in bch2_btree_write_buffer_flush_locked()
410 		     sizeof(wb->flushing.keys.data[0]),  in bch2_btree_write_buffer_flush_locked()
413 		darray_for_each(wb->flushing.keys, i) {  in bch2_btree_write_buffer_flush_locked()
424 				bch2_journal_pin_update(j, i->journal_seq, &wb->flushing.pin,  in bch2_btree_write_buffer_flush_locked()
461 			struct btree_write_buffered_key *dst = wb->flushing.keys.data;  in bch2_btree_write_buffer_flush_locked()
463 			darray_for_each(wb->flushing.keys, i)  in bch2_btree_write_buffer_flush_locked()
466 			wb->flushing.keys.nr = dst - wb->flushing.keys.data;  in bch2_btree_write_buffer_flush_locked()
471 		bch2_journal_pin_drop(j, &wb->flushing.pin);  in bch2_btree_write_buffer_flush_locked()
472 		wb->flushing.keys.nr = 0;  in bch2_btree_write_buffer_flush_locked()
476 	trace_write_buffer_flush(trans, wb->flushing.keys.nr, overwritten, fast, 0);  in bch2_btree_write_buffer_flush_locked()
509 		mutex_lock(&wb->flushing.lock);  in btree_write_buffer_flush_seq()
511 		mutex_unlock(&wb->flushing.lock);  in btree_write_buffer_flush_seq()
515 		  (wb->flushing.pin.seq && wb->flushing.pin.seq <= seq)));  in btree_write_buffer_flush_seq()
543 	if (mutex_trylock(&wb->flushing.lock)) {  in bch2_btree_write_buffer_flush_nocheck_rw()
545 		mutex_unlock(&wb->flushing.lock);  in bch2_btree_write_buffer_flush_nocheck_rw()
604 	mutex_lock(&wb->flushing.lock);  in bch2_btree_write_buffer_flush_work()
608 	mutex_unlock(&wb->flushing.lock);  in bch2_btree_write_buffer_flush_work()
644 	if (!ret && dst->wb == &wb->flushing)  in bch2_journal_key_to_wb_slowpath()
645 		ret = darray_resize(&wb->sorted, wb->flushing.keys.size);  in bch2_journal_key_to_wb_slowpath()
648 		if (dst->wb == &c->btree_write_buffer.flushing) {  in bch2_journal_key_to_wb_slowpath()
660 	if (dst->wb == &wb->flushing)  in bch2_journal_key_to_wb_slowpath()
661 		dst->room = min(dst->room, wb->sorted.size - wb->flushing.keys.nr);  in bch2_journal_key_to_wb_slowpath()
678 	if (mutex_trylock(&wb->flushing.lock)) {  in bch2_journal_keys_to_write_buffer_start()
684 		 * wb->flushing, saving us a copy later:  in bch2_journal_keys_to_write_buffer_start()
688 			dst->wb = &wb->flushing;  in bch2_journal_keys_to_write_buffer_start()
690 			mutex_unlock(&wb->flushing.lock);  in bch2_journal_keys_to_write_buffer_start()
699 	if (dst->wb == &wb->flushing)  in bch2_journal_keys_to_write_buffer_start()
700 		dst->room = min(dst->room, wb->sorted.size - wb->flushing.keys.nr);  in bch2_journal_keys_to_write_buffer_start()
743 	if (dst->wb == &wb->flushing)  in bch2_journal_keys_to_write_buffer_end()
744 		mutex_unlock(&wb->flushing.lock);  in bch2_journal_keys_to_write_buffer_end()
792 	return wb_keys_resize(&wb->flushing, new_size) ?:  in bch2_btree_write_buffer_resize()
800 	BUG_ON((wb->inc.keys.nr || wb->flushing.keys.nr) &&  in bch2_fs_btree_write_buffer_exit()
805 	darray_exit(&wb->flushing.keys);  in bch2_fs_btree_write_buffer_exit()
814 	mutex_init(&wb->flushing.lock);  in bch2_fs_btree_write_buffer_init()
821 		darray_make_room(&wb->flushing.keys, initial_size) ?:  in bch2_fs_btree_write_buffer_init()