124  * look find the first ordered struct that has this offset, otherwise
162 	 * ordered extent by calling btrfs_qgroup_release_data() and tracking  in alloc_ordered_extent()
163 	 * the qgroup reserved amount in the ordered extent, so that later after  in alloc_ordered_extent()
164 	 * completing the ordered extent, when running the data delayed ref it  in alloc_ordered_extent()
245 				"inconsistency in ordered tree at offset %llu",  in insert_ordered_extent()
263  * Add an ordered extent to the per-inode tree.
276  * tree is given a single reference on the ordered extent that was inserted, and
279  * Return: the new ordered extent or error pointer.
295 	 * For PREALLOC, we do not use ordered extent members, but  in btrfs_alloc_ordered_extent()
298 	 * So here we always pass 0 as offset for NOCOW/PREALLOC ordered extents,  in btrfs_alloc_ordered_extent()
323  * when an ordered extent is finished.  If the list covers more than one
324  * ordered extent, it is split across multiples.
336 void btrfs_mark_ordered_extent_error(struct btrfs_ordered_extent *ordered)  in btrfs_mark_ordered_extent_error()  argument
338 	if (!test_and_set_bit(BTRFS_ORDERED_IOERR, &ordered->flags))  in btrfs_mark_ordered_extent_error()
339 		mapping_set_error(ordered->inode->vfs_inode.i_mapping, -EIO);  in btrfs_mark_ordered_extent_error()
350 static bool can_finish_ordered_extent(struct btrfs_ordered_extent *ordered,  in can_finish_ordered_extent()  argument
354 	struct btrfs_inode *inode = ordered->inode;  in can_finish_ordered_extent()
365 		 * Ordered flag indicates whether we still have  in can_finish_ordered_extent()
366 		 * pending io unfinished for the ordered extent.  in can_finish_ordered_extent()
376 	if (WARN_ON_ONCE(len > ordered->bytes_left)) {  in can_finish_ordered_extent()
378 "bad ordered extent accounting, root=%llu ino=%llu OE offset=%llu OE len=%llu to_dec=%llu left=%llu…  in can_finish_ordered_extent()
380 			   ordered->file_offset, ordered->num_bytes,  in can_finish_ordered_extent()
381 			   len, ordered->bytes_left);  in can_finish_ordered_extent()
382 		ordered->bytes_left = 0;  in can_finish_ordered_extent()
384 		ordered->bytes_left -= len;  in can_finish_ordered_extent()
388 		set_bit(BTRFS_ORDERED_IOERR, &ordered->flags);  in can_finish_ordered_extent()
390 	if (ordered->bytes_left)  in can_finish_ordered_extent()
394 	 * All the IO of the ordered extent is finished, we need to queue  in can_finish_ordered_extent()
397 	set_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags);  in can_finish_ordered_extent()
398 	cond_wake_up(&ordered->wait);  in can_finish_ordered_extent()
399 	refcount_inc(&ordered->refs);  in can_finish_ordered_extent()
400 	trace_btrfs_ordered_extent_mark_finished(inode, ordered);  in can_finish_ordered_extent()
404 static void btrfs_queue_ordered_fn(struct btrfs_ordered_extent *ordered)  in btrfs_queue_ordered_fn()  argument
406 	struct btrfs_inode *inode = ordered->inode;  in btrfs_queue_ordered_fn()
411 	btrfs_init_work(&ordered->work, finish_ordered_fn, NULL);  in btrfs_queue_ordered_fn()
412 	btrfs_queue_work(wq, &ordered->work);  in btrfs_queue_ordered_fn()
415 void btrfs_finish_ordered_extent(struct btrfs_ordered_extent *ordered,  in btrfs_finish_ordered_extent()  argument
419 	struct btrfs_inode *inode = ordered->inode;  in btrfs_finish_ordered_extent()
426 	ret = can_finish_ordered_extent(ordered, folio, file_offset, len,  in btrfs_finish_ordered_extent()
435 	 * We have marked the ordered extent with BTRFS_ORDERED_IOERR, and we  in btrfs_finish_ordered_extent()
446 	 * logging before ordered extent completion runs in the work queue.  in btrfs_finish_ordered_extent()
449 	 * finds, so if by the time it collects extent maps the ordered extent  in btrfs_finish_ordered_extent()
453 	 * wait for completion of ordered extents in order to reduce latency.  in btrfs_finish_ordered_extent()
456 	 * ordered extents to complete before starting to log.  in btrfs_finish_ordered_extent()
458 	if (!uptodate && !test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags))  in btrfs_finish_ordered_extent()
462 		btrfs_queue_ordered_fn(ordered);  in btrfs_finish_ordered_extent()
466  * Mark all ordered extents io inside the specified range finished.
470  *		 updated to indicate whether the pending ordered io is finished.
475  * This function is called for endio, thus the range must have ordered
498 		/* No ordered extents at all */  in btrfs_mark_ordered_io_finished()
511 			/* No more ordered extents, exit */  in btrfs_mark_ordered_io_finished()
517 			/* Go to next ordered extent and continue */  in btrfs_mark_ordered_io_finished()
532 		 * Now we are definitely inside one ordered extent.  in btrfs_mark_ordered_io_finished()
554  * Finish IO for one ordered extent across a given range.  The range can only
555  * contain one ordered extent.
557  * @cached:	 The cached ordered extent. If not NULL, we can skip the tree
558  *               search and use the ordered extent directly.
559  * 		 Will be also used to store the finished ordered extent.
563  * Return true if the ordered extent is finished in the range, and update
567  * NOTE: The range can NOT cross multiple ordered extents.
568  * Thus caller should ensure the range doesn't cross ordered extents.
596 			   "bad ordered accounting left %llu size %llu",  in btrfs_dec_test_ordered_pending()
621  * used to drop a reference on an ordered extent.  This will free
643  * remove an ordered extent from the tree.  No references are dropped
656 	 * If this is a free space inode the thread has not acquired the ordered  in btrfs_remove_ordered_extent()
740 	struct btrfs_ordered_extent *ordered;  in btrfs_run_ordered_extent_work()  local
742 	ordered = container_of(work, struct btrfs_ordered_extent, flush_work);  in btrfs_run_ordered_extent_work()
743 	btrfs_start_ordered_extent(ordered);  in btrfs_run_ordered_extent_work()
744 	complete(&ordered->completion);  in btrfs_run_ordered_extent_work()
748  * Wait for all the ordered extents in a root. Use @bg as range or do whole
758 	struct btrfs_ordered_extent *ordered, *next;  in btrfs_wait_ordered_extents()  local
776 		ordered = list_first_entry(&splice, struct btrfs_ordered_extent,  in btrfs_wait_ordered_extents()
779 		if (range_end <= ordered->disk_bytenr ||  in btrfs_wait_ordered_extents()
780 		    ordered->disk_bytenr + ordered->disk_num_bytes <= range_start) {  in btrfs_wait_ordered_extents()
781 			list_move_tail(&ordered->root_extent_list, &skipped);  in btrfs_wait_ordered_extents()
786 		list_move_tail(&ordered->root_extent_list,  in btrfs_wait_ordered_extents()
788 		refcount_inc(&ordered->refs);  in btrfs_wait_ordered_extents()
791 		btrfs_init_work(&ordered->flush_work,  in btrfs_wait_ordered_extents()
793 		list_add_tail(&ordered->work_list, &works);  in btrfs_wait_ordered_extents()
794 		btrfs_queue_work(fs_info->flush_workers, &ordered->flush_work);  in btrfs_wait_ordered_extents()
806 	list_for_each_entry_safe(ordered, next, &works, work_list) {  in btrfs_wait_ordered_extents()
807 		list_del_init(&ordered->work_list);  in btrfs_wait_ordered_extents()
808 		wait_for_completion(&ordered->completion);  in btrfs_wait_ordered_extents()
809 		btrfs_put_ordered_extent(ordered);  in btrfs_wait_ordered_extents()
818  * Wait for @nr ordered extents that intersect the @bg, or the whole range of
854  * Start IO and wait for a given ordered extent to finish.
871 	 * If this is a free space inode do not take the ordered extents lockdep  in btrfs_start_ordered_extent_nowriteback()
901  * Used to wait on ordered extents across a large range of bytes.
909 	struct btrfs_ordered_extent *ordered;  in btrfs_wait_ordered_range()  local
928 	 * for any ordered extents that haven't completed yet. This is to make  in btrfs_wait_ordered_range()
930 	 * before the ordered extents complete - to avoid failures (-EEXIST)  in btrfs_wait_ordered_range()
931 	 * when adding the new ordered extents to the ordered tree.  in btrfs_wait_ordered_range()
937 		ordered = btrfs_lookup_first_ordered_extent(inode, end);  in btrfs_wait_ordered_range()
938 		if (!ordered)  in btrfs_wait_ordered_range()
940 		if (ordered->file_offset > orig_end) {  in btrfs_wait_ordered_range()
941 			btrfs_put_ordered_extent(ordered);  in btrfs_wait_ordered_range()
944 		if (ordered->file_offset + ordered->num_bytes <= start) {  in btrfs_wait_ordered_range()
945 			btrfs_put_ordered_extent(ordered);  in btrfs_wait_ordered_range()
948 		btrfs_start_ordered_extent(ordered);  in btrfs_wait_ordered_range()
949 		end = ordered->file_offset;  in btrfs_wait_ordered_range()
951 		 * If the ordered extent had an error save the error but don't  in btrfs_wait_ordered_range()
952 		 * exit without waiting first for all other ordered extents in  in btrfs_wait_ordered_range()
955 		if (test_bit(BTRFS_ORDERED_IOERR, &ordered->flags))  in btrfs_wait_ordered_range()
957 		btrfs_put_ordered_extent(ordered);  in btrfs_wait_ordered_range()
966  * find an ordered extent corresponding to file_offset.  return NULL if
993 /* Since the DIO code tries to lock a wide area we need to look for any ordered
1034  * Adds all ordered extents to the given list. The list ends up sorted by the
1035  * file_offset of the ordered extents.
1046 		struct btrfs_ordered_extent *ordered;  in btrfs_get_ordered_extents_for_logging()  local
1048 		ordered = rb_entry(n, struct btrfs_ordered_extent, rb_node);  in btrfs_get_ordered_extents_for_logging()
1050 		if (test_bit(BTRFS_ORDERED_LOGGED, &ordered->flags))  in btrfs_get_ordered_extents_for_logging()
1053 		ASSERT(list_empty(&ordered->log_list));  in btrfs_get_ordered_extents_for_logging()
1054 		list_add_tail(&ordered->log_list, list);  in btrfs_get_ordered_extents_for_logging()
1055 		refcount_inc(&ordered->refs);  in btrfs_get_ordered_extents_for_logging()
1056 		trace_btrfs_ordered_extent_lookup_for_logging(inode, ordered);  in btrfs_get_ordered_extents_for_logging()
1085  * Lookup the first ordered extent that overlaps the range
1089  * that this one won't return any ordered extent that does not overlap the range.
1091  * ensures the first ordered extent gets returned.
1107 	 * And __tree_search() can't return the adjacent ordered extents  in btrfs_lookup_first_ordered_range()
1119 			 * Direct hit, got an ordered extent that starts at  in btrfs_lookup_first_ordered_range()
1149 	/* No ordered extent in the range */  in btrfs_lookup_first_ordered_range()
1162  * Lock the passed range and ensures all pending ordered extents in it are run
1165  * @inode:        Inode whose ordered tree is to be searched
1179 	struct btrfs_ordered_extent *ordered;  in btrfs_lock_and_flush_ordered_range()  local
1188 		ordered = btrfs_lookup_ordered_range(inode, start,  in btrfs_lock_and_flush_ordered_range()
1190 		if (!ordered) {  in btrfs_lock_and_flush_ordered_range()
1201 		btrfs_start_ordered_extent(ordered);  in btrfs_lock_and_flush_ordered_range()
1202 		btrfs_put_ordered_extent(ordered);  in btrfs_lock_and_flush_ordered_range()
1207  * Lock the passed range and ensure all pending ordered extents in it are run
1216 	struct btrfs_ordered_extent *ordered;  in btrfs_try_lock_ordered_range()  local
1221 	ordered = btrfs_lookup_ordered_range(inode, start, end - start + 1);  in btrfs_try_lock_ordered_range()
1222 	if (!ordered)  in btrfs_try_lock_ordered_range()
1225 	btrfs_put_ordered_extent(ordered);  in btrfs_try_lock_ordered_range()
1231 /* Split out a new ordered extent for this first @len bytes of @ordered. */
1233 			struct btrfs_ordered_extent *ordered, u64 len)  in btrfs_split_ordered_extent()  argument
1235 	struct btrfs_inode *inode = ordered->inode;  in btrfs_split_ordered_extent()
1238 	u64 file_offset = ordered->file_offset;  in btrfs_split_ordered_extent()
1239 	u64 disk_bytenr = ordered->disk_bytenr;  in btrfs_split_ordered_extent()
1240 	unsigned long flags = ordered->flags;  in btrfs_split_ordered_extent()
1246 	trace_btrfs_ordered_extent_split(inode, ordered);  in btrfs_split_ordered_extent()
1251 	 * The entire bio must be covered by the ordered extent, but we can't  in btrfs_split_ordered_extent()
1254 	if (WARN_ON_ONCE(len >= ordered->num_bytes))  in btrfs_split_ordered_extent()
1257 	 * If our ordered extent had an error there's no point in continuing.  in btrfs_split_ordered_extent()
1260 	 * iterates over all existing ordered extents and sets the flag  in btrfs_split_ordered_extent()
1268 	/* We cannot split partially completed ordered extents. */  in btrfs_split_ordered_extent()
1269 	if (ordered->bytes_left) {  in btrfs_split_ordered_extent()
1271 		if (WARN_ON_ONCE(ordered->bytes_left != ordered->disk_num_bytes))  in btrfs_split_ordered_extent()
1274 	/* We cannot split a compressed ordered extent. */  in btrfs_split_ordered_extent()
1275 	if (WARN_ON_ONCE(ordered->disk_num_bytes != ordered->num_bytes))  in btrfs_split_ordered_extent()
1279 				   len, 0, flags, ordered->compress_type);  in btrfs_split_ordered_extent()
1289 	 * disk_num_bytes fields of the ordered extent below. And we disable  in btrfs_split_ordered_extent()
1294 	 * btrfs_wait_ordered_extents() getting the trimmed ordered extent  in btrfs_split_ordered_extent()
1295 	 * before we insert the new one, because even if it gets the ordered  in btrfs_split_ordered_extent()
1297 	 * uses it or during its use, the ordered extent might have been  in btrfs_split_ordered_extent()
1298 	 * trimmed in the meanwhile, and it missed the new ordered extent.  in btrfs_split_ordered_extent()
1307 	 * We don't have overlapping ordered extents (that would imply double  in btrfs_split_ordered_extent()
1309 	 * does not cross the ordered extent's num_bytes field, so there's  in btrfs_split_ordered_extent()
1312 	ordered->file_offset += len;  in btrfs_split_ordered_extent()
1313 	ordered->disk_bytenr += len;  in btrfs_split_ordered_extent()
1314 	ordered->num_bytes -= len;  in btrfs_split_ordered_extent()
1315 	ordered->disk_num_bytes -= len;  in btrfs_split_ordered_extent()
1316 	ordered->ram_bytes -= len;  in btrfs_split_ordered_extent()
1318 	if (test_bit(BTRFS_ORDERED_IO_DONE, &ordered->flags)) {  in btrfs_split_ordered_extent()
1319 		ASSERT(ordered->bytes_left == 0);  in btrfs_split_ordered_extent()
1322 		ordered->bytes_left -= len;  in btrfs_split_ordered_extent()
1325 	if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags)) {  in btrfs_split_ordered_extent()
1326 		if (ordered->truncated_len > len) {  in btrfs_split_ordered_extent()
1327 			ordered->truncated_len -= len;  in btrfs_split_ordered_extent()
1329 			new->truncated_len = ordered->truncated_len;  in btrfs_split_ordered_extent()
1330 			ordered->truncated_len = 0;  in btrfs_split_ordered_extent()
1334 	list_for_each_entry_safe(sum, tmpsum, &ordered->list, list) {  in btrfs_split_ordered_extent()
1344 			"inconsistency in ordered tree at offset %llu after split",  in btrfs_split_ordered_extent()