724 static void ceph_msg_data_bio_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_cursor_init()  argument
727 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bio_cursor_init()
728 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_cursor_init()
730 	cursor->resid = min_t(size_t, length, data->bio_length);  in ceph_msg_data_bio_cursor_init()
732 	if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_cursor_init()
733 		it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_cursor_init()
735 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_cursor_init()
738 static struct page *ceph_msg_data_bio_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_next()  argument
742 	struct bio_vec bv = bio_iter_iovec(cursor->bio_iter.bio,  in ceph_msg_data_bio_next()
743 					   cursor->bio_iter.iter);  in ceph_msg_data_bio_next()
750 static bool ceph_msg_data_bio_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bio_advance()  argument
753 	struct ceph_bio_iter *it = &cursor->bio_iter;  in ceph_msg_data_bio_advance()
756 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bio_advance()
758 	cursor->resid -= bytes;  in ceph_msg_data_bio_advance()
761 	if (!cursor->resid)  in ceph_msg_data_bio_advance()
771 		if (cursor->resid < it->iter.bi_size)  in ceph_msg_data_bio_advance()
772 			it->iter.bi_size = cursor->resid;  in ceph_msg_data_bio_advance()
775 	BUG_ON(cursor->resid < bio_iter_len(it->bio, it->iter));  in ceph_msg_data_bio_advance()
780 static void ceph_msg_data_bvecs_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_cursor_init()  argument
783 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_bvecs_cursor_init()
786 	cursor->resid = min_t(size_t, length, data->bvec_pos.iter.bi_size);  in ceph_msg_data_bvecs_cursor_init()
787 	cursor->bvec_iter = data->bvec_pos.iter;  in ceph_msg_data_bvecs_cursor_init()
788 	cursor->bvec_iter.bi_size = cursor->resid;  in ceph_msg_data_bvecs_cursor_init()
790 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_cursor_init()
793 static struct page *ceph_msg_data_bvecs_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_next()  argument
797 	struct bio_vec bv = bvec_iter_bvec(cursor->data->bvec_pos.bvecs,  in ceph_msg_data_bvecs_next()
798 					   cursor->bvec_iter);  in ceph_msg_data_bvecs_next()
805 static bool ceph_msg_data_bvecs_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_bvecs_advance()  argument
808 	struct bio_vec *bvecs = cursor->data->bvec_pos.bvecs;  in ceph_msg_data_bvecs_advance()
809 	struct page *page = bvec_iter_page(bvecs, cursor->bvec_iter);  in ceph_msg_data_bvecs_advance()
811 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_bvecs_advance()
812 	BUG_ON(bytes > bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
813 	cursor->resid -= bytes;  in ceph_msg_data_bvecs_advance()
814 	bvec_iter_advance(bvecs, &cursor->bvec_iter, bytes);  in ceph_msg_data_bvecs_advance()
816 	if (!cursor->resid)  in ceph_msg_data_bvecs_advance()
819 	if (!bytes || (cursor->bvec_iter.bi_bvec_done &&  in ceph_msg_data_bvecs_advance()
820 		       page == bvec_iter_page(bvecs, cursor->bvec_iter)))  in ceph_msg_data_bvecs_advance()
823 	BUG_ON(cursor->resid < bvec_iter_len(bvecs, cursor->bvec_iter));  in ceph_msg_data_bvecs_advance()
831 static void ceph_msg_data_pages_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_cursor_init()  argument
834 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_cursor_init()
842 	cursor->resid = min(length, data->length);  in ceph_msg_data_pages_cursor_init()
844 	cursor->page_offset = data->alignment & ~PAGE_MASK;  in ceph_msg_data_pages_cursor_init()
845 	cursor->page_index = 0;  in ceph_msg_data_pages_cursor_init()
847 	cursor->page_count = (unsigned short)page_count;  in ceph_msg_data_pages_cursor_init()
848 	BUG_ON(length > SIZE_MAX - cursor->page_offset);  in ceph_msg_data_pages_cursor_init()
852 ceph_msg_data_pages_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_next()  argument
855 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pages_next()
859 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_next()
860 	BUG_ON(cursor->page_offset >= PAGE_SIZE);  in ceph_msg_data_pages_next()
862 	*page_offset = cursor->page_offset;  in ceph_msg_data_pages_next()
863 	*length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset);  in ceph_msg_data_pages_next()
864 	return data->pages[cursor->page_index];  in ceph_msg_data_pages_next()
867 static bool ceph_msg_data_pages_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pages_advance()  argument
870 	BUG_ON(cursor->data->type != CEPH_MSG_DATA_PAGES);  in ceph_msg_data_pages_advance()
872 	BUG_ON(cursor->page_offset + bytes > PAGE_SIZE);  in ceph_msg_data_pages_advance()
874 	/* Advance the cursor page offset */  in ceph_msg_data_pages_advance()
876 	cursor->resid -= bytes;  in ceph_msg_data_pages_advance()
877 	cursor->page_offset = (cursor->page_offset + bytes) & ~PAGE_MASK;  in ceph_msg_data_pages_advance()
878 	if (!bytes || cursor->page_offset)  in ceph_msg_data_pages_advance()
881 	if (!cursor->resid)  in ceph_msg_data_pages_advance()
886 	BUG_ON(cursor->page_index >= cursor->page_count);  in ceph_msg_data_pages_advance()
887 	cursor->page_index++;  in ceph_msg_data_pages_advance()
896 ceph_msg_data_pagelist_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_cursor_init()  argument
899 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_cursor_init()
914 	cursor->resid = min(length, pagelist->length);  in ceph_msg_data_pagelist_cursor_init()
915 	cursor->page = page;  in ceph_msg_data_pagelist_cursor_init()
916 	cursor->offset = 0;  in ceph_msg_data_pagelist_cursor_init()
920 ceph_msg_data_pagelist_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_next()  argument
923 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_next()
931 	BUG_ON(!cursor->page);  in ceph_msg_data_pagelist_next()
932 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_next()
935 	*page_offset = cursor->offset & ~PAGE_MASK;  in ceph_msg_data_pagelist_next()
936 	*length = min_t(size_t, cursor->resid, PAGE_SIZE - *page_offset);  in ceph_msg_data_pagelist_next()
937 	return cursor->page;  in ceph_msg_data_pagelist_next()
940 static bool ceph_msg_data_pagelist_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_pagelist_advance()  argument
943 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_pagelist_advance()
951 	BUG_ON(cursor->offset + cursor->resid != pagelist->length);  in ceph_msg_data_pagelist_advance()
952 	BUG_ON((cursor->offset & ~PAGE_MASK) + bytes > PAGE_SIZE);  in ceph_msg_data_pagelist_advance()
954 	/* Advance the cursor offset */  in ceph_msg_data_pagelist_advance()
956 	cursor->resid -= bytes;  in ceph_msg_data_pagelist_advance()
957 	cursor->offset += bytes;  in ceph_msg_data_pagelist_advance()
959 	if (!bytes || cursor->offset & ~PAGE_MASK)  in ceph_msg_data_pagelist_advance()
962 	if (!cursor->resid)  in ceph_msg_data_pagelist_advance()
967 	BUG_ON(list_is_last(&cursor->page->lru, &pagelist->head));  in ceph_msg_data_pagelist_advance()
968 	cursor->page = list_next_entry(cursor->page, lru);  in ceph_msg_data_pagelist_advance()
972 static void ceph_msg_data_iter_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_iter_cursor_init()  argument
975 	struct ceph_msg_data *data = cursor->data;  in ceph_msg_data_iter_cursor_init()
977 	cursor->iov_iter = data->iter;  in ceph_msg_data_iter_cursor_init()
978 	cursor->lastlen = 0;  in ceph_msg_data_iter_cursor_init()
979 	iov_iter_truncate(&cursor->iov_iter, length);  in ceph_msg_data_iter_cursor_init()
980 	cursor->resid = iov_iter_count(&cursor->iov_iter);  in ceph_msg_data_iter_cursor_init()
983 static struct page *ceph_msg_data_iter_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_iter_next()  argument
989 	if (cursor->lastlen)  in ceph_msg_data_iter_next()
990 		iov_iter_revert(&cursor->iov_iter, cursor->lastlen);  in ceph_msg_data_iter_next()
992 	len = iov_iter_get_pages2(&cursor->iov_iter, &page, PAGE_SIZE,  in ceph_msg_data_iter_next()
996 	cursor->lastlen = len;  in ceph_msg_data_iter_next()
1008 	*length = min_t(size_t, len, cursor->resid);  in ceph_msg_data_iter_next()
1012 static bool ceph_msg_data_iter_advance(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_iter_advance()  argument
1015 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_iter_advance()
1016 	cursor->resid -= bytes;  in ceph_msg_data_iter_advance()
1018 	if (bytes < cursor->lastlen) {  in ceph_msg_data_iter_advance()
1019 		cursor->lastlen -= bytes;  in ceph_msg_data_iter_advance()
1021 		iov_iter_advance(&cursor->iov_iter, bytes - cursor->lastlen);  in ceph_msg_data_iter_advance()
1022 		cursor->lastlen = 0;  in ceph_msg_data_iter_advance()
1025 	return cursor->resid;  in ceph_msg_data_iter_advance()
1031  * consume an entire piece at once.  A data item's cursor keeps
1036 static void __ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor)  in __ceph_msg_data_cursor_init()  argument
1038 	size_t length = cursor->total_resid;  in __ceph_msg_data_cursor_init()
1040 	switch (cursor->data->type) {  in __ceph_msg_data_cursor_init()
1042 		ceph_msg_data_pagelist_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1045 		ceph_msg_data_pages_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1049 		ceph_msg_data_bio_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1053 		ceph_msg_data_bvecs_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1056 		ceph_msg_data_iter_cursor_init(cursor, length);  in __ceph_msg_data_cursor_init()
1063 	cursor->need_crc = true;  in __ceph_msg_data_cursor_init()
1066 void ceph_msg_data_cursor_init(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_cursor_init()  argument
1073 	cursor->total_resid = length;  in ceph_msg_data_cursor_init()
1074 	cursor->data = msg->data;  in ceph_msg_data_cursor_init()
1075 	cursor->sr_resid = 0;  in ceph_msg_data_cursor_init()
1077 	__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_cursor_init()
1085 struct page *ceph_msg_data_next(struct ceph_msg_data_cursor *cursor,  in ceph_msg_data_next()  argument
1090 	switch (cursor->data->type) {  in ceph_msg_data_next()
1092 		page = ceph_msg_data_pagelist_next(cursor, page_offset, length);  in ceph_msg_data_next()
1095 		page = ceph_msg_data_pages_next(cursor, page_offset, length);  in ceph_msg_data_next()
1099 		page = ceph_msg_data_bio_next(cursor, page_offset, length);  in ceph_msg_data_next()
1103 		page = ceph_msg_data_bvecs_next(cursor, page_offset, length);  in ceph_msg_data_next()
1106 		page = ceph_msg_data_iter_next(cursor, page_offset, length);  in ceph_msg_data_next()
1117 	BUG_ON(*length > cursor->resid);  in ceph_msg_data_next()
1123  * Returns true if the result moves the cursor on to the next piece
1126 void ceph_msg_data_advance(struct ceph_msg_data_cursor *cursor, size_t bytes)  in ceph_msg_data_advance()  argument
1130 	BUG_ON(bytes > cursor->resid);  in ceph_msg_data_advance()
1131 	switch (cursor->data->type) {  in ceph_msg_data_advance()
1133 		new_piece = ceph_msg_data_pagelist_advance(cursor, bytes);  in ceph_msg_data_advance()
1136 		new_piece = ceph_msg_data_pages_advance(cursor, bytes);  in ceph_msg_data_advance()
1140 		new_piece = ceph_msg_data_bio_advance(cursor, bytes);  in ceph_msg_data_advance()
1144 		new_piece = ceph_msg_data_bvecs_advance(cursor, bytes);  in ceph_msg_data_advance()
1147 		new_piece = ceph_msg_data_iter_advance(cursor, bytes);  in ceph_msg_data_advance()
1154 	cursor->total_resid -= bytes;  in ceph_msg_data_advance()
1156 	if (!cursor->resid && cursor->total_resid) {  in ceph_msg_data_advance()
1157 		cursor->data++;  in ceph_msg_data_advance()
1158 		__ceph_msg_data_cursor_init(cursor);  in ceph_msg_data_advance()
1161 	cursor->need_crc = new_piece;  in ceph_msg_data_advance()