313 #define DATA_SIZE(data_ring)		_DATA_SIZE((data_ring)->size_bits)  argument
314 #define DATA_SIZE_MASK(data_ring)	(DATA_SIZE(data_ring) - 1)  argument
320 #define DATA_INDEX(data_ring, lpos)	((lpos) & DATA_SIZE_MASK(data_ring))  argument
326 #define DATA_WRAPS(data_ring, lpos)	((lpos) >> (data_ring)->size_bits)  argument
334 #define DATA_THIS_WRAP_START_LPOS(data_ring, lpos) \  argument
335 ((lpos) & ~DATA_SIZE_MASK(data_ring))
374 static struct prb_data_block *to_block(struct prb_data_ring *data_ring,  in to_block()  argument
377 	return (void *)&data_ring->data[DATA_INDEX(data_ring, begin_lpos)];  in to_block()
400 static bool data_check_size(struct prb_data_ring *data_ring, unsigned int size)  in data_check_size()  argument
411 	return to_blk_size(size) <= DATA_SIZE(data_ring) / 2;  in data_check_size()
424 static bool need_more_space(struct prb_data_ring *data_ring,  in need_more_space()  argument
428 	return lpos_target - lpos_current - 1 < DATA_SIZE(data_ring);  in need_more_space()
588 	struct prb_data_ring *data_ring = &rb->text_data_ring;  in data_make_reusable()  local
597 	while (need_more_space(data_ring, lpos_begin, lpos_end)) {  in data_make_reusable()
598 		blk = to_block(data_ring, lpos_begin);  in data_make_reusable()
652 	struct prb_data_ring *data_ring = &rb->text_data_ring;  in data_push_tail()  local
679 	tail_lpos = atomic_long_read(&data_ring->tail_lpos); /* LMM(data_push_tail:A) */  in data_push_tail()
688 	while (need_more_space(data_ring, tail_lpos, lpos)) {  in data_push_tail()
751 			tail_lpos_new = atomic_long_read(&data_ring->tail_lpos  in data_push_tail()
768 		if (atomic_long_try_cmpxchg(&data_ring->tail_lpos, &tail_lpos,  in data_push_tail()
1019 static bool is_blk_wrapped(struct prb_data_ring *data_ring,  in is_blk_wrapped()  argument
1026 	return DATA_WRAPS(data_ring, begin_lpos) !=  in is_blk_wrapped()
1027 	       DATA_WRAPS(data_ring, next_lpos - 1);  in is_blk_wrapped()
1031 static unsigned long get_next_lpos(struct prb_data_ring *data_ring,  in get_next_lpos()  argument
1041 	if (!is_blk_wrapped(data_ring, begin_lpos, next_lpos))  in get_next_lpos()
1045 	return (DATA_THIS_WRAP_START_LPOS(data_ring, next_lpos) + size);  in get_next_lpos()
1056 	struct prb_data_ring *data_ring = &rb->text_data_ring;  in data_alloc()  local
1074 	begin_lpos = atomic_long_read(&data_ring->head_lpos);  in data_alloc()
1077 		next_lpos = get_next_lpos(data_ring, begin_lpos, size);  in data_alloc()
1088 		if (WARN_ON_ONCE(next_lpos - begin_lpos > DATA_SIZE(data_ring)) ||  in data_alloc()
1089 		    !data_push_tail(rb, next_lpos - DATA_SIZE(data_ring))) {  in data_alloc()
1112 	} while (!atomic_long_try_cmpxchg(&data_ring->head_lpos, &begin_lpos,  in data_alloc()
1115 	blk = to_block(data_ring, begin_lpos);  in data_alloc()
1118 	if (is_blk_wrapped(data_ring, begin_lpos, next_lpos)) {  in data_alloc()
1120 		blk = to_block(data_ring, 0);  in data_alloc()
1150 	struct prb_data_ring *data_ring = &rb->text_data_ring;  in data_realloc()  local
1157 	head_lpos = atomic_long_read(&data_ring->head_lpos);  in data_realloc()
1162 	wrapped = is_blk_wrapped(data_ring, blk_lpos->begin, blk_lpos->next);  in data_realloc()
1166 	next_lpos = get_next_lpos(data_ring, blk_lpos->begin, size);  in data_realloc()
1176 	if (!need_more_space(data_ring, head_lpos, next_lpos)) {  in data_realloc()
1178 			blk = to_block(data_ring, 0);  in data_realloc()
1180 			blk = to_block(data_ring, blk_lpos->begin);  in data_realloc()
1192 	if (WARN_ON_ONCE(next_lpos - blk_lpos->begin > DATA_SIZE(data_ring)) ||  in data_realloc()
1193 	    !data_push_tail(rb, next_lpos - DATA_SIZE(data_ring))) {  in data_realloc()
1198 	if (!atomic_long_try_cmpxchg(&data_ring->head_lpos, &head_lpos,  in data_realloc()
1203 	blk = to_block(data_ring, blk_lpos->begin);  in data_realloc()
1205 	if (is_blk_wrapped(data_ring, blk_lpos->begin, next_lpos)) {  in data_realloc()
1209 		blk = to_block(data_ring, 0);  in data_realloc()
1234 static unsigned int space_used(struct prb_data_ring *data_ring,  in space_used()  argument
1241 	if (!is_blk_wrapped(data_ring, blk_lpos->begin, blk_lpos->next)) {  in space_used()
1243 		return (DATA_INDEX(data_ring, blk_lpos->next) -  in space_used()
1244 			DATA_INDEX(data_ring, blk_lpos->begin));  in space_used()
1251 	return (DATA_INDEX(data_ring, blk_lpos->next) +  in space_used()
1252 		DATA_SIZE(data_ring) - DATA_INDEX(data_ring, blk_lpos->begin));  in space_used()
1264 static const char *get_data(struct prb_data_ring *data_ring,  in get_data()  argument
1288 	if (!is_blk_wrapped(data_ring, blk_lpos->begin, blk_lpos->next)) {  in get_data()
1289 		db = to_block(data_ring, blk_lpos->begin);  in get_data()
1293 	} else if (!is_blk_wrapped(data_ring,  in get_data()
1294 				   blk_lpos->begin + DATA_SIZE(data_ring),  in get_data()
1296 		db = to_block(data_ring, 0);  in get_data()
1297 		*data_size = DATA_INDEX(data_ring, blk_lpos->next);  in get_data()
1306 	if (WARN_ON_ONCE(!data_check_size(data_ring, *data_size) || !*data_size))  in get_data()
1873 static bool copy_data(struct prb_data_ring *data_ring,  in copy_data()  argument
1884 	data = get_data(data_ring, blk_lpos, &data_size);  in copy_data()