1 // SPDX-License-Identifier: GPL-2.0-only
29  * When we write to the ring buffer, check if the host needs to
32  *	1. The host guarantees that while it is draining the
37  *	2. The host guarantees that it will completely drain
40  *	   interrupt_mask and re-check to see if new data has
46  * the interrupt. The host expects interrupts only when the ring
47  * transitions from empty to non-empty (or full to non full on the guest
48  * to host ring).
50  * host logic is fixed.
55 	struct hv_ring_buffer_info *rbi = &channel->outbound;  in hv_signal_on_write()
58 	if (READ_ONCE(rbi->ring_buffer->interrupt_mask))  in hv_signal_on_write()
64 	 * This is the only case we need to signal when the  in hv_signal_on_write()
65 	 * ring transitions from being empty to non-empty.  in hv_signal_on_write()
67 	if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) {  in hv_signal_on_write()
68 		++channel->intr_out_empty;  in hv_signal_on_write()
77 	u32 next = ring_info->ring_buffer->write_index;  in hv_get_next_write_location()
87 	ring_info->ring_buffer->write_index = next_write_location;  in hv_set_next_write_location()
94 	return ring_info->ring_datasize;  in hv_get_ring_buffersize()
101 	return (u64)ring_info->ring_buffer->write_index << 32;  in hv_get_ring_bufferindices()
106  * Assume there is enough room. Handles wrap-around in dest case only!!
121 		start_write_offset -= ring_buffer_size;  in hv_copyto_ringbuffer()
140 	read_loc = READ_ONCE(rbi->ring_buffer->read_index);  in hv_get_ringbuffer_availbytes()
141 	write_loc = READ_ONCE(rbi->ring_buffer->write_index);  in hv_get_ringbuffer_availbytes()
142 	dsize = rbi->ring_datasize;  in hv_get_ringbuffer_availbytes()
144 	*write = write_loc >= read_loc ? dsize - (write_loc - read_loc) :  in hv_get_ringbuffer_availbytes()
145 		read_loc - write_loc;  in hv_get_ringbuffer_availbytes()
146 	*read = dsize - *write;  in hv_get_ringbuffer_availbytes()
156 	mutex_lock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
158 	if (!ring_info->ring_buffer) {  in hv_ringbuffer_get_debuginfo()
159 		mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
160 		return -EINVAL;  in hv_ringbuffer_get_debuginfo()
166 	debug_info->bytes_avail_toread = bytes_avail_toread;  in hv_ringbuffer_get_debuginfo()
167 	debug_info->bytes_avail_towrite = bytes_avail_towrite;  in hv_ringbuffer_get_debuginfo()
168 	debug_info->current_read_index = ring_info->ring_buffer->read_index;  in hv_ringbuffer_get_debuginfo()
169 	debug_info->current_write_index = ring_info->ring_buffer->write_index;  in hv_ringbuffer_get_debuginfo()
170 	debug_info->current_interrupt_mask  in hv_ringbuffer_get_debuginfo()
171 		= ring_info->ring_buffer->interrupt_mask;  in hv_ringbuffer_get_debuginfo()
172 	mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_get_debuginfo()
181 	mutex_init(&channel->inbound.ring_buffer_mutex);  in hv_ringbuffer_pre_init()
182 	mutex_init(&channel->outbound.ring_buffer_mutex);  in hv_ringbuffer_pre_init()
198 	pages_wraparound = kcalloc(page_cnt * 2 - 1,  in hv_ringbuffer_init()
202 		return -ENOMEM;  in hv_ringbuffer_init()
205 	for (i = 0; i < 2 * (page_cnt - 1); i++)  in hv_ringbuffer_init()
207 			&pages[i % (page_cnt - 1) + 1];  in hv_ringbuffer_init()
209 	ring_info->ring_buffer = (struct hv_ring_buffer *)  in hv_ringbuffer_init()
210 		vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP,  in hv_ringbuffer_init()
214 	if (!ring_info->ring_buffer)  in hv_ringbuffer_init()
215 		return -ENOMEM;  in hv_ringbuffer_init()
221 	memset(ring_info->ring_buffer, 0, HV_HYP_PAGE_SIZE);  in hv_ringbuffer_init()
223 	ring_info->ring_buffer->read_index =  in hv_ringbuffer_init()
224 		ring_info->ring_buffer->write_index = 0;  in hv_ringbuffer_init()
227 	ring_info->ring_buffer->feature_bits.value = 1;  in hv_ringbuffer_init()
229 	ring_info->ring_size = page_cnt << PAGE_SHIFT;  in hv_ringbuffer_init()
230 	ring_info->ring_size_div10_reciprocal =  in hv_ringbuffer_init()
231 		reciprocal_value(ring_info->ring_size / 10);  in hv_ringbuffer_init()
232 	ring_info->ring_datasize = ring_info->ring_size -  in hv_ringbuffer_init()
234 	ring_info->priv_read_index = 0;  in hv_ringbuffer_init()
238 		ring_info->pkt_buffer = kzalloc(max_pkt_size, GFP_KERNEL);  in hv_ringbuffer_init()
239 		if (!ring_info->pkt_buffer)  in hv_ringbuffer_init()
240 			return -ENOMEM;  in hv_ringbuffer_init()
241 		ring_info->pkt_buffer_size = max_pkt_size;  in hv_ringbuffer_init()
244 	spin_lock_init(&ring_info->ring_lock);  in hv_ringbuffer_init()
252 	mutex_lock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_cleanup()
253 	vunmap(ring_info->ring_buffer);  in hv_ringbuffer_cleanup()
254 	ring_info->ring_buffer = NULL;  in hv_ringbuffer_cleanup()
255 	mutex_unlock(&ring_info->ring_buffer_mutex);  in hv_ringbuffer_cleanup()
257 	kfree(ring_info->pkt_buffer);  in hv_ringbuffer_cleanup()
258 	ring_info->pkt_buffer = NULL;  in hv_ringbuffer_cleanup()
259 	ring_info->pkt_buffer_size = 0;  in hv_ringbuffer_cleanup()
264  * atomic contexts, like panic path (see the Hyper-V framebuffer driver).
269 	struct hv_ring_buffer_info *rinfo = &channel->outbound;  in hv_ringbuffer_spinlock_busy()
271 	return spin_is_locked(&rinfo->ring_lock);  in hv_ringbuffer_spinlock_busy()
287 	struct hv_ring_buffer_info *outring_info = &channel->outbound;  in hv_ringbuffer_write()
291 	if (channel->rescind)  in hv_ringbuffer_write()
292 		return -ENODEV;  in hv_ringbuffer_write()
297 	spin_lock_irqsave(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
302 	 * If there is only room for the packet, assume it is full.  in hv_ringbuffer_write()
307 		++channel->out_full_total;  in hv_ringbuffer_write()
309 		if (!channel->out_full_flag) {  in hv_ringbuffer_write()
310 			++channel->out_full_first;  in hv_ringbuffer_write()
311 			channel->out_full_flag = true;  in hv_ringbuffer_write()
314 		spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
315 		return -EAGAIN;  in hv_ringbuffer_write()
318 	channel->out_full_flag = false;  in hv_ringbuffer_write()
338 	if (desc->flags == VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED) {  in hv_ringbuffer_write()
339 		if (channel->next_request_id_callback != NULL) {  in hv_ringbuffer_write()
340 			rqst_id = channel->next_request_id_callback(channel, requestid);  in hv_ringbuffer_write()
342 				spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
343 				return -EAGAIN;  in hv_ringbuffer_write()
352 	 * Hyper-V host and subject to being changed at any time.  in hv_ringbuffer_write()
354 	WRITE_ONCE(desc->trans_id, __trans_id);  in hv_ringbuffer_write()
373 	spin_unlock_irqrestore(&outring_info->ring_lock, flags);  in hv_ringbuffer_write()
377 	if (channel->rescind) {  in hv_ringbuffer_write()
380 			if (channel->request_addr_callback != NULL)  in hv_ringbuffer_write()
381 				channel->request_addr_callback(channel, rqst_id);  in hv_ringbuffer_write()
383 		return -ENODEV;  in hv_ringbuffer_write()
397 		return -EINVAL;  in hv_ringbuffer_read()
412 	offset = raw ? 0 : (desc->offset8 << 3);  in hv_ringbuffer_read()
413 	packetlen = (desc->len8 << 3) - offset;  in hv_ringbuffer_read()
415 	*requestid = desc->trans_id;  in hv_ringbuffer_read()
418 		return -ENOBUFS;  in hv_ringbuffer_read()
420 	/* since ring is double mapped, only one copy is necessary */  in hv_ringbuffer_read()
426 	/* Notify host of update */  in hv_ringbuffer_read()
441 	u32 priv_read_loc = rbi->priv_read_index;  in hv_pkt_iter_avail()
445 	 * The Hyper-V host writes the packet data, then uses  in hv_pkt_iter_avail()
447 	 * here to prevent loads of the packet data from being re-ordered  in hv_pkt_iter_avail()
451 	write_loc = virt_load_acquire(&rbi->ring_buffer->write_index);  in hv_pkt_iter_avail()
454 		return write_loc - priv_read_loc;  in hv_pkt_iter_avail()
456 		return (rbi->ring_datasize - priv_read_loc) + write_loc;  in hv_pkt_iter_avail()
466 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_first()
475 	bytes_avail = min(rbi->pkt_buffer_size, bytes_avail);  in hv_pkt_iter_first()
477 	desc = (struct vmpacket_descriptor *)(hv_get_ring_buffer(rbi) + rbi->priv_read_index);  in hv_pkt_iter_first()
480 	 * Ensure the compiler does not use references to incoming Hyper-V values (which  in hv_pkt_iter_first()
483 	pkt_len = READ_ONCE(desc->len8) << 3;  in hv_pkt_iter_first()
484 	pkt_offset = READ_ONCE(desc->offset8) << 3;  in hv_pkt_iter_first()
488 	 * rbi->pkt_buffer_size  in hv_pkt_iter_first()
500 	/* Copy the Hyper-V packet out of the ring buffer */  in hv_pkt_iter_first()
501 	desc_copy = (struct vmpacket_descriptor *)rbi->pkt_buffer;  in hv_pkt_iter_first()
505 	 * Hyper-V could still change len8 and offset8 after the earlier read.  in hv_pkt_iter_first()
509 	desc_copy->len8 = pkt_len >> 3;  in hv_pkt_iter_first()
510 	desc_copy->offset8 = pkt_offset >> 3;  in hv_pkt_iter_first()
526 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in __hv_pkt_iter_next()
527 	u32 packetlen = desc->len8 << 3;  in __hv_pkt_iter_next()
528 	u32 dsize = rbi->ring_datasize;  in __hv_pkt_iter_next()
532 	rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER;  in __hv_pkt_iter_next()
533 	if (rbi->priv_read_index >= dsize)  in __hv_pkt_iter_next()
534 		rbi->priv_read_index -= dsize;  in __hv_pkt_iter_next()
545 	if (rbi->priv_read_index >= start_read_index)  in hv_pkt_iter_bytes_read()
546 		return rbi->priv_read_index - start_read_index;  in hv_pkt_iter_bytes_read()
548 		return rbi->ring_datasize - start_read_index +  in hv_pkt_iter_bytes_read()
549 			rbi->priv_read_index;  in hv_pkt_iter_bytes_read()
553  * Update host ring buffer after iterating over packets. If the host has
555  * sufficient space is being freed up, signal the host. But be careful to
556  * only signal the host when necessary, both for performance reasons and
557  * because Hyper-V protects itself by throttling guests that signal
568  * interrupt_mask is used only on the guest->host ring buffer when
569  * sending requests to the host. The host does not use it on the host->
574 	struct hv_ring_buffer_info *rbi = &channel->inbound;  in hv_pkt_iter_close()
583 	start_read_index = rbi->ring_buffer->read_index;  in hv_pkt_iter_close()
584 	rbi->ring_buffer->read_index = rbi->priv_read_index;  in hv_pkt_iter_close()
587 	 * Older versions of Hyper-V (before WS2102 and Win8) do not  in hv_pkt_iter_close()
588 	 * implement pending_send_sz and simply poll if the host->guest  in hv_pkt_iter_close()
591 	if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz)  in hv_pkt_iter_close()
598 	 * host were to set the pending_send_sz after we have sampled  in hv_pkt_iter_close()
610 	pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz);  in hv_pkt_iter_close()
623 	 * We want to signal the host only if we're transitioning  in hv_pkt_iter_close()
626 	 * could run and free up enough space to signal the host, and then  in hv_pkt_iter_close()
627 	 * run again and free up additional space before the host has a  in hv_pkt_iter_close()
639 	 * before we began the iteration. If so, the host was not  in hv_pkt_iter_close()
642 	if (curr_write_sz - bytes_read > pending_sz)  in hv_pkt_iter_close()
652 	++channel->intr_in_full;  in hv_pkt_iter_close()