Lines Matching +full:i +full:- +full:leak +full:- +full:current
1 // SPDX-License-Identifier: GPL-2.0-only
39 * interrupt_mask and re-check to see if new data has
46 * transitions from empty to non-empty (or full to non full on the guest
54 struct hv_ring_buffer_info *rbi = &channel->outbound; in hv_signal_on_write()
57 if (READ_ONCE(rbi->ring_buffer->interrupt_mask)) in hv_signal_on_write()
64 * ring transitions from being empty to non-empty. in hv_signal_on_write()
66 if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) { in hv_signal_on_write()
67 ++channel->intr_out_empty; in hv_signal_on_write()
76 u32 next = ring_info->ring_buffer->write_index; in hv_get_next_write_location()
86 ring_info->ring_buffer->write_index = next_write_location; in hv_set_next_write_location()
93 return ring_info->ring_datasize; in hv_get_ring_buffersize()
100 return (u64)ring_info->ring_buffer->write_index << 32; in hv_get_ring_bufferindices()
105 * Assume there is enough room. Handles wrap-around in dest case only!!
120 start_write_offset -= ring_buffer_size; in hv_copyto_ringbuffer()
139 read_loc = READ_ONCE(rbi->ring_buffer->read_index); in hv_get_ringbuffer_availbytes()
140 write_loc = READ_ONCE(rbi->ring_buffer->write_index); in hv_get_ringbuffer_availbytes()
141 dsize = rbi->ring_datasize; in hv_get_ringbuffer_availbytes()
143 *write = write_loc >= read_loc ? dsize - (write_loc - read_loc) : in hv_get_ringbuffer_availbytes()
144 read_loc - write_loc; in hv_get_ringbuffer_availbytes()
145 *read = dsize - *write; in hv_get_ringbuffer_availbytes()
155 mutex_lock(&ring_info->ring_buffer_mutex); in hv_ringbuffer_get_debuginfo()
157 if (!ring_info->ring_buffer) { in hv_ringbuffer_get_debuginfo()
158 mutex_unlock(&ring_info->ring_buffer_mutex); in hv_ringbuffer_get_debuginfo()
159 return -EINVAL; in hv_ringbuffer_get_debuginfo()
165 debug_info->bytes_avail_toread = bytes_avail_toread; in hv_ringbuffer_get_debuginfo()
166 debug_info->bytes_avail_towrite = bytes_avail_towrite; in hv_ringbuffer_get_debuginfo()
167 debug_info->current_read_index = ring_info->ring_buffer->read_index; in hv_ringbuffer_get_debuginfo()
168 debug_info->current_write_index = ring_info->ring_buffer->write_index; in hv_ringbuffer_get_debuginfo()
169 debug_info->current_interrupt_mask in hv_ringbuffer_get_debuginfo()
170 = ring_info->ring_buffer->interrupt_mask; in hv_ringbuffer_get_debuginfo()
171 mutex_unlock(&ring_info->ring_buffer_mutex); in hv_ringbuffer_get_debuginfo()
180 mutex_init(&channel->inbound.ring_buffer_mutex); in hv_ringbuffer_pre_init()
181 mutex_init(&channel->outbound.ring_buffer_mutex); in hv_ringbuffer_pre_init()
189 int i; in hv_ringbuffer_init() local
197 pages_wraparound = kcalloc(page_cnt * 2 - 1, in hv_ringbuffer_init()
201 return -ENOMEM; in hv_ringbuffer_init()
204 for (i = 0; i < 2 * (page_cnt - 1); i++) in hv_ringbuffer_init()
205 pages_wraparound[i + 1] = in hv_ringbuffer_init()
206 &pages[i % (page_cnt - 1) + 1]; in hv_ringbuffer_init()
208 ring_info->ring_buffer = (struct hv_ring_buffer *) in hv_ringbuffer_init()
209 vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, in hv_ringbuffer_init()
213 if (!ring_info->ring_buffer) in hv_ringbuffer_init()
214 return -ENOMEM; in hv_ringbuffer_init()
220 memset(ring_info->ring_buffer, 0, HV_HYP_PAGE_SIZE); in hv_ringbuffer_init()
222 ring_info->ring_buffer->read_index = in hv_ringbuffer_init()
223 ring_info->ring_buffer->write_index = 0; in hv_ringbuffer_init()
226 ring_info->ring_buffer->feature_bits.value = 1; in hv_ringbuffer_init()
228 ring_info->ring_size = page_cnt << PAGE_SHIFT; in hv_ringbuffer_init()
229 ring_info->ring_size_div10_reciprocal = in hv_ringbuffer_init()
230 reciprocal_value(ring_info->ring_size / 10); in hv_ringbuffer_init()
231 ring_info->ring_datasize = ring_info->ring_size - in hv_ringbuffer_init()
233 ring_info->priv_read_index = 0; in hv_ringbuffer_init()
237 ring_info->pkt_buffer = kzalloc(max_pkt_size, GFP_KERNEL); in hv_ringbuffer_init()
238 if (!ring_info->pkt_buffer) in hv_ringbuffer_init()
239 return -ENOMEM; in hv_ringbuffer_init()
240 ring_info->pkt_buffer_size = max_pkt_size; in hv_ringbuffer_init()
243 spin_lock_init(&ring_info->ring_lock); in hv_ringbuffer_init()
251 mutex_lock(&ring_info->ring_buffer_mutex); in hv_ringbuffer_cleanup()
252 vunmap(ring_info->ring_buffer); in hv_ringbuffer_cleanup()
253 ring_info->ring_buffer = NULL; in hv_ringbuffer_cleanup()
254 mutex_unlock(&ring_info->ring_buffer_mutex); in hv_ringbuffer_cleanup()
256 kfree(ring_info->pkt_buffer); in hv_ringbuffer_cleanup()
257 ring_info->pkt_buffer = NULL; in hv_ringbuffer_cleanup()
258 ring_info->pkt_buffer_size = 0; in hv_ringbuffer_cleanup()
263 * atomic contexts, like panic path (see the Hyper-V framebuffer driver).
268 struct hv_ring_buffer_info *rinfo = &channel->outbound; in hv_ringbuffer_spinlock_busy()
270 return spin_is_locked(&rinfo->ring_lock); in hv_ringbuffer_spinlock_busy()
279 int i; in hv_ringbuffer_write() local
286 struct hv_ring_buffer_info *outring_info = &channel->outbound; in hv_ringbuffer_write()
290 if (channel->rescind) in hv_ringbuffer_write()
291 return -ENODEV; in hv_ringbuffer_write()
293 for (i = 0; i < kv_count; i++) in hv_ringbuffer_write()
294 totalbytes_towrite += kv_list[i].iov_len; in hv_ringbuffer_write()
296 spin_lock_irqsave(&outring_info->ring_lock, flags); in hv_ringbuffer_write()
306 ++channel->out_full_total; in hv_ringbuffer_write()
308 if (!channel->out_full_flag) { in hv_ringbuffer_write()
309 ++channel->out_full_first; in hv_ringbuffer_write()
310 channel->out_full_flag = true; in hv_ringbuffer_write()
313 spin_unlock_irqrestore(&outring_info->ring_lock, flags); in hv_ringbuffer_write()
314 return -EAGAIN; in hv_ringbuffer_write()
317 channel->out_full_flag = false; in hv_ringbuffer_write()
324 for (i = 0; i < kv_count; i++) { in hv_ringbuffer_write()
327 kv_list[i].iov_base, in hv_ringbuffer_write()
328 kv_list[i].iov_len); in hv_ringbuffer_write()
337 if (desc->flags == VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED) { in hv_ringbuffer_write()
338 if (channel->next_request_id_callback != NULL) { in hv_ringbuffer_write()
339 rqst_id = channel->next_request_id_callback(channel, requestid); in hv_ringbuffer_write()
341 spin_unlock_irqrestore(&outring_info->ring_lock, flags); in hv_ringbuffer_write()
342 return -EAGAIN; in hv_ringbuffer_write()
351 * Hyper-V host and subject to being changed at any time. in hv_ringbuffer_write()
353 WRITE_ONCE(desc->trans_id, __trans_id); in hv_ringbuffer_write()
372 spin_unlock_irqrestore(&outring_info->ring_lock, flags); in hv_ringbuffer_write()
376 if (channel->rescind) { in hv_ringbuffer_write()
378 /* Reclaim request ID to avoid leak of IDs */ in hv_ringbuffer_write()
379 if (channel->request_addr_callback != NULL) in hv_ringbuffer_write()
380 channel->request_addr_callback(channel, rqst_id); in hv_ringbuffer_write()
382 return -ENODEV; in hv_ringbuffer_write()
396 return -EINVAL; in hv_ringbuffer_read()
411 offset = raw ? 0 : (desc->offset8 << 3); in hv_ringbuffer_read()
412 packetlen = (desc->len8 << 3) - offset; in hv_ringbuffer_read()
414 *requestid = desc->trans_id; in hv_ringbuffer_read()
417 return -ENOBUFS; in hv_ringbuffer_read()
433 * the current iterator (priv_read_index) location.
440 u32 priv_read_loc = rbi->priv_read_index; in hv_pkt_iter_avail()
444 * The Hyper-V host writes the packet data, then uses in hv_pkt_iter_avail()
446 * here to prevent loads of the packet data from being re-ordered in hv_pkt_iter_avail()
450 write_loc = virt_load_acquire(&rbi->ring_buffer->write_index); in hv_pkt_iter_avail()
453 return write_loc - priv_read_loc; in hv_pkt_iter_avail()
455 return (rbi->ring_datasize - priv_read_loc) + write_loc; in hv_pkt_iter_avail()
465 struct hv_ring_buffer_info *rbi = &channel->inbound; in hv_pkt_iter_first()
474 bytes_avail = min(rbi->pkt_buffer_size, bytes_avail); in hv_pkt_iter_first()
476 desc = (struct vmpacket_descriptor *)(hv_get_ring_buffer(rbi) + rbi->priv_read_index); in hv_pkt_iter_first()
479 * Ensure the compiler does not use references to incoming Hyper-V values (which in hv_pkt_iter_first()
482 pkt_len = READ_ONCE(desc->len8) << 3; in hv_pkt_iter_first()
483 pkt_offset = READ_ONCE(desc->offset8) << 3; in hv_pkt_iter_first()
487 * rbi->pkt_buffer_size in hv_pkt_iter_first()
499 /* Copy the Hyper-V packet out of the ring buffer */ in hv_pkt_iter_first()
500 desc_copy = (struct vmpacket_descriptor *)rbi->pkt_buffer; in hv_pkt_iter_first()
504 * Hyper-V could still change len8 and offset8 after the earlier read. in hv_pkt_iter_first()
508 desc_copy->len8 = pkt_len >> 3; in hv_pkt_iter_first()
509 desc_copy->offset8 = pkt_offset >> 3; in hv_pkt_iter_first()
518 * Advances the current location (priv_read_index) and checks for more
525 struct hv_ring_buffer_info *rbi = &channel->inbound; in __hv_pkt_iter_next()
526 u32 packetlen = desc->len8 << 3; in __hv_pkt_iter_next()
527 u32 dsize = rbi->ring_datasize; in __hv_pkt_iter_next()
531 rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER; in __hv_pkt_iter_next()
532 if (rbi->priv_read_index >= dsize) in __hv_pkt_iter_next()
533 rbi->priv_read_index -= dsize; in __hv_pkt_iter_next()
544 if (rbi->priv_read_index >= start_read_index) in hv_pkt_iter_bytes_read()
545 return rbi->priv_read_index - start_read_index; in hv_pkt_iter_bytes_read()
547 return rbi->ring_datasize - start_read_index + in hv_pkt_iter_bytes_read()
548 rbi->priv_read_index; in hv_pkt_iter_bytes_read()
556 * because Hyper-V protects itself by throttling guests that signal
564 * 3. Read the current write_index
567 * interrupt_mask is used only on the guest->host ring buffer when
568 * sending requests to the host. The host does not use it on the host->
573 struct hv_ring_buffer_info *rbi = &channel->inbound; in hv_pkt_iter_close()
582 start_read_index = rbi->ring_buffer->read_index; in hv_pkt_iter_close()
583 rbi->ring_buffer->read_index = rbi->priv_read_index; in hv_pkt_iter_close()
586 * Older versions of Hyper-V (before WS2102 and Win8) do not in hv_pkt_iter_close()
587 * implement pending_send_sz and simply poll if the host->guest in hv_pkt_iter_close()
590 if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz) in hv_pkt_iter_close()
609 pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz); in hv_pkt_iter_close()
641 if (curr_write_sz - bytes_read > pending_sz) in hv_pkt_iter_close()
651 ++channel->intr_in_full; in hv_pkt_iter_close()