1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2024, Microsoft Corporation. All rights reserved. 4 */ 5 6 #ifndef _MANA_SHADOW_QUEUE_H_ 7 #define _MANA_SHADOW_QUEUE_H_ 8 9 #include <linux/build_bug.h> 10 11 #define MANA_WQ_FENCE_WC BIT(0) 12 #define MANA_WQ_NO_SIGNAL_WC BIT(1) 13 #define MANA_WQE_OFFSET_MASK GENMASK(23, 0) 14 15 struct shadow_wqe_header { 16 u64 wr_id; 17 u64 wqe_offset_or_psn : 24; 18 u64 wqe_size_in_bu : 8; 19 u64 fsn : 24; 20 u64 send_opcode : 4; 21 u64 flags : 2; 22 }; 23 24 struct shadow_queue { 25 /* Unmasked producer index, Incremented on wqe posting */ 26 u64 prod_idx; 27 /* Unmasked consumer index, Incremented on cq polling */ 28 u64 cons_idx; 29 /* queue size in wqes */ 30 u32 length; 31 /* distance between elements in bytes */ 32 u32 stride; 33 /* ring buffer holding wqes */ 34 void *buffer; 35 }; 36 37 static inline int create_shadow_queue(struct shadow_queue *queue, uint32_t length, uint32_t stride) 38 { 39 queue->buffer = kvmalloc_array(length, stride, GFP_KERNEL); 40 if (!queue->buffer) 41 return -ENOMEM; 42 43 queue->length = length; 44 queue->stride = stride; 45 46 return 0; 47 } 48 49 static inline void reset_shadow_queue(struct shadow_queue *queue) 50 { 51 queue->prod_idx = 0; 52 queue->cons_idx = 0; 53 } 54 55 static inline void destroy_shadow_queue(struct shadow_queue *queue) 56 { 57 kvfree(queue->buffer); 58 } 59 60 static inline bool shadow_queue_full(struct shadow_queue *queue) 61 { 62 /* Do not reuse an entry until the poller has finished reading it. */ 63 return (queue->prod_idx - smp_load_acquire(&queue->cons_idx)) >= queue->length; 64 } 65 66 static inline bool shadow_queue_empty(struct shadow_queue *queue) 67 { 68 /* Pair with posting's release of the initialized shadow WQE. */ 69 return smp_load_acquire(&queue->prod_idx) == queue->cons_idx; 70 } 71 72 static inline void * 73 shadow_queue_get_element(const struct shadow_queue *queue, u64 unmasked_index) 74 { 75 u32 index = unmasked_index % queue->length; 76 77 return ((u8 *)queue->buffer + index * queue->stride); 78 } 79 80 static inline void * 81 shadow_queue_producer_entry(struct shadow_queue *queue) 82 { 83 return shadow_queue_get_element(queue, queue->prod_idx); 84 } 85 86 static inline void * 87 shadow_queue_get_next_to_consume(const struct shadow_queue *queue) 88 { 89 /* The producer publishes the WQE before advancing prod_idx. */ 90 if (queue->cons_idx == smp_load_acquire(&queue->prod_idx)) 91 return NULL; 92 93 return shadow_queue_get_element(queue, queue->cons_idx); 94 } 95 96 static inline void shadow_queue_advance_producer(struct shadow_queue *queue) 97 { 98 /* Publish all WQE fields to CQ polling on another CPU. */ 99 smp_store_release(&queue->prod_idx, queue->prod_idx + 1); 100 } 101 102 static inline void shadow_queue_advance_consumer(struct shadow_queue *queue) 103 { 104 /* Finish WC generation and queue-tail updates before allowing reuse. */ 105 smp_store_release(&queue->cons_idx, queue->cons_idx + 1); 106 } 107 108 #endif 109