1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Driver for Alibaba Elastic Ethernet Adapter. 4 * 5 * Copyright (C) 2025 Alibaba Inc. 6 */ 7 8 #include "eea_pci.h" 9 #include "eea_ring.h" 10 11 void eea_ering_irq_active(struct eea_ring *ering, struct eea_ring *tx_ering) 12 { 13 u64 value = 0, rx_idx, tx_idx; 14 15 tx_idx = (u64)tx_ering->cq.hw_idx; 16 rx_idx = (u64)ering->cq.hw_idx; 17 18 value |= EEA_IRQ_UNMASK << EEA_DB_FLAGS_OFF; 19 value |= tx_idx << EEA_DB_TX_CQ_HEAD_OFF; 20 value |= rx_idx << EEA_DB_RX_CQ_HEAD_OFF; 21 22 writeq(value, ering->db); 23 } 24 25 void *eea_ering_cq_get_desc(const struct eea_ring *ering) 26 { 27 u8 phase; 28 u8 *desc; 29 30 desc = ering->cq.desc + (ering->cq.head << ering->cq.desc_size_shift); 31 32 phase = READ_ONCE(*(u8 *)(desc + ering->cq.desc_size - 1)); 33 34 if ((phase & EEA_RING_DESC_F_CQ_PHASE) == ering->cq.phase) { 35 dma_rmb(); 36 return desc; 37 } 38 39 return NULL; 40 } 41 42 /* sq api */ 43 void *eea_ering_sq_alloc_desc(struct eea_ring *ering, u16 id, bool is_last, 44 u16 flags) 45 { 46 struct eea_ring_sq *sq = &ering->sq; 47 struct eea_common_desc *desc; 48 49 if (!sq->shadow_num) { 50 sq->shadow_idx = sq->head; 51 sq->shadow_id = cpu_to_le16(id); 52 } 53 54 if (!is_last) 55 flags |= EEA_RING_DESC_F_MORE; 56 57 desc = sq->desc + (sq->shadow_idx << sq->desc_size_shift); 58 59 desc->flags = cpu_to_le16(flags); 60 desc->id = sq->shadow_id; 61 62 if (unlikely(++sq->shadow_idx >= ering->num)) 63 sq->shadow_idx = 0; 64 65 ++sq->shadow_num; 66 67 return desc; 68 } 69 70 /* This is an allocation API for admin Q. For each call to admin Q, only one 71 * desc will be allocated. 72 */ 73 void *eea_ering_aq_alloc_desc(struct eea_ring *ering) 74 { 75 struct eea_ring_sq *sq = &ering->sq; 76 struct eea_common_desc *desc; 77 78 if (!sq->shadow_num) 79 sq->shadow_idx = sq->head; 80 81 desc = sq->desc + (sq->shadow_idx << sq->desc_size_shift); 82 83 if (unlikely(++sq->shadow_idx >= ering->num)) 84 sq->shadow_idx = 0; 85 86 ++sq->shadow_num; 87 88 return desc; 89 } 90 91 void eea_ering_sq_commit_desc(struct eea_ring *ering) 92 { 93 struct eea_ring_sq *sq = &ering->sq; 94 int num; 95 96 num = sq->shadow_num; 97 98 ering->num_free -= num; 99 100 sq->head = sq->shadow_idx; 101 sq->hw_idx += num; 102 sq->shadow_num = 0; 103 } 104 105 void eea_ering_sq_cancel(struct eea_ring *ering) 106 { 107 ering->sq.shadow_num = 0; 108 } 109 110 /* cq api */ 111 void eea_ering_cq_ack_desc(struct eea_ring *ering, u32 num) 112 { 113 struct eea_ring_cq *cq = &ering->cq; 114 115 cq->head += num; 116 cq->hw_idx += num; 117 118 if (unlikely(cq->head >= ering->num)) { 119 cq->head -= ering->num; 120 cq->phase ^= EEA_RING_DESC_F_CQ_PHASE; 121 } 122 123 ering->num_free += num; 124 } 125 126 /* notify */ 127 void eea_ering_kick(struct eea_ring *ering) 128 { 129 u64 value = 0, idx; 130 131 idx = (u64)ering->sq.hw_idx; 132 133 value |= EEA_IDX_PRESENT << EEA_DB_FLAGS_OFF; 134 value |= idx << EEA_DB_IDX_OFF; 135 136 writeq(value, ering->db); 137 } 138 139 /* ering alloc/free */ 140 static void ering_free_queue(struct eea_device *edev, size_t size, 141 void *queue, dma_addr_t dma_handle) 142 { 143 dma_free_coherent(edev->dma_dev, size, queue, dma_handle); 144 } 145 146 static void *ering_alloc_queue(struct eea_device *edev, size_t size, 147 dma_addr_t *dma_handle) 148 { 149 gfp_t flags = GFP_KERNEL | __GFP_NOWARN; 150 151 return dma_alloc_coherent(edev->dma_dev, size, dma_handle, flags); 152 } 153 154 static int ering_alloc_queues(struct eea_ring *ering, struct eea_device *edev, 155 size_t num, u8 sq_desc_size, u8 cq_desc_size) 156 { 157 dma_addr_t addr; 158 size_t size; 159 void *ring; 160 161 size = num * sq_desc_size; 162 163 ring = ering_alloc_queue(edev, size, &addr); 164 if (!ring) 165 return -ENOMEM; 166 167 ering->sq.desc = ring; 168 ering->sq.dma_addr = addr; 169 ering->sq.dma_size = size; 170 ering->sq.desc_size = sq_desc_size; 171 ering->sq.desc_size_shift = fls(sq_desc_size) - 1; 172 173 size = num * cq_desc_size; 174 175 ring = ering_alloc_queue(edev, size, &addr); 176 if (!ring) 177 goto err_free_sq; 178 179 ering->cq.desc = ring; 180 ering->cq.dma_addr = addr; 181 ering->cq.dma_size = size; 182 ering->cq.desc_size = cq_desc_size; 183 ering->cq.desc_size_shift = fls(cq_desc_size) - 1; 184 185 ering->num = num; 186 187 return 0; 188 189 err_free_sq: 190 ering_free_queue(ering->edev, ering->sq.dma_size, 191 ering->sq.desc, ering->sq.dma_addr); 192 return -ENOMEM; 193 } 194 195 static void ering_init(struct eea_ring *ering) 196 { 197 ering->cq.phase = EEA_RING_DESC_F_CQ_PHASE; 198 ering->num_free = ering->num; 199 } 200 201 struct eea_ring *eea_ering_alloc(u32 index, u32 num, struct eea_device *edev, 202 u8 sq_desc_size, u8 cq_desc_size, 203 const char *name) 204 { 205 struct eea_ring *ering; 206 207 if (num > EEA_NET_IO_HW_RING_DEPTH_MAX || 208 num < EEA_NET_IO_RING_DEPTH_MIN) 209 return NULL; 210 211 if (!is_power_of_2(num)) 212 return NULL; 213 214 if (!sq_desc_size || !is_power_of_2(sq_desc_size)) 215 return NULL; 216 217 if (!cq_desc_size || !is_power_of_2(cq_desc_size)) 218 return NULL; 219 220 ering = kzalloc(sizeof(*ering), GFP_KERNEL); 221 if (!ering) 222 return NULL; 223 224 ering->edev = edev; 225 ering->name = name; 226 ering->index = index; 227 228 if (ering_alloc_queues(ering, edev, num, sq_desc_size, cq_desc_size)) 229 goto err_free; 230 231 ering_init(ering); 232 233 return ering; 234 235 err_free: 236 kfree(ering); 237 return NULL; 238 } 239 240 void eea_ering_free(struct eea_ring *ering) 241 { 242 ering_free_queue(ering->edev, ering->cq.dma_size, 243 ering->cq.desc, ering->cq.dma_addr); 244 245 ering_free_queue(ering->edev, ering->sq.dma_size, 246 ering->sq.desc, ering->sq.dma_addr); 247 248 kfree(ering); 249 } 250