1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Physical Function ethernet driver 3 * 4 * Copyright (C) 2023 Marvell. 5 * 6 */ 7 8 #include <linux/netdevice.h> 9 #include <net/tso.h> 10 11 #include "cn10k.h" 12 #include "otx2_reg.h" 13 #include "otx2_common.h" 14 #include "otx2_txrx.h" 15 #include "otx2_struct.h" 16 17 #define OTX2_QOS_MAX_LEAF_NODES 16 18 19 static void otx2_qos_aura_pool_free(struct otx2_nic *pfvf, int pool_id) 20 { 21 struct otx2_pool *pool; 22 23 if (!pfvf->qset.pool) 24 return; 25 26 pool = &pfvf->qset.pool[pool_id]; 27 qmem_free(pfvf->dev, pool->stack); 28 qmem_free(pfvf->dev, pool->fc_addr); 29 pool->stack = NULL; 30 pool->fc_addr = NULL; 31 } 32 33 static int otx2_qos_sq_aura_pool_init(struct otx2_nic *pfvf, int qidx) 34 { 35 struct otx2_qset *qset = &pfvf->qset; 36 int pool_id, stack_pages, num_sqbs; 37 struct otx2_hw *hw = &pfvf->hw; 38 struct otx2_snd_queue *sq; 39 struct otx2_pool *pool; 40 dma_addr_t bufptr; 41 int err, ptr; 42 u64 iova, pa; 43 44 /* Calculate number of SQBs needed. 45 * 46 * For a 128byte SQE, and 4K size SQB, 31 SQEs will fit in one SQB. 47 * Last SQE is used for pointing to next SQB. 48 */ 49 num_sqbs = (hw->sqb_size / 128) - 1; 50 num_sqbs = (qset->sqe_cnt + num_sqbs) / num_sqbs; 51 52 /* Get no of stack pages needed */ 53 stack_pages = 54 (num_sqbs + hw->stack_pg_ptrs - 1) / hw->stack_pg_ptrs; 55 56 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, qidx); 57 pool = &pfvf->qset.pool[pool_id]; 58 59 /* Initialize aura context */ 60 err = otx2_aura_init(pfvf, pool_id, pool_id, num_sqbs); 61 if (err) 62 return err; 63 64 /* Initialize pool context */ 65 err = otx2_pool_init(pfvf, pool_id, stack_pages, 66 num_sqbs, hw->sqb_size, AURA_NIX_SQ); 67 if (err) 68 goto aura_free; 69 70 /* Flush accumulated messages */ 71 err = otx2_sync_mbox_msg(&pfvf->mbox); 72 if (err) 73 goto pool_free; 74 75 /* Allocate pointers and free them to aura/pool */ 76 sq = &qset->sq[qidx]; 77 sq->sqb_count = 0; 78 sq->sqb_ptrs = kcalloc(num_sqbs, sizeof(*sq->sqb_ptrs), GFP_KERNEL); 79 if (!sq->sqb_ptrs) { 80 err = -ENOMEM; 81 goto pool_free; 82 } 83 84 for (ptr = 0; ptr < num_sqbs; ptr++) { 85 err = otx2_alloc_rbuf(pfvf, pool, &bufptr); 86 if (err) 87 goto sqb_free; 88 pfvf->hw_ops->aura_freeptr(pfvf, pool_id, bufptr); 89 sq->sqb_ptrs[sq->sqb_count++] = (u64)bufptr; 90 } 91 92 return 0; 93 94 sqb_free: 95 while (ptr--) { 96 if (!sq->sqb_ptrs[ptr]) 97 continue; 98 iova = sq->sqb_ptrs[ptr]; 99 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); 100 dma_unmap_page_attrs(pfvf->dev, iova, hw->sqb_size, 101 DMA_FROM_DEVICE, 102 DMA_ATTR_SKIP_CPU_SYNC); 103 put_page(virt_to_page(phys_to_virt(pa))); 104 otx2_aura_allocptr(pfvf, pool_id); 105 } 106 sq->sqb_count = 0; 107 kfree(sq->sqb_ptrs); 108 pool_free: 109 qmem_free(pfvf->dev, pool->stack); 110 aura_free: 111 qmem_free(pfvf->dev, pool->fc_addr); 112 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 113 return err; 114 } 115 116 static void otx2_qos_sq_free_sqbs(struct otx2_nic *pfvf, int qidx) 117 { 118 struct otx2_qset *qset = &pfvf->qset; 119 struct otx2_hw *hw = &pfvf->hw; 120 struct otx2_snd_queue *sq; 121 u64 iova, pa; 122 int sqb; 123 124 sq = &qset->sq[qidx]; 125 if (!sq->sqb_ptrs) 126 return; 127 for (sqb = 0; sqb < sq->sqb_count; sqb++) { 128 if (!sq->sqb_ptrs[sqb]) 129 continue; 130 iova = sq->sqb_ptrs[sqb]; 131 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova); 132 dma_unmap_page_attrs(pfvf->dev, iova, hw->sqb_size, 133 DMA_FROM_DEVICE, 134 DMA_ATTR_SKIP_CPU_SYNC); 135 put_page(virt_to_page(phys_to_virt(pa))); 136 } 137 138 sq->sqb_count = 0; 139 140 sq = &qset->sq[qidx]; 141 qmem_free(pfvf->dev, sq->sqe); 142 qmem_free(pfvf->dev, sq->tso_hdrs); 143 kfree(sq->sg); 144 kfree(sq->sqb_ptrs); 145 qmem_free(pfvf->dev, sq->timestamps); 146 147 memset((void *)sq, 0, sizeof(*sq)); 148 } 149 150 /* send queue id */ 151 static void otx2_qos_sqb_flush(struct otx2_nic *pfvf, int qidx) 152 { 153 int sqe_tail, sqe_head; 154 u64 incr, *ptr, val; 155 156 ptr = (__force u64 *)otx2_get_regaddr(pfvf, NIX_LF_SQ_OP_STATUS); 157 incr = (u64)qidx << 32; 158 val = otx2_atomic64_add(incr, ptr); 159 sqe_head = (val >> 20) & 0x3F; 160 sqe_tail = (val >> 28) & 0x3F; 161 if (sqe_head != sqe_tail) 162 usleep_range(50, 60); 163 } 164 165 static int otx2_qos_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id) 166 { 167 struct nix_cn10k_aq_enq_req *cn10k_sq_aq; 168 struct npa_aq_enq_req *aura_aq; 169 struct npa_aq_enq_req *pool_aq; 170 struct nix_aq_enq_req *sq_aq; 171 172 if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 173 cn10k_sq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 174 if (!cn10k_sq_aq) 175 return -ENOMEM; 176 cn10k_sq_aq->qidx = qidx; 177 cn10k_sq_aq->sq.ena = 0; 178 cn10k_sq_aq->sq_mask.ena = 1; 179 cn10k_sq_aq->ctype = NIX_AQ_CTYPE_SQ; 180 cn10k_sq_aq->op = NIX_AQ_INSTOP_WRITE; 181 } else { 182 sq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 183 if (!sq_aq) 184 return -ENOMEM; 185 sq_aq->qidx = qidx; 186 sq_aq->sq.ena = 0; 187 sq_aq->sq_mask.ena = 1; 188 sq_aq->ctype = NIX_AQ_CTYPE_SQ; 189 sq_aq->op = NIX_AQ_INSTOP_WRITE; 190 } 191 192 aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 193 if (!aura_aq) { 194 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 195 return -ENOMEM; 196 } 197 198 aura_aq->aura_id = aura_id; 199 aura_aq->aura.ena = 0; 200 aura_aq->aura_mask.ena = 1; 201 aura_aq->ctype = NPA_AQ_CTYPE_AURA; 202 aura_aq->op = NPA_AQ_INSTOP_WRITE; 203 204 pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 205 if (!pool_aq) { 206 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 207 return -ENOMEM; 208 } 209 210 pool_aq->aura_id = aura_id; 211 pool_aq->pool.ena = 0; 212 pool_aq->pool_mask.ena = 1; 213 214 pool_aq->ctype = NPA_AQ_CTYPE_POOL; 215 pool_aq->op = NPA_AQ_INSTOP_WRITE; 216 217 return otx2_sync_mbox_msg(&pfvf->mbox); 218 } 219 220 int otx2_qos_get_qid(struct otx2_nic *pfvf) 221 { 222 int qidx; 223 224 qidx = find_first_zero_bit(pfvf->qos.qos_sq_bmap, 225 pfvf->hw.tc_tx_queues); 226 227 return qidx == pfvf->hw.tc_tx_queues ? -ENOSPC : qidx; 228 } 229 230 void otx2_qos_free_qid(struct otx2_nic *pfvf, int qidx) 231 { 232 clear_bit(qidx, pfvf->qos.qos_sq_bmap); 233 } 234 235 int otx2_qos_enable_sq(struct otx2_nic *pfvf, int qidx) 236 { 237 struct otx2_hw *hw = &pfvf->hw; 238 int pool_id, sq_idx, err; 239 240 if (pfvf->flags & OTX2_FLAG_INTF_DOWN) 241 return -EPERM; 242 243 sq_idx = hw->non_qos_queues + qidx; 244 245 mutex_lock(&pfvf->mbox.lock); 246 err = otx2_qos_sq_aura_pool_init(pfvf, sq_idx); 247 if (err) 248 goto out; 249 250 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, sq_idx); 251 err = otx2_sq_init(pfvf, sq_idx, pool_id); 252 if (err) 253 goto out; 254 out: 255 mutex_unlock(&pfvf->mbox.lock); 256 return err; 257 } 258 259 void otx2_qos_disable_sq(struct otx2_nic *pfvf, int qidx) 260 { 261 struct otx2_qset *qset = &pfvf->qset; 262 struct otx2_hw *hw = &pfvf->hw; 263 struct otx2_snd_queue *sq; 264 struct otx2_cq_queue *cq; 265 int pool_id, sq_idx; 266 267 sq_idx = hw->non_qos_queues + qidx; 268 269 /* If the DOWN flag is set SQs are already freed */ 270 if (pfvf->flags & OTX2_FLAG_INTF_DOWN) 271 return; 272 273 sq = &pfvf->qset.sq[sq_idx]; 274 if (!sq->sqb_ptrs) 275 return; 276 277 if (sq_idx < hw->non_qos_queues || 278 sq_idx >= otx2_get_total_tx_queues(pfvf)) { 279 netdev_err(pfvf->netdev, "Send Queue is not a QoS queue\n"); 280 return; 281 } 282 283 cq = &qset->cq[pfvf->hw.rx_queues + sq_idx]; 284 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_SQ, sq_idx); 285 286 otx2_qos_sqb_flush(pfvf, sq_idx); 287 otx2_smq_flush(pfvf, otx2_get_smq_idx(pfvf, sq_idx)); 288 otx2_cleanup_tx_cqes(pfvf, cq); 289 290 mutex_lock(&pfvf->mbox.lock); 291 otx2_qos_ctx_disable(pfvf, sq_idx, pool_id); 292 mutex_unlock(&pfvf->mbox.lock); 293 294 otx2_qos_sq_free_sqbs(pfvf, sq_idx); 295 otx2_qos_aura_pool_free(pfvf, pool_id); 296 } 297