1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include "cn10k.h" 9 #include "otx2_reg.h" 10 #include "otx2_struct.h" 11 12 static struct dev_hw_ops otx2_hw_ops = { 13 .sq_aq_init = otx2_sq_aq_init, 14 .sqe_flush = otx2_sqe_flush, 15 .aura_freeptr = otx2_aura_freeptr, 16 .refill_pool_ptrs = otx2_refill_pool_ptrs, 17 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler, 18 }; 19 20 static struct dev_hw_ops cn10k_hw_ops = { 21 .sq_aq_init = cn10k_sq_aq_init, 22 .sqe_flush = cn10k_sqe_flush, 23 .aura_freeptr = cn10k_aura_freeptr, 24 .refill_pool_ptrs = cn10k_refill_pool_ptrs, 25 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler, 26 }; 27 28 void otx2_init_hw_ops(struct otx2_nic *pfvf) 29 { 30 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 31 pfvf->hw_ops = &otx2_hw_ops; 32 return; 33 } 34 35 pfvf->hw_ops = &cn10k_hw_ops; 36 } 37 EXPORT_SYMBOL(otx2_init_hw_ops); 38 39 int cn10k_lmtst_init(struct otx2_nic *pfvf) 40 { 41 42 struct lmtst_tbl_setup_req *req; 43 struct otx2_lmt_info *lmt_info; 44 int err, cpu; 45 46 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) 47 return 0; 48 49 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/ 50 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE); 51 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info); 52 53 mutex_lock(&pfvf->mbox.lock); 54 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox); 55 if (!req) { 56 mutex_unlock(&pfvf->mbox.lock); 57 return -ENOMEM; 58 } 59 60 req->use_local_lmt_region = true; 61 62 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines, 63 LMT_LINE_SIZE); 64 if (err) { 65 mutex_unlock(&pfvf->mbox.lock); 66 return err; 67 } 68 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base; 69 req->lmt_iova = (u64)pfvf->dync_lmt->iova; 70 71 err = otx2_sync_mbox_msg(&pfvf->mbox); 72 mutex_unlock(&pfvf->mbox.lock); 73 74 for_each_possible_cpu(cpu) { 75 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu); 76 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base + 77 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE)); 78 lmt_info->lmt_id = cpu * LMT_BURST_SIZE; 79 } 80 81 return 0; 82 } 83 EXPORT_SYMBOL(cn10k_lmtst_init); 84 85 int cn10k_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura) 86 { 87 struct nix_cn10k_aq_enq_req *aq; 88 struct otx2_nic *pfvf = dev; 89 90 /* Get memory to put this msg */ 91 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 92 if (!aq) 93 return -ENOMEM; 94 95 aq->sq.cq = pfvf->hw.rx_queues + qidx; 96 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 97 aq->sq.cq_ena = 1; 98 aq->sq.ena = 1; 99 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); 100 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); 101 aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset; 102 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 103 aq->sq.sqb_aura = sqb_aura; 104 aq->sq.sq_int_ena = NIX_SQINT_BITS; 105 aq->sq.qint_idx = 0; 106 /* Due pipelining impact minimum 2000 unused SQ CQE's 107 * need to maintain to avoid CQ overflow. 108 */ 109 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 110 111 /* Fill AQ info */ 112 aq->qidx = qidx; 113 aq->ctype = NIX_AQ_CTYPE_SQ; 114 aq->op = NIX_AQ_INSTOP_INIT; 115 116 return otx2_sync_mbox_msg(&pfvf->mbox); 117 } 118 119 #define NPA_MAX_BURST 16 120 int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 121 { 122 struct otx2_nic *pfvf = dev; 123 int cnt = cq->pool_ptrs; 124 u64 ptrs[NPA_MAX_BURST]; 125 struct otx2_pool *pool; 126 dma_addr_t bufptr; 127 int num_ptrs = 1; 128 129 pool = &pfvf->qset.pool[cq->cq_idx]; 130 131 /* Refill pool with new buffers */ 132 while (cq->pool_ptrs) { 133 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { 134 if (num_ptrs--) 135 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 136 num_ptrs); 137 break; 138 } 139 cq->pool_ptrs--; 140 ptrs[num_ptrs] = pool->xsk_pool ? 141 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM; 142 143 num_ptrs++; 144 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { 145 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 146 num_ptrs); 147 num_ptrs = 1; 148 } 149 } 150 return cnt - cq->pool_ptrs; 151 } 152 153 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) 154 { 155 struct otx2_lmt_info *lmt_info; 156 struct otx2_nic *pfvf = dev; 157 u64 val = 0, tar_addr = 0; 158 159 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id()); 160 /* FIXME: val[0:10] LMT_ID. 161 * [12:15] no of LMTST - 1 in the burst. 162 * [19:63] data size of each LMTST in the burst except first. 163 */ 164 val = (lmt_info->lmt_id & 0x7FF); 165 /* Target address for LMTST flush tells HW how many 128bit 166 * words are present. 167 * tar_addr[6:4] size of first LMTST - 1 in units of 128b. 168 */ 169 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; 170 dma_wmb(); 171 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size); 172 cn10k_lmt_flush(val, tar_addr); 173 174 sq->head++; 175 sq->head &= (sq->sqe_cnt - 1); 176 } 177 178 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) 179 { 180 struct nix_bandprof_free_req *req; 181 int rc; 182 183 if (is_dev_otx2(pfvf->pdev)) 184 return 0; 185 186 mutex_lock(&pfvf->mbox.lock); 187 188 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 189 if (!req) { 190 rc = -ENOMEM; 191 goto out; 192 } 193 194 /* Free all bandwidth profiles allocated */ 195 req->free_all = true; 196 197 rc = otx2_sync_mbox_msg(&pfvf->mbox); 198 out: 199 mutex_unlock(&pfvf->mbox.lock); 200 return rc; 201 } 202 203 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) 204 { 205 struct nix_bandprof_alloc_req *req; 206 struct nix_bandprof_alloc_rsp *rsp; 207 int rc; 208 209 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox); 210 if (!req) 211 return -ENOMEM; 212 213 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 214 215 rc = otx2_sync_mbox_msg(&pfvf->mbox); 216 if (rc) 217 goto out; 218 219 rsp = (struct nix_bandprof_alloc_rsp *) 220 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 221 if (IS_ERR(rsp)) { 222 rc = PTR_ERR(rsp); 223 goto out; 224 } 225 226 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { 227 rc = -EIO; 228 goto out; 229 } 230 231 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; 232 out: 233 if (rc) { 234 dev_warn(pfvf->dev, 235 "Failed to allocate ingress bandwidth policer\n"); 236 } 237 238 return rc; 239 } 240 241 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) 242 { 243 struct otx2_hw *hw = &pfvf->hw; 244 int ret; 245 246 mutex_lock(&pfvf->mbox.lock); 247 248 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer); 249 250 mutex_unlock(&pfvf->mbox.lock); 251 252 return ret; 253 } 254 255 #define POLICER_TIMESTAMP 1 /* 1 second */ 256 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ 257 258 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, 259 u32 *burst_mantissa) 260 { 261 int tmp; 262 263 /* Burst is calculated as 264 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] 265 * This is the upper limit on number tokens (bytes) that 266 * can be accumulated in the bucket. 267 */ 268 *burst_exp = ilog2(burst); 269 if (burst < 256) { 270 /* No float: can't express mantissa in this case */ 271 *burst_mantissa = 0; 272 return; 273 } 274 275 if (*burst_exp > MAX_RATE_EXP) 276 *burst_exp = MAX_RATE_EXP; 277 278 /* Calculate mantissa 279 * Find remaining bytes 'burst - 2^burst_exp' 280 * mantissa = (remaining bytes) / 2^ (burst_exp - 8) 281 */ 282 tmp = burst - rounddown_pow_of_two(burst); 283 *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); 284 } 285 286 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, 287 u32 *rate_mantissa, u32 *rdiv) 288 { 289 u32 div = 0; 290 u32 exp = 0; 291 u64 tmp; 292 293 /* Figure out mantissa, exponent and divider from given max pkt rate 294 * 295 * To achieve desired rate HW adds 296 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every 297 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. 298 * Here policer timeunit is 2 usecs and rate is in bits per sec. 299 * Since floating point cannot be used below algorithm uses 1000000 300 * scale factor to support rates upto 100Gbps. 301 */ 302 tmp = rate * 32 * 2; 303 if (tmp < 256000000) { 304 while (tmp < 256000000) { 305 tmp = tmp * 2; 306 div++; 307 } 308 } else { 309 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) 310 tmp = tmp / 2; 311 312 if (exp > MAX_RATE_EXP) 313 exp = MAX_RATE_EXP; 314 } 315 316 *rate_mantissa = (tmp - 256000000) / 1000000; 317 *rate_exp = exp; 318 *rdiv = div; 319 } 320 321 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, 322 u16 policer, bool map) 323 { 324 struct nix_cn10k_aq_enq_req *aq; 325 326 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 327 if (!aq) 328 return -ENOMEM; 329 330 /* Enable policing and set the bandwidth profile (policer) index */ 331 if (map) 332 aq->rq.policer_ena = 1; 333 else 334 aq->rq.policer_ena = 0; 335 aq->rq_mask.policer_ena = 1; 336 337 aq->rq.band_prof_id = policer; 338 aq->rq_mask.band_prof_id = GENMASK(9, 0); 339 340 /* Fill AQ info */ 341 aq->qidx = rq_idx; 342 aq->ctype = NIX_AQ_CTYPE_RQ; 343 aq->op = NIX_AQ_INSTOP_WRITE; 344 345 return otx2_sync_mbox_msg(&pfvf->mbox); 346 } 347 348 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) 349 { 350 struct nix_bandprof_free_req *req; 351 352 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 353 if (!req) 354 return -ENOMEM; 355 356 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 357 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; 358 359 return otx2_sync_mbox_msg(&pfvf->mbox); 360 } 361 362 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) 363 { 364 struct otx2_hw *hw = &pfvf->hw; 365 int qidx, rc; 366 367 mutex_lock(&pfvf->mbox.lock); 368 369 /* Remove RQ's policer mapping */ 370 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 371 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, hw->matchall_ipolicer, false); 372 if (rc) 373 dev_warn(pfvf->dev, "Failed to unmap RQ %d's policer (error %d).", 374 qidx, rc); 375 } 376 377 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); 378 379 mutex_unlock(&pfvf->mbox.lock); 380 return rc; 381 } 382 383 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, 384 u32 burst, u64 rate, bool pps) 385 { 386 struct nix_cn10k_aq_enq_req *aq; 387 u32 burst_exp, burst_mantissa; 388 u32 rate_exp, rate_mantissa; 389 u32 rdiv; 390 391 /* Get exponent and mantissa values for the desired rate */ 392 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa); 393 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv); 394 395 /* Init bandwidth profile */ 396 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 397 if (!aq) 398 return -ENOMEM; 399 400 /* Set initial color mode to blind */ 401 aq->prof.icolor = 0x03; 402 aq->prof_mask.icolor = 0x03; 403 404 /* Set rate and burst values */ 405 aq->prof.cir_exponent = rate_exp; 406 aq->prof_mask.cir_exponent = 0x1F; 407 408 aq->prof.cir_mantissa = rate_mantissa; 409 aq->prof_mask.cir_mantissa = 0xFF; 410 411 aq->prof.cbs_exponent = burst_exp; 412 aq->prof_mask.cbs_exponent = 0x1F; 413 414 aq->prof.cbs_mantissa = burst_mantissa; 415 aq->prof_mask.cbs_mantissa = 0xFF; 416 417 aq->prof.rdiv = rdiv; 418 aq->prof_mask.rdiv = 0xF; 419 420 if (pps) { 421 /* The amount of decremented tokens is calculated according to 422 * the following equation: 423 * max([ LMODE ? 0 : (packet_length - LXPTR)] + 424 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], 425 * 1/256) 426 * if LMODE is 1 then rate limiting will be based on 427 * PPS otherwise bps. 428 * The aim of the ADJUST value is to specify a token cost per 429 * packet in contrary to the packet length that specifies a 430 * cost per byte. To rate limit based on PPS adjust mantissa 431 * is set as 384 and exponent as 1 so that number of tokens 432 * decremented becomes 1 i.e, 1 token per packeet. 433 */ 434 aq->prof.adjust_exponent = 1; 435 aq->prof_mask.adjust_exponent = 0x1F; 436 437 aq->prof.adjust_mantissa = 384; 438 aq->prof_mask.adjust_mantissa = 0x1FF; 439 440 aq->prof.lmode = 0x1; 441 aq->prof_mask.lmode = 0x1; 442 } 443 444 /* Two rate three color marker 445 * With PEIR/EIR set to zero, color will be either green or red 446 */ 447 aq->prof.meter_algo = 2; 448 aq->prof_mask.meter_algo = 0x3; 449 450 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; 451 aq->prof_mask.rc_action = 0x3; 452 453 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 454 aq->prof_mask.yc_action = 0x3; 455 456 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 457 aq->prof_mask.gc_action = 0x3; 458 459 /* Setting exponent value as 24 and mantissa as 0 configures 460 * the bucket with zero values making bucket unused. Peak 461 * information rate and Excess information rate buckets are 462 * unused here. 463 */ 464 aq->prof.peir_exponent = 24; 465 aq->prof_mask.peir_exponent = 0x1F; 466 467 aq->prof.peir_mantissa = 0; 468 aq->prof_mask.peir_mantissa = 0xFF; 469 470 aq->prof.pebs_exponent = 24; 471 aq->prof_mask.pebs_exponent = 0x1F; 472 473 aq->prof.pebs_mantissa = 0; 474 aq->prof_mask.pebs_mantissa = 0xFF; 475 476 aq->prof.hl_en = 0; 477 aq->prof_mask.hl_en = 1; 478 479 /* Fill AQ info */ 480 aq->qidx = profile; 481 aq->ctype = NIX_AQ_CTYPE_BANDPROF; 482 aq->op = NIX_AQ_INSTOP_WRITE; 483 484 return otx2_sync_mbox_msg(&pfvf->mbox); 485 } 486 487 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, 488 u32 burst, u64 rate) 489 { 490 struct otx2_hw *hw = &pfvf->hw; 491 int qidx, rc; 492 493 mutex_lock(&pfvf->mbox.lock); 494 495 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst, 496 rate, false); 497 if (rc) 498 goto out; 499 500 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 501 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, 502 hw->matchall_ipolicer, true); 503 if (rc) 504 break; 505 } 506 507 out: 508 mutex_unlock(&pfvf->mbox.lock); 509 return rc; 510 } 511