1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include "cn10k.h" 9 #include "otx2_reg.h" 10 #include "otx2_struct.h" 11 12 static struct dev_hw_ops otx2_hw_ops = { 13 .sq_aq_init = otx2_sq_aq_init, 14 .sqe_flush = otx2_sqe_flush, 15 .aura_freeptr = otx2_aura_freeptr, 16 .refill_pool_ptrs = otx2_refill_pool_ptrs, 17 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler, 18 .aura_aq_init = otx2_aura_aq_init, 19 .pool_aq_init = otx2_pool_aq_init, 20 }; 21 22 static struct dev_hw_ops cn10k_hw_ops = { 23 .sq_aq_init = cn10k_sq_aq_init, 24 .sqe_flush = cn10k_sqe_flush, 25 .aura_freeptr = cn10k_aura_freeptr, 26 .refill_pool_ptrs = cn10k_refill_pool_ptrs, 27 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler, 28 .aura_aq_init = otx2_aura_aq_init, 29 .pool_aq_init = otx2_pool_aq_init, 30 }; 31 32 void otx2_init_hw_ops(struct otx2_nic *pfvf) 33 { 34 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 35 pfvf->hw_ops = &otx2_hw_ops; 36 return; 37 } 38 39 pfvf->hw_ops = &cn10k_hw_ops; 40 } 41 EXPORT_SYMBOL(otx2_init_hw_ops); 42 43 int cn10k_lmtst_init(struct otx2_nic *pfvf) 44 { 45 46 struct lmtst_tbl_setup_req *req; 47 struct otx2_lmt_info *lmt_info; 48 int err, cpu; 49 50 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) 51 return 0; 52 53 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/ 54 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE); 55 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info); 56 57 mutex_lock(&pfvf->mbox.lock); 58 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox); 59 if (!req) { 60 mutex_unlock(&pfvf->mbox.lock); 61 return -ENOMEM; 62 } 63 64 req->use_local_lmt_region = true; 65 66 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines, 67 LMT_LINE_SIZE); 68 if (err) { 69 mutex_unlock(&pfvf->mbox.lock); 70 return err; 71 } 72 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base; 73 req->lmt_iova = (u64)pfvf->dync_lmt->iova; 74 75 err = otx2_sync_mbox_msg(&pfvf->mbox); 76 mutex_unlock(&pfvf->mbox.lock); 77 78 for_each_possible_cpu(cpu) { 79 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu); 80 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base + 81 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE)); 82 lmt_info->lmt_id = cpu * LMT_BURST_SIZE; 83 } 84 85 return 0; 86 } 87 EXPORT_SYMBOL(cn10k_lmtst_init); 88 89 int cn10k_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura) 90 { 91 struct nix_cn10k_aq_enq_req *aq; 92 struct otx2_nic *pfvf = dev; 93 94 /* Get memory to put this msg */ 95 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 96 if (!aq) 97 return -ENOMEM; 98 99 aq->sq.cq = pfvf->hw.rx_queues + qidx; 100 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 101 aq->sq.cq_ena = 1; 102 aq->sq.ena = 1; 103 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx); 104 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen); 105 aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset; 106 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 107 aq->sq.sqb_aura = sqb_aura; 108 aq->sq.sq_int_ena = NIX_SQINT_BITS; 109 aq->sq.qint_idx = 0; 110 /* Due pipelining impact minimum 2000 unused SQ CQE's 111 * need to maintain to avoid CQ overflow. 112 */ 113 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 114 115 /* Fill AQ info */ 116 aq->qidx = qidx; 117 aq->ctype = NIX_AQ_CTYPE_SQ; 118 aq->op = NIX_AQ_INSTOP_INIT; 119 120 return otx2_sync_mbox_msg(&pfvf->mbox); 121 } 122 123 #define NPA_MAX_BURST 16 124 int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 125 { 126 struct otx2_nic *pfvf = dev; 127 int cnt = cq->pool_ptrs; 128 u64 ptrs[NPA_MAX_BURST]; 129 struct otx2_pool *pool; 130 dma_addr_t bufptr; 131 int num_ptrs = 1; 132 133 pool = &pfvf->qset.pool[cq->cq_idx]; 134 135 /* Refill pool with new buffers */ 136 while (cq->pool_ptrs) { 137 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { 138 if (num_ptrs--) 139 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 140 num_ptrs); 141 break; 142 } 143 cq->pool_ptrs--; 144 ptrs[num_ptrs] = pool->xsk_pool ? 145 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM; 146 147 num_ptrs++; 148 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { 149 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 150 num_ptrs); 151 num_ptrs = 1; 152 } 153 } 154 return cnt - cq->pool_ptrs; 155 } 156 157 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) 158 { 159 struct otx2_lmt_info *lmt_info; 160 struct otx2_nic *pfvf = dev; 161 u64 val = 0, tar_addr = 0; 162 163 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id()); 164 /* FIXME: val[0:10] LMT_ID. 165 * [12:15] no of LMTST - 1 in the burst. 166 * [19:63] data size of each LMTST in the burst except first. 167 */ 168 val = (lmt_info->lmt_id & 0x7FF); 169 /* Target address for LMTST flush tells HW how many 128bit 170 * words are present. 171 * tar_addr[6:4] size of first LMTST - 1 in units of 128b. 172 */ 173 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; 174 dma_wmb(); 175 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size); 176 cn10k_lmt_flush(val, tar_addr); 177 178 sq->head++; 179 sq->head &= (sq->sqe_cnt - 1); 180 } 181 182 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) 183 { 184 struct nix_bandprof_free_req *req; 185 int rc; 186 187 if (is_dev_otx2(pfvf->pdev)) 188 return 0; 189 190 mutex_lock(&pfvf->mbox.lock); 191 192 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 193 if (!req) { 194 rc = -ENOMEM; 195 goto out; 196 } 197 198 /* Free all bandwidth profiles allocated */ 199 req->free_all = true; 200 201 rc = otx2_sync_mbox_msg(&pfvf->mbox); 202 out: 203 mutex_unlock(&pfvf->mbox.lock); 204 return rc; 205 } 206 207 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) 208 { 209 struct nix_bandprof_alloc_req *req; 210 struct nix_bandprof_alloc_rsp *rsp; 211 int rc; 212 213 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox); 214 if (!req) 215 return -ENOMEM; 216 217 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 218 219 rc = otx2_sync_mbox_msg(&pfvf->mbox); 220 if (rc) 221 goto out; 222 223 rsp = (struct nix_bandprof_alloc_rsp *) 224 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 225 if (IS_ERR(rsp)) { 226 rc = PTR_ERR(rsp); 227 goto out; 228 } 229 230 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { 231 rc = -EIO; 232 goto out; 233 } 234 235 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; 236 out: 237 if (rc) { 238 dev_warn(pfvf->dev, 239 "Failed to allocate ingress bandwidth policer\n"); 240 } 241 242 return rc; 243 } 244 245 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) 246 { 247 struct otx2_hw *hw = &pfvf->hw; 248 int ret; 249 250 mutex_lock(&pfvf->mbox.lock); 251 252 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer); 253 254 mutex_unlock(&pfvf->mbox.lock); 255 256 return ret; 257 } 258 259 #define POLICER_TIMESTAMP 1 /* 1 second */ 260 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ 261 262 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, 263 u32 *burst_mantissa) 264 { 265 int tmp; 266 267 /* Burst is calculated as 268 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] 269 * This is the upper limit on number tokens (bytes) that 270 * can be accumulated in the bucket. 271 */ 272 *burst_exp = ilog2(burst); 273 if (burst < 256) { 274 /* No float: can't express mantissa in this case */ 275 *burst_mantissa = 0; 276 return; 277 } 278 279 if (*burst_exp > MAX_RATE_EXP) 280 *burst_exp = MAX_RATE_EXP; 281 282 /* Calculate mantissa 283 * Find remaining bytes 'burst - 2^burst_exp' 284 * mantissa = (remaining bytes) / 2^ (burst_exp - 8) 285 */ 286 tmp = burst - rounddown_pow_of_two(burst); 287 *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); 288 } 289 290 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, 291 u32 *rate_mantissa, u32 *rdiv) 292 { 293 u32 div = 0; 294 u32 exp = 0; 295 u64 tmp; 296 297 /* Figure out mantissa, exponent and divider from given max pkt rate 298 * 299 * To achieve desired rate HW adds 300 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every 301 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. 302 * Here policer timeunit is 2 usecs and rate is in bits per sec. 303 * Since floating point cannot be used below algorithm uses 1000000 304 * scale factor to support rates upto 100Gbps. 305 */ 306 tmp = rate * 32 * 2; 307 if (tmp < 256000000) { 308 while (tmp < 256000000) { 309 tmp = tmp * 2; 310 div++; 311 } 312 } else { 313 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) 314 tmp = tmp / 2; 315 316 if (exp > MAX_RATE_EXP) 317 exp = MAX_RATE_EXP; 318 } 319 320 *rate_mantissa = (tmp - 256000000) / 1000000; 321 *rate_exp = exp; 322 *rdiv = div; 323 } 324 325 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, 326 u16 policer, bool map) 327 { 328 struct nix_cn10k_aq_enq_req *aq; 329 330 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 331 if (!aq) 332 return -ENOMEM; 333 334 /* Enable policing and set the bandwidth profile (policer) index */ 335 if (map) 336 aq->rq.policer_ena = 1; 337 else 338 aq->rq.policer_ena = 0; 339 aq->rq_mask.policer_ena = 1; 340 341 aq->rq.band_prof_id = policer; 342 aq->rq_mask.band_prof_id = GENMASK(9, 0); 343 344 /* If policer id is greater than 1023 then it implies hardware supports 345 * more leaf profiles. In that case use band_prof_id_h for 4 MSBs. 346 */ 347 aq->rq.band_prof_id_h = policer >> 10; 348 aq->rq_mask.band_prof_id_h = GENMASK(3, 0); 349 350 /* Fill AQ info */ 351 aq->qidx = rq_idx; 352 aq->ctype = NIX_AQ_CTYPE_RQ; 353 aq->op = NIX_AQ_INSTOP_WRITE; 354 355 return otx2_sync_mbox_msg(&pfvf->mbox); 356 } 357 358 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) 359 { 360 struct nix_bandprof_free_req *req; 361 362 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 363 if (!req) 364 return -ENOMEM; 365 366 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 367 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; 368 369 return otx2_sync_mbox_msg(&pfvf->mbox); 370 } 371 372 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) 373 { 374 struct otx2_hw *hw = &pfvf->hw; 375 int qidx, rc; 376 377 mutex_lock(&pfvf->mbox.lock); 378 379 /* Remove RQ's policer mapping */ 380 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 381 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, hw->matchall_ipolicer, false); 382 if (rc) 383 dev_warn(pfvf->dev, "Failed to unmap RQ %d's policer (error %d).", 384 qidx, rc); 385 } 386 387 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); 388 389 mutex_unlock(&pfvf->mbox.lock); 390 return rc; 391 } 392 393 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, 394 u32 burst, u64 rate, bool pps) 395 { 396 struct nix_cn10k_aq_enq_req *aq; 397 u32 burst_exp, burst_mantissa; 398 u32 rate_exp, rate_mantissa; 399 u32 rdiv; 400 401 /* Get exponent and mantissa values for the desired rate */ 402 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa); 403 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv); 404 405 /* Init bandwidth profile */ 406 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 407 if (!aq) 408 return -ENOMEM; 409 410 /* Set initial color mode to blind */ 411 aq->prof.icolor = 0x03; 412 aq->prof_mask.icolor = 0x03; 413 414 /* Set rate and burst values */ 415 aq->prof.cir_exponent = rate_exp; 416 aq->prof_mask.cir_exponent = 0x1F; 417 418 aq->prof.cir_mantissa = rate_mantissa; 419 aq->prof_mask.cir_mantissa = 0xFF; 420 421 aq->prof.cbs_exponent = burst_exp; 422 aq->prof_mask.cbs_exponent = 0x1F; 423 424 aq->prof.cbs_mantissa = burst_mantissa; 425 aq->prof_mask.cbs_mantissa = 0xFF; 426 427 aq->prof.rdiv = rdiv; 428 aq->prof_mask.rdiv = 0xF; 429 430 if (pps) { 431 /* The amount of decremented tokens is calculated according to 432 * the following equation: 433 * max([ LMODE ? 0 : (packet_length - LXPTR)] + 434 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], 435 * 1/256) 436 * if LMODE is 1 then rate limiting will be based on 437 * PPS otherwise bps. 438 * The aim of the ADJUST value is to specify a token cost per 439 * packet in contrary to the packet length that specifies a 440 * cost per byte. To rate limit based on PPS adjust mantissa 441 * is set as 384 and exponent as 1 so that number of tokens 442 * decremented becomes 1 i.e, 1 token per packeet. 443 */ 444 aq->prof.adjust_exponent = 1; 445 aq->prof_mask.adjust_exponent = 0x1F; 446 447 aq->prof.adjust_mantissa = 384; 448 aq->prof_mask.adjust_mantissa = 0x1FF; 449 450 aq->prof.lmode = 0x1; 451 aq->prof_mask.lmode = 0x1; 452 } 453 454 /* Two rate three color marker 455 * With PEIR/EIR set to zero, color will be either green or red 456 */ 457 aq->prof.meter_algo = 2; 458 aq->prof_mask.meter_algo = 0x3; 459 460 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; 461 aq->prof_mask.rc_action = 0x3; 462 463 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 464 aq->prof_mask.yc_action = 0x3; 465 466 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 467 aq->prof_mask.gc_action = 0x3; 468 469 /* Setting exponent value as 24 and mantissa as 0 configures 470 * the bucket with zero values making bucket unused. Peak 471 * information rate and Excess information rate buckets are 472 * unused here. 473 */ 474 aq->prof.peir_exponent = 24; 475 aq->prof_mask.peir_exponent = 0x1F; 476 477 aq->prof.peir_mantissa = 0; 478 aq->prof_mask.peir_mantissa = 0xFF; 479 480 aq->prof.pebs_exponent = 24; 481 aq->prof_mask.pebs_exponent = 0x1F; 482 483 aq->prof.pebs_mantissa = 0; 484 aq->prof_mask.pebs_mantissa = 0xFF; 485 486 aq->prof.hl_en = 0; 487 aq->prof_mask.hl_en = 1; 488 489 /* Fill AQ info */ 490 aq->qidx = profile; 491 aq->ctype = NIX_AQ_CTYPE_BANDPROF; 492 aq->op = NIX_AQ_INSTOP_WRITE; 493 494 return otx2_sync_mbox_msg(&pfvf->mbox); 495 } 496 497 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, 498 u32 burst, u64 rate) 499 { 500 struct otx2_hw *hw = &pfvf->hw; 501 int qidx, rc; 502 503 mutex_lock(&pfvf->mbox.lock); 504 505 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst, 506 rate, false); 507 if (rc) 508 goto out; 509 510 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 511 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, 512 hw->matchall_ipolicer, true); 513 if (rc) 514 break; 515 } 516 517 out: 518 mutex_unlock(&pfvf->mbox.lock); 519 return rc; 520 } 521