1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2021 Marvell. 5 * 6 */ 7 8 #include "cn10k.h" 9 #include "otx2_reg.h" 10 #include "otx2_struct.h" 11 12 static struct dev_hw_ops otx2_hw_ops = { 13 .sq_aq_init = otx2_sq_aq_init, 14 .sqe_flush = otx2_sqe_flush, 15 .aura_freeptr = otx2_aura_freeptr, 16 .refill_pool_ptrs = otx2_refill_pool_ptrs, 17 }; 18 19 static struct dev_hw_ops cn10k_hw_ops = { 20 .sq_aq_init = cn10k_sq_aq_init, 21 .sqe_flush = cn10k_sqe_flush, 22 .aura_freeptr = cn10k_aura_freeptr, 23 .refill_pool_ptrs = cn10k_refill_pool_ptrs, 24 }; 25 26 int cn10k_lmtst_init(struct otx2_nic *pfvf) 27 { 28 29 struct lmtst_tbl_setup_req *req; 30 struct otx2_lmt_info *lmt_info; 31 int err, cpu; 32 33 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 34 pfvf->hw_ops = &otx2_hw_ops; 35 return 0; 36 } 37 38 pfvf->hw_ops = &cn10k_hw_ops; 39 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/ 40 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE); 41 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info); 42 43 mutex_lock(&pfvf->mbox.lock); 44 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox); 45 if (!req) { 46 mutex_unlock(&pfvf->mbox.lock); 47 return -ENOMEM; 48 } 49 50 req->use_local_lmt_region = true; 51 52 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines, 53 LMT_LINE_SIZE); 54 if (err) { 55 mutex_unlock(&pfvf->mbox.lock); 56 return err; 57 } 58 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base; 59 req->lmt_iova = (u64)pfvf->dync_lmt->iova; 60 61 err = otx2_sync_mbox_msg(&pfvf->mbox); 62 mutex_unlock(&pfvf->mbox.lock); 63 64 for_each_possible_cpu(cpu) { 65 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu); 66 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base + 67 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE)); 68 lmt_info->lmt_id = cpu * LMT_BURST_SIZE; 69 } 70 71 return 0; 72 } 73 EXPORT_SYMBOL(cn10k_lmtst_init); 74 75 int cn10k_sq_aq_init(void *dev, u16 qidx, u16 sqb_aura) 76 { 77 struct nix_cn10k_aq_enq_req *aq; 78 struct otx2_nic *pfvf = dev; 79 80 /* Get memory to put this msg */ 81 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 82 if (!aq) 83 return -ENOMEM; 84 85 aq->sq.cq = pfvf->hw.rx_queues + qidx; 86 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */ 87 aq->sq.cq_ena = 1; 88 aq->sq.ena = 1; 89 /* Only one SMQ is allocated, map all SQ's to that SMQ */ 90 aq->sq.smq = pfvf->hw.txschq_list[NIX_TXSCH_LVL_SMQ][0]; 91 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->max_frs); 92 aq->sq.default_chan = pfvf->hw.tx_chan_base; 93 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */ 94 aq->sq.sqb_aura = sqb_aura; 95 aq->sq.sq_int_ena = NIX_SQINT_BITS; 96 aq->sq.qint_idx = 0; 97 /* Due pipelining impact minimum 2000 unused SQ CQE's 98 * need to maintain to avoid CQ overflow. 99 */ 100 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt)); 101 102 /* Fill AQ info */ 103 aq->qidx = qidx; 104 aq->ctype = NIX_AQ_CTYPE_SQ; 105 aq->op = NIX_AQ_INSTOP_INIT; 106 107 return otx2_sync_mbox_msg(&pfvf->mbox); 108 } 109 110 #define NPA_MAX_BURST 16 111 void cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq) 112 { 113 struct otx2_nic *pfvf = dev; 114 u64 ptrs[NPA_MAX_BURST]; 115 int num_ptrs = 1; 116 dma_addr_t bufptr; 117 118 /* Refill pool with new buffers */ 119 while (cq->pool_ptrs) { 120 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) { 121 if (num_ptrs--) 122 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 123 num_ptrs); 124 break; 125 } 126 cq->pool_ptrs--; 127 ptrs[num_ptrs] = (u64)bufptr + OTX2_HEAD_ROOM; 128 num_ptrs++; 129 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) { 130 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs, 131 num_ptrs); 132 num_ptrs = 1; 133 } 134 } 135 } 136 137 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx) 138 { 139 struct otx2_lmt_info *lmt_info; 140 struct otx2_nic *pfvf = dev; 141 u64 val = 0, tar_addr = 0; 142 143 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id()); 144 /* FIXME: val[0:10] LMT_ID. 145 * [12:15] no of LMTST - 1 in the burst. 146 * [19:63] data size of each LMTST in the burst except first. 147 */ 148 val = (lmt_info->lmt_id & 0x7FF); 149 /* Target address for LMTST flush tells HW how many 128bit 150 * words are present. 151 * tar_addr[6:4] size of first LMTST - 1 in units of 128b. 152 */ 153 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4; 154 dma_wmb(); 155 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size); 156 cn10k_lmt_flush(val, tar_addr); 157 158 sq->head++; 159 sq->head &= (sq->sqe_cnt - 1); 160 } 161 162 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf) 163 { 164 struct nix_bandprof_free_req *req; 165 int rc; 166 167 if (is_dev_otx2(pfvf->pdev)) 168 return 0; 169 170 mutex_lock(&pfvf->mbox.lock); 171 172 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 173 if (!req) { 174 rc = -ENOMEM; 175 goto out; 176 } 177 178 /* Free all bandwidth profiles allocated */ 179 req->free_all = true; 180 181 rc = otx2_sync_mbox_msg(&pfvf->mbox); 182 out: 183 mutex_unlock(&pfvf->mbox.lock); 184 return rc; 185 } 186 187 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf) 188 { 189 struct nix_bandprof_alloc_req *req; 190 struct nix_bandprof_alloc_rsp *rsp; 191 int rc; 192 193 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox); 194 if (!req) 195 return -ENOMEM; 196 197 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 198 199 rc = otx2_sync_mbox_msg(&pfvf->mbox); 200 if (rc) 201 goto out; 202 203 rsp = (struct nix_bandprof_alloc_rsp *) 204 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr); 205 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) { 206 rc = -EIO; 207 goto out; 208 } 209 210 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0]; 211 out: 212 if (rc) { 213 dev_warn(pfvf->dev, 214 "Failed to allocate ingress bandwidth policer\n"); 215 } 216 217 return rc; 218 } 219 220 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf) 221 { 222 struct otx2_hw *hw = &pfvf->hw; 223 int ret; 224 225 mutex_lock(&pfvf->mbox.lock); 226 227 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer); 228 229 mutex_unlock(&pfvf->mbox.lock); 230 231 return ret; 232 } 233 234 #define POLICER_TIMESTAMP 1 /* 1 second */ 235 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */ 236 237 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp, 238 u32 *burst_mantissa) 239 { 240 int tmp; 241 242 /* Burst is calculated as 243 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT] 244 * This is the upper limit on number tokens (bytes) that 245 * can be accumulated in the bucket. 246 */ 247 *burst_exp = ilog2(burst); 248 if (burst < 256) { 249 /* No float: can't express mantissa in this case */ 250 *burst_mantissa = 0; 251 return; 252 } 253 254 if (*burst_exp > MAX_RATE_EXP) 255 *burst_exp = MAX_RATE_EXP; 256 257 /* Calculate mantissa 258 * Find remaining bytes 'burst - 2^burst_exp' 259 * mantissa = (remaining bytes) / 2^ (burst_exp - 8) 260 */ 261 tmp = burst - rounddown_pow_of_two(burst); 262 *burst_mantissa = tmp / (1UL << (*burst_exp - 8)); 263 } 264 265 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp, 266 u32 *rate_mantissa, u32 *rdiv) 267 { 268 u32 div = 0; 269 u32 exp = 0; 270 u64 tmp; 271 272 /* Figure out mantissa, exponent and divider from given max pkt rate 273 * 274 * To achieve desired rate HW adds 275 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every 276 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket. 277 * Here policer timeunit is 2 usecs and rate is in bits per sec. 278 * Since floating point cannot be used below algorithm uses 1000000 279 * scale factor to support rates upto 100Gbps. 280 */ 281 tmp = rate * 32 * 2; 282 if (tmp < 256000000) { 283 while (tmp < 256000000) { 284 tmp = tmp * 2; 285 div++; 286 } 287 } else { 288 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++) 289 tmp = tmp / 2; 290 291 if (exp > MAX_RATE_EXP) 292 exp = MAX_RATE_EXP; 293 } 294 295 *rate_mantissa = (tmp - 256000000) / 1000000; 296 *rate_exp = exp; 297 *rdiv = div; 298 } 299 300 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx, 301 u16 policer, bool map) 302 { 303 struct nix_cn10k_aq_enq_req *aq; 304 305 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 306 if (!aq) 307 return -ENOMEM; 308 309 /* Enable policing and set the bandwidth profile (policer) index */ 310 if (map) 311 aq->rq.policer_ena = 1; 312 else 313 aq->rq.policer_ena = 0; 314 aq->rq_mask.policer_ena = 1; 315 316 aq->rq.band_prof_id = policer; 317 aq->rq_mask.band_prof_id = GENMASK(9, 0); 318 319 /* Fill AQ info */ 320 aq->qidx = rq_idx; 321 aq->ctype = NIX_AQ_CTYPE_RQ; 322 aq->op = NIX_AQ_INSTOP_WRITE; 323 324 return otx2_sync_mbox_msg(&pfvf->mbox); 325 } 326 327 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf) 328 { 329 struct nix_bandprof_free_req *req; 330 331 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox); 332 if (!req) 333 return -ENOMEM; 334 335 req->prof_count[BAND_PROF_LEAF_LAYER] = 1; 336 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf; 337 338 return otx2_sync_mbox_msg(&pfvf->mbox); 339 } 340 341 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf) 342 { 343 struct otx2_hw *hw = &pfvf->hw; 344 int qidx, rc; 345 346 mutex_lock(&pfvf->mbox.lock); 347 348 /* Remove RQ's policer mapping */ 349 for (qidx = 0; qidx < hw->rx_queues; qidx++) 350 cn10k_map_unmap_rq_policer(pfvf, qidx, 351 hw->matchall_ipolicer, false); 352 353 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer); 354 355 mutex_unlock(&pfvf->mbox.lock); 356 return rc; 357 } 358 359 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile, 360 u32 burst, u64 rate, bool pps) 361 { 362 struct nix_cn10k_aq_enq_req *aq; 363 u32 burst_exp, burst_mantissa; 364 u32 rate_exp, rate_mantissa; 365 u32 rdiv; 366 367 /* Get exponent and mantissa values for the desired rate */ 368 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa); 369 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv); 370 371 /* Init bandwidth profile */ 372 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 373 if (!aq) 374 return -ENOMEM; 375 376 /* Set initial color mode to blind */ 377 aq->prof.icolor = 0x03; 378 aq->prof_mask.icolor = 0x03; 379 380 /* Set rate and burst values */ 381 aq->prof.cir_exponent = rate_exp; 382 aq->prof_mask.cir_exponent = 0x1F; 383 384 aq->prof.cir_mantissa = rate_mantissa; 385 aq->prof_mask.cir_mantissa = 0xFF; 386 387 aq->prof.cbs_exponent = burst_exp; 388 aq->prof_mask.cbs_exponent = 0x1F; 389 390 aq->prof.cbs_mantissa = burst_mantissa; 391 aq->prof_mask.cbs_mantissa = 0xFF; 392 393 aq->prof.rdiv = rdiv; 394 aq->prof_mask.rdiv = 0xF; 395 396 if (pps) { 397 /* The amount of decremented tokens is calculated according to 398 * the following equation: 399 * max([ LMODE ? 0 : (packet_length - LXPTR)] + 400 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT], 401 * 1/256) 402 * if LMODE is 1 then rate limiting will be based on 403 * PPS otherwise bps. 404 * The aim of the ADJUST value is to specify a token cost per 405 * packet in contrary to the packet length that specifies a 406 * cost per byte. To rate limit based on PPS adjust mantissa 407 * is set as 384 and exponent as 1 so that number of tokens 408 * decremented becomes 1 i.e, 1 token per packeet. 409 */ 410 aq->prof.adjust_exponent = 1; 411 aq->prof_mask.adjust_exponent = 0x1F; 412 413 aq->prof.adjust_mantissa = 384; 414 aq->prof_mask.adjust_mantissa = 0x1FF; 415 416 aq->prof.lmode = 0x1; 417 aq->prof_mask.lmode = 0x1; 418 } 419 420 /* Two rate three color marker 421 * With PEIR/EIR set to zero, color will be either green or red 422 */ 423 aq->prof.meter_algo = 2; 424 aq->prof_mask.meter_algo = 0x3; 425 426 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP; 427 aq->prof_mask.rc_action = 0x3; 428 429 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 430 aq->prof_mask.yc_action = 0x3; 431 432 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS; 433 aq->prof_mask.gc_action = 0x3; 434 435 /* Setting exponent value as 24 and mantissa as 0 configures 436 * the bucket with zero values making bucket unused. Peak 437 * information rate and Excess information rate buckets are 438 * unused here. 439 */ 440 aq->prof.peir_exponent = 24; 441 aq->prof_mask.peir_exponent = 0x1F; 442 443 aq->prof.peir_mantissa = 0; 444 aq->prof_mask.peir_mantissa = 0xFF; 445 446 aq->prof.pebs_exponent = 24; 447 aq->prof_mask.pebs_exponent = 0x1F; 448 449 aq->prof.pebs_mantissa = 0; 450 aq->prof_mask.pebs_mantissa = 0xFF; 451 452 /* Fill AQ info */ 453 aq->qidx = profile; 454 aq->ctype = NIX_AQ_CTYPE_BANDPROF; 455 aq->op = NIX_AQ_INSTOP_WRITE; 456 457 return otx2_sync_mbox_msg(&pfvf->mbox); 458 } 459 460 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf, 461 u32 burst, u64 rate) 462 { 463 struct otx2_hw *hw = &pfvf->hw; 464 int qidx, rc; 465 466 mutex_lock(&pfvf->mbox.lock); 467 468 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst, 469 rate, false); 470 if (rc) 471 goto out; 472 473 for (qidx = 0; qidx < hw->rx_queues; qidx++) { 474 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, 475 hw->matchall_ipolicer, true); 476 if (rc) 477 break; 478 } 479 480 out: 481 mutex_unlock(&pfvf->mbox.lock); 482 return rc; 483 } 484