1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2021 Marvell.
5 *
6 */
7
8 #include "cn10k.h"
9 #include "otx2_reg.h"
10 #include "otx2_struct.h"
11
12 static struct dev_hw_ops otx2_hw_ops = {
13 .sq_aq_init = otx2_sq_aq_init,
14 .sqe_flush = otx2_sqe_flush,
15 .aura_freeptr = otx2_aura_freeptr,
16 .refill_pool_ptrs = otx2_refill_pool_ptrs,
17 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler,
18 };
19
20 static struct dev_hw_ops cn10k_hw_ops = {
21 .sq_aq_init = cn10k_sq_aq_init,
22 .sqe_flush = cn10k_sqe_flush,
23 .aura_freeptr = cn10k_aura_freeptr,
24 .refill_pool_ptrs = cn10k_refill_pool_ptrs,
25 .pfaf_mbox_intr_handler = otx2_pfaf_mbox_intr_handler,
26 };
27
otx2_init_hw_ops(struct otx2_nic * pfvf)28 void otx2_init_hw_ops(struct otx2_nic *pfvf)
29 {
30 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) {
31 pfvf->hw_ops = &otx2_hw_ops;
32 return;
33 }
34
35 pfvf->hw_ops = &cn10k_hw_ops;
36 }
37 EXPORT_SYMBOL(otx2_init_hw_ops);
38
cn10k_lmtst_init(struct otx2_nic * pfvf)39 int cn10k_lmtst_init(struct otx2_nic *pfvf)
40 {
41
42 struct lmtst_tbl_setup_req *req;
43 struct otx2_lmt_info *lmt_info;
44 int err, cpu;
45
46 if (!test_bit(CN10K_LMTST, &pfvf->hw.cap_flag))
47 return 0;
48
49 /* Total LMTLINES = num_online_cpus() * 32 (For Burst flush).*/
50 pfvf->tot_lmt_lines = (num_online_cpus() * LMT_BURST_SIZE);
51 pfvf->hw.lmt_info = alloc_percpu(struct otx2_lmt_info);
52
53 mutex_lock(&pfvf->mbox.lock);
54 req = otx2_mbox_alloc_msg_lmtst_tbl_setup(&pfvf->mbox);
55 if (!req) {
56 mutex_unlock(&pfvf->mbox.lock);
57 return -ENOMEM;
58 }
59
60 req->use_local_lmt_region = true;
61
62 err = qmem_alloc(pfvf->dev, &pfvf->dync_lmt, pfvf->tot_lmt_lines,
63 LMT_LINE_SIZE);
64 if (err) {
65 mutex_unlock(&pfvf->mbox.lock);
66 return err;
67 }
68 pfvf->hw.lmt_base = (u64 *)pfvf->dync_lmt->base;
69 req->lmt_iova = (u64)pfvf->dync_lmt->iova;
70
71 err = otx2_sync_mbox_msg(&pfvf->mbox);
72 mutex_unlock(&pfvf->mbox.lock);
73
74 for_each_possible_cpu(cpu) {
75 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, cpu);
76 lmt_info->lmt_addr = ((u64)pfvf->hw.lmt_base +
77 (cpu * LMT_BURST_SIZE * LMT_LINE_SIZE));
78 lmt_info->lmt_id = cpu * LMT_BURST_SIZE;
79 }
80
81 return 0;
82 }
83 EXPORT_SYMBOL(cn10k_lmtst_init);
84
cn10k_sq_aq_init(void * dev,u16 qidx,u8 chan_offset,u16 sqb_aura)85 int cn10k_sq_aq_init(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura)
86 {
87 struct nix_cn10k_aq_enq_req *aq;
88 struct otx2_nic *pfvf = dev;
89
90 /* Get memory to put this msg */
91 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
92 if (!aq)
93 return -ENOMEM;
94
95 aq->sq.cq = pfvf->hw.rx_queues + qidx;
96 aq->sq.max_sqe_size = NIX_MAXSQESZ_W16; /* 128 byte */
97 aq->sq.cq_ena = 1;
98 aq->sq.ena = 1;
99 aq->sq.smq = otx2_get_smq_idx(pfvf, qidx);
100 aq->sq.smq_rr_weight = mtu_to_dwrr_weight(pfvf, pfvf->tx_max_pktlen);
101 aq->sq.default_chan = pfvf->hw.tx_chan_base + chan_offset;
102 aq->sq.sqe_stype = NIX_STYPE_STF; /* Cache SQB */
103 aq->sq.sqb_aura = sqb_aura;
104 aq->sq.sq_int_ena = NIX_SQINT_BITS;
105 aq->sq.qint_idx = 0;
106 /* Due pipelining impact minimum 2000 unused SQ CQE's
107 * need to maintain to avoid CQ overflow.
108 */
109 aq->sq.cq_limit = ((SEND_CQ_SKID * 256) / (pfvf->qset.sqe_cnt));
110
111 /* Fill AQ info */
112 aq->qidx = qidx;
113 aq->ctype = NIX_AQ_CTYPE_SQ;
114 aq->op = NIX_AQ_INSTOP_INIT;
115
116 return otx2_sync_mbox_msg(&pfvf->mbox);
117 }
118
119 #define NPA_MAX_BURST 16
cn10k_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)120 int cn10k_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
121 {
122 struct otx2_nic *pfvf = dev;
123 int cnt = cq->pool_ptrs;
124 u64 ptrs[NPA_MAX_BURST];
125 struct otx2_pool *pool;
126 dma_addr_t bufptr;
127 int num_ptrs = 1;
128
129 pool = &pfvf->qset.pool[cq->cq_idx];
130
131 /* Refill pool with new buffers */
132 while (cq->pool_ptrs) {
133 if (otx2_alloc_buffer(pfvf, cq, &bufptr)) {
134 if (num_ptrs--)
135 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
136 num_ptrs);
137 break;
138 }
139 cq->pool_ptrs--;
140 ptrs[num_ptrs] = pool->xsk_pool ?
141 (u64)bufptr : (u64)bufptr + OTX2_HEAD_ROOM;
142
143 num_ptrs++;
144 if (num_ptrs == NPA_MAX_BURST || cq->pool_ptrs == 0) {
145 __cn10k_aura_freeptr(pfvf, cq->cq_idx, ptrs,
146 num_ptrs);
147 num_ptrs = 1;
148 }
149 }
150 return cnt - cq->pool_ptrs;
151 }
152
cn10k_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)153 void cn10k_sqe_flush(void *dev, struct otx2_snd_queue *sq, int size, int qidx)
154 {
155 struct otx2_lmt_info *lmt_info;
156 struct otx2_nic *pfvf = dev;
157 u64 val = 0, tar_addr = 0;
158
159 lmt_info = per_cpu_ptr(pfvf->hw.lmt_info, smp_processor_id());
160 /* FIXME: val[0:10] LMT_ID.
161 * [12:15] no of LMTST - 1 in the burst.
162 * [19:63] data size of each LMTST in the burst except first.
163 */
164 val = (lmt_info->lmt_id & 0x7FF);
165 /* Target address for LMTST flush tells HW how many 128bit
166 * words are present.
167 * tar_addr[6:4] size of first LMTST - 1 in units of 128b.
168 */
169 tar_addr |= sq->io_addr | (((size / 16) - 1) & 0x7) << 4;
170 dma_wmb();
171 memcpy((u64 *)lmt_info->lmt_addr, sq->sqe_base, size);
172 cn10k_lmt_flush(val, tar_addr);
173
174 sq->head++;
175 sq->head &= (sq->sqe_cnt - 1);
176 }
177
cn10k_free_all_ipolicers(struct otx2_nic * pfvf)178 int cn10k_free_all_ipolicers(struct otx2_nic *pfvf)
179 {
180 struct nix_bandprof_free_req *req;
181 int rc;
182
183 if (is_dev_otx2(pfvf->pdev))
184 return 0;
185
186 mutex_lock(&pfvf->mbox.lock);
187
188 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
189 if (!req) {
190 rc = -ENOMEM;
191 goto out;
192 }
193
194 /* Free all bandwidth profiles allocated */
195 req->free_all = true;
196
197 rc = otx2_sync_mbox_msg(&pfvf->mbox);
198 out:
199 mutex_unlock(&pfvf->mbox.lock);
200 return rc;
201 }
202
cn10k_alloc_leaf_profile(struct otx2_nic * pfvf,u16 * leaf)203 int cn10k_alloc_leaf_profile(struct otx2_nic *pfvf, u16 *leaf)
204 {
205 struct nix_bandprof_alloc_req *req;
206 struct nix_bandprof_alloc_rsp *rsp;
207 int rc;
208
209 req = otx2_mbox_alloc_msg_nix_bandprof_alloc(&pfvf->mbox);
210 if (!req)
211 return -ENOMEM;
212
213 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
214
215 rc = otx2_sync_mbox_msg(&pfvf->mbox);
216 if (rc)
217 goto out;
218
219 rsp = (struct nix_bandprof_alloc_rsp *)
220 otx2_mbox_get_rsp(&pfvf->mbox.mbox, 0, &req->hdr);
221 if (IS_ERR(rsp)) {
222 rc = PTR_ERR(rsp);
223 goto out;
224 }
225
226 if (!rsp->prof_count[BAND_PROF_LEAF_LAYER]) {
227 rc = -EIO;
228 goto out;
229 }
230
231 *leaf = rsp->prof_idx[BAND_PROF_LEAF_LAYER][0];
232 out:
233 if (rc) {
234 dev_warn(pfvf->dev,
235 "Failed to allocate ingress bandwidth policer\n");
236 }
237
238 return rc;
239 }
240
cn10k_alloc_matchall_ipolicer(struct otx2_nic * pfvf)241 int cn10k_alloc_matchall_ipolicer(struct otx2_nic *pfvf)
242 {
243 struct otx2_hw *hw = &pfvf->hw;
244 int ret;
245
246 mutex_lock(&pfvf->mbox.lock);
247
248 ret = cn10k_alloc_leaf_profile(pfvf, &hw->matchall_ipolicer);
249
250 mutex_unlock(&pfvf->mbox.lock);
251
252 return ret;
253 }
254
255 #define POLICER_TIMESTAMP 1 /* 1 second */
256 #define MAX_RATE_EXP 22 /* Valid rate exponent range: 0 - 22 */
257
cn10k_get_ingress_burst_cfg(u32 burst,u32 * burst_exp,u32 * burst_mantissa)258 static void cn10k_get_ingress_burst_cfg(u32 burst, u32 *burst_exp,
259 u32 *burst_mantissa)
260 {
261 int tmp;
262
263 /* Burst is calculated as
264 * (1+[BURST_MANTISSA]/256)*2^[BURST_EXPONENT]
265 * This is the upper limit on number tokens (bytes) that
266 * can be accumulated in the bucket.
267 */
268 *burst_exp = ilog2(burst);
269 if (burst < 256) {
270 /* No float: can't express mantissa in this case */
271 *burst_mantissa = 0;
272 return;
273 }
274
275 if (*burst_exp > MAX_RATE_EXP)
276 *burst_exp = MAX_RATE_EXP;
277
278 /* Calculate mantissa
279 * Find remaining bytes 'burst - 2^burst_exp'
280 * mantissa = (remaining bytes) / 2^ (burst_exp - 8)
281 */
282 tmp = burst - rounddown_pow_of_two(burst);
283 *burst_mantissa = tmp / (1UL << (*burst_exp - 8));
284 }
285
cn10k_get_ingress_rate_cfg(u64 rate,u32 * rate_exp,u32 * rate_mantissa,u32 * rdiv)286 static void cn10k_get_ingress_rate_cfg(u64 rate, u32 *rate_exp,
287 u32 *rate_mantissa, u32 *rdiv)
288 {
289 u32 div = 0;
290 u32 exp = 0;
291 u64 tmp;
292
293 /* Figure out mantissa, exponent and divider from given max pkt rate
294 *
295 * To achieve desired rate HW adds
296 * (1+[RATE_MANTISSA]/256)*2^[RATE_EXPONENT] tokens (bytes) at every
297 * policer timeunit * 2^rdiv ie 2 * 2^rdiv usecs, to the token bucket.
298 * Here policer timeunit is 2 usecs and rate is in bits per sec.
299 * Since floating point cannot be used below algorithm uses 1000000
300 * scale factor to support rates upto 100Gbps.
301 */
302 tmp = rate * 32 * 2;
303 if (tmp < 256000000) {
304 while (tmp < 256000000) {
305 tmp = tmp * 2;
306 div++;
307 }
308 } else {
309 for (exp = 0; tmp >= 512000000 && exp <= MAX_RATE_EXP; exp++)
310 tmp = tmp / 2;
311
312 if (exp > MAX_RATE_EXP)
313 exp = MAX_RATE_EXP;
314 }
315
316 *rate_mantissa = (tmp - 256000000) / 1000000;
317 *rate_exp = exp;
318 *rdiv = div;
319 }
320
cn10k_map_unmap_rq_policer(struct otx2_nic * pfvf,int rq_idx,u16 policer,bool map)321 int cn10k_map_unmap_rq_policer(struct otx2_nic *pfvf, int rq_idx,
322 u16 policer, bool map)
323 {
324 struct nix_cn10k_aq_enq_req *aq;
325
326 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
327 if (!aq)
328 return -ENOMEM;
329
330 /* Enable policing and set the bandwidth profile (policer) index */
331 if (map)
332 aq->rq.policer_ena = 1;
333 else
334 aq->rq.policer_ena = 0;
335 aq->rq_mask.policer_ena = 1;
336
337 aq->rq.band_prof_id = policer;
338 aq->rq_mask.band_prof_id = GENMASK(9, 0);
339
340 /* Fill AQ info */
341 aq->qidx = rq_idx;
342 aq->ctype = NIX_AQ_CTYPE_RQ;
343 aq->op = NIX_AQ_INSTOP_WRITE;
344
345 return otx2_sync_mbox_msg(&pfvf->mbox);
346 }
347
cn10k_free_leaf_profile(struct otx2_nic * pfvf,u16 leaf)348 int cn10k_free_leaf_profile(struct otx2_nic *pfvf, u16 leaf)
349 {
350 struct nix_bandprof_free_req *req;
351
352 req = otx2_mbox_alloc_msg_nix_bandprof_free(&pfvf->mbox);
353 if (!req)
354 return -ENOMEM;
355
356 req->prof_count[BAND_PROF_LEAF_LAYER] = 1;
357 req->prof_idx[BAND_PROF_LEAF_LAYER][0] = leaf;
358
359 return otx2_sync_mbox_msg(&pfvf->mbox);
360 }
361
cn10k_free_matchall_ipolicer(struct otx2_nic * pfvf)362 int cn10k_free_matchall_ipolicer(struct otx2_nic *pfvf)
363 {
364 struct otx2_hw *hw = &pfvf->hw;
365 int qidx, rc;
366
367 mutex_lock(&pfvf->mbox.lock);
368
369 /* Remove RQ's policer mapping */
370 for (qidx = 0; qidx < hw->rx_queues; qidx++) {
371 rc = cn10k_map_unmap_rq_policer(pfvf, qidx, hw->matchall_ipolicer, false);
372 if (rc)
373 dev_warn(pfvf->dev, "Failed to unmap RQ %d's policer (error %d).",
374 qidx, rc);
375 }
376
377 rc = cn10k_free_leaf_profile(pfvf, hw->matchall_ipolicer);
378
379 mutex_unlock(&pfvf->mbox.lock);
380 return rc;
381 }
382
cn10k_set_ipolicer_rate(struct otx2_nic * pfvf,u16 profile,u32 burst,u64 rate,bool pps)383 int cn10k_set_ipolicer_rate(struct otx2_nic *pfvf, u16 profile,
384 u32 burst, u64 rate, bool pps)
385 {
386 struct nix_cn10k_aq_enq_req *aq;
387 u32 burst_exp, burst_mantissa;
388 u32 rate_exp, rate_mantissa;
389 u32 rdiv;
390
391 /* Get exponent and mantissa values for the desired rate */
392 cn10k_get_ingress_burst_cfg(burst, &burst_exp, &burst_mantissa);
393 cn10k_get_ingress_rate_cfg(rate, &rate_exp, &rate_mantissa, &rdiv);
394
395 /* Init bandwidth profile */
396 aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox);
397 if (!aq)
398 return -ENOMEM;
399
400 /* Set initial color mode to blind */
401 aq->prof.icolor = 0x03;
402 aq->prof_mask.icolor = 0x03;
403
404 /* Set rate and burst values */
405 aq->prof.cir_exponent = rate_exp;
406 aq->prof_mask.cir_exponent = 0x1F;
407
408 aq->prof.cir_mantissa = rate_mantissa;
409 aq->prof_mask.cir_mantissa = 0xFF;
410
411 aq->prof.cbs_exponent = burst_exp;
412 aq->prof_mask.cbs_exponent = 0x1F;
413
414 aq->prof.cbs_mantissa = burst_mantissa;
415 aq->prof_mask.cbs_mantissa = 0xFF;
416
417 aq->prof.rdiv = rdiv;
418 aq->prof_mask.rdiv = 0xF;
419
420 if (pps) {
421 /* The amount of decremented tokens is calculated according to
422 * the following equation:
423 * max([ LMODE ? 0 : (packet_length - LXPTR)] +
424 * ([ADJUST_MANTISSA]/256 - 1) * 2^[ADJUST_EXPONENT],
425 * 1/256)
426 * if LMODE is 1 then rate limiting will be based on
427 * PPS otherwise bps.
428 * The aim of the ADJUST value is to specify a token cost per
429 * packet in contrary to the packet length that specifies a
430 * cost per byte. To rate limit based on PPS adjust mantissa
431 * is set as 384 and exponent as 1 so that number of tokens
432 * decremented becomes 1 i.e, 1 token per packeet.
433 */
434 aq->prof.adjust_exponent = 1;
435 aq->prof_mask.adjust_exponent = 0x1F;
436
437 aq->prof.adjust_mantissa = 384;
438 aq->prof_mask.adjust_mantissa = 0x1FF;
439
440 aq->prof.lmode = 0x1;
441 aq->prof_mask.lmode = 0x1;
442 }
443
444 /* Two rate three color marker
445 * With PEIR/EIR set to zero, color will be either green or red
446 */
447 aq->prof.meter_algo = 2;
448 aq->prof_mask.meter_algo = 0x3;
449
450 aq->prof.rc_action = NIX_RX_BAND_PROF_ACTIONRESULT_DROP;
451 aq->prof_mask.rc_action = 0x3;
452
453 aq->prof.yc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
454 aq->prof_mask.yc_action = 0x3;
455
456 aq->prof.gc_action = NIX_RX_BAND_PROF_ACTIONRESULT_PASS;
457 aq->prof_mask.gc_action = 0x3;
458
459 /* Setting exponent value as 24 and mantissa as 0 configures
460 * the bucket with zero values making bucket unused. Peak
461 * information rate and Excess information rate buckets are
462 * unused here.
463 */
464 aq->prof.peir_exponent = 24;
465 aq->prof_mask.peir_exponent = 0x1F;
466
467 aq->prof.peir_mantissa = 0;
468 aq->prof_mask.peir_mantissa = 0xFF;
469
470 aq->prof.pebs_exponent = 24;
471 aq->prof_mask.pebs_exponent = 0x1F;
472
473 aq->prof.pebs_mantissa = 0;
474 aq->prof_mask.pebs_mantissa = 0xFF;
475
476 aq->prof.hl_en = 0;
477 aq->prof_mask.hl_en = 1;
478
479 /* Fill AQ info */
480 aq->qidx = profile;
481 aq->ctype = NIX_AQ_CTYPE_BANDPROF;
482 aq->op = NIX_AQ_INSTOP_WRITE;
483
484 return otx2_sync_mbox_msg(&pfvf->mbox);
485 }
486
cn10k_set_matchall_ipolicer_rate(struct otx2_nic * pfvf,u32 burst,u64 rate)487 int cn10k_set_matchall_ipolicer_rate(struct otx2_nic *pfvf,
488 u32 burst, u64 rate)
489 {
490 struct otx2_hw *hw = &pfvf->hw;
491 int qidx, rc;
492
493 mutex_lock(&pfvf->mbox.lock);
494
495 rc = cn10k_set_ipolicer_rate(pfvf, hw->matchall_ipolicer, burst,
496 rate, false);
497 if (rc)
498 goto out;
499
500 for (qidx = 0; qidx < hw->rx_queues; qidx++) {
501 rc = cn10k_map_unmap_rq_policer(pfvf, qidx,
502 hw->matchall_ipolicer, true);
503 if (rc)
504 break;
505 }
506
507 out:
508 mutex_unlock(&pfvf->mbox.lock);
509 return rc;
510 }
511