xref: /linux/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c (revision 9a87ffc99ec8eb8d35eed7c4f816d75f5cc9662e)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 
4 #include <linux/math64.h>
5 #include "lib/aso.h"
6 #include "en/tc/post_act.h"
7 #include "meter.h"
8 #include "en/tc_priv.h"
9 
10 #define MLX5_START_COLOR_SHIFT 28
11 #define MLX5_METER_MODE_SHIFT 24
12 #define MLX5_CBS_EXP_SHIFT 24
13 #define MLX5_CBS_MAN_SHIFT 16
14 #define MLX5_CIR_EXP_SHIFT 8
15 
16 /* cir = 8*(10^9)*cir_mantissa/(2^cir_exponent)) bits/s */
17 #define MLX5_CONST_CIR 8000000000ULL
18 #define MLX5_CALC_CIR(m, e)  ((MLX5_CONST_CIR * (m)) >> (e))
19 #define MLX5_MAX_CIR ((MLX5_CONST_CIR * 0x100) - 1)
20 
21 /* cbs = cbs_mantissa*2^cbs_exponent */
22 #define MLX5_CALC_CBS(m, e)  ((m) << (e))
23 #define MLX5_MAX_CBS ((0x100ULL << 0x1F) - 1)
24 #define MLX5_MAX_HW_CBS 0x7FFFFFFF
25 
26 struct mlx5e_flow_meter_aso_obj {
27 	struct list_head entry;
28 	int base_id;
29 	int total_meters;
30 
31 	unsigned long meters_map[]; /* must be at the end of this struct */
32 };
33 
34 struct mlx5e_flow_meters {
35 	enum mlx5_flow_namespace_type ns_type;
36 	struct mlx5_aso *aso;
37 	struct mutex aso_lock; /* Protects aso operations */
38 	int log_granularity;
39 	u32 pdn;
40 
41 	DECLARE_HASHTABLE(hashtbl, 8);
42 
43 	struct mutex sync_lock; /* protect flow meter operations */
44 	struct list_head partial_list;
45 	struct list_head full_list;
46 
47 	struct mlx5_core_dev *mdev;
48 	struct mlx5e_post_act *post_act;
49 };
50 
51 static void
mlx5e_flow_meter_cir_calc(u64 cir,u8 * man,u8 * exp)52 mlx5e_flow_meter_cir_calc(u64 cir, u8 *man, u8 *exp)
53 {
54 	s64 _cir, _delta, delta = S64_MAX;
55 	u8 e, _man = 0, _exp = 0;
56 	u64 m;
57 
58 	for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
59 		m = cir << e;
60 		if ((s64)m < 0) /* overflow */
61 			break;
62 		m = div64_u64(m, MLX5_CONST_CIR);
63 		if (m > 0xFF) /* man width 8 bit */
64 			continue;
65 		_cir = MLX5_CALC_CIR(m, e);
66 		_delta = cir - _cir;
67 		if (_delta < delta) {
68 			_man = m;
69 			_exp = e;
70 			if (!_delta)
71 				goto found;
72 			delta = _delta;
73 		}
74 	}
75 
76 found:
77 	*man = _man;
78 	*exp = _exp;
79 }
80 
81 static void
mlx5e_flow_meter_cbs_calc(u64 cbs,u8 * man,u8 * exp)82 mlx5e_flow_meter_cbs_calc(u64 cbs, u8 *man, u8 *exp)
83 {
84 	s64 _cbs, _delta, delta = S64_MAX;
85 	u8 e, _man = 0, _exp = 0;
86 	u64 m;
87 
88 	for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */
89 		m = cbs >> e;
90 		if (m > 0xFF) /* man width 8 bit */
91 			continue;
92 		_cbs = MLX5_CALC_CBS(m, e);
93 		_delta = cbs - _cbs;
94 		if (_delta < delta) {
95 			_man = m;
96 			_exp = e;
97 			if (!_delta)
98 				goto found;
99 			delta = _delta;
100 		}
101 	}
102 
103 found:
104 	*man = _man;
105 	*exp = _exp;
106 }
107 
108 int
mlx5e_tc_meter_modify(struct mlx5_core_dev * mdev,struct mlx5e_flow_meter_handle * meter,struct mlx5e_flow_meter_params * meter_params)109 mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev,
110 		      struct mlx5e_flow_meter_handle *meter,
111 		      struct mlx5e_flow_meter_params *meter_params)
112 {
113 	struct mlx5_wqe_aso_ctrl_seg *aso_ctrl;
114 	struct mlx5_wqe_aso_data_seg *aso_data;
115 	struct mlx5e_flow_meters *flow_meters;
116 	u8 cir_man, cir_exp, cbs_man, cbs_exp;
117 	struct mlx5_aso_wqe *aso_wqe;
118 	unsigned long expires;
119 	struct mlx5_aso *aso;
120 	u64 rate, burst;
121 	u8 ds_cnt;
122 	int err;
123 
124 	rate = meter_params->rate;
125 	burst = meter_params->burst;
126 
127 	/* HW treats each packet as 128 bytes in PPS mode */
128 	if (meter_params->mode == MLX5_RATE_LIMIT_PPS) {
129 		rate <<= 10;
130 		burst <<= 7;
131 	}
132 
133 	if (!rate || rate > MLX5_MAX_CIR || !burst || burst > MLX5_MAX_CBS)
134 		return -EINVAL;
135 
136 	/* HW has limitation of total 31 bits for cbs */
137 	if (burst > MLX5_MAX_HW_CBS) {
138 		mlx5_core_warn(mdev,
139 			       "burst(%lld) is too large, use HW allowed value(%d)\n",
140 			       burst, MLX5_MAX_HW_CBS);
141 		burst = MLX5_MAX_HW_CBS;
142 	}
143 
144 	mlx5_core_dbg(mdev, "meter mode=%d\n", meter_params->mode);
145 	mlx5e_flow_meter_cir_calc(rate, &cir_man, &cir_exp);
146 	mlx5_core_dbg(mdev, "rate=%lld, cir=%lld, exp=%d, man=%d\n",
147 		      rate, MLX5_CALC_CIR(cir_man, cir_exp), cir_exp, cir_man);
148 	mlx5e_flow_meter_cbs_calc(burst, &cbs_man, &cbs_exp);
149 	mlx5_core_dbg(mdev, "burst=%lld, cbs=%lld, exp=%d, man=%d\n",
150 		      burst, MLX5_CALC_CBS((u64)cbs_man, cbs_exp), cbs_exp, cbs_man);
151 
152 	if (!cir_man || !cbs_man)
153 		return -EINVAL;
154 
155 	flow_meters = meter->flow_meters;
156 	aso = flow_meters->aso;
157 
158 	mutex_lock(&flow_meters->aso_lock);
159 	aso_wqe = mlx5_aso_get_wqe(aso);
160 	ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_DS);
161 	mlx5_aso_build_wqe(aso, ds_cnt, aso_wqe, meter->obj_id,
162 			   MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER);
163 
164 	aso_ctrl = &aso_wqe->aso_ctrl;
165 	aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6;
166 	aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE |
167 					  MLX5_ASO_ALWAYS_TRUE << 4;
168 	aso_ctrl->data_offset_condition_operand = MLX5_ASO_LOGICAL_OR << 6;
169 	aso_ctrl->data_mask = cpu_to_be64(0x80FFFFFFULL << (meter->idx ? 0 : 32));
170 
171 	aso_data = (struct mlx5_wqe_aso_data_seg *)(aso_wqe + 1);
172 	memset(aso_data, 0, sizeof(*aso_data));
173 	aso_data->bytewise_data[meter->idx * 8] = cpu_to_be32((0x1 << 31) | /* valid */
174 					(MLX5_FLOW_METER_COLOR_GREEN << MLX5_START_COLOR_SHIFT));
175 	if (meter_params->mode == MLX5_RATE_LIMIT_PPS)
176 		aso_data->bytewise_data[meter->idx * 8] |=
177 			cpu_to_be32(MLX5_FLOW_METER_MODE_NUM_PACKETS << MLX5_METER_MODE_SHIFT);
178 	else
179 		aso_data->bytewise_data[meter->idx * 8] |=
180 			cpu_to_be32(MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH << MLX5_METER_MODE_SHIFT);
181 
182 	aso_data->bytewise_data[meter->idx * 8 + 2] = cpu_to_be32((cbs_exp << MLX5_CBS_EXP_SHIFT) |
183 								  (cbs_man << MLX5_CBS_MAN_SHIFT) |
184 								  (cir_exp << MLX5_CIR_EXP_SHIFT) |
185 								  cir_man);
186 
187 	mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl);
188 
189 	/* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */
190 	expires = jiffies + msecs_to_jiffies(10);
191 	do {
192 		err = mlx5_aso_poll_cq(aso, true);
193 		if (err)
194 			usleep_range(2, 10);
195 	} while (err && time_is_after_jiffies(expires));
196 	mutex_unlock(&flow_meters->aso_lock);
197 
198 	return err;
199 }
200 
201 static int
mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters * flow_meters,int * obj_id)202 mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_id)
203 {
204 	u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {};
205 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
206 	struct mlx5_core_dev *mdev = flow_meters->mdev;
207 	void *obj, *param;
208 	int err;
209 
210 	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
211 	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
212 		 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO);
213 	param = MLX5_ADDR_OF(general_obj_in_cmd_hdr, in, op_param);
214 	MLX5_SET(general_obj_create_param, param, log_obj_range,
215 		 flow_meters->log_granularity);
216 
217 	obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj);
218 	MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn);
219 
220 	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
221 	if (!err) {
222 		*obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
223 		mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) created\n", *obj_id);
224 	}
225 
226 	return err;
227 }
228 
229 static void
mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev * mdev,u32 obj_id)230 mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id)
231 {
232 	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
233 	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
234 
235 	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
236 	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
237 		 MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO);
238 	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id);
239 
240 	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
241 	mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) destroyed\n", obj_id);
242 }
243 
244 static struct mlx5e_flow_meter_handle *
__mlx5e_flow_meter_alloc(struct mlx5e_flow_meters * flow_meters,bool alloc_aso)245 __mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters, bool alloc_aso)
246 {
247 	struct mlx5_core_dev *mdev = flow_meters->mdev;
248 	struct mlx5e_flow_meter_aso_obj *meters_obj;
249 	struct mlx5e_flow_meter_handle *meter;
250 	struct mlx5_fc *counter;
251 	int err, pos, total;
252 	u32 id;
253 
254 	meter = kzalloc(sizeof(*meter), GFP_KERNEL);
255 	if (!meter)
256 		return ERR_PTR(-ENOMEM);
257 
258 	counter = mlx5_fc_create(mdev, true);
259 	if (IS_ERR(counter)) {
260 		err = PTR_ERR(counter);
261 		goto err_drop_counter;
262 	}
263 	meter->drop_counter = counter;
264 
265 	counter = mlx5_fc_create(mdev, true);
266 	if (IS_ERR(counter)) {
267 		err = PTR_ERR(counter);
268 		goto err_act_counter;
269 	}
270 	meter->act_counter = counter;
271 
272 	if (!alloc_aso)
273 		goto no_aso;
274 
275 	meters_obj = list_first_entry_or_null(&flow_meters->partial_list,
276 					      struct mlx5e_flow_meter_aso_obj,
277 					      entry);
278 	/* 2 meters in one object */
279 	total = 1 << (flow_meters->log_granularity + 1);
280 	if (!meters_obj) {
281 		err = mlx5e_flow_meter_create_aso_obj(flow_meters, &id);
282 		if (err) {
283 			mlx5_core_err(mdev, "Failed to create flow meter ASO object\n");
284 			goto err_create;
285 		}
286 
287 		meters_obj = kzalloc(sizeof(*meters_obj) + BITS_TO_BYTES(total),
288 				     GFP_KERNEL);
289 		if (!meters_obj) {
290 			err = -ENOMEM;
291 			goto err_mem;
292 		}
293 
294 		meters_obj->base_id = id;
295 		meters_obj->total_meters = total;
296 		list_add(&meters_obj->entry, &flow_meters->partial_list);
297 		pos = 0;
298 	} else {
299 		pos = find_first_zero_bit(meters_obj->meters_map, total);
300 		if (bitmap_weight(meters_obj->meters_map, total) == total - 1) {
301 			list_del(&meters_obj->entry);
302 			list_add(&meters_obj->entry, &flow_meters->full_list);
303 		}
304 	}
305 
306 	bitmap_set(meters_obj->meters_map, pos, 1);
307 	meter->meters_obj = meters_obj;
308 	meter->obj_id = meters_obj->base_id + pos / 2;
309 	meter->idx = pos % 2;
310 
311 no_aso:
312 	meter->flow_meters = flow_meters;
313 	mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n",
314 		      meter->obj_id, meter->idx);
315 
316 	return meter;
317 
318 err_mem:
319 	mlx5e_flow_meter_destroy_aso_obj(mdev, id);
320 err_create:
321 	mlx5_fc_destroy(mdev, meter->act_counter);
322 err_act_counter:
323 	mlx5_fc_destroy(mdev, meter->drop_counter);
324 err_drop_counter:
325 	kfree(meter);
326 	return ERR_PTR(err);
327 }
328 
329 static void
__mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle * meter)330 __mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter)
331 {
332 	struct mlx5e_flow_meters *flow_meters = meter->flow_meters;
333 	struct mlx5_core_dev *mdev = flow_meters->mdev;
334 	struct mlx5e_flow_meter_aso_obj *meters_obj;
335 	int n, pos;
336 
337 	mlx5_fc_destroy(mdev, meter->act_counter);
338 	mlx5_fc_destroy(mdev, meter->drop_counter);
339 
340 	if (meter->params.mtu)
341 		goto out_no_aso;
342 
343 	meters_obj = meter->meters_obj;
344 	pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx;
345 	bitmap_clear(meters_obj->meters_map, pos, 1);
346 	n = bitmap_weight(meters_obj->meters_map, meters_obj->total_meters);
347 	if (n == 0) {
348 		list_del(&meters_obj->entry);
349 		mlx5e_flow_meter_destroy_aso_obj(mdev, meters_obj->base_id);
350 		kfree(meters_obj);
351 	} else if (n == meters_obj->total_meters - 1) {
352 		list_del(&meters_obj->entry);
353 		list_add(&meters_obj->entry, &flow_meters->partial_list);
354 	}
355 
356 out_no_aso:
357 	mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n",
358 		      meter->obj_id, meter->idx);
359 	kfree(meter);
360 }
361 
362 static struct mlx5e_flow_meter_handle *
__mlx5e_tc_meter_get(struct mlx5e_flow_meters * flow_meters,u32 index)363 __mlx5e_tc_meter_get(struct mlx5e_flow_meters *flow_meters, u32 index)
364 {
365 	struct mlx5e_flow_meter_handle *meter;
366 
367 	hash_for_each_possible(flow_meters->hashtbl, meter, hlist, index)
368 		if (meter->params.index == index)
369 			goto add_ref;
370 
371 	return ERR_PTR(-ENOENT);
372 
373 add_ref:
374 	meter->refcnt++;
375 
376 	return meter;
377 }
378 
379 struct mlx5e_flow_meter_handle *
mlx5e_tc_meter_get(struct mlx5_core_dev * mdev,struct mlx5e_flow_meter_params * params)380 mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params)
381 {
382 	struct mlx5e_flow_meters *flow_meters;
383 	struct mlx5e_flow_meter_handle *meter;
384 
385 	flow_meters = mlx5e_get_flow_meters(mdev);
386 	if (!flow_meters)
387 		return ERR_PTR(-EOPNOTSUPP);
388 
389 	mutex_lock(&flow_meters->sync_lock);
390 	meter = __mlx5e_tc_meter_get(flow_meters, params->index);
391 	mutex_unlock(&flow_meters->sync_lock);
392 
393 	return meter;
394 }
395 
396 static void
__mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle * meter)397 __mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter)
398 {
399 	if (--meter->refcnt == 0) {
400 		hash_del(&meter->hlist);
401 		__mlx5e_flow_meter_free(meter);
402 	}
403 }
404 
405 void
mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle * meter)406 mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter)
407 {
408 	struct mlx5e_flow_meters *flow_meters = meter->flow_meters;
409 
410 	mutex_lock(&flow_meters->sync_lock);
411 	__mlx5e_tc_meter_put(meter);
412 	mutex_unlock(&flow_meters->sync_lock);
413 }
414 
415 static struct mlx5e_flow_meter_handle *
mlx5e_tc_meter_alloc(struct mlx5e_flow_meters * flow_meters,struct mlx5e_flow_meter_params * params)416 mlx5e_tc_meter_alloc(struct mlx5e_flow_meters *flow_meters,
417 		     struct mlx5e_flow_meter_params *params)
418 {
419 	struct mlx5e_flow_meter_handle *meter;
420 
421 	meter = __mlx5e_flow_meter_alloc(flow_meters, !params->mtu);
422 	if (IS_ERR(meter))
423 		return meter;
424 
425 	hash_add(flow_meters->hashtbl, &meter->hlist, params->index);
426 	meter->params.index = params->index;
427 	meter->params.mtu = params->mtu;
428 	meter->refcnt++;
429 
430 	return meter;
431 }
432 
433 static int
__mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle * meter,struct mlx5e_flow_meter_params * params)434 __mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter,
435 			struct mlx5e_flow_meter_params *params)
436 {
437 	struct mlx5_core_dev *mdev = meter->flow_meters->mdev;
438 	int err = 0;
439 
440 	if (meter->params.mode != params->mode || meter->params.rate != params->rate ||
441 	    meter->params.burst != params->burst) {
442 		err = mlx5e_tc_meter_modify(mdev, meter, params);
443 		if (err)
444 			goto out;
445 
446 		meter->params.mode = params->mode;
447 		meter->params.rate = params->rate;
448 		meter->params.burst = params->burst;
449 	}
450 
451 out:
452 	return err;
453 }
454 
455 int
mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle * meter,struct mlx5e_flow_meter_params * params)456 mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter,
457 		      struct mlx5e_flow_meter_params *params)
458 {
459 	struct mlx5_core_dev *mdev = meter->flow_meters->mdev;
460 	struct mlx5e_flow_meters *flow_meters;
461 	int err;
462 
463 	flow_meters = mlx5e_get_flow_meters(mdev);
464 	if (!flow_meters)
465 		return -EOPNOTSUPP;
466 
467 	mutex_lock(&flow_meters->sync_lock);
468 	err = __mlx5e_tc_meter_update(meter, params);
469 	mutex_unlock(&flow_meters->sync_lock);
470 	return err;
471 }
472 
473 struct mlx5e_flow_meter_handle *
mlx5e_tc_meter_replace(struct mlx5_core_dev * mdev,struct mlx5e_flow_meter_params * params)474 mlx5e_tc_meter_replace(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params)
475 {
476 	struct mlx5e_flow_meters *flow_meters;
477 	struct mlx5e_flow_meter_handle *meter;
478 	int err;
479 
480 	flow_meters = mlx5e_get_flow_meters(mdev);
481 	if (!flow_meters)
482 		return ERR_PTR(-EOPNOTSUPP);
483 
484 	mutex_lock(&flow_meters->sync_lock);
485 	meter = __mlx5e_tc_meter_get(flow_meters, params->index);
486 	if (IS_ERR(meter)) {
487 		meter = mlx5e_tc_meter_alloc(flow_meters, params);
488 		if (IS_ERR(meter)) {
489 			err = PTR_ERR(meter);
490 			goto err_get;
491 		}
492 	}
493 
494 	err = __mlx5e_tc_meter_update(meter, params);
495 	if (err)
496 		goto err_update;
497 
498 	mutex_unlock(&flow_meters->sync_lock);
499 	return meter;
500 
501 err_update:
502 	__mlx5e_tc_meter_put(meter);
503 err_get:
504 	mutex_unlock(&flow_meters->sync_lock);
505 	return ERR_PTR(err);
506 }
507 
508 enum mlx5_flow_namespace_type
mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters * flow_meters)509 mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters *flow_meters)
510 {
511 	return flow_meters->ns_type;
512 }
513 
514 struct mlx5e_flow_meters *
mlx5e_flow_meters_init(struct mlx5e_priv * priv,enum mlx5_flow_namespace_type ns_type,struct mlx5e_post_act * post_act)515 mlx5e_flow_meters_init(struct mlx5e_priv *priv,
516 		       enum mlx5_flow_namespace_type ns_type,
517 		       struct mlx5e_post_act *post_act)
518 {
519 	struct mlx5_core_dev *mdev = priv->mdev;
520 	struct mlx5e_flow_meters *flow_meters;
521 	int err;
522 
523 	if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
524 	      MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO))
525 		return ERR_PTR(-EOPNOTSUPP);
526 
527 	if (IS_ERR_OR_NULL(post_act)) {
528 		netdev_dbg(priv->netdev,
529 			   "flow meter offload is not supported, post action is missing\n");
530 		return ERR_PTR(-EOPNOTSUPP);
531 	}
532 
533 	flow_meters = kzalloc(sizeof(*flow_meters), GFP_KERNEL);
534 	if (!flow_meters)
535 		return ERR_PTR(-ENOMEM);
536 
537 	err = mlx5_core_alloc_pd(mdev, &flow_meters->pdn);
538 	if (err) {
539 		mlx5_core_err(mdev, "Failed to alloc pd for flow meter aso, err=%d\n", err);
540 		goto err_out;
541 	}
542 
543 	flow_meters->aso = mlx5_aso_create(mdev, flow_meters->pdn);
544 	if (IS_ERR(flow_meters->aso)) {
545 		mlx5_core_warn(mdev, "Failed to create aso wqe for flow meter\n");
546 		err = PTR_ERR(flow_meters->aso);
547 		goto err_sq;
548 	}
549 
550 	mutex_init(&flow_meters->sync_lock);
551 	INIT_LIST_HEAD(&flow_meters->partial_list);
552 	INIT_LIST_HEAD(&flow_meters->full_list);
553 
554 	flow_meters->ns_type = ns_type;
555 	flow_meters->mdev = mdev;
556 	flow_meters->post_act = post_act;
557 	mutex_init(&flow_meters->aso_lock);
558 	flow_meters->log_granularity = min_t(int, 6,
559 					     MLX5_CAP_QOS(mdev, log_meter_aso_max_alloc));
560 
561 	return flow_meters;
562 
563 err_sq:
564 	mlx5_core_dealloc_pd(mdev, flow_meters->pdn);
565 err_out:
566 	kfree(flow_meters);
567 	return ERR_PTR(err);
568 }
569 
570 void
mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters * flow_meters)571 mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters)
572 {
573 	if (IS_ERR_OR_NULL(flow_meters))
574 		return;
575 
576 	mlx5_aso_destroy(flow_meters->aso);
577 	mlx5_core_dealloc_pd(flow_meters->mdev, flow_meters->pdn);
578 	kfree(flow_meters);
579 }
580 
581 void
mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle * meter,u64 * bytes,u64 * packets,u64 * drops,u64 * lastuse)582 mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter,
583 			 u64 *bytes, u64 *packets, u64 *drops, u64 *lastuse)
584 {
585 	u64 bytes1, packets1, lastuse1;
586 	u64 bytes2, packets2, lastuse2;
587 
588 	mlx5_fc_query_cached(meter->act_counter, &bytes1, &packets1, &lastuse1);
589 	mlx5_fc_query_cached(meter->drop_counter, &bytes2, &packets2, &lastuse2);
590 
591 	*bytes = bytes1 + bytes2;
592 	*packets = packets1 + packets2;
593 	*drops = packets2;
594 	*lastuse = max_t(u64, lastuse1, lastuse2);
595 }
596