1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <net/netdev_lock.h>
4
5 #include "common.h"
6 #include "netlink.h"
7
8 struct rss_req_info {
9 struct ethnl_req_info base;
10 u32 rss_context;
11 };
12
13 struct rss_reply_data {
14 struct ethnl_reply_data base;
15 bool has_flow_hash;
16 bool no_key_fields;
17 u32 indir_size;
18 u32 hkey_size;
19 u32 hfunc;
20 u32 input_xfrm;
21 u32 *indir_table;
22 u8 *hkey;
23 int flow_hash[__ETHTOOL_A_FLOW_CNT];
24 };
25
26 static const u8 ethtool_rxfh_ft_nl2ioctl[] = {
27 [ETHTOOL_A_FLOW_ETHER] = ETHER_FLOW,
28 [ETHTOOL_A_FLOW_IP4] = IPV4_FLOW,
29 [ETHTOOL_A_FLOW_IP6] = IPV6_FLOW,
30 [ETHTOOL_A_FLOW_TCP4] = TCP_V4_FLOW,
31 [ETHTOOL_A_FLOW_UDP4] = UDP_V4_FLOW,
32 [ETHTOOL_A_FLOW_SCTP4] = SCTP_V4_FLOW,
33 [ETHTOOL_A_FLOW_AH_ESP4] = AH_ESP_V4_FLOW,
34 [ETHTOOL_A_FLOW_TCP6] = TCP_V6_FLOW,
35 [ETHTOOL_A_FLOW_UDP6] = UDP_V6_FLOW,
36 [ETHTOOL_A_FLOW_SCTP6] = SCTP_V6_FLOW,
37 [ETHTOOL_A_FLOW_AH_ESP6] = AH_ESP_V6_FLOW,
38 [ETHTOOL_A_FLOW_AH4] = AH_V4_FLOW,
39 [ETHTOOL_A_FLOW_ESP4] = ESP_V4_FLOW,
40 [ETHTOOL_A_FLOW_AH6] = AH_V6_FLOW,
41 [ETHTOOL_A_FLOW_ESP6] = ESP_V6_FLOW,
42 [ETHTOOL_A_FLOW_GTPU4] = GTPU_V4_FLOW,
43 [ETHTOOL_A_FLOW_GTPU6] = GTPU_V6_FLOW,
44 [ETHTOOL_A_FLOW_GTPC4] = GTPC_V4_FLOW,
45 [ETHTOOL_A_FLOW_GTPC6] = GTPC_V6_FLOW,
46 [ETHTOOL_A_FLOW_GTPC_TEID4] = GTPC_TEID_V4_FLOW,
47 [ETHTOOL_A_FLOW_GTPC_TEID6] = GTPC_TEID_V6_FLOW,
48 [ETHTOOL_A_FLOW_GTPU_EH4] = GTPU_EH_V4_FLOW,
49 [ETHTOOL_A_FLOW_GTPU_EH6] = GTPU_EH_V6_FLOW,
50 [ETHTOOL_A_FLOW_GTPU_UL4] = GTPU_UL_V4_FLOW,
51 [ETHTOOL_A_FLOW_GTPU_UL6] = GTPU_UL_V6_FLOW,
52 [ETHTOOL_A_FLOW_GTPU_DL4] = GTPU_DL_V4_FLOW,
53 [ETHTOOL_A_FLOW_GTPU_DL6] = GTPU_DL_V6_FLOW,
54 };
55
56 #define RSS_REQINFO(__req_base) \
57 container_of(__req_base, struct rss_req_info, base)
58
59 #define RSS_REPDATA(__reply_base) \
60 container_of(__reply_base, struct rss_reply_data, base)
61
62 const struct nla_policy ethnl_rss_get_policy[] = {
63 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
64 [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32 },
65 [ETHTOOL_A_RSS_START_CONTEXT] = { .type = NLA_U32 },
66 };
67
68 static int
rss_parse_request(struct ethnl_req_info * req_info,const struct genl_info * info,struct nlattr ** tb,struct netlink_ext_ack * extack)69 rss_parse_request(struct ethnl_req_info *req_info,
70 const struct genl_info *info,
71 struct nlattr **tb,
72 struct netlink_ext_ack *extack)
73 {
74 struct rss_req_info *request = RSS_REQINFO(req_info);
75
76 if (tb[ETHTOOL_A_RSS_CONTEXT])
77 request->rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]);
78 if (tb[ETHTOOL_A_RSS_START_CONTEXT]) {
79 NL_SET_BAD_ATTR(extack, tb[ETHTOOL_A_RSS_START_CONTEXT]);
80 return -EINVAL;
81 }
82
83 return 0;
84 }
85
86 static void
rss_prepare_flow_hash(const struct rss_req_info * req,struct net_device * dev,struct rss_reply_data * data,const struct genl_info * info)87 rss_prepare_flow_hash(const struct rss_req_info *req, struct net_device *dev,
88 struct rss_reply_data *data, const struct genl_info *info)
89 {
90 int i;
91
92 data->has_flow_hash = false;
93
94 if (!dev->ethtool_ops->get_rxfh_fields)
95 return;
96 if (req->rss_context && !dev->ethtool_ops->rxfh_per_ctx_fields)
97 return;
98
99 mutex_lock(&dev->ethtool->rss_lock);
100 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) {
101 struct ethtool_rxfh_fields fields = {
102 .flow_type = ethtool_rxfh_ft_nl2ioctl[i],
103 .rss_context = req->rss_context,
104 };
105
106 if (dev->ethtool_ops->get_rxfh_fields(dev, &fields)) {
107 data->flow_hash[i] = -1; /* Unsupported */
108 continue;
109 }
110
111 data->flow_hash[i] = fields.data;
112 data->has_flow_hash = true;
113 }
114 mutex_unlock(&dev->ethtool->rss_lock);
115 }
116
117 static int
rss_get_data_alloc(struct net_device * dev,struct rss_reply_data * data)118 rss_get_data_alloc(struct net_device *dev, struct rss_reply_data *data)
119 {
120 const struct ethtool_ops *ops = dev->ethtool_ops;
121 u32 total_size, indir_bytes;
122 u8 *rss_config;
123
124 data->indir_size = 0;
125 data->hkey_size = 0;
126 if (ops->get_rxfh_indir_size)
127 data->indir_size = ops->get_rxfh_indir_size(dev);
128 if (ops->get_rxfh_key_size)
129 data->hkey_size = ops->get_rxfh_key_size(dev);
130
131 indir_bytes = data->indir_size * sizeof(u32);
132 total_size = indir_bytes + data->hkey_size;
133 rss_config = kzalloc(total_size, GFP_KERNEL);
134 if (!rss_config)
135 return -ENOMEM;
136
137 data->indir_table = (u32 *)rss_config;
138 if (data->hkey_size)
139 data->hkey = rss_config + indir_bytes;
140
141 return 0;
142 }
143
rss_get_data_free(const struct rss_reply_data * data)144 static void rss_get_data_free(const struct rss_reply_data *data)
145 {
146 kfree(data->indir_table);
147 }
148
149 static int
rss_prepare_get(const struct rss_req_info * request,struct net_device * dev,struct rss_reply_data * data,const struct genl_info * info)150 rss_prepare_get(const struct rss_req_info *request, struct net_device *dev,
151 struct rss_reply_data *data, const struct genl_info *info)
152 {
153 const struct ethtool_ops *ops = dev->ethtool_ops;
154 struct ethtool_rxfh_param rxfh = {};
155 int ret;
156
157 ret = ethnl_ops_begin(dev);
158 if (ret < 0)
159 return ret;
160 mutex_lock(&dev->ethtool->rss_lock);
161
162 ret = rss_get_data_alloc(dev, data);
163 if (ret)
164 goto out_unlock;
165
166 rxfh.indir_size = data->indir_size;
167 rxfh.indir = data->indir_table;
168 rxfh.key_size = data->hkey_size;
169 rxfh.key = data->hkey;
170
171 ret = ops->get_rxfh(dev, &rxfh);
172 if (ret) {
173 rss_get_data_free(data);
174 goto out_unlock;
175 }
176
177 data->hfunc = rxfh.hfunc;
178 data->input_xfrm = rxfh.input_xfrm;
179 out_unlock:
180 mutex_unlock(&dev->ethtool->rss_lock);
181 ethnl_ops_complete(dev);
182 return ret;
183 }
184
185 static void
__rss_prepare_ctx(struct net_device * dev,struct rss_reply_data * data,struct ethtool_rxfh_context * ctx)186 __rss_prepare_ctx(struct net_device *dev, struct rss_reply_data *data,
187 struct ethtool_rxfh_context *ctx)
188 {
189 if (WARN_ON_ONCE(data->indir_size != ctx->indir_size ||
190 data->hkey_size != ctx->key_size))
191 return;
192
193 data->no_key_fields = !dev->ethtool_ops->rxfh_per_ctx_key;
194
195 data->hfunc = ctx->hfunc;
196 data->input_xfrm = ctx->input_xfrm;
197 memcpy(data->indir_table, ethtool_rxfh_context_indir(ctx),
198 data->indir_size * sizeof(u32));
199 if (data->hkey_size)
200 memcpy(data->hkey, ethtool_rxfh_context_key(ctx),
201 data->hkey_size);
202 }
203
204 static int
rss_prepare_ctx(const struct rss_req_info * request,struct net_device * dev,struct rss_reply_data * data,const struct genl_info * info)205 rss_prepare_ctx(const struct rss_req_info *request, struct net_device *dev,
206 struct rss_reply_data *data, const struct genl_info *info)
207 {
208 struct ethtool_rxfh_context *ctx;
209 u32 total_size, indir_bytes;
210 u8 *rss_config;
211 int ret;
212
213 mutex_lock(&dev->ethtool->rss_lock);
214 ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context);
215 if (!ctx) {
216 ret = -ENOENT;
217 goto out_unlock;
218 }
219
220 data->indir_size = ctx->indir_size;
221 data->hkey_size = ctx->key_size;
222
223 indir_bytes = data->indir_size * sizeof(u32);
224 total_size = indir_bytes + data->hkey_size;
225 rss_config = kzalloc(total_size, GFP_KERNEL);
226 if (!rss_config) {
227 ret = -ENOMEM;
228 goto out_unlock;
229 }
230
231 data->indir_table = (u32 *)rss_config;
232 if (data->hkey_size)
233 data->hkey = rss_config + indir_bytes;
234
235 __rss_prepare_ctx(dev, data, ctx);
236
237 ret = 0;
238 out_unlock:
239 mutex_unlock(&dev->ethtool->rss_lock);
240 return ret;
241 }
242
243 static int
rss_prepare(const struct rss_req_info * request,struct net_device * dev,struct rss_reply_data * data,const struct genl_info * info)244 rss_prepare(const struct rss_req_info *request, struct net_device *dev,
245 struct rss_reply_data *data, const struct genl_info *info)
246 {
247 rss_prepare_flow_hash(request, dev, data, info);
248
249 /* Coming from RSS_SET, driver may only have flow_hash_fields ops */
250 if (!dev->ethtool_ops->get_rxfh)
251 return 0;
252
253 if (request->rss_context)
254 return rss_prepare_ctx(request, dev, data, info);
255 return rss_prepare_get(request, dev, data, info);
256 }
257
258 static int
rss_prepare_data(const struct ethnl_req_info * req_base,struct ethnl_reply_data * reply_base,const struct genl_info * info)259 rss_prepare_data(const struct ethnl_req_info *req_base,
260 struct ethnl_reply_data *reply_base,
261 const struct genl_info *info)
262 {
263 struct rss_reply_data *data = RSS_REPDATA(reply_base);
264 struct rss_req_info *request = RSS_REQINFO(req_base);
265 struct net_device *dev = reply_base->dev;
266 const struct ethtool_ops *ops;
267
268 ops = dev->ethtool_ops;
269 if (!ops->get_rxfh)
270 return -EOPNOTSUPP;
271
272 /* Some drivers don't handle rss_context */
273 if (request->rss_context && !ops->create_rxfh_context)
274 return -EOPNOTSUPP;
275
276 return rss_prepare(request, dev, data, info);
277 }
278
279 static int
rss_reply_size(const struct ethnl_req_info * req_base,const struct ethnl_reply_data * reply_base)280 rss_reply_size(const struct ethnl_req_info *req_base,
281 const struct ethnl_reply_data *reply_base)
282 {
283 const struct rss_reply_data *data = RSS_REPDATA(reply_base);
284 int len;
285
286 len = nla_total_size(sizeof(u32)) + /* _RSS_CONTEXT */
287 nla_total_size(sizeof(u32)) + /* _RSS_HFUNC */
288 nla_total_size(sizeof(u32)) + /* _RSS_INPUT_XFRM */
289 nla_total_size(sizeof(u32) * data->indir_size) + /* _RSS_INDIR */
290 nla_total_size(data->hkey_size) + /* _RSS_HKEY */
291 nla_total_size(0) + /* _RSS_FLOW_HASH */
292 nla_total_size(sizeof(u32)) * ETHTOOL_A_FLOW_MAX +
293 0;
294
295 return len;
296 }
297
298 static int
rss_fill_reply(struct sk_buff * skb,const struct ethnl_req_info * req_base,const struct ethnl_reply_data * reply_base)299 rss_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base,
300 const struct ethnl_reply_data *reply_base)
301 {
302 const struct rss_reply_data *data = RSS_REPDATA(reply_base);
303 struct rss_req_info *request = RSS_REQINFO(req_base);
304
305 if (request->rss_context &&
306 nla_put_u32(skb, ETHTOOL_A_RSS_CONTEXT, request->rss_context))
307 return -EMSGSIZE;
308
309 if ((data->indir_size &&
310 nla_put(skb, ETHTOOL_A_RSS_INDIR,
311 sizeof(u32) * data->indir_size, data->indir_table)))
312 return -EMSGSIZE;
313
314 if (!data->no_key_fields &&
315 ((data->hfunc &&
316 nla_put_u32(skb, ETHTOOL_A_RSS_HFUNC, data->hfunc)) ||
317 (data->input_xfrm &&
318 nla_put_u32(skb, ETHTOOL_A_RSS_INPUT_XFRM, data->input_xfrm)) ||
319 (data->hkey_size &&
320 nla_put(skb, ETHTOOL_A_RSS_HKEY, data->hkey_size, data->hkey))))
321 return -EMSGSIZE;
322
323 if (data->has_flow_hash) {
324 struct nlattr *nest;
325 int i;
326
327 nest = nla_nest_start(skb, ETHTOOL_A_RSS_FLOW_HASH);
328 if (!nest)
329 return -EMSGSIZE;
330
331 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) {
332 if (data->flow_hash[i] >= 0 &&
333 nla_put_uint(skb, i, data->flow_hash[i])) {
334 nla_nest_cancel(skb, nest);
335 return -EMSGSIZE;
336 }
337 }
338
339 nla_nest_end(skb, nest);
340 }
341
342 return 0;
343 }
344
rss_cleanup_data(struct ethnl_reply_data * reply_base)345 static void rss_cleanup_data(struct ethnl_reply_data *reply_base)
346 {
347 const struct rss_reply_data *data = RSS_REPDATA(reply_base);
348
349 rss_get_data_free(data);
350 }
351
352 struct rss_nl_dump_ctx {
353 unsigned long ifindex;
354 unsigned long ctx_idx;
355
356 /* User wants to only dump contexts from given ifindex */
357 unsigned int match_ifindex;
358 unsigned int start_ctx;
359 };
360
rss_dump_ctx(struct netlink_callback * cb)361 static struct rss_nl_dump_ctx *rss_dump_ctx(struct netlink_callback *cb)
362 {
363 NL_ASSERT_CTX_FITS(struct rss_nl_dump_ctx);
364
365 return (struct rss_nl_dump_ctx *)cb->ctx;
366 }
367
ethnl_rss_dump_start(struct netlink_callback * cb)368 int ethnl_rss_dump_start(struct netlink_callback *cb)
369 {
370 const struct genl_info *info = genl_info_dump(cb);
371 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb);
372 struct ethnl_req_info req_info = {};
373 struct nlattr **tb = info->attrs;
374 int ret;
375
376 /* Filtering by context not supported */
377 if (tb[ETHTOOL_A_RSS_CONTEXT]) {
378 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT]);
379 return -EINVAL;
380 }
381 if (tb[ETHTOOL_A_RSS_START_CONTEXT]) {
382 ctx->start_ctx = nla_get_u32(tb[ETHTOOL_A_RSS_START_CONTEXT]);
383 ctx->ctx_idx = ctx->start_ctx;
384 }
385
386 ret = ethnl_parse_header_dev_get(&req_info,
387 tb[ETHTOOL_A_RSS_HEADER],
388 sock_net(cb->skb->sk), cb->extack,
389 false);
390 if (req_info.dev) {
391 ctx->match_ifindex = req_info.dev->ifindex;
392 ctx->ifindex = ctx->match_ifindex;
393 ethnl_parse_header_dev_put(&req_info);
394 req_info.dev = NULL;
395 }
396
397 return ret;
398 }
399
400 static int
rss_dump_one_ctx(struct sk_buff * skb,struct netlink_callback * cb,struct net_device * dev,u32 rss_context)401 rss_dump_one_ctx(struct sk_buff *skb, struct netlink_callback *cb,
402 struct net_device *dev, u32 rss_context)
403 {
404 const struct genl_info *info = genl_info_dump(cb);
405 struct rss_reply_data data = {};
406 struct rss_req_info req = {};
407 void *ehdr;
408 int ret;
409
410 req.rss_context = rss_context;
411
412 ehdr = ethnl_dump_put(skb, cb, ETHTOOL_MSG_RSS_GET_REPLY);
413 if (!ehdr)
414 return -EMSGSIZE;
415
416 ret = ethnl_fill_reply_header(skb, dev, ETHTOOL_A_RSS_HEADER);
417 if (ret < 0)
418 goto err_cancel;
419
420 ret = rss_prepare(&req, dev, &data, info);
421 if (ret)
422 goto err_cancel;
423
424 ret = rss_fill_reply(skb, &req.base, &data.base);
425 if (ret)
426 goto err_cleanup;
427 genlmsg_end(skb, ehdr);
428
429 rss_cleanup_data(&data.base);
430 return 0;
431
432 err_cleanup:
433 rss_cleanup_data(&data.base);
434 err_cancel:
435 genlmsg_cancel(skb, ehdr);
436 return ret;
437 }
438
439 static int
rss_dump_one_dev(struct sk_buff * skb,struct netlink_callback * cb,struct net_device * dev)440 rss_dump_one_dev(struct sk_buff *skb, struct netlink_callback *cb,
441 struct net_device *dev)
442 {
443 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb);
444 int ret;
445
446 if (!dev->ethtool_ops->get_rxfh)
447 return 0;
448
449 if (!ctx->ctx_idx) {
450 ret = rss_dump_one_ctx(skb, cb, dev, 0);
451 if (ret)
452 return ret;
453 ctx->ctx_idx++;
454 }
455
456 for (; xa_find(&dev->ethtool->rss_ctx, &ctx->ctx_idx,
457 ULONG_MAX, XA_PRESENT); ctx->ctx_idx++) {
458 ret = rss_dump_one_ctx(skb, cb, dev, ctx->ctx_idx);
459 if (ret)
460 return ret;
461 }
462 ctx->ctx_idx = ctx->start_ctx;
463
464 return 0;
465 }
466
ethnl_rss_dumpit(struct sk_buff * skb,struct netlink_callback * cb)467 int ethnl_rss_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
468 {
469 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb);
470 struct net *net = sock_net(skb->sk);
471 struct net_device *dev;
472 int ret = 0;
473
474 rtnl_lock();
475 for_each_netdev_dump(net, dev, ctx->ifindex) {
476 if (ctx->match_ifindex && ctx->match_ifindex != ctx->ifindex)
477 break;
478
479 netdev_lock_ops(dev);
480 ret = rss_dump_one_dev(skb, cb, dev);
481 netdev_unlock_ops(dev);
482 if (ret)
483 break;
484 }
485 rtnl_unlock();
486
487 return ret;
488 }
489
490 /* RSS_NTF */
491
ethnl_rss_delete_notify(struct net_device * dev,u32 rss_context)492 static void ethnl_rss_delete_notify(struct net_device *dev, u32 rss_context)
493 {
494 struct sk_buff *ntf;
495 size_t ntf_size;
496 void *hdr;
497
498 ntf_size = ethnl_reply_header_size() +
499 nla_total_size(sizeof(u32)); /* _RSS_CONTEXT */
500
501 ntf = genlmsg_new(ntf_size, GFP_KERNEL);
502 if (!ntf)
503 goto out_warn;
504
505 hdr = ethnl_bcastmsg_put(ntf, ETHTOOL_MSG_RSS_DELETE_NTF);
506 if (!hdr)
507 goto out_free_ntf;
508
509 if (ethnl_fill_reply_header(ntf, dev, ETHTOOL_A_RSS_HEADER) ||
510 nla_put_u32(ntf, ETHTOOL_A_RSS_CONTEXT, rss_context))
511 goto out_free_ntf;
512
513 genlmsg_end(ntf, hdr);
514 if (ethnl_multicast(ntf, dev))
515 goto out_warn;
516
517 return;
518
519 out_free_ntf:
520 nlmsg_free(ntf);
521 out_warn:
522 pr_warn_once("Failed to send a RSS delete notification");
523 }
524
ethtool_rss_notify(struct net_device * dev,u32 type,u32 rss_context)525 void ethtool_rss_notify(struct net_device *dev, u32 type, u32 rss_context)
526 {
527 struct rss_req_info req_info = {
528 .rss_context = rss_context,
529 };
530
531 if (type == ETHTOOL_MSG_RSS_DELETE_NTF)
532 ethnl_rss_delete_notify(dev, rss_context);
533 else
534 ethnl_notify(dev, type, &req_info.base);
535 }
536
537 /* RSS_SET */
538
539 #define RFH_MASK (RXH_L2DA | RXH_VLAN | RXH_IP_SRC | RXH_IP_DST | \
540 RXH_L3_PROTO | RXH_L4_B_0_1 | RXH_L4_B_2_3 | \
541 RXH_GTP_TEID | RXH_DISCARD)
542 #define RFH_MASKv6 (RFH_MASK | RXH_IP6_FL)
543
544 static const struct nla_policy ethnl_rss_flows_policy[] = {
545 [ETHTOOL_A_FLOW_ETHER] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
546 [ETHTOOL_A_FLOW_IP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
547 [ETHTOOL_A_FLOW_IP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
548 [ETHTOOL_A_FLOW_TCP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
549 [ETHTOOL_A_FLOW_UDP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
550 [ETHTOOL_A_FLOW_SCTP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
551 [ETHTOOL_A_FLOW_AH_ESP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
552 [ETHTOOL_A_FLOW_TCP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
553 [ETHTOOL_A_FLOW_UDP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
554 [ETHTOOL_A_FLOW_SCTP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
555 [ETHTOOL_A_FLOW_AH_ESP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
556 [ETHTOOL_A_FLOW_AH4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
557 [ETHTOOL_A_FLOW_ESP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
558 [ETHTOOL_A_FLOW_AH6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
559 [ETHTOOL_A_FLOW_ESP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
560 [ETHTOOL_A_FLOW_GTPU4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
561 [ETHTOOL_A_FLOW_GTPU6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
562 [ETHTOOL_A_FLOW_GTPC4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
563 [ETHTOOL_A_FLOW_GTPC6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
564 [ETHTOOL_A_FLOW_GTPC_TEID4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
565 [ETHTOOL_A_FLOW_GTPC_TEID6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
566 [ETHTOOL_A_FLOW_GTPU_EH4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
567 [ETHTOOL_A_FLOW_GTPU_EH6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
568 [ETHTOOL_A_FLOW_GTPU_UL4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
569 [ETHTOOL_A_FLOW_GTPU_UL6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
570 [ETHTOOL_A_FLOW_GTPU_DL4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK),
571 [ETHTOOL_A_FLOW_GTPU_DL6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASKv6),
572 };
573
574 const struct nla_policy ethnl_rss_set_policy[ETHTOOL_A_RSS_FLOW_HASH + 1] = {
575 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
576 [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32, },
577 [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1),
578 [ETHTOOL_A_RSS_INDIR] = { .type = NLA_BINARY, },
579 [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1),
580 [ETHTOOL_A_RSS_INPUT_XFRM] =
581 NLA_POLICY_MAX(NLA_U32, RXH_XFRM_SYM_OR_XOR),
582 [ETHTOOL_A_RSS_FLOW_HASH] = NLA_POLICY_NESTED(ethnl_rss_flows_policy),
583 };
584
585 static int
ethnl_rss_set_validate(struct ethnl_req_info * req_info,struct genl_info * info)586 ethnl_rss_set_validate(struct ethnl_req_info *req_info, struct genl_info *info)
587 {
588 const struct ethtool_ops *ops = req_info->dev->ethtool_ops;
589 struct rss_req_info *request = RSS_REQINFO(req_info);
590 struct nlattr **tb = info->attrs;
591 struct nlattr *bad_attr = NULL;
592 u32 input_xfrm;
593
594 if (request->rss_context && !ops->create_rxfh_context)
595 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_CONTEXT];
596
597 if (request->rss_context && !ops->rxfh_per_ctx_key) {
598 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HFUNC];
599 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HKEY];
600 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
601 }
602
603 input_xfrm = nla_get_u32_default(tb[ETHTOOL_A_RSS_INPUT_XFRM], 0);
604 if (input_xfrm & ~ops->supported_input_xfrm)
605 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
606
607 if (tb[ETHTOOL_A_RSS_FLOW_HASH] && !ops->set_rxfh_fields)
608 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_FLOW_HASH];
609 if (request->rss_context &&
610 tb[ETHTOOL_A_RSS_FLOW_HASH] && !ops->rxfh_per_ctx_fields)
611 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_FLOW_HASH];
612
613 if (bad_attr) {
614 NL_SET_BAD_ATTR(info->extack, bad_attr);
615 return -EOPNOTSUPP;
616 }
617
618 return 1;
619 }
620
621 static int
rss_set_prep_indir(struct net_device * dev,struct genl_info * info,struct rss_reply_data * data,struct ethtool_rxfh_param * rxfh,bool * reset,bool * mod)622 rss_set_prep_indir(struct net_device *dev, struct genl_info *info,
623 struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh,
624 bool *reset, bool *mod)
625 {
626 struct netlink_ext_ack *extack = info->extack;
627 struct nlattr **tb = info->attrs;
628 size_t alloc_size;
629 int num_rx_rings;
630 u32 user_size;
631 int i, err;
632
633 if (!tb[ETHTOOL_A_RSS_INDIR])
634 return 0;
635 if (!data->indir_size)
636 return -EOPNOTSUPP;
637
638 err = ethtool_get_rx_ring_count(dev);
639 if (err < 0)
640 return err;
641 num_rx_rings = err;
642
643 if (nla_len(tb[ETHTOOL_A_RSS_INDIR]) % 4) {
644 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_INDIR]);
645 return -EINVAL;
646 }
647 user_size = nla_len(tb[ETHTOOL_A_RSS_INDIR]) / 4;
648 if (!user_size) {
649 if (rxfh->rss_context) {
650 NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_RSS_INDIR],
651 "can't reset table for a context");
652 return -EINVAL;
653 }
654 *reset = true;
655 } else if (data->indir_size % user_size) {
656 NL_SET_ERR_MSG_ATTR_FMT(extack, tb[ETHTOOL_A_RSS_INDIR],
657 "size (%d) mismatch with device indir table (%d)",
658 user_size, data->indir_size);
659 return -EINVAL;
660 }
661
662 rxfh->indir_size = data->indir_size;
663 alloc_size = array_size(data->indir_size, sizeof(rxfh->indir[0]));
664 rxfh->indir = kzalloc(alloc_size, GFP_KERNEL);
665 if (!rxfh->indir)
666 return -ENOMEM;
667
668 nla_memcpy(rxfh->indir, tb[ETHTOOL_A_RSS_INDIR], alloc_size);
669 for (i = 0; i < user_size; i++) {
670 if (rxfh->indir[i] < num_rx_rings)
671 continue;
672
673 NL_SET_ERR_MSG_ATTR_FMT(extack, tb[ETHTOOL_A_RSS_INDIR],
674 "entry %d: queue out of range (%d)",
675 i, rxfh->indir[i]);
676 err = -EINVAL;
677 goto err_free;
678 }
679
680 if (user_size) {
681 /* Replicate the user-provided table to fill the device table */
682 for (i = user_size; i < data->indir_size; i++)
683 rxfh->indir[i] = rxfh->indir[i % user_size];
684 } else {
685 for (i = 0; i < data->indir_size; i++)
686 rxfh->indir[i] =
687 ethtool_rxfh_indir_default(i, num_rx_rings);
688 }
689
690 *mod |= memcmp(rxfh->indir, data->indir_table, alloc_size);
691
692 return user_size;
693
694 err_free:
695 kfree(rxfh->indir);
696 rxfh->indir = NULL;
697 return err;
698 }
699
700 static int
rss_set_prep_hkey(struct net_device * dev,struct genl_info * info,struct rss_reply_data * data,struct ethtool_rxfh_param * rxfh,bool * mod)701 rss_set_prep_hkey(struct net_device *dev, struct genl_info *info,
702 struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh,
703 bool *mod)
704 {
705 struct nlattr **tb = info->attrs;
706
707 if (!tb[ETHTOOL_A_RSS_HKEY])
708 return 0;
709
710 if (nla_len(tb[ETHTOOL_A_RSS_HKEY]) != data->hkey_size) {
711 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_HKEY]);
712 return -EINVAL;
713 }
714
715 rxfh->key_size = data->hkey_size;
716 rxfh->key = kmemdup(data->hkey, data->hkey_size, GFP_KERNEL);
717 if (!rxfh->key)
718 return -ENOMEM;
719
720 ethnl_update_binary(rxfh->key, rxfh->key_size, tb[ETHTOOL_A_RSS_HKEY],
721 mod);
722 return 0;
723 }
724
725 static int
rss_check_rxfh_fields_sym(struct net_device * dev,struct genl_info * info,struct rss_reply_data * data,bool xfrm_sym)726 rss_check_rxfh_fields_sym(struct net_device *dev, struct genl_info *info,
727 struct rss_reply_data *data, bool xfrm_sym)
728 {
729 struct nlattr **tb = info->attrs;
730 int i;
731
732 if (!xfrm_sym)
733 return 0;
734 if (!data->has_flow_hash) {
735 NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_RSS_INPUT_XFRM],
736 "hash field config not reported");
737 return -EINVAL;
738 }
739
740 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++)
741 if (data->flow_hash[i] >= 0 &&
742 !ethtool_rxfh_config_is_sym(data->flow_hash[i])) {
743 NL_SET_ERR_MSG_ATTR(info->extack,
744 tb[ETHTOOL_A_RSS_INPUT_XFRM],
745 "hash field config is not symmetric");
746 return -EINVAL;
747 }
748
749 return 0;
750 }
751
752 static int
ethnl_set_rss_fields(struct net_device * dev,struct genl_info * info,u32 rss_context,struct rss_reply_data * data,bool xfrm_sym,bool * mod)753 ethnl_set_rss_fields(struct net_device *dev, struct genl_info *info,
754 u32 rss_context, struct rss_reply_data *data,
755 bool xfrm_sym, bool *mod)
756 {
757 struct nlattr *flow_nest = info->attrs[ETHTOOL_A_RSS_FLOW_HASH];
758 struct nlattr *flows[ETHTOOL_A_FLOW_MAX + 1];
759 const struct ethtool_ops *ops;
760 int i, ret;
761
762 ops = dev->ethtool_ops;
763
764 ret = rss_check_rxfh_fields_sym(dev, info, data, xfrm_sym);
765 if (ret)
766 return ret;
767
768 if (!flow_nest)
769 return 0;
770
771 ret = nla_parse_nested(flows, ARRAY_SIZE(ethnl_rss_flows_policy) - 1,
772 flow_nest, ethnl_rss_flows_policy, info->extack);
773 if (ret < 0)
774 return ret;
775
776 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) {
777 struct ethtool_rxfh_fields fields = {
778 .flow_type = ethtool_rxfh_ft_nl2ioctl[i],
779 .rss_context = rss_context,
780 };
781
782 if (!flows[i])
783 continue;
784
785 fields.data = nla_get_u32(flows[i]);
786 if (data->has_flow_hash && data->flow_hash[i] == fields.data)
787 continue;
788
789 if (xfrm_sym && !ethtool_rxfh_config_is_sym(fields.data)) {
790 NL_SET_ERR_MSG_ATTR(info->extack, flows[i],
791 "conflict with xfrm-input");
792 return -EINVAL;
793 }
794
795 ret = ops->set_rxfh_fields(dev, &fields, info->extack);
796 if (ret)
797 return ret;
798
799 *mod = true;
800 }
801
802 return 0;
803 }
804
805 static void
rss_set_ctx_update(struct ethtool_rxfh_context * ctx,struct nlattr ** tb,struct rss_reply_data * data,struct ethtool_rxfh_param * rxfh)806 rss_set_ctx_update(struct ethtool_rxfh_context *ctx, struct nlattr **tb,
807 struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh)
808 {
809 int i;
810
811 if (rxfh->indir) {
812 for (i = 0; i < data->indir_size; i++)
813 ethtool_rxfh_context_indir(ctx)[i] = rxfh->indir[i];
814 ctx->indir_configured = !!nla_len(tb[ETHTOOL_A_RSS_INDIR]);
815 }
816 if (rxfh->key) {
817 memcpy(ethtool_rxfh_context_key(ctx), rxfh->key,
818 data->hkey_size);
819 ctx->key_configured = !!rxfh->key_size;
820 }
821 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE)
822 ctx->hfunc = rxfh->hfunc;
823 if (rxfh->input_xfrm != RXH_XFRM_NO_CHANGE)
824 ctx->input_xfrm = rxfh->input_xfrm;
825 }
826
827 static int
ethnl_rss_set(struct ethnl_req_info * req_info,struct genl_info * info)828 ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info)
829 {
830 struct rss_req_info *request = RSS_REQINFO(req_info);
831 bool indir_reset = false, indir_mod, xfrm_sym;
832 struct ethtool_rxfh_context *ctx = NULL;
833 struct net_device *dev = req_info->dev;
834 bool mod = false, fields_mod = false;
835 struct ethtool_rxfh_param rxfh = {};
836 struct nlattr **tb = info->attrs;
837 struct rss_reply_data data = {};
838 const struct ethtool_ops *ops;
839 u32 indir_user_size;
840 int ret;
841
842 ops = dev->ethtool_ops;
843 data.base.dev = dev;
844
845 ret = rss_prepare(request, dev, &data, info);
846 if (ret)
847 return ret;
848
849 rxfh.rss_context = request->rss_context;
850
851 ret = rss_set_prep_indir(dev, info, &data, &rxfh, &indir_reset, &mod);
852 if (ret < 0)
853 goto exit_clean_data;
854 indir_user_size = ret;
855 indir_mod = !!tb[ETHTOOL_A_RSS_INDIR];
856
857 rxfh.hfunc = data.hfunc;
858 ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod);
859 if (rxfh.hfunc == data.hfunc)
860 rxfh.hfunc = ETH_RSS_HASH_NO_CHANGE;
861
862 ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod);
863 if (ret)
864 goto exit_free_indir;
865
866 rxfh.input_xfrm = data.input_xfrm;
867 ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod);
868 xfrm_sym = rxfh.input_xfrm || data.input_xfrm;
869 if (rxfh.input_xfrm == data.input_xfrm)
870 rxfh.input_xfrm = RXH_XFRM_NO_CHANGE;
871
872 mutex_lock(&dev->ethtool->rss_lock);
873 if (request->rss_context) {
874 ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context);
875 if (!ctx) {
876 ret = -ENOENT;
877 goto exit_unlock;
878 }
879 }
880
881 ret = ethnl_set_rss_fields(dev, info, request->rss_context,
882 &data, xfrm_sym, &fields_mod);
883 if (ret)
884 goto exit_unlock;
885
886 if (!mod)
887 ret = 0; /* nothing to tell the driver */
888 else if (!ops->set_rxfh)
889 ret = -EOPNOTSUPP;
890 else if (!rxfh.rss_context)
891 ret = ops->set_rxfh(dev, &rxfh, info->extack);
892 else
893 ret = ops->modify_rxfh_context(dev, ctx, &rxfh, info->extack);
894 if (ret)
895 goto exit_unlock;
896
897 if (ctx) {
898 rss_set_ctx_update(ctx, tb, &data, &rxfh);
899 if (indir_user_size)
900 ctx->indir_user_size = indir_user_size;
901 } else if (indir_reset) {
902 dev->ethtool->rss_indir_user_size = 0;
903 } else if (indir_mod) {
904 dev->ethtool->rss_indir_user_size = indir_user_size;
905 }
906
907 exit_unlock:
908 mutex_unlock(&dev->ethtool->rss_lock);
909 kfree(rxfh.key);
910 exit_free_indir:
911 kfree(rxfh.indir);
912 exit_clean_data:
913 rss_cleanup_data(&data.base);
914
915 return ret ?: mod || fields_mod;
916 }
917
918 const struct ethnl_request_ops ethnl_rss_request_ops = {
919 .request_cmd = ETHTOOL_MSG_RSS_GET,
920 .reply_cmd = ETHTOOL_MSG_RSS_GET_REPLY,
921 .hdr_attr = ETHTOOL_A_RSS_HEADER,
922 .req_info_size = sizeof(struct rss_req_info),
923 .reply_data_size = sizeof(struct rss_reply_data),
924
925 .parse_request = rss_parse_request,
926 .prepare_data = rss_prepare_data,
927 .reply_size = rss_reply_size,
928 .fill_reply = rss_fill_reply,
929 .cleanup_data = rss_cleanup_data,
930
931 .set_validate = ethnl_rss_set_validate,
932 .set = ethnl_rss_set,
933 .set_ntf_cmd = ETHTOOL_MSG_RSS_NTF,
934 };
935
936 /* RSS_CREATE */
937
938 const struct nla_policy ethnl_rss_create_policy[ETHTOOL_A_RSS_INPUT_XFRM + 1] = {
939 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
940 [ETHTOOL_A_RSS_CONTEXT] = NLA_POLICY_MIN(NLA_U32, 1),
941 [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1),
942 [ETHTOOL_A_RSS_INDIR] = NLA_POLICY_MIN(NLA_BINARY, 1),
943 [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1),
944 [ETHTOOL_A_RSS_INPUT_XFRM] =
945 NLA_POLICY_MAX(NLA_U32, RXH_XFRM_SYM_OR_XOR),
946 };
947
948 static int
ethnl_rss_create_validate(struct net_device * dev,struct genl_info * info)949 ethnl_rss_create_validate(struct net_device *dev, struct genl_info *info)
950 {
951 const struct ethtool_ops *ops = dev->ethtool_ops;
952 struct nlattr **tb = info->attrs;
953 struct nlattr *bad_attr = NULL;
954 u32 rss_context, input_xfrm;
955
956 if (!ops->create_rxfh_context)
957 return -EOPNOTSUPP;
958
959 rss_context = nla_get_u32_default(tb[ETHTOOL_A_RSS_CONTEXT], 0);
960 if (ops->rxfh_max_num_contexts &&
961 ops->rxfh_max_num_contexts <= rss_context) {
962 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT]);
963 return -ERANGE;
964 }
965
966 if (!ops->rxfh_per_ctx_key) {
967 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HFUNC];
968 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HKEY];
969 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
970 }
971
972 input_xfrm = nla_get_u32_default(tb[ETHTOOL_A_RSS_INPUT_XFRM], 0);
973 if (input_xfrm & ~ops->supported_input_xfrm)
974 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM];
975
976 if (bad_attr) {
977 NL_SET_BAD_ATTR(info->extack, bad_attr);
978 return -EOPNOTSUPP;
979 }
980
981 return 0;
982 }
983
984 static void
ethnl_rss_create_send_ntf(const struct sk_buff * rsp,struct net_device * dev)985 ethnl_rss_create_send_ntf(const struct sk_buff *rsp, struct net_device *dev)
986 {
987 struct genlmsghdr *genl_hdr;
988 struct nlmsghdr *nlh;
989 struct sk_buff *ntf;
990
991 ntf = skb_copy_expand(rsp, 0, 0, GFP_KERNEL);
992 if (!ntf)
993 return;
994
995 nlh = nlmsg_hdr(ntf);
996 /* Convert the reply into a notification */
997 nlh->nlmsg_pid = 0;
998 nlh->nlmsg_seq = ethnl_bcast_seq_next();
999
1000 genl_hdr = nlmsg_data(nlh);
1001 genl_hdr->cmd = ETHTOOL_MSG_RSS_CREATE_NTF;
1002
1003 ethnl_multicast(ntf, dev);
1004 }
1005
ethnl_rss_create_doit(struct sk_buff * skb,struct genl_info * info)1006 int ethnl_rss_create_doit(struct sk_buff *skb, struct genl_info *info)
1007 {
1008 bool indir_dflt = false, mod = false, ntf_fail = false;
1009 struct ethtool_rxfh_param rxfh = {};
1010 struct ethtool_rxfh_context *ctx;
1011 struct nlattr **tb = info->attrs;
1012 struct rss_reply_data data = {};
1013 const struct ethtool_ops *ops;
1014 struct rss_req_info req = {};
1015 struct net_device *dev;
1016 u32 indir_user_size;
1017 struct sk_buff *rsp;
1018 void *hdr;
1019 u32 limit;
1020 int ret;
1021
1022 rsp = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
1023 if (!rsp)
1024 return -ENOMEM;
1025
1026 ret = ethnl_parse_header_dev_get(&req.base, tb[ETHTOOL_A_RSS_HEADER],
1027 genl_info_net(info), info->extack,
1028 true);
1029 if (ret < 0)
1030 goto exit_free_rsp;
1031
1032 dev = req.base.dev;
1033 ops = dev->ethtool_ops;
1034
1035 req.rss_context = nla_get_u32_default(tb[ETHTOOL_A_RSS_CONTEXT], 0);
1036
1037 ret = ethnl_rss_create_validate(dev, info);
1038 if (ret)
1039 goto exit_free_dev;
1040
1041 rtnl_lock();
1042 netdev_lock_ops(dev);
1043
1044 ret = ethnl_ops_begin(dev);
1045 if (ret < 0)
1046 goto exit_dev_unlock;
1047
1048 ret = rss_get_data_alloc(dev, &data);
1049 if (ret)
1050 goto exit_ops;
1051
1052 ret = rss_set_prep_indir(dev, info, &data, &rxfh, &indir_dflt, &mod);
1053 if (ret < 0)
1054 goto exit_clean_data;
1055 indir_user_size = ret;
1056
1057 ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod);
1058
1059 ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod);
1060 if (ret)
1061 goto exit_free_indir;
1062
1063 rxfh.input_xfrm = RXH_XFRM_NO_CHANGE;
1064 ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod);
1065
1066 ctx = ethtool_rxfh_ctx_alloc(ops, data.indir_size, data.hkey_size);
1067 if (!ctx) {
1068 ret = -ENOMEM;
1069 goto exit_free_hkey;
1070 }
1071
1072 mutex_lock(&dev->ethtool->rss_lock);
1073 if (!req.rss_context) {
1074 limit = ops->rxfh_max_num_contexts ?: U32_MAX;
1075 ret = xa_alloc(&dev->ethtool->rss_ctx, &req.rss_context, ctx,
1076 XA_LIMIT(1, limit - 1), GFP_KERNEL_ACCOUNT);
1077 } else {
1078 ret = xa_insert(&dev->ethtool->rss_ctx,
1079 req.rss_context, ctx, GFP_KERNEL_ACCOUNT);
1080 }
1081 if (ret < 0) {
1082 NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT],
1083 "error allocating context ID");
1084 goto err_unlock_free_ctx;
1085 }
1086 rxfh.rss_context = req.rss_context;
1087
1088 ret = ops->create_rxfh_context(dev, ctx, &rxfh, info->extack);
1089 if (ret)
1090 goto err_ctx_id_free;
1091
1092 /* Make sure driver populates defaults */
1093 WARN_ON_ONCE(!rxfh.key && ops->rxfh_per_ctx_key &&
1094 !memchr_inv(ethtool_rxfh_context_key(ctx), 0,
1095 ctx->key_size));
1096
1097 /* Store the config from rxfh to Xarray.. */
1098 rss_set_ctx_update(ctx, tb, &data, &rxfh);
1099 ctx->indir_user_size = indir_user_size;
1100 /* .. copy from Xarray to data. */
1101 __rss_prepare_ctx(dev, &data, ctx);
1102
1103 hdr = ethnl_unicast_put(rsp, info->snd_portid, info->snd_seq,
1104 ETHTOOL_MSG_RSS_CREATE_ACT_REPLY);
1105 ntf_fail = ethnl_fill_reply_header(rsp, dev, ETHTOOL_A_RSS_HEADER);
1106 ntf_fail |= rss_fill_reply(rsp, &req.base, &data.base);
1107 if (WARN_ON(!hdr || ntf_fail)) {
1108 ret = -EMSGSIZE;
1109 goto err_remove_ctx;
1110 }
1111
1112 genlmsg_end(rsp, hdr);
1113
1114 ethnl_rss_create_send_ntf(rsp, dev);
1115 ret = genlmsg_reply(rsp, info);
1116 rsp = NULL;
1117
1118 exit_unlock:
1119 mutex_unlock(&dev->ethtool->rss_lock);
1120 exit_free_hkey:
1121 kfree(rxfh.key);
1122 exit_free_indir:
1123 kfree(rxfh.indir);
1124 exit_clean_data:
1125 rss_get_data_free(&data);
1126 exit_ops:
1127 ethnl_ops_complete(dev);
1128 exit_dev_unlock:
1129 netdev_unlock_ops(dev);
1130 rtnl_unlock();
1131 exit_free_dev:
1132 ethnl_parse_header_dev_put(&req.base);
1133 exit_free_rsp:
1134 nlmsg_free(rsp);
1135 return ret;
1136
1137 err_remove_ctx:
1138 if (ops->remove_rxfh_context(dev, ctx, req.rss_context, NULL))
1139 /* leave the context on failure, like ethnl_rss_delete_doit() */
1140 goto exit_unlock;
1141 err_ctx_id_free:
1142 xa_erase(&dev->ethtool->rss_ctx, req.rss_context);
1143 err_unlock_free_ctx:
1144 kfree(ctx);
1145 goto exit_unlock;
1146 }
1147
1148 /* RSS_DELETE */
1149
1150 const struct nla_policy ethnl_rss_delete_policy[ETHTOOL_A_RSS_CONTEXT + 1] = {
1151 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy),
1152 [ETHTOOL_A_RSS_CONTEXT] = NLA_POLICY_MIN(NLA_U32, 1),
1153 };
1154
ethnl_rss_delete_doit(struct sk_buff * skb,struct genl_info * info)1155 int ethnl_rss_delete_doit(struct sk_buff *skb, struct genl_info *info)
1156 {
1157 struct ethtool_rxfh_context *ctx;
1158 struct nlattr **tb = info->attrs;
1159 struct ethnl_req_info req = {};
1160 const struct ethtool_ops *ops;
1161 struct net_device *dev;
1162 u32 rss_context;
1163 int ret;
1164
1165 if (GENL_REQ_ATTR_CHECK(info, ETHTOOL_A_RSS_CONTEXT))
1166 return -EINVAL;
1167 rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]);
1168
1169 ret = ethnl_parse_header_dev_get(&req, tb[ETHTOOL_A_RSS_HEADER],
1170 genl_info_net(info), info->extack,
1171 true);
1172 if (ret < 0)
1173 return ret;
1174
1175 dev = req.dev;
1176 ops = dev->ethtool_ops;
1177
1178 if (!ops->create_rxfh_context) {
1179 ret = -EOPNOTSUPP;
1180 goto exit_free_dev;
1181 }
1182
1183 rtnl_lock();
1184 netdev_lock_ops(dev);
1185
1186 ret = ethnl_ops_begin(dev);
1187 if (ret < 0)
1188 goto exit_dev_unlock;
1189
1190 mutex_lock(&dev->ethtool->rss_lock);
1191 ret = ethtool_check_rss_ctx_busy(dev, rss_context);
1192 if (ret)
1193 goto exit_unlock;
1194
1195 ctx = xa_load(&dev->ethtool->rss_ctx, rss_context);
1196 if (!ctx) {
1197 ret = -ENOENT;
1198 goto exit_unlock;
1199 }
1200
1201 ret = ops->remove_rxfh_context(dev, ctx, rss_context, info->extack);
1202 if (ret)
1203 goto exit_unlock;
1204
1205 WARN_ON(xa_erase(&dev->ethtool->rss_ctx, rss_context) != ctx);
1206 kfree(ctx);
1207
1208 ethnl_rss_delete_notify(dev, rss_context);
1209
1210 exit_unlock:
1211 mutex_unlock(&dev->ethtool->rss_lock);
1212 ethnl_ops_complete(dev);
1213 exit_dev_unlock:
1214 netdev_unlock_ops(dev);
1215 rtnl_unlock();
1216 exit_free_dev:
1217 ethnl_parse_header_dev_put(&req);
1218 return ret;
1219 }
1220