1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <net/netdev_lock.h> 4 5 #include "netlink.h" 6 #include "common.h" 7 8 struct rss_req_info { 9 struct ethnl_req_info base; 10 u32 rss_context; 11 }; 12 13 struct rss_reply_data { 14 struct ethnl_reply_data base; 15 bool has_flow_hash; 16 bool no_key_fields; 17 u32 indir_size; 18 u32 hkey_size; 19 u32 hfunc; 20 u32 input_xfrm; 21 u32 *indir_table; 22 u8 *hkey; 23 int flow_hash[__ETHTOOL_A_FLOW_CNT]; 24 }; 25 26 static const u8 ethtool_rxfh_ft_nl2ioctl[] = { 27 [ETHTOOL_A_FLOW_ETHER] = ETHER_FLOW, 28 [ETHTOOL_A_FLOW_IP4] = IPV4_FLOW, 29 [ETHTOOL_A_FLOW_IP6] = IPV6_FLOW, 30 [ETHTOOL_A_FLOW_TCP4] = TCP_V4_FLOW, 31 [ETHTOOL_A_FLOW_UDP4] = UDP_V4_FLOW, 32 [ETHTOOL_A_FLOW_SCTP4] = SCTP_V4_FLOW, 33 [ETHTOOL_A_FLOW_AH_ESP4] = AH_ESP_V4_FLOW, 34 [ETHTOOL_A_FLOW_TCP6] = TCP_V6_FLOW, 35 [ETHTOOL_A_FLOW_UDP6] = UDP_V6_FLOW, 36 [ETHTOOL_A_FLOW_SCTP6] = SCTP_V6_FLOW, 37 [ETHTOOL_A_FLOW_AH_ESP6] = AH_ESP_V6_FLOW, 38 [ETHTOOL_A_FLOW_AH4] = AH_V4_FLOW, 39 [ETHTOOL_A_FLOW_ESP4] = ESP_V4_FLOW, 40 [ETHTOOL_A_FLOW_AH6] = AH_V6_FLOW, 41 [ETHTOOL_A_FLOW_ESP6] = ESP_V6_FLOW, 42 [ETHTOOL_A_FLOW_GTPU4] = GTPU_V4_FLOW, 43 [ETHTOOL_A_FLOW_GTPU6] = GTPU_V6_FLOW, 44 [ETHTOOL_A_FLOW_GTPC4] = GTPC_V4_FLOW, 45 [ETHTOOL_A_FLOW_GTPC6] = GTPC_V6_FLOW, 46 [ETHTOOL_A_FLOW_GTPC_TEID4] = GTPC_TEID_V4_FLOW, 47 [ETHTOOL_A_FLOW_GTPC_TEID6] = GTPC_TEID_V6_FLOW, 48 [ETHTOOL_A_FLOW_GTPU_EH4] = GTPU_EH_V4_FLOW, 49 [ETHTOOL_A_FLOW_GTPU_EH6] = GTPU_EH_V6_FLOW, 50 [ETHTOOL_A_FLOW_GTPU_UL4] = GTPU_UL_V4_FLOW, 51 [ETHTOOL_A_FLOW_GTPU_UL6] = GTPU_UL_V6_FLOW, 52 [ETHTOOL_A_FLOW_GTPU_DL4] = GTPU_DL_V4_FLOW, 53 [ETHTOOL_A_FLOW_GTPU_DL6] = GTPU_DL_V6_FLOW, 54 }; 55 56 #define RSS_REQINFO(__req_base) \ 57 container_of(__req_base, struct rss_req_info, base) 58 59 #define RSS_REPDATA(__reply_base) \ 60 container_of(__reply_base, struct rss_reply_data, base) 61 62 const struct nla_policy ethnl_rss_get_policy[] = { 63 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), 64 [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32 }, 65 [ETHTOOL_A_RSS_START_CONTEXT] = { .type = NLA_U32 }, 66 }; 67 68 static int 69 rss_parse_request(struct ethnl_req_info *req_info, struct nlattr **tb, 70 struct netlink_ext_ack *extack) 71 { 72 struct rss_req_info *request = RSS_REQINFO(req_info); 73 74 if (tb[ETHTOOL_A_RSS_CONTEXT]) 75 request->rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]); 76 if (tb[ETHTOOL_A_RSS_START_CONTEXT]) { 77 NL_SET_BAD_ATTR(extack, tb[ETHTOOL_A_RSS_START_CONTEXT]); 78 return -EINVAL; 79 } 80 81 return 0; 82 } 83 84 static void 85 rss_prepare_flow_hash(const struct rss_req_info *req, struct net_device *dev, 86 struct rss_reply_data *data, const struct genl_info *info) 87 { 88 int i; 89 90 data->has_flow_hash = false; 91 92 if (!dev->ethtool_ops->get_rxfh_fields) 93 return; 94 if (req->rss_context && !dev->ethtool_ops->rxfh_per_ctx_fields) 95 return; 96 97 mutex_lock(&dev->ethtool->rss_lock); 98 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) { 99 struct ethtool_rxfh_fields fields = { 100 .flow_type = ethtool_rxfh_ft_nl2ioctl[i], 101 .rss_context = req->rss_context, 102 }; 103 104 if (dev->ethtool_ops->get_rxfh_fields(dev, &fields)) { 105 data->flow_hash[i] = -1; /* Unsupported */ 106 continue; 107 } 108 109 data->flow_hash[i] = fields.data; 110 data->has_flow_hash = true; 111 } 112 mutex_unlock(&dev->ethtool->rss_lock); 113 } 114 115 static int 116 rss_prepare_get(const struct rss_req_info *request, struct net_device *dev, 117 struct rss_reply_data *data, const struct genl_info *info) 118 { 119 struct ethtool_rxfh_param rxfh = {}; 120 const struct ethtool_ops *ops; 121 u32 total_size, indir_bytes; 122 u8 *rss_config; 123 int ret; 124 125 ops = dev->ethtool_ops; 126 127 ret = ethnl_ops_begin(dev); 128 if (ret < 0) 129 return ret; 130 mutex_lock(&dev->ethtool->rss_lock); 131 132 data->indir_size = 0; 133 data->hkey_size = 0; 134 if (ops->get_rxfh_indir_size) 135 data->indir_size = ops->get_rxfh_indir_size(dev); 136 if (ops->get_rxfh_key_size) 137 data->hkey_size = ops->get_rxfh_key_size(dev); 138 139 indir_bytes = data->indir_size * sizeof(u32); 140 total_size = indir_bytes + data->hkey_size; 141 rss_config = kzalloc(total_size, GFP_KERNEL); 142 if (!rss_config) { 143 ret = -ENOMEM; 144 goto out_unlock; 145 } 146 147 if (data->indir_size) 148 data->indir_table = (u32 *)rss_config; 149 if (data->hkey_size) 150 data->hkey = rss_config + indir_bytes; 151 152 rxfh.indir_size = data->indir_size; 153 rxfh.indir = data->indir_table; 154 rxfh.key_size = data->hkey_size; 155 rxfh.key = data->hkey; 156 157 ret = ops->get_rxfh(dev, &rxfh); 158 if (ret) 159 goto out_unlock; 160 161 data->hfunc = rxfh.hfunc; 162 data->input_xfrm = rxfh.input_xfrm; 163 out_unlock: 164 mutex_unlock(&dev->ethtool->rss_lock); 165 ethnl_ops_complete(dev); 166 return ret; 167 } 168 169 static int 170 rss_prepare_ctx(const struct rss_req_info *request, struct net_device *dev, 171 struct rss_reply_data *data, const struct genl_info *info) 172 { 173 struct ethtool_rxfh_context *ctx; 174 u32 total_size, indir_bytes; 175 u8 *rss_config; 176 int ret; 177 178 data->no_key_fields = !dev->ethtool_ops->rxfh_per_ctx_key; 179 180 mutex_lock(&dev->ethtool->rss_lock); 181 ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context); 182 if (!ctx) { 183 ret = -ENOENT; 184 goto out_unlock; 185 } 186 187 data->indir_size = ctx->indir_size; 188 data->hkey_size = ctx->key_size; 189 data->hfunc = ctx->hfunc; 190 data->input_xfrm = ctx->input_xfrm; 191 192 indir_bytes = data->indir_size * sizeof(u32); 193 total_size = indir_bytes + data->hkey_size; 194 rss_config = kzalloc(total_size, GFP_KERNEL); 195 if (!rss_config) { 196 ret = -ENOMEM; 197 goto out_unlock; 198 } 199 200 data->indir_table = (u32 *)rss_config; 201 memcpy(data->indir_table, ethtool_rxfh_context_indir(ctx), indir_bytes); 202 203 if (data->hkey_size) { 204 data->hkey = rss_config + indir_bytes; 205 memcpy(data->hkey, ethtool_rxfh_context_key(ctx), 206 data->hkey_size); 207 } 208 209 ret = 0; 210 out_unlock: 211 mutex_unlock(&dev->ethtool->rss_lock); 212 return ret; 213 } 214 215 static int 216 rss_prepare(const struct rss_req_info *request, struct net_device *dev, 217 struct rss_reply_data *data, const struct genl_info *info) 218 { 219 rss_prepare_flow_hash(request, dev, data, info); 220 221 /* Coming from RSS_SET, driver may only have flow_hash_fields ops */ 222 if (!dev->ethtool_ops->get_rxfh) 223 return 0; 224 225 if (request->rss_context) 226 return rss_prepare_ctx(request, dev, data, info); 227 return rss_prepare_get(request, dev, data, info); 228 } 229 230 static int 231 rss_prepare_data(const struct ethnl_req_info *req_base, 232 struct ethnl_reply_data *reply_base, 233 const struct genl_info *info) 234 { 235 struct rss_reply_data *data = RSS_REPDATA(reply_base); 236 struct rss_req_info *request = RSS_REQINFO(req_base); 237 struct net_device *dev = reply_base->dev; 238 const struct ethtool_ops *ops; 239 240 ops = dev->ethtool_ops; 241 if (!ops->get_rxfh) 242 return -EOPNOTSUPP; 243 244 /* Some drivers don't handle rss_context */ 245 if (request->rss_context && !ops->create_rxfh_context) 246 return -EOPNOTSUPP; 247 248 return rss_prepare(request, dev, data, info); 249 } 250 251 static int 252 rss_reply_size(const struct ethnl_req_info *req_base, 253 const struct ethnl_reply_data *reply_base) 254 { 255 const struct rss_reply_data *data = RSS_REPDATA(reply_base); 256 int len; 257 258 len = nla_total_size(sizeof(u32)) + /* _RSS_CONTEXT */ 259 nla_total_size(sizeof(u32)) + /* _RSS_HFUNC */ 260 nla_total_size(sizeof(u32)) + /* _RSS_INPUT_XFRM */ 261 nla_total_size(sizeof(u32) * data->indir_size) + /* _RSS_INDIR */ 262 nla_total_size(data->hkey_size) + /* _RSS_HKEY */ 263 nla_total_size(0) + /* _RSS_FLOW_HASH */ 264 nla_total_size(sizeof(u32)) * ETHTOOL_A_FLOW_MAX + 265 0; 266 267 return len; 268 } 269 270 static int 271 rss_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base, 272 const struct ethnl_reply_data *reply_base) 273 { 274 const struct rss_reply_data *data = RSS_REPDATA(reply_base); 275 struct rss_req_info *request = RSS_REQINFO(req_base); 276 277 if (request->rss_context && 278 nla_put_u32(skb, ETHTOOL_A_RSS_CONTEXT, request->rss_context)) 279 return -EMSGSIZE; 280 281 if ((data->indir_size && 282 nla_put(skb, ETHTOOL_A_RSS_INDIR, 283 sizeof(u32) * data->indir_size, data->indir_table))) 284 return -EMSGSIZE; 285 286 if (!data->no_key_fields && 287 ((data->hfunc && 288 nla_put_u32(skb, ETHTOOL_A_RSS_HFUNC, data->hfunc)) || 289 (data->input_xfrm && 290 nla_put_u32(skb, ETHTOOL_A_RSS_INPUT_XFRM, data->input_xfrm)) || 291 (data->hkey_size && 292 nla_put(skb, ETHTOOL_A_RSS_HKEY, data->hkey_size, data->hkey)))) 293 return -EMSGSIZE; 294 295 if (data->has_flow_hash) { 296 struct nlattr *nest; 297 int i; 298 299 nest = nla_nest_start(skb, ETHTOOL_A_RSS_FLOW_HASH); 300 if (!nest) 301 return -EMSGSIZE; 302 303 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) { 304 if (data->flow_hash[i] >= 0 && 305 nla_put_uint(skb, i, data->flow_hash[i])) { 306 nla_nest_cancel(skb, nest); 307 return -EMSGSIZE; 308 } 309 } 310 311 nla_nest_end(skb, nest); 312 } 313 314 return 0; 315 } 316 317 static void rss_cleanup_data(struct ethnl_reply_data *reply_base) 318 { 319 const struct rss_reply_data *data = RSS_REPDATA(reply_base); 320 321 kfree(data->indir_table); 322 } 323 324 struct rss_nl_dump_ctx { 325 unsigned long ifindex; 326 unsigned long ctx_idx; 327 328 /* User wants to only dump contexts from given ifindex */ 329 unsigned int match_ifindex; 330 unsigned int start_ctx; 331 }; 332 333 static struct rss_nl_dump_ctx *rss_dump_ctx(struct netlink_callback *cb) 334 { 335 NL_ASSERT_CTX_FITS(struct rss_nl_dump_ctx); 336 337 return (struct rss_nl_dump_ctx *)cb->ctx; 338 } 339 340 int ethnl_rss_dump_start(struct netlink_callback *cb) 341 { 342 const struct genl_info *info = genl_info_dump(cb); 343 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb); 344 struct ethnl_req_info req_info = {}; 345 struct nlattr **tb = info->attrs; 346 int ret; 347 348 /* Filtering by context not supported */ 349 if (tb[ETHTOOL_A_RSS_CONTEXT]) { 350 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT]); 351 return -EINVAL; 352 } 353 if (tb[ETHTOOL_A_RSS_START_CONTEXT]) { 354 ctx->start_ctx = nla_get_u32(tb[ETHTOOL_A_RSS_START_CONTEXT]); 355 ctx->ctx_idx = ctx->start_ctx; 356 } 357 358 ret = ethnl_parse_header_dev_get(&req_info, 359 tb[ETHTOOL_A_RSS_HEADER], 360 sock_net(cb->skb->sk), cb->extack, 361 false); 362 if (req_info.dev) { 363 ctx->match_ifindex = req_info.dev->ifindex; 364 ctx->ifindex = ctx->match_ifindex; 365 ethnl_parse_header_dev_put(&req_info); 366 req_info.dev = NULL; 367 } 368 369 return ret; 370 } 371 372 static int 373 rss_dump_one_ctx(struct sk_buff *skb, struct netlink_callback *cb, 374 struct net_device *dev, u32 rss_context) 375 { 376 const struct genl_info *info = genl_info_dump(cb); 377 struct rss_reply_data data = {}; 378 struct rss_req_info req = {}; 379 void *ehdr; 380 int ret; 381 382 req.rss_context = rss_context; 383 384 ehdr = ethnl_dump_put(skb, cb, ETHTOOL_MSG_RSS_GET_REPLY); 385 if (!ehdr) 386 return -EMSGSIZE; 387 388 ret = ethnl_fill_reply_header(skb, dev, ETHTOOL_A_RSS_HEADER); 389 if (ret < 0) 390 goto err_cancel; 391 392 ret = rss_prepare(&req, dev, &data, info); 393 if (ret) 394 goto err_cancel; 395 396 ret = rss_fill_reply(skb, &req.base, &data.base); 397 if (ret) 398 goto err_cleanup; 399 genlmsg_end(skb, ehdr); 400 401 rss_cleanup_data(&data.base); 402 return 0; 403 404 err_cleanup: 405 rss_cleanup_data(&data.base); 406 err_cancel: 407 genlmsg_cancel(skb, ehdr); 408 return ret; 409 } 410 411 static int 412 rss_dump_one_dev(struct sk_buff *skb, struct netlink_callback *cb, 413 struct net_device *dev) 414 { 415 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb); 416 int ret; 417 418 if (!dev->ethtool_ops->get_rxfh) 419 return 0; 420 421 if (!ctx->ctx_idx) { 422 ret = rss_dump_one_ctx(skb, cb, dev, 0); 423 if (ret) 424 return ret; 425 ctx->ctx_idx++; 426 } 427 428 for (; xa_find(&dev->ethtool->rss_ctx, &ctx->ctx_idx, 429 ULONG_MAX, XA_PRESENT); ctx->ctx_idx++) { 430 ret = rss_dump_one_ctx(skb, cb, dev, ctx->ctx_idx); 431 if (ret) 432 return ret; 433 } 434 ctx->ctx_idx = ctx->start_ctx; 435 436 return 0; 437 } 438 439 int ethnl_rss_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 440 { 441 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb); 442 struct net *net = sock_net(skb->sk); 443 struct net_device *dev; 444 int ret = 0; 445 446 rtnl_lock(); 447 for_each_netdev_dump(net, dev, ctx->ifindex) { 448 if (ctx->match_ifindex && ctx->match_ifindex != ctx->ifindex) 449 break; 450 451 netdev_lock_ops(dev); 452 ret = rss_dump_one_dev(skb, cb, dev); 453 netdev_unlock_ops(dev); 454 if (ret) 455 break; 456 } 457 rtnl_unlock(); 458 459 return ret; 460 } 461 462 /* RSS_NTF */ 463 464 void ethtool_rss_notify(struct net_device *dev, u32 rss_context) 465 { 466 struct rss_req_info req_info = { 467 .rss_context = rss_context, 468 }; 469 470 ethnl_notify(dev, ETHTOOL_MSG_RSS_NTF, &req_info.base); 471 } 472 473 /* RSS_SET */ 474 475 #define RFH_MASK (RXH_L2DA | RXH_VLAN | RXH_IP_SRC | RXH_IP_DST | \ 476 RXH_L3_PROTO | RXH_L4_B_0_1 | RXH_L4_B_2_3 | \ 477 RXH_GTP_TEID | RXH_DISCARD) 478 479 static const struct nla_policy ethnl_rss_flows_policy[] = { 480 [ETHTOOL_A_FLOW_ETHER] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 481 [ETHTOOL_A_FLOW_IP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 482 [ETHTOOL_A_FLOW_IP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 483 [ETHTOOL_A_FLOW_TCP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 484 [ETHTOOL_A_FLOW_UDP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 485 [ETHTOOL_A_FLOW_SCTP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 486 [ETHTOOL_A_FLOW_AH_ESP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 487 [ETHTOOL_A_FLOW_TCP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 488 [ETHTOOL_A_FLOW_UDP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 489 [ETHTOOL_A_FLOW_SCTP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 490 [ETHTOOL_A_FLOW_AH_ESP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 491 [ETHTOOL_A_FLOW_AH4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 492 [ETHTOOL_A_FLOW_ESP4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 493 [ETHTOOL_A_FLOW_AH6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 494 [ETHTOOL_A_FLOW_ESP6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 495 [ETHTOOL_A_FLOW_GTPU4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 496 [ETHTOOL_A_FLOW_GTPU6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 497 [ETHTOOL_A_FLOW_GTPC4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 498 [ETHTOOL_A_FLOW_GTPC6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 499 [ETHTOOL_A_FLOW_GTPC_TEID4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 500 [ETHTOOL_A_FLOW_GTPC_TEID6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 501 [ETHTOOL_A_FLOW_GTPU_EH4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 502 [ETHTOOL_A_FLOW_GTPU_EH6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 503 [ETHTOOL_A_FLOW_GTPU_UL4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 504 [ETHTOOL_A_FLOW_GTPU_UL6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 505 [ETHTOOL_A_FLOW_GTPU_DL4] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 506 [ETHTOOL_A_FLOW_GTPU_DL6] = NLA_POLICY_MASK(NLA_UINT, RFH_MASK), 507 }; 508 509 const struct nla_policy ethnl_rss_set_policy[ETHTOOL_A_RSS_FLOW_HASH + 1] = { 510 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), 511 [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32, }, 512 [ETHTOOL_A_RSS_HFUNC] = NLA_POLICY_MIN(NLA_U32, 1), 513 [ETHTOOL_A_RSS_INDIR] = { .type = NLA_BINARY, }, 514 [ETHTOOL_A_RSS_HKEY] = NLA_POLICY_MIN(NLA_BINARY, 1), 515 [ETHTOOL_A_RSS_INPUT_XFRM] = 516 NLA_POLICY_MAX(NLA_U32, RXH_XFRM_SYM_OR_XOR), 517 [ETHTOOL_A_RSS_FLOW_HASH] = NLA_POLICY_NESTED(ethnl_rss_flows_policy), 518 }; 519 520 static int 521 ethnl_rss_set_validate(struct ethnl_req_info *req_info, struct genl_info *info) 522 { 523 const struct ethtool_ops *ops = req_info->dev->ethtool_ops; 524 struct rss_req_info *request = RSS_REQINFO(req_info); 525 struct nlattr **tb = info->attrs; 526 struct nlattr *bad_attr = NULL; 527 u32 input_xfrm; 528 529 if (request->rss_context && !ops->create_rxfh_context) 530 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_CONTEXT]; 531 532 if (request->rss_context && !ops->rxfh_per_ctx_key) { 533 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HFUNC]; 534 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_HKEY]; 535 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM]; 536 } 537 538 input_xfrm = nla_get_u32_default(tb[ETHTOOL_A_RSS_INPUT_XFRM], 0); 539 if (input_xfrm & ~ops->supported_input_xfrm) 540 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_INPUT_XFRM]; 541 542 if (tb[ETHTOOL_A_RSS_FLOW_HASH] && !ops->set_rxfh_fields) 543 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_FLOW_HASH]; 544 if (request->rss_context && 545 tb[ETHTOOL_A_RSS_FLOW_HASH] && !ops->rxfh_per_ctx_fields) 546 bad_attr = bad_attr ?: tb[ETHTOOL_A_RSS_FLOW_HASH]; 547 548 if (bad_attr) { 549 NL_SET_BAD_ATTR(info->extack, bad_attr); 550 return -EOPNOTSUPP; 551 } 552 553 return 1; 554 } 555 556 static int 557 rss_set_prep_indir(struct net_device *dev, struct genl_info *info, 558 struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh, 559 bool *reset, bool *mod) 560 { 561 const struct ethtool_ops *ops = dev->ethtool_ops; 562 struct netlink_ext_ack *extack = info->extack; 563 struct nlattr **tb = info->attrs; 564 struct ethtool_rxnfc rx_rings; 565 size_t alloc_size; 566 u32 user_size; 567 int i, err; 568 569 if (!tb[ETHTOOL_A_RSS_INDIR]) 570 return 0; 571 if (!data->indir_size || !ops->get_rxnfc) 572 return -EOPNOTSUPP; 573 574 rx_rings.cmd = ETHTOOL_GRXRINGS; 575 err = ops->get_rxnfc(dev, &rx_rings, NULL); 576 if (err) 577 return err; 578 579 if (nla_len(tb[ETHTOOL_A_RSS_INDIR]) % 4) { 580 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_INDIR]); 581 return -EINVAL; 582 } 583 user_size = nla_len(tb[ETHTOOL_A_RSS_INDIR]) / 4; 584 if (!user_size) { 585 if (rxfh->rss_context) { 586 NL_SET_ERR_MSG_ATTR(extack, tb[ETHTOOL_A_RSS_INDIR], 587 "can't reset table for a context"); 588 return -EINVAL; 589 } 590 *reset = true; 591 } else if (data->indir_size % user_size) { 592 NL_SET_ERR_MSG_ATTR_FMT(extack, tb[ETHTOOL_A_RSS_INDIR], 593 "size (%d) mismatch with device indir table (%d)", 594 user_size, data->indir_size); 595 return -EINVAL; 596 } 597 598 rxfh->indir_size = data->indir_size; 599 alloc_size = array_size(data->indir_size, sizeof(rxfh->indir[0])); 600 rxfh->indir = kzalloc(alloc_size, GFP_KERNEL); 601 if (!rxfh->indir) 602 return -ENOMEM; 603 604 nla_memcpy(rxfh->indir, tb[ETHTOOL_A_RSS_INDIR], alloc_size); 605 for (i = 0; i < user_size; i++) { 606 if (rxfh->indir[i] < rx_rings.data) 607 continue; 608 609 NL_SET_ERR_MSG_ATTR_FMT(extack, tb[ETHTOOL_A_RSS_INDIR], 610 "entry %d: queue out of range (%d)", 611 i, rxfh->indir[i]); 612 err = -EINVAL; 613 goto err_free; 614 } 615 616 if (user_size) { 617 /* Replicate the user-provided table to fill the device table */ 618 for (i = user_size; i < data->indir_size; i++) 619 rxfh->indir[i] = rxfh->indir[i % user_size]; 620 } else { 621 for (i = 0; i < data->indir_size; i++) 622 rxfh->indir[i] = 623 ethtool_rxfh_indir_default(i, rx_rings.data); 624 } 625 626 *mod |= memcmp(rxfh->indir, data->indir_table, data->indir_size); 627 628 return 0; 629 630 err_free: 631 kfree(rxfh->indir); 632 rxfh->indir = NULL; 633 return err; 634 } 635 636 static int 637 rss_set_prep_hkey(struct net_device *dev, struct genl_info *info, 638 struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh, 639 bool *mod) 640 { 641 struct nlattr **tb = info->attrs; 642 643 if (!tb[ETHTOOL_A_RSS_HKEY]) 644 return 0; 645 646 if (nla_len(tb[ETHTOOL_A_RSS_HKEY]) != data->hkey_size) { 647 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_HKEY]); 648 return -EINVAL; 649 } 650 651 rxfh->key_size = data->hkey_size; 652 rxfh->key = kmemdup(data->hkey, data->hkey_size, GFP_KERNEL); 653 if (!rxfh->key) 654 return -ENOMEM; 655 656 ethnl_update_binary(rxfh->key, rxfh->key_size, tb[ETHTOOL_A_RSS_HKEY], 657 mod); 658 return 0; 659 } 660 661 static int 662 rss_check_rxfh_fields_sym(struct net_device *dev, struct genl_info *info, 663 struct rss_reply_data *data, bool xfrm_sym) 664 { 665 struct nlattr **tb = info->attrs; 666 int i; 667 668 if (!xfrm_sym) 669 return 0; 670 if (!data->has_flow_hash) { 671 NL_SET_ERR_MSG_ATTR(info->extack, tb[ETHTOOL_A_RSS_INPUT_XFRM], 672 "hash field config not reported"); 673 return -EINVAL; 674 } 675 676 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) 677 if (data->flow_hash[i] >= 0 && 678 !ethtool_rxfh_config_is_sym(data->flow_hash[i])) { 679 NL_SET_ERR_MSG_ATTR(info->extack, 680 tb[ETHTOOL_A_RSS_INPUT_XFRM], 681 "hash field config is not symmetric"); 682 return -EINVAL; 683 } 684 685 return 0; 686 } 687 688 static int 689 ethnl_set_rss_fields(struct net_device *dev, struct genl_info *info, 690 u32 rss_context, struct rss_reply_data *data, 691 bool xfrm_sym, bool *mod) 692 { 693 struct nlattr *flow_nest = info->attrs[ETHTOOL_A_RSS_FLOW_HASH]; 694 struct nlattr *flows[ETHTOOL_A_FLOW_MAX + 1]; 695 const struct ethtool_ops *ops; 696 int i, ret; 697 698 ops = dev->ethtool_ops; 699 700 ret = rss_check_rxfh_fields_sym(dev, info, data, xfrm_sym); 701 if (ret) 702 return ret; 703 704 if (!flow_nest) 705 return 0; 706 707 ret = nla_parse_nested(flows, ARRAY_SIZE(ethnl_rss_flows_policy) - 1, 708 flow_nest, ethnl_rss_flows_policy, info->extack); 709 if (ret < 0) 710 return ret; 711 712 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) { 713 struct ethtool_rxfh_fields fields = { 714 .flow_type = ethtool_rxfh_ft_nl2ioctl[i], 715 .rss_context = rss_context, 716 }; 717 718 if (!flows[i]) 719 continue; 720 721 fields.data = nla_get_u32(flows[i]); 722 if (data->has_flow_hash && data->flow_hash[i] == fields.data) 723 continue; 724 725 if (xfrm_sym && !ethtool_rxfh_config_is_sym(fields.data)) { 726 NL_SET_ERR_MSG_ATTR(info->extack, flows[i], 727 "conflict with xfrm-input"); 728 return -EINVAL; 729 } 730 731 ret = ops->set_rxfh_fields(dev, &fields, info->extack); 732 if (ret) 733 return ret; 734 735 *mod = true; 736 } 737 738 return 0; 739 } 740 741 static void 742 rss_set_ctx_update(struct ethtool_rxfh_context *ctx, struct nlattr **tb, 743 struct rss_reply_data *data, struct ethtool_rxfh_param *rxfh) 744 { 745 int i; 746 747 if (rxfh->indir) { 748 for (i = 0; i < data->indir_size; i++) 749 ethtool_rxfh_context_indir(ctx)[i] = rxfh->indir[i]; 750 ctx->indir_configured = !!nla_len(tb[ETHTOOL_A_RSS_INDIR]); 751 } 752 if (rxfh->key) { 753 memcpy(ethtool_rxfh_context_key(ctx), rxfh->key, 754 data->hkey_size); 755 ctx->key_configured = !!rxfh->key_size; 756 } 757 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE) 758 ctx->hfunc = rxfh->hfunc; 759 if (rxfh->input_xfrm != RXH_XFRM_NO_CHANGE) 760 ctx->input_xfrm = rxfh->input_xfrm; 761 } 762 763 static int 764 ethnl_rss_set(struct ethnl_req_info *req_info, struct genl_info *info) 765 { 766 bool indir_reset = false, indir_mod, xfrm_sym = false; 767 struct rss_req_info *request = RSS_REQINFO(req_info); 768 struct ethtool_rxfh_context *ctx = NULL; 769 struct net_device *dev = req_info->dev; 770 bool mod = false, fields_mod = false; 771 struct ethtool_rxfh_param rxfh = {}; 772 struct nlattr **tb = info->attrs; 773 struct rss_reply_data data = {}; 774 const struct ethtool_ops *ops; 775 int ret; 776 777 ops = dev->ethtool_ops; 778 data.base.dev = dev; 779 780 ret = rss_prepare(request, dev, &data, info); 781 if (ret) 782 return ret; 783 784 rxfh.rss_context = request->rss_context; 785 786 ret = rss_set_prep_indir(dev, info, &data, &rxfh, &indir_reset, &mod); 787 if (ret) 788 goto exit_clean_data; 789 indir_mod = !!tb[ETHTOOL_A_RSS_INDIR]; 790 791 rxfh.hfunc = data.hfunc; 792 ethnl_update_u8(&rxfh.hfunc, tb[ETHTOOL_A_RSS_HFUNC], &mod); 793 if (rxfh.hfunc == data.hfunc) 794 rxfh.hfunc = ETH_RSS_HASH_NO_CHANGE; 795 796 ret = rss_set_prep_hkey(dev, info, &data, &rxfh, &mod); 797 if (ret) 798 goto exit_free_indir; 799 800 rxfh.input_xfrm = data.input_xfrm; 801 ethnl_update_u8(&rxfh.input_xfrm, tb[ETHTOOL_A_RSS_INPUT_XFRM], &mod); 802 /* For drivers which don't support input_xfrm it will be set to 0xff 803 * in the RSS context info. In all other case input_xfrm != 0 means 804 * symmetric hashing is requested. 805 */ 806 if (!request->rss_context || ops->rxfh_per_ctx_key) 807 xfrm_sym = rxfh.input_xfrm || data.input_xfrm; 808 if (rxfh.input_xfrm == data.input_xfrm) 809 rxfh.input_xfrm = RXH_XFRM_NO_CHANGE; 810 811 mutex_lock(&dev->ethtool->rss_lock); 812 if (request->rss_context) { 813 ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context); 814 if (!ctx) { 815 ret = -ENOENT; 816 goto exit_unlock; 817 } 818 } 819 820 ret = ethnl_set_rss_fields(dev, info, request->rss_context, 821 &data, xfrm_sym, &fields_mod); 822 if (ret) 823 goto exit_unlock; 824 825 if (!mod) 826 ret = 0; /* nothing to tell the driver */ 827 else if (!ops->set_rxfh) 828 ret = -EOPNOTSUPP; 829 else if (!rxfh.rss_context) 830 ret = ops->set_rxfh(dev, &rxfh, info->extack); 831 else 832 ret = ops->modify_rxfh_context(dev, ctx, &rxfh, info->extack); 833 if (ret) 834 goto exit_unlock; 835 836 if (ctx) 837 rss_set_ctx_update(ctx, tb, &data, &rxfh); 838 else if (indir_reset) 839 dev->priv_flags &= ~IFF_RXFH_CONFIGURED; 840 else if (indir_mod) 841 dev->priv_flags |= IFF_RXFH_CONFIGURED; 842 843 exit_unlock: 844 mutex_unlock(&dev->ethtool->rss_lock); 845 kfree(rxfh.key); 846 exit_free_indir: 847 kfree(rxfh.indir); 848 exit_clean_data: 849 rss_cleanup_data(&data.base); 850 851 return ret ?: mod || fields_mod; 852 } 853 854 const struct ethnl_request_ops ethnl_rss_request_ops = { 855 .request_cmd = ETHTOOL_MSG_RSS_GET, 856 .reply_cmd = ETHTOOL_MSG_RSS_GET_REPLY, 857 .hdr_attr = ETHTOOL_A_RSS_HEADER, 858 .req_info_size = sizeof(struct rss_req_info), 859 .reply_data_size = sizeof(struct rss_reply_data), 860 861 .parse_request = rss_parse_request, 862 .prepare_data = rss_prepare_data, 863 .reply_size = rss_reply_size, 864 .fill_reply = rss_fill_reply, 865 .cleanup_data = rss_cleanup_data, 866 867 .set_validate = ethnl_rss_set_validate, 868 .set = ethnl_rss_set, 869 .set_ntf_cmd = ETHTOOL_MSG_RSS_NTF, 870 }; 871