1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <net/netdev_lock.h> 4 5 #include "netlink.h" 6 #include "common.h" 7 8 struct rss_req_info { 9 struct ethnl_req_info base; 10 u32 rss_context; 11 }; 12 13 struct rss_reply_data { 14 struct ethnl_reply_data base; 15 bool has_flow_hash; 16 bool no_key_fields; 17 u32 indir_size; 18 u32 hkey_size; 19 u32 hfunc; 20 u32 input_xfrm; 21 u32 *indir_table; 22 u8 *hkey; 23 int flow_hash[__ETHTOOL_A_FLOW_CNT]; 24 }; 25 26 static const u8 ethtool_rxfh_ft_nl2ioctl[] = { 27 [ETHTOOL_A_FLOW_ETHER] = ETHER_FLOW, 28 [ETHTOOL_A_FLOW_IP4] = IPV4_FLOW, 29 [ETHTOOL_A_FLOW_IP6] = IPV6_FLOW, 30 [ETHTOOL_A_FLOW_TCP4] = TCP_V4_FLOW, 31 [ETHTOOL_A_FLOW_UDP4] = UDP_V4_FLOW, 32 [ETHTOOL_A_FLOW_SCTP4] = SCTP_V4_FLOW, 33 [ETHTOOL_A_FLOW_AH_ESP4] = AH_ESP_V4_FLOW, 34 [ETHTOOL_A_FLOW_TCP6] = TCP_V6_FLOW, 35 [ETHTOOL_A_FLOW_UDP6] = UDP_V6_FLOW, 36 [ETHTOOL_A_FLOW_SCTP6] = SCTP_V6_FLOW, 37 [ETHTOOL_A_FLOW_AH_ESP6] = AH_ESP_V6_FLOW, 38 [ETHTOOL_A_FLOW_AH4] = AH_V4_FLOW, 39 [ETHTOOL_A_FLOW_ESP4] = ESP_V4_FLOW, 40 [ETHTOOL_A_FLOW_AH6] = AH_V6_FLOW, 41 [ETHTOOL_A_FLOW_ESP6] = ESP_V6_FLOW, 42 [ETHTOOL_A_FLOW_GTPU4] = GTPU_V4_FLOW, 43 [ETHTOOL_A_FLOW_GTPU6] = GTPU_V6_FLOW, 44 [ETHTOOL_A_FLOW_GTPC4] = GTPC_V4_FLOW, 45 [ETHTOOL_A_FLOW_GTPC6] = GTPC_V6_FLOW, 46 [ETHTOOL_A_FLOW_GTPC_TEID4] = GTPC_TEID_V4_FLOW, 47 [ETHTOOL_A_FLOW_GTPC_TEID6] = GTPC_TEID_V6_FLOW, 48 [ETHTOOL_A_FLOW_GTPU_EH4] = GTPU_EH_V4_FLOW, 49 [ETHTOOL_A_FLOW_GTPU_EH6] = GTPU_EH_V6_FLOW, 50 [ETHTOOL_A_FLOW_GTPU_UL4] = GTPU_UL_V4_FLOW, 51 [ETHTOOL_A_FLOW_GTPU_UL6] = GTPU_UL_V6_FLOW, 52 [ETHTOOL_A_FLOW_GTPU_DL4] = GTPU_DL_V4_FLOW, 53 [ETHTOOL_A_FLOW_GTPU_DL6] = GTPU_DL_V6_FLOW, 54 }; 55 56 #define RSS_REQINFO(__req_base) \ 57 container_of(__req_base, struct rss_req_info, base) 58 59 #define RSS_REPDATA(__reply_base) \ 60 container_of(__reply_base, struct rss_reply_data, base) 61 62 const struct nla_policy ethnl_rss_get_policy[] = { 63 [ETHTOOL_A_RSS_HEADER] = NLA_POLICY_NESTED(ethnl_header_policy), 64 [ETHTOOL_A_RSS_CONTEXT] = { .type = NLA_U32 }, 65 [ETHTOOL_A_RSS_START_CONTEXT] = { .type = NLA_U32 }, 66 }; 67 68 static int 69 rss_parse_request(struct ethnl_req_info *req_info, struct nlattr **tb, 70 struct netlink_ext_ack *extack) 71 { 72 struct rss_req_info *request = RSS_REQINFO(req_info); 73 74 if (tb[ETHTOOL_A_RSS_CONTEXT]) 75 request->rss_context = nla_get_u32(tb[ETHTOOL_A_RSS_CONTEXT]); 76 if (tb[ETHTOOL_A_RSS_START_CONTEXT]) { 77 NL_SET_BAD_ATTR(extack, tb[ETHTOOL_A_RSS_START_CONTEXT]); 78 return -EINVAL; 79 } 80 81 return 0; 82 } 83 84 static void 85 rss_prepare_flow_hash(const struct rss_req_info *req, struct net_device *dev, 86 struct rss_reply_data *data, const struct genl_info *info) 87 { 88 int i; 89 90 data->has_flow_hash = false; 91 92 if (!dev->ethtool_ops->get_rxfh_fields) 93 return; 94 if (req->rss_context && !dev->ethtool_ops->rxfh_per_ctx_fields) 95 return; 96 97 mutex_lock(&dev->ethtool->rss_lock); 98 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) { 99 struct ethtool_rxfh_fields fields = { 100 .flow_type = ethtool_rxfh_ft_nl2ioctl[i], 101 .rss_context = req->rss_context, 102 }; 103 104 if (dev->ethtool_ops->get_rxfh_fields(dev, &fields)) { 105 data->flow_hash[i] = -1; /* Unsupported */ 106 continue; 107 } 108 109 data->flow_hash[i] = fields.data; 110 data->has_flow_hash = true; 111 } 112 mutex_unlock(&dev->ethtool->rss_lock); 113 } 114 115 static int 116 rss_prepare_get(const struct rss_req_info *request, struct net_device *dev, 117 struct rss_reply_data *data, const struct genl_info *info) 118 { 119 struct ethtool_rxfh_param rxfh = {}; 120 const struct ethtool_ops *ops; 121 u32 total_size, indir_bytes; 122 u8 *rss_config; 123 int ret; 124 125 ops = dev->ethtool_ops; 126 127 ret = ethnl_ops_begin(dev); 128 if (ret < 0) 129 return ret; 130 mutex_lock(&dev->ethtool->rss_lock); 131 132 data->indir_size = 0; 133 data->hkey_size = 0; 134 if (ops->get_rxfh_indir_size) 135 data->indir_size = ops->get_rxfh_indir_size(dev); 136 if (ops->get_rxfh_key_size) 137 data->hkey_size = ops->get_rxfh_key_size(dev); 138 139 indir_bytes = data->indir_size * sizeof(u32); 140 total_size = indir_bytes + data->hkey_size; 141 rss_config = kzalloc(total_size, GFP_KERNEL); 142 if (!rss_config) { 143 ret = -ENOMEM; 144 goto out_unlock; 145 } 146 147 if (data->indir_size) 148 data->indir_table = (u32 *)rss_config; 149 if (data->hkey_size) 150 data->hkey = rss_config + indir_bytes; 151 152 rxfh.indir_size = data->indir_size; 153 rxfh.indir = data->indir_table; 154 rxfh.key_size = data->hkey_size; 155 rxfh.key = data->hkey; 156 157 ret = ops->get_rxfh(dev, &rxfh); 158 if (ret) 159 goto out_unlock; 160 161 data->hfunc = rxfh.hfunc; 162 data->input_xfrm = rxfh.input_xfrm; 163 out_unlock: 164 mutex_unlock(&dev->ethtool->rss_lock); 165 ethnl_ops_complete(dev); 166 return ret; 167 } 168 169 static int 170 rss_prepare_ctx(const struct rss_req_info *request, struct net_device *dev, 171 struct rss_reply_data *data, const struct genl_info *info) 172 { 173 struct ethtool_rxfh_context *ctx; 174 u32 total_size, indir_bytes; 175 u8 *rss_config; 176 int ret; 177 178 data->no_key_fields = !dev->ethtool_ops->rxfh_per_ctx_key; 179 180 mutex_lock(&dev->ethtool->rss_lock); 181 ctx = xa_load(&dev->ethtool->rss_ctx, request->rss_context); 182 if (!ctx) { 183 ret = -ENOENT; 184 goto out_unlock; 185 } 186 187 data->indir_size = ctx->indir_size; 188 data->hkey_size = ctx->key_size; 189 data->hfunc = ctx->hfunc; 190 data->input_xfrm = ctx->input_xfrm; 191 192 indir_bytes = data->indir_size * sizeof(u32); 193 total_size = indir_bytes + data->hkey_size; 194 rss_config = kzalloc(total_size, GFP_KERNEL); 195 if (!rss_config) { 196 ret = -ENOMEM; 197 goto out_unlock; 198 } 199 200 data->indir_table = (u32 *)rss_config; 201 memcpy(data->indir_table, ethtool_rxfh_context_indir(ctx), indir_bytes); 202 203 if (data->hkey_size) { 204 data->hkey = rss_config + indir_bytes; 205 memcpy(data->hkey, ethtool_rxfh_context_key(ctx), 206 data->hkey_size); 207 } 208 209 ret = 0; 210 out_unlock: 211 mutex_unlock(&dev->ethtool->rss_lock); 212 return ret; 213 } 214 215 static int 216 rss_prepare(const struct rss_req_info *request, struct net_device *dev, 217 struct rss_reply_data *data, const struct genl_info *info) 218 { 219 rss_prepare_flow_hash(request, dev, data, info); 220 221 if (request->rss_context) 222 return rss_prepare_ctx(request, dev, data, info); 223 return rss_prepare_get(request, dev, data, info); 224 } 225 226 static int 227 rss_prepare_data(const struct ethnl_req_info *req_base, 228 struct ethnl_reply_data *reply_base, 229 const struct genl_info *info) 230 { 231 struct rss_reply_data *data = RSS_REPDATA(reply_base); 232 struct rss_req_info *request = RSS_REQINFO(req_base); 233 struct net_device *dev = reply_base->dev; 234 const struct ethtool_ops *ops; 235 236 ops = dev->ethtool_ops; 237 if (!ops->get_rxfh) 238 return -EOPNOTSUPP; 239 240 /* Some drivers don't handle rss_context */ 241 if (request->rss_context && !ops->create_rxfh_context) 242 return -EOPNOTSUPP; 243 244 return rss_prepare(request, dev, data, info); 245 } 246 247 static int 248 rss_reply_size(const struct ethnl_req_info *req_base, 249 const struct ethnl_reply_data *reply_base) 250 { 251 const struct rss_reply_data *data = RSS_REPDATA(reply_base); 252 int len; 253 254 len = nla_total_size(sizeof(u32)) + /* _RSS_CONTEXT */ 255 nla_total_size(sizeof(u32)) + /* _RSS_HFUNC */ 256 nla_total_size(sizeof(u32)) + /* _RSS_INPUT_XFRM */ 257 nla_total_size(sizeof(u32) * data->indir_size) + /* _RSS_INDIR */ 258 nla_total_size(data->hkey_size) + /* _RSS_HKEY */ 259 nla_total_size(0) + /* _RSS_FLOW_HASH */ 260 nla_total_size(sizeof(u32)) * ETHTOOL_A_FLOW_MAX + 261 0; 262 263 return len; 264 } 265 266 static int 267 rss_fill_reply(struct sk_buff *skb, const struct ethnl_req_info *req_base, 268 const struct ethnl_reply_data *reply_base) 269 { 270 const struct rss_reply_data *data = RSS_REPDATA(reply_base); 271 struct rss_req_info *request = RSS_REQINFO(req_base); 272 273 if (request->rss_context && 274 nla_put_u32(skb, ETHTOOL_A_RSS_CONTEXT, request->rss_context)) 275 return -EMSGSIZE; 276 277 if ((data->indir_size && 278 nla_put(skb, ETHTOOL_A_RSS_INDIR, 279 sizeof(u32) * data->indir_size, data->indir_table))) 280 return -EMSGSIZE; 281 282 if (!data->no_key_fields && 283 ((data->hfunc && 284 nla_put_u32(skb, ETHTOOL_A_RSS_HFUNC, data->hfunc)) || 285 (data->input_xfrm && 286 nla_put_u32(skb, ETHTOOL_A_RSS_INPUT_XFRM, data->input_xfrm)) || 287 (data->hkey_size && 288 nla_put(skb, ETHTOOL_A_RSS_HKEY, data->hkey_size, data->hkey)))) 289 return -EMSGSIZE; 290 291 if (data->has_flow_hash) { 292 struct nlattr *nest; 293 int i; 294 295 nest = nla_nest_start(skb, ETHTOOL_A_RSS_FLOW_HASH); 296 if (!nest) 297 return -EMSGSIZE; 298 299 for (i = 1; i < __ETHTOOL_A_FLOW_CNT; i++) { 300 if (data->flow_hash[i] >= 0 && 301 nla_put_uint(skb, i, data->flow_hash[i])) { 302 nla_nest_cancel(skb, nest); 303 return -EMSGSIZE; 304 } 305 } 306 307 nla_nest_end(skb, nest); 308 } 309 310 return 0; 311 } 312 313 static void rss_cleanup_data(struct ethnl_reply_data *reply_base) 314 { 315 const struct rss_reply_data *data = RSS_REPDATA(reply_base); 316 317 kfree(data->indir_table); 318 } 319 320 struct rss_nl_dump_ctx { 321 unsigned long ifindex; 322 unsigned long ctx_idx; 323 324 /* User wants to only dump contexts from given ifindex */ 325 unsigned int match_ifindex; 326 unsigned int start_ctx; 327 }; 328 329 static struct rss_nl_dump_ctx *rss_dump_ctx(struct netlink_callback *cb) 330 { 331 NL_ASSERT_CTX_FITS(struct rss_nl_dump_ctx); 332 333 return (struct rss_nl_dump_ctx *)cb->ctx; 334 } 335 336 int ethnl_rss_dump_start(struct netlink_callback *cb) 337 { 338 const struct genl_info *info = genl_info_dump(cb); 339 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb); 340 struct ethnl_req_info req_info = {}; 341 struct nlattr **tb = info->attrs; 342 int ret; 343 344 /* Filtering by context not supported */ 345 if (tb[ETHTOOL_A_RSS_CONTEXT]) { 346 NL_SET_BAD_ATTR(info->extack, tb[ETHTOOL_A_RSS_CONTEXT]); 347 return -EINVAL; 348 } 349 if (tb[ETHTOOL_A_RSS_START_CONTEXT]) { 350 ctx->start_ctx = nla_get_u32(tb[ETHTOOL_A_RSS_START_CONTEXT]); 351 ctx->ctx_idx = ctx->start_ctx; 352 } 353 354 ret = ethnl_parse_header_dev_get(&req_info, 355 tb[ETHTOOL_A_RSS_HEADER], 356 sock_net(cb->skb->sk), cb->extack, 357 false); 358 if (req_info.dev) { 359 ctx->match_ifindex = req_info.dev->ifindex; 360 ctx->ifindex = ctx->match_ifindex; 361 ethnl_parse_header_dev_put(&req_info); 362 req_info.dev = NULL; 363 } 364 365 return ret; 366 } 367 368 static int 369 rss_dump_one_ctx(struct sk_buff *skb, struct netlink_callback *cb, 370 struct net_device *dev, u32 rss_context) 371 { 372 const struct genl_info *info = genl_info_dump(cb); 373 struct rss_reply_data data = {}; 374 struct rss_req_info req = {}; 375 void *ehdr; 376 int ret; 377 378 req.rss_context = rss_context; 379 380 ehdr = ethnl_dump_put(skb, cb, ETHTOOL_MSG_RSS_GET_REPLY); 381 if (!ehdr) 382 return -EMSGSIZE; 383 384 ret = ethnl_fill_reply_header(skb, dev, ETHTOOL_A_RSS_HEADER); 385 if (ret < 0) 386 goto err_cancel; 387 388 ret = rss_prepare(&req, dev, &data, info); 389 if (ret) 390 goto err_cancel; 391 392 ret = rss_fill_reply(skb, &req.base, &data.base); 393 if (ret) 394 goto err_cleanup; 395 genlmsg_end(skb, ehdr); 396 397 rss_cleanup_data(&data.base); 398 return 0; 399 400 err_cleanup: 401 rss_cleanup_data(&data.base); 402 err_cancel: 403 genlmsg_cancel(skb, ehdr); 404 return ret; 405 } 406 407 static int 408 rss_dump_one_dev(struct sk_buff *skb, struct netlink_callback *cb, 409 struct net_device *dev) 410 { 411 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb); 412 int ret; 413 414 if (!dev->ethtool_ops->get_rxfh) 415 return 0; 416 417 if (!ctx->ctx_idx) { 418 ret = rss_dump_one_ctx(skb, cb, dev, 0); 419 if (ret) 420 return ret; 421 ctx->ctx_idx++; 422 } 423 424 for (; xa_find(&dev->ethtool->rss_ctx, &ctx->ctx_idx, 425 ULONG_MAX, XA_PRESENT); ctx->ctx_idx++) { 426 ret = rss_dump_one_ctx(skb, cb, dev, ctx->ctx_idx); 427 if (ret) 428 return ret; 429 } 430 ctx->ctx_idx = ctx->start_ctx; 431 432 return 0; 433 } 434 435 int ethnl_rss_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 436 { 437 struct rss_nl_dump_ctx *ctx = rss_dump_ctx(cb); 438 struct net *net = sock_net(skb->sk); 439 struct net_device *dev; 440 int ret = 0; 441 442 rtnl_lock(); 443 for_each_netdev_dump(net, dev, ctx->ifindex) { 444 if (ctx->match_ifindex && ctx->match_ifindex != ctx->ifindex) 445 break; 446 447 netdev_lock_ops(dev); 448 ret = rss_dump_one_dev(skb, cb, dev); 449 netdev_unlock_ops(dev); 450 if (ret) 451 break; 452 } 453 rtnl_unlock(); 454 455 return ret; 456 } 457 458 /* RSS_NTF */ 459 460 void ethtool_rss_notify(struct net_device *dev, u32 rss_context) 461 { 462 struct rss_req_info req_info = { 463 .rss_context = rss_context, 464 }; 465 466 ethnl_notify(dev, ETHTOOL_MSG_RSS_NTF, &req_info.base); 467 } 468 469 const struct ethnl_request_ops ethnl_rss_request_ops = { 470 .request_cmd = ETHTOOL_MSG_RSS_GET, 471 .reply_cmd = ETHTOOL_MSG_RSS_GET_REPLY, 472 .hdr_attr = ETHTOOL_A_RSS_HEADER, 473 .req_info_size = sizeof(struct rss_req_info), 474 .reply_data_size = sizeof(struct rss_reply_data), 475 476 .parse_request = rss_parse_request, 477 .prepare_data = rss_prepare_data, 478 .reply_size = rss_reply_size, 479 .fill_reply = rss_fill_reply, 480 .cleanup_data = rss_cleanup_data, 481 }; 482