xref: /linux/net/ethtool/ioctl.c (revision 07fdad3a93756b872da7b53647715c48d0f4a2d0)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * net/core/ethtool.c - Ethtool ioctl handler
4  * Copyright (c) 2003 Matthew Wilcox <matthew@wil.cx>
5  *
6  * This file is where we call all the ethtool_ops commands to get
7  * the information ethtool needs.
8  */
9 
10 #include <linux/compat.h>
11 #include <linux/etherdevice.h>
12 #include <linux/module.h>
13 #include <linux/types.h>
14 #include <linux/capability.h>
15 #include <linux/errno.h>
16 #include <linux/ethtool.h>
17 #include <linux/netdevice.h>
18 #include <linux/net_tstamp.h>
19 #include <linux/phy.h>
20 #include <linux/bitops.h>
21 #include <linux/uaccess.h>
22 #include <linux/vmalloc.h>
23 #include <linux/sfp.h>
24 #include <linux/slab.h>
25 #include <linux/rtnetlink.h>
26 #include <linux/sched/signal.h>
27 #include <linux/net.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/utsname.h>
30 #include <net/devlink.h>
31 #include <net/ipv6.h>
32 #include <net/xdp_sock_drv.h>
33 #include <net/flow_offload.h>
34 #include <net/netdev_lock.h>
35 #include <linux/ethtool_netlink.h>
36 #include "common.h"
37 
38 /* State held across locks and calls for commands which have devlink fallback */
39 struct ethtool_devlink_compat {
40 	struct devlink *devlink;
41 	union {
42 		struct ethtool_flash efl;
43 		struct ethtool_drvinfo info;
44 	};
45 };
46 
47 static struct devlink *netdev_to_devlink_get(struct net_device *dev)
48 {
49 	if (!dev->devlink_port)
50 		return NULL;
51 	return devlink_try_get(dev->devlink_port->devlink);
52 }
53 
54 /*
55  * Some useful ethtool_ops methods that're device independent.
56  * If we find that all drivers want to do the same thing here,
57  * we can turn these into dev_() function calls.
58  */
59 
60 u32 ethtool_op_get_link(struct net_device *dev)
61 {
62 	/* Synchronize carrier state with link watch, see also rtnl_getlink() */
63 	__linkwatch_sync_dev(dev);
64 
65 	return netif_carrier_ok(dev) ? 1 : 0;
66 }
67 EXPORT_SYMBOL(ethtool_op_get_link);
68 
69 int ethtool_op_get_ts_info(struct net_device *dev,
70 			   struct kernel_ethtool_ts_info *info)
71 {
72 	info->so_timestamping =
73 		SOF_TIMESTAMPING_TX_SOFTWARE |
74 		SOF_TIMESTAMPING_RX_SOFTWARE |
75 		SOF_TIMESTAMPING_SOFTWARE;
76 	info->phc_index = -1;
77 	return 0;
78 }
79 EXPORT_SYMBOL(ethtool_op_get_ts_info);
80 
81 /* Handlers for each ethtool command */
82 
83 static int ethtool_get_features(struct net_device *dev, void __user *useraddr)
84 {
85 	struct ethtool_gfeatures cmd = {
86 		.cmd = ETHTOOL_GFEATURES,
87 		.size = ETHTOOL_DEV_FEATURE_WORDS,
88 	};
89 	struct ethtool_get_features_block features[ETHTOOL_DEV_FEATURE_WORDS];
90 	u32 __user *sizeaddr;
91 	u32 copy_size;
92 	int i;
93 
94 	/* in case feature bits run out again */
95 	BUILD_BUG_ON(ETHTOOL_DEV_FEATURE_WORDS * sizeof(u32) > sizeof(netdev_features_t));
96 
97 	for (i = 0; i < ETHTOOL_DEV_FEATURE_WORDS; ++i) {
98 		features[i].available = (u32)(dev->hw_features >> (32 * i));
99 		features[i].requested = (u32)(dev->wanted_features >> (32 * i));
100 		features[i].active = (u32)(dev->features >> (32 * i));
101 		features[i].never_changed =
102 			(u32)(NETIF_F_NEVER_CHANGE >> (32 * i));
103 	}
104 
105 	sizeaddr = useraddr + offsetof(struct ethtool_gfeatures, size);
106 	if (get_user(copy_size, sizeaddr))
107 		return -EFAULT;
108 
109 	if (copy_size > ETHTOOL_DEV_FEATURE_WORDS)
110 		copy_size = ETHTOOL_DEV_FEATURE_WORDS;
111 
112 	if (copy_to_user(useraddr, &cmd, sizeof(cmd)))
113 		return -EFAULT;
114 	useraddr += sizeof(cmd);
115 	if (copy_to_user(useraddr, features,
116 			 array_size(copy_size, sizeof(*features))))
117 		return -EFAULT;
118 
119 	return 0;
120 }
121 
122 static int ethtool_set_features(struct net_device *dev, void __user *useraddr)
123 {
124 	struct ethtool_sfeatures cmd;
125 	struct ethtool_set_features_block features[ETHTOOL_DEV_FEATURE_WORDS];
126 	netdev_features_t wanted = 0, valid = 0;
127 	int i, ret = 0;
128 
129 	if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
130 		return -EFAULT;
131 	useraddr += sizeof(cmd);
132 
133 	if (cmd.size != ETHTOOL_DEV_FEATURE_WORDS)
134 		return -EINVAL;
135 
136 	if (copy_from_user(features, useraddr, sizeof(features)))
137 		return -EFAULT;
138 
139 	for (i = 0; i < ETHTOOL_DEV_FEATURE_WORDS; ++i) {
140 		valid |= (netdev_features_t)features[i].valid << (32 * i);
141 		wanted |= (netdev_features_t)features[i].requested << (32 * i);
142 	}
143 
144 	if (valid & ~NETIF_F_ETHTOOL_BITS)
145 		return -EINVAL;
146 
147 	if (valid & ~dev->hw_features) {
148 		valid &= dev->hw_features;
149 		ret |= ETHTOOL_F_UNSUPPORTED;
150 	}
151 
152 	dev->wanted_features &= ~valid;
153 	dev->wanted_features |= wanted & valid;
154 	__netdev_update_features(dev);
155 
156 	if ((dev->wanted_features ^ dev->features) & valid)
157 		ret |= ETHTOOL_F_WISH;
158 
159 	return ret;
160 }
161 
162 static int __ethtool_get_sset_count(struct net_device *dev, int sset)
163 {
164 	const struct ethtool_phy_ops *phy_ops = ethtool_phy_ops;
165 	const struct ethtool_ops *ops = dev->ethtool_ops;
166 
167 	if (sset == ETH_SS_FEATURES)
168 		return ARRAY_SIZE(netdev_features_strings);
169 
170 	if (sset == ETH_SS_RSS_HASH_FUNCS)
171 		return ARRAY_SIZE(rss_hash_func_strings);
172 
173 	if (sset == ETH_SS_TUNABLES)
174 		return ARRAY_SIZE(tunable_strings);
175 
176 	if (sset == ETH_SS_PHY_TUNABLES)
177 		return ARRAY_SIZE(phy_tunable_strings);
178 
179 	if (sset == ETH_SS_PHY_STATS && dev->phydev &&
180 	    !ops->get_ethtool_phy_stats &&
181 	    phy_ops && phy_ops->get_sset_count)
182 		return phy_ops->get_sset_count(dev->phydev);
183 
184 	if (sset == ETH_SS_LINK_MODES)
185 		return __ETHTOOL_LINK_MODE_MASK_NBITS;
186 
187 	if (ops->get_sset_count && ops->get_strings)
188 		return ops->get_sset_count(dev, sset);
189 	else
190 		return -EOPNOTSUPP;
191 }
192 
193 static void __ethtool_get_strings(struct net_device *dev,
194 	u32 stringset, u8 *data)
195 {
196 	const struct ethtool_phy_ops *phy_ops = ethtool_phy_ops;
197 	const struct ethtool_ops *ops = dev->ethtool_ops;
198 
199 	if (stringset == ETH_SS_FEATURES)
200 		memcpy(data, netdev_features_strings,
201 			sizeof(netdev_features_strings));
202 	else if (stringset == ETH_SS_RSS_HASH_FUNCS)
203 		memcpy(data, rss_hash_func_strings,
204 		       sizeof(rss_hash_func_strings));
205 	else if (stringset == ETH_SS_TUNABLES)
206 		memcpy(data, tunable_strings, sizeof(tunable_strings));
207 	else if (stringset == ETH_SS_PHY_TUNABLES)
208 		memcpy(data, phy_tunable_strings, sizeof(phy_tunable_strings));
209 	else if (stringset == ETH_SS_PHY_STATS && dev->phydev &&
210 		 !ops->get_ethtool_phy_stats && phy_ops &&
211 		 phy_ops->get_strings)
212 		phy_ops->get_strings(dev->phydev, data);
213 	else if (stringset == ETH_SS_LINK_MODES)
214 		memcpy(data, link_mode_names,
215 		       __ETHTOOL_LINK_MODE_MASK_NBITS * ETH_GSTRING_LEN);
216 	else
217 		/* ops->get_strings is valid because checked earlier */
218 		ops->get_strings(dev, stringset, data);
219 }
220 
221 static netdev_features_t ethtool_get_feature_mask(u32 eth_cmd)
222 {
223 	/* feature masks of legacy discrete ethtool ops */
224 
225 	switch (eth_cmd) {
226 	case ETHTOOL_GTXCSUM:
227 	case ETHTOOL_STXCSUM:
228 		return NETIF_F_CSUM_MASK | NETIF_F_FCOE_CRC |
229 		       NETIF_F_SCTP_CRC;
230 	case ETHTOOL_GRXCSUM:
231 	case ETHTOOL_SRXCSUM:
232 		return NETIF_F_RXCSUM;
233 	case ETHTOOL_GSG:
234 	case ETHTOOL_SSG:
235 		return NETIF_F_SG | NETIF_F_FRAGLIST;
236 	case ETHTOOL_GTSO:
237 	case ETHTOOL_STSO:
238 		return NETIF_F_ALL_TSO;
239 	case ETHTOOL_GGSO:
240 	case ETHTOOL_SGSO:
241 		return NETIF_F_GSO;
242 	case ETHTOOL_GGRO:
243 	case ETHTOOL_SGRO:
244 		return NETIF_F_GRO;
245 	default:
246 		BUG();
247 	}
248 }
249 
250 static int ethtool_get_one_feature(struct net_device *dev,
251 	char __user *useraddr, u32 ethcmd)
252 {
253 	netdev_features_t mask = ethtool_get_feature_mask(ethcmd);
254 	struct ethtool_value edata = {
255 		.cmd = ethcmd,
256 		.data = !!(dev->features & mask),
257 	};
258 
259 	if (copy_to_user(useraddr, &edata, sizeof(edata)))
260 		return -EFAULT;
261 	return 0;
262 }
263 
264 static int ethtool_set_one_feature(struct net_device *dev,
265 	void __user *useraddr, u32 ethcmd)
266 {
267 	struct ethtool_value edata;
268 	netdev_features_t mask;
269 
270 	if (copy_from_user(&edata, useraddr, sizeof(edata)))
271 		return -EFAULT;
272 
273 	mask = ethtool_get_feature_mask(ethcmd);
274 	mask &= dev->hw_features;
275 	if (!mask)
276 		return -EOPNOTSUPP;
277 
278 	if (edata.data)
279 		dev->wanted_features |= mask;
280 	else
281 		dev->wanted_features &= ~mask;
282 
283 	__netdev_update_features(dev);
284 
285 	return 0;
286 }
287 
288 #define ETH_ALL_FLAGS    (ETH_FLAG_LRO | ETH_FLAG_RXVLAN | ETH_FLAG_TXVLAN | \
289 			  ETH_FLAG_NTUPLE | ETH_FLAG_RXHASH)
290 #define ETH_ALL_FEATURES (NETIF_F_LRO | NETIF_F_HW_VLAN_CTAG_RX | \
291 			  NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_NTUPLE | \
292 			  NETIF_F_RXHASH)
293 
294 static u32 __ethtool_get_flags(struct net_device *dev)
295 {
296 	u32 flags = 0;
297 
298 	if (dev->features & NETIF_F_LRO)
299 		flags |= ETH_FLAG_LRO;
300 	if (dev->features & NETIF_F_HW_VLAN_CTAG_RX)
301 		flags |= ETH_FLAG_RXVLAN;
302 	if (dev->features & NETIF_F_HW_VLAN_CTAG_TX)
303 		flags |= ETH_FLAG_TXVLAN;
304 	if (dev->features & NETIF_F_NTUPLE)
305 		flags |= ETH_FLAG_NTUPLE;
306 	if (dev->features & NETIF_F_RXHASH)
307 		flags |= ETH_FLAG_RXHASH;
308 
309 	return flags;
310 }
311 
312 static int __ethtool_set_flags(struct net_device *dev, u32 data)
313 {
314 	netdev_features_t features = 0, changed;
315 
316 	if (data & ~ETH_ALL_FLAGS)
317 		return -EINVAL;
318 
319 	if (data & ETH_FLAG_LRO)
320 		features |= NETIF_F_LRO;
321 	if (data & ETH_FLAG_RXVLAN)
322 		features |= NETIF_F_HW_VLAN_CTAG_RX;
323 	if (data & ETH_FLAG_TXVLAN)
324 		features |= NETIF_F_HW_VLAN_CTAG_TX;
325 	if (data & ETH_FLAG_NTUPLE)
326 		features |= NETIF_F_NTUPLE;
327 	if (data & ETH_FLAG_RXHASH)
328 		features |= NETIF_F_RXHASH;
329 
330 	/* allow changing only bits set in hw_features */
331 	changed = (features ^ dev->features) & ETH_ALL_FEATURES;
332 	if (changed & ~dev->hw_features)
333 		return (changed & dev->hw_features) ? -EINVAL : -EOPNOTSUPP;
334 
335 	dev->wanted_features =
336 		(dev->wanted_features & ~changed) | (features & changed);
337 
338 	__netdev_update_features(dev);
339 
340 	return 0;
341 }
342 
343 /* Given two link masks, AND them together and save the result in dst. */
344 void ethtool_intersect_link_masks(struct ethtool_link_ksettings *dst,
345 				  struct ethtool_link_ksettings *src)
346 {
347 	unsigned int size = BITS_TO_LONGS(__ETHTOOL_LINK_MODE_MASK_NBITS);
348 	unsigned int idx = 0;
349 
350 	for (; idx < size; idx++) {
351 		dst->link_modes.supported[idx] &=
352 			src->link_modes.supported[idx];
353 		dst->link_modes.advertising[idx] &=
354 			src->link_modes.advertising[idx];
355 	}
356 }
357 EXPORT_SYMBOL(ethtool_intersect_link_masks);
358 
359 void ethtool_convert_legacy_u32_to_link_mode(unsigned long *dst,
360 					     u32 legacy_u32)
361 {
362 	linkmode_zero(dst);
363 	dst[0] = legacy_u32;
364 }
365 EXPORT_SYMBOL(ethtool_convert_legacy_u32_to_link_mode);
366 
367 /* return false if src had higher bits set. lower bits always updated. */
368 bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32,
369 					     const unsigned long *src)
370 {
371 	*legacy_u32 = src[0];
372 	return find_next_bit(src, __ETHTOOL_LINK_MODE_MASK_NBITS, 32) ==
373 		__ETHTOOL_LINK_MODE_MASK_NBITS;
374 }
375 EXPORT_SYMBOL(ethtool_convert_link_mode_to_legacy_u32);
376 
377 /* return false if ksettings link modes had higher bits
378  * set. legacy_settings always updated (best effort)
379  */
380 static bool
381 convert_link_ksettings_to_legacy_settings(
382 	struct ethtool_cmd *legacy_settings,
383 	const struct ethtool_link_ksettings *link_ksettings)
384 {
385 	bool retval = true;
386 
387 	memset(legacy_settings, 0, sizeof(*legacy_settings));
388 	/* this also clears the deprecated fields in legacy structure:
389 	 * __u8		transceiver;
390 	 * __u32	maxtxpkt;
391 	 * __u32	maxrxpkt;
392 	 */
393 
394 	retval &= ethtool_convert_link_mode_to_legacy_u32(
395 		&legacy_settings->supported,
396 		link_ksettings->link_modes.supported);
397 	retval &= ethtool_convert_link_mode_to_legacy_u32(
398 		&legacy_settings->advertising,
399 		link_ksettings->link_modes.advertising);
400 	retval &= ethtool_convert_link_mode_to_legacy_u32(
401 		&legacy_settings->lp_advertising,
402 		link_ksettings->link_modes.lp_advertising);
403 	ethtool_cmd_speed_set(legacy_settings, link_ksettings->base.speed);
404 	legacy_settings->duplex
405 		= link_ksettings->base.duplex;
406 	legacy_settings->port
407 		= link_ksettings->base.port;
408 	legacy_settings->phy_address
409 		= link_ksettings->base.phy_address;
410 	legacy_settings->autoneg
411 		= link_ksettings->base.autoneg;
412 	legacy_settings->mdio_support
413 		= link_ksettings->base.mdio_support;
414 	legacy_settings->eth_tp_mdix
415 		= link_ksettings->base.eth_tp_mdix;
416 	legacy_settings->eth_tp_mdix_ctrl
417 		= link_ksettings->base.eth_tp_mdix_ctrl;
418 	legacy_settings->transceiver
419 		= link_ksettings->base.transceiver;
420 	return retval;
421 }
422 
423 /* number of 32-bit words to store the user's link mode bitmaps */
424 #define __ETHTOOL_LINK_MODE_MASK_NU32			\
425 	DIV_ROUND_UP(__ETHTOOL_LINK_MODE_MASK_NBITS, 32)
426 
427 /* layout of the struct passed from/to userland */
428 struct ethtool_link_usettings {
429 	struct ethtool_link_settings base;
430 	struct {
431 		__u32 supported[__ETHTOOL_LINK_MODE_MASK_NU32];
432 		__u32 advertising[__ETHTOOL_LINK_MODE_MASK_NU32];
433 		__u32 lp_advertising[__ETHTOOL_LINK_MODE_MASK_NU32];
434 	} link_modes;
435 };
436 
437 /* Internal kernel helper to query a device ethtool_link_settings. */
438 int __ethtool_get_link_ksettings(struct net_device *dev,
439 				 struct ethtool_link_ksettings *link_ksettings)
440 {
441 	ASSERT_RTNL();
442 
443 	if (!dev->ethtool_ops->get_link_ksettings)
444 		return -EOPNOTSUPP;
445 
446 	if (!netif_device_present(dev))
447 		return -ENODEV;
448 
449 	memset(link_ksettings, 0, sizeof(*link_ksettings));
450 	return dev->ethtool_ops->get_link_ksettings(dev, link_ksettings);
451 }
452 EXPORT_SYMBOL(__ethtool_get_link_ksettings);
453 
454 /* convert ethtool_link_usettings in user space to a kernel internal
455  * ethtool_link_ksettings. return 0 on success, errno on error.
456  */
457 static int load_link_ksettings_from_user(struct ethtool_link_ksettings *to,
458 					 const void __user *from)
459 {
460 	struct ethtool_link_usettings link_usettings;
461 
462 	if (copy_from_user(&link_usettings, from, sizeof(link_usettings)))
463 		return -EFAULT;
464 
465 	memcpy(&to->base, &link_usettings.base, sizeof(to->base));
466 	bitmap_from_arr32(to->link_modes.supported,
467 			  link_usettings.link_modes.supported,
468 			  __ETHTOOL_LINK_MODE_MASK_NBITS);
469 	bitmap_from_arr32(to->link_modes.advertising,
470 			  link_usettings.link_modes.advertising,
471 			  __ETHTOOL_LINK_MODE_MASK_NBITS);
472 	bitmap_from_arr32(to->link_modes.lp_advertising,
473 			  link_usettings.link_modes.lp_advertising,
474 			  __ETHTOOL_LINK_MODE_MASK_NBITS);
475 
476 	return 0;
477 }
478 
479 /* Check if the user is trying to change anything besides speed/duplex */
480 bool ethtool_virtdev_validate_cmd(const struct ethtool_link_ksettings *cmd)
481 {
482 	struct ethtool_link_settings base2 = {};
483 
484 	base2.speed = cmd->base.speed;
485 	base2.port = PORT_OTHER;
486 	base2.duplex = cmd->base.duplex;
487 	base2.cmd = cmd->base.cmd;
488 	base2.link_mode_masks_nwords = cmd->base.link_mode_masks_nwords;
489 
490 	return !memcmp(&base2, &cmd->base, sizeof(base2)) &&
491 		bitmap_empty(cmd->link_modes.supported,
492 			     __ETHTOOL_LINK_MODE_MASK_NBITS) &&
493 		bitmap_empty(cmd->link_modes.lp_advertising,
494 			     __ETHTOOL_LINK_MODE_MASK_NBITS);
495 }
496 
497 /* convert a kernel internal ethtool_link_ksettings to
498  * ethtool_link_usettings in user space. return 0 on success, errno on
499  * error.
500  */
501 static int
502 store_link_ksettings_for_user(void __user *to,
503 			      const struct ethtool_link_ksettings *from)
504 {
505 	struct ethtool_link_usettings link_usettings;
506 
507 	memcpy(&link_usettings, from, sizeof(link_usettings));
508 	bitmap_to_arr32(link_usettings.link_modes.supported,
509 			from->link_modes.supported,
510 			__ETHTOOL_LINK_MODE_MASK_NBITS);
511 	bitmap_to_arr32(link_usettings.link_modes.advertising,
512 			from->link_modes.advertising,
513 			__ETHTOOL_LINK_MODE_MASK_NBITS);
514 	bitmap_to_arr32(link_usettings.link_modes.lp_advertising,
515 			from->link_modes.lp_advertising,
516 			__ETHTOOL_LINK_MODE_MASK_NBITS);
517 
518 	if (copy_to_user(to, &link_usettings, sizeof(link_usettings)))
519 		return -EFAULT;
520 
521 	return 0;
522 }
523 
524 /* Query device for its ethtool_link_settings. */
525 static int ethtool_get_link_ksettings(struct net_device *dev,
526 				      void __user *useraddr)
527 {
528 	int err = 0;
529 	struct ethtool_link_ksettings link_ksettings;
530 
531 	ASSERT_RTNL();
532 	if (!dev->ethtool_ops->get_link_ksettings)
533 		return -EOPNOTSUPP;
534 
535 	/* handle bitmap nbits handshake */
536 	if (copy_from_user(&link_ksettings.base, useraddr,
537 			   sizeof(link_ksettings.base)))
538 		return -EFAULT;
539 
540 	if (__ETHTOOL_LINK_MODE_MASK_NU32
541 	    != link_ksettings.base.link_mode_masks_nwords) {
542 		/* wrong link mode nbits requested */
543 		memset(&link_ksettings, 0, sizeof(link_ksettings));
544 		link_ksettings.base.cmd = ETHTOOL_GLINKSETTINGS;
545 		/* send back number of words required as negative val */
546 		compiletime_assert(__ETHTOOL_LINK_MODE_MASK_NU32 <= S8_MAX,
547 				   "need too many bits for link modes!");
548 		link_ksettings.base.link_mode_masks_nwords
549 			= -((s8)__ETHTOOL_LINK_MODE_MASK_NU32);
550 
551 		/* copy the base fields back to user, not the link
552 		 * mode bitmaps
553 		 */
554 		if (copy_to_user(useraddr, &link_ksettings.base,
555 				 sizeof(link_ksettings.base)))
556 			return -EFAULT;
557 
558 		return 0;
559 	}
560 
561 	/* handshake successful: user/kernel agree on
562 	 * link_mode_masks_nwords
563 	 */
564 
565 	memset(&link_ksettings, 0, sizeof(link_ksettings));
566 	err = dev->ethtool_ops->get_link_ksettings(dev, &link_ksettings);
567 	if (err < 0)
568 		return err;
569 
570 	/* make sure we tell the right values to user */
571 	link_ksettings.base.cmd = ETHTOOL_GLINKSETTINGS;
572 	link_ksettings.base.link_mode_masks_nwords
573 		= __ETHTOOL_LINK_MODE_MASK_NU32;
574 	link_ksettings.base.master_slave_cfg = MASTER_SLAVE_CFG_UNSUPPORTED;
575 	link_ksettings.base.master_slave_state = MASTER_SLAVE_STATE_UNSUPPORTED;
576 	link_ksettings.base.rate_matching = RATE_MATCH_NONE;
577 
578 	return store_link_ksettings_for_user(useraddr, &link_ksettings);
579 }
580 
581 /* Update device ethtool_link_settings. */
582 static int ethtool_set_link_ksettings(struct net_device *dev,
583 				      void __user *useraddr)
584 {
585 	struct ethtool_link_ksettings link_ksettings = {};
586 	int err;
587 
588 	ASSERT_RTNL();
589 
590 	if (!dev->ethtool_ops->set_link_ksettings)
591 		return -EOPNOTSUPP;
592 
593 	/* make sure nbits field has expected value */
594 	if (copy_from_user(&link_ksettings.base, useraddr,
595 			   sizeof(link_ksettings.base)))
596 		return -EFAULT;
597 
598 	if (__ETHTOOL_LINK_MODE_MASK_NU32
599 	    != link_ksettings.base.link_mode_masks_nwords)
600 		return -EINVAL;
601 
602 	/* copy the whole structure, now that we know it has expected
603 	 * format
604 	 */
605 	err = load_link_ksettings_from_user(&link_ksettings, useraddr);
606 	if (err)
607 		return err;
608 
609 	/* re-check nwords field, just in case */
610 	if (__ETHTOOL_LINK_MODE_MASK_NU32
611 	    != link_ksettings.base.link_mode_masks_nwords)
612 		return -EINVAL;
613 
614 	if (link_ksettings.base.master_slave_cfg ||
615 	    link_ksettings.base.master_slave_state)
616 		return -EINVAL;
617 
618 	err = dev->ethtool_ops->set_link_ksettings(dev, &link_ksettings);
619 	if (err >= 0) {
620 		ethtool_notify(dev, ETHTOOL_MSG_LINKINFO_NTF);
621 		ethtool_notify(dev, ETHTOOL_MSG_LINKMODES_NTF);
622 	}
623 	return err;
624 }
625 
626 int ethtool_virtdev_set_link_ksettings(struct net_device *dev,
627 				       const struct ethtool_link_ksettings *cmd,
628 				       u32 *dev_speed, u8 *dev_duplex)
629 {
630 	u32 speed;
631 	u8 duplex;
632 
633 	speed = cmd->base.speed;
634 	duplex = cmd->base.duplex;
635 	/* don't allow custom speed and duplex */
636 	if (!ethtool_validate_speed(speed) ||
637 	    !ethtool_validate_duplex(duplex) ||
638 	    !ethtool_virtdev_validate_cmd(cmd))
639 		return -EINVAL;
640 	*dev_speed = speed;
641 	*dev_duplex = duplex;
642 
643 	return 0;
644 }
645 EXPORT_SYMBOL(ethtool_virtdev_set_link_ksettings);
646 
647 /* Query device for its ethtool_cmd settings.
648  *
649  * Backward compatibility note: for compatibility with legacy ethtool, this is
650  * now implemented via get_link_ksettings. When driver reports higher link mode
651  * bits, a kernel warning is logged once (with name of 1st driver/device) to
652  * recommend user to upgrade ethtool, but the command is successful (only the
653  * lower link mode bits reported back to user). Deprecated fields from
654  * ethtool_cmd (transceiver/maxrxpkt/maxtxpkt) are always set to zero.
655  */
656 static int ethtool_get_settings(struct net_device *dev, void __user *useraddr)
657 {
658 	struct ethtool_link_ksettings link_ksettings;
659 	struct ethtool_cmd cmd;
660 	int err;
661 
662 	ASSERT_RTNL();
663 	if (!dev->ethtool_ops->get_link_ksettings)
664 		return -EOPNOTSUPP;
665 
666 	if (dev->ethtool->module_fw_flash_in_progress)
667 		return -EBUSY;
668 
669 	memset(&link_ksettings, 0, sizeof(link_ksettings));
670 	err = dev->ethtool_ops->get_link_ksettings(dev, &link_ksettings);
671 	if (err < 0)
672 		return err;
673 	convert_link_ksettings_to_legacy_settings(&cmd, &link_ksettings);
674 
675 	/* send a sensible cmd tag back to user */
676 	cmd.cmd = ETHTOOL_GSET;
677 
678 	if (copy_to_user(useraddr, &cmd, sizeof(cmd)))
679 		return -EFAULT;
680 
681 	return 0;
682 }
683 
684 /* Update device link settings with given ethtool_cmd.
685  *
686  * Backward compatibility note: for compatibility with legacy ethtool, this is
687  * now always implemented via set_link_settings. When user's request updates
688  * deprecated ethtool_cmd fields (transceiver/maxrxpkt/maxtxpkt), a kernel
689  * warning is logged once (with name of 1st driver/device) to recommend user to
690  * upgrade ethtool, and the request is rejected.
691  */
692 static int ethtool_set_settings(struct net_device *dev, void __user *useraddr)
693 {
694 	struct ethtool_link_ksettings link_ksettings;
695 	struct ethtool_cmd cmd;
696 	int ret;
697 
698 	ASSERT_RTNL();
699 
700 	if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
701 		return -EFAULT;
702 	if (!dev->ethtool_ops->set_link_ksettings)
703 		return -EOPNOTSUPP;
704 
705 	if (!convert_legacy_settings_to_link_ksettings(&link_ksettings, &cmd))
706 		return -EINVAL;
707 	link_ksettings.base.link_mode_masks_nwords =
708 		__ETHTOOL_LINK_MODE_MASK_NU32;
709 	ret = dev->ethtool_ops->set_link_ksettings(dev, &link_ksettings);
710 	if (ret >= 0) {
711 		ethtool_notify(dev, ETHTOOL_MSG_LINKINFO_NTF);
712 		ethtool_notify(dev, ETHTOOL_MSG_LINKMODES_NTF);
713 	}
714 	return ret;
715 }
716 
717 static int
718 ethtool_get_drvinfo(struct net_device *dev, struct ethtool_devlink_compat *rsp)
719 {
720 	const struct ethtool_ops *ops = dev->ethtool_ops;
721 	struct device *parent = dev->dev.parent;
722 
723 	rsp->info.cmd = ETHTOOL_GDRVINFO;
724 	strscpy(rsp->info.version, init_uts_ns.name.release,
725 		sizeof(rsp->info.version));
726 	if (ops->get_drvinfo) {
727 		ops->get_drvinfo(dev, &rsp->info);
728 		if (!rsp->info.bus_info[0] && parent)
729 			strscpy(rsp->info.bus_info, dev_name(parent),
730 				sizeof(rsp->info.bus_info));
731 		if (!rsp->info.driver[0] && parent && parent->driver)
732 			strscpy(rsp->info.driver, parent->driver->name,
733 				sizeof(rsp->info.driver));
734 	} else if (parent && parent->driver) {
735 		strscpy(rsp->info.bus_info, dev_name(parent),
736 			sizeof(rsp->info.bus_info));
737 		strscpy(rsp->info.driver, parent->driver->name,
738 			sizeof(rsp->info.driver));
739 	} else if (dev->rtnl_link_ops) {
740 		strscpy(rsp->info.driver, dev->rtnl_link_ops->kind,
741 			sizeof(rsp->info.driver));
742 	} else {
743 		return -EOPNOTSUPP;
744 	}
745 
746 	/*
747 	 * this method of obtaining string set info is deprecated;
748 	 * Use ETHTOOL_GSSET_INFO instead.
749 	 */
750 	if (ops->get_sset_count) {
751 		int rc;
752 
753 		rc = ops->get_sset_count(dev, ETH_SS_TEST);
754 		if (rc >= 0)
755 			rsp->info.testinfo_len = rc;
756 		rc = ops->get_sset_count(dev, ETH_SS_STATS);
757 		if (rc >= 0)
758 			rsp->info.n_stats = rc;
759 		rc = ops->get_sset_count(dev, ETH_SS_PRIV_FLAGS);
760 		if (rc >= 0)
761 			rsp->info.n_priv_flags = rc;
762 	}
763 	if (ops->get_regs_len) {
764 		int ret = ops->get_regs_len(dev);
765 
766 		if (ret > 0)
767 			rsp->info.regdump_len = ret;
768 	}
769 
770 	if (ops->get_eeprom_len)
771 		rsp->info.eedump_len = ops->get_eeprom_len(dev);
772 
773 	if (!rsp->info.fw_version[0])
774 		rsp->devlink = netdev_to_devlink_get(dev);
775 
776 	return 0;
777 }
778 
779 static noinline_for_stack int ethtool_get_sset_info(struct net_device *dev,
780 						    void __user *useraddr)
781 {
782 	struct ethtool_sset_info info;
783 	u64 sset_mask;
784 	int i, idx = 0, n_bits = 0, ret, rc;
785 	u32 *info_buf = NULL;
786 
787 	if (copy_from_user(&info, useraddr, sizeof(info)))
788 		return -EFAULT;
789 
790 	/* store copy of mask, because we zero struct later on */
791 	sset_mask = info.sset_mask;
792 	if (!sset_mask)
793 		return 0;
794 
795 	/* calculate size of return buffer */
796 	n_bits = hweight64(sset_mask);
797 
798 	memset(&info, 0, sizeof(info));
799 	info.cmd = ETHTOOL_GSSET_INFO;
800 
801 	info_buf = kcalloc(n_bits, sizeof(u32), GFP_USER);
802 	if (!info_buf)
803 		return -ENOMEM;
804 
805 	/*
806 	 * fill return buffer based on input bitmask and successful
807 	 * get_sset_count return
808 	 */
809 	for (i = 0; i < 64; i++) {
810 		if (!(sset_mask & (1ULL << i)))
811 			continue;
812 
813 		rc = __ethtool_get_sset_count(dev, i);
814 		if (rc >= 0) {
815 			info.sset_mask |= (1ULL << i);
816 			info_buf[idx++] = rc;
817 		}
818 	}
819 
820 	ret = -EFAULT;
821 	if (copy_to_user(useraddr, &info, sizeof(info)))
822 		goto out;
823 
824 	useraddr += offsetof(struct ethtool_sset_info, data);
825 	if (copy_to_user(useraddr, info_buf, array_size(idx, sizeof(u32))))
826 		goto out;
827 
828 	ret = 0;
829 
830 out:
831 	kfree(info_buf);
832 	return ret;
833 }
834 
835 static noinline_for_stack int
836 ethtool_rxnfc_copy_from_compat(struct ethtool_rxnfc *rxnfc,
837 			       const struct compat_ethtool_rxnfc __user *useraddr,
838 			       size_t size)
839 {
840 	struct compat_ethtool_rxnfc crxnfc = {};
841 
842 	/* We expect there to be holes between fs.m_ext and
843 	 * fs.ring_cookie and at the end of fs, but nowhere else.
844 	 * On non-x86, no conversion should be needed.
845 	 */
846 	BUILD_BUG_ON(!IS_ENABLED(CONFIG_X86_64) &&
847 		     sizeof(struct compat_ethtool_rxnfc) !=
848 		     sizeof(struct ethtool_rxnfc));
849 	BUILD_BUG_ON(offsetof(struct compat_ethtool_rxnfc, fs.m_ext) +
850 		     sizeof(useraddr->fs.m_ext) !=
851 		     offsetof(struct ethtool_rxnfc, fs.m_ext) +
852 		     sizeof(rxnfc->fs.m_ext));
853 	BUILD_BUG_ON(offsetof(struct compat_ethtool_rxnfc, fs.location) -
854 		     offsetof(struct compat_ethtool_rxnfc, fs.ring_cookie) !=
855 		     offsetof(struct ethtool_rxnfc, fs.location) -
856 		     offsetof(struct ethtool_rxnfc, fs.ring_cookie));
857 
858 	if (copy_from_user(&crxnfc, useraddr, min(size, sizeof(crxnfc))))
859 		return -EFAULT;
860 
861 	*rxnfc = (struct ethtool_rxnfc) {
862 		.cmd		= crxnfc.cmd,
863 		.flow_type	= crxnfc.flow_type,
864 		.data		= crxnfc.data,
865 		.fs		= {
866 			.flow_type	= crxnfc.fs.flow_type,
867 			.h_u		= crxnfc.fs.h_u,
868 			.h_ext		= crxnfc.fs.h_ext,
869 			.m_u		= crxnfc.fs.m_u,
870 			.m_ext		= crxnfc.fs.m_ext,
871 			.ring_cookie	= crxnfc.fs.ring_cookie,
872 			.location	= crxnfc.fs.location,
873 		},
874 		.rule_cnt	= crxnfc.rule_cnt,
875 	};
876 
877 	return 0;
878 }
879 
880 static int ethtool_rxnfc_copy_from_user(struct ethtool_rxnfc *rxnfc,
881 					const void __user *useraddr,
882 					size_t size)
883 {
884 	if (compat_need_64bit_alignment_fixup())
885 		return ethtool_rxnfc_copy_from_compat(rxnfc, useraddr, size);
886 
887 	if (copy_from_user(rxnfc, useraddr, size))
888 		return -EFAULT;
889 
890 	return 0;
891 }
892 
893 static int ethtool_rxnfc_copy_to_compat(void __user *useraddr,
894 					const struct ethtool_rxnfc *rxnfc,
895 					size_t size, const u32 *rule_buf)
896 {
897 	struct compat_ethtool_rxnfc crxnfc;
898 
899 	memset(&crxnfc, 0, sizeof(crxnfc));
900 	crxnfc = (struct compat_ethtool_rxnfc) {
901 		.cmd		= rxnfc->cmd,
902 		.flow_type	= rxnfc->flow_type,
903 		.data		= rxnfc->data,
904 		.fs		= {
905 			.flow_type	= rxnfc->fs.flow_type,
906 			.h_u		= rxnfc->fs.h_u,
907 			.h_ext		= rxnfc->fs.h_ext,
908 			.m_u		= rxnfc->fs.m_u,
909 			.m_ext		= rxnfc->fs.m_ext,
910 			.ring_cookie	= rxnfc->fs.ring_cookie,
911 			.location	= rxnfc->fs.location,
912 		},
913 		.rule_cnt	= rxnfc->rule_cnt,
914 	};
915 
916 	if (copy_to_user(useraddr, &crxnfc, min(size, sizeof(crxnfc))))
917 		return -EFAULT;
918 
919 	return 0;
920 }
921 
922 static int ethtool_rxnfc_copy_struct(u32 cmd, struct ethtool_rxnfc *info,
923 				     size_t *info_size, void __user *useraddr)
924 {
925 	/* struct ethtool_rxnfc was originally defined for
926 	 * ETHTOOL_{G,S}RXFH with only the cmd, flow_type and data
927 	 * members.  User-space might still be using that
928 	 * definition.
929 	 */
930 	if (cmd == ETHTOOL_GRXFH || cmd == ETHTOOL_SRXFH)
931 		*info_size = (offsetof(struct ethtool_rxnfc, data) +
932 			      sizeof(info->data));
933 
934 	if (ethtool_rxnfc_copy_from_user(info, useraddr, *info_size))
935 		return -EFAULT;
936 
937 	if ((cmd == ETHTOOL_GRXFH || cmd == ETHTOOL_SRXFH) && info->flow_type & FLOW_RSS) {
938 		*info_size = sizeof(*info);
939 		if (ethtool_rxnfc_copy_from_user(info, useraddr, *info_size))
940 			return -EFAULT;
941 		/* Since malicious users may modify the original data,
942 		 * we need to check whether FLOW_RSS is still requested.
943 		 */
944 		if (!(info->flow_type & FLOW_RSS))
945 			return -EINVAL;
946 	}
947 
948 	if (info->cmd != cmd)
949 		return -EINVAL;
950 
951 	return 0;
952 }
953 
954 static int ethtool_rxnfc_copy_to_user(void __user *useraddr,
955 				      const struct ethtool_rxnfc *rxnfc,
956 				      size_t size, const u32 *rule_buf)
957 {
958 	int ret;
959 
960 	if (compat_need_64bit_alignment_fixup()) {
961 		ret = ethtool_rxnfc_copy_to_compat(useraddr, rxnfc, size,
962 						   rule_buf);
963 		useraddr += offsetof(struct compat_ethtool_rxnfc, rule_locs);
964 	} else {
965 		ret = copy_to_user(useraddr, rxnfc, size);
966 		useraddr += offsetof(struct ethtool_rxnfc, rule_locs);
967 	}
968 
969 	if (ret)
970 		return -EFAULT;
971 
972 	if (rule_buf) {
973 		if (copy_to_user(useraddr, rule_buf,
974 				 rxnfc->rule_cnt * sizeof(u32)))
975 			return -EFAULT;
976 	}
977 
978 	return 0;
979 }
980 
981 static bool flow_type_hashable(u32 flow_type)
982 {
983 	switch (flow_type) {
984 	case ETHER_FLOW:
985 	case TCP_V4_FLOW:
986 	case UDP_V4_FLOW:
987 	case SCTP_V4_FLOW:
988 	case AH_ESP_V4_FLOW:
989 	case TCP_V6_FLOW:
990 	case UDP_V6_FLOW:
991 	case SCTP_V6_FLOW:
992 	case AH_ESP_V6_FLOW:
993 	case AH_V4_FLOW:
994 	case ESP_V4_FLOW:
995 	case AH_V6_FLOW:
996 	case ESP_V6_FLOW:
997 	case IPV4_FLOW:
998 	case IPV6_FLOW:
999 	case GTPU_V4_FLOW:
1000 	case GTPU_V6_FLOW:
1001 	case GTPC_V4_FLOW:
1002 	case GTPC_V6_FLOW:
1003 	case GTPC_TEID_V4_FLOW:
1004 	case GTPC_TEID_V6_FLOW:
1005 	case GTPU_EH_V4_FLOW:
1006 	case GTPU_EH_V6_FLOW:
1007 	case GTPU_UL_V4_FLOW:
1008 	case GTPU_UL_V6_FLOW:
1009 	case GTPU_DL_V4_FLOW:
1010 	case GTPU_DL_V6_FLOW:
1011 		return true;
1012 	}
1013 
1014 	return false;
1015 }
1016 
1017 static bool flow_type_v6(u32 flow_type)
1018 {
1019 	switch (flow_type) {
1020 	case TCP_V6_FLOW:
1021 	case UDP_V6_FLOW:
1022 	case SCTP_V6_FLOW:
1023 	case AH_ESP_V6_FLOW:
1024 	case AH_V6_FLOW:
1025 	case ESP_V6_FLOW:
1026 	case IPV6_FLOW:
1027 	case GTPU_V6_FLOW:
1028 	case GTPC_V6_FLOW:
1029 	case GTPC_TEID_V6_FLOW:
1030 	case GTPU_EH_V6_FLOW:
1031 	case GTPU_UL_V6_FLOW:
1032 	case GTPU_DL_V6_FLOW:
1033 		return true;
1034 	}
1035 
1036 	return false;
1037 }
1038 
1039 /* When adding a new type, update the assert and, if it's hashable, add it to
1040  * the flow_type_hashable switch case.
1041  */
1042 static_assert(GTPU_DL_V6_FLOW + 1 == __FLOW_TYPE_COUNT);
1043 
1044 static int ethtool_check_xfrm_rxfh(u32 input_xfrm, u64 rxfh)
1045 {
1046 	/* Sanity check: if symmetric-xor/symmetric-or-xor is set, then:
1047 	 * 1 - no other fields besides IP src/dst and/or L4 src/dst are set
1048 	 * 2 - If src is set, dst must also be set
1049 	 */
1050 	if ((input_xfrm != RXH_XFRM_NO_CHANGE &&
1051 	     input_xfrm & (RXH_XFRM_SYM_XOR | RXH_XFRM_SYM_OR_XOR)) &&
1052 	    !ethtool_rxfh_config_is_sym(rxfh))
1053 		return -EINVAL;
1054 
1055 	return 0;
1056 }
1057 
1058 static int ethtool_check_flow_types(struct net_device *dev, u32 input_xfrm)
1059 {
1060 	const struct ethtool_ops *ops = dev->ethtool_ops;
1061 	int err;
1062 	u32 i;
1063 
1064 	if (!input_xfrm || input_xfrm == RXH_XFRM_NO_CHANGE)
1065 		return 0;
1066 
1067 	for (i = 0; i < __FLOW_TYPE_COUNT; i++) {
1068 		struct ethtool_rxfh_fields fields = {
1069 			.flow_type	= i,
1070 		};
1071 
1072 		if (!flow_type_hashable(i))
1073 			continue;
1074 
1075 		if (ops->get_rxfh_fields(dev, &fields))
1076 			continue;
1077 
1078 		err = ethtool_check_xfrm_rxfh(input_xfrm, fields.data);
1079 		if (err)
1080 			return err;
1081 	}
1082 
1083 	return 0;
1084 }
1085 
1086 static noinline_for_stack int
1087 ethtool_set_rxfh_fields(struct net_device *dev, u32 cmd, void __user *useraddr)
1088 {
1089 	const struct ethtool_ops *ops = dev->ethtool_ops;
1090 	struct ethtool_rxfh_fields fields = {};
1091 	struct ethtool_rxnfc info;
1092 	size_t info_size = sizeof(info);
1093 	int rc;
1094 
1095 	if (!ops->set_rxfh_fields)
1096 		return -EOPNOTSUPP;
1097 
1098 	rc = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr);
1099 	if (rc)
1100 		return rc;
1101 
1102 	if (info.data & RXH_IP6_FL && !flow_type_v6(info.flow_type))
1103 		return -EINVAL;
1104 
1105 	if (info.flow_type & FLOW_RSS && info.rss_context &&
1106 	    !ops->rxfh_per_ctx_fields)
1107 		return -EINVAL;
1108 
1109 	mutex_lock(&dev->ethtool->rss_lock);
1110 	if (ops->get_rxfh) {
1111 		struct ethtool_rxfh_param rxfh = {};
1112 
1113 		rc = ops->get_rxfh(dev, &rxfh);
1114 		if (rc)
1115 			goto exit_unlock;
1116 
1117 		rc = ethtool_check_xfrm_rxfh(rxfh.input_xfrm, info.data);
1118 		if (rc)
1119 			goto exit_unlock;
1120 	}
1121 
1122 	fields.data = info.data;
1123 	fields.flow_type = info.flow_type & ~FLOW_RSS;
1124 	if (info.flow_type & FLOW_RSS)
1125 		fields.rss_context = info.rss_context;
1126 
1127 	rc = ops->set_rxfh_fields(dev, &fields, NULL);
1128 exit_unlock:
1129 	mutex_unlock(&dev->ethtool->rss_lock);
1130 	if (rc)
1131 		return rc;
1132 
1133 	ethtool_rss_notify(dev, ETHTOOL_MSG_RSS_NTF, fields.rss_context);
1134 	return 0;
1135 }
1136 
1137 static noinline_for_stack int
1138 ethtool_get_rxfh_fields(struct net_device *dev, u32 cmd, void __user *useraddr)
1139 {
1140 	struct ethtool_rxnfc info;
1141 	size_t info_size = sizeof(info);
1142 	const struct ethtool_ops *ops = dev->ethtool_ops;
1143 	struct ethtool_rxfh_fields fields = {};
1144 	int ret;
1145 
1146 	if (!ops->get_rxfh_fields)
1147 		return -EOPNOTSUPP;
1148 
1149 	ret = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr);
1150 	if (ret)
1151 		return ret;
1152 
1153 	if (info.flow_type & FLOW_RSS && info.rss_context &&
1154 	    !ops->rxfh_per_ctx_fields)
1155 		return -EINVAL;
1156 
1157 	fields.flow_type = info.flow_type & ~FLOW_RSS;
1158 	if (info.flow_type & FLOW_RSS)
1159 		fields.rss_context = info.rss_context;
1160 
1161 	mutex_lock(&dev->ethtool->rss_lock);
1162 	ret = ops->get_rxfh_fields(dev, &fields);
1163 	mutex_unlock(&dev->ethtool->rss_lock);
1164 	if (ret < 0)
1165 		return ret;
1166 
1167 	info.data = fields.data;
1168 
1169 	return ethtool_rxnfc_copy_to_user(useraddr, &info, info_size, NULL);
1170 }
1171 
1172 static noinline_for_stack int ethtool_set_rxnfc(struct net_device *dev,
1173 						u32 cmd, void __user *useraddr)
1174 {
1175 	const struct ethtool_ops *ops = dev->ethtool_ops;
1176 	struct ethtool_rxnfc info;
1177 	size_t info_size = sizeof(info);
1178 	int rc;
1179 
1180 	if (!ops->set_rxnfc)
1181 		return -EOPNOTSUPP;
1182 
1183 	rc = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr);
1184 	if (rc)
1185 		return rc;
1186 
1187 	if (cmd == ETHTOOL_SRXCLSRLINS && info.fs.flow_type & FLOW_RSS) {
1188 		/* Nonzero ring with RSS only makes sense
1189 		 * if NIC adds them together
1190 		 */
1191 		if (!ops->cap_rss_rxnfc_adds &&
1192 		    ethtool_get_flow_spec_ring(info.fs.ring_cookie))
1193 			return -EINVAL;
1194 
1195 		if (info.rss_context &&
1196 		    !xa_load(&dev->ethtool->rss_ctx, info.rss_context))
1197 			return -EINVAL;
1198 	}
1199 
1200 	rc = ops->set_rxnfc(dev, &info);
1201 	if (rc)
1202 		return rc;
1203 
1204 	if (cmd == ETHTOOL_SRXCLSRLINS &&
1205 	    ethtool_rxnfc_copy_to_user(useraddr, &info, info_size, NULL))
1206 		return -EFAULT;
1207 
1208 	return 0;
1209 }
1210 
1211 static noinline_for_stack int ethtool_get_rxrings(struct net_device *dev,
1212 						  u32 cmd,
1213 						  void __user *useraddr)
1214 {
1215 	struct ethtool_rxnfc info;
1216 	size_t info_size;
1217 	int ret;
1218 
1219 	info_size = sizeof(info);
1220 	ret = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr);
1221 	if (ret)
1222 		return ret;
1223 
1224 	ret = ethtool_get_rx_ring_count(dev);
1225 	if (ret < 0)
1226 		return ret;
1227 
1228 	info.data = ret;
1229 
1230 	return ethtool_rxnfc_copy_to_user(useraddr, &info, info_size, NULL);
1231 }
1232 
1233 static noinline_for_stack int ethtool_get_rxnfc(struct net_device *dev,
1234 						u32 cmd, void __user *useraddr)
1235 {
1236 	const struct ethtool_ops *ops = dev->ethtool_ops;
1237 	struct ethtool_rxnfc info;
1238 	void *rule_buf = NULL;
1239 	size_t info_size;
1240 	int ret;
1241 
1242 	if (!ops->get_rxnfc)
1243 		return -EOPNOTSUPP;
1244 
1245 	info_size = sizeof(info);
1246 	ret = ethtool_rxnfc_copy_struct(cmd, &info, &info_size, useraddr);
1247 	if (ret)
1248 		return ret;
1249 
1250 	if (info.cmd == ETHTOOL_GRXCLSRLALL) {
1251 		if (info.rule_cnt > 0) {
1252 			if (info.rule_cnt <= KMALLOC_MAX_SIZE / sizeof(u32))
1253 				rule_buf = kcalloc(info.rule_cnt, sizeof(u32),
1254 						   GFP_USER);
1255 			if (!rule_buf)
1256 				return -ENOMEM;
1257 		}
1258 	}
1259 
1260 	ret = ops->get_rxnfc(dev, &info, rule_buf);
1261 	if (ret < 0)
1262 		goto err_out;
1263 
1264 	ret = ethtool_rxnfc_copy_to_user(useraddr, &info, info_size, rule_buf);
1265 err_out:
1266 	kfree(rule_buf);
1267 
1268 	return ret;
1269 }
1270 
1271 static int ethtool_copy_validate_indir(u32 *indir, void __user *useraddr,
1272 				       int num_rx_rings,
1273 				       u32 size)
1274 {
1275 	int i;
1276 
1277 	if (copy_from_user(indir, useraddr, array_size(size, sizeof(indir[0]))))
1278 		return -EFAULT;
1279 
1280 	/* Validate ring indices */
1281 	for (i = 0; i < size; i++)
1282 		if (indir[i] >= num_rx_rings)
1283 			return -EINVAL;
1284 
1285 	return 0;
1286 }
1287 
1288 u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
1289 
1290 void netdev_rss_key_fill(void *buffer, size_t len)
1291 {
1292 	BUG_ON(len > sizeof(netdev_rss_key));
1293 	net_get_random_once(netdev_rss_key, sizeof(netdev_rss_key));
1294 	memcpy(buffer, netdev_rss_key, len);
1295 }
1296 EXPORT_SYMBOL(netdev_rss_key_fill);
1297 
1298 static noinline_for_stack int ethtool_get_rxfh_indir(struct net_device *dev,
1299 						     void __user *useraddr)
1300 {
1301 	struct ethtool_rxfh_param rxfh = {};
1302 	u32 user_size;
1303 	int ret;
1304 
1305 	if (!dev->ethtool_ops->get_rxfh_indir_size ||
1306 	    !dev->ethtool_ops->get_rxfh)
1307 		return -EOPNOTSUPP;
1308 	rxfh.indir_size = dev->ethtool_ops->get_rxfh_indir_size(dev);
1309 	if (rxfh.indir_size == 0)
1310 		return -EOPNOTSUPP;
1311 
1312 	if (copy_from_user(&user_size,
1313 			   useraddr + offsetof(struct ethtool_rxfh_indir, size),
1314 			   sizeof(user_size)))
1315 		return -EFAULT;
1316 
1317 	if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh_indir, size),
1318 			 &rxfh.indir_size, sizeof(rxfh.indir_size)))
1319 		return -EFAULT;
1320 
1321 	/* If the user buffer size is 0, this is just a query for the
1322 	 * device table size.  Otherwise, if it's smaller than the
1323 	 * device table size it's an error.
1324 	 */
1325 	if (user_size < rxfh.indir_size)
1326 		return user_size == 0 ? 0 : -EINVAL;
1327 
1328 	rxfh.indir = kcalloc(rxfh.indir_size, sizeof(rxfh.indir[0]), GFP_USER);
1329 	if (!rxfh.indir)
1330 		return -ENOMEM;
1331 
1332 	mutex_lock(&dev->ethtool->rss_lock);
1333 	ret = dev->ethtool_ops->get_rxfh(dev, &rxfh);
1334 	mutex_unlock(&dev->ethtool->rss_lock);
1335 	if (ret)
1336 		goto out;
1337 	if (copy_to_user(useraddr +
1338 			 offsetof(struct ethtool_rxfh_indir, ring_index[0]),
1339 			 rxfh.indir, rxfh.indir_size * sizeof(*rxfh.indir)))
1340 		ret = -EFAULT;
1341 
1342 out:
1343 	kfree(rxfh.indir);
1344 	return ret;
1345 }
1346 
1347 static noinline_for_stack int ethtool_set_rxfh_indir(struct net_device *dev,
1348 						     void __user *useraddr)
1349 {
1350 	const struct ethtool_ops *ops = dev->ethtool_ops;
1351 	struct ethtool_rxfh_param rxfh_dev = {};
1352 	struct netlink_ext_ack *extack = NULL;
1353 	int num_rx_rings;
1354 	u32 user_size, i;
1355 	int ret;
1356 	u32 ringidx_offset = offsetof(struct ethtool_rxfh_indir, ring_index[0]);
1357 
1358 	if (!ops->get_rxfh_indir_size || !ops->set_rxfh)
1359 		return -EOPNOTSUPP;
1360 
1361 	rxfh_dev.indir_size = ops->get_rxfh_indir_size(dev);
1362 	if (rxfh_dev.indir_size == 0)
1363 		return -EOPNOTSUPP;
1364 
1365 	if (copy_from_user(&user_size,
1366 			   useraddr + offsetof(struct ethtool_rxfh_indir, size),
1367 			   sizeof(user_size)))
1368 		return -EFAULT;
1369 
1370 	if (user_size != 0 && user_size != rxfh_dev.indir_size)
1371 		return -EINVAL;
1372 
1373 	rxfh_dev.indir = kcalloc(rxfh_dev.indir_size,
1374 				 sizeof(rxfh_dev.indir[0]), GFP_USER);
1375 	if (!rxfh_dev.indir)
1376 		return -ENOMEM;
1377 
1378 	num_rx_rings = ethtool_get_rx_ring_count(dev);
1379 	if (num_rx_rings < 0) {
1380 		ret = num_rx_rings;
1381 		goto out;
1382 	}
1383 
1384 	if (user_size == 0) {
1385 		u32 *indir = rxfh_dev.indir;
1386 
1387 		for (i = 0; i < rxfh_dev.indir_size; i++)
1388 			indir[i] = ethtool_rxfh_indir_default(i, num_rx_rings);
1389 	} else {
1390 		ret = ethtool_copy_validate_indir(rxfh_dev.indir,
1391 						  useraddr + ringidx_offset,
1392 						  num_rx_rings,
1393 						  rxfh_dev.indir_size);
1394 		if (ret)
1395 			goto out;
1396 	}
1397 
1398 	rxfh_dev.hfunc = ETH_RSS_HASH_NO_CHANGE;
1399 
1400 	mutex_lock(&dev->ethtool->rss_lock);
1401 	ret = ops->set_rxfh(dev, &rxfh_dev, extack);
1402 	if (ret)
1403 		goto out_unlock;
1404 
1405 	/* indicate whether rxfh was set to default */
1406 	if (user_size == 0)
1407 		dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
1408 	else
1409 		dev->priv_flags |= IFF_RXFH_CONFIGURED;
1410 
1411 out_unlock:
1412 	mutex_unlock(&dev->ethtool->rss_lock);
1413 out:
1414 	kfree(rxfh_dev.indir);
1415 	return ret;
1416 }
1417 
1418 static noinline_for_stack int ethtool_get_rxfh(struct net_device *dev,
1419 					       void __user *useraddr)
1420 {
1421 	const struct ethtool_ops *ops = dev->ethtool_ops;
1422 	struct ethtool_rxfh_param rxfh_dev = {};
1423 	u32 user_indir_size, user_key_size;
1424 	struct ethtool_rxfh_context *ctx;
1425 	struct ethtool_rxfh rxfh;
1426 	u32 indir_bytes;
1427 	u8 *rss_config;
1428 	u32 total_size;
1429 	int ret;
1430 
1431 	if (!ops->get_rxfh)
1432 		return -EOPNOTSUPP;
1433 
1434 	if (ops->get_rxfh_indir_size)
1435 		rxfh_dev.indir_size = ops->get_rxfh_indir_size(dev);
1436 	if (ops->get_rxfh_key_size)
1437 		rxfh_dev.key_size = ops->get_rxfh_key_size(dev);
1438 
1439 	if (copy_from_user(&rxfh, useraddr, sizeof(rxfh)))
1440 		return -EFAULT;
1441 	user_indir_size = rxfh.indir_size;
1442 	user_key_size = rxfh.key_size;
1443 
1444 	/* Check that reserved fields are 0 for now */
1445 	if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd32)
1446 		return -EINVAL;
1447 	/* Most drivers don't handle rss_context, check it's 0 as well */
1448 	if (rxfh.rss_context && !ops->create_rxfh_context)
1449 		return -EOPNOTSUPP;
1450 
1451 	rxfh.indir_size = rxfh_dev.indir_size;
1452 	rxfh.key_size = rxfh_dev.key_size;
1453 	if (copy_to_user(useraddr, &rxfh, sizeof(rxfh)))
1454 		return -EFAULT;
1455 
1456 	if ((user_indir_size && user_indir_size != rxfh_dev.indir_size) ||
1457 	    (user_key_size && user_key_size != rxfh_dev.key_size))
1458 		return -EINVAL;
1459 
1460 	indir_bytes = user_indir_size * sizeof(rxfh_dev.indir[0]);
1461 	total_size = indir_bytes + user_key_size;
1462 	rss_config = kzalloc(total_size, GFP_USER);
1463 	if (!rss_config)
1464 		return -ENOMEM;
1465 
1466 	if (user_indir_size)
1467 		rxfh_dev.indir = (u32 *)rss_config;
1468 
1469 	if (user_key_size)
1470 		rxfh_dev.key = rss_config + indir_bytes;
1471 
1472 	mutex_lock(&dev->ethtool->rss_lock);
1473 	if (rxfh.rss_context) {
1474 		ctx = xa_load(&dev->ethtool->rss_ctx, rxfh.rss_context);
1475 		if (!ctx) {
1476 			ret = -ENOENT;
1477 			goto out;
1478 		}
1479 		if (rxfh_dev.indir)
1480 			memcpy(rxfh_dev.indir, ethtool_rxfh_context_indir(ctx),
1481 			       indir_bytes);
1482 		if (!ops->rxfh_per_ctx_key) {
1483 			rxfh_dev.key_size = 0;
1484 		} else {
1485 			if (rxfh_dev.key)
1486 				memcpy(rxfh_dev.key,
1487 				       ethtool_rxfh_context_key(ctx),
1488 				       user_key_size);
1489 			rxfh_dev.hfunc = ctx->hfunc;
1490 		}
1491 		rxfh_dev.input_xfrm = ctx->input_xfrm;
1492 		ret = 0;
1493 	} else {
1494 		ret = dev->ethtool_ops->get_rxfh(dev, &rxfh_dev);
1495 		if (ret)
1496 			goto out;
1497 	}
1498 
1499 	if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh, hfunc),
1500 			 &rxfh_dev.hfunc, sizeof(rxfh.hfunc))) {
1501 		ret = -EFAULT;
1502 	} else if (copy_to_user(useraddr +
1503 				offsetof(struct ethtool_rxfh, input_xfrm),
1504 				&rxfh_dev.input_xfrm,
1505 				sizeof(rxfh.input_xfrm))) {
1506 		ret = -EFAULT;
1507 	} else if (copy_to_user(useraddr +
1508 				offsetof(struct ethtool_rxfh, key_size),
1509 				&rxfh_dev.key_size,
1510 				sizeof(rxfh.key_size))) {
1511 		ret = -EFAULT;
1512 	} else if (copy_to_user(useraddr +
1513 			      offsetof(struct ethtool_rxfh, rss_config[0]),
1514 			      rss_config, total_size)) {
1515 		ret = -EFAULT;
1516 	}
1517 out:
1518 	mutex_unlock(&dev->ethtool->rss_lock);
1519 	kfree(rss_config);
1520 
1521 	return ret;
1522 }
1523 
1524 static noinline_for_stack int ethtool_set_rxfh(struct net_device *dev,
1525 					       void __user *useraddr)
1526 {
1527 	u32 rss_cfg_offset = offsetof(struct ethtool_rxfh, rss_config[0]);
1528 	const struct ethtool_ops *ops = dev->ethtool_ops;
1529 	u32 dev_indir_size = 0, dev_key_size = 0, i;
1530 	u32 user_indir_len = 0, indir_bytes = 0;
1531 	struct ethtool_rxfh_param rxfh_dev = {};
1532 	struct ethtool_rxfh_context *ctx = NULL;
1533 	struct netlink_ext_ack *extack = NULL;
1534 	struct ethtool_rxfh rxfh;
1535 	bool create = false;
1536 	int num_rx_rings;
1537 	u8 *rss_config;
1538 	int ntf = 0;
1539 	int ret;
1540 
1541 	if (!ops->set_rxfh)
1542 		return -EOPNOTSUPP;
1543 
1544 	if (ops->get_rxfh_indir_size)
1545 		dev_indir_size = ops->get_rxfh_indir_size(dev);
1546 	if (ops->get_rxfh_key_size)
1547 		dev_key_size = ops->get_rxfh_key_size(dev);
1548 
1549 	if (copy_from_user(&rxfh, useraddr, sizeof(rxfh)))
1550 		return -EFAULT;
1551 
1552 	/* Check that reserved fields are 0 for now */
1553 	if (rxfh.rsvd8[0] || rxfh.rsvd8[1] || rxfh.rsvd32)
1554 		return -EINVAL;
1555 	/* Most drivers don't handle rss_context, check it's 0 as well */
1556 	if (rxfh.rss_context && !ops->create_rxfh_context)
1557 		return -EOPNOTSUPP;
1558 	/* Check input data transformation capabilities */
1559 	if (rxfh.input_xfrm && rxfh.input_xfrm != RXH_XFRM_SYM_XOR &&
1560 	    rxfh.input_xfrm != RXH_XFRM_SYM_OR_XOR &&
1561 	    rxfh.input_xfrm != RXH_XFRM_NO_CHANGE)
1562 		return -EINVAL;
1563 	if (rxfh.input_xfrm != RXH_XFRM_NO_CHANGE &&
1564 	    rxfh.input_xfrm & ~ops->supported_input_xfrm)
1565 		return -EOPNOTSUPP;
1566 	create = rxfh.rss_context == ETH_RXFH_CONTEXT_ALLOC;
1567 
1568 	if ((rxfh.indir_size &&
1569 	     rxfh.indir_size != ETH_RXFH_INDIR_NO_CHANGE &&
1570 	     rxfh.indir_size != dev_indir_size) ||
1571 	    (rxfh.key_size && rxfh.key_size != dev_key_size))
1572 		return -EINVAL;
1573 
1574 	/* Must request at least one change: indir size, hash key, function
1575 	 * or input transformation.
1576 	 * There's no need for any of it in case of context creation.
1577 	 */
1578 	if (!create &&
1579 	    (rxfh.indir_size == ETH_RXFH_INDIR_NO_CHANGE &&
1580 	     rxfh.key_size == 0 && rxfh.hfunc == ETH_RSS_HASH_NO_CHANGE &&
1581 	     rxfh.input_xfrm == RXH_XFRM_NO_CHANGE))
1582 		return -EINVAL;
1583 
1584 	indir_bytes = dev_indir_size * sizeof(rxfh_dev.indir[0]);
1585 
1586 	/* Check settings which may be global rather than per RSS-context */
1587 	if (rxfh.rss_context && !ops->rxfh_per_ctx_key)
1588 		if (rxfh.key_size ||
1589 		    (rxfh.hfunc && rxfh.hfunc != ETH_RSS_HASH_NO_CHANGE) ||
1590 		    (rxfh.input_xfrm && rxfh.input_xfrm != RXH_XFRM_NO_CHANGE))
1591 			return -EOPNOTSUPP;
1592 
1593 	rss_config = kzalloc(indir_bytes + dev_key_size, GFP_USER);
1594 	if (!rss_config)
1595 		return -ENOMEM;
1596 
1597 	num_rx_rings = ethtool_get_rx_ring_count(dev);
1598 	if (num_rx_rings < 0) {
1599 		ret = num_rx_rings;
1600 		goto out_free;
1601 	}
1602 
1603 	/* rxfh.indir_size == 0 means reset the indir table to default (master
1604 	 * context) or delete the context (other RSS contexts).
1605 	 * rxfh.indir_size == ETH_RXFH_INDIR_NO_CHANGE means leave it unchanged.
1606 	 */
1607 	if (rxfh.indir_size &&
1608 	    rxfh.indir_size != ETH_RXFH_INDIR_NO_CHANGE) {
1609 		user_indir_len = indir_bytes;
1610 		rxfh_dev.indir = (u32 *)rss_config;
1611 		rxfh_dev.indir_size = dev_indir_size;
1612 		ret = ethtool_copy_validate_indir(rxfh_dev.indir,
1613 						  useraddr + rss_cfg_offset,
1614 						  num_rx_rings,
1615 						  rxfh.indir_size);
1616 		if (ret)
1617 			goto out_free;
1618 	} else if (rxfh.indir_size == 0) {
1619 		if (rxfh.rss_context == 0) {
1620 			u32 *indir;
1621 
1622 			rxfh_dev.indir = (u32 *)rss_config;
1623 			rxfh_dev.indir_size = dev_indir_size;
1624 			indir = rxfh_dev.indir;
1625 			for (i = 0; i < dev_indir_size; i++)
1626 				indir[i] =
1627 					ethtool_rxfh_indir_default(i, num_rx_rings);
1628 		} else {
1629 			rxfh_dev.rss_delete = true;
1630 		}
1631 	}
1632 
1633 	if (rxfh.key_size) {
1634 		rxfh_dev.key_size = dev_key_size;
1635 		rxfh_dev.key = rss_config + indir_bytes;
1636 		if (copy_from_user(rxfh_dev.key,
1637 				   useraddr + rss_cfg_offset + user_indir_len,
1638 				   rxfh.key_size)) {
1639 			ret = -EFAULT;
1640 			goto out_free;
1641 		}
1642 	}
1643 
1644 	mutex_lock(&dev->ethtool->rss_lock);
1645 
1646 	ret = ethtool_check_flow_types(dev, rxfh.input_xfrm);
1647 	if (ret)
1648 		goto out_unlock;
1649 
1650 	if (rxfh.rss_context && rxfh_dev.rss_delete) {
1651 		ret = ethtool_check_rss_ctx_busy(dev, rxfh.rss_context);
1652 		if (ret)
1653 			goto out_unlock;
1654 	}
1655 
1656 	if (create) {
1657 		u32 limit, ctx_id;
1658 
1659 		if (rxfh_dev.rss_delete) {
1660 			ret = -EINVAL;
1661 			goto out_unlock;
1662 		}
1663 		ctx = ethtool_rxfh_ctx_alloc(ops, dev_indir_size, dev_key_size);
1664 		if (!ctx) {
1665 			ret = -ENOMEM;
1666 			goto out_unlock;
1667 		}
1668 
1669 		limit = ops->rxfh_max_num_contexts ?: U32_MAX;
1670 		ret = xa_alloc(&dev->ethtool->rss_ctx, &ctx_id, ctx,
1671 			       XA_LIMIT(1, limit - 1), GFP_KERNEL_ACCOUNT);
1672 		if (ret < 0) {
1673 			kfree(ctx);
1674 			goto out_unlock;
1675 		}
1676 		WARN_ON(!ctx_id); /* can't happen */
1677 		rxfh.rss_context = ctx_id;
1678 	} else if (rxfh.rss_context) {
1679 		ctx = xa_load(&dev->ethtool->rss_ctx, rxfh.rss_context);
1680 		if (!ctx) {
1681 			ret = -ENOENT;
1682 			goto out_unlock;
1683 		}
1684 	}
1685 	rxfh_dev.hfunc = rxfh.hfunc;
1686 	rxfh_dev.rss_context = rxfh.rss_context;
1687 	rxfh_dev.input_xfrm = rxfh.input_xfrm;
1688 
1689 	if (!rxfh.rss_context) {
1690 		ntf = ETHTOOL_MSG_RSS_NTF;
1691 		ret = ops->set_rxfh(dev, &rxfh_dev, extack);
1692 	} else if (create) {
1693 		ntf = ETHTOOL_MSG_RSS_CREATE_NTF;
1694 		ret = ops->create_rxfh_context(dev, ctx, &rxfh_dev, extack);
1695 		/* Make sure driver populates defaults */
1696 		WARN_ON_ONCE(!ret && !rxfh_dev.key && ops->rxfh_per_ctx_key &&
1697 			     !memchr_inv(ethtool_rxfh_context_key(ctx), 0,
1698 					 ctx->key_size));
1699 	} else if (rxfh_dev.rss_delete) {
1700 		ntf = ETHTOOL_MSG_RSS_DELETE_NTF;
1701 		ret = ops->remove_rxfh_context(dev, ctx, rxfh.rss_context,
1702 					       extack);
1703 	} else {
1704 		ntf = ETHTOOL_MSG_RSS_NTF;
1705 		ret = ops->modify_rxfh_context(dev, ctx, &rxfh_dev, extack);
1706 	}
1707 	if (ret) {
1708 		ntf = 0;
1709 		if (create) {
1710 			/* failed to create, free our new tracking entry */
1711 			xa_erase(&dev->ethtool->rss_ctx, rxfh.rss_context);
1712 			kfree(ctx);
1713 		}
1714 		goto out_unlock;
1715 	}
1716 
1717 	if (copy_to_user(useraddr + offsetof(struct ethtool_rxfh, rss_context),
1718 			 &rxfh_dev.rss_context, sizeof(rxfh_dev.rss_context)))
1719 		ret = -EFAULT;
1720 
1721 	if (!rxfh_dev.rss_context) {
1722 		/* indicate whether rxfh was set to default */
1723 		if (rxfh.indir_size == 0)
1724 			dev->priv_flags &= ~IFF_RXFH_CONFIGURED;
1725 		else if (rxfh.indir_size != ETH_RXFH_INDIR_NO_CHANGE)
1726 			dev->priv_flags |= IFF_RXFH_CONFIGURED;
1727 	}
1728 	/* Update rss_ctx tracking */
1729 	if (rxfh_dev.rss_delete) {
1730 		WARN_ON(xa_erase(&dev->ethtool->rss_ctx, rxfh.rss_context) != ctx);
1731 		kfree(ctx);
1732 	} else if (ctx) {
1733 		if (rxfh_dev.indir) {
1734 			for (i = 0; i < dev_indir_size; i++)
1735 				ethtool_rxfh_context_indir(ctx)[i] = rxfh_dev.indir[i];
1736 			ctx->indir_configured =
1737 				rxfh.indir_size &&
1738 				rxfh.indir_size != ETH_RXFH_INDIR_NO_CHANGE;
1739 		}
1740 		if (rxfh_dev.key) {
1741 			memcpy(ethtool_rxfh_context_key(ctx), rxfh_dev.key,
1742 			       dev_key_size);
1743 			ctx->key_configured = !!rxfh.key_size;
1744 		}
1745 		if (rxfh_dev.hfunc != ETH_RSS_HASH_NO_CHANGE)
1746 			ctx->hfunc = rxfh_dev.hfunc;
1747 		if (rxfh_dev.input_xfrm != RXH_XFRM_NO_CHANGE)
1748 			ctx->input_xfrm = rxfh_dev.input_xfrm;
1749 	}
1750 
1751 out_unlock:
1752 	mutex_unlock(&dev->ethtool->rss_lock);
1753 out_free:
1754 	kfree(rss_config);
1755 	if (ntf)
1756 		ethtool_rss_notify(dev, ntf, rxfh.rss_context);
1757 	return ret;
1758 }
1759 
1760 static int ethtool_get_regs(struct net_device *dev, char __user *useraddr)
1761 {
1762 	struct ethtool_regs regs;
1763 	const struct ethtool_ops *ops = dev->ethtool_ops;
1764 	void *regbuf;
1765 	int reglen, ret;
1766 
1767 	if (!ops->get_regs || !ops->get_regs_len)
1768 		return -EOPNOTSUPP;
1769 
1770 	if (copy_from_user(&regs, useraddr, sizeof(regs)))
1771 		return -EFAULT;
1772 
1773 	reglen = ops->get_regs_len(dev);
1774 	if (reglen <= 0)
1775 		return reglen;
1776 
1777 	if (regs.len > reglen)
1778 		regs.len = reglen;
1779 
1780 	regbuf = vzalloc(reglen);
1781 	if (!regbuf)
1782 		return -ENOMEM;
1783 
1784 	if (regs.len < reglen)
1785 		reglen = regs.len;
1786 
1787 	ops->get_regs(dev, &regs, regbuf);
1788 
1789 	ret = -EFAULT;
1790 	if (copy_to_user(useraddr, &regs, sizeof(regs)))
1791 		goto out;
1792 	useraddr += offsetof(struct ethtool_regs, data);
1793 	if (copy_to_user(useraddr, regbuf, reglen))
1794 		goto out;
1795 	ret = 0;
1796 
1797  out:
1798 	vfree(regbuf);
1799 	return ret;
1800 }
1801 
1802 static int ethtool_reset(struct net_device *dev, char __user *useraddr)
1803 {
1804 	struct ethtool_value reset;
1805 	int ret;
1806 
1807 	if (!dev->ethtool_ops->reset)
1808 		return -EOPNOTSUPP;
1809 
1810 	if (dev->ethtool->module_fw_flash_in_progress)
1811 		return -EBUSY;
1812 
1813 	if (copy_from_user(&reset, useraddr, sizeof(reset)))
1814 		return -EFAULT;
1815 
1816 	ret = dev->ethtool_ops->reset(dev, &reset.data);
1817 	if (ret)
1818 		return ret;
1819 
1820 	if (copy_to_user(useraddr, &reset, sizeof(reset)))
1821 		return -EFAULT;
1822 	return 0;
1823 }
1824 
1825 static int ethtool_get_wol(struct net_device *dev, char __user *useraddr)
1826 {
1827 	struct ethtool_wolinfo wol;
1828 
1829 	if (!dev->ethtool_ops->get_wol)
1830 		return -EOPNOTSUPP;
1831 
1832 	memset(&wol, 0, sizeof(struct ethtool_wolinfo));
1833 	wol.cmd = ETHTOOL_GWOL;
1834 	dev->ethtool_ops->get_wol(dev, &wol);
1835 
1836 	if (copy_to_user(useraddr, &wol, sizeof(wol)))
1837 		return -EFAULT;
1838 	return 0;
1839 }
1840 
1841 static int ethtool_set_wol(struct net_device *dev, char __user *useraddr)
1842 {
1843 	struct ethtool_wolinfo wol, cur_wol;
1844 	int ret;
1845 
1846 	if (!dev->ethtool_ops->get_wol || !dev->ethtool_ops->set_wol)
1847 		return -EOPNOTSUPP;
1848 
1849 	memset(&cur_wol, 0, sizeof(struct ethtool_wolinfo));
1850 	cur_wol.cmd = ETHTOOL_GWOL;
1851 	dev->ethtool_ops->get_wol(dev, &cur_wol);
1852 
1853 	if (copy_from_user(&wol, useraddr, sizeof(wol)))
1854 		return -EFAULT;
1855 
1856 	if (wol.wolopts & ~cur_wol.supported)
1857 		return -EINVAL;
1858 
1859 	if (wol.wolopts == cur_wol.wolopts &&
1860 	    !memcmp(wol.sopass, cur_wol.sopass, sizeof(wol.sopass)))
1861 		return 0;
1862 
1863 	ret = dev->ethtool_ops->set_wol(dev, &wol);
1864 	if (ret)
1865 		return ret;
1866 
1867 	dev->ethtool->wol_enabled = !!wol.wolopts;
1868 	ethtool_notify(dev, ETHTOOL_MSG_WOL_NTF);
1869 
1870 	return 0;
1871 }
1872 
1873 static void eee_to_keee(struct ethtool_keee *keee,
1874 			const struct ethtool_eee *eee)
1875 {
1876 	memset(keee, 0, sizeof(*keee));
1877 
1878 	keee->eee_enabled = eee->eee_enabled;
1879 	keee->tx_lpi_enabled = eee->tx_lpi_enabled;
1880 	keee->tx_lpi_timer = eee->tx_lpi_timer;
1881 
1882 	ethtool_convert_legacy_u32_to_link_mode(keee->advertised,
1883 						eee->advertised);
1884 }
1885 
1886 static void keee_to_eee(struct ethtool_eee *eee,
1887 			const struct ethtool_keee *keee)
1888 {
1889 	bool overflow;
1890 
1891 	memset(eee, 0, sizeof(*eee));
1892 
1893 	eee->eee_active = keee->eee_active;
1894 	eee->eee_enabled = keee->eee_enabled;
1895 	eee->tx_lpi_enabled = keee->tx_lpi_enabled;
1896 	eee->tx_lpi_timer = keee->tx_lpi_timer;
1897 
1898 	overflow = !ethtool_convert_link_mode_to_legacy_u32(&eee->supported,
1899 							    keee->supported);
1900 	ethtool_convert_link_mode_to_legacy_u32(&eee->advertised,
1901 						keee->advertised);
1902 	ethtool_convert_link_mode_to_legacy_u32(&eee->lp_advertised,
1903 						keee->lp_advertised);
1904 	if (overflow)
1905 		pr_warn("Ethtool ioctl interface doesn't support passing EEE linkmodes beyond bit 32\n");
1906 }
1907 
1908 static int ethtool_get_eee(struct net_device *dev, char __user *useraddr)
1909 {
1910 	struct ethtool_keee keee;
1911 	struct ethtool_eee eee;
1912 	int rc;
1913 
1914 	if (!dev->ethtool_ops->get_eee)
1915 		return -EOPNOTSUPP;
1916 
1917 	memset(&keee, 0, sizeof(keee));
1918 	rc = dev->ethtool_ops->get_eee(dev, &keee);
1919 	if (rc)
1920 		return rc;
1921 
1922 	keee_to_eee(&eee, &keee);
1923 	if (copy_to_user(useraddr, &eee, sizeof(eee)))
1924 		return -EFAULT;
1925 
1926 	return 0;
1927 }
1928 
1929 static int ethtool_set_eee(struct net_device *dev, char __user *useraddr)
1930 {
1931 	struct ethtool_keee keee;
1932 	struct ethtool_eee eee;
1933 	int ret;
1934 
1935 	if (!dev->ethtool_ops->set_eee)
1936 		return -EOPNOTSUPP;
1937 
1938 	if (copy_from_user(&eee, useraddr, sizeof(eee)))
1939 		return -EFAULT;
1940 
1941 	eee_to_keee(&keee, &eee);
1942 	ret = dev->ethtool_ops->set_eee(dev, &keee);
1943 	if (!ret)
1944 		ethtool_notify(dev, ETHTOOL_MSG_EEE_NTF);
1945 	return ret;
1946 }
1947 
1948 static int ethtool_nway_reset(struct net_device *dev)
1949 {
1950 	if (!dev->ethtool_ops->nway_reset)
1951 		return -EOPNOTSUPP;
1952 
1953 	return dev->ethtool_ops->nway_reset(dev);
1954 }
1955 
1956 static int ethtool_get_link(struct net_device *dev, char __user *useraddr)
1957 {
1958 	struct ethtool_value edata = { .cmd = ETHTOOL_GLINK };
1959 	int link = __ethtool_get_link(dev);
1960 
1961 	if (link < 0)
1962 		return link;
1963 
1964 	edata.data = link;
1965 	if (copy_to_user(useraddr, &edata, sizeof(edata)))
1966 		return -EFAULT;
1967 	return 0;
1968 }
1969 
1970 static int ethtool_get_any_eeprom(struct net_device *dev, void __user *useraddr,
1971 				  int (*getter)(struct net_device *,
1972 						struct ethtool_eeprom *, u8 *),
1973 				  u32 total_len)
1974 {
1975 	struct ethtool_eeprom eeprom;
1976 	void __user *userbuf = useraddr + sizeof(eeprom);
1977 	u32 bytes_remaining;
1978 	u8 *data;
1979 	int ret = 0;
1980 
1981 	if (copy_from_user(&eeprom, useraddr, sizeof(eeprom)))
1982 		return -EFAULT;
1983 
1984 	/* Check for wrap and zero */
1985 	if (eeprom.offset + eeprom.len <= eeprom.offset)
1986 		return -EINVAL;
1987 
1988 	/* Check for exceeding total eeprom len */
1989 	if (eeprom.offset + eeprom.len > total_len)
1990 		return -EINVAL;
1991 
1992 	data = kzalloc(PAGE_SIZE, GFP_USER);
1993 	if (!data)
1994 		return -ENOMEM;
1995 
1996 	bytes_remaining = eeprom.len;
1997 	while (bytes_remaining > 0) {
1998 		eeprom.len = min(bytes_remaining, (u32)PAGE_SIZE);
1999 
2000 		ret = getter(dev, &eeprom, data);
2001 		if (ret)
2002 			break;
2003 		if (!eeprom.len) {
2004 			ret = -EIO;
2005 			break;
2006 		}
2007 		if (copy_to_user(userbuf, data, eeprom.len)) {
2008 			ret = -EFAULT;
2009 			break;
2010 		}
2011 		userbuf += eeprom.len;
2012 		eeprom.offset += eeprom.len;
2013 		bytes_remaining -= eeprom.len;
2014 	}
2015 
2016 	eeprom.len = userbuf - (useraddr + sizeof(eeprom));
2017 	eeprom.offset -= eeprom.len;
2018 	if (copy_to_user(useraddr, &eeprom, sizeof(eeprom)))
2019 		ret = -EFAULT;
2020 
2021 	kfree(data);
2022 	return ret;
2023 }
2024 
2025 static int ethtool_get_eeprom(struct net_device *dev, void __user *useraddr)
2026 {
2027 	const struct ethtool_ops *ops = dev->ethtool_ops;
2028 
2029 	if (!ops->get_eeprom || !ops->get_eeprom_len ||
2030 	    !ops->get_eeprom_len(dev))
2031 		return -EOPNOTSUPP;
2032 
2033 	return ethtool_get_any_eeprom(dev, useraddr, ops->get_eeprom,
2034 				      ops->get_eeprom_len(dev));
2035 }
2036 
2037 static int ethtool_set_eeprom(struct net_device *dev, void __user *useraddr)
2038 {
2039 	struct ethtool_eeprom eeprom;
2040 	const struct ethtool_ops *ops = dev->ethtool_ops;
2041 	void __user *userbuf = useraddr + sizeof(eeprom);
2042 	u32 bytes_remaining;
2043 	u8 *data;
2044 	int ret = 0;
2045 
2046 	if (!ops->set_eeprom || !ops->get_eeprom_len ||
2047 	    !ops->get_eeprom_len(dev))
2048 		return -EOPNOTSUPP;
2049 
2050 	if (copy_from_user(&eeprom, useraddr, sizeof(eeprom)))
2051 		return -EFAULT;
2052 
2053 	/* Check for wrap and zero */
2054 	if (eeprom.offset + eeprom.len <= eeprom.offset)
2055 		return -EINVAL;
2056 
2057 	/* Check for exceeding total eeprom len */
2058 	if (eeprom.offset + eeprom.len > ops->get_eeprom_len(dev))
2059 		return -EINVAL;
2060 
2061 	data = kzalloc(PAGE_SIZE, GFP_USER);
2062 	if (!data)
2063 		return -ENOMEM;
2064 
2065 	bytes_remaining = eeprom.len;
2066 	while (bytes_remaining > 0) {
2067 		eeprom.len = min(bytes_remaining, (u32)PAGE_SIZE);
2068 
2069 		if (copy_from_user(data, userbuf, eeprom.len)) {
2070 			ret = -EFAULT;
2071 			break;
2072 		}
2073 		ret = ops->set_eeprom(dev, &eeprom, data);
2074 		if (ret)
2075 			break;
2076 		userbuf += eeprom.len;
2077 		eeprom.offset += eeprom.len;
2078 		bytes_remaining -= eeprom.len;
2079 	}
2080 
2081 	kfree(data);
2082 	return ret;
2083 }
2084 
2085 static noinline_for_stack int ethtool_get_coalesce(struct net_device *dev,
2086 						   void __user *useraddr)
2087 {
2088 	struct ethtool_coalesce coalesce = { .cmd = ETHTOOL_GCOALESCE };
2089 	struct kernel_ethtool_coalesce kernel_coalesce = {};
2090 	int ret;
2091 
2092 	if (!dev->ethtool_ops->get_coalesce)
2093 		return -EOPNOTSUPP;
2094 
2095 	ret = dev->ethtool_ops->get_coalesce(dev, &coalesce, &kernel_coalesce,
2096 					     NULL);
2097 	if (ret)
2098 		return ret;
2099 
2100 	if (copy_to_user(useraddr, &coalesce, sizeof(coalesce)))
2101 		return -EFAULT;
2102 	return 0;
2103 }
2104 
2105 static bool
2106 ethtool_set_coalesce_supported(struct net_device *dev,
2107 			       struct ethtool_coalesce *coalesce)
2108 {
2109 	u32 supported_params = dev->ethtool_ops->supported_coalesce_params;
2110 	u32 nonzero_params = 0;
2111 
2112 	if (coalesce->rx_coalesce_usecs)
2113 		nonzero_params |= ETHTOOL_COALESCE_RX_USECS;
2114 	if (coalesce->rx_max_coalesced_frames)
2115 		nonzero_params |= ETHTOOL_COALESCE_RX_MAX_FRAMES;
2116 	if (coalesce->rx_coalesce_usecs_irq)
2117 		nonzero_params |= ETHTOOL_COALESCE_RX_USECS_IRQ;
2118 	if (coalesce->rx_max_coalesced_frames_irq)
2119 		nonzero_params |= ETHTOOL_COALESCE_RX_MAX_FRAMES_IRQ;
2120 	if (coalesce->tx_coalesce_usecs)
2121 		nonzero_params |= ETHTOOL_COALESCE_TX_USECS;
2122 	if (coalesce->tx_max_coalesced_frames)
2123 		nonzero_params |= ETHTOOL_COALESCE_TX_MAX_FRAMES;
2124 	if (coalesce->tx_coalesce_usecs_irq)
2125 		nonzero_params |= ETHTOOL_COALESCE_TX_USECS_IRQ;
2126 	if (coalesce->tx_max_coalesced_frames_irq)
2127 		nonzero_params |= ETHTOOL_COALESCE_TX_MAX_FRAMES_IRQ;
2128 	if (coalesce->stats_block_coalesce_usecs)
2129 		nonzero_params |= ETHTOOL_COALESCE_STATS_BLOCK_USECS;
2130 	if (coalesce->use_adaptive_rx_coalesce)
2131 		nonzero_params |= ETHTOOL_COALESCE_USE_ADAPTIVE_RX;
2132 	if (coalesce->use_adaptive_tx_coalesce)
2133 		nonzero_params |= ETHTOOL_COALESCE_USE_ADAPTIVE_TX;
2134 	if (coalesce->pkt_rate_low)
2135 		nonzero_params |= ETHTOOL_COALESCE_PKT_RATE_LOW;
2136 	if (coalesce->rx_coalesce_usecs_low)
2137 		nonzero_params |= ETHTOOL_COALESCE_RX_USECS_LOW;
2138 	if (coalesce->rx_max_coalesced_frames_low)
2139 		nonzero_params |= ETHTOOL_COALESCE_RX_MAX_FRAMES_LOW;
2140 	if (coalesce->tx_coalesce_usecs_low)
2141 		nonzero_params |= ETHTOOL_COALESCE_TX_USECS_LOW;
2142 	if (coalesce->tx_max_coalesced_frames_low)
2143 		nonzero_params |= ETHTOOL_COALESCE_TX_MAX_FRAMES_LOW;
2144 	if (coalesce->pkt_rate_high)
2145 		nonzero_params |= ETHTOOL_COALESCE_PKT_RATE_HIGH;
2146 	if (coalesce->rx_coalesce_usecs_high)
2147 		nonzero_params |= ETHTOOL_COALESCE_RX_USECS_HIGH;
2148 	if (coalesce->rx_max_coalesced_frames_high)
2149 		nonzero_params |= ETHTOOL_COALESCE_RX_MAX_FRAMES_HIGH;
2150 	if (coalesce->tx_coalesce_usecs_high)
2151 		nonzero_params |= ETHTOOL_COALESCE_TX_USECS_HIGH;
2152 	if (coalesce->tx_max_coalesced_frames_high)
2153 		nonzero_params |= ETHTOOL_COALESCE_TX_MAX_FRAMES_HIGH;
2154 	if (coalesce->rate_sample_interval)
2155 		nonzero_params |= ETHTOOL_COALESCE_RATE_SAMPLE_INTERVAL;
2156 
2157 	return (supported_params & nonzero_params) == nonzero_params;
2158 }
2159 
2160 static noinline_for_stack int ethtool_set_coalesce(struct net_device *dev,
2161 						   void __user *useraddr)
2162 {
2163 	struct kernel_ethtool_coalesce kernel_coalesce = {};
2164 	struct ethtool_coalesce coalesce;
2165 	int ret;
2166 
2167 	if (!dev->ethtool_ops->set_coalesce || !dev->ethtool_ops->get_coalesce)
2168 		return -EOPNOTSUPP;
2169 
2170 	ret = dev->ethtool_ops->get_coalesce(dev, &coalesce, &kernel_coalesce,
2171 					     NULL);
2172 	if (ret)
2173 		return ret;
2174 
2175 	if (copy_from_user(&coalesce, useraddr, sizeof(coalesce)))
2176 		return -EFAULT;
2177 
2178 	if (!ethtool_set_coalesce_supported(dev, &coalesce))
2179 		return -EOPNOTSUPP;
2180 
2181 	ret = dev->ethtool_ops->set_coalesce(dev, &coalesce, &kernel_coalesce,
2182 					     NULL);
2183 	if (!ret)
2184 		ethtool_notify(dev, ETHTOOL_MSG_COALESCE_NTF);
2185 	return ret;
2186 }
2187 
2188 static int ethtool_get_ringparam(struct net_device *dev, void __user *useraddr)
2189 {
2190 	struct ethtool_ringparam ringparam = { .cmd = ETHTOOL_GRINGPARAM };
2191 	struct kernel_ethtool_ringparam kernel_ringparam = {};
2192 
2193 	if (!dev->ethtool_ops->get_ringparam)
2194 		return -EOPNOTSUPP;
2195 
2196 	dev->ethtool_ops->get_ringparam(dev, &ringparam,
2197 					&kernel_ringparam, NULL);
2198 
2199 	if (copy_to_user(useraddr, &ringparam, sizeof(ringparam)))
2200 		return -EFAULT;
2201 	return 0;
2202 }
2203 
2204 static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr)
2205 {
2206 	struct kernel_ethtool_ringparam kernel_ringparam;
2207 	struct ethtool_ringparam ringparam, max;
2208 	int ret;
2209 
2210 	if (!dev->ethtool_ops->set_ringparam || !dev->ethtool_ops->get_ringparam)
2211 		return -EOPNOTSUPP;
2212 
2213 	if (copy_from_user(&ringparam, useraddr, sizeof(ringparam)))
2214 		return -EFAULT;
2215 
2216 	ethtool_ringparam_get_cfg(dev, &max, &kernel_ringparam, NULL);
2217 
2218 	/* ensure new ring parameters are within the maximums */
2219 	if (ringparam.rx_pending > max.rx_max_pending ||
2220 	    ringparam.rx_mini_pending > max.rx_mini_max_pending ||
2221 	    ringparam.rx_jumbo_pending > max.rx_jumbo_max_pending ||
2222 	    ringparam.tx_pending > max.tx_max_pending)
2223 		return -EINVAL;
2224 
2225 	ret = dev->ethtool_ops->set_ringparam(dev, &ringparam,
2226 					      &kernel_ringparam, NULL);
2227 	if (!ret)
2228 		ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF);
2229 	return ret;
2230 }
2231 
2232 static noinline_for_stack int ethtool_get_channels(struct net_device *dev,
2233 						   void __user *useraddr)
2234 {
2235 	struct ethtool_channels channels = { .cmd = ETHTOOL_GCHANNELS };
2236 
2237 	if (!dev->ethtool_ops->get_channels)
2238 		return -EOPNOTSUPP;
2239 
2240 	dev->ethtool_ops->get_channels(dev, &channels);
2241 
2242 	if (copy_to_user(useraddr, &channels, sizeof(channels)))
2243 		return -EFAULT;
2244 	return 0;
2245 }
2246 
2247 static noinline_for_stack int ethtool_set_channels(struct net_device *dev,
2248 						   void __user *useraddr)
2249 {
2250 	struct ethtool_channels channels, curr = { .cmd = ETHTOOL_GCHANNELS };
2251 	u16 from_channel, to_channel;
2252 	unsigned int i;
2253 	int ret;
2254 
2255 	if (!dev->ethtool_ops->set_channels || !dev->ethtool_ops->get_channels)
2256 		return -EOPNOTSUPP;
2257 
2258 	if (copy_from_user(&channels, useraddr, sizeof(channels)))
2259 		return -EFAULT;
2260 
2261 	dev->ethtool_ops->get_channels(dev, &curr);
2262 
2263 	if (channels.rx_count == curr.rx_count &&
2264 	    channels.tx_count == curr.tx_count &&
2265 	    channels.combined_count == curr.combined_count &&
2266 	    channels.other_count == curr.other_count)
2267 		return 0;
2268 
2269 	/* ensure new counts are within the maximums */
2270 	if (channels.rx_count > curr.max_rx ||
2271 	    channels.tx_count > curr.max_tx ||
2272 	    channels.combined_count > curr.max_combined ||
2273 	    channels.other_count > curr.max_other)
2274 		return -EINVAL;
2275 
2276 	/* ensure there is at least one RX and one TX channel */
2277 	if (!channels.combined_count &&
2278 	    (!channels.rx_count || !channels.tx_count))
2279 		return -EINVAL;
2280 
2281 	ret = ethtool_check_max_channel(dev, channels, NULL);
2282 	if (ret)
2283 		return ret;
2284 
2285 	/* Disabling channels, query zero-copy AF_XDP sockets */
2286 	from_channel = channels.combined_count +
2287 		min(channels.rx_count, channels.tx_count);
2288 	to_channel = curr.combined_count + max(curr.rx_count, curr.tx_count);
2289 	for (i = from_channel; i < to_channel; i++)
2290 		if (xsk_get_pool_from_qid(dev, i))
2291 			return -EINVAL;
2292 
2293 	ret = dev->ethtool_ops->set_channels(dev, &channels);
2294 	if (!ret)
2295 		ethtool_notify(dev, ETHTOOL_MSG_CHANNELS_NTF);
2296 	return ret;
2297 }
2298 
2299 static int ethtool_get_pauseparam(struct net_device *dev, void __user *useraddr)
2300 {
2301 	struct ethtool_pauseparam pauseparam = { .cmd = ETHTOOL_GPAUSEPARAM };
2302 
2303 	if (!dev->ethtool_ops->get_pauseparam)
2304 		return -EOPNOTSUPP;
2305 
2306 	dev->ethtool_ops->get_pauseparam(dev, &pauseparam);
2307 
2308 	if (copy_to_user(useraddr, &pauseparam, sizeof(pauseparam)))
2309 		return -EFAULT;
2310 	return 0;
2311 }
2312 
2313 static int ethtool_set_pauseparam(struct net_device *dev, void __user *useraddr)
2314 {
2315 	struct ethtool_pauseparam pauseparam;
2316 	int ret;
2317 
2318 	if (!dev->ethtool_ops->set_pauseparam)
2319 		return -EOPNOTSUPP;
2320 
2321 	if (copy_from_user(&pauseparam, useraddr, sizeof(pauseparam)))
2322 		return -EFAULT;
2323 
2324 	ret = dev->ethtool_ops->set_pauseparam(dev, &pauseparam);
2325 	if (!ret)
2326 		ethtool_notify(dev, ETHTOOL_MSG_PAUSE_NTF);
2327 	return ret;
2328 }
2329 
2330 static int ethtool_self_test(struct net_device *dev, char __user *useraddr)
2331 {
2332 	struct ethtool_test test;
2333 	const struct ethtool_ops *ops = dev->ethtool_ops;
2334 	u64 *data;
2335 	int ret, test_len;
2336 
2337 	if (!ops->self_test || !ops->get_sset_count)
2338 		return -EOPNOTSUPP;
2339 
2340 	test_len = ops->get_sset_count(dev, ETH_SS_TEST);
2341 	if (test_len < 0)
2342 		return test_len;
2343 	WARN_ON(test_len == 0);
2344 
2345 	if (copy_from_user(&test, useraddr, sizeof(test)))
2346 		return -EFAULT;
2347 
2348 	test.len = test_len;
2349 	data = kcalloc(test_len, sizeof(u64), GFP_USER);
2350 	if (!data)
2351 		return -ENOMEM;
2352 
2353 	netif_testing_on(dev);
2354 	ops->self_test(dev, &test, data);
2355 	netif_testing_off(dev);
2356 
2357 	ret = -EFAULT;
2358 	if (copy_to_user(useraddr, &test, sizeof(test)))
2359 		goto out;
2360 	useraddr += sizeof(test);
2361 	if (copy_to_user(useraddr, data, array_size(test.len, sizeof(u64))))
2362 		goto out;
2363 	ret = 0;
2364 
2365  out:
2366 	kfree(data);
2367 	return ret;
2368 }
2369 
2370 static int ethtool_get_strings(struct net_device *dev, void __user *useraddr)
2371 {
2372 	struct ethtool_gstrings gstrings;
2373 	u8 *data;
2374 	int ret;
2375 
2376 	if (copy_from_user(&gstrings, useraddr, sizeof(gstrings)))
2377 		return -EFAULT;
2378 
2379 	ret = __ethtool_get_sset_count(dev, gstrings.string_set);
2380 	if (ret < 0)
2381 		return ret;
2382 	if (ret > S32_MAX / ETH_GSTRING_LEN)
2383 		return -ENOMEM;
2384 	WARN_ON_ONCE(!ret);
2385 
2386 	gstrings.len = ret;
2387 
2388 	if (gstrings.len) {
2389 		data = vzalloc(array_size(gstrings.len, ETH_GSTRING_LEN));
2390 		if (!data)
2391 			return -ENOMEM;
2392 
2393 		__ethtool_get_strings(dev, gstrings.string_set, data);
2394 	} else {
2395 		data = NULL;
2396 	}
2397 
2398 	ret = -EFAULT;
2399 	if (copy_to_user(useraddr, &gstrings, sizeof(gstrings)))
2400 		goto out;
2401 	useraddr += sizeof(gstrings);
2402 	if (gstrings.len &&
2403 	    copy_to_user(useraddr, data,
2404 			 array_size(gstrings.len, ETH_GSTRING_LEN)))
2405 		goto out;
2406 	ret = 0;
2407 
2408 out:
2409 	vfree(data);
2410 	return ret;
2411 }
2412 
2413 __printf(2, 3) void ethtool_sprintf(u8 **data, const char *fmt, ...)
2414 {
2415 	va_list args;
2416 
2417 	va_start(args, fmt);
2418 	vsnprintf(*data, ETH_GSTRING_LEN, fmt, args);
2419 	va_end(args);
2420 
2421 	*data += ETH_GSTRING_LEN;
2422 }
2423 EXPORT_SYMBOL(ethtool_sprintf);
2424 
2425 void ethtool_puts(u8 **data, const char *str)
2426 {
2427 	strscpy(*data, str, ETH_GSTRING_LEN);
2428 	*data += ETH_GSTRING_LEN;
2429 }
2430 EXPORT_SYMBOL(ethtool_puts);
2431 
2432 static int ethtool_phys_id(struct net_device *dev, void __user *useraddr)
2433 {
2434 	struct ethtool_value id;
2435 	static bool busy;
2436 	const struct ethtool_ops *ops = dev->ethtool_ops;
2437 	netdevice_tracker dev_tracker;
2438 	int rc;
2439 
2440 	if (!ops->set_phys_id)
2441 		return -EOPNOTSUPP;
2442 
2443 	if (busy)
2444 		return -EBUSY;
2445 
2446 	if (copy_from_user(&id, useraddr, sizeof(id)))
2447 		return -EFAULT;
2448 
2449 	rc = ops->set_phys_id(dev, ETHTOOL_ID_ACTIVE);
2450 	if (rc < 0)
2451 		return rc;
2452 
2453 	/* Drop the RTNL lock while waiting, but prevent reentry or
2454 	 * removal of the device.
2455 	 */
2456 	busy = true;
2457 	netdev_hold(dev, &dev_tracker, GFP_KERNEL);
2458 	netdev_unlock_ops(dev);
2459 	rtnl_unlock();
2460 
2461 	if (rc == 0) {
2462 		/* Driver will handle this itself */
2463 		schedule_timeout_interruptible(
2464 			id.data ? (id.data * HZ) : MAX_SCHEDULE_TIMEOUT);
2465 	} else {
2466 		/* Driver expects to be called at twice the frequency in rc */
2467 		int n = rc * 2, interval = HZ / n;
2468 		u64 count = mul_u32_u32(n, id.data);
2469 		u64 i = 0;
2470 
2471 		do {
2472 			rtnl_lock();
2473 			netdev_lock_ops(dev);
2474 			rc = ops->set_phys_id(dev,
2475 				    (i++ & 1) ? ETHTOOL_ID_OFF : ETHTOOL_ID_ON);
2476 			netdev_unlock_ops(dev);
2477 			rtnl_unlock();
2478 			if (rc)
2479 				break;
2480 			schedule_timeout_interruptible(interval);
2481 		} while (!signal_pending(current) && (!id.data || i < count));
2482 	}
2483 
2484 	rtnl_lock();
2485 	netdev_lock_ops(dev);
2486 	netdev_put(dev, &dev_tracker);
2487 	busy = false;
2488 
2489 	(void) ops->set_phys_id(dev, ETHTOOL_ID_INACTIVE);
2490 	return rc;
2491 }
2492 
2493 static int ethtool_get_stats(struct net_device *dev, void __user *useraddr)
2494 {
2495 	struct ethtool_stats stats;
2496 	const struct ethtool_ops *ops = dev->ethtool_ops;
2497 	u64 *data;
2498 	int ret, n_stats;
2499 
2500 	if (!ops->get_ethtool_stats || !ops->get_sset_count)
2501 		return -EOPNOTSUPP;
2502 
2503 	n_stats = ops->get_sset_count(dev, ETH_SS_STATS);
2504 	if (n_stats < 0)
2505 		return n_stats;
2506 	if (n_stats > S32_MAX / sizeof(u64))
2507 		return -ENOMEM;
2508 	WARN_ON_ONCE(!n_stats);
2509 	if (copy_from_user(&stats, useraddr, sizeof(stats)))
2510 		return -EFAULT;
2511 
2512 	stats.n_stats = n_stats;
2513 
2514 	if (n_stats) {
2515 		data = vzalloc(array_size(n_stats, sizeof(u64)));
2516 		if (!data)
2517 			return -ENOMEM;
2518 		ops->get_ethtool_stats(dev, &stats, data);
2519 	} else {
2520 		data = NULL;
2521 	}
2522 
2523 	ret = -EFAULT;
2524 	if (copy_to_user(useraddr, &stats, sizeof(stats)))
2525 		goto out;
2526 	useraddr += sizeof(stats);
2527 	if (n_stats && copy_to_user(useraddr, data, array_size(n_stats, sizeof(u64))))
2528 		goto out;
2529 	ret = 0;
2530 
2531  out:
2532 	vfree(data);
2533 	return ret;
2534 }
2535 
2536 static int ethtool_vzalloc_stats_array(int n_stats, u64 **data)
2537 {
2538 	if (n_stats < 0)
2539 		return n_stats;
2540 	if (n_stats > S32_MAX / sizeof(u64))
2541 		return -ENOMEM;
2542 	if (WARN_ON_ONCE(!n_stats))
2543 		return -EOPNOTSUPP;
2544 
2545 	*data = vzalloc(array_size(n_stats, sizeof(u64)));
2546 	if (!*data)
2547 		return -ENOMEM;
2548 
2549 	return 0;
2550 }
2551 
2552 static int ethtool_get_phy_stats_phydev(struct phy_device *phydev,
2553 					 struct ethtool_stats *stats,
2554 					 u64 **data)
2555  {
2556 	const struct ethtool_phy_ops *phy_ops = ethtool_phy_ops;
2557 	int n_stats, ret;
2558 
2559 	if (!phy_ops || !phy_ops->get_sset_count || !phy_ops->get_stats)
2560 		return -EOPNOTSUPP;
2561 
2562 	n_stats = phy_ops->get_sset_count(phydev);
2563 
2564 	ret = ethtool_vzalloc_stats_array(n_stats, data);
2565 	if (ret)
2566 		return ret;
2567 
2568 	stats->n_stats = n_stats;
2569 	return phy_ops->get_stats(phydev, stats, *data);
2570 }
2571 
2572 static int ethtool_get_phy_stats_ethtool(struct net_device *dev,
2573 					  struct ethtool_stats *stats,
2574 					  u64 **data)
2575 {
2576 	const struct ethtool_ops *ops = dev->ethtool_ops;
2577 	int n_stats, ret;
2578 
2579 	if (!ops || !ops->get_sset_count || !ops->get_ethtool_phy_stats)
2580 		return -EOPNOTSUPP;
2581 
2582 	n_stats = ops->get_sset_count(dev, ETH_SS_PHY_STATS);
2583 
2584 	ret = ethtool_vzalloc_stats_array(n_stats, data);
2585 	if (ret)
2586 		return ret;
2587 
2588 	stats->n_stats = n_stats;
2589 	ops->get_ethtool_phy_stats(dev, stats, *data);
2590 
2591 	return 0;
2592 }
2593 
2594 static int ethtool_get_phy_stats(struct net_device *dev, void __user *useraddr)
2595 {
2596 	struct phy_device *phydev = dev->phydev;
2597 	struct ethtool_stats stats;
2598 	u64 *data = NULL;
2599 	int ret = -EOPNOTSUPP;
2600 
2601 	if (copy_from_user(&stats, useraddr, sizeof(stats)))
2602 		return -EFAULT;
2603 
2604 	if (phydev)
2605 		ret = ethtool_get_phy_stats_phydev(phydev, &stats, &data);
2606 
2607 	if (ret == -EOPNOTSUPP)
2608 		ret = ethtool_get_phy_stats_ethtool(dev, &stats, &data);
2609 
2610 	if (ret)
2611 		goto out;
2612 
2613 	if (copy_to_user(useraddr, &stats, sizeof(stats))) {
2614 		ret = -EFAULT;
2615 		goto out;
2616 	}
2617 
2618 	useraddr += sizeof(stats);
2619 	if (copy_to_user(useraddr, data, array_size(stats.n_stats, sizeof(u64))))
2620 		ret = -EFAULT;
2621 
2622  out:
2623 	vfree(data);
2624 	return ret;
2625 }
2626 
2627 static int ethtool_get_perm_addr(struct net_device *dev, void __user *useraddr)
2628 {
2629 	struct ethtool_perm_addr epaddr;
2630 
2631 	if (copy_from_user(&epaddr, useraddr, sizeof(epaddr)))
2632 		return -EFAULT;
2633 
2634 	if (epaddr.size < dev->addr_len)
2635 		return -ETOOSMALL;
2636 	epaddr.size = dev->addr_len;
2637 
2638 	if (copy_to_user(useraddr, &epaddr, sizeof(epaddr)))
2639 		return -EFAULT;
2640 	useraddr += sizeof(epaddr);
2641 	if (copy_to_user(useraddr, dev->perm_addr, epaddr.size))
2642 		return -EFAULT;
2643 	return 0;
2644 }
2645 
2646 static int ethtool_get_value(struct net_device *dev, char __user *useraddr,
2647 			     u32 cmd, u32 (*actor)(struct net_device *))
2648 {
2649 	struct ethtool_value edata = { .cmd = cmd };
2650 
2651 	if (!actor)
2652 		return -EOPNOTSUPP;
2653 
2654 	edata.data = actor(dev);
2655 
2656 	if (copy_to_user(useraddr, &edata, sizeof(edata)))
2657 		return -EFAULT;
2658 	return 0;
2659 }
2660 
2661 static int ethtool_set_value_void(struct net_device *dev, char __user *useraddr,
2662 			     void (*actor)(struct net_device *, u32))
2663 {
2664 	struct ethtool_value edata;
2665 
2666 	if (!actor)
2667 		return -EOPNOTSUPP;
2668 
2669 	if (copy_from_user(&edata, useraddr, sizeof(edata)))
2670 		return -EFAULT;
2671 
2672 	actor(dev, edata.data);
2673 	return 0;
2674 }
2675 
2676 static int ethtool_set_value(struct net_device *dev, char __user *useraddr,
2677 			     int (*actor)(struct net_device *, u32))
2678 {
2679 	struct ethtool_value edata;
2680 
2681 	if (!actor)
2682 		return -EOPNOTSUPP;
2683 
2684 	if (copy_from_user(&edata, useraddr, sizeof(edata)))
2685 		return -EFAULT;
2686 
2687 	return actor(dev, edata.data);
2688 }
2689 
2690 static int
2691 ethtool_flash_device(struct net_device *dev, struct ethtool_devlink_compat *req)
2692 {
2693 	if (!dev->ethtool_ops->flash_device) {
2694 		req->devlink = netdev_to_devlink_get(dev);
2695 		return 0;
2696 	}
2697 
2698 	return dev->ethtool_ops->flash_device(dev, &req->efl);
2699 }
2700 
2701 static int ethtool_set_dump(struct net_device *dev,
2702 			void __user *useraddr)
2703 {
2704 	struct ethtool_dump dump;
2705 
2706 	if (!dev->ethtool_ops->set_dump)
2707 		return -EOPNOTSUPP;
2708 
2709 	if (copy_from_user(&dump, useraddr, sizeof(dump)))
2710 		return -EFAULT;
2711 
2712 	return dev->ethtool_ops->set_dump(dev, &dump);
2713 }
2714 
2715 static int ethtool_get_dump_flag(struct net_device *dev,
2716 				void __user *useraddr)
2717 {
2718 	int ret;
2719 	struct ethtool_dump dump;
2720 	const struct ethtool_ops *ops = dev->ethtool_ops;
2721 
2722 	if (!ops->get_dump_flag)
2723 		return -EOPNOTSUPP;
2724 
2725 	if (copy_from_user(&dump, useraddr, sizeof(dump)))
2726 		return -EFAULT;
2727 
2728 	ret = ops->get_dump_flag(dev, &dump);
2729 	if (ret)
2730 		return ret;
2731 
2732 	if (copy_to_user(useraddr, &dump, sizeof(dump)))
2733 		return -EFAULT;
2734 	return 0;
2735 }
2736 
2737 static int ethtool_get_dump_data(struct net_device *dev,
2738 				void __user *useraddr)
2739 {
2740 	int ret;
2741 	__u32 len;
2742 	struct ethtool_dump dump, tmp;
2743 	const struct ethtool_ops *ops = dev->ethtool_ops;
2744 	void *data = NULL;
2745 
2746 	if (!ops->get_dump_data || !ops->get_dump_flag)
2747 		return -EOPNOTSUPP;
2748 
2749 	if (copy_from_user(&dump, useraddr, sizeof(dump)))
2750 		return -EFAULT;
2751 
2752 	memset(&tmp, 0, sizeof(tmp));
2753 	tmp.cmd = ETHTOOL_GET_DUMP_FLAG;
2754 	ret = ops->get_dump_flag(dev, &tmp);
2755 	if (ret)
2756 		return ret;
2757 
2758 	len = min(tmp.len, dump.len);
2759 	if (!len)
2760 		return -EFAULT;
2761 
2762 	/* Don't ever let the driver think there's more space available
2763 	 * than it requested with .get_dump_flag().
2764 	 */
2765 	dump.len = len;
2766 
2767 	/* Always allocate enough space to hold the whole thing so that the
2768 	 * driver does not need to check the length and bother with partial
2769 	 * dumping.
2770 	 */
2771 	data = vzalloc(tmp.len);
2772 	if (!data)
2773 		return -ENOMEM;
2774 	ret = ops->get_dump_data(dev, &dump, data);
2775 	if (ret)
2776 		goto out;
2777 
2778 	/* There are two sane possibilities:
2779 	 * 1. The driver's .get_dump_data() does not touch dump.len.
2780 	 * 2. Or it may set dump.len to how much it really writes, which
2781 	 *    should be tmp.len (or len if it can do a partial dump).
2782 	 * In any case respond to userspace with the actual length of data
2783 	 * it's receiving.
2784 	 */
2785 	WARN_ON(dump.len != len && dump.len != tmp.len);
2786 	dump.len = len;
2787 
2788 	if (copy_to_user(useraddr, &dump, sizeof(dump))) {
2789 		ret = -EFAULT;
2790 		goto out;
2791 	}
2792 	useraddr += offsetof(struct ethtool_dump, data);
2793 	if (copy_to_user(useraddr, data, len))
2794 		ret = -EFAULT;
2795 out:
2796 	vfree(data);
2797 	return ret;
2798 }
2799 
2800 static int ethtool_get_ts_info(struct net_device *dev, void __user *useraddr)
2801 {
2802 	struct kernel_ethtool_ts_info kernel_info;
2803 	struct ethtool_ts_info info = {};
2804 	int err;
2805 
2806 	err = __ethtool_get_ts_info(dev, &kernel_info);
2807 	if (err)
2808 		return err;
2809 
2810 	info.cmd = kernel_info.cmd;
2811 	info.so_timestamping = kernel_info.so_timestamping;
2812 	info.phc_index = kernel_info.phc_index;
2813 	info.tx_types = kernel_info.tx_types;
2814 	info.rx_filters = kernel_info.rx_filters;
2815 
2816 	if (copy_to_user(useraddr, &info, sizeof(info)))
2817 		return -EFAULT;
2818 
2819 	return 0;
2820 }
2821 
2822 int ethtool_get_module_info_call(struct net_device *dev,
2823 				 struct ethtool_modinfo *modinfo)
2824 {
2825 	const struct ethtool_ops *ops = dev->ethtool_ops;
2826 	struct phy_device *phydev = dev->phydev;
2827 
2828 	if (dev->ethtool->module_fw_flash_in_progress)
2829 		return -EBUSY;
2830 
2831 	if (dev->sfp_bus)
2832 		return sfp_get_module_info(dev->sfp_bus, modinfo);
2833 
2834 	if (phydev && phydev->drv && phydev->drv->module_info)
2835 		return phydev->drv->module_info(phydev, modinfo);
2836 
2837 	if (ops->get_module_info)
2838 		return ops->get_module_info(dev, modinfo);
2839 
2840 	return -EOPNOTSUPP;
2841 }
2842 
2843 static int ethtool_get_module_info(struct net_device *dev,
2844 				   void __user *useraddr)
2845 {
2846 	int ret;
2847 	struct ethtool_modinfo modinfo;
2848 
2849 	if (copy_from_user(&modinfo, useraddr, sizeof(modinfo)))
2850 		return -EFAULT;
2851 
2852 	ret = ethtool_get_module_info_call(dev, &modinfo);
2853 	if (ret)
2854 		return ret;
2855 
2856 	if (copy_to_user(useraddr, &modinfo, sizeof(modinfo)))
2857 		return -EFAULT;
2858 
2859 	return 0;
2860 }
2861 
2862 int ethtool_get_module_eeprom_call(struct net_device *dev,
2863 				   struct ethtool_eeprom *ee, u8 *data)
2864 {
2865 	const struct ethtool_ops *ops = dev->ethtool_ops;
2866 	struct phy_device *phydev = dev->phydev;
2867 
2868 	if (dev->ethtool->module_fw_flash_in_progress)
2869 		return -EBUSY;
2870 
2871 	if (dev->sfp_bus)
2872 		return sfp_get_module_eeprom(dev->sfp_bus, ee, data);
2873 
2874 	if (phydev && phydev->drv && phydev->drv->module_eeprom)
2875 		return phydev->drv->module_eeprom(phydev, ee, data);
2876 
2877 	if (ops->get_module_eeprom)
2878 		return ops->get_module_eeprom(dev, ee, data);
2879 
2880 	return -EOPNOTSUPP;
2881 }
2882 
2883 static int ethtool_get_module_eeprom(struct net_device *dev,
2884 				     void __user *useraddr)
2885 {
2886 	int ret;
2887 	struct ethtool_modinfo modinfo;
2888 
2889 	ret = ethtool_get_module_info_call(dev, &modinfo);
2890 	if (ret)
2891 		return ret;
2892 
2893 	return ethtool_get_any_eeprom(dev, useraddr,
2894 				      ethtool_get_module_eeprom_call,
2895 				      modinfo.eeprom_len);
2896 }
2897 
2898 static int ethtool_tunable_valid(const struct ethtool_tunable *tuna)
2899 {
2900 	switch (tuna->id) {
2901 	case ETHTOOL_RX_COPYBREAK:
2902 	case ETHTOOL_TX_COPYBREAK:
2903 	case ETHTOOL_TX_COPYBREAK_BUF_SIZE:
2904 		if (tuna->len != sizeof(u32) ||
2905 		    tuna->type_id != ETHTOOL_TUNABLE_U32)
2906 			return -EINVAL;
2907 		break;
2908 	case ETHTOOL_PFC_PREVENTION_TOUT:
2909 		if (tuna->len != sizeof(u16) ||
2910 		    tuna->type_id != ETHTOOL_TUNABLE_U16)
2911 			return -EINVAL;
2912 		break;
2913 	default:
2914 		return -EINVAL;
2915 	}
2916 
2917 	return 0;
2918 }
2919 
2920 static int ethtool_get_tunable(struct net_device *dev, void __user *useraddr)
2921 {
2922 	int ret;
2923 	struct ethtool_tunable tuna;
2924 	const struct ethtool_ops *ops = dev->ethtool_ops;
2925 	void *data;
2926 
2927 	if (!ops->get_tunable)
2928 		return -EOPNOTSUPP;
2929 	if (copy_from_user(&tuna, useraddr, sizeof(tuna)))
2930 		return -EFAULT;
2931 	ret = ethtool_tunable_valid(&tuna);
2932 	if (ret)
2933 		return ret;
2934 	data = kzalloc(tuna.len, GFP_USER);
2935 	if (!data)
2936 		return -ENOMEM;
2937 	ret = ops->get_tunable(dev, &tuna, data);
2938 	if (ret)
2939 		goto out;
2940 	useraddr += sizeof(tuna);
2941 	ret = -EFAULT;
2942 	if (copy_to_user(useraddr, data, tuna.len))
2943 		goto out;
2944 	ret = 0;
2945 
2946 out:
2947 	kfree(data);
2948 	return ret;
2949 }
2950 
2951 static int ethtool_set_tunable(struct net_device *dev, void __user *useraddr)
2952 {
2953 	int ret;
2954 	struct ethtool_tunable tuna;
2955 	const struct ethtool_ops *ops = dev->ethtool_ops;
2956 	void *data;
2957 
2958 	if (!ops->set_tunable)
2959 		return -EOPNOTSUPP;
2960 	if (copy_from_user(&tuna, useraddr, sizeof(tuna)))
2961 		return -EFAULT;
2962 	ret = ethtool_tunable_valid(&tuna);
2963 	if (ret)
2964 		return ret;
2965 	useraddr += sizeof(tuna);
2966 	data = memdup_user(useraddr, tuna.len);
2967 	if (IS_ERR(data))
2968 		return PTR_ERR(data);
2969 	ret = ops->set_tunable(dev, &tuna, data);
2970 
2971 	kfree(data);
2972 	return ret;
2973 }
2974 
2975 static noinline_for_stack int
2976 ethtool_get_per_queue_coalesce(struct net_device *dev,
2977 			       void __user *useraddr,
2978 			       struct ethtool_per_queue_op *per_queue_opt)
2979 {
2980 	u32 bit;
2981 	int ret;
2982 	DECLARE_BITMAP(queue_mask, MAX_NUM_QUEUE);
2983 
2984 	if (!dev->ethtool_ops->get_per_queue_coalesce)
2985 		return -EOPNOTSUPP;
2986 
2987 	useraddr += sizeof(*per_queue_opt);
2988 
2989 	bitmap_from_arr32(queue_mask, per_queue_opt->queue_mask,
2990 			  MAX_NUM_QUEUE);
2991 
2992 	for_each_set_bit(bit, queue_mask, MAX_NUM_QUEUE) {
2993 		struct ethtool_coalesce coalesce = { .cmd = ETHTOOL_GCOALESCE };
2994 
2995 		ret = dev->ethtool_ops->get_per_queue_coalesce(dev, bit, &coalesce);
2996 		if (ret != 0)
2997 			return ret;
2998 		if (copy_to_user(useraddr, &coalesce, sizeof(coalesce)))
2999 			return -EFAULT;
3000 		useraddr += sizeof(coalesce);
3001 	}
3002 
3003 	return 0;
3004 }
3005 
3006 static noinline_for_stack int
3007 ethtool_set_per_queue_coalesce(struct net_device *dev,
3008 			       void __user *useraddr,
3009 			       struct ethtool_per_queue_op *per_queue_opt)
3010 {
3011 	u32 bit;
3012 	int i, ret = 0;
3013 	int n_queue;
3014 	struct ethtool_coalesce *backup = NULL, *tmp = NULL;
3015 	DECLARE_BITMAP(queue_mask, MAX_NUM_QUEUE);
3016 
3017 	if ((!dev->ethtool_ops->set_per_queue_coalesce) ||
3018 	    (!dev->ethtool_ops->get_per_queue_coalesce))
3019 		return -EOPNOTSUPP;
3020 
3021 	useraddr += sizeof(*per_queue_opt);
3022 
3023 	bitmap_from_arr32(queue_mask, per_queue_opt->queue_mask, MAX_NUM_QUEUE);
3024 	n_queue = bitmap_weight(queue_mask, MAX_NUM_QUEUE);
3025 	tmp = backup = kmalloc_array(n_queue, sizeof(*backup), GFP_KERNEL);
3026 	if (!backup)
3027 		return -ENOMEM;
3028 
3029 	for_each_set_bit(bit, queue_mask, MAX_NUM_QUEUE) {
3030 		struct ethtool_coalesce coalesce;
3031 
3032 		ret = dev->ethtool_ops->get_per_queue_coalesce(dev, bit, tmp);
3033 		if (ret != 0)
3034 			goto roll_back;
3035 
3036 		tmp++;
3037 
3038 		if (copy_from_user(&coalesce, useraddr, sizeof(coalesce))) {
3039 			ret = -EFAULT;
3040 			goto roll_back;
3041 		}
3042 
3043 		if (!ethtool_set_coalesce_supported(dev, &coalesce)) {
3044 			ret = -EOPNOTSUPP;
3045 			goto roll_back;
3046 		}
3047 
3048 		ret = dev->ethtool_ops->set_per_queue_coalesce(dev, bit, &coalesce);
3049 		if (ret != 0)
3050 			goto roll_back;
3051 
3052 		useraddr += sizeof(coalesce);
3053 	}
3054 
3055 roll_back:
3056 	if (ret != 0) {
3057 		tmp = backup;
3058 		for_each_set_bit(i, queue_mask, bit) {
3059 			dev->ethtool_ops->set_per_queue_coalesce(dev, i, tmp);
3060 			tmp++;
3061 		}
3062 	}
3063 	kfree(backup);
3064 
3065 	return ret;
3066 }
3067 
3068 static int noinline_for_stack ethtool_set_per_queue(struct net_device *dev,
3069 				 void __user *useraddr, u32 sub_cmd)
3070 {
3071 	struct ethtool_per_queue_op per_queue_opt;
3072 
3073 	if (copy_from_user(&per_queue_opt, useraddr, sizeof(per_queue_opt)))
3074 		return -EFAULT;
3075 
3076 	if (per_queue_opt.sub_command != sub_cmd)
3077 		return -EINVAL;
3078 
3079 	switch (per_queue_opt.sub_command) {
3080 	case ETHTOOL_GCOALESCE:
3081 		return ethtool_get_per_queue_coalesce(dev, useraddr, &per_queue_opt);
3082 	case ETHTOOL_SCOALESCE:
3083 		return ethtool_set_per_queue_coalesce(dev, useraddr, &per_queue_opt);
3084 	default:
3085 		return -EOPNOTSUPP;
3086 	}
3087 }
3088 
3089 static int ethtool_phy_tunable_valid(const struct ethtool_tunable *tuna)
3090 {
3091 	switch (tuna->id) {
3092 	case ETHTOOL_PHY_DOWNSHIFT:
3093 	case ETHTOOL_PHY_FAST_LINK_DOWN:
3094 		if (tuna->len != sizeof(u8) ||
3095 		    tuna->type_id != ETHTOOL_TUNABLE_U8)
3096 			return -EINVAL;
3097 		break;
3098 	case ETHTOOL_PHY_EDPD:
3099 		if (tuna->len != sizeof(u16) ||
3100 		    tuna->type_id != ETHTOOL_TUNABLE_U16)
3101 			return -EINVAL;
3102 		break;
3103 	default:
3104 		return -EINVAL;
3105 	}
3106 
3107 	return 0;
3108 }
3109 
3110 static int get_phy_tunable(struct net_device *dev, void __user *useraddr)
3111 {
3112 	struct phy_device *phydev = dev->phydev;
3113 	struct ethtool_tunable tuna;
3114 	bool phy_drv_tunable;
3115 	void *data;
3116 	int ret;
3117 
3118 	phy_drv_tunable = phydev && phydev->drv && phydev->drv->get_tunable;
3119 	if (!phy_drv_tunable && !dev->ethtool_ops->get_phy_tunable)
3120 		return -EOPNOTSUPP;
3121 	if (copy_from_user(&tuna, useraddr, sizeof(tuna)))
3122 		return -EFAULT;
3123 	ret = ethtool_phy_tunable_valid(&tuna);
3124 	if (ret)
3125 		return ret;
3126 	data = kzalloc(tuna.len, GFP_USER);
3127 	if (!data)
3128 		return -ENOMEM;
3129 	if (phy_drv_tunable) {
3130 		mutex_lock(&phydev->lock);
3131 		ret = phydev->drv->get_tunable(phydev, &tuna, data);
3132 		mutex_unlock(&phydev->lock);
3133 	} else {
3134 		ret = dev->ethtool_ops->get_phy_tunable(dev, &tuna, data);
3135 	}
3136 	if (ret)
3137 		goto out;
3138 	useraddr += sizeof(tuna);
3139 	ret = -EFAULT;
3140 	if (copy_to_user(useraddr, data, tuna.len))
3141 		goto out;
3142 	ret = 0;
3143 
3144 out:
3145 	kfree(data);
3146 	return ret;
3147 }
3148 
3149 static int set_phy_tunable(struct net_device *dev, void __user *useraddr)
3150 {
3151 	struct phy_device *phydev = dev->phydev;
3152 	struct ethtool_tunable tuna;
3153 	bool phy_drv_tunable;
3154 	void *data;
3155 	int ret;
3156 
3157 	phy_drv_tunable = phydev && phydev->drv && phydev->drv->get_tunable;
3158 	if (!phy_drv_tunable && !dev->ethtool_ops->set_phy_tunable)
3159 		return -EOPNOTSUPP;
3160 	if (copy_from_user(&tuna, useraddr, sizeof(tuna)))
3161 		return -EFAULT;
3162 	ret = ethtool_phy_tunable_valid(&tuna);
3163 	if (ret)
3164 		return ret;
3165 	useraddr += sizeof(tuna);
3166 	data = memdup_user(useraddr, tuna.len);
3167 	if (IS_ERR(data))
3168 		return PTR_ERR(data);
3169 	if (phy_drv_tunable) {
3170 		mutex_lock(&phydev->lock);
3171 		ret = phydev->drv->set_tunable(phydev, &tuna, data);
3172 		mutex_unlock(&phydev->lock);
3173 	} else {
3174 		ret = dev->ethtool_ops->set_phy_tunable(dev, &tuna, data);
3175 	}
3176 
3177 	kfree(data);
3178 	return ret;
3179 }
3180 
3181 static int ethtool_get_fecparam(struct net_device *dev, void __user *useraddr)
3182 {
3183 	struct ethtool_fecparam fecparam = { .cmd = ETHTOOL_GFECPARAM };
3184 	int rc;
3185 
3186 	if (!dev->ethtool_ops->get_fecparam)
3187 		return -EOPNOTSUPP;
3188 
3189 	rc = dev->ethtool_ops->get_fecparam(dev, &fecparam);
3190 	if (rc)
3191 		return rc;
3192 
3193 	if (WARN_ON_ONCE(fecparam.reserved))
3194 		fecparam.reserved = 0;
3195 
3196 	if (copy_to_user(useraddr, &fecparam, sizeof(fecparam)))
3197 		return -EFAULT;
3198 	return 0;
3199 }
3200 
3201 static int ethtool_set_fecparam(struct net_device *dev, void __user *useraddr)
3202 {
3203 	struct ethtool_fecparam fecparam;
3204 
3205 	if (!dev->ethtool_ops->set_fecparam)
3206 		return -EOPNOTSUPP;
3207 
3208 	if (copy_from_user(&fecparam, useraddr, sizeof(fecparam)))
3209 		return -EFAULT;
3210 
3211 	if (!fecparam.fec || fecparam.fec & ETHTOOL_FEC_NONE)
3212 		return -EINVAL;
3213 
3214 	fecparam.active_fec = 0;
3215 	fecparam.reserved = 0;
3216 
3217 	return dev->ethtool_ops->set_fecparam(dev, &fecparam);
3218 }
3219 
3220 /* The main entry point in this file.  Called from net/core/dev_ioctl.c */
3221 
3222 static int
3223 __dev_ethtool(struct net *net, struct ifreq *ifr, void __user *useraddr,
3224 	      u32 ethcmd, struct ethtool_devlink_compat *devlink_state)
3225 {
3226 	struct net_device *dev;
3227 	u32 sub_cmd;
3228 	int rc;
3229 	netdev_features_t old_features;
3230 
3231 	dev = __dev_get_by_name(net, ifr->ifr_name);
3232 	if (!dev)
3233 		return -ENODEV;
3234 
3235 	if (ethcmd == ETHTOOL_PERQUEUE) {
3236 		if (copy_from_user(&sub_cmd, useraddr + sizeof(ethcmd), sizeof(sub_cmd)))
3237 			return -EFAULT;
3238 	} else {
3239 		sub_cmd = ethcmd;
3240 	}
3241 	/* Allow some commands to be done by anyone */
3242 	switch (sub_cmd) {
3243 	case ETHTOOL_GSET:
3244 	case ETHTOOL_GDRVINFO:
3245 	case ETHTOOL_GMSGLVL:
3246 	case ETHTOOL_GLINK:
3247 	case ETHTOOL_GCOALESCE:
3248 	case ETHTOOL_GRINGPARAM:
3249 	case ETHTOOL_GPAUSEPARAM:
3250 	case ETHTOOL_GRXCSUM:
3251 	case ETHTOOL_GTXCSUM:
3252 	case ETHTOOL_GSG:
3253 	case ETHTOOL_GSSET_INFO:
3254 	case ETHTOOL_GSTRINGS:
3255 	case ETHTOOL_GSTATS:
3256 	case ETHTOOL_GPHYSTATS:
3257 	case ETHTOOL_GTSO:
3258 	case ETHTOOL_GPERMADDR:
3259 	case ETHTOOL_GUFO:
3260 	case ETHTOOL_GGSO:
3261 	case ETHTOOL_GGRO:
3262 	case ETHTOOL_GFLAGS:
3263 	case ETHTOOL_GPFLAGS:
3264 	case ETHTOOL_GRXFH:
3265 	case ETHTOOL_GRXRINGS:
3266 	case ETHTOOL_GRXCLSRLCNT:
3267 	case ETHTOOL_GRXCLSRULE:
3268 	case ETHTOOL_GRXCLSRLALL:
3269 	case ETHTOOL_GRXFHINDIR:
3270 	case ETHTOOL_GRSSH:
3271 	case ETHTOOL_GFEATURES:
3272 	case ETHTOOL_GCHANNELS:
3273 	case ETHTOOL_GET_TS_INFO:
3274 	case ETHTOOL_GEEE:
3275 	case ETHTOOL_GTUNABLE:
3276 	case ETHTOOL_PHY_GTUNABLE:
3277 	case ETHTOOL_GLINKSETTINGS:
3278 	case ETHTOOL_GFECPARAM:
3279 		break;
3280 	default:
3281 		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3282 			return -EPERM;
3283 	}
3284 
3285 	netdev_lock_ops(dev);
3286 	if (dev->dev.parent)
3287 		pm_runtime_get_sync(dev->dev.parent);
3288 
3289 	if (!netif_device_present(dev)) {
3290 		rc = -ENODEV;
3291 		goto out;
3292 	}
3293 
3294 	if (dev->ethtool_ops->begin) {
3295 		rc = dev->ethtool_ops->begin(dev);
3296 		if (rc < 0)
3297 			goto out;
3298 	}
3299 	old_features = dev->features;
3300 
3301 	switch (ethcmd) {
3302 	case ETHTOOL_GSET:
3303 		rc = ethtool_get_settings(dev, useraddr);
3304 		break;
3305 	case ETHTOOL_SSET:
3306 		rc = ethtool_set_settings(dev, useraddr);
3307 		break;
3308 	case ETHTOOL_GDRVINFO:
3309 		rc = ethtool_get_drvinfo(dev, devlink_state);
3310 		break;
3311 	case ETHTOOL_GREGS:
3312 		rc = ethtool_get_regs(dev, useraddr);
3313 		break;
3314 	case ETHTOOL_GWOL:
3315 		rc = ethtool_get_wol(dev, useraddr);
3316 		break;
3317 	case ETHTOOL_SWOL:
3318 		rc = ethtool_set_wol(dev, useraddr);
3319 		break;
3320 	case ETHTOOL_GMSGLVL:
3321 		rc = ethtool_get_value(dev, useraddr, ethcmd,
3322 				       dev->ethtool_ops->get_msglevel);
3323 		break;
3324 	case ETHTOOL_SMSGLVL:
3325 		rc = ethtool_set_value_void(dev, useraddr,
3326 				       dev->ethtool_ops->set_msglevel);
3327 		if (!rc)
3328 			ethtool_notify(dev, ETHTOOL_MSG_DEBUG_NTF);
3329 		break;
3330 	case ETHTOOL_GEEE:
3331 		rc = ethtool_get_eee(dev, useraddr);
3332 		break;
3333 	case ETHTOOL_SEEE:
3334 		rc = ethtool_set_eee(dev, useraddr);
3335 		break;
3336 	case ETHTOOL_NWAY_RST:
3337 		rc = ethtool_nway_reset(dev);
3338 		break;
3339 	case ETHTOOL_GLINK:
3340 		rc = ethtool_get_link(dev, useraddr);
3341 		break;
3342 	case ETHTOOL_GEEPROM:
3343 		rc = ethtool_get_eeprom(dev, useraddr);
3344 		break;
3345 	case ETHTOOL_SEEPROM:
3346 		rc = ethtool_set_eeprom(dev, useraddr);
3347 		break;
3348 	case ETHTOOL_GCOALESCE:
3349 		rc = ethtool_get_coalesce(dev, useraddr);
3350 		break;
3351 	case ETHTOOL_SCOALESCE:
3352 		rc = ethtool_set_coalesce(dev, useraddr);
3353 		break;
3354 	case ETHTOOL_GRINGPARAM:
3355 		rc = ethtool_get_ringparam(dev, useraddr);
3356 		break;
3357 	case ETHTOOL_SRINGPARAM:
3358 		rc = ethtool_set_ringparam(dev, useraddr);
3359 		break;
3360 	case ETHTOOL_GPAUSEPARAM:
3361 		rc = ethtool_get_pauseparam(dev, useraddr);
3362 		break;
3363 	case ETHTOOL_SPAUSEPARAM:
3364 		rc = ethtool_set_pauseparam(dev, useraddr);
3365 		break;
3366 	case ETHTOOL_TEST:
3367 		rc = ethtool_self_test(dev, useraddr);
3368 		break;
3369 	case ETHTOOL_GSTRINGS:
3370 		rc = ethtool_get_strings(dev, useraddr);
3371 		break;
3372 	case ETHTOOL_PHYS_ID:
3373 		rc = ethtool_phys_id(dev, useraddr);
3374 		break;
3375 	case ETHTOOL_GSTATS:
3376 		rc = ethtool_get_stats(dev, useraddr);
3377 		break;
3378 	case ETHTOOL_GPERMADDR:
3379 		rc = ethtool_get_perm_addr(dev, useraddr);
3380 		break;
3381 	case ETHTOOL_GFLAGS:
3382 		rc = ethtool_get_value(dev, useraddr, ethcmd,
3383 					__ethtool_get_flags);
3384 		break;
3385 	case ETHTOOL_SFLAGS:
3386 		rc = ethtool_set_value(dev, useraddr, __ethtool_set_flags);
3387 		break;
3388 	case ETHTOOL_GPFLAGS:
3389 		rc = ethtool_get_value(dev, useraddr, ethcmd,
3390 				       dev->ethtool_ops->get_priv_flags);
3391 		if (!rc)
3392 			ethtool_notify(dev, ETHTOOL_MSG_PRIVFLAGS_NTF);
3393 		break;
3394 	case ETHTOOL_SPFLAGS:
3395 		rc = ethtool_set_value(dev, useraddr,
3396 				       dev->ethtool_ops->set_priv_flags);
3397 		break;
3398 	case ETHTOOL_GRXFH:
3399 		rc = ethtool_get_rxfh_fields(dev, ethcmd, useraddr);
3400 		break;
3401 	case ETHTOOL_SRXFH:
3402 		rc = ethtool_set_rxfh_fields(dev, ethcmd, useraddr);
3403 		break;
3404 	case ETHTOOL_GRXRINGS:
3405 		rc = ethtool_get_rxrings(dev, ethcmd, useraddr);
3406 		break;
3407 	case ETHTOOL_GRXCLSRLCNT:
3408 	case ETHTOOL_GRXCLSRULE:
3409 	case ETHTOOL_GRXCLSRLALL:
3410 		rc = ethtool_get_rxnfc(dev, ethcmd, useraddr);
3411 		break;
3412 	case ETHTOOL_SRXCLSRLDEL:
3413 	case ETHTOOL_SRXCLSRLINS:
3414 		rc = ethtool_set_rxnfc(dev, ethcmd, useraddr);
3415 		break;
3416 	case ETHTOOL_FLASHDEV:
3417 		rc = ethtool_flash_device(dev, devlink_state);
3418 		break;
3419 	case ETHTOOL_RESET:
3420 		rc = ethtool_reset(dev, useraddr);
3421 		break;
3422 	case ETHTOOL_GSSET_INFO:
3423 		rc = ethtool_get_sset_info(dev, useraddr);
3424 		break;
3425 	case ETHTOOL_GRXFHINDIR:
3426 		rc = ethtool_get_rxfh_indir(dev, useraddr);
3427 		break;
3428 	case ETHTOOL_SRXFHINDIR:
3429 		rc = ethtool_set_rxfh_indir(dev, useraddr);
3430 		break;
3431 	case ETHTOOL_GRSSH:
3432 		rc = ethtool_get_rxfh(dev, useraddr);
3433 		break;
3434 	case ETHTOOL_SRSSH:
3435 		rc = ethtool_set_rxfh(dev, useraddr);
3436 		break;
3437 	case ETHTOOL_GFEATURES:
3438 		rc = ethtool_get_features(dev, useraddr);
3439 		break;
3440 	case ETHTOOL_SFEATURES:
3441 		rc = ethtool_set_features(dev, useraddr);
3442 		break;
3443 	case ETHTOOL_GTXCSUM:
3444 	case ETHTOOL_GRXCSUM:
3445 	case ETHTOOL_GSG:
3446 	case ETHTOOL_GTSO:
3447 	case ETHTOOL_GGSO:
3448 	case ETHTOOL_GGRO:
3449 		rc = ethtool_get_one_feature(dev, useraddr, ethcmd);
3450 		break;
3451 	case ETHTOOL_STXCSUM:
3452 	case ETHTOOL_SRXCSUM:
3453 	case ETHTOOL_SSG:
3454 	case ETHTOOL_STSO:
3455 	case ETHTOOL_SGSO:
3456 	case ETHTOOL_SGRO:
3457 		rc = ethtool_set_one_feature(dev, useraddr, ethcmd);
3458 		break;
3459 	case ETHTOOL_GCHANNELS:
3460 		rc = ethtool_get_channels(dev, useraddr);
3461 		break;
3462 	case ETHTOOL_SCHANNELS:
3463 		rc = ethtool_set_channels(dev, useraddr);
3464 		break;
3465 	case ETHTOOL_SET_DUMP:
3466 		rc = ethtool_set_dump(dev, useraddr);
3467 		break;
3468 	case ETHTOOL_GET_DUMP_FLAG:
3469 		rc = ethtool_get_dump_flag(dev, useraddr);
3470 		break;
3471 	case ETHTOOL_GET_DUMP_DATA:
3472 		rc = ethtool_get_dump_data(dev, useraddr);
3473 		break;
3474 	case ETHTOOL_GET_TS_INFO:
3475 		rc = ethtool_get_ts_info(dev, useraddr);
3476 		break;
3477 	case ETHTOOL_GMODULEINFO:
3478 		rc = ethtool_get_module_info(dev, useraddr);
3479 		break;
3480 	case ETHTOOL_GMODULEEEPROM:
3481 		rc = ethtool_get_module_eeprom(dev, useraddr);
3482 		break;
3483 	case ETHTOOL_GTUNABLE:
3484 		rc = ethtool_get_tunable(dev, useraddr);
3485 		break;
3486 	case ETHTOOL_STUNABLE:
3487 		rc = ethtool_set_tunable(dev, useraddr);
3488 		break;
3489 	case ETHTOOL_GPHYSTATS:
3490 		rc = ethtool_get_phy_stats(dev, useraddr);
3491 		break;
3492 	case ETHTOOL_PERQUEUE:
3493 		rc = ethtool_set_per_queue(dev, useraddr, sub_cmd);
3494 		break;
3495 	case ETHTOOL_GLINKSETTINGS:
3496 		rc = ethtool_get_link_ksettings(dev, useraddr);
3497 		break;
3498 	case ETHTOOL_SLINKSETTINGS:
3499 		rc = ethtool_set_link_ksettings(dev, useraddr);
3500 		break;
3501 	case ETHTOOL_PHY_GTUNABLE:
3502 		rc = get_phy_tunable(dev, useraddr);
3503 		break;
3504 	case ETHTOOL_PHY_STUNABLE:
3505 		rc = set_phy_tunable(dev, useraddr);
3506 		break;
3507 	case ETHTOOL_GFECPARAM:
3508 		rc = ethtool_get_fecparam(dev, useraddr);
3509 		break;
3510 	case ETHTOOL_SFECPARAM:
3511 		rc = ethtool_set_fecparam(dev, useraddr);
3512 		break;
3513 	default:
3514 		rc = -EOPNOTSUPP;
3515 	}
3516 
3517 	if (dev->ethtool_ops->complete)
3518 		dev->ethtool_ops->complete(dev);
3519 
3520 	if (old_features != dev->features)
3521 		netdev_features_change(dev);
3522 out:
3523 	if (dev->dev.parent)
3524 		pm_runtime_put(dev->dev.parent);
3525 	netdev_unlock_ops(dev);
3526 
3527 	return rc;
3528 }
3529 
3530 int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *useraddr)
3531 {
3532 	struct ethtool_devlink_compat *state;
3533 	u32 ethcmd;
3534 	int rc;
3535 
3536 	if (copy_from_user(&ethcmd, useraddr, sizeof(ethcmd)))
3537 		return -EFAULT;
3538 
3539 	state = kzalloc(sizeof(*state), GFP_KERNEL);
3540 	if (!state)
3541 		return -ENOMEM;
3542 
3543 	switch (ethcmd) {
3544 	case ETHTOOL_FLASHDEV:
3545 		if (copy_from_user(&state->efl, useraddr, sizeof(state->efl))) {
3546 			rc = -EFAULT;
3547 			goto exit_free;
3548 		}
3549 		state->efl.data[ETHTOOL_FLASH_MAX_FILENAME - 1] = 0;
3550 		break;
3551 	}
3552 
3553 	rtnl_lock();
3554 	rc = __dev_ethtool(net, ifr, useraddr, ethcmd, state);
3555 	rtnl_unlock();
3556 	if (rc)
3557 		goto exit_free;
3558 
3559 	switch (ethcmd) {
3560 	case ETHTOOL_FLASHDEV:
3561 		if (state->devlink)
3562 			rc = devlink_compat_flash_update(state->devlink,
3563 							 state->efl.data);
3564 		break;
3565 	case ETHTOOL_GDRVINFO:
3566 		if (state->devlink)
3567 			devlink_compat_running_version(state->devlink,
3568 						       state->info.fw_version,
3569 						       sizeof(state->info.fw_version));
3570 		if (copy_to_user(useraddr, &state->info, sizeof(state->info))) {
3571 			rc = -EFAULT;
3572 			goto exit_free;
3573 		}
3574 		break;
3575 	}
3576 
3577 exit_free:
3578 	if (state->devlink)
3579 		devlink_put(state->devlink);
3580 	kfree(state);
3581 	return rc;
3582 }
3583 
3584 struct ethtool_rx_flow_key {
3585 	struct flow_dissector_key_basic			basic;
3586 	union {
3587 		struct flow_dissector_key_ipv4_addrs	ipv4;
3588 		struct flow_dissector_key_ipv6_addrs	ipv6;
3589 	};
3590 	struct flow_dissector_key_ports			tp;
3591 	struct flow_dissector_key_ip			ip;
3592 	struct flow_dissector_key_vlan			vlan;
3593 	struct flow_dissector_key_eth_addrs		eth_addrs;
3594 } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
3595 
3596 struct ethtool_rx_flow_match {
3597 	struct flow_dissector		dissector;
3598 	struct ethtool_rx_flow_key	key;
3599 	struct ethtool_rx_flow_key	mask;
3600 };
3601 
3602 struct ethtool_rx_flow_rule *
3603 ethtool_rx_flow_rule_create(const struct ethtool_rx_flow_spec_input *input)
3604 {
3605 	const struct ethtool_rx_flow_spec *fs = input->fs;
3606 	struct ethtool_rx_flow_match *match;
3607 	struct ethtool_rx_flow_rule *flow;
3608 	struct flow_action_entry *act;
3609 
3610 	flow = kzalloc(sizeof(struct ethtool_rx_flow_rule) +
3611 		       sizeof(struct ethtool_rx_flow_match), GFP_KERNEL);
3612 	if (!flow)
3613 		return ERR_PTR(-ENOMEM);
3614 
3615 	/* ethtool_rx supports only one single action per rule. */
3616 	flow->rule = flow_rule_alloc(1);
3617 	if (!flow->rule) {
3618 		kfree(flow);
3619 		return ERR_PTR(-ENOMEM);
3620 	}
3621 
3622 	match = (struct ethtool_rx_flow_match *)flow->priv;
3623 	flow->rule->match.dissector	= &match->dissector;
3624 	flow->rule->match.mask		= &match->mask;
3625 	flow->rule->match.key		= &match->key;
3626 
3627 	match->mask.basic.n_proto = htons(0xffff);
3628 
3629 	switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS)) {
3630 	case ETHER_FLOW: {
3631 		const struct ethhdr *ether_spec, *ether_m_spec;
3632 
3633 		ether_spec = &fs->h_u.ether_spec;
3634 		ether_m_spec = &fs->m_u.ether_spec;
3635 
3636 		if (!is_zero_ether_addr(ether_m_spec->h_source)) {
3637 			ether_addr_copy(match->key.eth_addrs.src,
3638 					ether_spec->h_source);
3639 			ether_addr_copy(match->mask.eth_addrs.src,
3640 					ether_m_spec->h_source);
3641 		}
3642 		if (!is_zero_ether_addr(ether_m_spec->h_dest)) {
3643 			ether_addr_copy(match->key.eth_addrs.dst,
3644 					ether_spec->h_dest);
3645 			ether_addr_copy(match->mask.eth_addrs.dst,
3646 					ether_m_spec->h_dest);
3647 		}
3648 		if (ether_m_spec->h_proto) {
3649 			match->key.basic.n_proto = ether_spec->h_proto;
3650 			match->mask.basic.n_proto = ether_m_spec->h_proto;
3651 		}
3652 		}
3653 		break;
3654 	case TCP_V4_FLOW:
3655 	case UDP_V4_FLOW: {
3656 		const struct ethtool_tcpip4_spec *v4_spec, *v4_m_spec;
3657 
3658 		match->key.basic.n_proto = htons(ETH_P_IP);
3659 
3660 		v4_spec = &fs->h_u.tcp_ip4_spec;
3661 		v4_m_spec = &fs->m_u.tcp_ip4_spec;
3662 
3663 		if (v4_m_spec->ip4src) {
3664 			match->key.ipv4.src = v4_spec->ip4src;
3665 			match->mask.ipv4.src = v4_m_spec->ip4src;
3666 		}
3667 		if (v4_m_spec->ip4dst) {
3668 			match->key.ipv4.dst = v4_spec->ip4dst;
3669 			match->mask.ipv4.dst = v4_m_spec->ip4dst;
3670 		}
3671 		if (v4_m_spec->ip4src ||
3672 		    v4_m_spec->ip4dst) {
3673 			match->dissector.used_keys |=
3674 				BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS);
3675 			match->dissector.offset[FLOW_DISSECTOR_KEY_IPV4_ADDRS] =
3676 				offsetof(struct ethtool_rx_flow_key, ipv4);
3677 		}
3678 		if (v4_m_spec->psrc) {
3679 			match->key.tp.src = v4_spec->psrc;
3680 			match->mask.tp.src = v4_m_spec->psrc;
3681 		}
3682 		if (v4_m_spec->pdst) {
3683 			match->key.tp.dst = v4_spec->pdst;
3684 			match->mask.tp.dst = v4_m_spec->pdst;
3685 		}
3686 		if (v4_m_spec->psrc ||
3687 		    v4_m_spec->pdst) {
3688 			match->dissector.used_keys |=
3689 				BIT_ULL(FLOW_DISSECTOR_KEY_PORTS);
3690 			match->dissector.offset[FLOW_DISSECTOR_KEY_PORTS] =
3691 				offsetof(struct ethtool_rx_flow_key, tp);
3692 		}
3693 		if (v4_m_spec->tos) {
3694 			match->key.ip.tos = v4_spec->tos;
3695 			match->mask.ip.tos = v4_m_spec->tos;
3696 			match->dissector.used_keys |=
3697 				BIT(FLOW_DISSECTOR_KEY_IP);
3698 			match->dissector.offset[FLOW_DISSECTOR_KEY_IP] =
3699 				offsetof(struct ethtool_rx_flow_key, ip);
3700 		}
3701 		}
3702 		break;
3703 	case TCP_V6_FLOW:
3704 	case UDP_V6_FLOW: {
3705 		const struct ethtool_tcpip6_spec *v6_spec, *v6_m_spec;
3706 
3707 		match->key.basic.n_proto = htons(ETH_P_IPV6);
3708 
3709 		v6_spec = &fs->h_u.tcp_ip6_spec;
3710 		v6_m_spec = &fs->m_u.tcp_ip6_spec;
3711 		if (!ipv6_addr_any((struct in6_addr *)v6_m_spec->ip6src)) {
3712 			memcpy(&match->key.ipv6.src, v6_spec->ip6src,
3713 			       sizeof(match->key.ipv6.src));
3714 			memcpy(&match->mask.ipv6.src, v6_m_spec->ip6src,
3715 			       sizeof(match->mask.ipv6.src));
3716 		}
3717 		if (!ipv6_addr_any((struct in6_addr *)v6_m_spec->ip6dst)) {
3718 			memcpy(&match->key.ipv6.dst, v6_spec->ip6dst,
3719 			       sizeof(match->key.ipv6.dst));
3720 			memcpy(&match->mask.ipv6.dst, v6_m_spec->ip6dst,
3721 			       sizeof(match->mask.ipv6.dst));
3722 		}
3723 		if (!ipv6_addr_any((struct in6_addr *)v6_m_spec->ip6src) ||
3724 		    !ipv6_addr_any((struct in6_addr *)v6_m_spec->ip6dst)) {
3725 			match->dissector.used_keys |=
3726 				BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS);
3727 			match->dissector.offset[FLOW_DISSECTOR_KEY_IPV6_ADDRS] =
3728 				offsetof(struct ethtool_rx_flow_key, ipv6);
3729 		}
3730 		if (v6_m_spec->psrc) {
3731 			match->key.tp.src = v6_spec->psrc;
3732 			match->mask.tp.src = v6_m_spec->psrc;
3733 		}
3734 		if (v6_m_spec->pdst) {
3735 			match->key.tp.dst = v6_spec->pdst;
3736 			match->mask.tp.dst = v6_m_spec->pdst;
3737 		}
3738 		if (v6_m_spec->psrc ||
3739 		    v6_m_spec->pdst) {
3740 			match->dissector.used_keys |=
3741 				BIT_ULL(FLOW_DISSECTOR_KEY_PORTS);
3742 			match->dissector.offset[FLOW_DISSECTOR_KEY_PORTS] =
3743 				offsetof(struct ethtool_rx_flow_key, tp);
3744 		}
3745 		if (v6_m_spec->tclass) {
3746 			match->key.ip.tos = v6_spec->tclass;
3747 			match->mask.ip.tos = v6_m_spec->tclass;
3748 			match->dissector.used_keys |=
3749 				BIT_ULL(FLOW_DISSECTOR_KEY_IP);
3750 			match->dissector.offset[FLOW_DISSECTOR_KEY_IP] =
3751 				offsetof(struct ethtool_rx_flow_key, ip);
3752 		}
3753 		}
3754 		break;
3755 	default:
3756 		ethtool_rx_flow_rule_destroy(flow);
3757 		return ERR_PTR(-EINVAL);
3758 	}
3759 
3760 	switch (fs->flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS)) {
3761 	case TCP_V4_FLOW:
3762 	case TCP_V6_FLOW:
3763 		match->key.basic.ip_proto = IPPROTO_TCP;
3764 		match->mask.basic.ip_proto = 0xff;
3765 		break;
3766 	case UDP_V4_FLOW:
3767 	case UDP_V6_FLOW:
3768 		match->key.basic.ip_proto = IPPROTO_UDP;
3769 		match->mask.basic.ip_proto = 0xff;
3770 		break;
3771 	}
3772 
3773 	match->dissector.used_keys |= BIT_ULL(FLOW_DISSECTOR_KEY_BASIC);
3774 	match->dissector.offset[FLOW_DISSECTOR_KEY_BASIC] =
3775 		offsetof(struct ethtool_rx_flow_key, basic);
3776 
3777 	if (fs->flow_type & FLOW_EXT) {
3778 		const struct ethtool_flow_ext *ext_h_spec = &fs->h_ext;
3779 		const struct ethtool_flow_ext *ext_m_spec = &fs->m_ext;
3780 
3781 		if (ext_m_spec->vlan_etype) {
3782 			match->key.vlan.vlan_tpid = ext_h_spec->vlan_etype;
3783 			match->mask.vlan.vlan_tpid = ext_m_spec->vlan_etype;
3784 		}
3785 
3786 		if (ext_m_spec->vlan_tci) {
3787 			match->key.vlan.vlan_id =
3788 				ntohs(ext_h_spec->vlan_tci) & 0x0fff;
3789 			match->mask.vlan.vlan_id =
3790 				ntohs(ext_m_spec->vlan_tci) & 0x0fff;
3791 
3792 			match->key.vlan.vlan_dei =
3793 				!!(ext_h_spec->vlan_tci & htons(0x1000));
3794 			match->mask.vlan.vlan_dei =
3795 				!!(ext_m_spec->vlan_tci & htons(0x1000));
3796 
3797 			match->key.vlan.vlan_priority =
3798 				(ntohs(ext_h_spec->vlan_tci) & 0xe000) >> 13;
3799 			match->mask.vlan.vlan_priority =
3800 				(ntohs(ext_m_spec->vlan_tci) & 0xe000) >> 13;
3801 		}
3802 
3803 		if (ext_m_spec->vlan_etype ||
3804 		    ext_m_spec->vlan_tci) {
3805 			match->dissector.used_keys |=
3806 				BIT_ULL(FLOW_DISSECTOR_KEY_VLAN);
3807 			match->dissector.offset[FLOW_DISSECTOR_KEY_VLAN] =
3808 				offsetof(struct ethtool_rx_flow_key, vlan);
3809 		}
3810 	}
3811 	if (fs->flow_type & FLOW_MAC_EXT) {
3812 		const struct ethtool_flow_ext *ext_h_spec = &fs->h_ext;
3813 		const struct ethtool_flow_ext *ext_m_spec = &fs->m_ext;
3814 
3815 		memcpy(match->key.eth_addrs.dst, ext_h_spec->h_dest,
3816 		       ETH_ALEN);
3817 		memcpy(match->mask.eth_addrs.dst, ext_m_spec->h_dest,
3818 		       ETH_ALEN);
3819 
3820 		match->dissector.used_keys |=
3821 			BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS);
3822 		match->dissector.offset[FLOW_DISSECTOR_KEY_ETH_ADDRS] =
3823 			offsetof(struct ethtool_rx_flow_key, eth_addrs);
3824 	}
3825 
3826 	act = &flow->rule->action.entries[0];
3827 	switch (fs->ring_cookie) {
3828 	case RX_CLS_FLOW_DISC:
3829 		act->id = FLOW_ACTION_DROP;
3830 		break;
3831 	case RX_CLS_FLOW_WAKE:
3832 		act->id = FLOW_ACTION_WAKE;
3833 		break;
3834 	default:
3835 		act->id = FLOW_ACTION_QUEUE;
3836 		if (fs->flow_type & FLOW_RSS)
3837 			act->queue.ctx = input->rss_ctx;
3838 
3839 		act->queue.vf = ethtool_get_flow_spec_ring_vf(fs->ring_cookie);
3840 		act->queue.index = ethtool_get_flow_spec_ring(fs->ring_cookie);
3841 		break;
3842 	}
3843 
3844 	return flow;
3845 }
3846 EXPORT_SYMBOL(ethtool_rx_flow_rule_create);
3847 
3848 void ethtool_rx_flow_rule_destroy(struct ethtool_rx_flow_rule *flow)
3849 {
3850 	kfree(flow->rule);
3851 	kfree(flow);
3852 }
3853 EXPORT_SYMBOL(ethtool_rx_flow_rule_destroy);
3854