1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* net/sched/sch_teql.c "True" (or "trivial") link equalizer. 3 * 4 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 5 */ 6 7 #include <linux/module.h> 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/string.h> 12 #include <linux/errno.h> 13 #include <linux/if_arp.h> 14 #include <linux/netdevice.h> 15 #include <linux/init.h> 16 #include <linux/skbuff.h> 17 #include <linux/moduleparam.h> 18 #include <net/arp.h> 19 #include <net/dst.h> 20 #include <net/ndisc.h> 21 #include <net/neighbour.h> 22 #include <net/pkt_sched.h> 23 24 /* 25 How to setup it. 26 ---------------- 27 28 After loading this module you will find a new device teqlN 29 and new qdisc with the same name. To join a slave to the equalizer 30 you should just set this qdisc on a device f.e. 31 32 # tc qdisc add dev eth0 root teql0 33 # tc qdisc add dev eth1 root teql0 34 35 That's all. Full PnP 8) 36 37 Applicability. 38 -------------- 39 40 1. Slave devices MUST be active devices, i.e., they must raise the tbusy 41 signal and generate EOI events. If you want to equalize virtual devices 42 like tunnels, use a normal eql device. 43 2. This device puts no limitations on physical slave characteristics 44 f.e. it will equalize 9600baud line and 100Mb ethernet perfectly :-) 45 Certainly, large difference in link speeds will make the resulting 46 eqalized link unusable, because of huge packet reordering. 47 I estimate an upper useful difference as ~10 times. 48 3. If the slave requires address resolution, only protocols using 49 neighbour cache (IPv4/IPv6) will work over the equalized link. 50 Other protocols are still allowed to use the slave device directly, 51 which will not break load balancing, though native slave 52 traffic will have the highest priority. */ 53 54 struct teql_master { 55 struct Qdisc_ops qops; 56 struct net_device *dev; 57 struct Qdisc __rcu *slaves; 58 spinlock_t slaves_lock; /* serializes writes to ->slaves */ 59 struct list_head master_list; 60 unsigned long tx_bytes; 61 unsigned long tx_packets; 62 unsigned long tx_errors; 63 unsigned long tx_dropped; 64 }; 65 66 struct teql_sched_data { 67 struct Qdisc __rcu *next; 68 struct teql_master *m; 69 struct sk_buff_head q; 70 }; 71 72 #define NEXT_SLAVE(q) (((struct teql_sched_data *)qdisc_priv(q))->next) 73 74 #define FMASK (IFF_BROADCAST | IFF_POINTOPOINT) 75 76 /* "teql*" qdisc routines */ 77 78 static int 79 teql_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free) 80 { 81 struct net_device *dev = qdisc_dev(sch); 82 struct teql_sched_data *q = qdisc_priv(sch); 83 84 if (q->q.qlen < READ_ONCE(dev->tx_queue_len)) { 85 __skb_queue_tail(&q->q, skb); 86 return NET_XMIT_SUCCESS; 87 } 88 89 return qdisc_drop(skb, sch, to_free); 90 } 91 92 static struct sk_buff * 93 teql_dequeue(struct Qdisc *sch) 94 { 95 struct teql_sched_data *dat = qdisc_priv(sch); 96 struct netdev_queue *dat_queue; 97 struct sk_buff *skb; 98 struct Qdisc *q; 99 100 skb = __skb_dequeue(&dat->q); 101 dat_queue = netdev_get_tx_queue(dat->m->dev, 0); 102 q = rcu_dereference_bh(dat_queue->qdisc); 103 104 if (skb == NULL) { 105 struct net_device *m = qdisc_dev(q); 106 if (m) { 107 spin_lock_bh(&dat->m->slaves_lock); 108 rcu_assign_pointer(dat->m->slaves, sch); 109 spin_unlock_bh(&dat->m->slaves_lock); 110 netif_wake_queue(m); 111 } 112 } else { 113 qdisc_bstats_update(sch, skb); 114 } 115 WRITE_ONCE(sch->q.qlen, dat->q.qlen + READ_ONCE(q->q.qlen)); 116 return skb; 117 } 118 119 static struct sk_buff * 120 teql_peek(struct Qdisc *sch) 121 { 122 /* teql is meant to be used as root qdisc */ 123 return NULL; 124 } 125 126 static void 127 teql_reset(struct Qdisc *sch) 128 { 129 struct teql_sched_data *dat = qdisc_priv(sch); 130 131 skb_queue_purge(&dat->q); 132 } 133 134 static void 135 teql_destroy(struct Qdisc *sch) 136 { 137 struct Qdisc *q, *prev; 138 struct teql_sched_data *dat = qdisc_priv(sch); 139 struct teql_master *master = dat->m; 140 struct netdev_queue *txq = NULL; 141 bool reset_master_queue = false; 142 143 if (!master) 144 return; 145 146 spin_lock_bh(&master->slaves_lock); 147 prev = rcu_dereference_protected(master->slaves, 148 lockdep_is_held(&master->slaves_lock)); 149 if (prev) { 150 do { 151 struct Qdisc *head, *next; 152 153 q = rcu_dereference_protected(NEXT_SLAVE(prev), 154 lockdep_is_held(&master->slaves_lock)); 155 if (q != sch) { 156 prev = q; 157 continue; 158 } 159 160 next = rcu_dereference_protected(NEXT_SLAVE(q), 161 lockdep_is_held(&master->slaves_lock)); 162 rcu_assign_pointer(NEXT_SLAVE(prev), next); 163 164 head = rcu_dereference_protected(master->slaves, 165 lockdep_is_held(&master->slaves_lock)); 166 if (q == head) { 167 rcu_assign_pointer(master->slaves, next); 168 if (q == next) { 169 txq = netdev_get_tx_queue(master->dev, 0); 170 rcu_assign_pointer(master->slaves, NULL); 171 reset_master_queue = true; 172 } 173 } 174 skb_queue_purge(&dat->q); 175 break; 176 } while (prev != rcu_dereference_protected(master->slaves, 177 lockdep_is_held(&master->slaves_lock))); 178 } 179 spin_unlock_bh(&master->slaves_lock); 180 181 if (reset_master_queue) 182 dev_reset_queue(master->dev, txq, NULL); 183 } 184 185 static int teql_qdisc_init(struct Qdisc *sch, struct nlattr *opt, 186 struct netlink_ext_ack *extack) 187 { 188 struct net_device *dev = qdisc_dev(sch); 189 struct teql_master *m = (struct teql_master *)sch->ops; 190 struct teql_sched_data *q = qdisc_priv(sch); 191 struct Qdisc *first; 192 193 if (dev->hard_header_len > m->dev->hard_header_len) 194 return -EINVAL; 195 196 if (m->dev == dev) 197 return -ELOOP; 198 199 if (sch->parent != TC_H_ROOT) { 200 NL_SET_ERR_MSG_MOD(extack, "teql can only be used as root"); 201 return -EOPNOTSUPP; 202 } 203 204 q->m = m; 205 206 skb_queue_head_init(&q->q); 207 208 spin_lock_bh(&m->slaves_lock); 209 first = rcu_dereference_protected(m->slaves, lockdep_is_held(&m->slaves_lock)); 210 if (first) { 211 if (m->dev->flags & IFF_UP) { 212 if ((m->dev->flags & IFF_POINTOPOINT && 213 !(dev->flags & IFF_POINTOPOINT)) || 214 (m->dev->flags & IFF_BROADCAST && 215 !(dev->flags & IFF_BROADCAST)) || 216 (m->dev->flags & IFF_MULTICAST && 217 !(dev->flags & IFF_MULTICAST)) || 218 dev->mtu < m->dev->mtu) { 219 spin_unlock_bh(&m->slaves_lock); 220 return -EINVAL; 221 } 222 } else { 223 if (!(dev->flags&IFF_POINTOPOINT)) 224 m->dev->flags &= ~IFF_POINTOPOINT; 225 if (!(dev->flags&IFF_BROADCAST)) 226 m->dev->flags &= ~IFF_BROADCAST; 227 if (!(dev->flags&IFF_MULTICAST)) 228 m->dev->flags &= ~IFF_MULTICAST; 229 if (dev->mtu < m->dev->mtu) 230 m->dev->mtu = dev->mtu; 231 } 232 rcu_assign_pointer(q->next, 233 rcu_dereference_protected(NEXT_SLAVE(first), 234 lockdep_is_held(&m->slaves_lock))); 235 rcu_assign_pointer(NEXT_SLAVE(first), sch); 236 } else { 237 rcu_assign_pointer(q->next, sch); 238 rcu_assign_pointer(m->slaves, sch); 239 m->dev->mtu = dev->mtu; 240 m->dev->flags = (m->dev->flags&~FMASK)|(dev->flags&FMASK); 241 } 242 spin_unlock_bh(&m->slaves_lock); 243 return 0; 244 } 245 246 247 static int 248 __teql_resolve(struct sk_buff *skb, struct sk_buff *skb_res, 249 struct net_device *dev, struct netdev_queue *txq, 250 struct dst_entry *dst) 251 { 252 struct neighbour *n; 253 int err = 0; 254 255 n = dst_neigh_lookup_skb(dst, skb); 256 if (!n) 257 return -ENOENT; 258 259 if (dst->dev != dev) { 260 struct neighbour *mn; 261 262 #if IS_ENABLED(CONFIG_IPV6) 263 if (n->tbl->family == AF_INET6) { 264 mn = ipv6_neigh_lookup(dev, n->primary_key); 265 if (!mn) 266 mn = ipv6_neigh_create(dev, n->primary_key); 267 } else 268 #endif 269 { 270 mn = ipv4_neigh_lookup(dev, n->primary_key); 271 if (!mn) 272 mn = ipv4_neigh_create(dev, n->primary_key); 273 } 274 neigh_release(n); 275 if (IS_ERR(mn)) 276 return PTR_ERR(mn); 277 n = mn; 278 } 279 280 if (neigh_event_send(n, skb_res) == 0) { 281 char haddr[MAX_ADDR_LEN]; 282 283 neigh_ha_snapshot(haddr, n, dev); 284 if (dev_hard_header(skb, dev, ntohs(skb_protocol(skb, false)), 285 haddr, NULL, skb->len) < 0) 286 err = -EINVAL; 287 } else { 288 err = (skb_res == NULL) ? -EAGAIN : 1; 289 } 290 neigh_release(n); 291 return err; 292 } 293 294 static inline int teql_resolve(struct sk_buff *skb, 295 struct sk_buff *skb_res, 296 struct net_device *dev, 297 struct netdev_queue *txq) 298 { 299 struct dst_entry *dst = skb_dst(skb); 300 int res; 301 302 if (rcu_access_pointer(txq->qdisc) == &noop_qdisc) 303 return -ENODEV; 304 305 if (!dev->header_ops || !dst) 306 return 0; 307 308 rcu_read_lock(); 309 res = __teql_resolve(skb, skb_res, dev, txq, dst); 310 rcu_read_unlock(); 311 312 return res; 313 } 314 315 static netdev_tx_t teql_master_xmit(struct sk_buff *skb, struct net_device *dev) 316 { 317 struct teql_master *master = netdev_priv(dev); 318 struct Qdisc *start, *q; 319 int busy; 320 int nores; 321 int subq = skb_get_queue_mapping(skb); 322 struct sk_buff *skb_res = NULL; 323 324 restart: 325 nores = 0; 326 busy = 0; 327 328 rcu_read_lock(); 329 330 start = rcu_dereference(master->slaves); 331 332 q = start; 333 if (!q) 334 goto drop; 335 336 do { 337 struct net_device *slave = qdisc_dev(q); 338 struct netdev_queue *slave_txq = netdev_get_tx_queue(slave, 0); 339 340 if (rcu_access_pointer(slave_txq->qdisc_sleeping) != q) 341 continue; 342 if (netif_xmit_stopped(netdev_get_tx_queue(slave, subq)) || 343 !netif_running(slave)) { 344 busy = 1; 345 continue; 346 } 347 348 switch (teql_resolve(skb, skb_res, slave, slave_txq)) { 349 case 0: 350 if (__netif_tx_trylock(slave_txq)) { 351 unsigned int length = qdisc_pkt_len(skb); 352 353 skb->dev = slave; 354 if (!netif_xmit_frozen_or_stopped(slave_txq) && 355 netdev_start_xmit(skb, slave, slave_txq, false) == 356 NETDEV_TX_OK) { 357 __netif_tx_unlock(slave_txq); 358 spin_lock(&master->slaves_lock); 359 if (rcu_dereference_protected(master->slaves, 360 lockdep_is_held(&master->slaves_lock)) == q) 361 rcu_assign_pointer(master->slaves, 362 rcu_dereference_protected(NEXT_SLAVE(q), 363 lockdep_is_held(&master->slaves_lock))); 364 spin_unlock(&master->slaves_lock); 365 netif_wake_queue(dev); 366 master->tx_packets++; 367 master->tx_bytes += length; 368 rcu_read_unlock(); 369 return NETDEV_TX_OK; 370 } 371 __netif_tx_unlock(slave_txq); 372 } 373 if (netif_xmit_stopped(netdev_get_tx_queue(dev, 0))) 374 busy = 1; 375 break; 376 case 1: 377 spin_lock(&master->slaves_lock); 378 if (rcu_dereference_protected(master->slaves, 379 lockdep_is_held(&master->slaves_lock)) == q) 380 rcu_assign_pointer(master->slaves, 381 rcu_dereference_protected(NEXT_SLAVE(q), 382 lockdep_is_held(&master->slaves_lock))); 383 spin_unlock(&master->slaves_lock); 384 rcu_read_unlock(); 385 return NETDEV_TX_OK; 386 default: 387 nores = 1; 388 break; 389 } 390 skb->dev = dev; 391 __skb_pull(skb, skb_network_offset(skb)); 392 } while ((q = rcu_dereference(NEXT_SLAVE(q))) != start); 393 394 if (nores && skb_res == NULL) { 395 skb_res = skb; 396 rcu_read_unlock(); 397 goto restart; 398 } 399 400 if (busy) { 401 netif_stop_queue(dev); 402 rcu_read_unlock(); 403 return NETDEV_TX_BUSY; 404 } 405 master->tx_errors++; 406 407 drop: 408 master->tx_dropped++; 409 rcu_read_unlock(); 410 dev_kfree_skb(skb); 411 return NETDEV_TX_OK; 412 } 413 414 static int teql_master_open(struct net_device *dev) 415 { 416 struct Qdisc *q, *first; 417 struct teql_master *m = netdev_priv(dev); 418 int mtu = 0xFFFE; 419 unsigned int flags = IFF_NOARP | IFF_MULTICAST; 420 421 first = rtnl_dereference(m->slaves); 422 if (!first) 423 return -EUNATCH; 424 425 flags = FMASK; 426 427 q = first; 428 do { 429 struct net_device *slave = qdisc_dev(q); 430 431 if (slave == NULL) 432 return -EUNATCH; 433 434 if (slave->mtu < mtu) 435 mtu = slave->mtu; 436 if (slave->hard_header_len > LL_MAX_HEADER) 437 return -EINVAL; 438 439 /* If all the slaves are BROADCAST, master is BROADCAST 440 If all the slaves are PtP, master is PtP 441 Otherwise, master is NBMA. 442 */ 443 if (!(slave->flags&IFF_POINTOPOINT)) 444 flags &= ~IFF_POINTOPOINT; 445 if (!(slave->flags&IFF_BROADCAST)) 446 flags &= ~IFF_BROADCAST; 447 if (!(slave->flags&IFF_MULTICAST)) 448 flags &= ~IFF_MULTICAST; 449 } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); 450 451 m->dev->mtu = mtu; 452 m->dev->flags = (m->dev->flags&~FMASK) | flags; 453 netif_start_queue(m->dev); 454 return 0; 455 } 456 457 static int teql_master_close(struct net_device *dev) 458 { 459 netif_stop_queue(dev); 460 return 0; 461 } 462 463 static void teql_master_stats64(struct net_device *dev, 464 struct rtnl_link_stats64 *stats) 465 { 466 struct teql_master *m = netdev_priv(dev); 467 468 stats->tx_packets = m->tx_packets; 469 stats->tx_bytes = m->tx_bytes; 470 stats->tx_errors = m->tx_errors; 471 stats->tx_dropped = m->tx_dropped; 472 } 473 474 static int teql_master_mtu(struct net_device *dev, int new_mtu) 475 { 476 struct teql_master *m = netdev_priv(dev); 477 struct Qdisc *q, *first; 478 479 first = rtnl_dereference(m->slaves); 480 q = first; 481 if (q) { 482 do { 483 if (new_mtu > qdisc_dev(q)->mtu) 484 return -EINVAL; 485 } while ((q = rtnl_dereference(NEXT_SLAVE(q))) != first); 486 } 487 488 WRITE_ONCE(dev->mtu, new_mtu); 489 return 0; 490 } 491 492 static const struct net_device_ops teql_netdev_ops = { 493 .ndo_open = teql_master_open, 494 .ndo_stop = teql_master_close, 495 .ndo_start_xmit = teql_master_xmit, 496 .ndo_get_stats64 = teql_master_stats64, 497 .ndo_change_mtu = teql_master_mtu, 498 }; 499 500 static __init void teql_master_setup(struct net_device *dev) 501 { 502 struct teql_master *master = netdev_priv(dev); 503 struct Qdisc_ops *ops = &master->qops; 504 505 spin_lock_init(&master->slaves_lock); 506 master->dev = dev; 507 ops->priv_size = sizeof(struct teql_sched_data); 508 509 ops->enqueue = teql_enqueue; 510 ops->dequeue = teql_dequeue; 511 ops->peek = teql_peek; 512 ops->init = teql_qdisc_init; 513 ops->reset = teql_reset; 514 ops->destroy = teql_destroy; 515 ops->owner = THIS_MODULE; 516 517 dev->netdev_ops = &teql_netdev_ops; 518 dev->type = ARPHRD_VOID; 519 dev->mtu = 1500; 520 dev->min_mtu = 68; 521 dev->max_mtu = 65535; 522 dev->tx_queue_len = 100; 523 dev->flags = IFF_NOARP; 524 dev->hard_header_len = LL_MAX_HEADER; 525 netif_keep_dst(dev); 526 } 527 528 static LIST_HEAD(master_dev_list); 529 static int max_equalizers = 1; 530 module_param(max_equalizers, int, 0); 531 MODULE_PARM_DESC(max_equalizers, "Max number of link equalizers"); 532 533 static int __init teql_init(void) 534 { 535 int i; 536 int err = -ENODEV; 537 538 for (i = 0; i < max_equalizers; i++) { 539 struct net_device *dev; 540 struct teql_master *master; 541 542 dev = alloc_netdev(sizeof(struct teql_master), "teql%d", 543 NET_NAME_UNKNOWN, teql_master_setup); 544 if (!dev) { 545 err = -ENOMEM; 546 break; 547 } 548 549 if ((err = register_netdev(dev))) { 550 free_netdev(dev); 551 break; 552 } 553 554 master = netdev_priv(dev); 555 556 strscpy(master->qops.id, dev->name, IFNAMSIZ); 557 err = register_qdisc(&master->qops); 558 559 if (err) { 560 unregister_netdev(dev); 561 free_netdev(dev); 562 break; 563 } 564 565 list_add_tail(&master->master_list, &master_dev_list); 566 } 567 return i ? 0 : err; 568 } 569 570 static void __exit teql_exit(void) 571 { 572 struct teql_master *master, *nxt; 573 574 list_for_each_entry_safe(master, nxt, &master_dev_list, master_list) { 575 576 list_del(&master->master_list); 577 578 unregister_qdisc(&master->qops); 579 unregister_netdev(master->dev); 580 free_netdev(master->dev); 581 } 582 } 583 584 module_init(teql_init); 585 module_exit(teql_exit); 586 587 MODULE_LICENSE("GPL"); 588 MODULE_DESCRIPTION("True (or trivial) link equalizer qdisc"); 589