1e689cf4aSJeff Kirsher /* sunvnet.c: Sun LDOM Virtual Network Driver. 2e689cf4aSJeff Kirsher * 3e689cf4aSJeff Kirsher * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4e689cf4aSJeff Kirsher */ 5e689cf4aSJeff Kirsher 6e689cf4aSJeff Kirsher #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7e689cf4aSJeff Kirsher 8e689cf4aSJeff Kirsher #include <linux/module.h> 9e689cf4aSJeff Kirsher #include <linux/kernel.h> 10e689cf4aSJeff Kirsher #include <linux/types.h> 11e689cf4aSJeff Kirsher #include <linux/slab.h> 12e689cf4aSJeff Kirsher #include <linux/delay.h> 13e689cf4aSJeff Kirsher #include <linux/init.h> 14e689cf4aSJeff Kirsher #include <linux/netdevice.h> 15e689cf4aSJeff Kirsher #include <linux/ethtool.h> 16e689cf4aSJeff Kirsher #include <linux/etherdevice.h> 17e689cf4aSJeff Kirsher #include <linux/mutex.h> 18e4defc77SDavid L Stevens #include <linux/if_vlan.h> 19e689cf4aSJeff Kirsher 20a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 21a2b78e9bSDavid L Stevens #include <linux/icmpv6.h> 22a2b78e9bSDavid L Stevens #endif 23a2b78e9bSDavid L Stevens 24*6d0ba919SDavid L Stevens #include <net/ip.h> 25a2b78e9bSDavid L Stevens #include <net/icmp.h> 26a2b78e9bSDavid L Stevens #include <net/route.h> 27a2b78e9bSDavid L Stevens 28e689cf4aSJeff Kirsher #include <asm/vio.h> 29e689cf4aSJeff Kirsher #include <asm/ldc.h> 30e689cf4aSJeff Kirsher 31e689cf4aSJeff Kirsher #include "sunvnet.h" 32e689cf4aSJeff Kirsher 33e689cf4aSJeff Kirsher #define DRV_MODULE_NAME "sunvnet" 34e689cf4aSJeff Kirsher #define DRV_MODULE_VERSION "1.0" 35e689cf4aSJeff Kirsher #define DRV_MODULE_RELDATE "June 25, 2007" 36e689cf4aSJeff Kirsher 37f73d12bdSBill Pemberton static char version[] = 38e689cf4aSJeff Kirsher DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 39e689cf4aSJeff Kirsher MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 40e689cf4aSJeff Kirsher MODULE_DESCRIPTION("Sun LDOM virtual network driver"); 41e689cf4aSJeff Kirsher MODULE_LICENSE("GPL"); 42e689cf4aSJeff Kirsher MODULE_VERSION(DRV_MODULE_VERSION); 43e689cf4aSJeff Kirsher 44d51bffd1SSowmini Varadhan #define VNET_MAX_TXQS 16 45d51bffd1SSowmini Varadhan 46adddc32dSSowmini Varadhan /* Heuristic for the number of times to exponentially backoff and 47adddc32dSSowmini Varadhan * retry sending an LDC trigger when EAGAIN is encountered 48adddc32dSSowmini Varadhan */ 49adddc32dSSowmini Varadhan #define VNET_MAX_RETRIES 10 50adddc32dSSowmini Varadhan 51d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start); 52d1015645SSowmini Varadhan 53e689cf4aSJeff Kirsher /* Ordered from largest major to lowest */ 54e689cf4aSJeff Kirsher static struct vio_version vnet_versions[] = { 55*6d0ba919SDavid L Stevens { .major = 1, .minor = 8 }, 56*6d0ba919SDavid L Stevens { .major = 1, .minor = 7 }, 57e4defc77SDavid L Stevens { .major = 1, .minor = 6 }, 58e689cf4aSJeff Kirsher { .major = 1, .minor = 0 }, 59e689cf4aSJeff Kirsher }; 60e689cf4aSJeff Kirsher 61e689cf4aSJeff Kirsher static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr) 62e689cf4aSJeff Kirsher { 63e689cf4aSJeff Kirsher return vio_dring_avail(dr, VNET_TX_RING_SIZE); 64e689cf4aSJeff Kirsher } 65e689cf4aSJeff Kirsher 66e689cf4aSJeff Kirsher static int vnet_handle_unknown(struct vnet_port *port, void *arg) 67e689cf4aSJeff Kirsher { 68e689cf4aSJeff Kirsher struct vio_msg_tag *pkt = arg; 69e689cf4aSJeff Kirsher 70e689cf4aSJeff Kirsher pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n", 71e689cf4aSJeff Kirsher pkt->type, pkt->stype, pkt->stype_env, pkt->sid); 72e689cf4aSJeff Kirsher pr_err("Resetting connection\n"); 73e689cf4aSJeff Kirsher 74e689cf4aSJeff Kirsher ldc_disconnect(port->vio.lp); 75e689cf4aSJeff Kirsher 76e689cf4aSJeff Kirsher return -ECONNRESET; 77e689cf4aSJeff Kirsher } 78e689cf4aSJeff Kirsher 79d6732489SDavid L Stevens static int vnet_port_alloc_tx_ring(struct vnet_port *port); 80d6732489SDavid L Stevens 81e689cf4aSJeff Kirsher static int vnet_send_attr(struct vio_driver_state *vio) 82e689cf4aSJeff Kirsher { 83e689cf4aSJeff Kirsher struct vnet_port *port = to_vnet_port(vio); 84e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 85e689cf4aSJeff Kirsher struct vio_net_attr_info pkt; 86e4defc77SDavid L Stevens int framelen = ETH_FRAME_LEN; 87d6732489SDavid L Stevens int i, err; 88d6732489SDavid L Stevens 89d6732489SDavid L Stevens err = vnet_port_alloc_tx_ring(to_vnet_port(vio)); 90d6732489SDavid L Stevens if (err) 91d6732489SDavid L Stevens return err; 92e689cf4aSJeff Kirsher 93e689cf4aSJeff Kirsher memset(&pkt, 0, sizeof(pkt)); 94e689cf4aSJeff Kirsher pkt.tag.type = VIO_TYPE_CTRL; 95e689cf4aSJeff Kirsher pkt.tag.stype = VIO_SUBTYPE_INFO; 96e689cf4aSJeff Kirsher pkt.tag.stype_env = VIO_ATTR_INFO; 97e689cf4aSJeff Kirsher pkt.tag.sid = vio_send_sid(vio); 98e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2)) 99e689cf4aSJeff Kirsher pkt.xfer_mode = VIO_DRING_MODE; 100e4defc77SDavid L Stevens else 101e4defc77SDavid L Stevens pkt.xfer_mode = VIO_NEW_DRING_MODE; 102e689cf4aSJeff Kirsher pkt.addr_type = VNET_ADDR_ETHERMAC; 103e689cf4aSJeff Kirsher pkt.ack_freq = 0; 104e689cf4aSJeff Kirsher for (i = 0; i < 6; i++) 105e689cf4aSJeff Kirsher pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8); 106e4defc77SDavid L Stevens if (vio_version_after(vio, 1, 3)) { 107e4defc77SDavid L Stevens if (port->rmtu) { 108e4defc77SDavid L Stevens port->rmtu = min(VNET_MAXPACKET, port->rmtu); 109e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 110e4defc77SDavid L Stevens } else { 111e4defc77SDavid L Stevens port->rmtu = VNET_MAXPACKET; 112e4defc77SDavid L Stevens pkt.mtu = port->rmtu; 113e4defc77SDavid L Stevens } 114e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) 115e4defc77SDavid L Stevens pkt.options = VIO_TX_DRING; 116e4defc77SDavid L Stevens } else if (vio_version_before(vio, 1, 3)) { 117e4defc77SDavid L Stevens pkt.mtu = framelen; 118e4defc77SDavid L Stevens } else { /* v1.3 */ 119e4defc77SDavid L Stevens pkt.mtu = framelen + VLAN_HLEN; 120e4defc77SDavid L Stevens } 121e4defc77SDavid L Stevens 122e4defc77SDavid L Stevens pkt.plnk_updt = PHYSLINK_UPDATE_NONE; 123e4defc77SDavid L Stevens pkt.cflags = 0; 124e689cf4aSJeff Kirsher 125e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 126e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 127e4defc77SDavid L Stevens "cflags[0x%04x] lso_max[%u]\n", 128e689cf4aSJeff Kirsher pkt.xfer_mode, pkt.addr_type, 129e689cf4aSJeff Kirsher (unsigned long long)pkt.addr, 130e4defc77SDavid L Stevens pkt.ack_freq, pkt.plnk_updt, pkt.options, 131e4defc77SDavid L Stevens (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen); 132e4defc77SDavid L Stevens 133e689cf4aSJeff Kirsher 134e689cf4aSJeff Kirsher return vio_ldc_send(vio, &pkt, sizeof(pkt)); 135e689cf4aSJeff Kirsher } 136e689cf4aSJeff Kirsher 137e689cf4aSJeff Kirsher static int handle_attr_info(struct vio_driver_state *vio, 138e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 139e689cf4aSJeff Kirsher { 140e4defc77SDavid L Stevens struct vnet_port *port = to_vnet_port(vio); 141e4defc77SDavid L Stevens u64 localmtu; 142e4defc77SDavid L Stevens u8 xfer_mode; 143e4defc77SDavid L Stevens 144e4defc77SDavid L Stevens viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 145e4defc77SDavid L Stevens "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 146e4defc77SDavid L Stevens " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 147e689cf4aSJeff Kirsher pkt->xfer_mode, pkt->addr_type, 148e689cf4aSJeff Kirsher (unsigned long long)pkt->addr, 149e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 150e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 151e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 152e689cf4aSJeff Kirsher 153e689cf4aSJeff Kirsher pkt->tag.sid = vio_send_sid(vio); 154e689cf4aSJeff Kirsher 155e4defc77SDavid L Stevens xfer_mode = pkt->xfer_mode; 156e4defc77SDavid L Stevens /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */ 157e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE) 158e4defc77SDavid L Stevens xfer_mode = VIO_NEW_DRING_MODE; 159e4defc77SDavid L Stevens 160e4defc77SDavid L Stevens /* MTU negotiation: 161e4defc77SDavid L Stevens * < v1.3 - ETH_FRAME_LEN exactly 162e4defc77SDavid L Stevens * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change 163e4defc77SDavid L Stevens * pkt->mtu for ACK 164e4defc77SDavid L Stevens * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly 165e4defc77SDavid L Stevens */ 166e4defc77SDavid L Stevens if (vio_version_before(vio, 1, 3)) { 167e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN; 168e4defc77SDavid L Stevens } else if (vio_version_after(vio, 1, 3)) { 169e4defc77SDavid L Stevens localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET; 170e4defc77SDavid L Stevens localmtu = min(pkt->mtu, localmtu); 171e4defc77SDavid L Stevens pkt->mtu = localmtu; 172e4defc77SDavid L Stevens } else { /* v1.3 */ 173e4defc77SDavid L Stevens localmtu = ETH_FRAME_LEN + VLAN_HLEN; 174e4defc77SDavid L Stevens } 175e4defc77SDavid L Stevens port->rmtu = localmtu; 176e4defc77SDavid L Stevens 177e4defc77SDavid L Stevens /* for version >= 1.6, ACK packet mode we support */ 178e4defc77SDavid L Stevens if (vio_version_after_eq(vio, 1, 6)) { 179e4defc77SDavid L Stevens pkt->xfer_mode = VIO_NEW_DRING_MODE; 180e4defc77SDavid L Stevens pkt->options = VIO_TX_DRING; 181e4defc77SDavid L Stevens } 182e4defc77SDavid L Stevens 183e4defc77SDavid L Stevens if (!(xfer_mode | VIO_NEW_DRING_MODE) || 184e689cf4aSJeff Kirsher pkt->addr_type != VNET_ADDR_ETHERMAC || 185e4defc77SDavid L Stevens pkt->mtu != localmtu) { 186e689cf4aSJeff Kirsher viodbg(HS, "SEND NET ATTR NACK\n"); 187e689cf4aSJeff Kirsher 188e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_NACK; 189e689cf4aSJeff Kirsher 190e689cf4aSJeff Kirsher (void) vio_ldc_send(vio, pkt, sizeof(*pkt)); 191e689cf4aSJeff Kirsher 192e689cf4aSJeff Kirsher return -ECONNRESET; 193e689cf4aSJeff Kirsher } else { 194e4defc77SDavid L Stevens viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] " 195e4defc77SDavid L Stevens "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] " 196e4defc77SDavid L Stevens "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 197e4defc77SDavid L Stevens pkt->xfer_mode, pkt->addr_type, 198e4defc77SDavid L Stevens (unsigned long long)pkt->addr, 199e4defc77SDavid L Stevens pkt->ack_freq, pkt->plnk_updt, pkt->options, 200e4defc77SDavid L Stevens (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 201e4defc77SDavid L Stevens pkt->ipv4_lso_maxlen); 202e689cf4aSJeff Kirsher 203e689cf4aSJeff Kirsher pkt->tag.stype = VIO_SUBTYPE_ACK; 204e689cf4aSJeff Kirsher 205e689cf4aSJeff Kirsher return vio_ldc_send(vio, pkt, sizeof(*pkt)); 206e689cf4aSJeff Kirsher } 207e689cf4aSJeff Kirsher 208e689cf4aSJeff Kirsher } 209e689cf4aSJeff Kirsher 210e689cf4aSJeff Kirsher static int handle_attr_ack(struct vio_driver_state *vio, 211e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 212e689cf4aSJeff Kirsher { 213e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR ACK\n"); 214e689cf4aSJeff Kirsher 215e689cf4aSJeff Kirsher return 0; 216e689cf4aSJeff Kirsher } 217e689cf4aSJeff Kirsher 218e689cf4aSJeff Kirsher static int handle_attr_nack(struct vio_driver_state *vio, 219e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt) 220e689cf4aSJeff Kirsher { 221e689cf4aSJeff Kirsher viodbg(HS, "GOT NET ATTR NACK\n"); 222e689cf4aSJeff Kirsher 223e689cf4aSJeff Kirsher return -ECONNRESET; 224e689cf4aSJeff Kirsher } 225e689cf4aSJeff Kirsher 226e689cf4aSJeff Kirsher static int vnet_handle_attr(struct vio_driver_state *vio, void *arg) 227e689cf4aSJeff Kirsher { 228e689cf4aSJeff Kirsher struct vio_net_attr_info *pkt = arg; 229e689cf4aSJeff Kirsher 230e689cf4aSJeff Kirsher switch (pkt->tag.stype) { 231e689cf4aSJeff Kirsher case VIO_SUBTYPE_INFO: 232e689cf4aSJeff Kirsher return handle_attr_info(vio, pkt); 233e689cf4aSJeff Kirsher 234e689cf4aSJeff Kirsher case VIO_SUBTYPE_ACK: 235e689cf4aSJeff Kirsher return handle_attr_ack(vio, pkt); 236e689cf4aSJeff Kirsher 237e689cf4aSJeff Kirsher case VIO_SUBTYPE_NACK: 238e689cf4aSJeff Kirsher return handle_attr_nack(vio, pkt); 239e689cf4aSJeff Kirsher 240e689cf4aSJeff Kirsher default: 241e689cf4aSJeff Kirsher return -ECONNRESET; 242e689cf4aSJeff Kirsher } 243e689cf4aSJeff Kirsher } 244e689cf4aSJeff Kirsher 245e689cf4aSJeff Kirsher static void vnet_handshake_complete(struct vio_driver_state *vio) 246e689cf4aSJeff Kirsher { 247e689cf4aSJeff Kirsher struct vio_dring_state *dr; 248e689cf4aSJeff Kirsher 249e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_RX_RING]; 250e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 251e689cf4aSJeff Kirsher 252e689cf4aSJeff Kirsher dr = &vio->drings[VIO_DRIVER_TX_RING]; 253e689cf4aSJeff Kirsher dr->snd_nxt = dr->rcv_nxt = 1; 254e689cf4aSJeff Kirsher } 255e689cf4aSJeff Kirsher 256e689cf4aSJeff Kirsher /* The hypervisor interface that implements copying to/from imported 257e689cf4aSJeff Kirsher * memory from another domain requires that copies are done to 8-byte 258e689cf4aSJeff Kirsher * aligned buffers, and that the lengths of such copies are also 8-byte 259e689cf4aSJeff Kirsher * multiples. 260e689cf4aSJeff Kirsher * 261e689cf4aSJeff Kirsher * So we align skb->data to an 8-byte multiple and pad-out the data 262e689cf4aSJeff Kirsher * area so we can round the copy length up to the next multiple of 263e689cf4aSJeff Kirsher * 8 for the copy. 264e689cf4aSJeff Kirsher * 265e689cf4aSJeff Kirsher * The transmitter puts the actual start of the packet 6 bytes into 266e689cf4aSJeff Kirsher * the buffer it sends over, so that the IP headers after the ethernet 267e689cf4aSJeff Kirsher * header are aligned properly. These 6 bytes are not in the descriptor 268e689cf4aSJeff Kirsher * length, they are simply implied. This offset is represented using 269e689cf4aSJeff Kirsher * the VNET_PACKET_SKIP macro. 270e689cf4aSJeff Kirsher */ 271e689cf4aSJeff Kirsher static struct sk_buff *alloc_and_align_skb(struct net_device *dev, 272e689cf4aSJeff Kirsher unsigned int len) 273e689cf4aSJeff Kirsher { 274e689cf4aSJeff Kirsher struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8); 275e689cf4aSJeff Kirsher unsigned long addr, off; 276e689cf4aSJeff Kirsher 277e689cf4aSJeff Kirsher if (unlikely(!skb)) 278e689cf4aSJeff Kirsher return NULL; 279e689cf4aSJeff Kirsher 280e689cf4aSJeff Kirsher addr = (unsigned long) skb->data; 281e689cf4aSJeff Kirsher off = ((addr + 7UL) & ~7UL) - addr; 282e689cf4aSJeff Kirsher if (off) 283e689cf4aSJeff Kirsher skb_reserve(skb, off); 284e689cf4aSJeff Kirsher 285e689cf4aSJeff Kirsher return skb; 286e689cf4aSJeff Kirsher } 287e689cf4aSJeff Kirsher 288*6d0ba919SDavid L Stevens static inline void vnet_fullcsum(struct sk_buff *skb) 289*6d0ba919SDavid L Stevens { 290*6d0ba919SDavid L Stevens struct iphdr *iph = ip_hdr(skb); 291*6d0ba919SDavid L Stevens int offset = skb_transport_offset(skb); 292*6d0ba919SDavid L Stevens 293*6d0ba919SDavid L Stevens if (skb->protocol != htons(ETH_P_IP)) 294*6d0ba919SDavid L Stevens return; 295*6d0ba919SDavid L Stevens if (iph->protocol != IPPROTO_TCP && 296*6d0ba919SDavid L Stevens iph->protocol != IPPROTO_UDP) 297*6d0ba919SDavid L Stevens return; 298*6d0ba919SDavid L Stevens skb->ip_summed = CHECKSUM_NONE; 299*6d0ba919SDavid L Stevens skb->csum_level = 1; 300*6d0ba919SDavid L Stevens skb->csum = 0; 301*6d0ba919SDavid L Stevens if (iph->protocol == IPPROTO_TCP) { 302*6d0ba919SDavid L Stevens struct tcphdr *ptcp = tcp_hdr(skb); 303*6d0ba919SDavid L Stevens 304*6d0ba919SDavid L Stevens ptcp->check = 0; 305*6d0ba919SDavid L Stevens skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 306*6d0ba919SDavid L Stevens ptcp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 307*6d0ba919SDavid L Stevens skb->len - offset, IPPROTO_TCP, 308*6d0ba919SDavid L Stevens skb->csum); 309*6d0ba919SDavid L Stevens } else if (iph->protocol == IPPROTO_UDP) { 310*6d0ba919SDavid L Stevens struct udphdr *pudp = udp_hdr(skb); 311*6d0ba919SDavid L Stevens 312*6d0ba919SDavid L Stevens pudp->check = 0; 313*6d0ba919SDavid L Stevens skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 314*6d0ba919SDavid L Stevens pudp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 315*6d0ba919SDavid L Stevens skb->len - offset, IPPROTO_UDP, 316*6d0ba919SDavid L Stevens skb->csum); 317*6d0ba919SDavid L Stevens } 318*6d0ba919SDavid L Stevens } 319*6d0ba919SDavid L Stevens 320*6d0ba919SDavid L Stevens static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc) 321e689cf4aSJeff Kirsher { 322e689cf4aSJeff Kirsher struct net_device *dev = port->vp->dev; 323*6d0ba919SDavid L Stevens unsigned int len = desc->size; 324e689cf4aSJeff Kirsher unsigned int copy_len; 325e689cf4aSJeff Kirsher struct sk_buff *skb; 326e689cf4aSJeff Kirsher int err; 327e689cf4aSJeff Kirsher 328e689cf4aSJeff Kirsher err = -EMSGSIZE; 329e4defc77SDavid L Stevens if (unlikely(len < ETH_ZLEN || len > port->rmtu)) { 330e689cf4aSJeff Kirsher dev->stats.rx_length_errors++; 331e689cf4aSJeff Kirsher goto out_dropped; 332e689cf4aSJeff Kirsher } 333e689cf4aSJeff Kirsher 334e689cf4aSJeff Kirsher skb = alloc_and_align_skb(dev, len); 335e689cf4aSJeff Kirsher err = -ENOMEM; 336e689cf4aSJeff Kirsher if (unlikely(!skb)) { 337e689cf4aSJeff Kirsher dev->stats.rx_missed_errors++; 338e689cf4aSJeff Kirsher goto out_dropped; 339e689cf4aSJeff Kirsher } 340e689cf4aSJeff Kirsher 341e689cf4aSJeff Kirsher copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U; 342e689cf4aSJeff Kirsher skb_put(skb, copy_len); 343e689cf4aSJeff Kirsher err = ldc_copy(port->vio.lp, LDC_COPY_IN, 344e689cf4aSJeff Kirsher skb->data, copy_len, 0, 345*6d0ba919SDavid L Stevens desc->cookies, desc->ncookies); 346e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 347e689cf4aSJeff Kirsher dev->stats.rx_frame_errors++; 348e689cf4aSJeff Kirsher goto out_free_skb; 349e689cf4aSJeff Kirsher } 350e689cf4aSJeff Kirsher 351e689cf4aSJeff Kirsher skb_pull(skb, VNET_PACKET_SKIP); 352e689cf4aSJeff Kirsher skb_trim(skb, len); 353e689cf4aSJeff Kirsher skb->protocol = eth_type_trans(skb, dev); 354e689cf4aSJeff Kirsher 355*6d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 8)) { 356*6d0ba919SDavid L Stevens struct vio_net_dext *dext = vio_net_ext(desc); 357*6d0ba919SDavid L Stevens 358*6d0ba919SDavid L Stevens if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM) { 359*6d0ba919SDavid L Stevens if (skb->protocol == ETH_P_IP) { 360*6d0ba919SDavid L Stevens struct iphdr *iph = (struct iphdr *)skb->data; 361*6d0ba919SDavid L Stevens 362*6d0ba919SDavid L Stevens iph->check = 0; 363*6d0ba919SDavid L Stevens ip_send_check(iph); 364*6d0ba919SDavid L Stevens } 365*6d0ba919SDavid L Stevens } 366*6d0ba919SDavid L Stevens if ((dext->flags & VNET_PKT_HCK_FULLCKSUM) && 367*6d0ba919SDavid L Stevens skb->ip_summed == CHECKSUM_NONE) 368*6d0ba919SDavid L Stevens vnet_fullcsum(skb); 369*6d0ba919SDavid L Stevens if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) { 370*6d0ba919SDavid L Stevens skb->ip_summed = CHECKSUM_PARTIAL; 371*6d0ba919SDavid L Stevens skb->csum_level = 0; 372*6d0ba919SDavid L Stevens if (dext->flags & VNET_PKT_HCK_FULLCKSUM_OK) 373*6d0ba919SDavid L Stevens skb->csum_level = 1; 374*6d0ba919SDavid L Stevens } 375*6d0ba919SDavid L Stevens } 376*6d0ba919SDavid L Stevens 377e689cf4aSJeff Kirsher dev->stats.rx_packets++; 378e689cf4aSJeff Kirsher dev->stats.rx_bytes += len; 37969088822SSowmini Varadhan napi_gro_receive(&port->napi, skb); 380e689cf4aSJeff Kirsher return 0; 381e689cf4aSJeff Kirsher 382e689cf4aSJeff Kirsher out_free_skb: 383e689cf4aSJeff Kirsher kfree_skb(skb); 384e689cf4aSJeff Kirsher 385e689cf4aSJeff Kirsher out_dropped: 386e689cf4aSJeff Kirsher dev->stats.rx_dropped++; 387e689cf4aSJeff Kirsher return err; 388e689cf4aSJeff Kirsher } 389e689cf4aSJeff Kirsher 390e689cf4aSJeff Kirsher static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr, 391e689cf4aSJeff Kirsher u32 start, u32 end, u8 vio_dring_state) 392e689cf4aSJeff Kirsher { 393e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 394e689cf4aSJeff Kirsher .tag = { 395e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 396e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_ACK, 397e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 398e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 399e689cf4aSJeff Kirsher }, 400e689cf4aSJeff Kirsher .dring_ident = dr->ident, 401e689cf4aSJeff Kirsher .start_idx = start, 402e689cf4aSJeff Kirsher .end_idx = end, 403e689cf4aSJeff Kirsher .state = vio_dring_state, 404e689cf4aSJeff Kirsher }; 405e689cf4aSJeff Kirsher int err, delay; 406adddc32dSSowmini Varadhan int retries = 0; 407e689cf4aSJeff Kirsher 408e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 409e689cf4aSJeff Kirsher delay = 1; 410e689cf4aSJeff Kirsher do { 411e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 412e689cf4aSJeff Kirsher if (err > 0) { 413e689cf4aSJeff Kirsher dr->snd_nxt++; 414e689cf4aSJeff Kirsher break; 415e689cf4aSJeff Kirsher } 416e689cf4aSJeff Kirsher udelay(delay); 417e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 418e689cf4aSJeff Kirsher delay = 128; 419adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) { 420adddc32dSSowmini Varadhan pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n", 421adddc32dSSowmini Varadhan port->raddr[0], port->raddr[1], 422adddc32dSSowmini Varadhan port->raddr[2], port->raddr[3], 423adddc32dSSowmini Varadhan port->raddr[4], port->raddr[5]); 424d1015645SSowmini Varadhan break; 425adddc32dSSowmini Varadhan } 426e689cf4aSJeff Kirsher } while (err == -EAGAIN); 427e689cf4aSJeff Kirsher 428d1015645SSowmini Varadhan if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) { 429d1015645SSowmini Varadhan port->stop_rx_idx = end; 430d1015645SSowmini Varadhan port->stop_rx = true; 431d1015645SSowmini Varadhan } else { 432d1015645SSowmini Varadhan port->stop_rx_idx = 0; 433d1015645SSowmini Varadhan port->stop_rx = false; 434d1015645SSowmini Varadhan } 435d1015645SSowmini Varadhan 436e689cf4aSJeff Kirsher return err; 437e689cf4aSJeff Kirsher } 438e689cf4aSJeff Kirsher 439e689cf4aSJeff Kirsher static u32 next_idx(u32 idx, struct vio_dring_state *dr) 440e689cf4aSJeff Kirsher { 441e689cf4aSJeff Kirsher if (++idx == dr->num_entries) 442e689cf4aSJeff Kirsher idx = 0; 443e689cf4aSJeff Kirsher return idx; 444e689cf4aSJeff Kirsher } 445e689cf4aSJeff Kirsher 446e689cf4aSJeff Kirsher static u32 prev_idx(u32 idx, struct vio_dring_state *dr) 447e689cf4aSJeff Kirsher { 448e689cf4aSJeff Kirsher if (idx == 0) 449e689cf4aSJeff Kirsher idx = dr->num_entries - 1; 450e689cf4aSJeff Kirsher else 451e689cf4aSJeff Kirsher idx--; 452e689cf4aSJeff Kirsher 453e689cf4aSJeff Kirsher return idx; 454e689cf4aSJeff Kirsher } 455e689cf4aSJeff Kirsher 456e689cf4aSJeff Kirsher static struct vio_net_desc *get_rx_desc(struct vnet_port *port, 457e689cf4aSJeff Kirsher struct vio_dring_state *dr, 458e689cf4aSJeff Kirsher u32 index) 459e689cf4aSJeff Kirsher { 460e689cf4aSJeff Kirsher struct vio_net_desc *desc = port->vio.desc_buf; 461e689cf4aSJeff Kirsher int err; 462e689cf4aSJeff Kirsher 463e689cf4aSJeff Kirsher err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size, 464e689cf4aSJeff Kirsher (index * dr->entry_size), 465e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 466e689cf4aSJeff Kirsher if (err < 0) 467e689cf4aSJeff Kirsher return ERR_PTR(err); 468e689cf4aSJeff Kirsher 469e689cf4aSJeff Kirsher return desc; 470e689cf4aSJeff Kirsher } 471e689cf4aSJeff Kirsher 472e689cf4aSJeff Kirsher static int put_rx_desc(struct vnet_port *port, 473e689cf4aSJeff Kirsher struct vio_dring_state *dr, 474e689cf4aSJeff Kirsher struct vio_net_desc *desc, 475e689cf4aSJeff Kirsher u32 index) 476e689cf4aSJeff Kirsher { 477e689cf4aSJeff Kirsher int err; 478e689cf4aSJeff Kirsher 479e689cf4aSJeff Kirsher err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size, 480e689cf4aSJeff Kirsher (index * dr->entry_size), 481e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 482e689cf4aSJeff Kirsher if (err < 0) 483e689cf4aSJeff Kirsher return err; 484e689cf4aSJeff Kirsher 485e689cf4aSJeff Kirsher return 0; 486e689cf4aSJeff Kirsher } 487e689cf4aSJeff Kirsher 488e689cf4aSJeff Kirsher static int vnet_walk_rx_one(struct vnet_port *port, 489e689cf4aSJeff Kirsher struct vio_dring_state *dr, 490e689cf4aSJeff Kirsher u32 index, int *needs_ack) 491e689cf4aSJeff Kirsher { 492e689cf4aSJeff Kirsher struct vio_net_desc *desc = get_rx_desc(port, dr, index); 493e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 494e689cf4aSJeff Kirsher int err; 495e689cf4aSJeff Kirsher 49669088822SSowmini Varadhan BUG_ON(desc == NULL); 497e689cf4aSJeff Kirsher if (IS_ERR(desc)) 498e689cf4aSJeff Kirsher return PTR_ERR(desc); 499e689cf4aSJeff Kirsher 50078dcff7bSDavid L Stevens if (desc->hdr.state != VIO_DESC_READY) 50178dcff7bSDavid L Stevens return 1; 50278dcff7bSDavid L Stevens 50378dcff7bSDavid L Stevens rmb(); 50478dcff7bSDavid L Stevens 505e689cf4aSJeff Kirsher viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n", 506e689cf4aSJeff Kirsher desc->hdr.state, desc->hdr.ack, 507e689cf4aSJeff Kirsher desc->size, desc->ncookies, 508e689cf4aSJeff Kirsher desc->cookies[0].cookie_addr, 509e689cf4aSJeff Kirsher desc->cookies[0].cookie_size); 510e689cf4aSJeff Kirsher 511*6d0ba919SDavid L Stevens err = vnet_rx_one(port, desc); 512e689cf4aSJeff Kirsher if (err == -ECONNRESET) 513e689cf4aSJeff Kirsher return err; 514e689cf4aSJeff Kirsher desc->hdr.state = VIO_DESC_DONE; 515e689cf4aSJeff Kirsher err = put_rx_desc(port, dr, desc, index); 516e689cf4aSJeff Kirsher if (err < 0) 517e689cf4aSJeff Kirsher return err; 518e689cf4aSJeff Kirsher *needs_ack = desc->hdr.ack; 519e689cf4aSJeff Kirsher return 0; 520e689cf4aSJeff Kirsher } 521e689cf4aSJeff Kirsher 522e689cf4aSJeff Kirsher static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, 52369088822SSowmini Varadhan u32 start, u32 end, int *npkts, int budget) 524e689cf4aSJeff Kirsher { 525e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 526e689cf4aSJeff Kirsher int ack_start = -1, ack_end = -1; 52769088822SSowmini Varadhan bool send_ack = true; 528e689cf4aSJeff Kirsher 529e689cf4aSJeff Kirsher end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr); 530e689cf4aSJeff Kirsher 531e689cf4aSJeff Kirsher viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end); 532e689cf4aSJeff Kirsher 533e689cf4aSJeff Kirsher while (start != end) { 534e689cf4aSJeff Kirsher int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack); 535e689cf4aSJeff Kirsher if (err == -ECONNRESET) 536e689cf4aSJeff Kirsher return err; 537e689cf4aSJeff Kirsher if (err != 0) 538e689cf4aSJeff Kirsher break; 53969088822SSowmini Varadhan (*npkts)++; 540e689cf4aSJeff Kirsher if (ack_start == -1) 541e689cf4aSJeff Kirsher ack_start = start; 542e689cf4aSJeff Kirsher ack_end = start; 543e689cf4aSJeff Kirsher start = next_idx(start, dr); 544e689cf4aSJeff Kirsher if (ack && start != end) { 545e689cf4aSJeff Kirsher err = vnet_send_ack(port, dr, ack_start, ack_end, 546e689cf4aSJeff Kirsher VIO_DRING_ACTIVE); 547e689cf4aSJeff Kirsher if (err == -ECONNRESET) 548e689cf4aSJeff Kirsher return err; 549e689cf4aSJeff Kirsher ack_start = -1; 550e689cf4aSJeff Kirsher } 55169088822SSowmini Varadhan if ((*npkts) >= budget) { 55269088822SSowmini Varadhan send_ack = false; 55369088822SSowmini Varadhan break; 55469088822SSowmini Varadhan } 555e689cf4aSJeff Kirsher } 556e689cf4aSJeff Kirsher if (unlikely(ack_start == -1)) 557e689cf4aSJeff Kirsher ack_start = ack_end = prev_idx(start, dr); 55869088822SSowmini Varadhan if (send_ack) { 55969088822SSowmini Varadhan port->napi_resume = false; 56069088822SSowmini Varadhan return vnet_send_ack(port, dr, ack_start, ack_end, 56169088822SSowmini Varadhan VIO_DRING_STOPPED); 56269088822SSowmini Varadhan } else { 56369088822SSowmini Varadhan port->napi_resume = true; 56469088822SSowmini Varadhan port->napi_stop_idx = ack_end; 56569088822SSowmini Varadhan return 1; 56669088822SSowmini Varadhan } 567e689cf4aSJeff Kirsher } 568e689cf4aSJeff Kirsher 56969088822SSowmini Varadhan static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, 57069088822SSowmini Varadhan int budget) 571e689cf4aSJeff Kirsher { 572e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 573e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; 574e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 575e689cf4aSJeff Kirsher 576e689cf4aSJeff Kirsher viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n", 577e689cf4aSJeff Kirsher pkt->tag.stype_env, pkt->seq, dr->rcv_nxt); 578e689cf4aSJeff Kirsher 579e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 580e689cf4aSJeff Kirsher return 0; 581e689cf4aSJeff Kirsher if (unlikely(pkt->seq != dr->rcv_nxt)) { 582e689cf4aSJeff Kirsher pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n", 583e689cf4aSJeff Kirsher pkt->seq, dr->rcv_nxt); 584e689cf4aSJeff Kirsher return 0; 585e689cf4aSJeff Kirsher } 586e689cf4aSJeff Kirsher 58769088822SSowmini Varadhan if (!port->napi_resume) 588e689cf4aSJeff Kirsher dr->rcv_nxt++; 589e689cf4aSJeff Kirsher 590e689cf4aSJeff Kirsher /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ 591e689cf4aSJeff Kirsher 59269088822SSowmini Varadhan return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, 59369088822SSowmini Varadhan npkts, budget); 594e689cf4aSJeff Kirsher } 595e689cf4aSJeff Kirsher 596e689cf4aSJeff Kirsher static int idx_is_pending(struct vio_dring_state *dr, u32 end) 597e689cf4aSJeff Kirsher { 598e689cf4aSJeff Kirsher u32 idx = dr->cons; 599e689cf4aSJeff Kirsher int found = 0; 600e689cf4aSJeff Kirsher 601e689cf4aSJeff Kirsher while (idx != dr->prod) { 602e689cf4aSJeff Kirsher if (idx == end) { 603e689cf4aSJeff Kirsher found = 1; 604e689cf4aSJeff Kirsher break; 605e689cf4aSJeff Kirsher } 606e689cf4aSJeff Kirsher idx = next_idx(idx, dr); 607e689cf4aSJeff Kirsher } 608e689cf4aSJeff Kirsher return found; 609e689cf4aSJeff Kirsher } 610e689cf4aSJeff Kirsher 611e689cf4aSJeff Kirsher static int vnet_ack(struct vnet_port *port, void *msgbuf) 612e689cf4aSJeff Kirsher { 613e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 614e689cf4aSJeff Kirsher struct vio_dring_data *pkt = msgbuf; 615e689cf4aSJeff Kirsher struct net_device *dev; 616e689cf4aSJeff Kirsher struct vnet *vp; 617e689cf4aSJeff Kirsher u32 end; 618d1015645SSowmini Varadhan struct vio_net_desc *desc; 619d51bffd1SSowmini Varadhan struct netdev_queue *txq; 620d51bffd1SSowmini Varadhan 621e689cf4aSJeff Kirsher if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 622e689cf4aSJeff Kirsher return 0; 623e689cf4aSJeff Kirsher 624e689cf4aSJeff Kirsher end = pkt->end_idx; 62569088822SSowmini Varadhan vp = port->vp; 62669088822SSowmini Varadhan dev = vp->dev; 627b0cffed5SSowmini Varadhan netif_tx_lock(dev); 628b0cffed5SSowmini Varadhan if (unlikely(!idx_is_pending(dr, end))) { 629b0cffed5SSowmini Varadhan netif_tx_unlock(dev); 630b0cffed5SSowmini Varadhan return 0; 631b0cffed5SSowmini Varadhan } 632b0cffed5SSowmini Varadhan 633d1015645SSowmini Varadhan /* sync for race conditions with vnet_start_xmit() and tell xmit it 634d1015645SSowmini Varadhan * is time to send a trigger. 635d1015645SSowmini Varadhan */ 636e689cf4aSJeff Kirsher dr->cons = next_idx(end, dr); 637d1015645SSowmini Varadhan desc = vio_dring_entry(dr, dr->cons); 638777362d7SSowmini Varadhan if (desc->hdr.state == VIO_DESC_READY && !port->start_cons) { 639d1015645SSowmini Varadhan /* vnet_start_xmit() just populated this dring but missed 640d1015645SSowmini Varadhan * sending the "start" LDC message to the consumer. 641d1015645SSowmini Varadhan * Send a "start" trigger on its behalf. 642d1015645SSowmini Varadhan */ 643d1015645SSowmini Varadhan if (__vnet_tx_trigger(port, dr->cons) > 0) 644d1015645SSowmini Varadhan port->start_cons = false; 645d1015645SSowmini Varadhan else 646d1015645SSowmini Varadhan port->start_cons = true; 647d1015645SSowmini Varadhan } else { 648d1015645SSowmini Varadhan port->start_cons = true; 649d1015645SSowmini Varadhan } 65069088822SSowmini Varadhan netif_tx_unlock(dev); 651d1015645SSowmini Varadhan 652d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(dev, port->q_index); 653d51bffd1SSowmini Varadhan if (unlikely(netif_tx_queue_stopped(txq) && 654e689cf4aSJeff Kirsher vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) 655e689cf4aSJeff Kirsher return 1; 656e689cf4aSJeff Kirsher 657e689cf4aSJeff Kirsher return 0; 658e689cf4aSJeff Kirsher } 659e689cf4aSJeff Kirsher 660e689cf4aSJeff Kirsher static int vnet_nack(struct vnet_port *port, void *msgbuf) 661e689cf4aSJeff Kirsher { 662e689cf4aSJeff Kirsher /* XXX just reset or similar XXX */ 663e689cf4aSJeff Kirsher return 0; 664e689cf4aSJeff Kirsher } 665e689cf4aSJeff Kirsher 666e689cf4aSJeff Kirsher static int handle_mcast(struct vnet_port *port, void *msgbuf) 667e689cf4aSJeff Kirsher { 668e689cf4aSJeff Kirsher struct vio_net_mcast_info *pkt = msgbuf; 669e689cf4aSJeff Kirsher 670e689cf4aSJeff Kirsher if (pkt->tag.stype != VIO_SUBTYPE_ACK) 671e689cf4aSJeff Kirsher pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n", 672e689cf4aSJeff Kirsher port->vp->dev->name, 673e689cf4aSJeff Kirsher pkt->tag.type, 674e689cf4aSJeff Kirsher pkt->tag.stype, 675e689cf4aSJeff Kirsher pkt->tag.stype_env, 676e689cf4aSJeff Kirsher pkt->tag.sid); 677e689cf4aSJeff Kirsher 678e689cf4aSJeff Kirsher return 0; 679e689cf4aSJeff Kirsher } 680e689cf4aSJeff Kirsher 681d51bffd1SSowmini Varadhan /* Got back a STOPPED LDC message on port. If the queue is stopped, 682d51bffd1SSowmini Varadhan * wake it up so that we'll send out another START message at the 683d51bffd1SSowmini Varadhan * next TX. 684d51bffd1SSowmini Varadhan */ 685d51bffd1SSowmini Varadhan static void maybe_tx_wakeup(struct vnet_port *port) 686e689cf4aSJeff Kirsher { 687d51bffd1SSowmini Varadhan struct netdev_queue *txq; 688e689cf4aSJeff Kirsher 689d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(port->vp->dev, port->q_index); 690d51bffd1SSowmini Varadhan __netif_tx_lock(txq, smp_processor_id()); 691d51bffd1SSowmini Varadhan if (likely(netif_tx_queue_stopped(txq))) { 692e689cf4aSJeff Kirsher struct vio_dring_state *dr; 693e689cf4aSJeff Kirsher 694e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 695d51bffd1SSowmini Varadhan netif_tx_wake_queue(txq); 696e689cf4aSJeff Kirsher } 697d51bffd1SSowmini Varadhan __netif_tx_unlock(txq); 698e689cf4aSJeff Kirsher } 699e689cf4aSJeff Kirsher 70069088822SSowmini Varadhan static inline bool port_is_up(struct vnet_port *vnet) 701e689cf4aSJeff Kirsher { 70269088822SSowmini Varadhan struct vio_driver_state *vio = &vnet->vio; 70369088822SSowmini Varadhan 70469088822SSowmini Varadhan return !!(vio->hs_state & VIO_HS_COMPLETE); 70569088822SSowmini Varadhan } 70669088822SSowmini Varadhan 70769088822SSowmini Varadhan static int vnet_event_napi(struct vnet_port *port, int budget) 70869088822SSowmini Varadhan { 709e689cf4aSJeff Kirsher struct vio_driver_state *vio = &port->vio; 710e689cf4aSJeff Kirsher int tx_wakeup, err; 71169088822SSowmini Varadhan int npkts = 0; 71269088822SSowmini Varadhan int event = (port->rx_event & LDC_EVENT_RESET); 713e689cf4aSJeff Kirsher 71469088822SSowmini Varadhan ldc_ctrl: 715e689cf4aSJeff Kirsher if (unlikely(event == LDC_EVENT_RESET || 716e689cf4aSJeff Kirsher event == LDC_EVENT_UP)) { 717e689cf4aSJeff Kirsher vio_link_state_change(vio, event); 718e689cf4aSJeff Kirsher 719e4defc77SDavid L Stevens if (event == LDC_EVENT_RESET) { 720e4defc77SDavid L Stevens port->rmtu = 0; 721e689cf4aSJeff Kirsher vio_port_up(vio); 722e4defc77SDavid L Stevens } 72369088822SSowmini Varadhan port->rx_event = 0; 72469088822SSowmini Varadhan return 0; 725e689cf4aSJeff Kirsher } 72669088822SSowmini Varadhan /* We may have multiple LDC events in rx_event. Unroll send_events() */ 72769088822SSowmini Varadhan event = (port->rx_event & LDC_EVENT_UP); 72869088822SSowmini Varadhan port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); 72969088822SSowmini Varadhan if (event == LDC_EVENT_UP) 73069088822SSowmini Varadhan goto ldc_ctrl; 73169088822SSowmini Varadhan event = port->rx_event; 73269088822SSowmini Varadhan if (!(event & LDC_EVENT_DATA_READY)) 73369088822SSowmini Varadhan return 0; 734e689cf4aSJeff Kirsher 73569088822SSowmini Varadhan /* we dont expect any other bits than RESET, UP, DATA_READY */ 73669088822SSowmini Varadhan BUG_ON(event != LDC_EVENT_DATA_READY); 737e689cf4aSJeff Kirsher 738e689cf4aSJeff Kirsher tx_wakeup = err = 0; 739e689cf4aSJeff Kirsher while (1) { 740e689cf4aSJeff Kirsher union { 741e689cf4aSJeff Kirsher struct vio_msg_tag tag; 742e689cf4aSJeff Kirsher u64 raw[8]; 743e689cf4aSJeff Kirsher } msgbuf; 744e689cf4aSJeff Kirsher 74569088822SSowmini Varadhan if (port->napi_resume) { 74669088822SSowmini Varadhan struct vio_dring_data *pkt = 74769088822SSowmini Varadhan (struct vio_dring_data *)&msgbuf; 74869088822SSowmini Varadhan struct vio_dring_state *dr = 74969088822SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING]; 75069088822SSowmini Varadhan 75169088822SSowmini Varadhan pkt->tag.type = VIO_TYPE_DATA; 75269088822SSowmini Varadhan pkt->tag.stype = VIO_SUBTYPE_INFO; 75369088822SSowmini Varadhan pkt->tag.stype_env = VIO_DRING_DATA; 75469088822SSowmini Varadhan pkt->seq = dr->rcv_nxt; 75569088822SSowmini Varadhan pkt->start_idx = next_idx(port->napi_stop_idx, dr); 75669088822SSowmini Varadhan pkt->end_idx = -1; 75769088822SSowmini Varadhan goto napi_resume; 75869088822SSowmini Varadhan } 759e689cf4aSJeff Kirsher err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); 760e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 761e689cf4aSJeff Kirsher if (err == -ECONNRESET) 762e689cf4aSJeff Kirsher vio_conn_reset(vio); 763e689cf4aSJeff Kirsher break; 764e689cf4aSJeff Kirsher } 765e689cf4aSJeff Kirsher if (err == 0) 766e689cf4aSJeff Kirsher break; 767e689cf4aSJeff Kirsher viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n", 768e689cf4aSJeff Kirsher msgbuf.tag.type, 769e689cf4aSJeff Kirsher msgbuf.tag.stype, 770e689cf4aSJeff Kirsher msgbuf.tag.stype_env, 771e689cf4aSJeff Kirsher msgbuf.tag.sid); 772e689cf4aSJeff Kirsher err = vio_validate_sid(vio, &msgbuf.tag); 773e689cf4aSJeff Kirsher if (err < 0) 774e689cf4aSJeff Kirsher break; 77569088822SSowmini Varadhan napi_resume: 776e689cf4aSJeff Kirsher if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { 777e689cf4aSJeff Kirsher if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { 77869088822SSowmini Varadhan if (!port_is_up(port)) { 77969088822SSowmini Varadhan /* failures like handshake_failure() 78069088822SSowmini Varadhan * may have cleaned up dring, but 78169088822SSowmini Varadhan * NAPI polling may bring us here. 78269088822SSowmini Varadhan */ 78369088822SSowmini Varadhan err = -ECONNRESET; 78469088822SSowmini Varadhan break; 78569088822SSowmini Varadhan } 78669088822SSowmini Varadhan err = vnet_rx(port, &msgbuf, &npkts, budget); 78769088822SSowmini Varadhan if (npkts >= budget) 78869088822SSowmini Varadhan break; 7898c4ee3e7SSowmini Varadhan if (npkts == 0) 7908c4ee3e7SSowmini Varadhan break; 791e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { 792e689cf4aSJeff Kirsher err = vnet_ack(port, &msgbuf); 793e689cf4aSJeff Kirsher if (err > 0) 794e689cf4aSJeff Kirsher tx_wakeup |= err; 795e689cf4aSJeff Kirsher } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) { 796e689cf4aSJeff Kirsher err = vnet_nack(port, &msgbuf); 797e689cf4aSJeff Kirsher } 798e689cf4aSJeff Kirsher } else if (msgbuf.tag.type == VIO_TYPE_CTRL) { 799e689cf4aSJeff Kirsher if (msgbuf.tag.stype_env == VNET_MCAST_INFO) 800e689cf4aSJeff Kirsher err = handle_mcast(port, &msgbuf); 801e689cf4aSJeff Kirsher else 802e689cf4aSJeff Kirsher err = vio_control_pkt_engine(vio, &msgbuf); 803e689cf4aSJeff Kirsher if (err) 804e689cf4aSJeff Kirsher break; 805e689cf4aSJeff Kirsher } else { 806e689cf4aSJeff Kirsher err = vnet_handle_unknown(port, &msgbuf); 807e689cf4aSJeff Kirsher } 808e689cf4aSJeff Kirsher if (err == -ECONNRESET) 809e689cf4aSJeff Kirsher break; 810e689cf4aSJeff Kirsher } 811e689cf4aSJeff Kirsher if (unlikely(tx_wakeup && err != -ECONNRESET)) 812d51bffd1SSowmini Varadhan maybe_tx_wakeup(port); 81369088822SSowmini Varadhan return npkts; 81469088822SSowmini Varadhan } 8151d311ad2SSowmini Varadhan 81669088822SSowmini Varadhan static int vnet_poll(struct napi_struct *napi, int budget) 81769088822SSowmini Varadhan { 81869088822SSowmini Varadhan struct vnet_port *port = container_of(napi, struct vnet_port, napi); 81969088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 82069088822SSowmini Varadhan int processed = vnet_event_napi(port, budget); 82169088822SSowmini Varadhan 82269088822SSowmini Varadhan if (processed < budget) { 82369088822SSowmini Varadhan napi_complete(napi); 8247bd68bfdSSowmini Varadhan port->rx_event &= ~LDC_EVENT_DATA_READY; 82569088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); 82669088822SSowmini Varadhan } 82769088822SSowmini Varadhan return processed; 82869088822SSowmini Varadhan } 82969088822SSowmini Varadhan 83069088822SSowmini Varadhan static void vnet_event(void *arg, int event) 83169088822SSowmini Varadhan { 83269088822SSowmini Varadhan struct vnet_port *port = arg; 83369088822SSowmini Varadhan struct vio_driver_state *vio = &port->vio; 83469088822SSowmini Varadhan 83569088822SSowmini Varadhan port->rx_event |= event; 83669088822SSowmini Varadhan vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); 83769088822SSowmini Varadhan napi_schedule(&port->napi); 83869088822SSowmini Varadhan 839e689cf4aSJeff Kirsher } 840e689cf4aSJeff Kirsher 841d1015645SSowmini Varadhan static int __vnet_tx_trigger(struct vnet_port *port, u32 start) 842e689cf4aSJeff Kirsher { 843e689cf4aSJeff Kirsher struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 844e689cf4aSJeff Kirsher struct vio_dring_data hdr = { 845e689cf4aSJeff Kirsher .tag = { 846e689cf4aSJeff Kirsher .type = VIO_TYPE_DATA, 847e689cf4aSJeff Kirsher .stype = VIO_SUBTYPE_INFO, 848e689cf4aSJeff Kirsher .stype_env = VIO_DRING_DATA, 849e689cf4aSJeff Kirsher .sid = vio_send_sid(&port->vio), 850e689cf4aSJeff Kirsher }, 851e689cf4aSJeff Kirsher .dring_ident = dr->ident, 852d1015645SSowmini Varadhan .start_idx = start, 853e689cf4aSJeff Kirsher .end_idx = (u32) -1, 854e689cf4aSJeff Kirsher }; 855e689cf4aSJeff Kirsher int err, delay; 856adddc32dSSowmini Varadhan int retries = 0; 857e689cf4aSJeff Kirsher 858d1015645SSowmini Varadhan if (port->stop_rx) { 859d1015645SSowmini Varadhan err = vnet_send_ack(port, 860d1015645SSowmini Varadhan &port->vio.drings[VIO_DRIVER_RX_RING], 861d1015645SSowmini Varadhan port->stop_rx_idx, -1, 862d1015645SSowmini Varadhan VIO_DRING_STOPPED); 863d1015645SSowmini Varadhan if (err <= 0) 864d1015645SSowmini Varadhan return err; 865d1015645SSowmini Varadhan } 866d1015645SSowmini Varadhan 867e689cf4aSJeff Kirsher hdr.seq = dr->snd_nxt; 868e689cf4aSJeff Kirsher delay = 1; 869e689cf4aSJeff Kirsher do { 870e689cf4aSJeff Kirsher err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 871e689cf4aSJeff Kirsher if (err > 0) { 872e689cf4aSJeff Kirsher dr->snd_nxt++; 873e689cf4aSJeff Kirsher break; 874e689cf4aSJeff Kirsher } 875e689cf4aSJeff Kirsher udelay(delay); 876e689cf4aSJeff Kirsher if ((delay <<= 1) > 128) 877e689cf4aSJeff Kirsher delay = 128; 878adddc32dSSowmini Varadhan if (retries++ > VNET_MAX_RETRIES) 879adddc32dSSowmini Varadhan break; 880e689cf4aSJeff Kirsher } while (err == -EAGAIN); 881e689cf4aSJeff Kirsher 882e689cf4aSJeff Kirsher return err; 883e689cf4aSJeff Kirsher } 884e689cf4aSJeff Kirsher 885e689cf4aSJeff Kirsher struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) 886e689cf4aSJeff Kirsher { 887e689cf4aSJeff Kirsher unsigned int hash = vnet_hashfn(skb->data); 888e689cf4aSJeff Kirsher struct hlist_head *hp = &vp->port_hash[hash]; 889e689cf4aSJeff Kirsher struct vnet_port *port; 890e689cf4aSJeff Kirsher 8912a968dd8SSowmini Varadhan hlist_for_each_entry_rcu(port, hp, hash) { 8928266f5fcSDavid L Stevens if (!port_is_up(port)) 8938266f5fcSDavid L Stevens continue; 8942e42e474SJoe Perches if (ether_addr_equal(port->raddr, skb->data)) 895e689cf4aSJeff Kirsher return port; 896e689cf4aSJeff Kirsher } 8972a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 8988266f5fcSDavid L Stevens if (!port->switch_port) 8998266f5fcSDavid L Stevens continue; 9008266f5fcSDavid L Stevens if (!port_is_up(port)) 9018266f5fcSDavid L Stevens continue; 902e689cf4aSJeff Kirsher return port; 903e689cf4aSJeff Kirsher } 9048266f5fcSDavid L Stevens return NULL; 9058266f5fcSDavid L Stevens } 906e689cf4aSJeff Kirsher 9078e845f4cSDavid L Stevens static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, 9088e845f4cSDavid L Stevens unsigned *pending) 9098e845f4cSDavid L Stevens { 9108e845f4cSDavid L Stevens struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 9118e845f4cSDavid L Stevens struct sk_buff *skb = NULL; 9128e845f4cSDavid L Stevens int i, txi; 9138e845f4cSDavid L Stevens 9148e845f4cSDavid L Stevens *pending = 0; 9158e845f4cSDavid L Stevens 9168e845f4cSDavid L Stevens txi = dr->prod-1; 9178e845f4cSDavid L Stevens if (txi < 0) 9188e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 9198e845f4cSDavid L Stevens 9208e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 9218e845f4cSDavid L Stevens struct vio_net_desc *d; 9228e845f4cSDavid L Stevens 9238e845f4cSDavid L Stevens d = vio_dring_entry(dr, txi); 9248e845f4cSDavid L Stevens 9258e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_DONE) { 9268e845f4cSDavid L Stevens if (port->tx_bufs[txi].skb) { 9278e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb->next); 9288e845f4cSDavid L Stevens 9298e845f4cSDavid L Stevens port->tx_bufs[txi].skb->next = skb; 9308e845f4cSDavid L Stevens skb = port->tx_bufs[txi].skb; 9318e845f4cSDavid L Stevens port->tx_bufs[txi].skb = NULL; 9328e845f4cSDavid L Stevens 9338e845f4cSDavid L Stevens ldc_unmap(port->vio.lp, 9348e845f4cSDavid L Stevens port->tx_bufs[txi].cookies, 9358e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies); 9368e845f4cSDavid L Stevens } 9378e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 9388e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_READY) { 9398e845f4cSDavid L Stevens (*pending)++; 9408e845f4cSDavid L Stevens } else if (d->hdr.state == VIO_DESC_FREE) { 9418e845f4cSDavid L Stevens break; 9428e845f4cSDavid L Stevens } 9438e845f4cSDavid L Stevens --txi; 9448e845f4cSDavid L Stevens if (txi < 0) 9458e845f4cSDavid L Stevens txi = VNET_TX_RING_SIZE-1; 9468e845f4cSDavid L Stevens } 9478e845f4cSDavid L Stevens return skb; 9488e845f4cSDavid L Stevens } 9498e845f4cSDavid L Stevens 9508e845f4cSDavid L Stevens static inline void vnet_free_skbs(struct sk_buff *skb) 9518e845f4cSDavid L Stevens { 9528e845f4cSDavid L Stevens struct sk_buff *next; 9538e845f4cSDavid L Stevens 9548e845f4cSDavid L Stevens while (skb) { 9558e845f4cSDavid L Stevens next = skb->next; 9568e845f4cSDavid L Stevens skb->next = NULL; 9578e845f4cSDavid L Stevens dev_kfree_skb(skb); 9588e845f4cSDavid L Stevens skb = next; 9598e845f4cSDavid L Stevens } 9608e845f4cSDavid L Stevens } 9618e845f4cSDavid L Stevens 9628e845f4cSDavid L Stevens static void vnet_clean_timer_expire(unsigned long port0) 9638e845f4cSDavid L Stevens { 9648e845f4cSDavid L Stevens struct vnet_port *port = (struct vnet_port *)port0; 9658e845f4cSDavid L Stevens struct sk_buff *freeskbs; 9668e845f4cSDavid L Stevens unsigned pending; 9678e845f4cSDavid L Stevens 96813b13dd9SSowmini Varadhan netif_tx_lock(port->vp->dev); 9698e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 97013b13dd9SSowmini Varadhan netif_tx_unlock(port->vp->dev); 9718e845f4cSDavid L Stevens 9728e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 9738e845f4cSDavid L Stevens 9748e845f4cSDavid L Stevens if (pending) 9758e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 9768e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 9778e845f4cSDavid L Stevens else 9788e845f4cSDavid L Stevens del_timer(&port->clean_timer); 9798e845f4cSDavid L Stevens } 9808e845f4cSDavid L Stevens 9818e845f4cSDavid L Stevens static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, void **pstart, 9828e845f4cSDavid L Stevens int *plen) 9838e845f4cSDavid L Stevens { 9848e845f4cSDavid L Stevens struct sk_buff *nskb; 9858e845f4cSDavid L Stevens int len, pad; 9868e845f4cSDavid L Stevens 9878e845f4cSDavid L Stevens len = skb->len; 9888e845f4cSDavid L Stevens pad = 0; 9898e845f4cSDavid L Stevens if (len < ETH_ZLEN) { 9908e845f4cSDavid L Stevens pad += ETH_ZLEN - skb->len; 9918e845f4cSDavid L Stevens len += pad; 9928e845f4cSDavid L Stevens } 9938e845f4cSDavid L Stevens len += VNET_PACKET_SKIP; 9948e845f4cSDavid L Stevens pad += 8 - (len & 7); 9958e845f4cSDavid L Stevens len += 8 - (len & 7); 9968e845f4cSDavid L Stevens 9978e845f4cSDavid L Stevens if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP || 9988e845f4cSDavid L Stevens skb_tailroom(skb) < pad || 9998e845f4cSDavid L Stevens skb_headroom(skb) < VNET_PACKET_SKIP) { 1000*6d0ba919SDavid L Stevens int offset; 1001*6d0ba919SDavid L Stevens 10028e845f4cSDavid L Stevens nskb = alloc_and_align_skb(skb->dev, skb->len); 10038e845f4cSDavid L Stevens skb_reserve(nskb, VNET_PACKET_SKIP); 1004*6d0ba919SDavid L Stevens 1005*6d0ba919SDavid L Stevens nskb->protocol = skb->protocol; 1006*6d0ba919SDavid L Stevens offset = skb_mac_header(skb) - skb->data; 1007*6d0ba919SDavid L Stevens skb_set_mac_header(nskb, offset); 1008*6d0ba919SDavid L Stevens offset = skb_network_header(skb) - skb->data; 1009*6d0ba919SDavid L Stevens skb_set_network_header(nskb, offset); 1010*6d0ba919SDavid L Stevens offset = skb_transport_header(skb) - skb->data; 1011*6d0ba919SDavid L Stevens skb_set_transport_header(nskb, offset); 1012*6d0ba919SDavid L Stevens 1013*6d0ba919SDavid L Stevens nskb->csum_offset = skb->csum_offset; 1014*6d0ba919SDavid L Stevens nskb->ip_summed = skb->ip_summed; 1015*6d0ba919SDavid L Stevens 10168e845f4cSDavid L Stevens if (skb_copy_bits(skb, 0, nskb->data, skb->len)) { 10178e845f4cSDavid L Stevens dev_kfree_skb(nskb); 10188e845f4cSDavid L Stevens dev_kfree_skb(skb); 10198e845f4cSDavid L Stevens return NULL; 10208e845f4cSDavid L Stevens } 10218e845f4cSDavid L Stevens (void)skb_put(nskb, skb->len); 10228e845f4cSDavid L Stevens dev_kfree_skb(skb); 10238e845f4cSDavid L Stevens skb = nskb; 10248e845f4cSDavid L Stevens } 10258e845f4cSDavid L Stevens 10268e845f4cSDavid L Stevens *pstart = skb->data - VNET_PACKET_SKIP; 10278e845f4cSDavid L Stevens *plen = len; 10288e845f4cSDavid L Stevens return skb; 10298e845f4cSDavid L Stevens } 10308e845f4cSDavid L Stevens 1031d51bffd1SSowmini Varadhan static u16 1032d51bffd1SSowmini Varadhan vnet_select_queue(struct net_device *dev, struct sk_buff *skb, 1033d51bffd1SSowmini Varadhan void *accel_priv, select_queue_fallback_t fallback) 1034d51bffd1SSowmini Varadhan { 1035d51bffd1SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 1036d51bffd1SSowmini Varadhan struct vnet_port *port = __tx_port_find(vp, skb); 1037d51bffd1SSowmini Varadhan 1038c647cc3fSDavid L Stevens if (port == NULL) 1039c647cc3fSDavid L Stevens return 0; 1040d51bffd1SSowmini Varadhan return port->q_index; 1041d51bffd1SSowmini Varadhan } 1042d51bffd1SSowmini Varadhan 1043e689cf4aSJeff Kirsher static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 1044e689cf4aSJeff Kirsher { 1045e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 10462a968dd8SSowmini Varadhan struct vnet_port *port = NULL; 1047e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1048e689cf4aSJeff Kirsher struct vio_net_desc *d; 1049e689cf4aSJeff Kirsher unsigned int len; 10508e845f4cSDavid L Stevens struct sk_buff *freeskbs = NULL; 10518e845f4cSDavid L Stevens int i, err, txi; 10528e845f4cSDavid L Stevens void *start = NULL; 10538e845f4cSDavid L Stevens int nlen = 0; 10548e845f4cSDavid L Stevens unsigned pending = 0; 1055d51bffd1SSowmini Varadhan struct netdev_queue *txq; 1056e689cf4aSJeff Kirsher 10572a968dd8SSowmini Varadhan skb = vnet_skb_shape(skb, &start, &nlen); 10582a968dd8SSowmini Varadhan if (unlikely(!skb)) 1059e689cf4aSJeff Kirsher goto out_dropped; 1060e689cf4aSJeff Kirsher 10612a968dd8SSowmini Varadhan rcu_read_lock(); 106213b13dd9SSowmini Varadhan port = __tx_port_find(vp, skb); 1063df20286aSSowmini Varadhan if (unlikely(!port)) { 1064df20286aSSowmini Varadhan rcu_read_unlock(); 10658e845f4cSDavid L Stevens goto out_dropped; 1066df20286aSSowmini Varadhan } 10678e845f4cSDavid L Stevens 1068a2b78e9bSDavid L Stevens if (skb->len > port->rmtu) { 1069a2b78e9bSDavid L Stevens unsigned long localmtu = port->rmtu - ETH_HLEN; 1070a2b78e9bSDavid L Stevens 1071a2b78e9bSDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 3)) 1072a2b78e9bSDavid L Stevens localmtu -= VLAN_HLEN; 1073a2b78e9bSDavid L Stevens 1074a2b78e9bSDavid L Stevens if (skb->protocol == htons(ETH_P_IP)) { 1075a2b78e9bSDavid L Stevens struct flowi4 fl4; 1076a2b78e9bSDavid L Stevens struct rtable *rt = NULL; 1077a2b78e9bSDavid L Stevens 1078a2b78e9bSDavid L Stevens memset(&fl4, 0, sizeof(fl4)); 1079a2b78e9bSDavid L Stevens fl4.flowi4_oif = dev->ifindex; 1080a2b78e9bSDavid L Stevens fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); 1081a2b78e9bSDavid L Stevens fl4.daddr = ip_hdr(skb)->daddr; 1082a2b78e9bSDavid L Stevens fl4.saddr = ip_hdr(skb)->saddr; 1083a2b78e9bSDavid L Stevens 1084a2b78e9bSDavid L Stevens rt = ip_route_output_key(dev_net(dev), &fl4); 10852a968dd8SSowmini Varadhan rcu_read_unlock(); 1086a2b78e9bSDavid L Stevens if (!IS_ERR(rt)) { 1087a2b78e9bSDavid L Stevens skb_dst_set(skb, &rt->dst); 1088a2b78e9bSDavid L Stevens icmp_send(skb, ICMP_DEST_UNREACH, 1089a2b78e9bSDavid L Stevens ICMP_FRAG_NEEDED, 1090a2b78e9bSDavid L Stevens htonl(localmtu)); 1091a2b78e9bSDavid L Stevens } 1092a2b78e9bSDavid L Stevens } 1093a2b78e9bSDavid L Stevens #if IS_ENABLED(CONFIG_IPV6) 1094a2b78e9bSDavid L Stevens else if (skb->protocol == htons(ETH_P_IPV6)) 1095a2b78e9bSDavid L Stevens icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu); 1096a2b78e9bSDavid L Stevens #endif 109742db672dSDavid L Stevens goto out_dropped; 1098a2b78e9bSDavid L Stevens } 109942db672dSDavid L Stevens 1100e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1101d51bffd1SSowmini Varadhan i = skb_get_queue_mapping(skb); 1102d51bffd1SSowmini Varadhan txq = netdev_get_tx_queue(dev, i); 1103d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1104d51bffd1SSowmini Varadhan if (!netif_tx_queue_stopped(txq)) { 1105d51bffd1SSowmini Varadhan netif_tx_stop_queue(txq); 1106e689cf4aSJeff Kirsher 1107e689cf4aSJeff Kirsher /* This is a hard error, log it. */ 1108e689cf4aSJeff Kirsher netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); 1109e689cf4aSJeff Kirsher dev->stats.tx_errors++; 1110e689cf4aSJeff Kirsher } 11112a968dd8SSowmini Varadhan rcu_read_unlock(); 1112e689cf4aSJeff Kirsher return NETDEV_TX_BUSY; 1113e689cf4aSJeff Kirsher } 1114e689cf4aSJeff Kirsher 1115e689cf4aSJeff Kirsher d = vio_dring_cur(dr); 1116e689cf4aSJeff Kirsher 11178e845f4cSDavid L Stevens txi = dr->prod; 11188e845f4cSDavid L Stevens 11198e845f4cSDavid L Stevens freeskbs = vnet_clean_tx_ring(port, &pending); 11208e845f4cSDavid L Stevens 11218e845f4cSDavid L Stevens BUG_ON(port->tx_bufs[txi].skb); 1122e689cf4aSJeff Kirsher 1123e689cf4aSJeff Kirsher len = skb->len; 11248e845f4cSDavid L Stevens if (len < ETH_ZLEN) 1125e689cf4aSJeff Kirsher len = ETH_ZLEN; 11268e845f4cSDavid L Stevens 11278e845f4cSDavid L Stevens port->tx_bufs[txi].skb = skb; 11288e845f4cSDavid L Stevens skb = NULL; 11298e845f4cSDavid L Stevens 11308e845f4cSDavid L Stevens err = ldc_map_single(port->vio.lp, start, nlen, 113142db672dSDavid L Stevens port->tx_bufs[txi].cookies, VNET_MAXCOOKIES, 11328e845f4cSDavid L Stevens (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); 11338e845f4cSDavid L Stevens if (err < 0) { 11348e845f4cSDavid L Stevens netdev_info(dev, "tx buffer map error %d\n", err); 113513b13dd9SSowmini Varadhan goto out_dropped; 1136e689cf4aSJeff Kirsher } 11378e845f4cSDavid L Stevens port->tx_bufs[txi].ncookies = err; 1138e689cf4aSJeff Kirsher 11391f6394e3SSowmini Varadhan /* We don't rely on the ACKs to free the skb in vnet_start_xmit(), 11401f6394e3SSowmini Varadhan * thus it is safe to not set VIO_ACK_ENABLE for each transmission: 11411f6394e3SSowmini Varadhan * the protocol itself does not require it as long as the peer 11421f6394e3SSowmini Varadhan * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED. 11431f6394e3SSowmini Varadhan * 11441f6394e3SSowmini Varadhan * An ACK for every packet in the ring is expensive as the 11451f6394e3SSowmini Varadhan * sending of LDC messages is slow and affects performance. 11461f6394e3SSowmini Varadhan */ 11471f6394e3SSowmini Varadhan d->hdr.ack = VIO_ACK_DISABLE; 1148e689cf4aSJeff Kirsher d->size = len; 11498e845f4cSDavid L Stevens d->ncookies = port->tx_bufs[txi].ncookies; 1150e689cf4aSJeff Kirsher for (i = 0; i < d->ncookies; i++) 11518e845f4cSDavid L Stevens d->cookies[i] = port->tx_bufs[txi].cookies[i]; 1152*6d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 7)) { 1153*6d0ba919SDavid L Stevens struct vio_net_dext *dext = vio_net_ext(d); 1154*6d0ba919SDavid L Stevens 1155*6d0ba919SDavid L Stevens memset(dext, 0, sizeof(*dext)); 1156*6d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 8) && 1157*6d0ba919SDavid L Stevens !port->switch_port) { 1158*6d0ba919SDavid L Stevens dext->flags |= VNET_PKT_HCK_IPV4_HDRCKSUM_OK; 1159*6d0ba919SDavid L Stevens dext->flags |= VNET_PKT_HCK_FULLCKSUM_OK; 1160*6d0ba919SDavid L Stevens } 1161*6d0ba919SDavid L Stevens } 1162e689cf4aSJeff Kirsher 1163e689cf4aSJeff Kirsher /* This has to be a non-SMP write barrier because we are writing 1164e689cf4aSJeff Kirsher * to memory which is shared with the peer LDOM. 1165e689cf4aSJeff Kirsher */ 1166e689cf4aSJeff Kirsher wmb(); 1167e689cf4aSJeff Kirsher 1168e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_READY; 1169e689cf4aSJeff Kirsher 1170d1015645SSowmini Varadhan /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent 1171d1015645SSowmini Varadhan * to notify the consumer that some descriptors are READY. 1172d1015645SSowmini Varadhan * After that "start" trigger, no additional triggers are needed until 1173d1015645SSowmini Varadhan * a DRING_STOPPED is received from the consumer. The dr->cons field 1174d1015645SSowmini Varadhan * (set up by vnet_ack()) has the value of the next dring index 1175d1015645SSowmini Varadhan * that has not yet been ack-ed. We send a "start" trigger here 1176d1015645SSowmini Varadhan * if, and only if, start_cons is true (reset it afterward). Conversely, 1177d1015645SSowmini Varadhan * vnet_ack() should check if the dring corresponding to cons 1178d1015645SSowmini Varadhan * is marked READY, but start_cons was false. 1179d1015645SSowmini Varadhan * If so, vnet_ack() should send out the missed "start" trigger. 1180d1015645SSowmini Varadhan * 1181d1015645SSowmini Varadhan * Note that the wmb() above makes sure the cookies et al. are 1182d1015645SSowmini Varadhan * not globally visible before the VIO_DESC_READY, and that the 1183d1015645SSowmini Varadhan * stores are ordered correctly by the compiler. The consumer will 1184d1015645SSowmini Varadhan * not proceed until the VIO_DESC_READY is visible assuring that 1185d1015645SSowmini Varadhan * the consumer does not observe anything related to descriptors 1186d1015645SSowmini Varadhan * out of order. The HV trap from the LDC start trigger is the 1187d1015645SSowmini Varadhan * producer to consumer announcement that work is available to the 1188d1015645SSowmini Varadhan * consumer 1189d1015645SSowmini Varadhan */ 1190d1015645SSowmini Varadhan if (!port->start_cons) 1191d1015645SSowmini Varadhan goto ldc_start_done; /* previous trigger suffices */ 1192d1015645SSowmini Varadhan 1193d1015645SSowmini Varadhan err = __vnet_tx_trigger(port, dr->cons); 1194e689cf4aSJeff Kirsher if (unlikely(err < 0)) { 1195e689cf4aSJeff Kirsher netdev_info(dev, "TX trigger error %d\n", err); 1196e689cf4aSJeff Kirsher d->hdr.state = VIO_DESC_FREE; 1197e689cf4aSJeff Kirsher dev->stats.tx_carrier_errors++; 119813b13dd9SSowmini Varadhan goto out_dropped; 1199e689cf4aSJeff Kirsher } 1200e689cf4aSJeff Kirsher 1201d1015645SSowmini Varadhan ldc_start_done: 1202d1015645SSowmini Varadhan port->start_cons = false; 1203d1015645SSowmini Varadhan 1204e689cf4aSJeff Kirsher dev->stats.tx_packets++; 12058e845f4cSDavid L Stevens dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1206e689cf4aSJeff Kirsher 1207e689cf4aSJeff Kirsher dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1208d0aedcd4SDwight Engen if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1209d51bffd1SSowmini Varadhan netif_tx_stop_queue(txq); 1210e689cf4aSJeff Kirsher if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1211d51bffd1SSowmini Varadhan netif_tx_wake_queue(txq); 1212e689cf4aSJeff Kirsher } 1213e689cf4aSJeff Kirsher 12142a968dd8SSowmini Varadhan (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); 12152a968dd8SSowmini Varadhan rcu_read_unlock(); 1216e689cf4aSJeff Kirsher 12178e845f4cSDavid L Stevens vnet_free_skbs(freeskbs); 12188e845f4cSDavid L Stevens 1219e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1220e689cf4aSJeff Kirsher 1221e689cf4aSJeff Kirsher out_dropped: 12228e845f4cSDavid L Stevens if (pending) 12238e845f4cSDavid L Stevens (void)mod_timer(&port->clean_timer, 12248e845f4cSDavid L Stevens jiffies + VNET_CLEAN_TIMEOUT); 1225a29c9c43SDavid L Stevens else if (port) 12268e845f4cSDavid L Stevens del_timer(&port->clean_timer); 12272a968dd8SSowmini Varadhan if (port) 12282a968dd8SSowmini Varadhan rcu_read_unlock(); 12292a968dd8SSowmini Varadhan if (skb) 12302a968dd8SSowmini Varadhan dev_kfree_skb(skb); 12312a968dd8SSowmini Varadhan vnet_free_skbs(freeskbs); 1232e689cf4aSJeff Kirsher dev->stats.tx_dropped++; 1233e689cf4aSJeff Kirsher return NETDEV_TX_OK; 1234e689cf4aSJeff Kirsher } 1235e689cf4aSJeff Kirsher 1236e689cf4aSJeff Kirsher static void vnet_tx_timeout(struct net_device *dev) 1237e689cf4aSJeff Kirsher { 1238e689cf4aSJeff Kirsher /* XXX Implement me XXX */ 1239e689cf4aSJeff Kirsher } 1240e689cf4aSJeff Kirsher 1241e689cf4aSJeff Kirsher static int vnet_open(struct net_device *dev) 1242e689cf4aSJeff Kirsher { 1243e689cf4aSJeff Kirsher netif_carrier_on(dev); 1244d51bffd1SSowmini Varadhan netif_tx_start_all_queues(dev); 1245e689cf4aSJeff Kirsher 1246e689cf4aSJeff Kirsher return 0; 1247e689cf4aSJeff Kirsher } 1248e689cf4aSJeff Kirsher 1249e689cf4aSJeff Kirsher static int vnet_close(struct net_device *dev) 1250e689cf4aSJeff Kirsher { 1251d51bffd1SSowmini Varadhan netif_tx_stop_all_queues(dev); 1252e689cf4aSJeff Kirsher netif_carrier_off(dev); 1253e689cf4aSJeff Kirsher 1254e689cf4aSJeff Kirsher return 0; 1255e689cf4aSJeff Kirsher } 1256e689cf4aSJeff Kirsher 1257e689cf4aSJeff Kirsher static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr) 1258e689cf4aSJeff Kirsher { 1259e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1260e689cf4aSJeff Kirsher 1261e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 126200fa4ce9Sdingtianhong if (ether_addr_equal(m->addr, addr)) 1263e689cf4aSJeff Kirsher return m; 1264e689cf4aSJeff Kirsher } 1265e689cf4aSJeff Kirsher return NULL; 1266e689cf4aSJeff Kirsher } 1267e689cf4aSJeff Kirsher 1268e689cf4aSJeff Kirsher static void __update_mc_list(struct vnet *vp, struct net_device *dev) 1269e689cf4aSJeff Kirsher { 1270e689cf4aSJeff Kirsher struct netdev_hw_addr *ha; 1271e689cf4aSJeff Kirsher 1272e689cf4aSJeff Kirsher netdev_for_each_mc_addr(ha, dev) { 1273e689cf4aSJeff Kirsher struct vnet_mcast_entry *m; 1274e689cf4aSJeff Kirsher 1275e689cf4aSJeff Kirsher m = __vnet_mc_find(vp, ha->addr); 1276e689cf4aSJeff Kirsher if (m) { 1277e689cf4aSJeff Kirsher m->hit = 1; 1278e689cf4aSJeff Kirsher continue; 1279e689cf4aSJeff Kirsher } 1280e689cf4aSJeff Kirsher 1281e689cf4aSJeff Kirsher if (!m) { 1282e689cf4aSJeff Kirsher m = kzalloc(sizeof(*m), GFP_ATOMIC); 1283e689cf4aSJeff Kirsher if (!m) 1284e689cf4aSJeff Kirsher continue; 1285e689cf4aSJeff Kirsher memcpy(m->addr, ha->addr, ETH_ALEN); 1286e689cf4aSJeff Kirsher m->hit = 1; 1287e689cf4aSJeff Kirsher 1288e689cf4aSJeff Kirsher m->next = vp->mcast_list; 1289e689cf4aSJeff Kirsher vp->mcast_list = m; 1290e689cf4aSJeff Kirsher } 1291e689cf4aSJeff Kirsher } 1292e689cf4aSJeff Kirsher } 1293e689cf4aSJeff Kirsher 1294e689cf4aSJeff Kirsher static void __send_mc_list(struct vnet *vp, struct vnet_port *port) 1295e689cf4aSJeff Kirsher { 1296e689cf4aSJeff Kirsher struct vio_net_mcast_info info; 1297e689cf4aSJeff Kirsher struct vnet_mcast_entry *m, **pp; 1298e689cf4aSJeff Kirsher int n_addrs; 1299e689cf4aSJeff Kirsher 1300e689cf4aSJeff Kirsher memset(&info, 0, sizeof(info)); 1301e689cf4aSJeff Kirsher 1302e689cf4aSJeff Kirsher info.tag.type = VIO_TYPE_CTRL; 1303e689cf4aSJeff Kirsher info.tag.stype = VIO_SUBTYPE_INFO; 1304e689cf4aSJeff Kirsher info.tag.stype_env = VNET_MCAST_INFO; 1305e689cf4aSJeff Kirsher info.tag.sid = vio_send_sid(&port->vio); 1306e689cf4aSJeff Kirsher info.set = 1; 1307e689cf4aSJeff Kirsher 1308e689cf4aSJeff Kirsher n_addrs = 0; 1309e689cf4aSJeff Kirsher for (m = vp->mcast_list; m; m = m->next) { 1310e689cf4aSJeff Kirsher if (m->sent) 1311e689cf4aSJeff Kirsher continue; 1312e689cf4aSJeff Kirsher m->sent = 1; 1313e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1314e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1315e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1316e689cf4aSJeff Kirsher info.count = n_addrs; 1317e689cf4aSJeff Kirsher 1318e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1319e689cf4aSJeff Kirsher sizeof(info)); 1320e689cf4aSJeff Kirsher n_addrs = 0; 1321e689cf4aSJeff Kirsher } 1322e689cf4aSJeff Kirsher } 1323e689cf4aSJeff Kirsher if (n_addrs) { 1324e689cf4aSJeff Kirsher info.count = n_addrs; 1325e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1326e689cf4aSJeff Kirsher } 1327e689cf4aSJeff Kirsher 1328e689cf4aSJeff Kirsher info.set = 0; 1329e689cf4aSJeff Kirsher 1330e689cf4aSJeff Kirsher n_addrs = 0; 1331e689cf4aSJeff Kirsher pp = &vp->mcast_list; 1332e689cf4aSJeff Kirsher while ((m = *pp) != NULL) { 1333e689cf4aSJeff Kirsher if (m->hit) { 1334e689cf4aSJeff Kirsher m->hit = 0; 1335e689cf4aSJeff Kirsher pp = &m->next; 1336e689cf4aSJeff Kirsher continue; 1337e689cf4aSJeff Kirsher } 1338e689cf4aSJeff Kirsher 1339e689cf4aSJeff Kirsher memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1340e689cf4aSJeff Kirsher m->addr, ETH_ALEN); 1341e689cf4aSJeff Kirsher if (++n_addrs == VNET_NUM_MCAST) { 1342e689cf4aSJeff Kirsher info.count = n_addrs; 1343e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, 1344e689cf4aSJeff Kirsher sizeof(info)); 1345e689cf4aSJeff Kirsher n_addrs = 0; 1346e689cf4aSJeff Kirsher } 1347e689cf4aSJeff Kirsher 1348e689cf4aSJeff Kirsher *pp = m->next; 1349e689cf4aSJeff Kirsher kfree(m); 1350e689cf4aSJeff Kirsher } 1351e689cf4aSJeff Kirsher if (n_addrs) { 1352e689cf4aSJeff Kirsher info.count = n_addrs; 1353e689cf4aSJeff Kirsher (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1354e689cf4aSJeff Kirsher } 1355e689cf4aSJeff Kirsher } 1356e689cf4aSJeff Kirsher 1357e689cf4aSJeff Kirsher static void vnet_set_rx_mode(struct net_device *dev) 1358e689cf4aSJeff Kirsher { 1359e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1360e689cf4aSJeff Kirsher struct vnet_port *port; 1361e689cf4aSJeff Kirsher 13622a968dd8SSowmini Varadhan rcu_read_lock(); 13632a968dd8SSowmini Varadhan list_for_each_entry_rcu(port, &vp->port_list, list) { 1364e689cf4aSJeff Kirsher 1365e689cf4aSJeff Kirsher if (port->switch_port) { 1366e689cf4aSJeff Kirsher __update_mc_list(vp, dev); 1367e689cf4aSJeff Kirsher __send_mc_list(vp, port); 13682a968dd8SSowmini Varadhan break; 1369e689cf4aSJeff Kirsher } 1370e689cf4aSJeff Kirsher } 13712a968dd8SSowmini Varadhan rcu_read_unlock(); 1372e689cf4aSJeff Kirsher } 1373e689cf4aSJeff Kirsher 1374e689cf4aSJeff Kirsher static int vnet_change_mtu(struct net_device *dev, int new_mtu) 1375e689cf4aSJeff Kirsher { 137642db672dSDavid L Stevens if (new_mtu < 68 || new_mtu > 65535) 1377e689cf4aSJeff Kirsher return -EINVAL; 1378e689cf4aSJeff Kirsher 1379e689cf4aSJeff Kirsher dev->mtu = new_mtu; 1380e689cf4aSJeff Kirsher return 0; 1381e689cf4aSJeff Kirsher } 1382e689cf4aSJeff Kirsher 1383e689cf4aSJeff Kirsher static int vnet_set_mac_addr(struct net_device *dev, void *p) 1384e689cf4aSJeff Kirsher { 1385e689cf4aSJeff Kirsher return -EINVAL; 1386e689cf4aSJeff Kirsher } 1387e689cf4aSJeff Kirsher 1388e689cf4aSJeff Kirsher static void vnet_get_drvinfo(struct net_device *dev, 1389e689cf4aSJeff Kirsher struct ethtool_drvinfo *info) 1390e689cf4aSJeff Kirsher { 13917826d43fSJiri Pirko strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 13927826d43fSJiri Pirko strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); 1393e689cf4aSJeff Kirsher } 1394e689cf4aSJeff Kirsher 1395e689cf4aSJeff Kirsher static u32 vnet_get_msglevel(struct net_device *dev) 1396e689cf4aSJeff Kirsher { 1397e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1398e689cf4aSJeff Kirsher return vp->msg_enable; 1399e689cf4aSJeff Kirsher } 1400e689cf4aSJeff Kirsher 1401e689cf4aSJeff Kirsher static void vnet_set_msglevel(struct net_device *dev, u32 value) 1402e689cf4aSJeff Kirsher { 1403e689cf4aSJeff Kirsher struct vnet *vp = netdev_priv(dev); 1404e689cf4aSJeff Kirsher vp->msg_enable = value; 1405e689cf4aSJeff Kirsher } 1406e689cf4aSJeff Kirsher 1407e689cf4aSJeff Kirsher static const struct ethtool_ops vnet_ethtool_ops = { 1408e689cf4aSJeff Kirsher .get_drvinfo = vnet_get_drvinfo, 1409e689cf4aSJeff Kirsher .get_msglevel = vnet_get_msglevel, 1410e689cf4aSJeff Kirsher .set_msglevel = vnet_set_msglevel, 1411e689cf4aSJeff Kirsher .get_link = ethtool_op_get_link, 1412e689cf4aSJeff Kirsher }; 1413e689cf4aSJeff Kirsher 1414e689cf4aSJeff Kirsher static void vnet_port_free_tx_bufs(struct vnet_port *port) 1415e689cf4aSJeff Kirsher { 1416e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1417e689cf4aSJeff Kirsher int i; 1418e689cf4aSJeff Kirsher 1419e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1420e689cf4aSJeff Kirsher if (dr->base) { 1421e689cf4aSJeff Kirsher ldc_free_exp_dring(port->vio.lp, dr->base, 1422e689cf4aSJeff Kirsher (dr->entry_size * dr->num_entries), 1423e689cf4aSJeff Kirsher dr->cookies, dr->ncookies); 1424e689cf4aSJeff Kirsher dr->base = NULL; 1425e689cf4aSJeff Kirsher dr->entry_size = 0; 1426e689cf4aSJeff Kirsher dr->num_entries = 0; 1427e689cf4aSJeff Kirsher dr->pending = 0; 1428e689cf4aSJeff Kirsher dr->ncookies = 0; 1429e689cf4aSJeff Kirsher } 1430e689cf4aSJeff Kirsher 1431e689cf4aSJeff Kirsher for (i = 0; i < VNET_TX_RING_SIZE; i++) { 14328e845f4cSDavid L Stevens struct vio_net_desc *d; 14338e845f4cSDavid L Stevens void *skb = port->tx_bufs[i].skb; 1434e689cf4aSJeff Kirsher 14358e845f4cSDavid L Stevens if (!skb) 1436e689cf4aSJeff Kirsher continue; 1437e689cf4aSJeff Kirsher 14388e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 14398e845f4cSDavid L Stevens if (d->hdr.state == VIO_DESC_READY) 14408e845f4cSDavid L Stevens pr_warn("active transmit buffers freed\n"); 14418e845f4cSDavid L Stevens 1442e689cf4aSJeff Kirsher ldc_unmap(port->vio.lp, 1443e689cf4aSJeff Kirsher port->tx_bufs[i].cookies, 1444e689cf4aSJeff Kirsher port->tx_bufs[i].ncookies); 14458e845f4cSDavid L Stevens dev_kfree_skb(skb); 14468e845f4cSDavid L Stevens port->tx_bufs[i].skb = NULL; 14478e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 1448e689cf4aSJeff Kirsher } 1449e689cf4aSJeff Kirsher } 1450e689cf4aSJeff Kirsher 1451d6732489SDavid L Stevens static int vnet_port_alloc_tx_ring(struct vnet_port *port) 1452e689cf4aSJeff Kirsher { 1453e689cf4aSJeff Kirsher struct vio_dring_state *dr; 1454*6d0ba919SDavid L Stevens unsigned long len, elen; 1455e689cf4aSJeff Kirsher int i, err, ncookies; 1456e689cf4aSJeff Kirsher void *dring; 1457e689cf4aSJeff Kirsher 1458e689cf4aSJeff Kirsher dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1459e689cf4aSJeff Kirsher 1460*6d0ba919SDavid L Stevens elen = sizeof(struct vio_net_desc) + 1461*6d0ba919SDavid L Stevens sizeof(struct ldc_trans_cookie) * 2; 1462*6d0ba919SDavid L Stevens if (vio_version_after_eq(&port->vio, 1, 7)) 1463*6d0ba919SDavid L Stevens elen += sizeof(struct vio_net_dext); 1464*6d0ba919SDavid L Stevens len = VNET_TX_RING_SIZE * elen; 1465e689cf4aSJeff Kirsher 1466e689cf4aSJeff Kirsher ncookies = VIO_MAX_RING_COOKIES; 1467e689cf4aSJeff Kirsher dring = ldc_alloc_exp_dring(port->vio.lp, len, 1468e689cf4aSJeff Kirsher dr->cookies, &ncookies, 1469e689cf4aSJeff Kirsher (LDC_MAP_SHADOW | 1470e689cf4aSJeff Kirsher LDC_MAP_DIRECT | 1471e689cf4aSJeff Kirsher LDC_MAP_RW)); 1472e689cf4aSJeff Kirsher if (IS_ERR(dring)) { 1473e689cf4aSJeff Kirsher err = PTR_ERR(dring); 1474e689cf4aSJeff Kirsher goto err_out; 1475e689cf4aSJeff Kirsher } 1476e689cf4aSJeff Kirsher 1477e689cf4aSJeff Kirsher dr->base = dring; 1478*6d0ba919SDavid L Stevens dr->entry_size = elen; 1479e689cf4aSJeff Kirsher dr->num_entries = VNET_TX_RING_SIZE; 1480e689cf4aSJeff Kirsher dr->prod = dr->cons = 0; 1481d1015645SSowmini Varadhan port->start_cons = true; /* need an initial trigger */ 1482e689cf4aSJeff Kirsher dr->pending = VNET_TX_RING_SIZE; 1483e689cf4aSJeff Kirsher dr->ncookies = ncookies; 1484e689cf4aSJeff Kirsher 14858e845f4cSDavid L Stevens for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 14868e845f4cSDavid L Stevens struct vio_net_desc *d; 14878e845f4cSDavid L Stevens 14888e845f4cSDavid L Stevens d = vio_dring_entry(dr, i); 14898e845f4cSDavid L Stevens d->hdr.state = VIO_DESC_FREE; 14908e845f4cSDavid L Stevens } 1491e689cf4aSJeff Kirsher return 0; 1492e689cf4aSJeff Kirsher 1493e689cf4aSJeff Kirsher err_out: 1494e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1495e689cf4aSJeff Kirsher 1496e689cf4aSJeff Kirsher return err; 1497e689cf4aSJeff Kirsher } 1498e689cf4aSJeff Kirsher 149969088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 150069088822SSowmini Varadhan static void vnet_poll_controller(struct net_device *dev) 150169088822SSowmini Varadhan { 150269088822SSowmini Varadhan struct vnet *vp = netdev_priv(dev); 150369088822SSowmini Varadhan struct vnet_port *port; 150469088822SSowmini Varadhan unsigned long flags; 150569088822SSowmini Varadhan 150669088822SSowmini Varadhan spin_lock_irqsave(&vp->lock, flags); 150769088822SSowmini Varadhan if (!list_empty(&vp->port_list)) { 150869088822SSowmini Varadhan port = list_entry(vp->port_list.next, struct vnet_port, list); 150969088822SSowmini Varadhan napi_schedule(&port->napi); 151069088822SSowmini Varadhan } 151169088822SSowmini Varadhan spin_unlock_irqrestore(&vp->lock, flags); 151269088822SSowmini Varadhan } 151369088822SSowmini Varadhan #endif 1514e689cf4aSJeff Kirsher static LIST_HEAD(vnet_list); 1515e689cf4aSJeff Kirsher static DEFINE_MUTEX(vnet_list_mutex); 1516e689cf4aSJeff Kirsher 1517e689cf4aSJeff Kirsher static const struct net_device_ops vnet_ops = { 1518e689cf4aSJeff Kirsher .ndo_open = vnet_open, 1519e689cf4aSJeff Kirsher .ndo_stop = vnet_close, 1520afc4b13dSJiri Pirko .ndo_set_rx_mode = vnet_set_rx_mode, 1521e689cf4aSJeff Kirsher .ndo_set_mac_address = vnet_set_mac_addr, 1522e689cf4aSJeff Kirsher .ndo_validate_addr = eth_validate_addr, 1523e689cf4aSJeff Kirsher .ndo_tx_timeout = vnet_tx_timeout, 1524e689cf4aSJeff Kirsher .ndo_change_mtu = vnet_change_mtu, 1525e689cf4aSJeff Kirsher .ndo_start_xmit = vnet_start_xmit, 1526d51bffd1SSowmini Varadhan .ndo_select_queue = vnet_select_queue, 152769088822SSowmini Varadhan #ifdef CONFIG_NET_POLL_CONTROLLER 152869088822SSowmini Varadhan .ndo_poll_controller = vnet_poll_controller, 152969088822SSowmini Varadhan #endif 1530e689cf4aSJeff Kirsher }; 1531e689cf4aSJeff Kirsher 1532f73d12bdSBill Pemberton static struct vnet *vnet_new(const u64 *local_mac) 1533e689cf4aSJeff Kirsher { 1534e689cf4aSJeff Kirsher struct net_device *dev; 1535e689cf4aSJeff Kirsher struct vnet *vp; 1536e689cf4aSJeff Kirsher int err, i; 1537e689cf4aSJeff Kirsher 1538d51bffd1SSowmini Varadhan dev = alloc_etherdev_mqs(sizeof(*vp), VNET_MAX_TXQS, 1); 153941de8d4cSJoe Perches if (!dev) 1540e689cf4aSJeff Kirsher return ERR_PTR(-ENOMEM); 15418e845f4cSDavid L Stevens dev->needed_headroom = VNET_PACKET_SKIP + 8; 15428e845f4cSDavid L Stevens dev->needed_tailroom = 8; 1543e689cf4aSJeff Kirsher 1544e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1545e689cf4aSJeff Kirsher dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff; 1546e689cf4aSJeff Kirsher 1547e689cf4aSJeff Kirsher vp = netdev_priv(dev); 1548e689cf4aSJeff Kirsher 1549e689cf4aSJeff Kirsher spin_lock_init(&vp->lock); 1550e689cf4aSJeff Kirsher vp->dev = dev; 1551e689cf4aSJeff Kirsher 1552e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->port_list); 1553e689cf4aSJeff Kirsher for (i = 0; i < VNET_PORT_HASH_SIZE; i++) 1554e689cf4aSJeff Kirsher INIT_HLIST_HEAD(&vp->port_hash[i]); 1555e689cf4aSJeff Kirsher INIT_LIST_HEAD(&vp->list); 1556e689cf4aSJeff Kirsher vp->local_mac = *local_mac; 1557e689cf4aSJeff Kirsher 1558e689cf4aSJeff Kirsher dev->netdev_ops = &vnet_ops; 1559e689cf4aSJeff Kirsher dev->ethtool_ops = &vnet_ethtool_ops; 1560e689cf4aSJeff Kirsher dev->watchdog_timeo = VNET_TX_TIMEOUT; 1561e689cf4aSJeff Kirsher 1562e689cf4aSJeff Kirsher err = register_netdev(dev); 1563e689cf4aSJeff Kirsher if (err) { 1564e689cf4aSJeff Kirsher pr_err("Cannot register net device, aborting\n"); 1565e689cf4aSJeff Kirsher goto err_out_free_dev; 1566e689cf4aSJeff Kirsher } 1567e689cf4aSJeff Kirsher 1568e689cf4aSJeff Kirsher netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr); 1569e689cf4aSJeff Kirsher 1570e689cf4aSJeff Kirsher list_add(&vp->list, &vnet_list); 1571e689cf4aSJeff Kirsher 1572e689cf4aSJeff Kirsher return vp; 1573e689cf4aSJeff Kirsher 1574e689cf4aSJeff Kirsher err_out_free_dev: 1575e689cf4aSJeff Kirsher free_netdev(dev); 1576e689cf4aSJeff Kirsher 1577e689cf4aSJeff Kirsher return ERR_PTR(err); 1578e689cf4aSJeff Kirsher } 1579e689cf4aSJeff Kirsher 1580f73d12bdSBill Pemberton static struct vnet *vnet_find_or_create(const u64 *local_mac) 1581e689cf4aSJeff Kirsher { 1582e689cf4aSJeff Kirsher struct vnet *iter, *vp; 1583e689cf4aSJeff Kirsher 1584e689cf4aSJeff Kirsher mutex_lock(&vnet_list_mutex); 1585e689cf4aSJeff Kirsher vp = NULL; 1586e689cf4aSJeff Kirsher list_for_each_entry(iter, &vnet_list, list) { 1587e689cf4aSJeff Kirsher if (iter->local_mac == *local_mac) { 1588e689cf4aSJeff Kirsher vp = iter; 1589e689cf4aSJeff Kirsher break; 1590e689cf4aSJeff Kirsher } 1591e689cf4aSJeff Kirsher } 1592e689cf4aSJeff Kirsher if (!vp) 1593e689cf4aSJeff Kirsher vp = vnet_new(local_mac); 1594e689cf4aSJeff Kirsher mutex_unlock(&vnet_list_mutex); 1595e689cf4aSJeff Kirsher 1596e689cf4aSJeff Kirsher return vp; 1597e689cf4aSJeff Kirsher } 1598e689cf4aSJeff Kirsher 1599a4b70a07SSowmini Varadhan static void vnet_cleanup(void) 1600a4b70a07SSowmini Varadhan { 1601a4b70a07SSowmini Varadhan struct vnet *vp; 1602a4b70a07SSowmini Varadhan struct net_device *dev; 1603a4b70a07SSowmini Varadhan 1604a4b70a07SSowmini Varadhan mutex_lock(&vnet_list_mutex); 1605a4b70a07SSowmini Varadhan while (!list_empty(&vnet_list)) { 1606a4b70a07SSowmini Varadhan vp = list_first_entry(&vnet_list, struct vnet, list); 1607a4b70a07SSowmini Varadhan list_del(&vp->list); 1608a4b70a07SSowmini Varadhan dev = vp->dev; 1609a4b70a07SSowmini Varadhan /* vio_unregister_driver() should have cleaned up port_list */ 1610a4b70a07SSowmini Varadhan BUG_ON(!list_empty(&vp->port_list)); 1611a4b70a07SSowmini Varadhan unregister_netdev(dev); 1612a4b70a07SSowmini Varadhan free_netdev(dev); 1613a4b70a07SSowmini Varadhan } 1614a4b70a07SSowmini Varadhan mutex_unlock(&vnet_list_mutex); 1615a4b70a07SSowmini Varadhan } 1616a4b70a07SSowmini Varadhan 1617e689cf4aSJeff Kirsher static const char *local_mac_prop = "local-mac-address"; 1618e689cf4aSJeff Kirsher 1619f73d12bdSBill Pemberton static struct vnet *vnet_find_parent(struct mdesc_handle *hp, 1620e689cf4aSJeff Kirsher u64 port_node) 1621e689cf4aSJeff Kirsher { 1622e689cf4aSJeff Kirsher const u64 *local_mac = NULL; 1623e689cf4aSJeff Kirsher u64 a; 1624e689cf4aSJeff Kirsher 1625e689cf4aSJeff Kirsher mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) { 1626e689cf4aSJeff Kirsher u64 target = mdesc_arc_target(hp, a); 1627e689cf4aSJeff Kirsher const char *name; 1628e689cf4aSJeff Kirsher 1629e689cf4aSJeff Kirsher name = mdesc_get_property(hp, target, "name", NULL); 1630e689cf4aSJeff Kirsher if (!name || strcmp(name, "network")) 1631e689cf4aSJeff Kirsher continue; 1632e689cf4aSJeff Kirsher 1633e689cf4aSJeff Kirsher local_mac = mdesc_get_property(hp, target, 1634e689cf4aSJeff Kirsher local_mac_prop, NULL); 1635e689cf4aSJeff Kirsher if (local_mac) 1636e689cf4aSJeff Kirsher break; 1637e689cf4aSJeff Kirsher } 1638e689cf4aSJeff Kirsher if (!local_mac) 1639e689cf4aSJeff Kirsher return ERR_PTR(-ENODEV); 1640e689cf4aSJeff Kirsher 1641e689cf4aSJeff Kirsher return vnet_find_or_create(local_mac); 1642e689cf4aSJeff Kirsher } 1643e689cf4aSJeff Kirsher 1644e689cf4aSJeff Kirsher static struct ldc_channel_config vnet_ldc_cfg = { 1645e689cf4aSJeff Kirsher .event = vnet_event, 1646e689cf4aSJeff Kirsher .mtu = 64, 1647e689cf4aSJeff Kirsher .mode = LDC_MODE_UNRELIABLE, 1648e689cf4aSJeff Kirsher }; 1649e689cf4aSJeff Kirsher 1650e689cf4aSJeff Kirsher static struct vio_driver_ops vnet_vio_ops = { 1651e689cf4aSJeff Kirsher .send_attr = vnet_send_attr, 1652e689cf4aSJeff Kirsher .handle_attr = vnet_handle_attr, 1653e689cf4aSJeff Kirsher .handshake_complete = vnet_handshake_complete, 1654e689cf4aSJeff Kirsher }; 1655e689cf4aSJeff Kirsher 1656f73d12bdSBill Pemberton static void print_version(void) 1657e689cf4aSJeff Kirsher { 1658e689cf4aSJeff Kirsher printk_once(KERN_INFO "%s", version); 1659e689cf4aSJeff Kirsher } 1660e689cf4aSJeff Kirsher 1661e689cf4aSJeff Kirsher const char *remote_macaddr_prop = "remote-mac-address"; 1662e689cf4aSJeff Kirsher 1663d51bffd1SSowmini Varadhan static void 1664d51bffd1SSowmini Varadhan vnet_port_add_txq(struct vnet_port *port) 1665d51bffd1SSowmini Varadhan { 1666d51bffd1SSowmini Varadhan struct vnet *vp = port->vp; 1667d51bffd1SSowmini Varadhan int n; 1668d51bffd1SSowmini Varadhan 1669d51bffd1SSowmini Varadhan n = vp->nports++; 1670d51bffd1SSowmini Varadhan n = n & (VNET_MAX_TXQS - 1); 1671d51bffd1SSowmini Varadhan port->q_index = n; 1672d51bffd1SSowmini Varadhan netif_tx_wake_queue(netdev_get_tx_queue(vp->dev, port->q_index)); 1673d51bffd1SSowmini Varadhan } 1674d51bffd1SSowmini Varadhan 1675d51bffd1SSowmini Varadhan static void 1676d51bffd1SSowmini Varadhan vnet_port_rm_txq(struct vnet_port *port) 1677d51bffd1SSowmini Varadhan { 1678d51bffd1SSowmini Varadhan port->vp->nports--; 1679d51bffd1SSowmini Varadhan netif_tx_stop_queue(netdev_get_tx_queue(port->vp->dev, port->q_index)); 1680d51bffd1SSowmini Varadhan } 1681d51bffd1SSowmini Varadhan 16821dd06ae8SGreg Kroah-Hartman static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 1683e689cf4aSJeff Kirsher { 1684e689cf4aSJeff Kirsher struct mdesc_handle *hp; 1685e689cf4aSJeff Kirsher struct vnet_port *port; 1686e689cf4aSJeff Kirsher unsigned long flags; 1687e689cf4aSJeff Kirsher struct vnet *vp; 1688e689cf4aSJeff Kirsher const u64 *rmac; 1689e689cf4aSJeff Kirsher int len, i, err, switch_port; 1690e689cf4aSJeff Kirsher 1691e689cf4aSJeff Kirsher print_version(); 1692e689cf4aSJeff Kirsher 1693e689cf4aSJeff Kirsher hp = mdesc_grab(); 1694e689cf4aSJeff Kirsher 1695e689cf4aSJeff Kirsher vp = vnet_find_parent(hp, vdev->mp); 1696e689cf4aSJeff Kirsher if (IS_ERR(vp)) { 1697e689cf4aSJeff Kirsher pr_err("Cannot find port parent vnet\n"); 1698e689cf4aSJeff Kirsher err = PTR_ERR(vp); 1699e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1700e689cf4aSJeff Kirsher } 1701e689cf4aSJeff Kirsher 1702e689cf4aSJeff Kirsher rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len); 1703e689cf4aSJeff Kirsher err = -ENODEV; 1704e689cf4aSJeff Kirsher if (!rmac) { 1705e689cf4aSJeff Kirsher pr_err("Port lacks %s property\n", remote_macaddr_prop); 1706e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1707e689cf4aSJeff Kirsher } 1708e689cf4aSJeff Kirsher 1709e689cf4aSJeff Kirsher port = kzalloc(sizeof(*port), GFP_KERNEL); 1710e689cf4aSJeff Kirsher err = -ENOMEM; 1711e404decbSJoe Perches if (!port) 1712e689cf4aSJeff Kirsher goto err_out_put_mdesc; 1713e689cf4aSJeff Kirsher 1714e689cf4aSJeff Kirsher for (i = 0; i < ETH_ALEN; i++) 1715e689cf4aSJeff Kirsher port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff; 1716e689cf4aSJeff Kirsher 1717e689cf4aSJeff Kirsher port->vp = vp; 1718e689cf4aSJeff Kirsher 1719e689cf4aSJeff Kirsher err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK, 1720e689cf4aSJeff Kirsher vnet_versions, ARRAY_SIZE(vnet_versions), 1721e689cf4aSJeff Kirsher &vnet_vio_ops, vp->dev->name); 1722e689cf4aSJeff Kirsher if (err) 1723e689cf4aSJeff Kirsher goto err_out_free_port; 1724e689cf4aSJeff Kirsher 1725e689cf4aSJeff Kirsher err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port); 1726e689cf4aSJeff Kirsher if (err) 1727e689cf4aSJeff Kirsher goto err_out_free_port; 1728e689cf4aSJeff Kirsher 172969088822SSowmini Varadhan netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); 173069088822SSowmini Varadhan 1731e689cf4aSJeff Kirsher INIT_HLIST_NODE(&port->hash); 1732e689cf4aSJeff Kirsher INIT_LIST_HEAD(&port->list); 1733e689cf4aSJeff Kirsher 1734e689cf4aSJeff Kirsher switch_port = 0; 1735e689cf4aSJeff Kirsher if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL) 1736e689cf4aSJeff Kirsher switch_port = 1; 1737e689cf4aSJeff Kirsher port->switch_port = switch_port; 1738e689cf4aSJeff Kirsher 1739e689cf4aSJeff Kirsher spin_lock_irqsave(&vp->lock, flags); 1740e689cf4aSJeff Kirsher if (switch_port) 17412a968dd8SSowmini Varadhan list_add_rcu(&port->list, &vp->port_list); 1742e689cf4aSJeff Kirsher else 17432a968dd8SSowmini Varadhan list_add_tail_rcu(&port->list, &vp->port_list); 17442a968dd8SSowmini Varadhan hlist_add_head_rcu(&port->hash, 17452a968dd8SSowmini Varadhan &vp->port_hash[vnet_hashfn(port->raddr)]); 1746d51bffd1SSowmini Varadhan vnet_port_add_txq(port); 1747e689cf4aSJeff Kirsher spin_unlock_irqrestore(&vp->lock, flags); 1748e689cf4aSJeff Kirsher 1749e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, port); 1750e689cf4aSJeff Kirsher 1751e689cf4aSJeff Kirsher pr_info("%s: PORT ( remote-mac %pM%s )\n", 1752e689cf4aSJeff Kirsher vp->dev->name, port->raddr, switch_port ? " switch-port" : ""); 1753e689cf4aSJeff Kirsher 17548e845f4cSDavid L Stevens setup_timer(&port->clean_timer, vnet_clean_timer_expire, 17558e845f4cSDavid L Stevens (unsigned long)port); 17568e845f4cSDavid L Stevens 175769088822SSowmini Varadhan napi_enable(&port->napi); 1758e689cf4aSJeff Kirsher vio_port_up(&port->vio); 1759e689cf4aSJeff Kirsher 1760e689cf4aSJeff Kirsher mdesc_release(hp); 1761e689cf4aSJeff Kirsher 1762e689cf4aSJeff Kirsher return 0; 1763e689cf4aSJeff Kirsher 1764e689cf4aSJeff Kirsher err_out_free_port: 1765e689cf4aSJeff Kirsher kfree(port); 1766e689cf4aSJeff Kirsher 1767e689cf4aSJeff Kirsher err_out_put_mdesc: 1768e689cf4aSJeff Kirsher mdesc_release(hp); 1769e689cf4aSJeff Kirsher return err; 1770e689cf4aSJeff Kirsher } 1771e689cf4aSJeff Kirsher 1772e689cf4aSJeff Kirsher static int vnet_port_remove(struct vio_dev *vdev) 1773e689cf4aSJeff Kirsher { 1774e689cf4aSJeff Kirsher struct vnet_port *port = dev_get_drvdata(&vdev->dev); 1775e689cf4aSJeff Kirsher 1776e689cf4aSJeff Kirsher if (port) { 1777e689cf4aSJeff Kirsher 1778e689cf4aSJeff Kirsher del_timer_sync(&port->vio.timer); 1779e689cf4aSJeff Kirsher 178069088822SSowmini Varadhan napi_disable(&port->napi); 1781e689cf4aSJeff Kirsher 17822a968dd8SSowmini Varadhan list_del_rcu(&port->list); 17832a968dd8SSowmini Varadhan hlist_del_rcu(&port->hash); 17842a968dd8SSowmini Varadhan 17852a968dd8SSowmini Varadhan synchronize_rcu(); 17862a968dd8SSowmini Varadhan del_timer_sync(&port->clean_timer); 1787d51bffd1SSowmini Varadhan vnet_port_rm_txq(port); 178869088822SSowmini Varadhan netif_napi_del(&port->napi); 1789e689cf4aSJeff Kirsher vnet_port_free_tx_bufs(port); 1790e689cf4aSJeff Kirsher vio_ldc_free(&port->vio); 1791e689cf4aSJeff Kirsher 1792e689cf4aSJeff Kirsher dev_set_drvdata(&vdev->dev, NULL); 1793e689cf4aSJeff Kirsher 1794e689cf4aSJeff Kirsher kfree(port); 1795aabb9875SDave Kleikamp 1796e689cf4aSJeff Kirsher } 1797e689cf4aSJeff Kirsher return 0; 1798e689cf4aSJeff Kirsher } 1799e689cf4aSJeff Kirsher 1800e689cf4aSJeff Kirsher static const struct vio_device_id vnet_port_match[] = { 1801e689cf4aSJeff Kirsher { 1802e689cf4aSJeff Kirsher .type = "vnet-port", 1803e689cf4aSJeff Kirsher }, 1804e689cf4aSJeff Kirsher {}, 1805e689cf4aSJeff Kirsher }; 1806e689cf4aSJeff Kirsher MODULE_DEVICE_TABLE(vio, vnet_port_match); 1807e689cf4aSJeff Kirsher 1808e689cf4aSJeff Kirsher static struct vio_driver vnet_port_driver = { 1809e689cf4aSJeff Kirsher .id_table = vnet_port_match, 1810e689cf4aSJeff Kirsher .probe = vnet_port_probe, 1811e689cf4aSJeff Kirsher .remove = vnet_port_remove, 1812e689cf4aSJeff Kirsher .name = "vnet_port", 1813e689cf4aSJeff Kirsher }; 1814e689cf4aSJeff Kirsher 1815e689cf4aSJeff Kirsher static int __init vnet_init(void) 1816e689cf4aSJeff Kirsher { 1817e689cf4aSJeff Kirsher return vio_register_driver(&vnet_port_driver); 1818e689cf4aSJeff Kirsher } 1819e689cf4aSJeff Kirsher 1820e689cf4aSJeff Kirsher static void __exit vnet_exit(void) 1821e689cf4aSJeff Kirsher { 1822e689cf4aSJeff Kirsher vio_unregister_driver(&vnet_port_driver); 1823a4b70a07SSowmini Varadhan vnet_cleanup(); 1824e689cf4aSJeff Kirsher } 1825e689cf4aSJeff Kirsher 1826e689cf4aSJeff Kirsher module_init(vnet_init); 1827e689cf4aSJeff Kirsher module_exit(vnet_exit); 1828