1 /* sunvnet.c: Sun LDOM Virtual Network Driver. 2 * 3 * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4 */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <linux/module.h> 9 #include <linux/kernel.h> 10 #include <linux/types.h> 11 #include <linux/slab.h> 12 #include <linux/delay.h> 13 #include <linux/init.h> 14 #include <linux/netdevice.h> 15 #include <linux/ethtool.h> 16 #include <linux/etherdevice.h> 17 #include <linux/mutex.h> 18 #include <linux/highmem.h> 19 #include <linux/if_vlan.h> 20 21 #if IS_ENABLED(CONFIG_IPV6) 22 #include <linux/icmpv6.h> 23 #endif 24 25 #include <net/ip.h> 26 #include <net/icmp.h> 27 #include <net/route.h> 28 29 #include <asm/vio.h> 30 #include <asm/ldc.h> 31 32 #include "sunvnet.h" 33 34 #define DRV_MODULE_NAME "sunvnet" 35 #define DRV_MODULE_VERSION "1.0" 36 #define DRV_MODULE_RELDATE "June 25, 2007" 37 38 static char version[] = 39 DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)"); 41 MODULE_DESCRIPTION("Sun LDOM virtual network driver"); 42 MODULE_LICENSE("GPL"); 43 MODULE_VERSION(DRV_MODULE_VERSION); 44 45 #define VNET_MAX_TXQS 16 46 47 /* Heuristic for the number of times to exponentially backoff and 48 * retry sending an LDC trigger when EAGAIN is encountered 49 */ 50 #define VNET_MAX_RETRIES 10 51 52 static int __vnet_tx_trigger(struct vnet_port *port, u32 start); 53 54 /* Ordered from largest major to lowest */ 55 static struct vio_version vnet_versions[] = { 56 { .major = 1, .minor = 8 }, 57 { .major = 1, .minor = 7 }, 58 { .major = 1, .minor = 6 }, 59 { .major = 1, .minor = 0 }, 60 }; 61 62 static inline u32 vnet_tx_dring_avail(struct vio_dring_state *dr) 63 { 64 return vio_dring_avail(dr, VNET_TX_RING_SIZE); 65 } 66 67 static int vnet_handle_unknown(struct vnet_port *port, void *arg) 68 { 69 struct vio_msg_tag *pkt = arg; 70 71 pr_err("Received unknown msg [%02x:%02x:%04x:%08x]\n", 72 pkt->type, pkt->stype, pkt->stype_env, pkt->sid); 73 pr_err("Resetting connection\n"); 74 75 ldc_disconnect(port->vio.lp); 76 77 return -ECONNRESET; 78 } 79 80 static int vnet_port_alloc_tx_ring(struct vnet_port *port); 81 82 static int vnet_send_attr(struct vio_driver_state *vio) 83 { 84 struct vnet_port *port = to_vnet_port(vio); 85 struct net_device *dev = port->vp->dev; 86 struct vio_net_attr_info pkt; 87 int framelen = ETH_FRAME_LEN; 88 int i, err; 89 90 err = vnet_port_alloc_tx_ring(to_vnet_port(vio)); 91 if (err) 92 return err; 93 94 memset(&pkt, 0, sizeof(pkt)); 95 pkt.tag.type = VIO_TYPE_CTRL; 96 pkt.tag.stype = VIO_SUBTYPE_INFO; 97 pkt.tag.stype_env = VIO_ATTR_INFO; 98 pkt.tag.sid = vio_send_sid(vio); 99 if (vio_version_before(vio, 1, 2)) 100 pkt.xfer_mode = VIO_DRING_MODE; 101 else 102 pkt.xfer_mode = VIO_NEW_DRING_MODE; 103 pkt.addr_type = VNET_ADDR_ETHERMAC; 104 pkt.ack_freq = 0; 105 for (i = 0; i < 6; i++) 106 pkt.addr |= (u64)dev->dev_addr[i] << ((5 - i) * 8); 107 if (vio_version_after(vio, 1, 3)) { 108 if (port->rmtu) { 109 port->rmtu = min(VNET_MAXPACKET, port->rmtu); 110 pkt.mtu = port->rmtu; 111 } else { 112 port->rmtu = VNET_MAXPACKET; 113 pkt.mtu = port->rmtu; 114 } 115 if (vio_version_after_eq(vio, 1, 6)) 116 pkt.options = VIO_TX_DRING; 117 } else if (vio_version_before(vio, 1, 3)) { 118 pkt.mtu = framelen; 119 } else { /* v1.3 */ 120 pkt.mtu = framelen + VLAN_HLEN; 121 } 122 123 pkt.cflags = 0; 124 if (vio_version_after_eq(vio, 1, 7) && port->tso) { 125 pkt.cflags |= VNET_LSO_IPV4_CAPAB; 126 if (!port->tsolen) 127 port->tsolen = VNET_MAXTSO; 128 pkt.ipv4_lso_maxlen = port->tsolen; 129 } 130 131 pkt.plnk_updt = PHYSLINK_UPDATE_NONE; 132 133 viodbg(HS, "SEND NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 134 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 135 "cflags[0x%04x] lso_max[%u]\n", 136 pkt.xfer_mode, pkt.addr_type, 137 (unsigned long long)pkt.addr, 138 pkt.ack_freq, pkt.plnk_updt, pkt.options, 139 (unsigned long long)pkt.mtu, pkt.cflags, pkt.ipv4_lso_maxlen); 140 141 142 return vio_ldc_send(vio, &pkt, sizeof(pkt)); 143 } 144 145 static int handle_attr_info(struct vio_driver_state *vio, 146 struct vio_net_attr_info *pkt) 147 { 148 struct vnet_port *port = to_vnet_port(vio); 149 u64 localmtu; 150 u8 xfer_mode; 151 152 viodbg(HS, "GOT NET ATTR xmode[0x%x] atype[0x%x] addr[%llx] " 153 "ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] mtu[%llu] " 154 " (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 155 pkt->xfer_mode, pkt->addr_type, 156 (unsigned long long)pkt->addr, 157 pkt->ack_freq, pkt->plnk_updt, pkt->options, 158 (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 159 pkt->ipv4_lso_maxlen); 160 161 pkt->tag.sid = vio_send_sid(vio); 162 163 xfer_mode = pkt->xfer_mode; 164 /* for version < 1.2, VIO_DRING_MODE = 0x3 and no bitmask */ 165 if (vio_version_before(vio, 1, 2) && xfer_mode == VIO_DRING_MODE) 166 xfer_mode = VIO_NEW_DRING_MODE; 167 168 /* MTU negotiation: 169 * < v1.3 - ETH_FRAME_LEN exactly 170 * > v1.3 - MIN(pkt.mtu, VNET_MAXPACKET, port->rmtu) and change 171 * pkt->mtu for ACK 172 * = v1.3 - ETH_FRAME_LEN + VLAN_HLEN exactly 173 */ 174 if (vio_version_before(vio, 1, 3)) { 175 localmtu = ETH_FRAME_LEN; 176 } else if (vio_version_after(vio, 1, 3)) { 177 localmtu = port->rmtu ? port->rmtu : VNET_MAXPACKET; 178 localmtu = min(pkt->mtu, localmtu); 179 pkt->mtu = localmtu; 180 } else { /* v1.3 */ 181 localmtu = ETH_FRAME_LEN + VLAN_HLEN; 182 } 183 port->rmtu = localmtu; 184 185 /* LSO negotiation */ 186 if (vio_version_after_eq(vio, 1, 7)) 187 port->tso &= !!(pkt->cflags & VNET_LSO_IPV4_CAPAB); 188 else 189 port->tso = false; 190 if (port->tso) { 191 if (!port->tsolen) 192 port->tsolen = VNET_MAXTSO; 193 port->tsolen = min(port->tsolen, pkt->ipv4_lso_maxlen); 194 if (port->tsolen < VNET_MINTSO) { 195 port->tso = false; 196 port->tsolen = 0; 197 pkt->cflags &= ~VNET_LSO_IPV4_CAPAB; 198 } 199 pkt->ipv4_lso_maxlen = port->tsolen; 200 } else { 201 pkt->cflags &= ~VNET_LSO_IPV4_CAPAB; 202 pkt->ipv4_lso_maxlen = 0; 203 } 204 205 /* for version >= 1.6, ACK packet mode we support */ 206 if (vio_version_after_eq(vio, 1, 6)) { 207 pkt->xfer_mode = VIO_NEW_DRING_MODE; 208 pkt->options = VIO_TX_DRING; 209 } 210 211 if (!(xfer_mode | VIO_NEW_DRING_MODE) || 212 pkt->addr_type != VNET_ADDR_ETHERMAC || 213 pkt->mtu != localmtu) { 214 viodbg(HS, "SEND NET ATTR NACK\n"); 215 216 pkt->tag.stype = VIO_SUBTYPE_NACK; 217 218 (void) vio_ldc_send(vio, pkt, sizeof(*pkt)); 219 220 return -ECONNRESET; 221 } else { 222 viodbg(HS, "SEND NET ATTR ACK xmode[0x%x] atype[0x%x] " 223 "addr[%llx] ackfreq[%u] plnk_updt[0x%02x] opts[0x%02x] " 224 "mtu[%llu] (rmtu[%llu]) cflags[0x%04x] lso_max[%u]\n", 225 pkt->xfer_mode, pkt->addr_type, 226 (unsigned long long)pkt->addr, 227 pkt->ack_freq, pkt->plnk_updt, pkt->options, 228 (unsigned long long)pkt->mtu, port->rmtu, pkt->cflags, 229 pkt->ipv4_lso_maxlen); 230 231 pkt->tag.stype = VIO_SUBTYPE_ACK; 232 233 return vio_ldc_send(vio, pkt, sizeof(*pkt)); 234 } 235 236 } 237 238 static int handle_attr_ack(struct vio_driver_state *vio, 239 struct vio_net_attr_info *pkt) 240 { 241 viodbg(HS, "GOT NET ATTR ACK\n"); 242 243 return 0; 244 } 245 246 static int handle_attr_nack(struct vio_driver_state *vio, 247 struct vio_net_attr_info *pkt) 248 { 249 viodbg(HS, "GOT NET ATTR NACK\n"); 250 251 return -ECONNRESET; 252 } 253 254 static int vnet_handle_attr(struct vio_driver_state *vio, void *arg) 255 { 256 struct vio_net_attr_info *pkt = arg; 257 258 switch (pkt->tag.stype) { 259 case VIO_SUBTYPE_INFO: 260 return handle_attr_info(vio, pkt); 261 262 case VIO_SUBTYPE_ACK: 263 return handle_attr_ack(vio, pkt); 264 265 case VIO_SUBTYPE_NACK: 266 return handle_attr_nack(vio, pkt); 267 268 default: 269 return -ECONNRESET; 270 } 271 } 272 273 static void vnet_handshake_complete(struct vio_driver_state *vio) 274 { 275 struct vio_dring_state *dr; 276 277 dr = &vio->drings[VIO_DRIVER_RX_RING]; 278 dr->snd_nxt = dr->rcv_nxt = 1; 279 280 dr = &vio->drings[VIO_DRIVER_TX_RING]; 281 dr->snd_nxt = dr->rcv_nxt = 1; 282 } 283 284 /* The hypervisor interface that implements copying to/from imported 285 * memory from another domain requires that copies are done to 8-byte 286 * aligned buffers, and that the lengths of such copies are also 8-byte 287 * multiples. 288 * 289 * So we align skb->data to an 8-byte multiple and pad-out the data 290 * area so we can round the copy length up to the next multiple of 291 * 8 for the copy. 292 * 293 * The transmitter puts the actual start of the packet 6 bytes into 294 * the buffer it sends over, so that the IP headers after the ethernet 295 * header are aligned properly. These 6 bytes are not in the descriptor 296 * length, they are simply implied. This offset is represented using 297 * the VNET_PACKET_SKIP macro. 298 */ 299 static struct sk_buff *alloc_and_align_skb(struct net_device *dev, 300 unsigned int len) 301 { 302 struct sk_buff *skb = netdev_alloc_skb(dev, len+VNET_PACKET_SKIP+8+8); 303 unsigned long addr, off; 304 305 if (unlikely(!skb)) 306 return NULL; 307 308 addr = (unsigned long) skb->data; 309 off = ((addr + 7UL) & ~7UL) - addr; 310 if (off) 311 skb_reserve(skb, off); 312 313 return skb; 314 } 315 316 static inline void vnet_fullcsum(struct sk_buff *skb) 317 { 318 struct iphdr *iph = ip_hdr(skb); 319 int offset = skb_transport_offset(skb); 320 321 if (skb->protocol != htons(ETH_P_IP)) 322 return; 323 if (iph->protocol != IPPROTO_TCP && 324 iph->protocol != IPPROTO_UDP) 325 return; 326 skb->ip_summed = CHECKSUM_NONE; 327 skb->csum_level = 1; 328 skb->csum = 0; 329 if (iph->protocol == IPPROTO_TCP) { 330 struct tcphdr *ptcp = tcp_hdr(skb); 331 332 ptcp->check = 0; 333 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 334 ptcp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 335 skb->len - offset, IPPROTO_TCP, 336 skb->csum); 337 } else if (iph->protocol == IPPROTO_UDP) { 338 struct udphdr *pudp = udp_hdr(skb); 339 340 pudp->check = 0; 341 skb->csum = skb_checksum(skb, offset, skb->len - offset, 0); 342 pudp->check = csum_tcpudp_magic(iph->saddr, iph->daddr, 343 skb->len - offset, IPPROTO_UDP, 344 skb->csum); 345 } 346 } 347 348 static int vnet_rx_one(struct vnet_port *port, struct vio_net_desc *desc) 349 { 350 struct net_device *dev = port->vp->dev; 351 unsigned int len = desc->size; 352 unsigned int copy_len; 353 struct sk_buff *skb; 354 int err; 355 356 err = -EMSGSIZE; 357 if (unlikely(len < ETH_ZLEN || len > port->rmtu)) { 358 dev->stats.rx_length_errors++; 359 goto out_dropped; 360 } 361 362 skb = alloc_and_align_skb(dev, len); 363 err = -ENOMEM; 364 if (unlikely(!skb)) { 365 dev->stats.rx_missed_errors++; 366 goto out_dropped; 367 } 368 369 copy_len = (len + VNET_PACKET_SKIP + 7U) & ~7U; 370 skb_put(skb, copy_len); 371 err = ldc_copy(port->vio.lp, LDC_COPY_IN, 372 skb->data, copy_len, 0, 373 desc->cookies, desc->ncookies); 374 if (unlikely(err < 0)) { 375 dev->stats.rx_frame_errors++; 376 goto out_free_skb; 377 } 378 379 skb_pull(skb, VNET_PACKET_SKIP); 380 skb_trim(skb, len); 381 skb->protocol = eth_type_trans(skb, dev); 382 383 if (vio_version_after_eq(&port->vio, 1, 8)) { 384 struct vio_net_dext *dext = vio_net_ext(desc); 385 386 if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM) { 387 if (skb->protocol == ETH_P_IP) { 388 struct iphdr *iph = (struct iphdr *)skb->data; 389 390 iph->check = 0; 391 ip_send_check(iph); 392 } 393 } 394 if ((dext->flags & VNET_PKT_HCK_FULLCKSUM) && 395 skb->ip_summed == CHECKSUM_NONE) 396 vnet_fullcsum(skb); 397 if (dext->flags & VNET_PKT_HCK_IPV4_HDRCKSUM_OK) { 398 skb->ip_summed = CHECKSUM_PARTIAL; 399 skb->csum_level = 0; 400 if (dext->flags & VNET_PKT_HCK_FULLCKSUM_OK) 401 skb->csum_level = 1; 402 } 403 } 404 405 skb->ip_summed = port->switch_port ? CHECKSUM_NONE : CHECKSUM_PARTIAL; 406 407 dev->stats.rx_packets++; 408 dev->stats.rx_bytes += len; 409 napi_gro_receive(&port->napi, skb); 410 return 0; 411 412 out_free_skb: 413 kfree_skb(skb); 414 415 out_dropped: 416 dev->stats.rx_dropped++; 417 return err; 418 } 419 420 static int vnet_send_ack(struct vnet_port *port, struct vio_dring_state *dr, 421 u32 start, u32 end, u8 vio_dring_state) 422 { 423 struct vio_dring_data hdr = { 424 .tag = { 425 .type = VIO_TYPE_DATA, 426 .stype = VIO_SUBTYPE_ACK, 427 .stype_env = VIO_DRING_DATA, 428 .sid = vio_send_sid(&port->vio), 429 }, 430 .dring_ident = dr->ident, 431 .start_idx = start, 432 .end_idx = end, 433 .state = vio_dring_state, 434 }; 435 int err, delay; 436 int retries = 0; 437 438 hdr.seq = dr->snd_nxt; 439 delay = 1; 440 do { 441 err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 442 if (err > 0) { 443 dr->snd_nxt++; 444 break; 445 } 446 udelay(delay); 447 if ((delay <<= 1) > 128) 448 delay = 128; 449 if (retries++ > VNET_MAX_RETRIES) { 450 pr_info("ECONNRESET %x:%x:%x:%x:%x:%x\n", 451 port->raddr[0], port->raddr[1], 452 port->raddr[2], port->raddr[3], 453 port->raddr[4], port->raddr[5]); 454 break; 455 } 456 } while (err == -EAGAIN); 457 458 if (err <= 0 && vio_dring_state == VIO_DRING_STOPPED) { 459 port->stop_rx_idx = end; 460 port->stop_rx = true; 461 } else { 462 port->stop_rx_idx = 0; 463 port->stop_rx = false; 464 } 465 466 return err; 467 } 468 469 static u32 next_idx(u32 idx, struct vio_dring_state *dr) 470 { 471 if (++idx == dr->num_entries) 472 idx = 0; 473 return idx; 474 } 475 476 static u32 prev_idx(u32 idx, struct vio_dring_state *dr) 477 { 478 if (idx == 0) 479 idx = dr->num_entries - 1; 480 else 481 idx--; 482 483 return idx; 484 } 485 486 static struct vio_net_desc *get_rx_desc(struct vnet_port *port, 487 struct vio_dring_state *dr, 488 u32 index) 489 { 490 struct vio_net_desc *desc = port->vio.desc_buf; 491 int err; 492 493 err = ldc_get_dring_entry(port->vio.lp, desc, dr->entry_size, 494 (index * dr->entry_size), 495 dr->cookies, dr->ncookies); 496 if (err < 0) 497 return ERR_PTR(err); 498 499 return desc; 500 } 501 502 static int put_rx_desc(struct vnet_port *port, 503 struct vio_dring_state *dr, 504 struct vio_net_desc *desc, 505 u32 index) 506 { 507 int err; 508 509 err = ldc_put_dring_entry(port->vio.lp, desc, dr->entry_size, 510 (index * dr->entry_size), 511 dr->cookies, dr->ncookies); 512 if (err < 0) 513 return err; 514 515 return 0; 516 } 517 518 static int vnet_walk_rx_one(struct vnet_port *port, 519 struct vio_dring_state *dr, 520 u32 index, int *needs_ack) 521 { 522 struct vio_net_desc *desc = get_rx_desc(port, dr, index); 523 struct vio_driver_state *vio = &port->vio; 524 int err; 525 526 BUG_ON(desc == NULL); 527 if (IS_ERR(desc)) 528 return PTR_ERR(desc); 529 530 if (desc->hdr.state != VIO_DESC_READY) 531 return 1; 532 533 rmb(); 534 535 viodbg(DATA, "vio_walk_rx_one desc[%02x:%02x:%08x:%08x:%llx:%llx]\n", 536 desc->hdr.state, desc->hdr.ack, 537 desc->size, desc->ncookies, 538 desc->cookies[0].cookie_addr, 539 desc->cookies[0].cookie_size); 540 541 err = vnet_rx_one(port, desc); 542 if (err == -ECONNRESET) 543 return err; 544 desc->hdr.state = VIO_DESC_DONE; 545 err = put_rx_desc(port, dr, desc, index); 546 if (err < 0) 547 return err; 548 *needs_ack = desc->hdr.ack; 549 return 0; 550 } 551 552 static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, 553 u32 start, u32 end, int *npkts, int budget) 554 { 555 struct vio_driver_state *vio = &port->vio; 556 int ack_start = -1, ack_end = -1; 557 bool send_ack = true; 558 559 end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr); 560 561 viodbg(DATA, "vnet_walk_rx start[%08x] end[%08x]\n", start, end); 562 563 while (start != end) { 564 int ack = 0, err = vnet_walk_rx_one(port, dr, start, &ack); 565 if (err == -ECONNRESET) 566 return err; 567 if (err != 0) 568 break; 569 (*npkts)++; 570 if (ack_start == -1) 571 ack_start = start; 572 ack_end = start; 573 start = next_idx(start, dr); 574 if (ack && start != end) { 575 err = vnet_send_ack(port, dr, ack_start, ack_end, 576 VIO_DRING_ACTIVE); 577 if (err == -ECONNRESET) 578 return err; 579 ack_start = -1; 580 } 581 if ((*npkts) >= budget) { 582 send_ack = false; 583 break; 584 } 585 } 586 if (unlikely(ack_start == -1)) 587 ack_start = ack_end = prev_idx(start, dr); 588 if (send_ack) { 589 port->napi_resume = false; 590 return vnet_send_ack(port, dr, ack_start, ack_end, 591 VIO_DRING_STOPPED); 592 } else { 593 port->napi_resume = true; 594 port->napi_stop_idx = ack_end; 595 return 1; 596 } 597 } 598 599 static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, 600 int budget) 601 { 602 struct vio_dring_data *pkt = msgbuf; 603 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; 604 struct vio_driver_state *vio = &port->vio; 605 606 viodbg(DATA, "vnet_rx stype_env[%04x] seq[%016llx] rcv_nxt[%016llx]\n", 607 pkt->tag.stype_env, pkt->seq, dr->rcv_nxt); 608 609 if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 610 return 0; 611 if (unlikely(pkt->seq != dr->rcv_nxt)) { 612 pr_err("RX out of sequence seq[0x%llx] rcv_nxt[0x%llx]\n", 613 pkt->seq, dr->rcv_nxt); 614 return 0; 615 } 616 617 if (!port->napi_resume) 618 dr->rcv_nxt++; 619 620 /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ 621 622 return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, 623 npkts, budget); 624 } 625 626 static int idx_is_pending(struct vio_dring_state *dr, u32 end) 627 { 628 u32 idx = dr->cons; 629 int found = 0; 630 631 while (idx != dr->prod) { 632 if (idx == end) { 633 found = 1; 634 break; 635 } 636 idx = next_idx(idx, dr); 637 } 638 return found; 639 } 640 641 static int vnet_ack(struct vnet_port *port, void *msgbuf) 642 { 643 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 644 struct vio_dring_data *pkt = msgbuf; 645 struct net_device *dev; 646 struct vnet *vp; 647 u32 end; 648 struct vio_net_desc *desc; 649 struct netdev_queue *txq; 650 651 if (unlikely(pkt->tag.stype_env != VIO_DRING_DATA)) 652 return 0; 653 654 end = pkt->end_idx; 655 vp = port->vp; 656 dev = vp->dev; 657 netif_tx_lock(dev); 658 if (unlikely(!idx_is_pending(dr, end))) { 659 netif_tx_unlock(dev); 660 return 0; 661 } 662 663 /* sync for race conditions with vnet_start_xmit() and tell xmit it 664 * is time to send a trigger. 665 */ 666 dr->cons = next_idx(end, dr); 667 desc = vio_dring_entry(dr, dr->cons); 668 if (desc->hdr.state == VIO_DESC_READY && !port->start_cons) { 669 /* vnet_start_xmit() just populated this dring but missed 670 * sending the "start" LDC message to the consumer. 671 * Send a "start" trigger on its behalf. 672 */ 673 if (__vnet_tx_trigger(port, dr->cons) > 0) 674 port->start_cons = false; 675 else 676 port->start_cons = true; 677 } else { 678 port->start_cons = true; 679 } 680 netif_tx_unlock(dev); 681 682 txq = netdev_get_tx_queue(dev, port->q_index); 683 if (unlikely(netif_tx_queue_stopped(txq) && 684 vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) 685 return 1; 686 687 return 0; 688 } 689 690 static int vnet_nack(struct vnet_port *port, void *msgbuf) 691 { 692 /* XXX just reset or similar XXX */ 693 return 0; 694 } 695 696 static int handle_mcast(struct vnet_port *port, void *msgbuf) 697 { 698 struct vio_net_mcast_info *pkt = msgbuf; 699 700 if (pkt->tag.stype != VIO_SUBTYPE_ACK) 701 pr_err("%s: Got unexpected MCAST reply [%02x:%02x:%04x:%08x]\n", 702 port->vp->dev->name, 703 pkt->tag.type, 704 pkt->tag.stype, 705 pkt->tag.stype_env, 706 pkt->tag.sid); 707 708 return 0; 709 } 710 711 /* Got back a STOPPED LDC message on port. If the queue is stopped, 712 * wake it up so that we'll send out another START message at the 713 * next TX. 714 */ 715 static void maybe_tx_wakeup(struct vnet_port *port) 716 { 717 struct netdev_queue *txq; 718 719 txq = netdev_get_tx_queue(port->vp->dev, port->q_index); 720 __netif_tx_lock(txq, smp_processor_id()); 721 if (likely(netif_tx_queue_stopped(txq))) { 722 struct vio_dring_state *dr; 723 724 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 725 netif_tx_wake_queue(txq); 726 } 727 __netif_tx_unlock(txq); 728 } 729 730 static inline bool port_is_up(struct vnet_port *vnet) 731 { 732 struct vio_driver_state *vio = &vnet->vio; 733 734 return !!(vio->hs_state & VIO_HS_COMPLETE); 735 } 736 737 static int vnet_event_napi(struct vnet_port *port, int budget) 738 { 739 struct vio_driver_state *vio = &port->vio; 740 int tx_wakeup, err; 741 int npkts = 0; 742 int event = (port->rx_event & LDC_EVENT_RESET); 743 744 ldc_ctrl: 745 if (unlikely(event == LDC_EVENT_RESET || 746 event == LDC_EVENT_UP)) { 747 vio_link_state_change(vio, event); 748 749 if (event == LDC_EVENT_RESET) { 750 port->rmtu = 0; 751 port->tso = true; 752 port->tsolen = 0; 753 vio_port_up(vio); 754 } 755 port->rx_event = 0; 756 return 0; 757 } 758 /* We may have multiple LDC events in rx_event. Unroll send_events() */ 759 event = (port->rx_event & LDC_EVENT_UP); 760 port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); 761 if (event == LDC_EVENT_UP) 762 goto ldc_ctrl; 763 event = port->rx_event; 764 if (!(event & LDC_EVENT_DATA_READY)) 765 return 0; 766 767 /* we dont expect any other bits than RESET, UP, DATA_READY */ 768 BUG_ON(event != LDC_EVENT_DATA_READY); 769 770 tx_wakeup = err = 0; 771 while (1) { 772 union { 773 struct vio_msg_tag tag; 774 u64 raw[8]; 775 } msgbuf; 776 777 if (port->napi_resume) { 778 struct vio_dring_data *pkt = 779 (struct vio_dring_data *)&msgbuf; 780 struct vio_dring_state *dr = 781 &port->vio.drings[VIO_DRIVER_RX_RING]; 782 783 pkt->tag.type = VIO_TYPE_DATA; 784 pkt->tag.stype = VIO_SUBTYPE_INFO; 785 pkt->tag.stype_env = VIO_DRING_DATA; 786 pkt->seq = dr->rcv_nxt; 787 pkt->start_idx = next_idx(port->napi_stop_idx, dr); 788 pkt->end_idx = -1; 789 goto napi_resume; 790 } 791 err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); 792 if (unlikely(err < 0)) { 793 if (err == -ECONNRESET) 794 vio_conn_reset(vio); 795 break; 796 } 797 if (err == 0) 798 break; 799 viodbg(DATA, "TAG [%02x:%02x:%04x:%08x]\n", 800 msgbuf.tag.type, 801 msgbuf.tag.stype, 802 msgbuf.tag.stype_env, 803 msgbuf.tag.sid); 804 err = vio_validate_sid(vio, &msgbuf.tag); 805 if (err < 0) 806 break; 807 napi_resume: 808 if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { 809 if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { 810 if (!port_is_up(port)) { 811 /* failures like handshake_failure() 812 * may have cleaned up dring, but 813 * NAPI polling may bring us here. 814 */ 815 err = -ECONNRESET; 816 break; 817 } 818 err = vnet_rx(port, &msgbuf, &npkts, budget); 819 if (npkts >= budget) 820 break; 821 if (npkts == 0) 822 break; 823 } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { 824 err = vnet_ack(port, &msgbuf); 825 if (err > 0) 826 tx_wakeup |= err; 827 } else if (msgbuf.tag.stype == VIO_SUBTYPE_NACK) { 828 err = vnet_nack(port, &msgbuf); 829 } 830 } else if (msgbuf.tag.type == VIO_TYPE_CTRL) { 831 if (msgbuf.tag.stype_env == VNET_MCAST_INFO) 832 err = handle_mcast(port, &msgbuf); 833 else 834 err = vio_control_pkt_engine(vio, &msgbuf); 835 if (err) 836 break; 837 } else { 838 err = vnet_handle_unknown(port, &msgbuf); 839 } 840 if (err == -ECONNRESET) 841 break; 842 } 843 if (unlikely(tx_wakeup && err != -ECONNRESET)) 844 maybe_tx_wakeup(port); 845 return npkts; 846 } 847 848 static int vnet_poll(struct napi_struct *napi, int budget) 849 { 850 struct vnet_port *port = container_of(napi, struct vnet_port, napi); 851 struct vio_driver_state *vio = &port->vio; 852 int processed = vnet_event_napi(port, budget); 853 854 if (processed < budget) { 855 napi_complete(napi); 856 port->rx_event &= ~LDC_EVENT_DATA_READY; 857 vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); 858 } 859 return processed; 860 } 861 862 static void vnet_event(void *arg, int event) 863 { 864 struct vnet_port *port = arg; 865 struct vio_driver_state *vio = &port->vio; 866 867 port->rx_event |= event; 868 vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); 869 napi_schedule(&port->napi); 870 871 } 872 873 static int __vnet_tx_trigger(struct vnet_port *port, u32 start) 874 { 875 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 876 struct vio_dring_data hdr = { 877 .tag = { 878 .type = VIO_TYPE_DATA, 879 .stype = VIO_SUBTYPE_INFO, 880 .stype_env = VIO_DRING_DATA, 881 .sid = vio_send_sid(&port->vio), 882 }, 883 .dring_ident = dr->ident, 884 .start_idx = start, 885 .end_idx = (u32) -1, 886 }; 887 int err, delay; 888 int retries = 0; 889 890 if (port->stop_rx) { 891 err = vnet_send_ack(port, 892 &port->vio.drings[VIO_DRIVER_RX_RING], 893 port->stop_rx_idx, -1, 894 VIO_DRING_STOPPED); 895 if (err <= 0) 896 return err; 897 } 898 899 hdr.seq = dr->snd_nxt; 900 delay = 1; 901 do { 902 err = vio_ldc_send(&port->vio, &hdr, sizeof(hdr)); 903 if (err > 0) { 904 dr->snd_nxt++; 905 break; 906 } 907 udelay(delay); 908 if ((delay <<= 1) > 128) 909 delay = 128; 910 if (retries++ > VNET_MAX_RETRIES) 911 break; 912 } while (err == -EAGAIN); 913 914 return err; 915 } 916 917 struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) 918 { 919 unsigned int hash = vnet_hashfn(skb->data); 920 struct hlist_head *hp = &vp->port_hash[hash]; 921 struct vnet_port *port; 922 923 hlist_for_each_entry_rcu(port, hp, hash) { 924 if (!port_is_up(port)) 925 continue; 926 if (ether_addr_equal(port->raddr, skb->data)) 927 return port; 928 } 929 list_for_each_entry_rcu(port, &vp->port_list, list) { 930 if (!port->switch_port) 931 continue; 932 if (!port_is_up(port)) 933 continue; 934 return port; 935 } 936 return NULL; 937 } 938 939 static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, 940 unsigned *pending) 941 { 942 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 943 struct sk_buff *skb = NULL; 944 int i, txi; 945 946 *pending = 0; 947 948 txi = dr->prod-1; 949 if (txi < 0) 950 txi = VNET_TX_RING_SIZE-1; 951 952 for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 953 struct vio_net_desc *d; 954 955 d = vio_dring_entry(dr, txi); 956 957 if (d->hdr.state == VIO_DESC_DONE) { 958 if (port->tx_bufs[txi].skb) { 959 BUG_ON(port->tx_bufs[txi].skb->next); 960 961 port->tx_bufs[txi].skb->next = skb; 962 skb = port->tx_bufs[txi].skb; 963 port->tx_bufs[txi].skb = NULL; 964 965 ldc_unmap(port->vio.lp, 966 port->tx_bufs[txi].cookies, 967 port->tx_bufs[txi].ncookies); 968 } 969 d->hdr.state = VIO_DESC_FREE; 970 } else if (d->hdr.state == VIO_DESC_READY) { 971 (*pending)++; 972 } else if (d->hdr.state == VIO_DESC_FREE) { 973 break; 974 } 975 --txi; 976 if (txi < 0) 977 txi = VNET_TX_RING_SIZE-1; 978 } 979 return skb; 980 } 981 982 static inline void vnet_free_skbs(struct sk_buff *skb) 983 { 984 struct sk_buff *next; 985 986 while (skb) { 987 next = skb->next; 988 skb->next = NULL; 989 dev_kfree_skb(skb); 990 skb = next; 991 } 992 } 993 994 static void vnet_clean_timer_expire(unsigned long port0) 995 { 996 struct vnet_port *port = (struct vnet_port *)port0; 997 struct sk_buff *freeskbs; 998 unsigned pending; 999 1000 netif_tx_lock(port->vp->dev); 1001 freeskbs = vnet_clean_tx_ring(port, &pending); 1002 netif_tx_unlock(port->vp->dev); 1003 1004 vnet_free_skbs(freeskbs); 1005 1006 if (pending) 1007 (void)mod_timer(&port->clean_timer, 1008 jiffies + VNET_CLEAN_TIMEOUT); 1009 else 1010 del_timer(&port->clean_timer); 1011 } 1012 1013 static inline int vnet_skb_map(struct ldc_channel *lp, struct sk_buff *skb, 1014 struct ldc_trans_cookie *cookies, int ncookies, 1015 unsigned int map_perm) 1016 { 1017 int i, nc, err, blen; 1018 1019 /* header */ 1020 blen = skb_headlen(skb); 1021 if (blen < ETH_ZLEN) 1022 blen = ETH_ZLEN; 1023 blen += VNET_PACKET_SKIP; 1024 blen += 8 - (blen & 7); 1025 1026 err = ldc_map_single(lp, skb->data-VNET_PACKET_SKIP, blen, cookies, 1027 ncookies, map_perm); 1028 if (err < 0) 1029 return err; 1030 nc = err; 1031 1032 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1033 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1034 u8 *vaddr; 1035 1036 if (nc < ncookies) { 1037 vaddr = kmap_atomic(skb_frag_page(f)); 1038 blen = skb_frag_size(f); 1039 blen += 8 - (blen & 7); 1040 err = ldc_map_single(lp, vaddr + f->page_offset, 1041 blen, cookies + nc, ncookies - nc, 1042 map_perm); 1043 kunmap_atomic(vaddr); 1044 } else { 1045 err = -EMSGSIZE; 1046 } 1047 1048 if (err < 0) { 1049 ldc_unmap(lp, cookies, nc); 1050 return err; 1051 } 1052 nc += err; 1053 } 1054 return nc; 1055 } 1056 1057 static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, int ncookies) 1058 { 1059 struct sk_buff *nskb; 1060 int i, len, pad, docopy; 1061 1062 len = skb->len; 1063 pad = 0; 1064 if (len < ETH_ZLEN) { 1065 pad += ETH_ZLEN - skb->len; 1066 len += pad; 1067 } 1068 len += VNET_PACKET_SKIP; 1069 pad += 8 - (len & 7); 1070 1071 /* make sure we have enough cookies and alignment in every frag */ 1072 docopy = skb_shinfo(skb)->nr_frags >= ncookies; 1073 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1074 skb_frag_t *f = &skb_shinfo(skb)->frags[i]; 1075 1076 docopy |= f->page_offset & 7; 1077 } 1078 if (((unsigned long)skb->data & 7) != VNET_PACKET_SKIP || 1079 skb_tailroom(skb) < pad || 1080 skb_headroom(skb) < VNET_PACKET_SKIP || docopy) { 1081 int start = 0, offset; 1082 __wsum csum; 1083 1084 len = skb->len > ETH_ZLEN ? skb->len : ETH_ZLEN; 1085 nskb = alloc_and_align_skb(skb->dev, len); 1086 if (nskb == NULL) { 1087 dev_kfree_skb(skb); 1088 return NULL; 1089 } 1090 skb_reserve(nskb, VNET_PACKET_SKIP); 1091 1092 nskb->protocol = skb->protocol; 1093 offset = skb_mac_header(skb) - skb->data; 1094 skb_set_mac_header(nskb, offset); 1095 offset = skb_network_header(skb) - skb->data; 1096 skb_set_network_header(nskb, offset); 1097 offset = skb_transport_header(skb) - skb->data; 1098 skb_set_transport_header(nskb, offset); 1099 1100 offset = 0; 1101 nskb->csum_offset = skb->csum_offset; 1102 nskb->ip_summed = skb->ip_summed; 1103 1104 if (skb->ip_summed == CHECKSUM_PARTIAL) 1105 start = skb_checksum_start_offset(skb); 1106 if (start) { 1107 struct iphdr *iph = ip_hdr(nskb); 1108 int offset = start + nskb->csum_offset; 1109 1110 if (skb_copy_bits(skb, 0, nskb->data, start)) { 1111 dev_kfree_skb(nskb); 1112 dev_kfree_skb(skb); 1113 return NULL; 1114 } 1115 *(__sum16 *)(skb->data + offset) = 0; 1116 csum = skb_copy_and_csum_bits(skb, start, 1117 nskb->data + start, 1118 skb->len - start, 0); 1119 if (iph->protocol == IPPROTO_TCP || 1120 iph->protocol == IPPROTO_UDP) { 1121 csum = csum_tcpudp_magic(iph->saddr, iph->daddr, 1122 skb->len - start, 1123 iph->protocol, csum); 1124 } 1125 *(__sum16 *)(nskb->data + offset) = csum; 1126 1127 nskb->ip_summed = CHECKSUM_NONE; 1128 } else if (skb_copy_bits(skb, 0, nskb->data, skb->len)) { 1129 dev_kfree_skb(nskb); 1130 dev_kfree_skb(skb); 1131 return NULL; 1132 } 1133 (void)skb_put(nskb, skb->len); 1134 if (skb_is_gso(skb)) { 1135 skb_shinfo(nskb)->gso_size = skb_shinfo(skb)->gso_size; 1136 skb_shinfo(nskb)->gso_type = skb_shinfo(skb)->gso_type; 1137 } 1138 dev_kfree_skb(skb); 1139 skb = nskb; 1140 } 1141 return skb; 1142 } 1143 1144 static u16 1145 vnet_select_queue(struct net_device *dev, struct sk_buff *skb, 1146 void *accel_priv, select_queue_fallback_t fallback) 1147 { 1148 struct vnet *vp = netdev_priv(dev); 1149 struct vnet_port *port = __tx_port_find(vp, skb); 1150 1151 if (port == NULL) 1152 return 0; 1153 return port->q_index; 1154 } 1155 1156 static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev); 1157 1158 static int vnet_handle_offloads(struct vnet_port *port, struct sk_buff *skb) 1159 { 1160 struct net_device *dev = port->vp->dev; 1161 struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1162 struct sk_buff *segs; 1163 int maclen, datalen; 1164 int status; 1165 int gso_size, gso_type, gso_segs; 1166 int hlen = skb_transport_header(skb) - skb_mac_header(skb); 1167 int proto = IPPROTO_IP; 1168 1169 if (skb->protocol == htons(ETH_P_IP)) 1170 proto = ip_hdr(skb)->protocol; 1171 else if (skb->protocol == htons(ETH_P_IPV6)) 1172 proto = ipv6_hdr(skb)->nexthdr; 1173 1174 if (proto == IPPROTO_TCP) 1175 hlen += tcp_hdr(skb)->doff * 4; 1176 else if (proto == IPPROTO_UDP) 1177 hlen += sizeof(struct udphdr); 1178 else { 1179 pr_err("vnet_handle_offloads GSO with unknown transport " 1180 "protocol %d tproto %d\n", skb->protocol, proto); 1181 hlen = 128; /* XXX */ 1182 } 1183 datalen = port->tsolen - hlen; 1184 1185 gso_size = skb_shinfo(skb)->gso_size; 1186 gso_type = skb_shinfo(skb)->gso_type; 1187 gso_segs = skb_shinfo(skb)->gso_segs; 1188 1189 if (port->tso && gso_size < datalen) 1190 gso_segs = DIV_ROUND_UP(skb->len - hlen, datalen); 1191 1192 if (unlikely(vnet_tx_dring_avail(dr) < gso_segs)) { 1193 struct netdev_queue *txq; 1194 1195 txq = netdev_get_tx_queue(dev, port->q_index); 1196 netif_tx_stop_queue(txq); 1197 if (vnet_tx_dring_avail(dr) < skb_shinfo(skb)->gso_segs) 1198 return NETDEV_TX_BUSY; 1199 netif_tx_wake_queue(txq); 1200 } 1201 1202 maclen = skb_network_header(skb) - skb_mac_header(skb); 1203 skb_pull(skb, maclen); 1204 1205 if (port->tso && gso_size < datalen) { 1206 /* segment to TSO size */ 1207 skb_shinfo(skb)->gso_size = datalen; 1208 skb_shinfo(skb)->gso_segs = gso_segs; 1209 1210 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO); 1211 1212 /* restore gso_size & gso_segs */ 1213 skb_shinfo(skb)->gso_size = gso_size; 1214 skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len - hlen, 1215 gso_size); 1216 } else 1217 segs = skb_gso_segment(skb, dev->features & ~NETIF_F_TSO); 1218 if (IS_ERR(segs)) { 1219 dev->stats.tx_dropped++; 1220 return NETDEV_TX_OK; 1221 } 1222 1223 skb_push(skb, maclen); 1224 skb_reset_mac_header(skb); 1225 1226 status = 0; 1227 while (segs) { 1228 struct sk_buff *curr = segs; 1229 1230 segs = segs->next; 1231 curr->next = NULL; 1232 if (port->tso && curr->len > dev->mtu) { 1233 skb_shinfo(curr)->gso_size = gso_size; 1234 skb_shinfo(curr)->gso_type = gso_type; 1235 skb_shinfo(curr)->gso_segs = 1236 DIV_ROUND_UP(curr->len - hlen, gso_size); 1237 } else 1238 skb_shinfo(curr)->gso_size = 0; 1239 1240 skb_push(curr, maclen); 1241 skb_reset_mac_header(curr); 1242 memcpy(skb_mac_header(curr), skb_mac_header(skb), 1243 maclen); 1244 curr->csum_start = skb_transport_header(curr) - curr->head; 1245 if (ip_hdr(curr)->protocol == IPPROTO_TCP) 1246 curr->csum_offset = offsetof(struct tcphdr, check); 1247 else if (ip_hdr(curr)->protocol == IPPROTO_UDP) 1248 curr->csum_offset = offsetof(struct udphdr, check); 1249 1250 if (!(status & NETDEV_TX_MASK)) 1251 status = vnet_start_xmit(curr, dev); 1252 if (status & NETDEV_TX_MASK) 1253 dev_kfree_skb_any(curr); 1254 } 1255 1256 if (!(status & NETDEV_TX_MASK)) 1257 dev_kfree_skb_any(skb); 1258 return status; 1259 } 1260 1261 static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) 1262 { 1263 struct vnet *vp = netdev_priv(dev); 1264 struct vnet_port *port = NULL; 1265 struct vio_dring_state *dr; 1266 struct vio_net_desc *d; 1267 unsigned int len; 1268 struct sk_buff *freeskbs = NULL; 1269 int i, err, txi; 1270 unsigned pending = 0; 1271 struct netdev_queue *txq; 1272 1273 rcu_read_lock(); 1274 port = __tx_port_find(vp, skb); 1275 if (unlikely(!port)) { 1276 rcu_read_unlock(); 1277 goto out_dropped; 1278 } 1279 1280 if (skb_is_gso(skb) && skb->len > port->tsolen) { 1281 err = vnet_handle_offloads(port, skb); 1282 rcu_read_unlock(); 1283 return err; 1284 } 1285 1286 if (!skb_is_gso(skb) && skb->len > port->rmtu) { 1287 unsigned long localmtu = port->rmtu - ETH_HLEN; 1288 1289 if (vio_version_after_eq(&port->vio, 1, 3)) 1290 localmtu -= VLAN_HLEN; 1291 1292 if (skb->protocol == htons(ETH_P_IP)) { 1293 struct flowi4 fl4; 1294 struct rtable *rt = NULL; 1295 1296 memset(&fl4, 0, sizeof(fl4)); 1297 fl4.flowi4_oif = dev->ifindex; 1298 fl4.flowi4_tos = RT_TOS(ip_hdr(skb)->tos); 1299 fl4.daddr = ip_hdr(skb)->daddr; 1300 fl4.saddr = ip_hdr(skb)->saddr; 1301 1302 rt = ip_route_output_key(dev_net(dev), &fl4); 1303 rcu_read_unlock(); 1304 if (!IS_ERR(rt)) { 1305 skb_dst_set(skb, &rt->dst); 1306 icmp_send(skb, ICMP_DEST_UNREACH, 1307 ICMP_FRAG_NEEDED, 1308 htonl(localmtu)); 1309 } 1310 } 1311 #if IS_ENABLED(CONFIG_IPV6) 1312 else if (skb->protocol == htons(ETH_P_IPV6)) 1313 icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, localmtu); 1314 #endif 1315 goto out_dropped; 1316 } 1317 1318 skb = vnet_skb_shape(skb, 2); 1319 1320 if (unlikely(!skb)) 1321 goto out_dropped; 1322 1323 if (skb->ip_summed == CHECKSUM_PARTIAL) 1324 vnet_fullcsum(skb); 1325 1326 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1327 i = skb_get_queue_mapping(skb); 1328 txq = netdev_get_tx_queue(dev, i); 1329 if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1330 if (!netif_tx_queue_stopped(txq)) { 1331 netif_tx_stop_queue(txq); 1332 1333 /* This is a hard error, log it. */ 1334 netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); 1335 dev->stats.tx_errors++; 1336 } 1337 rcu_read_unlock(); 1338 return NETDEV_TX_BUSY; 1339 } 1340 1341 d = vio_dring_cur(dr); 1342 1343 txi = dr->prod; 1344 1345 freeskbs = vnet_clean_tx_ring(port, &pending); 1346 1347 BUG_ON(port->tx_bufs[txi].skb); 1348 1349 len = skb->len; 1350 if (len < ETH_ZLEN) 1351 len = ETH_ZLEN; 1352 1353 err = vnet_skb_map(port->vio.lp, skb, port->tx_bufs[txi].cookies, 2, 1354 (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); 1355 if (err < 0) { 1356 netdev_info(dev, "tx buffer map error %d\n", err); 1357 goto out_dropped; 1358 } 1359 1360 port->tx_bufs[txi].skb = skb; 1361 skb = NULL; 1362 port->tx_bufs[txi].ncookies = err; 1363 1364 /* We don't rely on the ACKs to free the skb in vnet_start_xmit(), 1365 * thus it is safe to not set VIO_ACK_ENABLE for each transmission: 1366 * the protocol itself does not require it as long as the peer 1367 * sends a VIO_SUBTYPE_ACK for VIO_DRING_STOPPED. 1368 * 1369 * An ACK for every packet in the ring is expensive as the 1370 * sending of LDC messages is slow and affects performance. 1371 */ 1372 d->hdr.ack = VIO_ACK_DISABLE; 1373 d->size = len; 1374 d->ncookies = port->tx_bufs[txi].ncookies; 1375 for (i = 0; i < d->ncookies; i++) 1376 d->cookies[i] = port->tx_bufs[txi].cookies[i]; 1377 if (vio_version_after_eq(&port->vio, 1, 7)) { 1378 struct vio_net_dext *dext = vio_net_ext(d); 1379 1380 memset(dext, 0, sizeof(*dext)); 1381 if (skb_is_gso(port->tx_bufs[txi].skb)) { 1382 dext->ipv4_lso_mss = skb_shinfo(port->tx_bufs[txi].skb) 1383 ->gso_size; 1384 dext->flags |= VNET_PKT_IPV4_LSO; 1385 } 1386 if (vio_version_after_eq(&port->vio, 1, 8) && 1387 !port->switch_port) { 1388 dext->flags |= VNET_PKT_HCK_IPV4_HDRCKSUM_OK; 1389 dext->flags |= VNET_PKT_HCK_FULLCKSUM_OK; 1390 } 1391 } 1392 1393 /* This has to be a non-SMP write barrier because we are writing 1394 * to memory which is shared with the peer LDOM. 1395 */ 1396 wmb(); 1397 1398 d->hdr.state = VIO_DESC_READY; 1399 1400 /* Exactly one ldc "start" trigger (for dr->cons) needs to be sent 1401 * to notify the consumer that some descriptors are READY. 1402 * After that "start" trigger, no additional triggers are needed until 1403 * a DRING_STOPPED is received from the consumer. The dr->cons field 1404 * (set up by vnet_ack()) has the value of the next dring index 1405 * that has not yet been ack-ed. We send a "start" trigger here 1406 * if, and only if, start_cons is true (reset it afterward). Conversely, 1407 * vnet_ack() should check if the dring corresponding to cons 1408 * is marked READY, but start_cons was false. 1409 * If so, vnet_ack() should send out the missed "start" trigger. 1410 * 1411 * Note that the wmb() above makes sure the cookies et al. are 1412 * not globally visible before the VIO_DESC_READY, and that the 1413 * stores are ordered correctly by the compiler. The consumer will 1414 * not proceed until the VIO_DESC_READY is visible assuring that 1415 * the consumer does not observe anything related to descriptors 1416 * out of order. The HV trap from the LDC start trigger is the 1417 * producer to consumer announcement that work is available to the 1418 * consumer 1419 */ 1420 if (!port->start_cons) 1421 goto ldc_start_done; /* previous trigger suffices */ 1422 1423 err = __vnet_tx_trigger(port, dr->cons); 1424 if (unlikely(err < 0)) { 1425 netdev_info(dev, "TX trigger error %d\n", err); 1426 d->hdr.state = VIO_DESC_FREE; 1427 dev->stats.tx_carrier_errors++; 1428 goto out_dropped; 1429 } 1430 1431 ldc_start_done: 1432 port->start_cons = false; 1433 1434 dev->stats.tx_packets++; 1435 dev->stats.tx_bytes += port->tx_bufs[txi].skb->len; 1436 1437 dr->prod = (dr->prod + 1) & (VNET_TX_RING_SIZE - 1); 1438 if (unlikely(vnet_tx_dring_avail(dr) < 1)) { 1439 netif_tx_stop_queue(txq); 1440 if (vnet_tx_dring_avail(dr) > VNET_TX_WAKEUP_THRESH(dr)) 1441 netif_tx_wake_queue(txq); 1442 } 1443 1444 (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); 1445 rcu_read_unlock(); 1446 1447 vnet_free_skbs(freeskbs); 1448 1449 return NETDEV_TX_OK; 1450 1451 out_dropped: 1452 if (pending) 1453 (void)mod_timer(&port->clean_timer, 1454 jiffies + VNET_CLEAN_TIMEOUT); 1455 else if (port) 1456 del_timer(&port->clean_timer); 1457 if (port) 1458 rcu_read_unlock(); 1459 if (skb) 1460 dev_kfree_skb(skb); 1461 vnet_free_skbs(freeskbs); 1462 dev->stats.tx_dropped++; 1463 return NETDEV_TX_OK; 1464 } 1465 1466 static void vnet_tx_timeout(struct net_device *dev) 1467 { 1468 /* XXX Implement me XXX */ 1469 } 1470 1471 static int vnet_open(struct net_device *dev) 1472 { 1473 netif_carrier_on(dev); 1474 netif_tx_start_all_queues(dev); 1475 1476 return 0; 1477 } 1478 1479 static int vnet_close(struct net_device *dev) 1480 { 1481 netif_tx_stop_all_queues(dev); 1482 netif_carrier_off(dev); 1483 1484 return 0; 1485 } 1486 1487 static struct vnet_mcast_entry *__vnet_mc_find(struct vnet *vp, u8 *addr) 1488 { 1489 struct vnet_mcast_entry *m; 1490 1491 for (m = vp->mcast_list; m; m = m->next) { 1492 if (ether_addr_equal(m->addr, addr)) 1493 return m; 1494 } 1495 return NULL; 1496 } 1497 1498 static void __update_mc_list(struct vnet *vp, struct net_device *dev) 1499 { 1500 struct netdev_hw_addr *ha; 1501 1502 netdev_for_each_mc_addr(ha, dev) { 1503 struct vnet_mcast_entry *m; 1504 1505 m = __vnet_mc_find(vp, ha->addr); 1506 if (m) { 1507 m->hit = 1; 1508 continue; 1509 } 1510 1511 if (!m) { 1512 m = kzalloc(sizeof(*m), GFP_ATOMIC); 1513 if (!m) 1514 continue; 1515 memcpy(m->addr, ha->addr, ETH_ALEN); 1516 m->hit = 1; 1517 1518 m->next = vp->mcast_list; 1519 vp->mcast_list = m; 1520 } 1521 } 1522 } 1523 1524 static void __send_mc_list(struct vnet *vp, struct vnet_port *port) 1525 { 1526 struct vio_net_mcast_info info; 1527 struct vnet_mcast_entry *m, **pp; 1528 int n_addrs; 1529 1530 memset(&info, 0, sizeof(info)); 1531 1532 info.tag.type = VIO_TYPE_CTRL; 1533 info.tag.stype = VIO_SUBTYPE_INFO; 1534 info.tag.stype_env = VNET_MCAST_INFO; 1535 info.tag.sid = vio_send_sid(&port->vio); 1536 info.set = 1; 1537 1538 n_addrs = 0; 1539 for (m = vp->mcast_list; m; m = m->next) { 1540 if (m->sent) 1541 continue; 1542 m->sent = 1; 1543 memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1544 m->addr, ETH_ALEN); 1545 if (++n_addrs == VNET_NUM_MCAST) { 1546 info.count = n_addrs; 1547 1548 (void) vio_ldc_send(&port->vio, &info, 1549 sizeof(info)); 1550 n_addrs = 0; 1551 } 1552 } 1553 if (n_addrs) { 1554 info.count = n_addrs; 1555 (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1556 } 1557 1558 info.set = 0; 1559 1560 n_addrs = 0; 1561 pp = &vp->mcast_list; 1562 while ((m = *pp) != NULL) { 1563 if (m->hit) { 1564 m->hit = 0; 1565 pp = &m->next; 1566 continue; 1567 } 1568 1569 memcpy(&info.mcast_addr[n_addrs * ETH_ALEN], 1570 m->addr, ETH_ALEN); 1571 if (++n_addrs == VNET_NUM_MCAST) { 1572 info.count = n_addrs; 1573 (void) vio_ldc_send(&port->vio, &info, 1574 sizeof(info)); 1575 n_addrs = 0; 1576 } 1577 1578 *pp = m->next; 1579 kfree(m); 1580 } 1581 if (n_addrs) { 1582 info.count = n_addrs; 1583 (void) vio_ldc_send(&port->vio, &info, sizeof(info)); 1584 } 1585 } 1586 1587 static void vnet_set_rx_mode(struct net_device *dev) 1588 { 1589 struct vnet *vp = netdev_priv(dev); 1590 struct vnet_port *port; 1591 1592 rcu_read_lock(); 1593 list_for_each_entry_rcu(port, &vp->port_list, list) { 1594 1595 if (port->switch_port) { 1596 __update_mc_list(vp, dev); 1597 __send_mc_list(vp, port); 1598 break; 1599 } 1600 } 1601 rcu_read_unlock(); 1602 } 1603 1604 static int vnet_change_mtu(struct net_device *dev, int new_mtu) 1605 { 1606 if (new_mtu < 68 || new_mtu > 65535) 1607 return -EINVAL; 1608 1609 dev->mtu = new_mtu; 1610 return 0; 1611 } 1612 1613 static int vnet_set_mac_addr(struct net_device *dev, void *p) 1614 { 1615 return -EINVAL; 1616 } 1617 1618 static void vnet_get_drvinfo(struct net_device *dev, 1619 struct ethtool_drvinfo *info) 1620 { 1621 strlcpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 1622 strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); 1623 } 1624 1625 static u32 vnet_get_msglevel(struct net_device *dev) 1626 { 1627 struct vnet *vp = netdev_priv(dev); 1628 return vp->msg_enable; 1629 } 1630 1631 static void vnet_set_msglevel(struct net_device *dev, u32 value) 1632 { 1633 struct vnet *vp = netdev_priv(dev); 1634 vp->msg_enable = value; 1635 } 1636 1637 static const struct ethtool_ops vnet_ethtool_ops = { 1638 .get_drvinfo = vnet_get_drvinfo, 1639 .get_msglevel = vnet_get_msglevel, 1640 .set_msglevel = vnet_set_msglevel, 1641 .get_link = ethtool_op_get_link, 1642 }; 1643 1644 static void vnet_port_free_tx_bufs(struct vnet_port *port) 1645 { 1646 struct vio_dring_state *dr; 1647 int i; 1648 1649 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1650 if (dr->base) { 1651 ldc_free_exp_dring(port->vio.lp, dr->base, 1652 (dr->entry_size * dr->num_entries), 1653 dr->cookies, dr->ncookies); 1654 dr->base = NULL; 1655 dr->entry_size = 0; 1656 dr->num_entries = 0; 1657 dr->pending = 0; 1658 dr->ncookies = 0; 1659 } 1660 1661 for (i = 0; i < VNET_TX_RING_SIZE; i++) { 1662 struct vio_net_desc *d; 1663 void *skb = port->tx_bufs[i].skb; 1664 1665 if (!skb) 1666 continue; 1667 1668 d = vio_dring_entry(dr, i); 1669 if (d->hdr.state == VIO_DESC_READY) 1670 pr_warn("active transmit buffers freed\n"); 1671 1672 ldc_unmap(port->vio.lp, 1673 port->tx_bufs[i].cookies, 1674 port->tx_bufs[i].ncookies); 1675 dev_kfree_skb(skb); 1676 port->tx_bufs[i].skb = NULL; 1677 d->hdr.state = VIO_DESC_FREE; 1678 } 1679 } 1680 1681 static int vnet_port_alloc_tx_ring(struct vnet_port *port) 1682 { 1683 struct vio_dring_state *dr; 1684 unsigned long len, elen; 1685 int i, err, ncookies; 1686 void *dring; 1687 1688 dr = &port->vio.drings[VIO_DRIVER_TX_RING]; 1689 1690 elen = sizeof(struct vio_net_desc) + 1691 sizeof(struct ldc_trans_cookie) * 2; 1692 if (vio_version_after_eq(&port->vio, 1, 7)) 1693 elen += sizeof(struct vio_net_dext); 1694 len = VNET_TX_RING_SIZE * elen; 1695 1696 ncookies = VIO_MAX_RING_COOKIES; 1697 dring = ldc_alloc_exp_dring(port->vio.lp, len, 1698 dr->cookies, &ncookies, 1699 (LDC_MAP_SHADOW | 1700 LDC_MAP_DIRECT | 1701 LDC_MAP_RW)); 1702 if (IS_ERR(dring)) { 1703 err = PTR_ERR(dring); 1704 goto err_out; 1705 } 1706 1707 dr->base = dring; 1708 dr->entry_size = elen; 1709 dr->num_entries = VNET_TX_RING_SIZE; 1710 dr->prod = dr->cons = 0; 1711 port->start_cons = true; /* need an initial trigger */ 1712 dr->pending = VNET_TX_RING_SIZE; 1713 dr->ncookies = ncookies; 1714 1715 for (i = 0; i < VNET_TX_RING_SIZE; ++i) { 1716 struct vio_net_desc *d; 1717 1718 d = vio_dring_entry(dr, i); 1719 d->hdr.state = VIO_DESC_FREE; 1720 } 1721 return 0; 1722 1723 err_out: 1724 vnet_port_free_tx_bufs(port); 1725 1726 return err; 1727 } 1728 1729 #ifdef CONFIG_NET_POLL_CONTROLLER 1730 static void vnet_poll_controller(struct net_device *dev) 1731 { 1732 struct vnet *vp = netdev_priv(dev); 1733 struct vnet_port *port; 1734 unsigned long flags; 1735 1736 spin_lock_irqsave(&vp->lock, flags); 1737 if (!list_empty(&vp->port_list)) { 1738 port = list_entry(vp->port_list.next, struct vnet_port, list); 1739 napi_schedule(&port->napi); 1740 } 1741 spin_unlock_irqrestore(&vp->lock, flags); 1742 } 1743 #endif 1744 static LIST_HEAD(vnet_list); 1745 static DEFINE_MUTEX(vnet_list_mutex); 1746 1747 static const struct net_device_ops vnet_ops = { 1748 .ndo_open = vnet_open, 1749 .ndo_stop = vnet_close, 1750 .ndo_set_rx_mode = vnet_set_rx_mode, 1751 .ndo_set_mac_address = vnet_set_mac_addr, 1752 .ndo_validate_addr = eth_validate_addr, 1753 .ndo_tx_timeout = vnet_tx_timeout, 1754 .ndo_change_mtu = vnet_change_mtu, 1755 .ndo_start_xmit = vnet_start_xmit, 1756 .ndo_select_queue = vnet_select_queue, 1757 #ifdef CONFIG_NET_POLL_CONTROLLER 1758 .ndo_poll_controller = vnet_poll_controller, 1759 #endif 1760 }; 1761 1762 static struct vnet *vnet_new(const u64 *local_mac) 1763 { 1764 struct net_device *dev; 1765 struct vnet *vp; 1766 int err, i; 1767 1768 dev = alloc_etherdev_mqs(sizeof(*vp), VNET_MAX_TXQS, 1); 1769 if (!dev) 1770 return ERR_PTR(-ENOMEM); 1771 dev->needed_headroom = VNET_PACKET_SKIP + 8; 1772 dev->needed_tailroom = 8; 1773 1774 for (i = 0; i < ETH_ALEN; i++) 1775 dev->dev_addr[i] = (*local_mac >> (5 - i) * 8) & 0xff; 1776 1777 vp = netdev_priv(dev); 1778 1779 spin_lock_init(&vp->lock); 1780 vp->dev = dev; 1781 1782 INIT_LIST_HEAD(&vp->port_list); 1783 for (i = 0; i < VNET_PORT_HASH_SIZE; i++) 1784 INIT_HLIST_HEAD(&vp->port_hash[i]); 1785 INIT_LIST_HEAD(&vp->list); 1786 vp->local_mac = *local_mac; 1787 1788 dev->netdev_ops = &vnet_ops; 1789 dev->ethtool_ops = &vnet_ethtool_ops; 1790 dev->watchdog_timeo = VNET_TX_TIMEOUT; 1791 1792 dev->hw_features = NETIF_F_TSO | NETIF_F_GSO | NETIF_F_GSO_SOFTWARE | 1793 NETIF_F_HW_CSUM | NETIF_F_SG; 1794 dev->features = dev->hw_features; 1795 1796 err = register_netdev(dev); 1797 if (err) { 1798 pr_err("Cannot register net device, aborting\n"); 1799 goto err_out_free_dev; 1800 } 1801 1802 netdev_info(dev, "Sun LDOM vnet %pM\n", dev->dev_addr); 1803 1804 list_add(&vp->list, &vnet_list); 1805 1806 return vp; 1807 1808 err_out_free_dev: 1809 free_netdev(dev); 1810 1811 return ERR_PTR(err); 1812 } 1813 1814 static struct vnet *vnet_find_or_create(const u64 *local_mac) 1815 { 1816 struct vnet *iter, *vp; 1817 1818 mutex_lock(&vnet_list_mutex); 1819 vp = NULL; 1820 list_for_each_entry(iter, &vnet_list, list) { 1821 if (iter->local_mac == *local_mac) { 1822 vp = iter; 1823 break; 1824 } 1825 } 1826 if (!vp) 1827 vp = vnet_new(local_mac); 1828 mutex_unlock(&vnet_list_mutex); 1829 1830 return vp; 1831 } 1832 1833 static void vnet_cleanup(void) 1834 { 1835 struct vnet *vp; 1836 struct net_device *dev; 1837 1838 mutex_lock(&vnet_list_mutex); 1839 while (!list_empty(&vnet_list)) { 1840 vp = list_first_entry(&vnet_list, struct vnet, list); 1841 list_del(&vp->list); 1842 dev = vp->dev; 1843 /* vio_unregister_driver() should have cleaned up port_list */ 1844 BUG_ON(!list_empty(&vp->port_list)); 1845 unregister_netdev(dev); 1846 free_netdev(dev); 1847 } 1848 mutex_unlock(&vnet_list_mutex); 1849 } 1850 1851 static const char *local_mac_prop = "local-mac-address"; 1852 1853 static struct vnet *vnet_find_parent(struct mdesc_handle *hp, 1854 u64 port_node) 1855 { 1856 const u64 *local_mac = NULL; 1857 u64 a; 1858 1859 mdesc_for_each_arc(a, hp, port_node, MDESC_ARC_TYPE_BACK) { 1860 u64 target = mdesc_arc_target(hp, a); 1861 const char *name; 1862 1863 name = mdesc_get_property(hp, target, "name", NULL); 1864 if (!name || strcmp(name, "network")) 1865 continue; 1866 1867 local_mac = mdesc_get_property(hp, target, 1868 local_mac_prop, NULL); 1869 if (local_mac) 1870 break; 1871 } 1872 if (!local_mac) 1873 return ERR_PTR(-ENODEV); 1874 1875 return vnet_find_or_create(local_mac); 1876 } 1877 1878 static struct ldc_channel_config vnet_ldc_cfg = { 1879 .event = vnet_event, 1880 .mtu = 64, 1881 .mode = LDC_MODE_UNRELIABLE, 1882 }; 1883 1884 static struct vio_driver_ops vnet_vio_ops = { 1885 .send_attr = vnet_send_attr, 1886 .handle_attr = vnet_handle_attr, 1887 .handshake_complete = vnet_handshake_complete, 1888 }; 1889 1890 static void print_version(void) 1891 { 1892 printk_once(KERN_INFO "%s", version); 1893 } 1894 1895 const char *remote_macaddr_prop = "remote-mac-address"; 1896 1897 static void 1898 vnet_port_add_txq(struct vnet_port *port) 1899 { 1900 struct vnet *vp = port->vp; 1901 int n; 1902 1903 n = vp->nports++; 1904 n = n & (VNET_MAX_TXQS - 1); 1905 port->q_index = n; 1906 netif_tx_wake_queue(netdev_get_tx_queue(vp->dev, port->q_index)); 1907 } 1908 1909 static void 1910 vnet_port_rm_txq(struct vnet_port *port) 1911 { 1912 port->vp->nports--; 1913 netif_tx_stop_queue(netdev_get_tx_queue(port->vp->dev, port->q_index)); 1914 } 1915 1916 static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) 1917 { 1918 struct mdesc_handle *hp; 1919 struct vnet_port *port; 1920 unsigned long flags; 1921 struct vnet *vp; 1922 const u64 *rmac; 1923 int len, i, err, switch_port; 1924 1925 print_version(); 1926 1927 hp = mdesc_grab(); 1928 1929 vp = vnet_find_parent(hp, vdev->mp); 1930 if (IS_ERR(vp)) { 1931 pr_err("Cannot find port parent vnet\n"); 1932 err = PTR_ERR(vp); 1933 goto err_out_put_mdesc; 1934 } 1935 1936 rmac = mdesc_get_property(hp, vdev->mp, remote_macaddr_prop, &len); 1937 err = -ENODEV; 1938 if (!rmac) { 1939 pr_err("Port lacks %s property\n", remote_macaddr_prop); 1940 goto err_out_put_mdesc; 1941 } 1942 1943 port = kzalloc(sizeof(*port), GFP_KERNEL); 1944 err = -ENOMEM; 1945 if (!port) 1946 goto err_out_put_mdesc; 1947 1948 for (i = 0; i < ETH_ALEN; i++) 1949 port->raddr[i] = (*rmac >> (5 - i) * 8) & 0xff; 1950 1951 port->vp = vp; 1952 1953 err = vio_driver_init(&port->vio, vdev, VDEV_NETWORK, 1954 vnet_versions, ARRAY_SIZE(vnet_versions), 1955 &vnet_vio_ops, vp->dev->name); 1956 if (err) 1957 goto err_out_free_port; 1958 1959 err = vio_ldc_alloc(&port->vio, &vnet_ldc_cfg, port); 1960 if (err) 1961 goto err_out_free_port; 1962 1963 netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); 1964 1965 INIT_HLIST_NODE(&port->hash); 1966 INIT_LIST_HEAD(&port->list); 1967 1968 switch_port = 0; 1969 if (mdesc_get_property(hp, vdev->mp, "switch-port", NULL) != NULL) 1970 switch_port = 1; 1971 port->switch_port = switch_port; 1972 port->tso = true; 1973 port->tsolen = 0; 1974 1975 spin_lock_irqsave(&vp->lock, flags); 1976 if (switch_port) 1977 list_add_rcu(&port->list, &vp->port_list); 1978 else 1979 list_add_tail_rcu(&port->list, &vp->port_list); 1980 hlist_add_head_rcu(&port->hash, 1981 &vp->port_hash[vnet_hashfn(port->raddr)]); 1982 vnet_port_add_txq(port); 1983 spin_unlock_irqrestore(&vp->lock, flags); 1984 1985 dev_set_drvdata(&vdev->dev, port); 1986 1987 pr_info("%s: PORT ( remote-mac %pM%s )\n", 1988 vp->dev->name, port->raddr, switch_port ? " switch-port" : ""); 1989 1990 setup_timer(&port->clean_timer, vnet_clean_timer_expire, 1991 (unsigned long)port); 1992 1993 napi_enable(&port->napi); 1994 vio_port_up(&port->vio); 1995 1996 mdesc_release(hp); 1997 1998 return 0; 1999 2000 err_out_free_port: 2001 kfree(port); 2002 2003 err_out_put_mdesc: 2004 mdesc_release(hp); 2005 return err; 2006 } 2007 2008 static int vnet_port_remove(struct vio_dev *vdev) 2009 { 2010 struct vnet_port *port = dev_get_drvdata(&vdev->dev); 2011 2012 if (port) { 2013 2014 del_timer_sync(&port->vio.timer); 2015 2016 napi_disable(&port->napi); 2017 2018 list_del_rcu(&port->list); 2019 hlist_del_rcu(&port->hash); 2020 2021 synchronize_rcu(); 2022 del_timer_sync(&port->clean_timer); 2023 vnet_port_rm_txq(port); 2024 netif_napi_del(&port->napi); 2025 vnet_port_free_tx_bufs(port); 2026 vio_ldc_free(&port->vio); 2027 2028 dev_set_drvdata(&vdev->dev, NULL); 2029 2030 kfree(port); 2031 2032 } 2033 return 0; 2034 } 2035 2036 static const struct vio_device_id vnet_port_match[] = { 2037 { 2038 .type = "vnet-port", 2039 }, 2040 {}, 2041 }; 2042 MODULE_DEVICE_TABLE(vio, vnet_port_match); 2043 2044 static struct vio_driver vnet_port_driver = { 2045 .id_table = vnet_port_match, 2046 .probe = vnet_port_probe, 2047 .remove = vnet_port_remove, 2048 .name = "vnet_port", 2049 }; 2050 2051 static int __init vnet_init(void) 2052 { 2053 return vio_register_driver(&vnet_port_driver); 2054 } 2055 2056 static void __exit vnet_exit(void) 2057 { 2058 vio_unregister_driver(&vnet_port_driver); 2059 vnet_cleanup(); 2060 } 2061 2062 module_init(vnet_init); 2063 module_exit(vnet_exit); 2064