1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * IBM Power Virtual Ethernet Device Driver 4 * 5 * Copyright (C) IBM Corporation, 2003, 2010 6 * 7 * Authors: Dave Larson <larson1@us.ibm.com> 8 * Santiago Leon <santil@linux.vnet.ibm.com> 9 * Brian King <brking@linux.vnet.ibm.com> 10 * Robert Jennings <rcj@linux.vnet.ibm.com> 11 * Anton Blanchard <anton@au.ibm.com> 12 */ 13 14 #include <linux/module.h> 15 #include <linux/types.h> 16 #include <linux/errno.h> 17 #include <linux/dma-mapping.h> 18 #include <linux/kernel.h> 19 #include <linux/netdevice.h> 20 #include <linux/etherdevice.h> 21 #include <linux/skbuff.h> 22 #include <linux/init.h> 23 #include <linux/interrupt.h> 24 #include <linux/mm.h> 25 #include <linux/pm.h> 26 #include <linux/ethtool.h> 27 #include <linux/in.h> 28 #include <linux/ip.h> 29 #include <linux/ipv6.h> 30 #include <linux/slab.h> 31 #include <asm/hvcall.h> 32 #include <linux/atomic.h> 33 #include <asm/vio.h> 34 #include <asm/iommu.h> 35 #include <asm/firmware.h> 36 #include <net/tcp.h> 37 #include <net/ip6_checksum.h> 38 39 #include "ibmveth.h" 40 41 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance); 42 static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev); 43 44 static struct kobj_type ktype_veth_pool; 45 46 47 static const char ibmveth_driver_name[] = "ibmveth"; 48 static const char ibmveth_driver_string[] = "IBM Power Virtual Ethernet Driver"; 49 #define ibmveth_driver_version "1.06" 50 51 MODULE_AUTHOR("Santiago Leon <santil@linux.vnet.ibm.com>"); 52 MODULE_DESCRIPTION("IBM Power Virtual Ethernet Driver"); 53 MODULE_LICENSE("GPL"); 54 MODULE_VERSION(ibmveth_driver_version); 55 56 static unsigned int tx_copybreak __read_mostly = 128; 57 module_param(tx_copybreak, uint, 0644); 58 MODULE_PARM_DESC(tx_copybreak, 59 "Maximum size of packet that is copied to a new buffer on transmit"); 60 61 static unsigned int rx_copybreak __read_mostly = 128; 62 module_param(rx_copybreak, uint, 0644); 63 MODULE_PARM_DESC(rx_copybreak, 64 "Maximum size of packet that is copied to a new buffer on receive"); 65 66 static unsigned int rx_flush __read_mostly = 0; 67 module_param(rx_flush, uint, 0644); 68 MODULE_PARM_DESC(rx_flush, "Flush receive buffers before use"); 69 70 static bool old_large_send __read_mostly; 71 module_param(old_large_send, bool, 0444); 72 MODULE_PARM_DESC(old_large_send, 73 "Use old large send method on firmware that supports the new method"); 74 75 struct ibmveth_stat { 76 char name[ETH_GSTRING_LEN]; 77 int offset; 78 }; 79 80 #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat) 81 #define IBMVETH_GET_STAT(a, off) *((u64 *)(((unsigned long)(a)) + off)) 82 83 static struct ibmveth_stat ibmveth_stats[] = { 84 { "replenish_task_cycles", IBMVETH_STAT_OFF(replenish_task_cycles) }, 85 { "replenish_no_mem", IBMVETH_STAT_OFF(replenish_no_mem) }, 86 { "replenish_add_buff_failure", 87 IBMVETH_STAT_OFF(replenish_add_buff_failure) }, 88 { "replenish_add_buff_success", 89 IBMVETH_STAT_OFF(replenish_add_buff_success) }, 90 { "rx_invalid_buffer", IBMVETH_STAT_OFF(rx_invalid_buffer) }, 91 { "rx_no_buffer", IBMVETH_STAT_OFF(rx_no_buffer) }, 92 { "tx_map_failed", IBMVETH_STAT_OFF(tx_map_failed) }, 93 { "tx_send_failed", IBMVETH_STAT_OFF(tx_send_failed) }, 94 { "fw_enabled_ipv4_csum", IBMVETH_STAT_OFF(fw_ipv4_csum_support) }, 95 { "fw_enabled_ipv6_csum", IBMVETH_STAT_OFF(fw_ipv6_csum_support) }, 96 { "tx_large_packets", IBMVETH_STAT_OFF(tx_large_packets) }, 97 { "rx_large_packets", IBMVETH_STAT_OFF(rx_large_packets) }, 98 { "fw_enabled_large_send", IBMVETH_STAT_OFF(fw_large_send_support) } 99 }; 100 101 /* simple methods of getting data from the current rxq entry */ 102 static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter) 103 { 104 return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].flags_off); 105 } 106 107 static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter) 108 { 109 return (ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_TOGGLE) >> 110 IBMVETH_RXQ_TOGGLE_SHIFT; 111 } 112 113 static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter) 114 { 115 return ibmveth_rxq_toggle(adapter) == adapter->rx_queue.toggle; 116 } 117 118 static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter) 119 { 120 return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_VALID; 121 } 122 123 static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter) 124 { 125 return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_OFF_MASK; 126 } 127 128 static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter) 129 { 130 return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_LRG_PKT; 131 } 132 133 static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter) 134 { 135 return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].length); 136 } 137 138 static u64 ibmveth_rxq_correlator(struct ibmveth_adapter *adapter) 139 { 140 unsigned int idx = adapter->rx_queue.index; 141 142 return READ_ONCE(adapter->rx_queue.queue_addr[idx].correlator); 143 } 144 145 static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter) 146 { 147 return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD; 148 } 149 150 static unsigned int ibmveth_real_max_tx_queues(void) 151 { 152 unsigned int n_cpu = num_online_cpus(); 153 154 return min(n_cpu, IBMVETH_MAX_QUEUES); 155 } 156 157 /* setup the initial settings for a buffer pool */ 158 static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool, 159 u32 pool_index, u32 pool_size, 160 u32 buff_size, u32 pool_active) 161 { 162 pool->size = pool_size; 163 pool->index = pool_index; 164 pool->buff_size = buff_size; 165 pool->threshold = pool_size * 7 / 8; 166 pool->active = pool_active; 167 } 168 169 /* allocate and setup an buffer pool - called during open */ 170 static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool) 171 { 172 int i; 173 174 pool->free_map = kmalloc_array(pool->size, sizeof(u16), GFP_KERNEL); 175 176 if (!pool->free_map) 177 return -1; 178 179 pool->dma_addr = kzalloc_objs(dma_addr_t, pool->size); 180 if (!pool->dma_addr) { 181 kfree(pool->free_map); 182 pool->free_map = NULL; 183 return -1; 184 } 185 186 pool->skbuff = kcalloc(pool->size, sizeof(void *), GFP_KERNEL); 187 188 if (!pool->skbuff) { 189 kfree(pool->dma_addr); 190 pool->dma_addr = NULL; 191 192 kfree(pool->free_map); 193 pool->free_map = NULL; 194 return -1; 195 } 196 197 for (i = 0; i < pool->size; ++i) 198 pool->free_map[i] = i; 199 200 atomic_set(&pool->available, 0); 201 pool->producer_index = 0; 202 pool->consumer_index = 0; 203 204 return 0; 205 } 206 207 static inline void ibmveth_flush_buffer(void *addr, unsigned long length) 208 { 209 unsigned long offset; 210 211 for (offset = 0; offset < length; offset += SMP_CACHE_BYTES) 212 asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset)); 213 } 214 215 /* replenish the buffers for a pool. note that we don't need to 216 * skb_reserve these since they are used for incoming... 217 */ 218 static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter, 219 struct ibmveth_buff_pool *pool) 220 { 221 union ibmveth_buf_desc descs[IBMVETH_MAX_RX_PER_HCALL] = {0}; 222 u32 remaining = pool->size - atomic_read(&pool->available); 223 u64 correlators[IBMVETH_MAX_RX_PER_HCALL] = {0}; 224 unsigned long lpar_rc; 225 u32 buffers_added = 0; 226 u32 i, filled, batch; 227 struct vio_dev *vdev; 228 dma_addr_t dma_addr; 229 struct device *dev; 230 u32 index; 231 232 vdev = adapter->vdev; 233 dev = &vdev->dev; 234 235 mb(); 236 237 batch = adapter->rx_buffers_per_hcall; 238 239 while (remaining > 0) { 240 unsigned int free_index = pool->consumer_index; 241 242 /* Fill a batch of descriptors */ 243 for (filled = 0; filled < min(remaining, batch); filled++) { 244 index = pool->free_map[free_index]; 245 if (WARN_ON(index == IBM_VETH_INVALID_MAP)) { 246 adapter->replenish_add_buff_failure++; 247 netdev_info(adapter->netdev, 248 "Invalid map index %u, reset\n", 249 index); 250 schedule_work(&adapter->work); 251 break; 252 } 253 254 if (!pool->skbuff[index]) { 255 struct sk_buff *skb = NULL; 256 257 skb = netdev_alloc_skb(adapter->netdev, 258 pool->buff_size); 259 if (!skb) { 260 adapter->replenish_no_mem++; 261 adapter->replenish_add_buff_failure++; 262 break; 263 } 264 265 dma_addr = dma_map_single(dev, skb->data, 266 pool->buff_size, 267 DMA_FROM_DEVICE); 268 if (dma_mapping_error(dev, dma_addr)) { 269 dev_kfree_skb_any(skb); 270 adapter->replenish_add_buff_failure++; 271 break; 272 } 273 274 pool->dma_addr[index] = dma_addr; 275 pool->skbuff[index] = skb; 276 } else { 277 /* re-use case */ 278 dma_addr = pool->dma_addr[index]; 279 } 280 281 if (rx_flush) { 282 unsigned int len; 283 284 len = adapter->netdev->mtu + IBMVETH_BUFF_OH; 285 len = min(pool->buff_size, len); 286 ibmveth_flush_buffer(pool->skbuff[index]->data, 287 len); 288 } 289 290 descs[filled].fields.flags_len = IBMVETH_BUF_VALID | 291 pool->buff_size; 292 descs[filled].fields.address = dma_addr; 293 294 correlators[filled] = ((u64)pool->index << 32) | index; 295 *(u64 *)pool->skbuff[index]->data = correlators[filled]; 296 297 free_index++; 298 if (free_index >= pool->size) 299 free_index = 0; 300 } 301 302 if (!filled) 303 break; 304 305 /* single buffer case*/ 306 if (filled == 1) 307 lpar_rc = h_add_logical_lan_buffer(vdev->unit_address, 308 descs[0].desc); 309 else 310 /* Multi-buffer hcall */ 311 lpar_rc = h_add_logical_lan_buffers(vdev->unit_address, 312 descs[0].desc, 313 descs[1].desc, 314 descs[2].desc, 315 descs[3].desc, 316 descs[4].desc, 317 descs[5].desc, 318 descs[6].desc, 319 descs[7].desc); 320 if (lpar_rc != H_SUCCESS) { 321 dev_warn_ratelimited(dev, 322 "RX h_add_logical_lan failed: filled=%u, rc=%lu, batch=%u\n", 323 filled, lpar_rc, batch); 324 goto hcall_failure; 325 } 326 327 /* Only update pool state after hcall succeeds */ 328 for (i = 0; i < filled; i++) { 329 free_index = pool->consumer_index; 330 pool->free_map[free_index] = IBM_VETH_INVALID_MAP; 331 332 pool->consumer_index++; 333 if (pool->consumer_index >= pool->size) 334 pool->consumer_index = 0; 335 } 336 337 buffers_added += filled; 338 adapter->replenish_add_buff_success += filled; 339 remaining -= filled; 340 341 memset(&descs, 0, sizeof(descs)); 342 memset(&correlators, 0, sizeof(correlators)); 343 continue; 344 345 hcall_failure: 346 for (i = 0; i < filled; i++) { 347 index = correlators[i] & 0xffffffffUL; 348 dma_addr = pool->dma_addr[index]; 349 350 if (pool->skbuff[index]) { 351 if (dma_addr && 352 !dma_mapping_error(dev, dma_addr)) 353 dma_unmap_single(dev, dma_addr, 354 pool->buff_size, 355 DMA_FROM_DEVICE); 356 357 dev_kfree_skb_any(pool->skbuff[index]); 358 pool->skbuff[index] = NULL; 359 } 360 } 361 adapter->replenish_add_buff_failure += filled; 362 363 /* 364 * If multi rx buffers hcall is no longer supported by FW 365 * e.g. in the case of Live Partition Migration 366 */ 367 if (batch > 1 && lpar_rc == H_FUNCTION) { 368 /* 369 * Instead of retry submit single buffer individually 370 * here just set the max rx buffer per hcall to 1 371 * buffers will be respleshed next time 372 * when ibmveth_replenish_buffer_pool() is called again 373 * with single-buffer case 374 */ 375 netdev_info(adapter->netdev, 376 "RX Multi buffers not supported by FW, rc=%lu\n", 377 lpar_rc); 378 adapter->rx_buffers_per_hcall = 1; 379 netdev_info(adapter->netdev, 380 "Next rx replesh will fall back to single-buffer hcall\n"); 381 } 382 break; 383 } 384 385 mb(); 386 atomic_add(buffers_added, &(pool->available)); 387 } 388 389 /* 390 * The final 8 bytes of the buffer list is a counter of frames dropped 391 * because there was not a buffer in the buffer list capable of holding 392 * the frame. 393 */ 394 static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter) 395 { 396 __be64 *p = adapter->buffer_list_addr + 4096 - 8; 397 398 adapter->rx_no_buffer = be64_to_cpup(p); 399 } 400 401 /* replenish routine */ 402 static void ibmveth_replenish_task(struct ibmveth_adapter *adapter) 403 { 404 int i; 405 406 adapter->replenish_task_cycles++; 407 408 for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) { 409 struct ibmveth_buff_pool *pool = &adapter->rx_buff_pool[i]; 410 411 if (pool->active && 412 (atomic_read(&pool->available) < pool->threshold)) 413 ibmveth_replenish_buffer_pool(adapter, pool); 414 } 415 416 ibmveth_update_rx_no_buffer(adapter); 417 } 418 419 /* empty and free ana buffer pool - also used to do cleanup in error paths */ 420 static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter, 421 struct ibmveth_buff_pool *pool) 422 { 423 int i; 424 425 kfree(pool->free_map); 426 pool->free_map = NULL; 427 428 if (pool->skbuff && pool->dma_addr) { 429 for (i = 0; i < pool->size; ++i) { 430 struct sk_buff *skb = pool->skbuff[i]; 431 if (skb) { 432 dma_unmap_single(&adapter->vdev->dev, 433 pool->dma_addr[i], 434 pool->buff_size, 435 DMA_FROM_DEVICE); 436 dev_kfree_skb_any(skb); 437 pool->skbuff[i] = NULL; 438 } 439 } 440 } 441 442 if (pool->dma_addr) { 443 kfree(pool->dma_addr); 444 pool->dma_addr = NULL; 445 } 446 447 if (pool->skbuff) { 448 kfree(pool->skbuff); 449 pool->skbuff = NULL; 450 } 451 } 452 453 /* The correlator comes back from PHYP; a bad one schedules a reset. */ 454 static bool ibmveth_rxq_correlator_valid(struct ibmveth_adapter *adapter, 455 u64 correlator) 456 { 457 unsigned int index = correlator & 0xffffffffUL; 458 unsigned int pool = correlator >> 32; 459 460 /* An inactive pool keeps its size but has no skbuff array. */ 461 if (pool < IBMVETH_NUM_BUFF_POOLS && 462 index < adapter->rx_buff_pool[pool].size && 463 adapter->rx_buff_pool[pool].skbuff) 464 return true; 465 466 if (net_ratelimit()) 467 netdev_err(adapter->netdev, 468 "invalid RX correlator %llx, resetting\n", 469 correlator); 470 schedule_work(&adapter->work); 471 return false; 472 } 473 474 static void ibmveth_rxq_no_skb(struct ibmveth_adapter *adapter, 475 u64 correlator) 476 { 477 if (net_ratelimit()) 478 netdev_err(adapter->netdev, 479 "no buffer for RX correlator %llx, resetting\n", 480 correlator); 481 schedule_work(&adapter->work); 482 } 483 484 /** 485 * ibmveth_remove_buffer_from_pool - remove a buffer from a pool 486 * @adapter: adapter instance 487 * @correlator: identifies pool and index 488 * @reuse: whether to reuse buffer 489 * 490 * Return: 491 * * %0 - success 492 * * %-EINVAL - correlator maps to pool or index out of range, or to an 493 * inactive pool 494 * * %-EFAULT - pool and index map to null skb 495 */ 496 static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter, 497 u64 correlator, bool reuse) 498 { 499 unsigned int pool = correlator >> 32; 500 unsigned int index = correlator & 0xffffffffUL; 501 unsigned int free_index; 502 struct sk_buff *skb; 503 504 if (!ibmveth_rxq_correlator_valid(adapter, correlator)) 505 return -EINVAL; 506 507 skb = adapter->rx_buff_pool[pool].skbuff[index]; 508 if (!skb) { 509 ibmveth_rxq_no_skb(adapter, correlator); 510 return -EFAULT; 511 } 512 513 /* if we are going to reuse the buffer then keep the pointers around 514 * but mark index as available. replenish will see the skb pointer and 515 * assume it is to be recycled. 516 */ 517 if (!reuse) { 518 /* remove the skb pointer to mark free. actual freeing is done 519 * by upper level networking after gro_receive 520 */ 521 adapter->rx_buff_pool[pool].skbuff[index] = NULL; 522 523 dma_unmap_single(&adapter->vdev->dev, 524 adapter->rx_buff_pool[pool].dma_addr[index], 525 adapter->rx_buff_pool[pool].buff_size, 526 DMA_FROM_DEVICE); 527 } 528 529 free_index = adapter->rx_buff_pool[pool].producer_index; 530 adapter->rx_buff_pool[pool].producer_index++; 531 if (adapter->rx_buff_pool[pool].producer_index >= 532 adapter->rx_buff_pool[pool].size) 533 adapter->rx_buff_pool[pool].producer_index = 0; 534 adapter->rx_buff_pool[pool].free_map[free_index] = index; 535 536 mb(); 537 538 atomic_dec(&(adapter->rx_buff_pool[pool].available)); 539 540 return 0; 541 } 542 543 /* get the buffer for @correlator, read once from the current rx queue entry */ 544 static struct sk_buff *ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter, 545 u64 correlator) 546 { 547 unsigned int pool = correlator >> 32; 548 unsigned int index = correlator & 0xffffffffUL; 549 struct sk_buff *skb; 550 551 if (!ibmveth_rxq_correlator_valid(adapter, correlator)) 552 return NULL; 553 554 skb = adapter->rx_buff_pool[pool].skbuff[index]; 555 if (!skb) 556 ibmveth_rxq_no_skb(adapter, correlator); 557 return skb; 558 } 559 560 static void ibmveth_rxq_advance(struct ibmveth_adapter *adapter) 561 { 562 if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) { 563 adapter->rx_queue.index = 0; 564 adapter->rx_queue.toggle = !adapter->rx_queue.toggle; 565 } 566 } 567 568 /** 569 * ibmveth_rxq_harvest_buffer - Harvest buffer from pool 570 * 571 * @adapter: pointer to adapter 572 * @correlator: correlator of the current rx queue entry, as read by poll 573 * @reuse: whether to reuse buffer 574 * 575 * Context: called from ibmveth_poll 576 * 577 * The ring advances even on error, so poll does not return to a bad 578 * slot before the scheduled reset can run. 579 * 580 * Return: 581 * * %0 - success 582 * * other - non-zero return from ibmveth_remove_buffer_from_pool 583 */ 584 static int ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter, 585 u64 correlator, bool reuse) 586 { 587 int rc; 588 589 rc = ibmveth_remove_buffer_from_pool(adapter, correlator, reuse); 590 ibmveth_rxq_advance(adapter); 591 592 return rc; 593 } 594 595 static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx) 596 { 597 void *ptr = adapter->tx_ltb_ptr[idx]; 598 599 if (!ptr) 600 return; 601 602 adapter->tx_ltb_ptr[idx] = NULL; 603 dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx], 604 adapter->tx_ltb_size, DMA_TO_DEVICE); 605 kfree(ptr); 606 } 607 608 static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx) 609 { 610 adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size, 611 GFP_KERNEL); 612 if (!adapter->tx_ltb_ptr[idx]) { 613 netdev_err(adapter->netdev, 614 "unable to allocate tx long term buffer\n"); 615 return -ENOMEM; 616 } 617 adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev, 618 adapter->tx_ltb_ptr[idx], 619 adapter->tx_ltb_size, 620 DMA_TO_DEVICE); 621 if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) { 622 netdev_err(adapter->netdev, 623 "unable to DMA map tx long term buffer\n"); 624 kfree(adapter->tx_ltb_ptr[idx]); 625 adapter->tx_ltb_ptr[idx] = NULL; 626 return -ENOMEM; 627 } 628 629 return 0; 630 } 631 632 static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter, 633 union ibmveth_buf_desc rxq_desc, u64 mac_address) 634 { 635 int rc, try_again = 1; 636 637 /* 638 * After a kexec the adapter will still be open, so our attempt to 639 * open it will fail. So if we get a failure we free the adapter and 640 * try again, but only once. 641 */ 642 retry: 643 rc = h_register_logical_lan(adapter->vdev->unit_address, 644 adapter->buffer_list_dma, rxq_desc.desc, 645 adapter->filter_list_dma, mac_address); 646 647 if (rc != H_SUCCESS && try_again) { 648 do { 649 rc = h_free_logical_lan(adapter->vdev->unit_address); 650 } while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY)); 651 652 try_again = 0; 653 goto retry; 654 } 655 656 return rc; 657 } 658 659 static int ibmveth_open(struct net_device *netdev) 660 { 661 struct ibmveth_adapter *adapter = netdev_priv(netdev); 662 u64 mac_address; 663 int rxq_entries = 1; 664 unsigned long lpar_rc; 665 int rc; 666 union ibmveth_buf_desc rxq_desc; 667 int i; 668 struct device *dev; 669 670 netdev_dbg(netdev, "open starting\n"); 671 672 for(i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 673 rxq_entries += adapter->rx_buff_pool[i].size; 674 675 rc = -ENOMEM; 676 adapter->buffer_list_addr = (void*) get_zeroed_page(GFP_KERNEL); 677 if (!adapter->buffer_list_addr) { 678 netdev_err(netdev, "unable to allocate list pages\n"); 679 goto out; 680 } 681 682 adapter->filter_list_addr = (void*) get_zeroed_page(GFP_KERNEL); 683 if (!adapter->filter_list_addr) { 684 netdev_err(netdev, "unable to allocate filter pages\n"); 685 goto out_free_buffer_list; 686 } 687 688 dev = &adapter->vdev->dev; 689 690 adapter->rx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) * 691 rxq_entries; 692 adapter->rx_queue.queue_addr = 693 dma_alloc_coherent(dev, adapter->rx_queue.queue_len, 694 &adapter->rx_queue.queue_dma, GFP_KERNEL); 695 if (!adapter->rx_queue.queue_addr) 696 goto out_free_filter_list; 697 698 adapter->buffer_list_dma = dma_map_single(dev, 699 adapter->buffer_list_addr, 4096, DMA_BIDIRECTIONAL); 700 if (dma_mapping_error(dev, adapter->buffer_list_dma)) { 701 netdev_err(netdev, "unable to map buffer list pages\n"); 702 goto out_free_queue_mem; 703 } 704 705 adapter->filter_list_dma = dma_map_single(dev, 706 adapter->filter_list_addr, 4096, DMA_BIDIRECTIONAL); 707 if (dma_mapping_error(dev, adapter->filter_list_dma)) { 708 netdev_err(netdev, "unable to map filter list pages\n"); 709 goto out_unmap_buffer_list; 710 } 711 712 for (i = 0; i < netdev->real_num_tx_queues; i++) { 713 if (ibmveth_allocate_tx_ltb(adapter, i)) 714 goto out_unmap_filter_list; 715 } 716 717 adapter->rx_queue.index = 0; 718 adapter->rx_queue.num_slots = rxq_entries; 719 adapter->rx_queue.toggle = 1; 720 721 mac_address = ether_addr_to_u64(netdev->dev_addr); 722 723 rxq_desc.fields.flags_len = IBMVETH_BUF_VALID | 724 adapter->rx_queue.queue_len; 725 rxq_desc.fields.address = adapter->rx_queue.queue_dma; 726 727 netdev_dbg(netdev, "buffer list @ 0x%p\n", adapter->buffer_list_addr); 728 netdev_dbg(netdev, "filter list @ 0x%p\n", adapter->filter_list_addr); 729 netdev_dbg(netdev, "receive q @ 0x%p\n", adapter->rx_queue.queue_addr); 730 731 h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE); 732 733 lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address); 734 735 if (lpar_rc != H_SUCCESS) { 736 netdev_err(netdev, "h_register_logical_lan failed with %ld\n", 737 lpar_rc); 738 netdev_err(netdev, "buffer TCE:0x%llx filter TCE:0x%llx rxq " 739 "desc:0x%llx MAC:0x%llx\n", 740 adapter->buffer_list_dma, 741 adapter->filter_list_dma, 742 rxq_desc.desc, 743 mac_address); 744 rc = -ENONET; 745 goto out_unmap_filter_list; 746 } 747 748 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { 749 if (!adapter->rx_buff_pool[i].active) 750 continue; 751 if (ibmveth_alloc_buffer_pool(&adapter->rx_buff_pool[i])) { 752 netdev_err(netdev, "unable to alloc pool\n"); 753 adapter->rx_buff_pool[i].active = 0; 754 rc = -ENOMEM; 755 goto out_free_buffer_pools; 756 } 757 } 758 759 /* NAPI can run as soon as it is enabled, from netpoll during the 760 * direct close()/open() pairs or from a direct ibmveth_interrupt() 761 * call, so enable it only once everything ibmveth_poll() touches 762 * exists. 763 */ 764 napi_enable(&adapter->napi); 765 766 netdev_dbg(netdev, "registering irq 0x%x\n", netdev->irq); 767 rc = request_irq(netdev->irq, ibmveth_interrupt, 0, netdev->name, 768 netdev); 769 if (rc != 0) { 770 napi_disable(&adapter->napi); 771 synchronize_net(); 772 netdev_err(netdev, "unable to request irq 0x%x, rc %d\n", 773 netdev->irq, rc); 774 goto out_free_buffer_pools; 775 } 776 777 rc = -ENOMEM; 778 779 netdev_dbg(netdev, "initial replenish cycle\n"); 780 ibmveth_interrupt(netdev->irq, netdev); 781 782 netif_tx_start_all_queues(netdev); 783 784 adapter->opened = true; 785 netdev_dbg(netdev, "open complete\n"); 786 787 return 0; 788 789 out_free_buffer_pools: 790 do { 791 lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); 792 } while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY)); 793 while (--i >= 0) { 794 if (adapter->rx_buff_pool[i].active) 795 ibmveth_free_buffer_pool(adapter, 796 &adapter->rx_buff_pool[i]); 797 } 798 out_unmap_filter_list: 799 dma_unmap_single(dev, adapter->filter_list_dma, 4096, 800 DMA_BIDIRECTIONAL); 801 802 for (i = netdev->real_num_tx_queues - 1; i >= 0; i--) 803 ibmveth_free_tx_ltb(adapter, i); 804 805 out_unmap_buffer_list: 806 dma_unmap_single(dev, adapter->buffer_list_dma, 4096, 807 DMA_BIDIRECTIONAL); 808 out_free_queue_mem: 809 dma_free_coherent(dev, adapter->rx_queue.queue_len, 810 adapter->rx_queue.queue_addr, 811 adapter->rx_queue.queue_dma); 812 out_free_filter_list: 813 free_page((unsigned long)adapter->filter_list_addr); 814 out_free_buffer_list: 815 free_page((unsigned long)adapter->buffer_list_addr); 816 out: 817 return rc; 818 } 819 820 static int ibmveth_close(struct net_device *netdev) 821 { 822 struct ibmveth_adapter *adapter = netdev_priv(netdev); 823 struct device *dev = &adapter->vdev->dev; 824 long lpar_rc; 825 int i; 826 827 /* change_mtu, pool sysfs, set_csum and set_tso call close() and 828 * open() directly. If that open() fails, IFF_UP stays set and 829 * NAPI is disabled; a second close() would hang in napi_disable(). 830 */ 831 if (!adapter->opened) 832 return 0; 833 adapter->opened = false; 834 835 netdev_dbg(netdev, "close starting\n"); 836 837 napi_disable(&adapter->napi); 838 /* napi_disable() returns once ibmveth_poll() has called 839 * napi_complete_done(), but the poll still re-enables the 840 * interrupt and reads the RX queue after that. 841 */ 842 synchronize_net(); 843 844 netif_tx_disable(netdev); 845 846 h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE); 847 848 do { 849 lpar_rc = h_free_logical_lan(adapter->vdev->unit_address); 850 } while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY)); 851 852 if (lpar_rc != H_SUCCESS) { 853 netdev_err(netdev, "h_free_logical_lan failed with %lx, " 854 "continuing with close\n", lpar_rc); 855 } 856 857 free_irq(netdev->irq, netdev); 858 859 ibmveth_update_rx_no_buffer(adapter); 860 861 dma_unmap_single(dev, adapter->buffer_list_dma, 4096, 862 DMA_BIDIRECTIONAL); 863 free_page((unsigned long)adapter->buffer_list_addr); 864 865 dma_unmap_single(dev, adapter->filter_list_dma, 4096, 866 DMA_BIDIRECTIONAL); 867 free_page((unsigned long)adapter->filter_list_addr); 868 869 dma_free_coherent(dev, adapter->rx_queue.queue_len, 870 adapter->rx_queue.queue_addr, 871 adapter->rx_queue.queue_dma); 872 873 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 874 if (adapter->rx_buff_pool[i].active) 875 ibmveth_free_buffer_pool(adapter, 876 &adapter->rx_buff_pool[i]); 877 878 for (i = 0; i < netdev->real_num_tx_queues; i++) 879 ibmveth_free_tx_ltb(adapter, i); 880 881 netdev_dbg(netdev, "close complete\n"); 882 883 return 0; 884 } 885 886 /** 887 * ibmveth_reset - Handle scheduled reset work 888 * 889 * @w: pointer to work_struct embedded in adapter structure 890 * 891 * Context: This routine acquires rtnl_mutex and, if the device is open, 892 * disables its NAPI through ibmveth_close. It can't be called 893 * directly in a context that has already acquired rtnl_mutex or 894 * disabled its NAPI, or directly from a poll routine. 895 * 896 * Return: void 897 */ 898 static void ibmveth_reset(struct work_struct *w) 899 { 900 struct ibmveth_adapter *adapter = container_of(w, struct ibmveth_adapter, work); 901 struct net_device *netdev = adapter->netdev; 902 903 netdev_dbg(netdev, "reset starting\n"); 904 905 rtnl_lock(); 906 907 dev_close(adapter->netdev); 908 dev_open(adapter->netdev, NULL); 909 910 rtnl_unlock(); 911 912 netdev_dbg(netdev, "reset complete\n"); 913 } 914 915 static int ibmveth_set_link_ksettings(struct net_device *dev, 916 const struct ethtool_link_ksettings *cmd) 917 { 918 struct ibmveth_adapter *adapter = netdev_priv(dev); 919 920 return ethtool_virtdev_set_link_ksettings(dev, cmd, 921 &adapter->speed, 922 &adapter->duplex); 923 } 924 925 static int ibmveth_get_link_ksettings(struct net_device *dev, 926 struct ethtool_link_ksettings *cmd) 927 { 928 struct ibmveth_adapter *adapter = netdev_priv(dev); 929 930 cmd->base.speed = adapter->speed; 931 cmd->base.duplex = adapter->duplex; 932 cmd->base.port = PORT_OTHER; 933 934 return 0; 935 } 936 937 static void ibmveth_init_link_settings(struct net_device *dev) 938 { 939 struct ibmveth_adapter *adapter = netdev_priv(dev); 940 941 adapter->speed = SPEED_1000; 942 adapter->duplex = DUPLEX_FULL; 943 } 944 945 static void netdev_get_drvinfo(struct net_device *dev, 946 struct ethtool_drvinfo *info) 947 { 948 strscpy(info->driver, ibmveth_driver_name, sizeof(info->driver)); 949 strscpy(info->version, ibmveth_driver_version, sizeof(info->version)); 950 } 951 952 static netdev_features_t ibmveth_fix_features(struct net_device *dev, 953 netdev_features_t features) 954 { 955 /* 956 * Since the ibmveth firmware interface does not have the 957 * concept of separate tx/rx checksum offload enable, if rx 958 * checksum is disabled we also have to disable tx checksum 959 * offload. Once we disable rx checksum offload, we are no 960 * longer allowed to send tx buffers that are not properly 961 * checksummed. 962 */ 963 964 if (!(features & NETIF_F_RXCSUM)) 965 features &= ~NETIF_F_CSUM_MASK; 966 967 return features; 968 } 969 970 static int ibmveth_set_csum_offload(struct net_device *dev, u32 data) 971 { 972 struct ibmveth_adapter *adapter = netdev_priv(dev); 973 unsigned long set_attr, clr_attr, ret_attr; 974 unsigned long set_attr6, clr_attr6; 975 long ret, ret4, ret6; 976 int rc1 = 0, rc2 = 0; 977 int restart = 0; 978 979 if (netif_running(dev)) { 980 restart = 1; 981 ibmveth_close(dev); 982 } 983 984 set_attr = 0; 985 clr_attr = 0; 986 set_attr6 = 0; 987 clr_attr6 = 0; 988 989 if (data) { 990 set_attr = IBMVETH_ILLAN_IPV4_TCP_CSUM; 991 set_attr6 = IBMVETH_ILLAN_IPV6_TCP_CSUM; 992 } else { 993 clr_attr = IBMVETH_ILLAN_IPV4_TCP_CSUM; 994 clr_attr6 = IBMVETH_ILLAN_IPV6_TCP_CSUM; 995 } 996 997 ret = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr); 998 999 if (ret == H_SUCCESS && 1000 (ret_attr & IBMVETH_ILLAN_PADDED_PKT_CSUM)) { 1001 ret4 = h_illan_attributes(adapter->vdev->unit_address, clr_attr, 1002 set_attr, &ret_attr); 1003 1004 if (ret4 != H_SUCCESS) { 1005 netdev_err(dev, "unable to change IPv4 checksum " 1006 "offload settings. %d rc=%ld\n", 1007 data, ret4); 1008 1009 h_illan_attributes(adapter->vdev->unit_address, 1010 set_attr, clr_attr, &ret_attr); 1011 1012 if (data == 1) 1013 dev->features &= ~NETIF_F_IP_CSUM; 1014 1015 } else { 1016 adapter->fw_ipv4_csum_support = data; 1017 } 1018 1019 ret6 = h_illan_attributes(adapter->vdev->unit_address, 1020 clr_attr6, set_attr6, &ret_attr); 1021 1022 if (ret6 != H_SUCCESS) { 1023 netdev_err(dev, "unable to change IPv6 checksum " 1024 "offload settings. %d rc=%ld\n", 1025 data, ret6); 1026 1027 h_illan_attributes(adapter->vdev->unit_address, 1028 set_attr6, clr_attr6, &ret_attr); 1029 1030 if (data == 1) 1031 dev->features &= ~NETIF_F_IPV6_CSUM; 1032 1033 } else 1034 adapter->fw_ipv6_csum_support = data; 1035 1036 if (ret4 == H_SUCCESS || ret6 == H_SUCCESS) 1037 adapter->rx_csum = data; 1038 else 1039 rc1 = -EIO; 1040 } else { 1041 rc1 = -EIO; 1042 netdev_err(dev, "unable to change checksum offload settings." 1043 " %d rc=%ld ret_attr=%lx\n", data, ret, 1044 ret_attr); 1045 } 1046 1047 if (restart) 1048 rc2 = ibmveth_open(dev); 1049 1050 return rc1 ? rc1 : rc2; 1051 } 1052 1053 static int ibmveth_set_tso(struct net_device *dev, u32 data) 1054 { 1055 struct ibmveth_adapter *adapter = netdev_priv(dev); 1056 unsigned long set_attr, clr_attr, ret_attr; 1057 long ret1, ret2; 1058 int rc1 = 0, rc2 = 0; 1059 int restart = 0; 1060 1061 if (netif_running(dev)) { 1062 restart = 1; 1063 ibmveth_close(dev); 1064 } 1065 1066 set_attr = 0; 1067 clr_attr = 0; 1068 1069 if (data) 1070 set_attr = IBMVETH_ILLAN_LRG_SR_ENABLED; 1071 else 1072 clr_attr = IBMVETH_ILLAN_LRG_SR_ENABLED; 1073 1074 ret1 = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr); 1075 1076 if (ret1 == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_LRG_SND_SUPPORT) && 1077 !old_large_send) { 1078 ret2 = h_illan_attributes(adapter->vdev->unit_address, clr_attr, 1079 set_attr, &ret_attr); 1080 1081 if (ret2 != H_SUCCESS) { 1082 netdev_err(dev, "unable to change tso settings. %d rc=%ld\n", 1083 data, ret2); 1084 1085 h_illan_attributes(adapter->vdev->unit_address, 1086 set_attr, clr_attr, &ret_attr); 1087 1088 if (data == 1) 1089 dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6); 1090 rc1 = -EIO; 1091 1092 } else { 1093 adapter->fw_large_send_support = data; 1094 adapter->large_send = data; 1095 } 1096 } else { 1097 /* Older firmware version of large send offload does not 1098 * support tcp6/ipv6 1099 */ 1100 if (data == 1) { 1101 dev->features &= ~NETIF_F_TSO6; 1102 netdev_info(dev, "TSO feature requires all partitions to have updated driver"); 1103 } 1104 adapter->large_send = data; 1105 } 1106 1107 if (restart) 1108 rc2 = ibmveth_open(dev); 1109 1110 return rc1 ? rc1 : rc2; 1111 } 1112 1113 static int ibmveth_set_features(struct net_device *dev, 1114 netdev_features_t features) 1115 { 1116 struct ibmveth_adapter *adapter = netdev_priv(dev); 1117 int rx_csum = !!(features & NETIF_F_RXCSUM); 1118 int large_send = !!(features & (NETIF_F_TSO | NETIF_F_TSO6)); 1119 int rc1 = 0, rc2 = 0; 1120 1121 if (rx_csum != adapter->rx_csum) { 1122 rc1 = ibmveth_set_csum_offload(dev, rx_csum); 1123 if (rc1 && !adapter->rx_csum) 1124 dev->features = 1125 features & ~(NETIF_F_CSUM_MASK | 1126 NETIF_F_RXCSUM); 1127 } 1128 1129 if (large_send != adapter->large_send) { 1130 rc2 = ibmveth_set_tso(dev, large_send); 1131 if (rc2 && !adapter->large_send) 1132 dev->features = 1133 features & ~(NETIF_F_TSO | NETIF_F_TSO6); 1134 } 1135 1136 return rc1 ? rc1 : rc2; 1137 } 1138 1139 static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data) 1140 { 1141 int i; 1142 1143 if (stringset != ETH_SS_STATS) 1144 return; 1145 1146 for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN) 1147 memcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN); 1148 } 1149 1150 static int ibmveth_get_sset_count(struct net_device *dev, int sset) 1151 { 1152 switch (sset) { 1153 case ETH_SS_STATS: 1154 return ARRAY_SIZE(ibmveth_stats); 1155 default: 1156 return -EOPNOTSUPP; 1157 } 1158 } 1159 1160 static void ibmveth_get_ethtool_stats(struct net_device *dev, 1161 struct ethtool_stats *stats, u64 *data) 1162 { 1163 int i; 1164 struct ibmveth_adapter *adapter = netdev_priv(dev); 1165 1166 for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++) 1167 data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset); 1168 } 1169 1170 static void ibmveth_get_channels(struct net_device *netdev, 1171 struct ethtool_channels *channels) 1172 { 1173 channels->max_tx = ibmveth_real_max_tx_queues(); 1174 channels->tx_count = netdev->real_num_tx_queues; 1175 1176 channels->max_rx = netdev->real_num_rx_queues; 1177 channels->rx_count = netdev->real_num_rx_queues; 1178 } 1179 1180 static int ibmveth_set_channels(struct net_device *netdev, 1181 struct ethtool_channels *channels) 1182 { 1183 struct ibmveth_adapter *adapter = netdev_priv(netdev); 1184 unsigned int old = netdev->real_num_tx_queues, 1185 goal = channels->tx_count; 1186 int rc, i, alloc_rc = 0; 1187 1188 /* If the device is not open (including a failed close/open with 1189 * IFF_UP still set) then don't allocate, just set desired 1190 * netdev_queue's and return 1191 */ 1192 if (!adapter->opened) 1193 return netif_set_real_num_tx_queues(netdev, goal); 1194 1195 /* We have IBMVETH_MAX_QUEUES netdev_queue's allocated 1196 * but we may need to alloc/free the ltb's. 1197 */ 1198 netif_tx_stop_all_queues(netdev); 1199 1200 /* Allocate any queue that we need */ 1201 for (i = old; i < goal; i++) { 1202 if (adapter->tx_ltb_ptr[i]) 1203 continue; 1204 1205 rc = ibmveth_allocate_tx_ltb(adapter, i); 1206 if (!rc) 1207 continue; 1208 1209 /* if something goes wrong, free everything we just allocated */ 1210 netdev_err(netdev, "Failed to allocate more tx queues, returning to %d queues\n", 1211 old); 1212 alloc_rc = rc; 1213 goal = old; 1214 old = i; 1215 break; 1216 } 1217 rc = netif_set_real_num_tx_queues(netdev, goal); 1218 if (rc) { 1219 netdev_err(netdev, "Failed to set real tx queues, returning to %d queues\n", 1220 old); 1221 goal = old; 1222 old = i; 1223 } else if (alloc_rc) { 1224 rc = alloc_rc; 1225 } 1226 /* Free any that are no longer needed */ 1227 for (i = old; i > goal; i--) { 1228 if (adapter->tx_ltb_ptr[i - 1]) 1229 ibmveth_free_tx_ltb(adapter, i - 1); 1230 } 1231 1232 netif_tx_wake_all_queues(netdev); 1233 1234 return rc; 1235 } 1236 1237 static const struct ethtool_ops netdev_ethtool_ops = { 1238 .get_drvinfo = netdev_get_drvinfo, 1239 .get_link = ethtool_op_get_link, 1240 .get_strings = ibmveth_get_strings, 1241 .get_sset_count = ibmveth_get_sset_count, 1242 .get_ethtool_stats = ibmveth_get_ethtool_stats, 1243 .get_link_ksettings = ibmveth_get_link_ksettings, 1244 .set_link_ksettings = ibmveth_set_link_ksettings, 1245 .get_channels = ibmveth_get_channels, 1246 .set_channels = ibmveth_set_channels 1247 }; 1248 1249 static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 1250 { 1251 return -EOPNOTSUPP; 1252 } 1253 1254 static int ibmveth_send(struct ibmveth_adapter *adapter, 1255 unsigned long desc, unsigned long mss) 1256 { 1257 unsigned long correlator; 1258 unsigned int retry_count; 1259 unsigned long ret; 1260 1261 /* 1262 * The retry count sets a maximum for the number of broadcast and 1263 * multicast destinations within the system. 1264 */ 1265 retry_count = 1024; 1266 correlator = 0; 1267 do { 1268 ret = h_send_logical_lan(adapter->vdev->unit_address, desc, 1269 correlator, &correlator, mss, 1270 adapter->fw_large_send_support); 1271 } while ((ret == H_BUSY) && (retry_count--)); 1272 1273 if (ret != H_SUCCESS && ret != H_DROPPED) { 1274 netdev_err(adapter->netdev, "tx: h_send_logical_lan failed " 1275 "with rc=%ld\n", ret); 1276 return 1; 1277 } 1278 1279 return 0; 1280 } 1281 1282 static int ibmveth_is_packet_unsupported(struct sk_buff *skb, 1283 struct net_device *netdev) 1284 { 1285 struct ethhdr *ether_header; 1286 int ret = 0; 1287 1288 ether_header = eth_hdr(skb); 1289 1290 if (ether_addr_equal(ether_header->h_dest, netdev->dev_addr)) { 1291 netdev_dbg(netdev, "veth doesn't support loopback packets, dropping packet.\n"); 1292 netdev->stats.tx_dropped++; 1293 ret = -EOPNOTSUPP; 1294 } 1295 1296 return ret; 1297 } 1298 1299 static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb, 1300 struct net_device *netdev) 1301 { 1302 struct ibmveth_adapter *adapter = netdev_priv(netdev); 1303 unsigned int desc_flags, total_bytes; 1304 union ibmveth_buf_desc desc; 1305 int i, queue_num = skb_get_queue_mapping(skb); 1306 unsigned long mss = 0; 1307 1308 if (ibmveth_is_packet_unsupported(skb, netdev)) 1309 goto out; 1310 /* veth can't checksum offload UDP */ 1311 if (skb->ip_summed == CHECKSUM_PARTIAL && 1312 ((skb->protocol == htons(ETH_P_IP) && 1313 ip_hdr(skb)->protocol != IPPROTO_TCP) || 1314 (skb->protocol == htons(ETH_P_IPV6) && 1315 ipv6_hdr(skb)->nexthdr != IPPROTO_TCP)) && 1316 skb_checksum_help(skb)) { 1317 1318 netdev_err(netdev, "tx: failed to checksum packet\n"); 1319 netdev->stats.tx_dropped++; 1320 goto out; 1321 } 1322 1323 desc_flags = IBMVETH_BUF_VALID; 1324 1325 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1326 unsigned char *buf = skb_transport_header(skb) + 1327 skb->csum_offset; 1328 1329 desc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD); 1330 1331 /* Need to zero out the checksum */ 1332 buf[0] = 0; 1333 buf[1] = 0; 1334 1335 if (skb_is_gso(skb) && adapter->fw_large_send_support) 1336 desc_flags |= IBMVETH_BUF_LRG_SND; 1337 } 1338 1339 if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) { 1340 if (adapter->fw_large_send_support) { 1341 mss = (unsigned long)skb_shinfo(skb)->gso_size; 1342 adapter->tx_large_packets++; 1343 } else if (!skb_is_gso_v6(skb)) { 1344 /* Put -1 in the IP checksum to tell phyp it 1345 * is a largesend packet. Put the mss in 1346 * the TCP checksum. 1347 */ 1348 ip_hdr(skb)->check = 0xffff; 1349 tcp_hdr(skb)->check = 1350 cpu_to_be16(skb_shinfo(skb)->gso_size); 1351 adapter->tx_large_packets++; 1352 } 1353 } 1354 1355 /* Copy header into mapped buffer */ 1356 if (unlikely(skb->len > adapter->tx_ltb_size)) { 1357 netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n", 1358 skb->len, adapter->tx_ltb_size); 1359 netdev->stats.tx_dropped++; 1360 goto out; 1361 } 1362 memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb)); 1363 total_bytes = skb_headlen(skb); 1364 /* Copy frags into mapped buffers */ 1365 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1366 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1367 1368 memcpy(adapter->tx_ltb_ptr[queue_num] + total_bytes, 1369 skb_frag_address_safe(frag), skb_frag_size(frag)); 1370 total_bytes += skb_frag_size(frag); 1371 } 1372 1373 if (unlikely(total_bytes != skb->len)) { 1374 netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n", 1375 skb->len, total_bytes); 1376 netdev->stats.tx_dropped++; 1377 goto out; 1378 } 1379 desc.fields.flags_len = desc_flags | skb->len; 1380 desc.fields.address = adapter->tx_ltb_dma[queue_num]; 1381 /* finish writing to long_term_buff before VIOS accessing it */ 1382 dma_wmb(); 1383 1384 if (ibmveth_send(adapter, desc.desc, mss)) { 1385 adapter->tx_send_failed++; 1386 netdev->stats.tx_dropped++; 1387 } else { 1388 netdev->stats.tx_packets++; 1389 netdev->stats.tx_bytes += skb->len; 1390 } 1391 1392 out: 1393 dev_consume_skb_any(skb); 1394 return NETDEV_TX_OK; 1395 1396 1397 } 1398 1399 /* Returns false when the IP or TCP headers fail the checks; the frame is 1400 * then left unchanged and not marked GSO. 1401 */ 1402 static bool ibmveth_rx_mss_helper(struct sk_buff *skb, u16 mss, int lrg_pkt) 1403 { 1404 unsigned int gso_type, ip_len; 1405 struct tcphdr *tcph; 1406 int offset = 0; 1407 int hdr_len; 1408 1409 /* only TCP packets will be aggregated */ 1410 if (skb->protocol == htons(ETH_P_IP)) { 1411 struct iphdr *iph = (struct iphdr *)skb->data; 1412 1413 if (skb_headlen(skb) < sizeof(*iph) || iph->ihl < 5) 1414 return false; 1415 if (iph->protocol == IPPROTO_TCP) { 1416 /* only a whole datagram has a TCP header to segment */ 1417 if (ip_is_fragment(iph)) 1418 return false; 1419 offset = iph->ihl * 4; 1420 ip_len = ntohs(iph->tot_len); 1421 gso_type = SKB_GSO_TCPV4; 1422 } else { 1423 return true; 1424 } 1425 } else if (skb->protocol == htons(ETH_P_IPV6)) { 1426 struct ipv6hdr *iph6 = (struct ipv6hdr *)skb->data; 1427 1428 if (skb_headlen(skb) < sizeof(*iph6)) 1429 return false; 1430 if (iph6->nexthdr == IPPROTO_TCP) { 1431 offset = sizeof(struct ipv6hdr); 1432 ip_len = offset + ntohs(iph6->payload_len); 1433 gso_type = SKB_GSO_TCPV6; 1434 } else { 1435 return true; 1436 } 1437 } else { 1438 return true; 1439 } 1440 /* if mss is not set through Large Packet bit/mss in rx buffer, 1441 * expect that the mss will be written to the tcp header checksum. 1442 */ 1443 tcph = (struct tcphdr *)(skb->data + offset); 1444 if (skb_headlen(skb) < offset + sizeof(*tcph) || tcph->doff < 5 || 1445 ip_len != skb_headlen(skb) || ip_len <= offset + tcph->doff * 4) 1446 return false; 1447 skb_shinfo(skb)->gso_type = gso_type; 1448 if (lrg_pkt) { 1449 skb_shinfo(skb)->gso_size = mss; 1450 } else if (offset) { 1451 skb_shinfo(skb)->gso_size = ntohs(tcph->check); 1452 tcph->check = 0; 1453 } 1454 1455 if (skb_shinfo(skb)->gso_size) { 1456 /* chosen by the sending partition: at least the floor 1457 * virtio_net_hdr_to_skb() uses, and never GSO_BY_FRAGS 1458 */ 1459 skb_shinfo(skb)->gso_size = 1460 clamp_t(u16, skb_shinfo(skb)->gso_size, 1461 TCP_MIN_GSO_SIZE, GSO_BY_FRAGS - 1); 1462 hdr_len = offset + tcph->doff * 4; 1463 skb_shinfo(skb)->gso_segs = 1464 DIV_ROUND_UP(ip_len - hdr_len, 1465 skb_shinfo(skb)->gso_size); 1466 } 1467 return true; 1468 } 1469 1470 static void ibmveth_rx_csum_helper(struct sk_buff *skb, 1471 struct ibmveth_adapter *adapter) 1472 { 1473 struct iphdr *iph = NULL; 1474 struct ipv6hdr *iph6 = NULL; 1475 __be16 skb_proto = 0; 1476 u16 iphlen = 0; 1477 u16 iph_proto = 0; 1478 u16 tcphdrlen = 0; 1479 1480 skb_proto = be16_to_cpu(skb->protocol); 1481 1482 if (skb_proto == ETH_P_IP) { 1483 iph = (struct iphdr *)skb->data; 1484 if (skb_headlen(skb) < sizeof(*iph) || iph->ihl < 5 || 1485 skb_headlen(skb) < iph->ihl * 4 || 1486 (iph->protocol == IPPROTO_TCP && ip_is_fragment(iph))) 1487 goto bad_hdr; 1488 1489 /* If the IP checksum is not offloaded and if the packet 1490 * is large send, the checksum must be rebuilt. 1491 */ 1492 if (iph->check == 0xffff) { 1493 iph->check = 0; 1494 iph->check = ip_fast_csum((unsigned char *)iph, 1495 iph->ihl); 1496 } 1497 1498 iphlen = iph->ihl * 4; 1499 iph_proto = iph->protocol; 1500 } else if (skb_proto == ETH_P_IPV6) { 1501 iph6 = (struct ipv6hdr *)skb->data; 1502 if (skb_headlen(skb) < sizeof(*iph6)) 1503 goto bad_hdr; 1504 iphlen = sizeof(struct ipv6hdr); 1505 iph_proto = iph6->nexthdr; 1506 } 1507 1508 /* When CSO is enabled the TCP checksum may have be set to NULL by 1509 * the sender given that we zeroed out TCP checksum field in 1510 * transmit path (refer ibmveth_start_xmit routine). In this case set 1511 * up CHECKSUM_PARTIAL. If the packet is forwarded, the checksum will 1512 * then be recalculated by the destination NIC (CSO must be enabled 1513 * on the destination NIC). 1514 * 1515 * In an OVS environment, when a flow is not cached, specifically for a 1516 * new TCP connection, the first packet information is passed up to 1517 * the user space for finding a flow. During this process, OVS computes 1518 * checksum on the first packet when CHECKSUM_PARTIAL flag is set. 1519 * 1520 * So, re-compute TCP pseudo header checksum. 1521 */ 1522 1523 if (iph_proto == IPPROTO_TCP) { 1524 struct tcphdr *tcph = (struct tcphdr *)(skb->data + iphlen); 1525 1526 if (skb_headlen(skb) < iphlen + sizeof(*tcph)) 1527 goto bad_hdr; 1528 if (tcph->check == 0x0000) { 1529 /* Recompute TCP pseudo header checksum */ 1530 tcphdrlen = skb->len - iphlen; 1531 if (skb_proto == ETH_P_IP) 1532 tcph->check = 1533 ~csum_tcpudp_magic(iph->saddr, 1534 iph->daddr, tcphdrlen, iph_proto, 0); 1535 else if (skb_proto == ETH_P_IPV6) 1536 tcph->check = 1537 ~csum_ipv6_magic(&iph6->saddr, 1538 &iph6->daddr, tcphdrlen, iph_proto, 0); 1539 /* Setup SKB fields for checksum offload */ 1540 skb_partial_csum_set(skb, iphlen, 1541 offsetof(struct tcphdr, check)); 1542 skb_reset_network_header(skb); 1543 } 1544 } 1545 return; 1546 1547 bad_hdr: 1548 /* headers do not fit the frame, or a fragment has no whole TCP 1549 * segment; let the stack check it 1550 */ 1551 skb->ip_summed = CHECKSUM_NONE; 1552 } 1553 1554 static int ibmveth_poll(struct napi_struct *napi, int budget) 1555 { 1556 struct ibmveth_adapter *adapter = 1557 container_of(napi, struct ibmveth_adapter, napi); 1558 struct net_device *netdev = adapter->netdev; 1559 int frames_processed = 0; 1560 unsigned long lpar_rc; 1561 u64 correlator; 1562 u16 mss = 0; 1563 int rc; 1564 1565 while (frames_processed < budget) { 1566 if (!ibmveth_rxq_pending_buffer(adapter)) 1567 break; 1568 1569 smp_rmb(); 1570 correlator = ibmveth_rxq_correlator(adapter); 1571 if (!ibmveth_rxq_buffer_valid(adapter)) { 1572 wmb(); /* suggested by larson1 */ 1573 adapter->rx_invalid_buffer++; 1574 netdev_dbg(netdev, "recycling invalid buffer\n"); 1575 rc = ibmveth_rxq_harvest_buffer(adapter, correlator, true); 1576 if (unlikely(rc)) { 1577 netdev->stats.rx_dropped++; 1578 frames_processed++; 1579 continue; 1580 } 1581 } else { 1582 struct sk_buff *skb, *new_skb; 1583 int length = ibmveth_rxq_frame_length(adapter); 1584 int offset = ibmveth_rxq_frame_offset(adapter); 1585 int csum_good = ibmveth_rxq_csum_good(adapter); 1586 int lrg_pkt = ibmveth_rxq_large_packet(adapter); 1587 unsigned int pool, room, off, len; 1588 __sum16 iph_check = 0; 1589 1590 skb = ibmveth_rxq_get_buffer(adapter, correlator); 1591 if (unlikely(!skb)) { 1592 ibmveth_rxq_advance(adapter); 1593 netdev->stats.rx_dropped++; 1594 frames_processed++; 1595 continue; 1596 } 1597 1598 pool = correlator >> 32; 1599 room = min_t(unsigned int, skb_tailroom(skb), 1600 adapter->rx_buff_pool[pool].buff_size); 1601 off = offset; 1602 len = length; 1603 if (unlikely(len < ETH_HLEN || off >= room || 1604 len > room - off)) { 1605 if (net_ratelimit()) 1606 netdev_err(netdev, 1607 "bad RX frame offset %u length %u (buffer %u), dropping\n", 1608 off, len, room); 1609 ibmveth_rxq_harvest_buffer(adapter, correlator, 1610 true); 1611 netdev->stats.rx_dropped++; 1612 frames_processed++; 1613 continue; 1614 } 1615 1616 /* if the large packet bit is set in the rx queue 1617 * descriptor, the mss will be written by PHYP eight 1618 * bytes from the start of the rx buffer, which is 1619 * skb->data at this stage 1620 */ 1621 if (lrg_pkt) { 1622 __be64 *rxmss = (__be64 *)(skb->data + 8); 1623 1624 mss = (u16)be64_to_cpu(*rxmss); 1625 } 1626 1627 new_skb = NULL; 1628 if (length < rx_copybreak) 1629 new_skb = netdev_alloc_skb(netdev, length); 1630 1631 if (new_skb) { 1632 skb_copy_to_linear_data(new_skb, 1633 skb->data + offset, 1634 length); 1635 if (rx_flush) 1636 ibmveth_flush_buffer(skb->data, 1637 length + offset); 1638 rc = ibmveth_rxq_harvest_buffer(adapter, 1639 correlator, 1640 true); 1641 if (unlikely(rc)) { 1642 dev_kfree_skb_any(new_skb); 1643 netdev->stats.rx_dropped++; 1644 frames_processed++; 1645 continue; 1646 } 1647 skb = new_skb; 1648 } else { 1649 rc = ibmveth_rxq_harvest_buffer(adapter, 1650 correlator, 1651 false); 1652 if (unlikely(rc)) { 1653 netdev->stats.rx_dropped++; 1654 frames_processed++; 1655 continue; 1656 } 1657 skb_reserve(skb, offset); 1658 } 1659 1660 skb_put(skb, length); 1661 skb->protocol = eth_type_trans(skb, netdev); 1662 1663 /* PHYP without PLSO support places a -1 in the ip 1664 * checksum for large send frames. 1665 */ 1666 if (skb->protocol == cpu_to_be16(ETH_P_IP) && 1667 skb_headlen(skb) >= sizeof(struct iphdr)) { 1668 struct iphdr *iph = (struct iphdr *)skb->data; 1669 1670 iph_check = iph->check; 1671 } 1672 1673 if ((length > netdev->mtu + ETH_HLEN) || 1674 lrg_pkt || iph_check == 0xffff) { 1675 if (!ibmveth_rx_mss_helper(skb, mss, lrg_pkt)) 1676 csum_good = 0; 1677 adapter->rx_large_packets++; 1678 } 1679 1680 if (csum_good) { 1681 skb->ip_summed = CHECKSUM_UNNECESSARY; 1682 ibmveth_rx_csum_helper(skb, adapter); 1683 } 1684 1685 napi_gro_receive(napi, skb); /* send it up */ 1686 1687 netdev->stats.rx_packets++; 1688 netdev->stats.rx_bytes += length; 1689 frames_processed++; 1690 } 1691 } 1692 1693 /* netpoll polls with budget 0 for TX only, and is not serialized 1694 * with a NAPI poll that was already running when it was set up 1695 */ 1696 if (budget) 1697 ibmveth_replenish_task(adapter); 1698 1699 if (frames_processed == budget) 1700 goto out; 1701 1702 if (!napi_complete_done(napi, frames_processed)) 1703 goto out; 1704 1705 /* We think we are done - reenable interrupts, 1706 * then check once more to make sure we are done. 1707 */ 1708 lpar_rc = h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_ENABLE); 1709 if (WARN_ON(lpar_rc != H_SUCCESS)) { 1710 schedule_work(&adapter->work); 1711 goto out; 1712 } 1713 1714 /* Mask before queueing, as ibmveth_interrupt() does: a threaded 1715 * NAPI can start polling, and unmask, as soon as it is queued. 1716 */ 1717 if (ibmveth_rxq_pending_buffer(adapter) && napi_schedule_prep(napi)) { 1718 lpar_rc = h_vio_signal(adapter->vdev->unit_address, 1719 VIO_IRQ_DISABLE); 1720 __napi_schedule(napi); 1721 } 1722 1723 out: 1724 return frames_processed; 1725 } 1726 1727 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance) 1728 { 1729 struct net_device *netdev = dev_instance; 1730 struct ibmveth_adapter *adapter = netdev_priv(netdev); 1731 unsigned long lpar_rc; 1732 1733 if (napi_schedule_prep(&adapter->napi)) { 1734 lpar_rc = h_vio_signal(adapter->vdev->unit_address, 1735 VIO_IRQ_DISABLE); 1736 WARN_ON(lpar_rc != H_SUCCESS); 1737 __napi_schedule(&adapter->napi); 1738 } 1739 return IRQ_HANDLED; 1740 } 1741 1742 static void ibmveth_set_multicast_list(struct net_device *netdev) 1743 { 1744 struct ibmveth_adapter *adapter = netdev_priv(netdev); 1745 unsigned long lpar_rc; 1746 1747 if ((netdev->flags & IFF_PROMISC) || 1748 (netdev_mc_count(netdev) > adapter->mcastFilterSize)) { 1749 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address, 1750 IbmVethMcastEnableRecv | 1751 IbmVethMcastDisableFiltering, 1752 0); 1753 if (lpar_rc != H_SUCCESS) { 1754 netdev_err(netdev, "h_multicast_ctrl rc=%ld when " 1755 "entering promisc mode\n", lpar_rc); 1756 } 1757 } else { 1758 struct netdev_hw_addr *ha; 1759 /* clear the filter table & disable filtering */ 1760 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address, 1761 IbmVethMcastEnableRecv | 1762 IbmVethMcastDisableFiltering | 1763 IbmVethMcastClearFilterTable, 1764 0); 1765 if (lpar_rc != H_SUCCESS) { 1766 netdev_err(netdev, "h_multicast_ctrl rc=%ld when " 1767 "attempting to clear filter table\n", 1768 lpar_rc); 1769 } 1770 /* add the addresses to the filter table */ 1771 netdev_for_each_mc_addr(ha, netdev) { 1772 /* add the multicast address to the filter table */ 1773 u64 mcast_addr; 1774 mcast_addr = ether_addr_to_u64(ha->addr); 1775 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address, 1776 IbmVethMcastAddFilter, 1777 mcast_addr); 1778 if (lpar_rc != H_SUCCESS) { 1779 netdev_err(netdev, "h_multicast_ctrl rc=%ld " 1780 "when adding an entry to the filter " 1781 "table\n", lpar_rc); 1782 } 1783 } 1784 1785 /* re-enable filtering */ 1786 lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address, 1787 IbmVethMcastEnableFiltering, 1788 0); 1789 if (lpar_rc != H_SUCCESS) { 1790 netdev_err(netdev, "h_multicast_ctrl rc=%ld when " 1791 "enabling filtering\n", lpar_rc); 1792 } 1793 } 1794 } 1795 1796 static int ibmveth_change_mtu(struct net_device *dev, int new_mtu) 1797 { 1798 struct ibmveth_adapter *adapter = netdev_priv(dev); 1799 struct vio_dev *viodev = adapter->vdev; 1800 int new_mtu_oh = new_mtu + IBMVETH_BUFF_OH; 1801 int i, rc; 1802 int need_restart = 0; 1803 1804 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 1805 if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) 1806 break; 1807 1808 if (i == IBMVETH_NUM_BUFF_POOLS) 1809 return -EINVAL; 1810 1811 /* Deactivate all the buffer pools so that the next loop can activate 1812 only the buffer pools necessary to hold the new MTU */ 1813 if (netif_running(adapter->netdev)) { 1814 need_restart = 1; 1815 ibmveth_close(adapter->netdev); 1816 } 1817 1818 /* Look for an active buffer pool that can hold the new MTU */ 1819 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { 1820 adapter->rx_buff_pool[i].active = 1; 1821 1822 if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) { 1823 WRITE_ONCE(dev->mtu, new_mtu); 1824 vio_cmo_set_dev_desired(viodev, 1825 ibmveth_get_desired_dma 1826 (viodev)); 1827 if (need_restart) { 1828 return ibmveth_open(adapter->netdev); 1829 } 1830 return 0; 1831 } 1832 } 1833 1834 if (need_restart && (rc = ibmveth_open(adapter->netdev))) 1835 return rc; 1836 1837 return -EINVAL; 1838 } 1839 1840 /** 1841 * ibmveth_get_desired_dma - Calculate IO memory desired by the driver 1842 * 1843 * @vdev: struct vio_dev for the device whose desired IO mem is to be returned 1844 * 1845 * Return value: 1846 * Number of bytes of IO data the driver will need to perform well. 1847 */ 1848 static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev) 1849 { 1850 struct net_device *netdev = dev_get_drvdata(&vdev->dev); 1851 struct ibmveth_adapter *adapter; 1852 struct iommu_table *tbl; 1853 unsigned long ret; 1854 int i; 1855 int rxqentries = 1; 1856 1857 tbl = get_iommu_table_base(&vdev->dev); 1858 1859 /* netdev inits at probe time along with the structures we need below*/ 1860 if (netdev == NULL) 1861 return IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl); 1862 1863 adapter = netdev_priv(netdev); 1864 1865 ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE; 1866 ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl); 1867 /* add size of mapped tx buffers */ 1868 ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl); 1869 1870 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { 1871 /* add the size of the active receive buffers */ 1872 if (adapter->rx_buff_pool[i].active) 1873 ret += 1874 adapter->rx_buff_pool[i].size * 1875 IOMMU_PAGE_ALIGN(adapter->rx_buff_pool[i]. 1876 buff_size, tbl); 1877 rxqentries += adapter->rx_buff_pool[i].size; 1878 } 1879 /* add the size of the receive queue entries */ 1880 ret += IOMMU_PAGE_ALIGN( 1881 rxqentries * sizeof(struct ibmveth_rx_q_entry), tbl); 1882 1883 return ret; 1884 } 1885 1886 static int ibmveth_set_mac_addr(struct net_device *dev, void *p) 1887 { 1888 struct ibmveth_adapter *adapter = netdev_priv(dev); 1889 struct sockaddr *addr = p; 1890 u64 mac_address; 1891 int rc; 1892 1893 if (!is_valid_ether_addr(addr->sa_data)) 1894 return -EADDRNOTAVAIL; 1895 1896 mac_address = ether_addr_to_u64(addr->sa_data); 1897 rc = h_change_logical_lan_mac(adapter->vdev->unit_address, mac_address); 1898 if (rc) { 1899 netdev_err(adapter->netdev, "h_change_logical_lan_mac failed with rc=%d\n", rc); 1900 return rc; 1901 } 1902 1903 eth_hw_addr_set(dev, addr->sa_data); 1904 1905 return 0; 1906 } 1907 1908 static netdev_features_t ibmveth_features_check(struct sk_buff *skb, 1909 struct net_device *dev, 1910 netdev_features_t features) 1911 { 1912 /* Some physical adapters do not support segmentation offload with 1913 * MSS < 224. Disable GSO for such packets to avoid adapter freeze. 1914 * Note: Single-segment packets (gso_segs == 1) don't need this check 1915 * as they bypass the LSO path and are transmitted without segmentation. 1916 */ 1917 if (skb_is_gso(skb)) { 1918 if (skb_shinfo(skb)->gso_size < IBMVETH_MIN_LSO_MSS) { 1919 netdev_warn_once(dev, 1920 "MSS %u too small for LSO, disabling GSO\n", 1921 skb_shinfo(skb)->gso_size); 1922 features &= ~NETIF_F_GSO_MASK; 1923 } 1924 } 1925 1926 return vlan_features_check(skb, features); 1927 } 1928 1929 static const struct net_device_ops ibmveth_netdev_ops = { 1930 .ndo_open = ibmveth_open, 1931 .ndo_stop = ibmveth_close, 1932 .ndo_start_xmit = ibmveth_start_xmit, 1933 .ndo_set_rx_mode = ibmveth_set_multicast_list, 1934 .ndo_eth_ioctl = ibmveth_ioctl, 1935 .ndo_change_mtu = ibmveth_change_mtu, 1936 .ndo_fix_features = ibmveth_fix_features, 1937 .ndo_set_features = ibmveth_set_features, 1938 .ndo_validate_addr = eth_validate_addr, 1939 .ndo_set_mac_address = ibmveth_set_mac_addr, 1940 .ndo_features_check = ibmveth_features_check, 1941 }; 1942 1943 /** 1944 * ibmveth_pool_kobj_release - Mark a pool kobject finished 1945 * @kobj: kobject embedded in the pool 1946 * 1947 * The pool kobjects live in netdev_priv(), so the last put must wait 1948 * for this before free_netdev(). 1949 */ 1950 static void ibmveth_pool_kobj_release(struct kobject *kobj) 1951 { 1952 struct ibmveth_buff_pool *pool = container_of(kobj, 1953 struct ibmveth_buff_pool, 1954 kobj); 1955 1956 complete(&pool->released); 1957 } 1958 1959 /** 1960 * ibmveth_put_pool_kobjs - Drop the pool kobjects and wait for release 1961 * @adapter: ibmveth adapter 1962 * 1963 * With CONFIG_DEBUG_KOBJECT_RELEASE the cleanup, including removing the 1964 * sysfs files, runs later from a work item in the kobject; wait for it 1965 * so free_netdev() cannot free the pools first. 1966 */ 1967 static void ibmveth_put_pool_kobjs(struct ibmveth_adapter *adapter) 1968 { 1969 int i; 1970 1971 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 1972 kobject_put(&adapter->rx_buff_pool[i].kobj); 1973 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 1974 wait_for_completion(&adapter->rx_buff_pool[i].released); 1975 } 1976 1977 static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id) 1978 { 1979 int rc, i, mac_len; 1980 struct net_device *netdev; 1981 struct ibmveth_adapter *adapter; 1982 unsigned char *mac_addr_p; 1983 __be32 *mcastFilterSize_p; 1984 long ret; 1985 unsigned long ret_attr; 1986 1987 dev_dbg(&dev->dev, "entering ibmveth_probe for UA 0x%x\n", 1988 dev->unit_address); 1989 1990 mac_addr_p = (unsigned char *)vio_get_attribute(dev, VETH_MAC_ADDR, 1991 &mac_len); 1992 if (!mac_addr_p) { 1993 dev_err(&dev->dev, "Can't find VETH_MAC_ADDR attribute\n"); 1994 return -EINVAL; 1995 } 1996 /* Workaround for old/broken pHyp */ 1997 if (mac_len == 8) 1998 mac_addr_p += 2; 1999 else if (mac_len != 6) { 2000 dev_err(&dev->dev, "VETH_MAC_ADDR attribute wrong len %d\n", 2001 mac_len); 2002 return -EINVAL; 2003 } 2004 2005 mcastFilterSize_p = (__be32 *)vio_get_attribute(dev, 2006 VETH_MCAST_FILTER_SIZE, 2007 NULL); 2008 if (!mcastFilterSize_p) { 2009 dev_err(&dev->dev, "Can't find VETH_MCAST_FILTER_SIZE " 2010 "attribute\n"); 2011 return -EINVAL; 2012 } 2013 2014 netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1); 2015 if (!netdev) 2016 return -ENOMEM; 2017 2018 adapter = netdev_priv(netdev); 2019 dev_set_drvdata(&dev->dev, netdev); 2020 2021 adapter->vdev = dev; 2022 adapter->netdev = netdev; 2023 INIT_WORK(&adapter->work, ibmveth_reset); 2024 adapter->mcastFilterSize = be32_to_cpu(*mcastFilterSize_p); 2025 ibmveth_init_link_settings(netdev); 2026 2027 netif_napi_add_weight(netdev, &adapter->napi, ibmveth_poll, 16); 2028 2029 netdev->irq = dev->irq; 2030 netdev->netdev_ops = &ibmveth_netdev_ops; 2031 netdev->ethtool_ops = &netdev_ethtool_ops; 2032 SET_NETDEV_DEV(netdev, &dev->dev); 2033 netdev->hw_features = NETIF_F_SG; 2034 if (vio_get_attribute(dev, "ibm,illan-options", NULL) != NULL) { 2035 netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 2036 NETIF_F_RXCSUM; 2037 } 2038 2039 netdev->features |= netdev->hw_features; 2040 2041 ret = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr); 2042 2043 /* If running older firmware, TSO should not be enabled by default */ 2044 if (ret == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_LRG_SND_SUPPORT) && 2045 !old_large_send) { 2046 netdev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 2047 netdev->features |= netdev->hw_features; 2048 } else { 2049 netdev->hw_features |= NETIF_F_TSO; 2050 } 2051 2052 adapter->is_active_trunk = false; 2053 if (ret == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_ACTIVE_TRUNK)) { 2054 adapter->is_active_trunk = true; 2055 netdev->hw_features |= NETIF_F_FRAGLIST; 2056 netdev->features |= NETIF_F_FRAGLIST; 2057 } 2058 2059 if (ret == H_SUCCESS && 2060 (ret_attr & IBMVETH_ILLAN_RX_MULTI_BUFF_SUPPORT)) { 2061 adapter->rx_buffers_per_hcall = IBMVETH_MAX_RX_PER_HCALL; 2062 netdev_dbg(netdev, 2063 "RX Multi-buffer hcall supported by FW, batch set to %u\n", 2064 adapter->rx_buffers_per_hcall); 2065 } else { 2066 adapter->rx_buffers_per_hcall = 1; 2067 netdev_dbg(netdev, 2068 "RX Single-buffer hcall mode, batch set to %u\n", 2069 adapter->rx_buffers_per_hcall); 2070 } 2071 2072 netdev->min_mtu = IBMVETH_MIN_MTU; 2073 netdev->max_mtu = ETH_MAX_MTU - IBMVETH_BUFF_OH; 2074 2075 eth_hw_addr_set(netdev, mac_addr_p); 2076 2077 if (firmware_has_feature(FW_FEATURE_CMO)) 2078 memcpy(pool_count, pool_count_cmo, sizeof(pool_count)); 2079 2080 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { 2081 struct kobject *kobj = &adapter->rx_buff_pool[i].kobj; 2082 int error; 2083 2084 ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, 2085 pool_count[i], pool_size[i], 2086 pool_active[i]); 2087 init_completion(&adapter->rx_buff_pool[i].released); 2088 error = kobject_init_and_add(kobj, &ktype_veth_pool, 2089 &dev->dev.kobj, "pool%d", i); 2090 if (!error) 2091 kobject_uevent(kobj, KOBJ_ADD); 2092 } 2093 2094 rc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(), 2095 IBMVETH_DEFAULT_QUEUES)); 2096 if (rc) { 2097 netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n", 2098 rc); 2099 goto err_put_pools; 2100 } 2101 adapter->tx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE); 2102 for (i = 0; i < IBMVETH_MAX_QUEUES; i++) 2103 adapter->tx_ltb_ptr[i] = NULL; 2104 2105 netdev_dbg(netdev, "adapter @ 0x%p\n", adapter); 2106 netdev_dbg(netdev, "registering netdev...\n"); 2107 2108 ibmveth_set_features(netdev, netdev->features); 2109 2110 rc = register_netdev(netdev); 2111 2112 if (rc) { 2113 netdev_dbg(netdev, "failed to register netdev rc=%d\n", rc); 2114 goto err_put_pools; 2115 } 2116 2117 netdev_dbg(netdev, "registered\n"); 2118 2119 return 0; 2120 2121 err_put_pools: 2122 ibmveth_put_pool_kobjs(adapter); 2123 free_netdev(netdev); 2124 return rc; 2125 } 2126 2127 static void ibmveth_remove(struct vio_dev *dev) 2128 { 2129 struct net_device *netdev = dev_get_drvdata(&dev->dev); 2130 struct ibmveth_adapter *adapter = netdev_priv(netdev); 2131 2132 disable_work_sync(&adapter->work); 2133 2134 ibmveth_put_pool_kobjs(adapter); 2135 2136 unregister_netdev(netdev); 2137 2138 free_netdev(netdev); 2139 dev_set_drvdata(&dev->dev, NULL); 2140 } 2141 2142 static struct attribute veth_active_attr; 2143 static struct attribute veth_num_attr; 2144 static struct attribute veth_size_attr; 2145 2146 static ssize_t veth_pool_show(struct kobject *kobj, 2147 struct attribute *attr, char *buf) 2148 { 2149 struct ibmveth_buff_pool *pool = container_of(kobj, 2150 struct ibmveth_buff_pool, 2151 kobj); 2152 2153 if (attr == &veth_active_attr) 2154 return sprintf(buf, "%d\n", pool->active); 2155 else if (attr == &veth_num_attr) 2156 return sprintf(buf, "%d\n", pool->size); 2157 else if (attr == &veth_size_attr) 2158 return sprintf(buf, "%d\n", pool->buff_size); 2159 return 0; 2160 } 2161 2162 /** 2163 * veth_pool_store - sysfs store handler for pool attributes 2164 * @kobj: kobject embedded in pool 2165 * @attr: attribute being changed 2166 * @buf: value being stored 2167 * @count: length of @buf in bytes 2168 * 2169 * Stores new value in pool attribute. Verifies the range of the new value for 2170 * size and buff_size. Verifies that at least one pool remains available to 2171 * receive MTU-sized packets. 2172 * 2173 * Context: Process context. 2174 * Takes and releases rtnl_mutex to ensure correct ordering of close 2175 * and open calls. 2176 * Return: 2177 * * %-EPERM - Not allowed to disabled all MTU-sized buffer pools 2178 * * %-EINVAL - New pool size or buffer size is out of range 2179 * * count - Return count for success 2180 * * other - Return value from a failed ibmveth_open call 2181 */ 2182 static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr, 2183 const char *buf, size_t count) 2184 { 2185 struct ibmveth_buff_pool *pool = container_of(kobj, 2186 struct ibmveth_buff_pool, 2187 kobj); 2188 struct net_device *netdev = dev_get_drvdata(kobj_to_dev(kobj->parent)); 2189 struct ibmveth_adapter *adapter = netdev_priv(netdev); 2190 long value = simple_strtol(buf, NULL, 10); 2191 bool change = false; 2192 u32 newbuff_size; 2193 u32 oldbuff_size; 2194 int newactive; 2195 int oldactive; 2196 u32 newsize; 2197 u32 oldsize; 2198 long rc; 2199 2200 rtnl_lock(); 2201 2202 oldbuff_size = pool->buff_size; 2203 oldactive = pool->active; 2204 oldsize = pool->size; 2205 2206 newbuff_size = oldbuff_size; 2207 newactive = oldactive; 2208 newsize = oldsize; 2209 2210 if (attr == &veth_active_attr) { 2211 if (value && !oldactive) { 2212 newactive = 1; 2213 change = true; 2214 } else if (!value && oldactive) { 2215 int mtu = netdev->mtu + IBMVETH_BUFF_OH; 2216 int i; 2217 /* Make sure there is a buffer pool with buffers that 2218 can hold a packet of the size of the MTU */ 2219 for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) { 2220 if (pool == &adapter->rx_buff_pool[i]) 2221 continue; 2222 if (!adapter->rx_buff_pool[i].active) 2223 continue; 2224 if (mtu <= adapter->rx_buff_pool[i].buff_size) 2225 break; 2226 } 2227 2228 if (i == IBMVETH_NUM_BUFF_POOLS) { 2229 netdev_err(netdev, "no active pool >= MTU\n"); 2230 rc = -EPERM; 2231 goto unlock_err; 2232 } 2233 2234 newactive = 0; 2235 change = true; 2236 } 2237 } else if (attr == &veth_num_attr) { 2238 if (value <= 0 || value > IBMVETH_MAX_POOL_COUNT) { 2239 rc = -EINVAL; 2240 goto unlock_err; 2241 } 2242 if (value != oldsize) { 2243 newsize = value; 2244 change = true; 2245 } 2246 } else if (attr == &veth_size_attr) { 2247 if (value <= IBMVETH_BUFF_OH || value > IBMVETH_MAX_BUF_SIZE) { 2248 rc = -EINVAL; 2249 goto unlock_err; 2250 } 2251 if (value != oldbuff_size) { 2252 newbuff_size = value; 2253 change = true; 2254 } 2255 } 2256 2257 if (change) { 2258 if (netif_running(netdev)) 2259 ibmveth_close(netdev); 2260 2261 pool->active = newactive; 2262 pool->buff_size = newbuff_size; 2263 pool->size = newsize; 2264 2265 if (netif_running(netdev)) { 2266 rc = ibmveth_open(netdev); 2267 if (rc) { 2268 pool->active = oldactive; 2269 pool->buff_size = oldbuff_size; 2270 pool->size = oldsize; 2271 goto unlock_err; 2272 } 2273 } 2274 } 2275 rtnl_unlock(); 2276 2277 /* kick the interrupt handler to allocate/deallocate pools */ 2278 ibmveth_interrupt(netdev->irq, netdev); 2279 return count; 2280 2281 unlock_err: 2282 rtnl_unlock(); 2283 return rc; 2284 } 2285 2286 2287 #define ATTR(_name, _mode) \ 2288 struct attribute veth_##_name##_attr = { \ 2289 .name = __stringify(_name), .mode = _mode, \ 2290 }; 2291 2292 static ATTR(active, 0644); 2293 static ATTR(num, 0644); 2294 static ATTR(size, 0644); 2295 2296 static struct attribute *veth_pool_attrs[] = { 2297 &veth_active_attr, 2298 &veth_num_attr, 2299 &veth_size_attr, 2300 NULL, 2301 }; 2302 ATTRIBUTE_GROUPS(veth_pool); 2303 2304 static const struct sysfs_ops veth_pool_ops = { 2305 .show = veth_pool_show, 2306 .store = veth_pool_store, 2307 }; 2308 2309 static struct kobj_type ktype_veth_pool = { 2310 .release = ibmveth_pool_kobj_release, 2311 .sysfs_ops = &veth_pool_ops, 2312 .default_groups = veth_pool_groups, 2313 }; 2314 2315 static int ibmveth_resume(struct device *dev) 2316 { 2317 struct net_device *netdev = dev_get_drvdata(dev); 2318 ibmveth_interrupt(netdev->irq, netdev); 2319 return 0; 2320 } 2321 2322 static const struct vio_device_id ibmveth_device_table[] = { 2323 { "network", "IBM,l-lan"}, 2324 { "", "" } 2325 }; 2326 MODULE_DEVICE_TABLE(vio, ibmveth_device_table); 2327 2328 static const struct dev_pm_ops ibmveth_pm_ops = { 2329 .resume = ibmveth_resume 2330 }; 2331 2332 static struct vio_driver ibmveth_driver = { 2333 .id_table = ibmveth_device_table, 2334 .probe = ibmveth_probe, 2335 .remove = ibmveth_remove, 2336 .get_desired_dma = ibmveth_get_desired_dma, 2337 .name = ibmveth_driver_name, 2338 .pm = &ibmveth_pm_ops, 2339 }; 2340 2341 static int __init ibmveth_module_init(void) 2342 { 2343 printk(KERN_DEBUG "%s: %s %s\n", ibmveth_driver_name, 2344 ibmveth_driver_string, ibmveth_driver_version); 2345 2346 return vio_register_driver(&ibmveth_driver); 2347 } 2348 2349 static void __exit ibmveth_module_exit(void) 2350 { 2351 vio_unregister_driver(&ibmveth_driver); 2352 } 2353 2354 module_init(ibmveth_module_init); 2355 module_exit(ibmveth_module_exit); 2356 2357 #ifdef CONFIG_IBMVETH_KUNIT_TEST 2358 #include <kunit/test.h> 2359 2360 /** 2361 * ibmveth_reset_kunit - reset routine for running in KUnit environment 2362 * 2363 * @w: pointer to work_struct embedded in adapter structure 2364 * 2365 * Context: Called in the KUnit environment. Does nothing. 2366 * 2367 * Return: void 2368 */ 2369 static void ibmveth_reset_kunit(struct work_struct *w) 2370 { 2371 netdev_dbg(NULL, "reset_kunit starting\n"); 2372 netdev_dbg(NULL, "reset_kunit complete\n"); 2373 } 2374 2375 /** 2376 * ibmveth_remove_buffer_from_pool_test - unit test for some of 2377 * ibmveth_remove_buffer_from_pool 2378 * @test: pointer to kunit structure 2379 * 2380 * Tests the error returns from ibmveth_remove_buffer_from_pool. 2381 * Each error also logs a ratelimited netdev_err. 2382 * 2383 * Return: void 2384 */ 2385 static void ibmveth_remove_buffer_from_pool_test(struct kunit *test) 2386 { 2387 struct ibmveth_adapter *adapter = kunit_kzalloc(test, sizeof(*adapter), GFP_KERNEL); 2388 struct ibmveth_buff_pool *pool; 2389 u64 correlator; 2390 int ret; 2391 2392 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter); 2393 2394 INIT_WORK(&adapter->work, ibmveth_reset_kunit); 2395 2396 /* Set sane values for buffer pools */ 2397 for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 2398 ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, 2399 pool_count[i], pool_size[i], 2400 pool_active[i]); 2401 2402 pool = &adapter->rx_buff_pool[0]; 2403 pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void *), GFP_KERNEL); 2404 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff); 2405 2406 correlator = ((u64)IBMVETH_NUM_BUFF_POOLS << 32) | 0; 2407 KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); 2408 KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); 2409 2410 correlator = ((u64)0 << 32) | adapter->rx_buff_pool[0].size; 2411 KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); 2412 KUNIT_EXPECT_EQ(test, -EINVAL, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); 2413 2414 /* Pool 2 is in range but has no skbuff array, like an inactive pool. */ 2415 correlator = ((u64)2 << 32) | 0; 2416 ret = ibmveth_remove_buffer_from_pool(adapter, correlator, false); 2417 KUNIT_EXPECT_EQ(test, -EINVAL, ret); 2418 ret = ibmveth_remove_buffer_from_pool(adapter, correlator, true); 2419 KUNIT_EXPECT_EQ(test, -EINVAL, ret); 2420 2421 correlator = (u64)0 | 0; 2422 pool->skbuff[0] = NULL; 2423 KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, false)); 2424 KUNIT_EXPECT_EQ(test, -EFAULT, ibmveth_remove_buffer_from_pool(adapter, correlator, true)); 2425 2426 flush_work(&adapter->work); 2427 } 2428 2429 /** 2430 * ibmveth_rxq_get_buffer_test - unit test for ibmveth_rxq_get_buffer 2431 * @test: pointer to kunit structure 2432 * 2433 * Tests ibmveth_rxq_get_buffer. Each NULL return also logs a ratelimited 2434 * netdev_err. 2435 * 2436 * Return: void 2437 */ 2438 static void ibmveth_rxq_get_buffer_test(struct kunit *test) 2439 { 2440 struct ibmveth_adapter *adapter = kunit_kzalloc(test, sizeof(*adapter), GFP_KERNEL); 2441 struct sk_buff *skb = kunit_kzalloc(test, sizeof(*skb), GFP_KERNEL); 2442 struct ibmveth_buff_pool *pool; 2443 u64 cor; 2444 2445 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter); 2446 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, skb); 2447 2448 INIT_WORK(&adapter->work, ibmveth_reset_kunit); 2449 2450 adapter->rx_queue.queue_len = 1; 2451 adapter->rx_queue.index = 0; 2452 adapter->rx_queue.queue_addr = kunit_kzalloc(test, sizeof(struct ibmveth_rx_q_entry), 2453 GFP_KERNEL); 2454 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter->rx_queue.queue_addr); 2455 2456 /* Set sane values for buffer pools */ 2457 for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 2458 ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, 2459 pool_count[i], pool_size[i], 2460 pool_active[i]); 2461 2462 pool = &adapter->rx_buff_pool[0]; 2463 pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void *), GFP_KERNEL); 2464 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff); 2465 2466 adapter->rx_queue.queue_addr[0].correlator = (u64)IBMVETH_NUM_BUFF_POOLS << 32 | 0; 2467 cor = ibmveth_rxq_correlator(adapter); 2468 KUNIT_EXPECT_PTR_EQ(test, NULL, 2469 ibmveth_rxq_get_buffer(adapter, cor)); 2470 2471 adapter->rx_queue.queue_addr[0].correlator = (u64)0 << 32 | adapter->rx_buff_pool[0].size; 2472 cor = ibmveth_rxq_correlator(adapter); 2473 KUNIT_EXPECT_PTR_EQ(test, NULL, 2474 ibmveth_rxq_get_buffer(adapter, cor)); 2475 2476 /* Pool 2 is in range but has no skbuff array, like an inactive pool. */ 2477 adapter->rx_queue.queue_addr[0].correlator = (u64)2 << 32 | 0; 2478 cor = ibmveth_rxq_correlator(adapter); 2479 KUNIT_EXPECT_PTR_EQ(test, NULL, 2480 ibmveth_rxq_get_buffer(adapter, cor)); 2481 2482 pool->skbuff[0] = skb; 2483 adapter->rx_queue.queue_addr[0].correlator = (u64)0 << 32 | 0; 2484 cor = ibmveth_rxq_correlator(adapter); 2485 KUNIT_EXPECT_PTR_EQ(test, skb, 2486 ibmveth_rxq_get_buffer(adapter, cor)); 2487 2488 flush_work(&adapter->work); 2489 } 2490 2491 /** 2492 * ibmveth_rxq_harvest_buffer_test - unit test for ibmveth_rxq_harvest_buffer 2493 * @test: pointer to kunit structure 2494 * 2495 * A bad correlator must still advance the RX ring, wrapping and flipping 2496 * the toggle at the end. This covers the harvest path; the advance after 2497 * ibmveth_rxq_get_buffer() fails in ibmveth_poll() is not tested here. 2498 * 2499 * Return: void 2500 */ 2501 static void ibmveth_rxq_harvest_buffer_test(struct kunit *test) 2502 { 2503 struct ibmveth_adapter *adapter; 2504 struct ibmveth_buff_pool *pool; 2505 u64 cor; 2506 int ret; 2507 2508 adapter = kunit_kzalloc(test, sizeof(*adapter), GFP_KERNEL); 2509 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter); 2510 2511 INIT_WORK(&adapter->work, ibmveth_reset_kunit); 2512 2513 adapter->rx_queue.num_slots = 2; 2514 adapter->rx_queue.index = 0; 2515 adapter->rx_queue.toggle = 1; 2516 adapter->rx_queue.queue_addr = 2517 kunit_kcalloc(test, 2, sizeof(struct ibmveth_rx_q_entry), 2518 GFP_KERNEL); 2519 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, adapter->rx_queue.queue_addr); 2520 2521 /* Set sane values for buffer pools */ 2522 for (int i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) 2523 ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i, 2524 pool_count[i], pool_size[i], 2525 pool_active[i]); 2526 2527 pool = &adapter->rx_buff_pool[0]; 2528 pool->skbuff = kunit_kcalloc(test, pool->size, sizeof(void *), 2529 GFP_KERNEL); 2530 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, pool->skbuff); 2531 2532 /* Slot 0: pool out of range. Slot 1: valid, but no skb. */ 2533 adapter->rx_queue.queue_addr[0].correlator = 2534 (u64)IBMVETH_NUM_BUFF_POOLS << 32 | 0; 2535 adapter->rx_queue.queue_addr[1].correlator = (u64)0 << 32 | 0; 2536 2537 cor = ibmveth_rxq_correlator(adapter); 2538 ret = ibmveth_rxq_harvest_buffer(adapter, cor, true); 2539 KUNIT_EXPECT_EQ(test, -EINVAL, ret); 2540 KUNIT_EXPECT_EQ(test, 1ULL, adapter->rx_queue.index); 2541 KUNIT_EXPECT_EQ(test, 1ULL, adapter->rx_queue.toggle); 2542 2543 cor = ibmveth_rxq_correlator(adapter); 2544 ret = ibmveth_rxq_harvest_buffer(adapter, cor, true); 2545 KUNIT_EXPECT_EQ(test, -EFAULT, ret); 2546 KUNIT_EXPECT_EQ(test, 0ULL, adapter->rx_queue.index); 2547 KUNIT_EXPECT_EQ(test, 0ULL, adapter->rx_queue.toggle); 2548 2549 flush_work(&adapter->work); 2550 } 2551 2552 KUNIT_DEFINE_ACTION_WRAPPER(ibmveth_kfree_skb, kfree_skb, struct sk_buff *); 2553 2554 /* A TCP frame after eth_type_trans(): IP header, 20 byte TCP header and 2555 * payload bytes of data, with the IP length matching the frame. 2556 */ 2557 static struct sk_buff *ibmveth_test_tcp_skb(struct kunit *test, bool ipv6, 2558 unsigned int payload) 2559 { 2560 unsigned int iplen = ipv6 ? sizeof(struct ipv6hdr) : 2561 sizeof(struct iphdr); 2562 unsigned int len = iplen + sizeof(struct tcphdr) + payload; 2563 struct sk_buff *skb = alloc_skb(len, GFP_KERNEL); 2564 struct tcphdr *tcph; 2565 2566 KUNIT_ASSERT_NOT_ERR_OR_NULL(test, skb); 2567 KUNIT_ASSERT_EQ(test, 0, kunit_add_action_or_reset(test, 2568 ibmveth_kfree_skb, 2569 skb)); 2570 skb_put_zero(skb, len); 2571 if (ipv6) { 2572 struct ipv6hdr *iph6 = (struct ipv6hdr *)skb->data; 2573 2574 skb->protocol = htons(ETH_P_IPV6); 2575 iph6->version = 6; 2576 iph6->nexthdr = IPPROTO_TCP; 2577 iph6->payload_len = htons(len - iplen); 2578 } else { 2579 struct iphdr *iph = (struct iphdr *)skb->data; 2580 2581 skb->protocol = htons(ETH_P_IP); 2582 iph->version = 4; 2583 iph->ihl = 5; 2584 iph->protocol = IPPROTO_TCP; 2585 iph->tot_len = htons(len); 2586 } 2587 tcph = (struct tcphdr *)(skb->data + iplen); 2588 tcph->doff = 5; 2589 return skb; 2590 } 2591 2592 /** 2593 * ibmveth_rx_mss_helper_test - unit test for ibmveth_rx_mss_helper 2594 * @test: pointer to kunit structure 2595 * 2596 * Large send frames get gso_size from the descriptor or the TCP checksum 2597 * field, clamped, and gso_segs from the IP length. Frames whose headers 2598 * do not fit, whose IP length is not the frame length, with no TCP 2599 * payload within the IP length, or that are IPv4 fragments, are refused 2600 * and left without GSO. 2601 * 2602 * Return: void 2603 */ 2604 static void ibmveth_rx_mss_helper_test(struct kunit *test) 2605 { 2606 struct sk_buff *skb; 2607 struct tcphdr *tcph; 2608 struct iphdr *iph; 2609 2610 skb = ibmveth_test_tcp_skb(test, false, 2896); 2611 tcph = (struct tcphdr *)(skb->data + sizeof(*iph)); 2612 tcph->check = (__force __sum16)htons(1448); 2613 KUNIT_EXPECT_TRUE(test, ibmveth_rx_mss_helper(skb, 0, 0)); 2614 KUNIT_EXPECT_EQ(test, SKB_GSO_TCPV4, skb_shinfo(skb)->gso_type); 2615 KUNIT_EXPECT_EQ(test, 1448, skb_shinfo(skb)->gso_size); 2616 KUNIT_EXPECT_EQ(test, 2, skb_shinfo(skb)->gso_segs); 2617 KUNIT_EXPECT_EQ(test, 0, (__force u16)tcph->check); 2618 2619 skb = ibmveth_test_tcp_skb(test, false, 2896); 2620 KUNIT_EXPECT_TRUE(test, ibmveth_rx_mss_helper(skb, 1000, 1)); 2621 KUNIT_EXPECT_EQ(test, 1000, skb_shinfo(skb)->gso_size); 2622 KUNIT_EXPECT_EQ(test, 3, skb_shinfo(skb)->gso_segs); 2623 2624 skb = ibmveth_test_tcp_skb(test, true, 2896); 2625 tcph = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr)); 2626 tcph->check = (__force __sum16)htons(1448); 2627 KUNIT_EXPECT_TRUE(test, ibmveth_rx_mss_helper(skb, 0, 0)); 2628 KUNIT_EXPECT_EQ(test, SKB_GSO_TCPV6, skb_shinfo(skb)->gso_type); 2629 KUNIT_EXPECT_EQ(test, 1448, skb_shinfo(skb)->gso_size); 2630 KUNIT_EXPECT_EQ(test, 2, skb_shinfo(skb)->gso_segs); 2631 2632 skb = ibmveth_test_tcp_skb(test, false, 2896); 2633 KUNIT_EXPECT_TRUE(test, ibmveth_rx_mss_helper(skb, 1, 1)); 2634 KUNIT_EXPECT_EQ(test, TCP_MIN_GSO_SIZE, skb_shinfo(skb)->gso_size); 2635 2636 skb = ibmveth_test_tcp_skb(test, false, 2896); 2637 KUNIT_EXPECT_TRUE(test, ibmveth_rx_mss_helper(skb, GSO_BY_FRAGS, 1)); 2638 KUNIT_EXPECT_EQ(test, GSO_BY_FRAGS - 1, skb_shinfo(skb)->gso_size); 2639 2640 /* refused: ihl 0, doff 4, IP length past the frame. With ihl 0 the 2641 * TCP header would overlap the IP header; saddr makes its doff 5. 2642 */ 2643 skb = ibmveth_test_tcp_skb(test, false, 2896); 2644 ((struct iphdr *)skb->data)->ihl = 0; 2645 ((struct iphdr *)skb->data)->saddr = htonl(0x50000000); 2646 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2647 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2648 2649 skb = ibmveth_test_tcp_skb(test, false, 2896); 2650 ((struct tcphdr *)(skb->data + sizeof(*iph)))->doff = 4; 2651 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2652 2653 skb = ibmveth_test_tcp_skb(test, false, 2896); 2654 ((struct iphdr *)skb->data)->tot_len = htons(skb->len + 1); 2655 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2656 2657 /* refused: bytes past the IP length, its large send fields left 2658 * alone; gso_segs from the IP length would undercount them 2659 */ 2660 skb = ibmveth_test_tcp_skb(test, false, 2896); 2661 ((struct iphdr *)skb->data)->tot_len = htons(skb->len - 100); 2662 tcph = (struct tcphdr *)(skb->data + sizeof(*iph)); 2663 tcph->check = (__force __sum16)htons(1448); 2664 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 0, 0)); 2665 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2666 KUNIT_EXPECT_EQ(test, 1448, ntohs((__force __be16)tcph->check)); 2667 2668 /* refused: a fragment past the first, its payload left alone */ 2669 skb = ibmveth_test_tcp_skb(test, false, 2896); 2670 ((struct iphdr *)skb->data)->frag_off = htons(185); 2671 tcph = (struct tcphdr *)(skb->data + sizeof(*iph)); 2672 tcph->check = (__force __sum16)htons(1448); 2673 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 0, 0)); 2674 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2675 KUNIT_EXPECT_EQ(test, 1448, ntohs((__force __be16)tcph->check)); 2676 2677 /* refused: only Ethernet padding past the TCP header */ 2678 skb = ibmveth_test_tcp_skb(test, false, 6); 2679 ((struct iphdr *)skb->data)->tot_len = htons(40); 2680 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2681 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2682 2683 skb = ibmveth_test_tcp_skb(test, true, 2896); 2684 ((struct ipv6hdr *)skb->data)->payload_len = 0; 2685 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2686 2687 /* refused: no TCP payload, with the IP length matching the frame */ 2688 skb = ibmveth_test_tcp_skb(test, false, 0); 2689 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2690 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2691 2692 skb = ibmveth_test_tcp_skb(test, true, 0); 2693 KUNIT_EXPECT_FALSE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2694 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2695 2696 /* not TCP: accepted and left alone */ 2697 skb = ibmveth_test_tcp_skb(test, false, 2896); 2698 ((struct iphdr *)skb->data)->protocol = IPPROTO_UDP; 2699 KUNIT_EXPECT_TRUE(test, ibmveth_rx_mss_helper(skb, 1448, 1)); 2700 KUNIT_EXPECT_EQ(test, 0, skb_shinfo(skb)->gso_size); 2701 } 2702 2703 /** 2704 * ibmveth_rx_csum_helper_test - unit test for ibmveth_rx_csum_helper 2705 * @test: pointer to kunit structure 2706 * 2707 * A large send IPv4 header checksum of 0xffff is rebuilt. Headers that 2708 * do not fit the frame, an IPv4 ihl below 5, or a TCP fragment, clear 2709 * CHECKSUM_UNNECESSARY instead of being read. 2710 * 2711 * Return: void 2712 */ 2713 static void ibmveth_rx_csum_helper_test(struct kunit *test) 2714 { 2715 struct sk_buff *skb; 2716 struct tcphdr *tcph; 2717 struct iphdr *iph; 2718 2719 skb = ibmveth_test_tcp_skb(test, false, 100); 2720 iph = (struct iphdr *)skb->data; 2721 iph->check = (__force __sum16)0xffff; 2722 tcph = (struct tcphdr *)(skb->data + sizeof(*iph)); 2723 tcph->check = (__force __sum16)htons(1); 2724 skb->ip_summed = CHECKSUM_UNNECESSARY; 2725 ibmveth_rx_csum_helper(skb, NULL); 2726 KUNIT_EXPECT_EQ(test, 0, (__force u16)ip_fast_csum(iph, iph->ihl)); 2727 KUNIT_EXPECT_EQ(test, CHECKSUM_UNNECESSARY, (int)skb->ip_summed); 2728 2729 skb = ibmveth_test_tcp_skb(test, false, 100); 2730 iph = (struct iphdr *)skb->data; 2731 iph->ihl = 0; 2732 iph->check = (__force __sum16)0xffff; 2733 skb->ip_summed = CHECKSUM_UNNECESSARY; 2734 ibmveth_rx_csum_helper(skb, NULL); 2735 KUNIT_EXPECT_EQ(test, CHECKSUM_NONE, (int)skb->ip_summed); 2736 KUNIT_EXPECT_EQ(test, 0xffff, (__force u16)iph->check); 2737 2738 /* IPv4 options past the frame; UDP, so no TCP check follows */ 2739 skb = ibmveth_test_tcp_skb(test, false, 0); 2740 iph = (struct iphdr *)skb->data; 2741 iph->protocol = IPPROTO_UDP; 2742 iph->ihl = 15; 2743 iph->check = (__force __sum16)0xffff; 2744 skb->ip_summed = CHECKSUM_UNNECESSARY; 2745 ibmveth_rx_csum_helper(skb, NULL); 2746 KUNIT_EXPECT_EQ(test, CHECKSUM_NONE, (int)skb->ip_summed); 2747 2748 /* a TCP fragment past the first: header and payload left alone */ 2749 skb = ibmveth_test_tcp_skb(test, false, 100); 2750 iph = (struct iphdr *)skb->data; 2751 iph->frag_off = htons(185); 2752 iph->check = (__force __sum16)0xffff; 2753 skb->ip_summed = CHECKSUM_UNNECESSARY; 2754 ibmveth_rx_csum_helper(skb, NULL); 2755 KUNIT_EXPECT_EQ(test, CHECKSUM_NONE, (int)skb->ip_summed); 2756 KUNIT_EXPECT_EQ(test, 0xffff, (__force u16)iph->check); 2757 KUNIT_EXPECT_EQ(test, 0, 2758 (__force u16)((struct tcphdr *)(skb->data + 2759 sizeof(*iph)))->check); 2760 2761 /* TCP header past the frame */ 2762 skb = ibmveth_test_tcp_skb(test, false, 0); 2763 skb_trim(skb, sizeof(struct iphdr)); 2764 skb->ip_summed = CHECKSUM_UNNECESSARY; 2765 ibmveth_rx_csum_helper(skb, NULL); 2766 KUNIT_EXPECT_EQ(test, CHECKSUM_NONE, (int)skb->ip_summed); 2767 2768 /* IPv6 header past the frame; UDP, so no TCP check follows */ 2769 skb = ibmveth_test_tcp_skb(test, true, 0); 2770 ((struct ipv6hdr *)skb->data)->nexthdr = IPPROTO_UDP; 2771 skb_trim(skb, sizeof(struct ipv6hdr) - 1); 2772 skb->ip_summed = CHECKSUM_UNNECESSARY; 2773 ibmveth_rx_csum_helper(skb, NULL); 2774 KUNIT_EXPECT_EQ(test, CHECKSUM_NONE, (int)skb->ip_summed); 2775 } 2776 2777 static struct kunit_case ibmveth_test_cases[] = { 2778 KUNIT_CASE(ibmveth_remove_buffer_from_pool_test), 2779 KUNIT_CASE(ibmveth_rxq_get_buffer_test), 2780 KUNIT_CASE(ibmveth_rxq_harvest_buffer_test), 2781 KUNIT_CASE(ibmveth_rx_mss_helper_test), 2782 KUNIT_CASE(ibmveth_rx_csum_helper_test), 2783 {} 2784 }; 2785 2786 static struct kunit_suite ibmveth_test_suite = { 2787 .name = "ibmveth-kunit-test", 2788 .test_cases = ibmveth_test_cases, 2789 }; 2790 2791 kunit_test_suite(ibmveth_test_suite); 2792 #endif 2793