xref: /linux/drivers/net/ethernet/ibm/ibmveth.c (revision 8e1bb4a41aa78d6105e59186af3dcd545fc66e70)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * IBM Power Virtual Ethernet Device Driver
4  *
5  * Copyright (C) IBM Corporation, 2003, 2010
6  *
7  * Authors: Dave Larson <larson1@us.ibm.com>
8  *	    Santiago Leon <santil@linux.vnet.ibm.com>
9  *	    Brian King <brking@linux.vnet.ibm.com>
10  *	    Robert Jennings <rcj@linux.vnet.ibm.com>
11  *	    Anton Blanchard <anton@au.ibm.com>
12  */
13 
14 #include <linux/module.h>
15 #include <linux/types.h>
16 #include <linux/errno.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/kernel.h>
19 #include <linux/netdevice.h>
20 #include <linux/etherdevice.h>
21 #include <linux/skbuff.h>
22 #include <linux/init.h>
23 #include <linux/interrupt.h>
24 #include <linux/mm.h>
25 #include <linux/pm.h>
26 #include <linux/ethtool.h>
27 #include <linux/in.h>
28 #include <linux/ip.h>
29 #include <linux/ipv6.h>
30 #include <linux/slab.h>
31 #include <asm/hvcall.h>
32 #include <linux/atomic.h>
33 #include <asm/vio.h>
34 #include <asm/iommu.h>
35 #include <asm/firmware.h>
36 #include <net/tcp.h>
37 #include <net/ip6_checksum.h>
38 
39 #include "ibmveth.h"
40 
41 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance);
42 static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter);
43 static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev);
44 
45 static struct kobj_type ktype_veth_pool;
46 
47 
48 static const char ibmveth_driver_name[] = "ibmveth";
49 static const char ibmveth_driver_string[] = "IBM Power Virtual Ethernet Driver";
50 #define ibmveth_driver_version "1.06"
51 
52 MODULE_AUTHOR("Santiago Leon <santil@linux.vnet.ibm.com>");
53 MODULE_DESCRIPTION("IBM Power Virtual Ethernet Driver");
54 MODULE_LICENSE("GPL");
55 MODULE_VERSION(ibmveth_driver_version);
56 
57 static unsigned int tx_copybreak __read_mostly = 128;
58 module_param(tx_copybreak, uint, 0644);
59 MODULE_PARM_DESC(tx_copybreak,
60 	"Maximum size of packet that is copied to a new buffer on transmit");
61 
62 static unsigned int rx_copybreak __read_mostly = 128;
63 module_param(rx_copybreak, uint, 0644);
64 MODULE_PARM_DESC(rx_copybreak,
65 	"Maximum size of packet that is copied to a new buffer on receive");
66 
67 static unsigned int rx_flush __read_mostly = 0;
68 module_param(rx_flush, uint, 0644);
69 MODULE_PARM_DESC(rx_flush, "Flush receive buffers before use");
70 
71 static bool old_large_send __read_mostly;
72 module_param(old_large_send, bool, 0444);
73 MODULE_PARM_DESC(old_large_send,
74 	"Use old large send method on firmware that supports the new method");
75 
76 struct ibmveth_stat {
77 	char name[ETH_GSTRING_LEN];
78 	int offset;
79 };
80 
81 #define IBMVETH_STAT_OFF(stat) offsetof(struct ibmveth_adapter, stat)
82 #define IBMVETH_GET_STAT(a, off) *((u64 *)(((unsigned long)(a)) + off))
83 
84 static struct ibmveth_stat ibmveth_stats[] = {
85 	{ "replenish_task_cycles", IBMVETH_STAT_OFF(replenish_task_cycles) },
86 	{ "replenish_no_mem", IBMVETH_STAT_OFF(replenish_no_mem) },
87 	{ "replenish_add_buff_failure",
88 			IBMVETH_STAT_OFF(replenish_add_buff_failure) },
89 	{ "replenish_add_buff_success",
90 			IBMVETH_STAT_OFF(replenish_add_buff_success) },
91 	{ "rx_invalid_buffer", IBMVETH_STAT_OFF(rx_invalid_buffer) },
92 	{ "rx_no_buffer", IBMVETH_STAT_OFF(rx_no_buffer) },
93 	{ "tx_map_failed", IBMVETH_STAT_OFF(tx_map_failed) },
94 	{ "tx_send_failed", IBMVETH_STAT_OFF(tx_send_failed) },
95 	{ "fw_enabled_ipv4_csum", IBMVETH_STAT_OFF(fw_ipv4_csum_support) },
96 	{ "fw_enabled_ipv6_csum", IBMVETH_STAT_OFF(fw_ipv6_csum_support) },
97 	{ "tx_large_packets", IBMVETH_STAT_OFF(tx_large_packets) },
98 	{ "rx_large_packets", IBMVETH_STAT_OFF(rx_large_packets) },
99 	{ "fw_enabled_large_send", IBMVETH_STAT_OFF(fw_large_send_support) }
100 };
101 
102 /* simple methods of getting data from the current rxq entry */
103 static inline u32 ibmveth_rxq_flags(struct ibmveth_adapter *adapter)
104 {
105 	return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].flags_off);
106 }
107 
108 static inline int ibmveth_rxq_toggle(struct ibmveth_adapter *adapter)
109 {
110 	return (ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_TOGGLE) >>
111 			IBMVETH_RXQ_TOGGLE_SHIFT;
112 }
113 
114 static inline int ibmveth_rxq_pending_buffer(struct ibmveth_adapter *adapter)
115 {
116 	return ibmveth_rxq_toggle(adapter) == adapter->rx_queue.toggle;
117 }
118 
119 static inline int ibmveth_rxq_buffer_valid(struct ibmveth_adapter *adapter)
120 {
121 	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_VALID;
122 }
123 
124 static inline int ibmveth_rxq_frame_offset(struct ibmveth_adapter *adapter)
125 {
126 	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_OFF_MASK;
127 }
128 
129 static inline int ibmveth_rxq_large_packet(struct ibmveth_adapter *adapter)
130 {
131 	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_LRG_PKT;
132 }
133 
134 static inline int ibmveth_rxq_frame_length(struct ibmveth_adapter *adapter)
135 {
136 	return be32_to_cpu(adapter->rx_queue.queue_addr[adapter->rx_queue.index].length);
137 }
138 
139 static inline int ibmveth_rxq_csum_good(struct ibmveth_adapter *adapter)
140 {
141 	return ibmveth_rxq_flags(adapter) & IBMVETH_RXQ_CSUM_GOOD;
142 }
143 
144 static unsigned int ibmveth_real_max_tx_queues(void)
145 {
146 	unsigned int n_cpu = num_online_cpus();
147 
148 	return min(n_cpu, IBMVETH_MAX_QUEUES);
149 }
150 
151 /* setup the initial settings for a buffer pool */
152 static void ibmveth_init_buffer_pool(struct ibmveth_buff_pool *pool,
153 				     u32 pool_index, u32 pool_size,
154 				     u32 buff_size, u32 pool_active)
155 {
156 	pool->size = pool_size;
157 	pool->index = pool_index;
158 	pool->buff_size = buff_size;
159 	pool->threshold = pool_size * 7 / 8;
160 	pool->active = pool_active;
161 }
162 
163 /* allocate and setup an buffer pool - called during open */
164 static int ibmveth_alloc_buffer_pool(struct ibmveth_buff_pool *pool)
165 {
166 	int i;
167 
168 	pool->free_map = kmalloc_array(pool->size, sizeof(u16), GFP_KERNEL);
169 
170 	if (!pool->free_map)
171 		return -1;
172 
173 	pool->dma_addr = kcalloc(pool->size, sizeof(dma_addr_t), GFP_KERNEL);
174 	if (!pool->dma_addr) {
175 		kfree(pool->free_map);
176 		pool->free_map = NULL;
177 		return -1;
178 	}
179 
180 	pool->skbuff = kcalloc(pool->size, sizeof(void *), GFP_KERNEL);
181 
182 	if (!pool->skbuff) {
183 		kfree(pool->dma_addr);
184 		pool->dma_addr = NULL;
185 
186 		kfree(pool->free_map);
187 		pool->free_map = NULL;
188 		return -1;
189 	}
190 
191 	for (i = 0; i < pool->size; ++i)
192 		pool->free_map[i] = i;
193 
194 	atomic_set(&pool->available, 0);
195 	pool->producer_index = 0;
196 	pool->consumer_index = 0;
197 
198 	return 0;
199 }
200 
201 static inline void ibmveth_flush_buffer(void *addr, unsigned long length)
202 {
203 	unsigned long offset;
204 
205 	for (offset = 0; offset < length; offset += SMP_CACHE_BYTES)
206 		asm("dcbf %0,%1,1" :: "b" (addr), "r" (offset));
207 }
208 
209 /* replenish the buffers for a pool.  note that we don't need to
210  * skb_reserve these since they are used for incoming...
211  */
212 static void ibmveth_replenish_buffer_pool(struct ibmveth_adapter *adapter,
213 					  struct ibmveth_buff_pool *pool)
214 {
215 	u32 i;
216 	u32 count = pool->size - atomic_read(&pool->available);
217 	u32 buffers_added = 0;
218 	struct sk_buff *skb;
219 	unsigned int free_index, index;
220 	u64 correlator;
221 	unsigned long lpar_rc;
222 	dma_addr_t dma_addr;
223 
224 	mb();
225 
226 	for (i = 0; i < count; ++i) {
227 		union ibmveth_buf_desc desc;
228 
229 		skb = netdev_alloc_skb(adapter->netdev, pool->buff_size);
230 
231 		if (!skb) {
232 			netdev_dbg(adapter->netdev,
233 				   "replenish: unable to allocate skb\n");
234 			adapter->replenish_no_mem++;
235 			break;
236 		}
237 
238 		free_index = pool->consumer_index;
239 		pool->consumer_index++;
240 		if (pool->consumer_index >= pool->size)
241 			pool->consumer_index = 0;
242 		index = pool->free_map[free_index];
243 
244 		BUG_ON(index == IBM_VETH_INVALID_MAP);
245 		BUG_ON(pool->skbuff[index] != NULL);
246 
247 		dma_addr = dma_map_single(&adapter->vdev->dev, skb->data,
248 				pool->buff_size, DMA_FROM_DEVICE);
249 
250 		if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
251 			goto failure;
252 
253 		pool->free_map[free_index] = IBM_VETH_INVALID_MAP;
254 		pool->dma_addr[index] = dma_addr;
255 		pool->skbuff[index] = skb;
256 
257 		correlator = ((u64)pool->index << 32) | index;
258 		*(u64 *)skb->data = correlator;
259 
260 		desc.fields.flags_len = IBMVETH_BUF_VALID | pool->buff_size;
261 		desc.fields.address = dma_addr;
262 
263 		if (rx_flush) {
264 			unsigned int len = min(pool->buff_size,
265 						adapter->netdev->mtu +
266 						IBMVETH_BUFF_OH);
267 			ibmveth_flush_buffer(skb->data, len);
268 		}
269 		lpar_rc = h_add_logical_lan_buffer(adapter->vdev->unit_address,
270 						   desc.desc);
271 
272 		if (lpar_rc != H_SUCCESS) {
273 			goto failure;
274 		} else {
275 			buffers_added++;
276 			adapter->replenish_add_buff_success++;
277 		}
278 	}
279 
280 	mb();
281 	atomic_add(buffers_added, &(pool->available));
282 	return;
283 
284 failure:
285 	pool->free_map[free_index] = index;
286 	pool->skbuff[index] = NULL;
287 	if (pool->consumer_index == 0)
288 		pool->consumer_index = pool->size - 1;
289 	else
290 		pool->consumer_index--;
291 	if (!dma_mapping_error(&adapter->vdev->dev, dma_addr))
292 		dma_unmap_single(&adapter->vdev->dev,
293 		                 pool->dma_addr[index], pool->buff_size,
294 		                 DMA_FROM_DEVICE);
295 	dev_kfree_skb_any(skb);
296 	adapter->replenish_add_buff_failure++;
297 
298 	mb();
299 	atomic_add(buffers_added, &(pool->available));
300 }
301 
302 /*
303  * The final 8 bytes of the buffer list is a counter of frames dropped
304  * because there was not a buffer in the buffer list capable of holding
305  * the frame.
306  */
307 static void ibmveth_update_rx_no_buffer(struct ibmveth_adapter *adapter)
308 {
309 	__be64 *p = adapter->buffer_list_addr + 4096 - 8;
310 
311 	adapter->rx_no_buffer = be64_to_cpup(p);
312 }
313 
314 /* replenish routine */
315 static void ibmveth_replenish_task(struct ibmveth_adapter *adapter)
316 {
317 	int i;
318 
319 	adapter->replenish_task_cycles++;
320 
321 	for (i = (IBMVETH_NUM_BUFF_POOLS - 1); i >= 0; i--) {
322 		struct ibmveth_buff_pool *pool = &adapter->rx_buff_pool[i];
323 
324 		if (pool->active &&
325 		    (atomic_read(&pool->available) < pool->threshold))
326 			ibmveth_replenish_buffer_pool(adapter, pool);
327 	}
328 
329 	ibmveth_update_rx_no_buffer(adapter);
330 }
331 
332 /* empty and free ana buffer pool - also used to do cleanup in error paths */
333 static void ibmveth_free_buffer_pool(struct ibmveth_adapter *adapter,
334 				     struct ibmveth_buff_pool *pool)
335 {
336 	int i;
337 
338 	kfree(pool->free_map);
339 	pool->free_map = NULL;
340 
341 	if (pool->skbuff && pool->dma_addr) {
342 		for (i = 0; i < pool->size; ++i) {
343 			struct sk_buff *skb = pool->skbuff[i];
344 			if (skb) {
345 				dma_unmap_single(&adapter->vdev->dev,
346 						 pool->dma_addr[i],
347 						 pool->buff_size,
348 						 DMA_FROM_DEVICE);
349 				dev_kfree_skb_any(skb);
350 				pool->skbuff[i] = NULL;
351 			}
352 		}
353 	}
354 
355 	if (pool->dma_addr) {
356 		kfree(pool->dma_addr);
357 		pool->dma_addr = NULL;
358 	}
359 
360 	if (pool->skbuff) {
361 		kfree(pool->skbuff);
362 		pool->skbuff = NULL;
363 	}
364 }
365 
366 /* remove a buffer from a pool */
367 static void ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,
368 					    u64 correlator)
369 {
370 	unsigned int pool  = correlator >> 32;
371 	unsigned int index = correlator & 0xffffffffUL;
372 	unsigned int free_index;
373 	struct sk_buff *skb;
374 
375 	BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
376 	BUG_ON(index >= adapter->rx_buff_pool[pool].size);
377 
378 	skb = adapter->rx_buff_pool[pool].skbuff[index];
379 
380 	BUG_ON(skb == NULL);
381 
382 	adapter->rx_buff_pool[pool].skbuff[index] = NULL;
383 
384 	dma_unmap_single(&adapter->vdev->dev,
385 			 adapter->rx_buff_pool[pool].dma_addr[index],
386 			 adapter->rx_buff_pool[pool].buff_size,
387 			 DMA_FROM_DEVICE);
388 
389 	free_index = adapter->rx_buff_pool[pool].producer_index;
390 	adapter->rx_buff_pool[pool].producer_index++;
391 	if (adapter->rx_buff_pool[pool].producer_index >=
392 	    adapter->rx_buff_pool[pool].size)
393 		adapter->rx_buff_pool[pool].producer_index = 0;
394 	adapter->rx_buff_pool[pool].free_map[free_index] = index;
395 
396 	mb();
397 
398 	atomic_dec(&(adapter->rx_buff_pool[pool].available));
399 }
400 
401 /* get the current buffer on the rx queue */
402 static inline struct sk_buff *ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter)
403 {
404 	u64 correlator = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator;
405 	unsigned int pool = correlator >> 32;
406 	unsigned int index = correlator & 0xffffffffUL;
407 
408 	BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
409 	BUG_ON(index >= adapter->rx_buff_pool[pool].size);
410 
411 	return adapter->rx_buff_pool[pool].skbuff[index];
412 }
413 
414 /* recycle the current buffer on the rx queue */
415 static int ibmveth_rxq_recycle_buffer(struct ibmveth_adapter *adapter)
416 {
417 	u32 q_index = adapter->rx_queue.index;
418 	u64 correlator = adapter->rx_queue.queue_addr[q_index].correlator;
419 	unsigned int pool = correlator >> 32;
420 	unsigned int index = correlator & 0xffffffffUL;
421 	union ibmveth_buf_desc desc;
422 	unsigned long lpar_rc;
423 	int ret = 1;
424 
425 	BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
426 	BUG_ON(index >= adapter->rx_buff_pool[pool].size);
427 
428 	if (!adapter->rx_buff_pool[pool].active) {
429 		ibmveth_rxq_harvest_buffer(adapter);
430 		ibmveth_free_buffer_pool(adapter, &adapter->rx_buff_pool[pool]);
431 		goto out;
432 	}
433 
434 	desc.fields.flags_len = IBMVETH_BUF_VALID |
435 		adapter->rx_buff_pool[pool].buff_size;
436 	desc.fields.address = adapter->rx_buff_pool[pool].dma_addr[index];
437 
438 	lpar_rc = h_add_logical_lan_buffer(adapter->vdev->unit_address, desc.desc);
439 
440 	if (lpar_rc != H_SUCCESS) {
441 		netdev_dbg(adapter->netdev, "h_add_logical_lan_buffer failed "
442 			   "during recycle rc=%ld", lpar_rc);
443 		ibmveth_remove_buffer_from_pool(adapter, adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator);
444 		ret = 0;
445 	}
446 
447 	if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
448 		adapter->rx_queue.index = 0;
449 		adapter->rx_queue.toggle = !adapter->rx_queue.toggle;
450 	}
451 
452 out:
453 	return ret;
454 }
455 
456 static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter)
457 {
458 	ibmveth_remove_buffer_from_pool(adapter, adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator);
459 
460 	if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
461 		adapter->rx_queue.index = 0;
462 		adapter->rx_queue.toggle = !adapter->rx_queue.toggle;
463 	}
464 }
465 
466 static void ibmveth_free_tx_ltb(struct ibmveth_adapter *adapter, int idx)
467 {
468 	dma_unmap_single(&adapter->vdev->dev, adapter->tx_ltb_dma[idx],
469 			 adapter->tx_ltb_size, DMA_TO_DEVICE);
470 	kfree(adapter->tx_ltb_ptr[idx]);
471 	adapter->tx_ltb_ptr[idx] = NULL;
472 }
473 
474 static int ibmveth_allocate_tx_ltb(struct ibmveth_adapter *adapter, int idx)
475 {
476 	adapter->tx_ltb_ptr[idx] = kzalloc(adapter->tx_ltb_size,
477 					   GFP_KERNEL);
478 	if (!adapter->tx_ltb_ptr[idx]) {
479 		netdev_err(adapter->netdev,
480 			   "unable to allocate tx long term buffer\n");
481 		return -ENOMEM;
482 	}
483 	adapter->tx_ltb_dma[idx] = dma_map_single(&adapter->vdev->dev,
484 						  adapter->tx_ltb_ptr[idx],
485 						  adapter->tx_ltb_size,
486 						  DMA_TO_DEVICE);
487 	if (dma_mapping_error(&adapter->vdev->dev, adapter->tx_ltb_dma[idx])) {
488 		netdev_err(adapter->netdev,
489 			   "unable to DMA map tx long term buffer\n");
490 		kfree(adapter->tx_ltb_ptr[idx]);
491 		adapter->tx_ltb_ptr[idx] = NULL;
492 		return -ENOMEM;
493 	}
494 
495 	return 0;
496 }
497 
498 static int ibmveth_register_logical_lan(struct ibmveth_adapter *adapter,
499         union ibmveth_buf_desc rxq_desc, u64 mac_address)
500 {
501 	int rc, try_again = 1;
502 
503 	/*
504 	 * After a kexec the adapter will still be open, so our attempt to
505 	 * open it will fail. So if we get a failure we free the adapter and
506 	 * try again, but only once.
507 	 */
508 retry:
509 	rc = h_register_logical_lan(adapter->vdev->unit_address,
510 				    adapter->buffer_list_dma, rxq_desc.desc,
511 				    adapter->filter_list_dma, mac_address);
512 
513 	if (rc != H_SUCCESS && try_again) {
514 		do {
515 			rc = h_free_logical_lan(adapter->vdev->unit_address);
516 		} while (H_IS_LONG_BUSY(rc) || (rc == H_BUSY));
517 
518 		try_again = 0;
519 		goto retry;
520 	}
521 
522 	return rc;
523 }
524 
525 static int ibmveth_open(struct net_device *netdev)
526 {
527 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
528 	u64 mac_address;
529 	int rxq_entries = 1;
530 	unsigned long lpar_rc;
531 	int rc;
532 	union ibmveth_buf_desc rxq_desc;
533 	int i;
534 	struct device *dev;
535 
536 	netdev_dbg(netdev, "open starting\n");
537 
538 	napi_enable(&adapter->napi);
539 
540 	for(i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
541 		rxq_entries += adapter->rx_buff_pool[i].size;
542 
543 	rc = -ENOMEM;
544 	adapter->buffer_list_addr = (void*) get_zeroed_page(GFP_KERNEL);
545 	if (!adapter->buffer_list_addr) {
546 		netdev_err(netdev, "unable to allocate list pages\n");
547 		goto out;
548 	}
549 
550 	adapter->filter_list_addr = (void*) get_zeroed_page(GFP_KERNEL);
551 	if (!adapter->filter_list_addr) {
552 		netdev_err(netdev, "unable to allocate filter pages\n");
553 		goto out_free_buffer_list;
554 	}
555 
556 	dev = &adapter->vdev->dev;
557 
558 	adapter->rx_queue.queue_len = sizeof(struct ibmveth_rx_q_entry) *
559 						rxq_entries;
560 	adapter->rx_queue.queue_addr =
561 		dma_alloc_coherent(dev, adapter->rx_queue.queue_len,
562 				   &adapter->rx_queue.queue_dma, GFP_KERNEL);
563 	if (!adapter->rx_queue.queue_addr)
564 		goto out_free_filter_list;
565 
566 	adapter->buffer_list_dma = dma_map_single(dev,
567 			adapter->buffer_list_addr, 4096, DMA_BIDIRECTIONAL);
568 	if (dma_mapping_error(dev, adapter->buffer_list_dma)) {
569 		netdev_err(netdev, "unable to map buffer list pages\n");
570 		goto out_free_queue_mem;
571 	}
572 
573 	adapter->filter_list_dma = dma_map_single(dev,
574 			adapter->filter_list_addr, 4096, DMA_BIDIRECTIONAL);
575 	if (dma_mapping_error(dev, adapter->filter_list_dma)) {
576 		netdev_err(netdev, "unable to map filter list pages\n");
577 		goto out_unmap_buffer_list;
578 	}
579 
580 	for (i = 0; i < netdev->real_num_tx_queues; i++) {
581 		if (ibmveth_allocate_tx_ltb(adapter, i))
582 			goto out_free_tx_ltb;
583 	}
584 
585 	adapter->rx_queue.index = 0;
586 	adapter->rx_queue.num_slots = rxq_entries;
587 	adapter->rx_queue.toggle = 1;
588 
589 	mac_address = ether_addr_to_u64(netdev->dev_addr);
590 
591 	rxq_desc.fields.flags_len = IBMVETH_BUF_VALID |
592 					adapter->rx_queue.queue_len;
593 	rxq_desc.fields.address = adapter->rx_queue.queue_dma;
594 
595 	netdev_dbg(netdev, "buffer list @ 0x%p\n", adapter->buffer_list_addr);
596 	netdev_dbg(netdev, "filter list @ 0x%p\n", adapter->filter_list_addr);
597 	netdev_dbg(netdev, "receive q   @ 0x%p\n", adapter->rx_queue.queue_addr);
598 
599 	h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
600 
601 	lpar_rc = ibmveth_register_logical_lan(adapter, rxq_desc, mac_address);
602 
603 	if (lpar_rc != H_SUCCESS) {
604 		netdev_err(netdev, "h_register_logical_lan failed with %ld\n",
605 			   lpar_rc);
606 		netdev_err(netdev, "buffer TCE:0x%llx filter TCE:0x%llx rxq "
607 			   "desc:0x%llx MAC:0x%llx\n",
608 				     adapter->buffer_list_dma,
609 				     adapter->filter_list_dma,
610 				     rxq_desc.desc,
611 				     mac_address);
612 		rc = -ENONET;
613 		goto out_unmap_filter_list;
614 	}
615 
616 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
617 		if (!adapter->rx_buff_pool[i].active)
618 			continue;
619 		if (ibmveth_alloc_buffer_pool(&adapter->rx_buff_pool[i])) {
620 			netdev_err(netdev, "unable to alloc pool\n");
621 			adapter->rx_buff_pool[i].active = 0;
622 			rc = -ENOMEM;
623 			goto out_free_buffer_pools;
624 		}
625 	}
626 
627 	netdev_dbg(netdev, "registering irq 0x%x\n", netdev->irq);
628 	rc = request_irq(netdev->irq, ibmveth_interrupt, 0, netdev->name,
629 			 netdev);
630 	if (rc != 0) {
631 		netdev_err(netdev, "unable to request irq 0x%x, rc %d\n",
632 			   netdev->irq, rc);
633 		do {
634 			lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
635 		} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
636 
637 		goto out_free_buffer_pools;
638 	}
639 
640 	rc = -ENOMEM;
641 
642 	netdev_dbg(netdev, "initial replenish cycle\n");
643 	ibmveth_interrupt(netdev->irq, netdev);
644 
645 	netif_tx_start_all_queues(netdev);
646 
647 	netdev_dbg(netdev, "open complete\n");
648 
649 	return 0;
650 
651 out_free_buffer_pools:
652 	while (--i >= 0) {
653 		if (adapter->rx_buff_pool[i].active)
654 			ibmveth_free_buffer_pool(adapter,
655 						 &adapter->rx_buff_pool[i]);
656 	}
657 out_unmap_filter_list:
658 	dma_unmap_single(dev, adapter->filter_list_dma, 4096,
659 			 DMA_BIDIRECTIONAL);
660 
661 out_free_tx_ltb:
662 	while (--i >= 0) {
663 		ibmveth_free_tx_ltb(adapter, i);
664 	}
665 
666 out_unmap_buffer_list:
667 	dma_unmap_single(dev, adapter->buffer_list_dma, 4096,
668 			 DMA_BIDIRECTIONAL);
669 out_free_queue_mem:
670 	dma_free_coherent(dev, adapter->rx_queue.queue_len,
671 			  adapter->rx_queue.queue_addr,
672 			  adapter->rx_queue.queue_dma);
673 out_free_filter_list:
674 	free_page((unsigned long)adapter->filter_list_addr);
675 out_free_buffer_list:
676 	free_page((unsigned long)adapter->buffer_list_addr);
677 out:
678 	napi_disable(&adapter->napi);
679 	return rc;
680 }
681 
682 static int ibmveth_close(struct net_device *netdev)
683 {
684 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
685 	struct device *dev = &adapter->vdev->dev;
686 	long lpar_rc;
687 	int i;
688 
689 	netdev_dbg(netdev, "close starting\n");
690 
691 	napi_disable(&adapter->napi);
692 
693 	netif_tx_stop_all_queues(netdev);
694 
695 	h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_DISABLE);
696 
697 	do {
698 		lpar_rc = h_free_logical_lan(adapter->vdev->unit_address);
699 	} while (H_IS_LONG_BUSY(lpar_rc) || (lpar_rc == H_BUSY));
700 
701 	if (lpar_rc != H_SUCCESS) {
702 		netdev_err(netdev, "h_free_logical_lan failed with %lx, "
703 			   "continuing with close\n", lpar_rc);
704 	}
705 
706 	free_irq(netdev->irq, netdev);
707 
708 	ibmveth_update_rx_no_buffer(adapter);
709 
710 	dma_unmap_single(dev, adapter->buffer_list_dma, 4096,
711 			 DMA_BIDIRECTIONAL);
712 	free_page((unsigned long)adapter->buffer_list_addr);
713 
714 	dma_unmap_single(dev, adapter->filter_list_dma, 4096,
715 			 DMA_BIDIRECTIONAL);
716 	free_page((unsigned long)adapter->filter_list_addr);
717 
718 	dma_free_coherent(dev, adapter->rx_queue.queue_len,
719 			  adapter->rx_queue.queue_addr,
720 			  adapter->rx_queue.queue_dma);
721 
722 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
723 		if (adapter->rx_buff_pool[i].active)
724 			ibmveth_free_buffer_pool(adapter,
725 						 &adapter->rx_buff_pool[i]);
726 
727 	for (i = 0; i < netdev->real_num_tx_queues; i++)
728 		ibmveth_free_tx_ltb(adapter, i);
729 
730 	netdev_dbg(netdev, "close complete\n");
731 
732 	return 0;
733 }
734 
735 static int ibmveth_set_link_ksettings(struct net_device *dev,
736 				      const struct ethtool_link_ksettings *cmd)
737 {
738 	struct ibmveth_adapter *adapter = netdev_priv(dev);
739 
740 	return ethtool_virtdev_set_link_ksettings(dev, cmd,
741 						  &adapter->speed,
742 						  &adapter->duplex);
743 }
744 
745 static int ibmveth_get_link_ksettings(struct net_device *dev,
746 				      struct ethtool_link_ksettings *cmd)
747 {
748 	struct ibmveth_adapter *adapter = netdev_priv(dev);
749 
750 	cmd->base.speed = adapter->speed;
751 	cmd->base.duplex = adapter->duplex;
752 	cmd->base.port = PORT_OTHER;
753 
754 	return 0;
755 }
756 
757 static void ibmveth_init_link_settings(struct net_device *dev)
758 {
759 	struct ibmveth_adapter *adapter = netdev_priv(dev);
760 
761 	adapter->speed = SPEED_1000;
762 	adapter->duplex = DUPLEX_FULL;
763 }
764 
765 static void netdev_get_drvinfo(struct net_device *dev,
766 			       struct ethtool_drvinfo *info)
767 {
768 	strscpy(info->driver, ibmveth_driver_name, sizeof(info->driver));
769 	strscpy(info->version, ibmveth_driver_version, sizeof(info->version));
770 }
771 
772 static netdev_features_t ibmveth_fix_features(struct net_device *dev,
773 	netdev_features_t features)
774 {
775 	/*
776 	 * Since the ibmveth firmware interface does not have the
777 	 * concept of separate tx/rx checksum offload enable, if rx
778 	 * checksum is disabled we also have to disable tx checksum
779 	 * offload. Once we disable rx checksum offload, we are no
780 	 * longer allowed to send tx buffers that are not properly
781 	 * checksummed.
782 	 */
783 
784 	if (!(features & NETIF_F_RXCSUM))
785 		features &= ~NETIF_F_CSUM_MASK;
786 
787 	return features;
788 }
789 
790 static int ibmveth_set_csum_offload(struct net_device *dev, u32 data)
791 {
792 	struct ibmveth_adapter *adapter = netdev_priv(dev);
793 	unsigned long set_attr, clr_attr, ret_attr;
794 	unsigned long set_attr6, clr_attr6;
795 	long ret, ret4, ret6;
796 	int rc1 = 0, rc2 = 0;
797 	int restart = 0;
798 
799 	if (netif_running(dev)) {
800 		restart = 1;
801 		ibmveth_close(dev);
802 	}
803 
804 	set_attr = 0;
805 	clr_attr = 0;
806 	set_attr6 = 0;
807 	clr_attr6 = 0;
808 
809 	if (data) {
810 		set_attr = IBMVETH_ILLAN_IPV4_TCP_CSUM;
811 		set_attr6 = IBMVETH_ILLAN_IPV6_TCP_CSUM;
812 	} else {
813 		clr_attr = IBMVETH_ILLAN_IPV4_TCP_CSUM;
814 		clr_attr6 = IBMVETH_ILLAN_IPV6_TCP_CSUM;
815 	}
816 
817 	ret = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr);
818 
819 	if (ret == H_SUCCESS &&
820 	    (ret_attr & IBMVETH_ILLAN_PADDED_PKT_CSUM)) {
821 		ret4 = h_illan_attributes(adapter->vdev->unit_address, clr_attr,
822 					 set_attr, &ret_attr);
823 
824 		if (ret4 != H_SUCCESS) {
825 			netdev_err(dev, "unable to change IPv4 checksum "
826 					"offload settings. %d rc=%ld\n",
827 					data, ret4);
828 
829 			h_illan_attributes(adapter->vdev->unit_address,
830 					   set_attr, clr_attr, &ret_attr);
831 
832 			if (data == 1)
833 				dev->features &= ~NETIF_F_IP_CSUM;
834 
835 		} else {
836 			adapter->fw_ipv4_csum_support = data;
837 		}
838 
839 		ret6 = h_illan_attributes(adapter->vdev->unit_address,
840 					 clr_attr6, set_attr6, &ret_attr);
841 
842 		if (ret6 != H_SUCCESS) {
843 			netdev_err(dev, "unable to change IPv6 checksum "
844 					"offload settings. %d rc=%ld\n",
845 					data, ret6);
846 
847 			h_illan_attributes(adapter->vdev->unit_address,
848 					   set_attr6, clr_attr6, &ret_attr);
849 
850 			if (data == 1)
851 				dev->features &= ~NETIF_F_IPV6_CSUM;
852 
853 		} else
854 			adapter->fw_ipv6_csum_support = data;
855 
856 		if (ret4 == H_SUCCESS || ret6 == H_SUCCESS)
857 			adapter->rx_csum = data;
858 		else
859 			rc1 = -EIO;
860 	} else {
861 		rc1 = -EIO;
862 		netdev_err(dev, "unable to change checksum offload settings."
863 				     " %d rc=%ld ret_attr=%lx\n", data, ret,
864 				     ret_attr);
865 	}
866 
867 	if (restart)
868 		rc2 = ibmveth_open(dev);
869 
870 	return rc1 ? rc1 : rc2;
871 }
872 
873 static int ibmveth_set_tso(struct net_device *dev, u32 data)
874 {
875 	struct ibmveth_adapter *adapter = netdev_priv(dev);
876 	unsigned long set_attr, clr_attr, ret_attr;
877 	long ret1, ret2;
878 	int rc1 = 0, rc2 = 0;
879 	int restart = 0;
880 
881 	if (netif_running(dev)) {
882 		restart = 1;
883 		ibmveth_close(dev);
884 	}
885 
886 	set_attr = 0;
887 	clr_attr = 0;
888 
889 	if (data)
890 		set_attr = IBMVETH_ILLAN_LRG_SR_ENABLED;
891 	else
892 		clr_attr = IBMVETH_ILLAN_LRG_SR_ENABLED;
893 
894 	ret1 = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr);
895 
896 	if (ret1 == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_LRG_SND_SUPPORT) &&
897 	    !old_large_send) {
898 		ret2 = h_illan_attributes(adapter->vdev->unit_address, clr_attr,
899 					  set_attr, &ret_attr);
900 
901 		if (ret2 != H_SUCCESS) {
902 			netdev_err(dev, "unable to change tso settings. %d rc=%ld\n",
903 				   data, ret2);
904 
905 			h_illan_attributes(adapter->vdev->unit_address,
906 					   set_attr, clr_attr, &ret_attr);
907 
908 			if (data == 1)
909 				dev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6);
910 			rc1 = -EIO;
911 
912 		} else {
913 			adapter->fw_large_send_support = data;
914 			adapter->large_send = data;
915 		}
916 	} else {
917 		/* Older firmware version of large send offload does not
918 		 * support tcp6/ipv6
919 		 */
920 		if (data == 1) {
921 			dev->features &= ~NETIF_F_TSO6;
922 			netdev_info(dev, "TSO feature requires all partitions to have updated driver");
923 		}
924 		adapter->large_send = data;
925 	}
926 
927 	if (restart)
928 		rc2 = ibmveth_open(dev);
929 
930 	return rc1 ? rc1 : rc2;
931 }
932 
933 static int ibmveth_set_features(struct net_device *dev,
934 	netdev_features_t features)
935 {
936 	struct ibmveth_adapter *adapter = netdev_priv(dev);
937 	int rx_csum = !!(features & NETIF_F_RXCSUM);
938 	int large_send = !!(features & (NETIF_F_TSO | NETIF_F_TSO6));
939 	int rc1 = 0, rc2 = 0;
940 
941 	if (rx_csum != adapter->rx_csum) {
942 		rc1 = ibmveth_set_csum_offload(dev, rx_csum);
943 		if (rc1 && !adapter->rx_csum)
944 			dev->features =
945 				features & ~(NETIF_F_CSUM_MASK |
946 					     NETIF_F_RXCSUM);
947 	}
948 
949 	if (large_send != adapter->large_send) {
950 		rc2 = ibmveth_set_tso(dev, large_send);
951 		if (rc2 && !adapter->large_send)
952 			dev->features =
953 				features & ~(NETIF_F_TSO | NETIF_F_TSO6);
954 	}
955 
956 	return rc1 ? rc1 : rc2;
957 }
958 
959 static void ibmveth_get_strings(struct net_device *dev, u32 stringset, u8 *data)
960 {
961 	int i;
962 
963 	if (stringset != ETH_SS_STATS)
964 		return;
965 
966 	for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++, data += ETH_GSTRING_LEN)
967 		memcpy(data, ibmveth_stats[i].name, ETH_GSTRING_LEN);
968 }
969 
970 static int ibmveth_get_sset_count(struct net_device *dev, int sset)
971 {
972 	switch (sset) {
973 	case ETH_SS_STATS:
974 		return ARRAY_SIZE(ibmveth_stats);
975 	default:
976 		return -EOPNOTSUPP;
977 	}
978 }
979 
980 static void ibmveth_get_ethtool_stats(struct net_device *dev,
981 				      struct ethtool_stats *stats, u64 *data)
982 {
983 	int i;
984 	struct ibmveth_adapter *adapter = netdev_priv(dev);
985 
986 	for (i = 0; i < ARRAY_SIZE(ibmveth_stats); i++)
987 		data[i] = IBMVETH_GET_STAT(adapter, ibmveth_stats[i].offset);
988 }
989 
990 static void ibmveth_get_channels(struct net_device *netdev,
991 				 struct ethtool_channels *channels)
992 {
993 	channels->max_tx = ibmveth_real_max_tx_queues();
994 	channels->tx_count = netdev->real_num_tx_queues;
995 
996 	channels->max_rx = netdev->real_num_rx_queues;
997 	channels->rx_count = netdev->real_num_rx_queues;
998 }
999 
1000 static int ibmveth_set_channels(struct net_device *netdev,
1001 				struct ethtool_channels *channels)
1002 {
1003 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
1004 	unsigned int old = netdev->real_num_tx_queues,
1005 		     goal = channels->tx_count;
1006 	int rc, i;
1007 
1008 	/* If ndo_open has not been called yet then don't allocate, just set
1009 	 * desired netdev_queue's and return
1010 	 */
1011 	if (!(netdev->flags & IFF_UP))
1012 		return netif_set_real_num_tx_queues(netdev, goal);
1013 
1014 	/* We have IBMVETH_MAX_QUEUES netdev_queue's allocated
1015 	 * but we may need to alloc/free the ltb's.
1016 	 */
1017 	netif_tx_stop_all_queues(netdev);
1018 
1019 	/* Allocate any queue that we need */
1020 	for (i = old; i < goal; i++) {
1021 		if (adapter->tx_ltb_ptr[i])
1022 			continue;
1023 
1024 		rc = ibmveth_allocate_tx_ltb(adapter, i);
1025 		if (!rc)
1026 			continue;
1027 
1028 		/* if something goes wrong, free everything we just allocated */
1029 		netdev_err(netdev, "Failed to allocate more tx queues, returning to %d queues\n",
1030 			   old);
1031 		goal = old;
1032 		old = i;
1033 		break;
1034 	}
1035 	rc = netif_set_real_num_tx_queues(netdev, goal);
1036 	if (rc) {
1037 		netdev_err(netdev, "Failed to set real tx queues, returning to %d queues\n",
1038 			   old);
1039 		goal = old;
1040 		old = i;
1041 	}
1042 	/* Free any that are no longer needed */
1043 	for (i = old; i > goal; i--) {
1044 		if (adapter->tx_ltb_ptr[i - 1])
1045 			ibmveth_free_tx_ltb(adapter, i - 1);
1046 	}
1047 
1048 	netif_tx_wake_all_queues(netdev);
1049 
1050 	return rc;
1051 }
1052 
1053 static const struct ethtool_ops netdev_ethtool_ops = {
1054 	.get_drvinfo		         = netdev_get_drvinfo,
1055 	.get_link		         = ethtool_op_get_link,
1056 	.get_strings		         = ibmveth_get_strings,
1057 	.get_sset_count		         = ibmveth_get_sset_count,
1058 	.get_ethtool_stats	         = ibmveth_get_ethtool_stats,
1059 	.get_link_ksettings	         = ibmveth_get_link_ksettings,
1060 	.set_link_ksettings              = ibmveth_set_link_ksettings,
1061 	.get_channels			 = ibmveth_get_channels,
1062 	.set_channels			 = ibmveth_set_channels
1063 };
1064 
1065 static int ibmveth_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1066 {
1067 	return -EOPNOTSUPP;
1068 }
1069 
1070 static int ibmveth_send(struct ibmveth_adapter *adapter,
1071 			unsigned long desc, unsigned long mss)
1072 {
1073 	unsigned long correlator;
1074 	unsigned int retry_count;
1075 	unsigned long ret;
1076 
1077 	/*
1078 	 * The retry count sets a maximum for the number of broadcast and
1079 	 * multicast destinations within the system.
1080 	 */
1081 	retry_count = 1024;
1082 	correlator = 0;
1083 	do {
1084 		ret = h_send_logical_lan(adapter->vdev->unit_address, desc,
1085 					 correlator, &correlator, mss,
1086 					 adapter->fw_large_send_support);
1087 	} while ((ret == H_BUSY) && (retry_count--));
1088 
1089 	if (ret != H_SUCCESS && ret != H_DROPPED) {
1090 		netdev_err(adapter->netdev, "tx: h_send_logical_lan failed "
1091 			   "with rc=%ld\n", ret);
1092 		return 1;
1093 	}
1094 
1095 	return 0;
1096 }
1097 
1098 static int ibmveth_is_packet_unsupported(struct sk_buff *skb,
1099 					 struct net_device *netdev)
1100 {
1101 	struct ethhdr *ether_header;
1102 	int ret = 0;
1103 
1104 	ether_header = eth_hdr(skb);
1105 
1106 	if (ether_addr_equal(ether_header->h_dest, netdev->dev_addr)) {
1107 		netdev_dbg(netdev, "veth doesn't support loopback packets, dropping packet.\n");
1108 		netdev->stats.tx_dropped++;
1109 		ret = -EOPNOTSUPP;
1110 	}
1111 
1112 	return ret;
1113 }
1114 
1115 static netdev_tx_t ibmveth_start_xmit(struct sk_buff *skb,
1116 				      struct net_device *netdev)
1117 {
1118 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
1119 	unsigned int desc_flags, total_bytes;
1120 	union ibmveth_buf_desc desc;
1121 	int i, queue_num = skb_get_queue_mapping(skb);
1122 	unsigned long mss = 0;
1123 
1124 	if (ibmveth_is_packet_unsupported(skb, netdev))
1125 		goto out;
1126 	/* veth can't checksum offload UDP */
1127 	if (skb->ip_summed == CHECKSUM_PARTIAL &&
1128 	    ((skb->protocol == htons(ETH_P_IP) &&
1129 	      ip_hdr(skb)->protocol != IPPROTO_TCP) ||
1130 	     (skb->protocol == htons(ETH_P_IPV6) &&
1131 	      ipv6_hdr(skb)->nexthdr != IPPROTO_TCP)) &&
1132 	    skb_checksum_help(skb)) {
1133 
1134 		netdev_err(netdev, "tx: failed to checksum packet\n");
1135 		netdev->stats.tx_dropped++;
1136 		goto out;
1137 	}
1138 
1139 	desc_flags = IBMVETH_BUF_VALID;
1140 
1141 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
1142 		unsigned char *buf = skb_transport_header(skb) +
1143 						skb->csum_offset;
1144 
1145 		desc_flags |= (IBMVETH_BUF_NO_CSUM | IBMVETH_BUF_CSUM_GOOD);
1146 
1147 		/* Need to zero out the checksum */
1148 		buf[0] = 0;
1149 		buf[1] = 0;
1150 
1151 		if (skb_is_gso(skb) && adapter->fw_large_send_support)
1152 			desc_flags |= IBMVETH_BUF_LRG_SND;
1153 	}
1154 
1155 	if (skb->ip_summed == CHECKSUM_PARTIAL && skb_is_gso(skb)) {
1156 		if (adapter->fw_large_send_support) {
1157 			mss = (unsigned long)skb_shinfo(skb)->gso_size;
1158 			adapter->tx_large_packets++;
1159 		} else if (!skb_is_gso_v6(skb)) {
1160 			/* Put -1 in the IP checksum to tell phyp it
1161 			 * is a largesend packet. Put the mss in
1162 			 * the TCP checksum.
1163 			 */
1164 			ip_hdr(skb)->check = 0xffff;
1165 			tcp_hdr(skb)->check =
1166 				cpu_to_be16(skb_shinfo(skb)->gso_size);
1167 			adapter->tx_large_packets++;
1168 		}
1169 	}
1170 
1171 	/* Copy header into mapped buffer */
1172 	if (unlikely(skb->len > adapter->tx_ltb_size)) {
1173 		netdev_err(adapter->netdev, "tx: packet size (%u) exceeds ltb (%u)\n",
1174 			   skb->len, adapter->tx_ltb_size);
1175 		netdev->stats.tx_dropped++;
1176 		goto out;
1177 	}
1178 	memcpy(adapter->tx_ltb_ptr[queue_num], skb->data, skb_headlen(skb));
1179 	total_bytes = skb_headlen(skb);
1180 	/* Copy frags into mapped buffers */
1181 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1182 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1183 
1184 		memcpy(adapter->tx_ltb_ptr[queue_num] + total_bytes,
1185 		       skb_frag_address_safe(frag), skb_frag_size(frag));
1186 		total_bytes += skb_frag_size(frag);
1187 	}
1188 
1189 	if (unlikely(total_bytes != skb->len)) {
1190 		netdev_err(adapter->netdev, "tx: incorrect packet len copied into ltb (%u != %u)\n",
1191 			   skb->len, total_bytes);
1192 		netdev->stats.tx_dropped++;
1193 		goto out;
1194 	}
1195 	desc.fields.flags_len = desc_flags | skb->len;
1196 	desc.fields.address = adapter->tx_ltb_dma[queue_num];
1197 	/* finish writing to long_term_buff before VIOS accessing it */
1198 	dma_wmb();
1199 
1200 	if (ibmveth_send(adapter, desc.desc, mss)) {
1201 		adapter->tx_send_failed++;
1202 		netdev->stats.tx_dropped++;
1203 	} else {
1204 		netdev->stats.tx_packets++;
1205 		netdev->stats.tx_bytes += skb->len;
1206 	}
1207 
1208 out:
1209 	dev_consume_skb_any(skb);
1210 	return NETDEV_TX_OK;
1211 
1212 
1213 }
1214 
1215 static void ibmveth_rx_mss_helper(struct sk_buff *skb, u16 mss, int lrg_pkt)
1216 {
1217 	struct tcphdr *tcph;
1218 	int offset = 0;
1219 	int hdr_len;
1220 
1221 	/* only TCP packets will be aggregated */
1222 	if (skb->protocol == htons(ETH_P_IP)) {
1223 		struct iphdr *iph = (struct iphdr *)skb->data;
1224 
1225 		if (iph->protocol == IPPROTO_TCP) {
1226 			offset = iph->ihl * 4;
1227 			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1228 		} else {
1229 			return;
1230 		}
1231 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
1232 		struct ipv6hdr *iph6 = (struct ipv6hdr *)skb->data;
1233 
1234 		if (iph6->nexthdr == IPPROTO_TCP) {
1235 			offset = sizeof(struct ipv6hdr);
1236 			skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1237 		} else {
1238 			return;
1239 		}
1240 	} else {
1241 		return;
1242 	}
1243 	/* if mss is not set through Large Packet bit/mss in rx buffer,
1244 	 * expect that the mss will be written to the tcp header checksum.
1245 	 */
1246 	tcph = (struct tcphdr *)(skb->data + offset);
1247 	if (lrg_pkt) {
1248 		skb_shinfo(skb)->gso_size = mss;
1249 	} else if (offset) {
1250 		skb_shinfo(skb)->gso_size = ntohs(tcph->check);
1251 		tcph->check = 0;
1252 	}
1253 
1254 	if (skb_shinfo(skb)->gso_size) {
1255 		hdr_len = offset + tcph->doff * 4;
1256 		skb_shinfo(skb)->gso_segs =
1257 				DIV_ROUND_UP(skb->len - hdr_len,
1258 					     skb_shinfo(skb)->gso_size);
1259 	}
1260 }
1261 
1262 static void ibmveth_rx_csum_helper(struct sk_buff *skb,
1263 				   struct ibmveth_adapter *adapter)
1264 {
1265 	struct iphdr *iph = NULL;
1266 	struct ipv6hdr *iph6 = NULL;
1267 	__be16 skb_proto = 0;
1268 	u16 iphlen = 0;
1269 	u16 iph_proto = 0;
1270 	u16 tcphdrlen = 0;
1271 
1272 	skb_proto = be16_to_cpu(skb->protocol);
1273 
1274 	if (skb_proto == ETH_P_IP) {
1275 		iph = (struct iphdr *)skb->data;
1276 
1277 		/* If the IP checksum is not offloaded and if the packet
1278 		 *  is large send, the checksum must be rebuilt.
1279 		 */
1280 		if (iph->check == 0xffff) {
1281 			iph->check = 0;
1282 			iph->check = ip_fast_csum((unsigned char *)iph,
1283 						  iph->ihl);
1284 		}
1285 
1286 		iphlen = iph->ihl * 4;
1287 		iph_proto = iph->protocol;
1288 	} else if (skb_proto == ETH_P_IPV6) {
1289 		iph6 = (struct ipv6hdr *)skb->data;
1290 		iphlen = sizeof(struct ipv6hdr);
1291 		iph_proto = iph6->nexthdr;
1292 	}
1293 
1294 	/* When CSO is enabled the TCP checksum may have be set to NULL by
1295 	 * the sender given that we zeroed out TCP checksum field in
1296 	 * transmit path (refer ibmveth_start_xmit routine). In this case set
1297 	 * up CHECKSUM_PARTIAL. If the packet is forwarded, the checksum will
1298 	 * then be recalculated by the destination NIC (CSO must be enabled
1299 	 * on the destination NIC).
1300 	 *
1301 	 * In an OVS environment, when a flow is not cached, specifically for a
1302 	 * new TCP connection, the first packet information is passed up to
1303 	 * the user space for finding a flow. During this process, OVS computes
1304 	 * checksum on the first packet when CHECKSUM_PARTIAL flag is set.
1305 	 *
1306 	 * So, re-compute TCP pseudo header checksum.
1307 	 */
1308 
1309 	if (iph_proto == IPPROTO_TCP) {
1310 		struct tcphdr *tcph = (struct tcphdr *)(skb->data + iphlen);
1311 
1312 		if (tcph->check == 0x0000) {
1313 			/* Recompute TCP pseudo header checksum  */
1314 			tcphdrlen = skb->len - iphlen;
1315 			if (skb_proto == ETH_P_IP)
1316 				tcph->check =
1317 				 ~csum_tcpudp_magic(iph->saddr,
1318 				iph->daddr, tcphdrlen, iph_proto, 0);
1319 			else if (skb_proto == ETH_P_IPV6)
1320 				tcph->check =
1321 				 ~csum_ipv6_magic(&iph6->saddr,
1322 				&iph6->daddr, tcphdrlen, iph_proto, 0);
1323 			/* Setup SKB fields for checksum offload */
1324 			skb_partial_csum_set(skb, iphlen,
1325 					     offsetof(struct tcphdr, check));
1326 			skb_reset_network_header(skb);
1327 		}
1328 	}
1329 }
1330 
1331 static int ibmveth_poll(struct napi_struct *napi, int budget)
1332 {
1333 	struct ibmveth_adapter *adapter =
1334 			container_of(napi, struct ibmveth_adapter, napi);
1335 	struct net_device *netdev = adapter->netdev;
1336 	int frames_processed = 0;
1337 	unsigned long lpar_rc;
1338 	u16 mss = 0;
1339 
1340 	while (frames_processed < budget) {
1341 		if (!ibmveth_rxq_pending_buffer(adapter))
1342 			break;
1343 
1344 		smp_rmb();
1345 		if (!ibmveth_rxq_buffer_valid(adapter)) {
1346 			wmb(); /* suggested by larson1 */
1347 			adapter->rx_invalid_buffer++;
1348 			netdev_dbg(netdev, "recycling invalid buffer\n");
1349 			ibmveth_rxq_recycle_buffer(adapter);
1350 		} else {
1351 			struct sk_buff *skb, *new_skb;
1352 			int length = ibmveth_rxq_frame_length(adapter);
1353 			int offset = ibmveth_rxq_frame_offset(adapter);
1354 			int csum_good = ibmveth_rxq_csum_good(adapter);
1355 			int lrg_pkt = ibmveth_rxq_large_packet(adapter);
1356 			__sum16 iph_check = 0;
1357 
1358 			skb = ibmveth_rxq_get_buffer(adapter);
1359 
1360 			/* if the large packet bit is set in the rx queue
1361 			 * descriptor, the mss will be written by PHYP eight
1362 			 * bytes from the start of the rx buffer, which is
1363 			 * skb->data at this stage
1364 			 */
1365 			if (lrg_pkt) {
1366 				__be64 *rxmss = (__be64 *)(skb->data + 8);
1367 
1368 				mss = (u16)be64_to_cpu(*rxmss);
1369 			}
1370 
1371 			new_skb = NULL;
1372 			if (length < rx_copybreak)
1373 				new_skb = netdev_alloc_skb(netdev, length);
1374 
1375 			if (new_skb) {
1376 				skb_copy_to_linear_data(new_skb,
1377 							skb->data + offset,
1378 							length);
1379 				if (rx_flush)
1380 					ibmveth_flush_buffer(skb->data,
1381 						length + offset);
1382 				if (!ibmveth_rxq_recycle_buffer(adapter))
1383 					kfree_skb(skb);
1384 				skb = new_skb;
1385 			} else {
1386 				ibmveth_rxq_harvest_buffer(adapter);
1387 				skb_reserve(skb, offset);
1388 			}
1389 
1390 			skb_put(skb, length);
1391 			skb->protocol = eth_type_trans(skb, netdev);
1392 
1393 			/* PHYP without PLSO support places a -1 in the ip
1394 			 * checksum for large send frames.
1395 			 */
1396 			if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
1397 				struct iphdr *iph = (struct iphdr *)skb->data;
1398 
1399 				iph_check = iph->check;
1400 			}
1401 
1402 			if ((length > netdev->mtu + ETH_HLEN) ||
1403 			    lrg_pkt || iph_check == 0xffff) {
1404 				ibmveth_rx_mss_helper(skb, mss, lrg_pkt);
1405 				adapter->rx_large_packets++;
1406 			}
1407 
1408 			if (csum_good) {
1409 				skb->ip_summed = CHECKSUM_UNNECESSARY;
1410 				ibmveth_rx_csum_helper(skb, adapter);
1411 			}
1412 
1413 			napi_gro_receive(napi, skb);	/* send it up */
1414 
1415 			netdev->stats.rx_packets++;
1416 			netdev->stats.rx_bytes += length;
1417 			frames_processed++;
1418 		}
1419 	}
1420 
1421 	ibmveth_replenish_task(adapter);
1422 
1423 	if (frames_processed < budget) {
1424 		napi_complete_done(napi, frames_processed);
1425 
1426 		/* We think we are done - reenable interrupts,
1427 		 * then check once more to make sure we are done.
1428 		 */
1429 		lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1430 				       VIO_IRQ_ENABLE);
1431 
1432 		BUG_ON(lpar_rc != H_SUCCESS);
1433 
1434 		if (ibmveth_rxq_pending_buffer(adapter) &&
1435 		    napi_schedule(napi)) {
1436 			lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1437 					       VIO_IRQ_DISABLE);
1438 		}
1439 	}
1440 
1441 	return frames_processed;
1442 }
1443 
1444 static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance)
1445 {
1446 	struct net_device *netdev = dev_instance;
1447 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
1448 	unsigned long lpar_rc;
1449 
1450 	if (napi_schedule_prep(&adapter->napi)) {
1451 		lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1452 				       VIO_IRQ_DISABLE);
1453 		BUG_ON(lpar_rc != H_SUCCESS);
1454 		__napi_schedule(&adapter->napi);
1455 	}
1456 	return IRQ_HANDLED;
1457 }
1458 
1459 static void ibmveth_set_multicast_list(struct net_device *netdev)
1460 {
1461 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
1462 	unsigned long lpar_rc;
1463 
1464 	if ((netdev->flags & IFF_PROMISC) ||
1465 	    (netdev_mc_count(netdev) > adapter->mcastFilterSize)) {
1466 		lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1467 					   IbmVethMcastEnableRecv |
1468 					   IbmVethMcastDisableFiltering,
1469 					   0);
1470 		if (lpar_rc != H_SUCCESS) {
1471 			netdev_err(netdev, "h_multicast_ctrl rc=%ld when "
1472 				   "entering promisc mode\n", lpar_rc);
1473 		}
1474 	} else {
1475 		struct netdev_hw_addr *ha;
1476 		/* clear the filter table & disable filtering */
1477 		lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1478 					   IbmVethMcastEnableRecv |
1479 					   IbmVethMcastDisableFiltering |
1480 					   IbmVethMcastClearFilterTable,
1481 					   0);
1482 		if (lpar_rc != H_SUCCESS) {
1483 			netdev_err(netdev, "h_multicast_ctrl rc=%ld when "
1484 				   "attempting to clear filter table\n",
1485 				   lpar_rc);
1486 		}
1487 		/* add the addresses to the filter table */
1488 		netdev_for_each_mc_addr(ha, netdev) {
1489 			/* add the multicast address to the filter table */
1490 			u64 mcast_addr;
1491 			mcast_addr = ether_addr_to_u64(ha->addr);
1492 			lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1493 						   IbmVethMcastAddFilter,
1494 						   mcast_addr);
1495 			if (lpar_rc != H_SUCCESS) {
1496 				netdev_err(netdev, "h_multicast_ctrl rc=%ld "
1497 					   "when adding an entry to the filter "
1498 					   "table\n", lpar_rc);
1499 			}
1500 		}
1501 
1502 		/* re-enable filtering */
1503 		lpar_rc = h_multicast_ctrl(adapter->vdev->unit_address,
1504 					   IbmVethMcastEnableFiltering,
1505 					   0);
1506 		if (lpar_rc != H_SUCCESS) {
1507 			netdev_err(netdev, "h_multicast_ctrl rc=%ld when "
1508 				   "enabling filtering\n", lpar_rc);
1509 		}
1510 	}
1511 }
1512 
1513 static int ibmveth_change_mtu(struct net_device *dev, int new_mtu)
1514 {
1515 	struct ibmveth_adapter *adapter = netdev_priv(dev);
1516 	struct vio_dev *viodev = adapter->vdev;
1517 	int new_mtu_oh = new_mtu + IBMVETH_BUFF_OH;
1518 	int i, rc;
1519 	int need_restart = 0;
1520 
1521 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
1522 		if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size)
1523 			break;
1524 
1525 	if (i == IBMVETH_NUM_BUFF_POOLS)
1526 		return -EINVAL;
1527 
1528 	/* Deactivate all the buffer pools so that the next loop can activate
1529 	   only the buffer pools necessary to hold the new MTU */
1530 	if (netif_running(adapter->netdev)) {
1531 		need_restart = 1;
1532 		ibmveth_close(adapter->netdev);
1533 	}
1534 
1535 	/* Look for an active buffer pool that can hold the new MTU */
1536 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1537 		adapter->rx_buff_pool[i].active = 1;
1538 
1539 		if (new_mtu_oh <= adapter->rx_buff_pool[i].buff_size) {
1540 			WRITE_ONCE(dev->mtu, new_mtu);
1541 			vio_cmo_set_dev_desired(viodev,
1542 						ibmveth_get_desired_dma
1543 						(viodev));
1544 			if (need_restart) {
1545 				return ibmveth_open(adapter->netdev);
1546 			}
1547 			return 0;
1548 		}
1549 	}
1550 
1551 	if (need_restart && (rc = ibmveth_open(adapter->netdev)))
1552 		return rc;
1553 
1554 	return -EINVAL;
1555 }
1556 
1557 #ifdef CONFIG_NET_POLL_CONTROLLER
1558 static void ibmveth_poll_controller(struct net_device *dev)
1559 {
1560 	ibmveth_replenish_task(netdev_priv(dev));
1561 	ibmveth_interrupt(dev->irq, dev);
1562 }
1563 #endif
1564 
1565 /**
1566  * ibmveth_get_desired_dma - Calculate IO memory desired by the driver
1567  *
1568  * @vdev: struct vio_dev for the device whose desired IO mem is to be returned
1569  *
1570  * Return value:
1571  *	Number of bytes of IO data the driver will need to perform well.
1572  */
1573 static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev)
1574 {
1575 	struct net_device *netdev = dev_get_drvdata(&vdev->dev);
1576 	struct ibmveth_adapter *adapter;
1577 	struct iommu_table *tbl;
1578 	unsigned long ret;
1579 	int i;
1580 	int rxqentries = 1;
1581 
1582 	tbl = get_iommu_table_base(&vdev->dev);
1583 
1584 	/* netdev inits at probe time along with the structures we need below*/
1585 	if (netdev == NULL)
1586 		return IOMMU_PAGE_ALIGN(IBMVETH_IO_ENTITLEMENT_DEFAULT, tbl);
1587 
1588 	adapter = netdev_priv(netdev);
1589 
1590 	ret = IBMVETH_BUFF_LIST_SIZE + IBMVETH_FILT_LIST_SIZE;
1591 	ret += IOMMU_PAGE_ALIGN(netdev->mtu, tbl);
1592 	/* add size of mapped tx buffers */
1593 	ret += IOMMU_PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE, tbl);
1594 
1595 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1596 		/* add the size of the active receive buffers */
1597 		if (adapter->rx_buff_pool[i].active)
1598 			ret +=
1599 			    adapter->rx_buff_pool[i].size *
1600 			    IOMMU_PAGE_ALIGN(adapter->rx_buff_pool[i].
1601 					     buff_size, tbl);
1602 		rxqentries += adapter->rx_buff_pool[i].size;
1603 	}
1604 	/* add the size of the receive queue entries */
1605 	ret += IOMMU_PAGE_ALIGN(
1606 		rxqentries * sizeof(struct ibmveth_rx_q_entry), tbl);
1607 
1608 	return ret;
1609 }
1610 
1611 static int ibmveth_set_mac_addr(struct net_device *dev, void *p)
1612 {
1613 	struct ibmveth_adapter *adapter = netdev_priv(dev);
1614 	struct sockaddr *addr = p;
1615 	u64 mac_address;
1616 	int rc;
1617 
1618 	if (!is_valid_ether_addr(addr->sa_data))
1619 		return -EADDRNOTAVAIL;
1620 
1621 	mac_address = ether_addr_to_u64(addr->sa_data);
1622 	rc = h_change_logical_lan_mac(adapter->vdev->unit_address, mac_address);
1623 	if (rc) {
1624 		netdev_err(adapter->netdev, "h_change_logical_lan_mac failed with rc=%d\n", rc);
1625 		return rc;
1626 	}
1627 
1628 	eth_hw_addr_set(dev, addr->sa_data);
1629 
1630 	return 0;
1631 }
1632 
1633 static const struct net_device_ops ibmveth_netdev_ops = {
1634 	.ndo_open		= ibmveth_open,
1635 	.ndo_stop		= ibmveth_close,
1636 	.ndo_start_xmit		= ibmveth_start_xmit,
1637 	.ndo_set_rx_mode	= ibmveth_set_multicast_list,
1638 	.ndo_eth_ioctl		= ibmveth_ioctl,
1639 	.ndo_change_mtu		= ibmveth_change_mtu,
1640 	.ndo_fix_features	= ibmveth_fix_features,
1641 	.ndo_set_features	= ibmveth_set_features,
1642 	.ndo_validate_addr	= eth_validate_addr,
1643 	.ndo_set_mac_address    = ibmveth_set_mac_addr,
1644 #ifdef CONFIG_NET_POLL_CONTROLLER
1645 	.ndo_poll_controller	= ibmveth_poll_controller,
1646 #endif
1647 };
1648 
1649 static int ibmveth_probe(struct vio_dev *dev, const struct vio_device_id *id)
1650 {
1651 	int rc, i, mac_len;
1652 	struct net_device *netdev;
1653 	struct ibmveth_adapter *adapter;
1654 	unsigned char *mac_addr_p;
1655 	__be32 *mcastFilterSize_p;
1656 	long ret;
1657 	unsigned long ret_attr;
1658 
1659 	dev_dbg(&dev->dev, "entering ibmveth_probe for UA 0x%x\n",
1660 		dev->unit_address);
1661 
1662 	mac_addr_p = (unsigned char *)vio_get_attribute(dev, VETH_MAC_ADDR,
1663 							&mac_len);
1664 	if (!mac_addr_p) {
1665 		dev_err(&dev->dev, "Can't find VETH_MAC_ADDR attribute\n");
1666 		return -EINVAL;
1667 	}
1668 	/* Workaround for old/broken pHyp */
1669 	if (mac_len == 8)
1670 		mac_addr_p += 2;
1671 	else if (mac_len != 6) {
1672 		dev_err(&dev->dev, "VETH_MAC_ADDR attribute wrong len %d\n",
1673 			mac_len);
1674 		return -EINVAL;
1675 	}
1676 
1677 	mcastFilterSize_p = (__be32 *)vio_get_attribute(dev,
1678 							VETH_MCAST_FILTER_SIZE,
1679 							NULL);
1680 	if (!mcastFilterSize_p) {
1681 		dev_err(&dev->dev, "Can't find VETH_MCAST_FILTER_SIZE "
1682 			"attribute\n");
1683 		return -EINVAL;
1684 	}
1685 
1686 	netdev = alloc_etherdev_mqs(sizeof(struct ibmveth_adapter), IBMVETH_MAX_QUEUES, 1);
1687 	if (!netdev)
1688 		return -ENOMEM;
1689 
1690 	adapter = netdev_priv(netdev);
1691 	dev_set_drvdata(&dev->dev, netdev);
1692 
1693 	adapter->vdev = dev;
1694 	adapter->netdev = netdev;
1695 	adapter->mcastFilterSize = be32_to_cpu(*mcastFilterSize_p);
1696 	ibmveth_init_link_settings(netdev);
1697 
1698 	netif_napi_add_weight(netdev, &adapter->napi, ibmveth_poll, 16);
1699 
1700 	netdev->irq = dev->irq;
1701 	netdev->netdev_ops = &ibmveth_netdev_ops;
1702 	netdev->ethtool_ops = &netdev_ethtool_ops;
1703 	SET_NETDEV_DEV(netdev, &dev->dev);
1704 	netdev->hw_features = NETIF_F_SG;
1705 	if (vio_get_attribute(dev, "ibm,illan-options", NULL) != NULL) {
1706 		netdev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
1707 				       NETIF_F_RXCSUM;
1708 	}
1709 
1710 	netdev->features |= netdev->hw_features;
1711 
1712 	ret = h_illan_attributes(adapter->vdev->unit_address, 0, 0, &ret_attr);
1713 
1714 	/* If running older firmware, TSO should not be enabled by default */
1715 	if (ret == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_LRG_SND_SUPPORT) &&
1716 	    !old_large_send) {
1717 		netdev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
1718 		netdev->features |= netdev->hw_features;
1719 	} else {
1720 		netdev->hw_features |= NETIF_F_TSO;
1721 	}
1722 
1723 	adapter->is_active_trunk = false;
1724 	if (ret == H_SUCCESS && (ret_attr & IBMVETH_ILLAN_ACTIVE_TRUNK)) {
1725 		adapter->is_active_trunk = true;
1726 		netdev->hw_features |= NETIF_F_FRAGLIST;
1727 		netdev->features |= NETIF_F_FRAGLIST;
1728 	}
1729 
1730 	netdev->min_mtu = IBMVETH_MIN_MTU;
1731 	netdev->max_mtu = ETH_MAX_MTU - IBMVETH_BUFF_OH;
1732 
1733 	eth_hw_addr_set(netdev, mac_addr_p);
1734 
1735 	if (firmware_has_feature(FW_FEATURE_CMO))
1736 		memcpy(pool_count, pool_count_cmo, sizeof(pool_count));
1737 
1738 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1739 		struct kobject *kobj = &adapter->rx_buff_pool[i].kobj;
1740 		int error;
1741 
1742 		ibmveth_init_buffer_pool(&adapter->rx_buff_pool[i], i,
1743 					 pool_count[i], pool_size[i],
1744 					 pool_active[i]);
1745 		error = kobject_init_and_add(kobj, &ktype_veth_pool,
1746 					     &dev->dev.kobj, "pool%d", i);
1747 		if (!error)
1748 			kobject_uevent(kobj, KOBJ_ADD);
1749 	}
1750 
1751 	rc = netif_set_real_num_tx_queues(netdev, min(num_online_cpus(),
1752 						      IBMVETH_DEFAULT_QUEUES));
1753 	if (rc) {
1754 		netdev_dbg(netdev, "failed to set number of tx queues rc=%d\n",
1755 			   rc);
1756 		free_netdev(netdev);
1757 		return rc;
1758 	}
1759 	adapter->tx_ltb_size = PAGE_ALIGN(IBMVETH_MAX_TX_BUF_SIZE);
1760 	for (i = 0; i < IBMVETH_MAX_QUEUES; i++)
1761 		adapter->tx_ltb_ptr[i] = NULL;
1762 
1763 	netdev_dbg(netdev, "adapter @ 0x%p\n", adapter);
1764 	netdev_dbg(netdev, "registering netdev...\n");
1765 
1766 	ibmveth_set_features(netdev, netdev->features);
1767 
1768 	rc = register_netdev(netdev);
1769 
1770 	if (rc) {
1771 		netdev_dbg(netdev, "failed to register netdev rc=%d\n", rc);
1772 		free_netdev(netdev);
1773 		return rc;
1774 	}
1775 
1776 	netdev_dbg(netdev, "registered\n");
1777 
1778 	return 0;
1779 }
1780 
1781 static void ibmveth_remove(struct vio_dev *dev)
1782 {
1783 	struct net_device *netdev = dev_get_drvdata(&dev->dev);
1784 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
1785 	int i;
1786 
1787 	for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++)
1788 		kobject_put(&adapter->rx_buff_pool[i].kobj);
1789 
1790 	unregister_netdev(netdev);
1791 
1792 	free_netdev(netdev);
1793 	dev_set_drvdata(&dev->dev, NULL);
1794 }
1795 
1796 static struct attribute veth_active_attr;
1797 static struct attribute veth_num_attr;
1798 static struct attribute veth_size_attr;
1799 
1800 static ssize_t veth_pool_show(struct kobject *kobj,
1801 			      struct attribute *attr, char *buf)
1802 {
1803 	struct ibmveth_buff_pool *pool = container_of(kobj,
1804 						      struct ibmveth_buff_pool,
1805 						      kobj);
1806 
1807 	if (attr == &veth_active_attr)
1808 		return sprintf(buf, "%d\n", pool->active);
1809 	else if (attr == &veth_num_attr)
1810 		return sprintf(buf, "%d\n", pool->size);
1811 	else if (attr == &veth_size_attr)
1812 		return sprintf(buf, "%d\n", pool->buff_size);
1813 	return 0;
1814 }
1815 
1816 static ssize_t veth_pool_store(struct kobject *kobj, struct attribute *attr,
1817 			       const char *buf, size_t count)
1818 {
1819 	struct ibmveth_buff_pool *pool = container_of(kobj,
1820 						      struct ibmveth_buff_pool,
1821 						      kobj);
1822 	struct net_device *netdev = dev_get_drvdata(kobj_to_dev(kobj->parent));
1823 	struct ibmveth_adapter *adapter = netdev_priv(netdev);
1824 	long value = simple_strtol(buf, NULL, 10);
1825 	long rc;
1826 
1827 	if (attr == &veth_active_attr) {
1828 		if (value && !pool->active) {
1829 			if (netif_running(netdev)) {
1830 				if (ibmveth_alloc_buffer_pool(pool)) {
1831 					netdev_err(netdev,
1832 						   "unable to alloc pool\n");
1833 					return -ENOMEM;
1834 				}
1835 				pool->active = 1;
1836 				ibmveth_close(netdev);
1837 				if ((rc = ibmveth_open(netdev)))
1838 					return rc;
1839 			} else {
1840 				pool->active = 1;
1841 			}
1842 		} else if (!value && pool->active) {
1843 			int mtu = netdev->mtu + IBMVETH_BUFF_OH;
1844 			int i;
1845 			/* Make sure there is a buffer pool with buffers that
1846 			   can hold a packet of the size of the MTU */
1847 			for (i = 0; i < IBMVETH_NUM_BUFF_POOLS; i++) {
1848 				if (pool == &adapter->rx_buff_pool[i])
1849 					continue;
1850 				if (!adapter->rx_buff_pool[i].active)
1851 					continue;
1852 				if (mtu <= adapter->rx_buff_pool[i].buff_size)
1853 					break;
1854 			}
1855 
1856 			if (i == IBMVETH_NUM_BUFF_POOLS) {
1857 				netdev_err(netdev, "no active pool >= MTU\n");
1858 				return -EPERM;
1859 			}
1860 
1861 			if (netif_running(netdev)) {
1862 				ibmveth_close(netdev);
1863 				pool->active = 0;
1864 				if ((rc = ibmveth_open(netdev)))
1865 					return rc;
1866 			}
1867 			pool->active = 0;
1868 		}
1869 	} else if (attr == &veth_num_attr) {
1870 		if (value <= 0 || value > IBMVETH_MAX_POOL_COUNT) {
1871 			return -EINVAL;
1872 		} else {
1873 			if (netif_running(netdev)) {
1874 				ibmveth_close(netdev);
1875 				pool->size = value;
1876 				if ((rc = ibmveth_open(netdev)))
1877 					return rc;
1878 			} else {
1879 				pool->size = value;
1880 			}
1881 		}
1882 	} else if (attr == &veth_size_attr) {
1883 		if (value <= IBMVETH_BUFF_OH || value > IBMVETH_MAX_BUF_SIZE) {
1884 			return -EINVAL;
1885 		} else {
1886 			if (netif_running(netdev)) {
1887 				ibmveth_close(netdev);
1888 				pool->buff_size = value;
1889 				if ((rc = ibmveth_open(netdev)))
1890 					return rc;
1891 			} else {
1892 				pool->buff_size = value;
1893 			}
1894 		}
1895 	}
1896 
1897 	/* kick the interrupt handler to allocate/deallocate pools */
1898 	ibmveth_interrupt(netdev->irq, netdev);
1899 	return count;
1900 }
1901 
1902 
1903 #define ATTR(_name, _mode)				\
1904 	struct attribute veth_##_name##_attr = {	\
1905 	.name = __stringify(_name), .mode = _mode,	\
1906 	};
1907 
1908 static ATTR(active, 0644);
1909 static ATTR(num, 0644);
1910 static ATTR(size, 0644);
1911 
1912 static struct attribute *veth_pool_attrs[] = {
1913 	&veth_active_attr,
1914 	&veth_num_attr,
1915 	&veth_size_attr,
1916 	NULL,
1917 };
1918 ATTRIBUTE_GROUPS(veth_pool);
1919 
1920 static const struct sysfs_ops veth_pool_ops = {
1921 	.show   = veth_pool_show,
1922 	.store  = veth_pool_store,
1923 };
1924 
1925 static struct kobj_type ktype_veth_pool = {
1926 	.release        = NULL,
1927 	.sysfs_ops      = &veth_pool_ops,
1928 	.default_groups = veth_pool_groups,
1929 };
1930 
1931 static int ibmveth_resume(struct device *dev)
1932 {
1933 	struct net_device *netdev = dev_get_drvdata(dev);
1934 	ibmveth_interrupt(netdev->irq, netdev);
1935 	return 0;
1936 }
1937 
1938 static const struct vio_device_id ibmveth_device_table[] = {
1939 	{ "network", "IBM,l-lan"},
1940 	{ "", "" }
1941 };
1942 MODULE_DEVICE_TABLE(vio, ibmveth_device_table);
1943 
1944 static const struct dev_pm_ops ibmveth_pm_ops = {
1945 	.resume = ibmveth_resume
1946 };
1947 
1948 static struct vio_driver ibmveth_driver = {
1949 	.id_table	= ibmveth_device_table,
1950 	.probe		= ibmveth_probe,
1951 	.remove		= ibmveth_remove,
1952 	.get_desired_dma = ibmveth_get_desired_dma,
1953 	.name		= ibmveth_driver_name,
1954 	.pm		= &ibmveth_pm_ops,
1955 };
1956 
1957 static int __init ibmveth_module_init(void)
1958 {
1959 	printk(KERN_DEBUG "%s: %s %s\n", ibmveth_driver_name,
1960 	       ibmveth_driver_string, ibmveth_driver_version);
1961 
1962 	return vio_register_driver(&ibmveth_driver);
1963 }
1964 
1965 static void __exit ibmveth_module_exit(void)
1966 {
1967 	vio_unregister_driver(&ibmveth_driver);
1968 }
1969 
1970 module_init(ibmveth_module_init);
1971 module_exit(ibmveth_module_exit);
1972