25 static inline u8 *ef4_tx_get_copy_buffer(struct ef4_tx_queue *tx_queue,  in ef4_tx_get_copy_buffer()  argument
28 	unsigned int index = ef4_tx_queue_get_insert_index(tx_queue);  in ef4_tx_get_copy_buffer()
30 		&tx_queue->cb_page[index >> (PAGE_SHIFT - EF4_TX_CB_ORDER)];  in ef4_tx_get_copy_buffer()
35 	    ef4_nic_alloc_buffer(tx_queue->efx, page_buf, PAGE_SIZE,  in ef4_tx_get_copy_buffer()
43 static void ef4_dequeue_buffer(struct ef4_tx_queue *tx_queue,  in ef4_dequeue_buffer()  argument
49 		struct device *dma_dev = &tx_queue->efx->pci_dev->dev;  in ef4_dequeue_buffer()
64 		netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,  in ef4_dequeue_buffer()
66 			   tx_queue->queue, tx_queue->read_count);  in ef4_dequeue_buffer()
140 static int ef4_enqueue_skb_copy(struct ef4_tx_queue *tx_queue,  in ef4_enqueue_skb_copy()  argument
143 	unsigned int min_len = tx_queue->tx_min_size;  in ef4_enqueue_skb_copy()
151 	buffer = ef4_tx_queue_get_insert_buffer(tx_queue);  in ef4_enqueue_skb_copy()
153 	copy_buffer = ef4_tx_get_copy_buffer(tx_queue, buffer);  in ef4_enqueue_skb_copy()
169 	++tx_queue->insert_count;  in ef4_enqueue_skb_copy()
173 static struct ef4_tx_buffer *ef4_tx_map_chunk(struct ef4_tx_queue *tx_queue,  in ef4_tx_map_chunk()  argument
177 	const struct ef4_nic_type *nic_type = tx_queue->efx->type;  in ef4_tx_map_chunk()
183 		buffer = ef4_tx_queue_get_insert_buffer(tx_queue);  in ef4_tx_map_chunk()
184 		dma_len = nic_type->tx_limit_len(tx_queue, dma_addr, len);  in ef4_tx_map_chunk()
191 		++tx_queue->insert_count;  in ef4_tx_map_chunk()
199 static int ef4_tx_map_data(struct ef4_tx_queue *tx_queue, struct sk_buff *skb)  in ef4_tx_map_data()  argument
201 	struct ef4_nic *efx = tx_queue->efx;  in ef4_tx_map_data()
226 		buffer = ef4_tx_map_chunk(tx_queue, dma_addr, len);  in ef4_tx_map_data()
261 static void ef4_enqueue_unwind(struct ef4_tx_queue *tx_queue)  in ef4_enqueue_unwind()  argument
266 	while (tx_queue->insert_count != tx_queue->write_count) {  in ef4_enqueue_unwind()
267 		--tx_queue->insert_count;  in ef4_enqueue_unwind()
268 		buffer = __ef4_tx_queue_get_insert_buffer(tx_queue);  in ef4_enqueue_unwind()
269 		ef4_dequeue_buffer(tx_queue, buffer, NULL, NULL);  in ef4_enqueue_unwind()
289 netdev_tx_t ef4_enqueue_skb(struct ef4_tx_queue *tx_queue, struct sk_buff *skb)  in ef4_enqueue_skb()  argument
297 	if (skb_len < tx_queue->tx_min_size ||  in ef4_enqueue_skb()
300 		if (ef4_enqueue_skb_copy(tx_queue, skb))  in ef4_enqueue_skb()
302 		tx_queue->cb_packets++;  in ef4_enqueue_skb()
307 	if (!data_mapped && (ef4_tx_map_data(tx_queue, skb)))  in ef4_enqueue_skb()
311 	netdev_tx_sent_queue(tx_queue->core_txq, skb_len);  in ef4_enqueue_skb()
314 	if (!netdev_xmit_more() || netif_xmit_stopped(tx_queue->core_txq)) {  in ef4_enqueue_skb()
315 		struct ef4_tx_queue *txq2 = ef4_tx_queue_partner(tx_queue);  in ef4_enqueue_skb()
324 		ef4_nic_push_buffers(tx_queue);  in ef4_enqueue_skb()
326 		tx_queue->xmit_more_available = netdev_xmit_more();  in ef4_enqueue_skb()
329 	tx_queue->tx_packets++;  in ef4_enqueue_skb()
331 	ef4_tx_maybe_stop_queue(tx_queue);  in ef4_enqueue_skb()
337 	ef4_enqueue_unwind(tx_queue);  in ef4_enqueue_skb()
347 static void ef4_dequeue_buffers(struct ef4_tx_queue *tx_queue,  in ef4_dequeue_buffers()  argument
352 	struct ef4_nic *efx = tx_queue->efx;  in ef4_dequeue_buffers()
355 	stop_index = (index + 1) & tx_queue->ptr_mask;  in ef4_dequeue_buffers()
356 	read_ptr = tx_queue->read_count & tx_queue->ptr_mask;  in ef4_dequeue_buffers()
359 		struct ef4_tx_buffer *buffer = &tx_queue->buffer[read_ptr];  in ef4_dequeue_buffers()
365 				  tx_queue->queue, read_ptr);  in ef4_dequeue_buffers()
370 		ef4_dequeue_buffer(tx_queue, buffer, pkts_compl, bytes_compl);  in ef4_dequeue_buffers()
372 		++tx_queue->read_count;  in ef4_dequeue_buffers()
373 		read_ptr = tx_queue->read_count & tx_queue->ptr_mask;  in ef4_dequeue_buffers()
390 	struct ef4_tx_queue *tx_queue;  in ef4_hard_start_xmit()  local
401 	tx_queue = ef4_get_tx_queue(efx, index, type);  in ef4_hard_start_xmit()
403 	return ef4_enqueue_skb(tx_queue, skb);  in ef4_hard_start_xmit()
406 void ef4_init_tx_queue_core_txq(struct ef4_tx_queue *tx_queue)  in ef4_init_tx_queue_core_txq()  argument
408 	struct ef4_nic *efx = tx_queue->efx;  in ef4_init_tx_queue_core_txq()
411 	tx_queue->core_txq =  in ef4_init_tx_queue_core_txq()
413 				    tx_queue->queue / EF4_TXQ_TYPES +  in ef4_init_tx_queue_core_txq()
414 				    ((tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI) ?  in ef4_init_tx_queue_core_txq()
424 	struct ef4_tx_queue *tx_queue;  in ef4_setup_tc()  local
449 			ef4_for_each_possible_channel_tx_queue(tx_queue,  in ef4_setup_tc()
451 				if (!(tx_queue->queue & EF4_TXQ_TYPE_HIGHPRI))  in ef4_setup_tc()
453 				if (!tx_queue->buffer) {  in ef4_setup_tc()
454 					rc = ef4_probe_tx_queue(tx_queue);  in ef4_setup_tc()
458 				if (!tx_queue->initialised)  in ef4_setup_tc()
459 					ef4_init_tx_queue(tx_queue);  in ef4_setup_tc()
460 				ef4_init_tx_queue_core_txq(tx_queue);  in ef4_setup_tc()
484 void ef4_xmit_done(struct ef4_tx_queue *tx_queue, unsigned int index)  in ef4_xmit_done()  argument
487 	struct ef4_nic *efx = tx_queue->efx;  in ef4_xmit_done()
491 	EF4_BUG_ON_PARANOID(index > tx_queue->ptr_mask);  in ef4_xmit_done()
493 	ef4_dequeue_buffers(tx_queue, index, &pkts_compl, &bytes_compl);  in ef4_xmit_done()
494 	tx_queue->pkts_compl += pkts_compl;  in ef4_xmit_done()
495 	tx_queue->bytes_compl += bytes_compl;  in ef4_xmit_done()
498 		++tx_queue->merge_events;  in ef4_xmit_done()
505 	if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&  in ef4_xmit_done()
508 		txq2 = ef4_tx_queue_partner(tx_queue);  in ef4_xmit_done()
509 		fill_level = max(tx_queue->insert_count - tx_queue->read_count,  in ef4_xmit_done()
512 			netif_tx_wake_queue(tx_queue->core_txq);  in ef4_xmit_done()
516 	if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {  in ef4_xmit_done()
517 		tx_queue->old_write_count = READ_ONCE(tx_queue->write_count);  in ef4_xmit_done()
518 		if (tx_queue->read_count == tx_queue->old_write_count) {  in ef4_xmit_done()
520 			tx_queue->empty_read_count =  in ef4_xmit_done()
521 				tx_queue->read_count | EF4_EMPTY_COUNT_VALID;  in ef4_xmit_done()
526 static unsigned int ef4_tx_cb_page_count(struct ef4_tx_queue *tx_queue)  in ef4_tx_cb_page_count()  argument
528 	return DIV_ROUND_UP(tx_queue->ptr_mask + 1, PAGE_SIZE >> EF4_TX_CB_ORDER);  in ef4_tx_cb_page_count()
531 int ef4_probe_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_probe_tx_queue()  argument
533 	struct ef4_nic *efx = tx_queue->efx;  in ef4_probe_tx_queue()
540 	tx_queue->ptr_mask = entries - 1;  in ef4_probe_tx_queue()
544 		  tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);  in ef4_probe_tx_queue()
547 	tx_queue->buffer = kzalloc_objs(*tx_queue->buffer, entries);  in ef4_probe_tx_queue()
548 	if (!tx_queue->buffer)  in ef4_probe_tx_queue()
551 	tx_queue->cb_page = kzalloc_objs(tx_queue->cb_page[0],  in ef4_probe_tx_queue()
552 					 ef4_tx_cb_page_count(tx_queue));  in ef4_probe_tx_queue()
553 	if (!tx_queue->cb_page) {  in ef4_probe_tx_queue()
559 	rc = ef4_nic_probe_tx(tx_queue);  in ef4_probe_tx_queue()
566 	kfree(tx_queue->cb_page);  in ef4_probe_tx_queue()
567 	tx_queue->cb_page = NULL;  in ef4_probe_tx_queue()
569 	kfree(tx_queue->buffer);  in ef4_probe_tx_queue()
570 	tx_queue->buffer = NULL;  in ef4_probe_tx_queue()
574 void ef4_init_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_init_tx_queue()  argument
576 	struct ef4_nic *efx = tx_queue->efx;  in ef4_init_tx_queue()
579 		  "initialising TX queue %d\n", tx_queue->queue);  in ef4_init_tx_queue()
581 	tx_queue->insert_count = 0;  in ef4_init_tx_queue()
582 	tx_queue->write_count = 0;  in ef4_init_tx_queue()
583 	tx_queue->old_write_count = 0;  in ef4_init_tx_queue()
584 	tx_queue->read_count = 0;  in ef4_init_tx_queue()
585 	tx_queue->old_read_count = 0;  in ef4_init_tx_queue()
586 	tx_queue->empty_read_count = 0 | EF4_EMPTY_COUNT_VALID;  in ef4_init_tx_queue()
587 	tx_queue->xmit_more_available = false;  in ef4_init_tx_queue()
590 	tx_queue->tx_min_size = EF4_WORKAROUND_15592(efx) ? 33 : 0;  in ef4_init_tx_queue()
593 	ef4_nic_init_tx(tx_queue);  in ef4_init_tx_queue()
595 	tx_queue->initialised = true;  in ef4_init_tx_queue()
598 void ef4_fini_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_fini_tx_queue()  argument
602 	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,  in ef4_fini_tx_queue()
603 		  "shutting down TX queue %d\n", tx_queue->queue);  in ef4_fini_tx_queue()
605 	if (!tx_queue->buffer)  in ef4_fini_tx_queue()
609 	while (tx_queue->read_count != tx_queue->write_count) {  in ef4_fini_tx_queue()
611 		buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];  in ef4_fini_tx_queue()
612 		ef4_dequeue_buffer(tx_queue, buffer, &pkts_compl, &bytes_compl);  in ef4_fini_tx_queue()
614 		++tx_queue->read_count;  in ef4_fini_tx_queue()
616 	tx_queue->xmit_more_available = false;  in ef4_fini_tx_queue()
617 	netdev_tx_reset_queue(tx_queue->core_txq);  in ef4_fini_tx_queue()
620 void ef4_remove_tx_queue(struct ef4_tx_queue *tx_queue)  in ef4_remove_tx_queue()  argument
624 	if (!tx_queue->buffer)  in ef4_remove_tx_queue()
627 	netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,  in ef4_remove_tx_queue()
628 		  "destroying TX queue %d\n", tx_queue->queue);  in ef4_remove_tx_queue()
629 	ef4_nic_remove_tx(tx_queue);  in ef4_remove_tx_queue()
631 	if (tx_queue->cb_page) {  in ef4_remove_tx_queue()
632 		for (i = 0; i < ef4_tx_cb_page_count(tx_queue); i++)  in ef4_remove_tx_queue()
633 			ef4_nic_free_buffer(tx_queue->efx,  in ef4_remove_tx_queue()
634 					    &tx_queue->cb_page[i]);  in ef4_remove_tx_queue()
635 		kfree(tx_queue->cb_page);  in ef4_remove_tx_queue()
636 		tx_queue->cb_page = NULL;  in ef4_remove_tx_queue()
639 	kfree(tx_queue->buffer);  in ef4_remove_tx_queue()
640 	tx_queue->buffer = NULL;  in ef4_remove_tx_queue()