Lines Matching +full:tx +full:- +full:queues +full:- +full:to +full:- +full:use
4 * Copyright (c) 2002-2005, K A Fraser
11 * software packages, subject to the following license:
13 * Permission is hereby granted, free of charge, to any person obtaining a copy
14 * of this source file (the "Software"), to deal in the Software without
15 * restriction, including without limitation the rights to use, copy, modify,
17 * and to permit persons to whom the Software is furnished to do so, subject to
24 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
28 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
67 "Maximum number of queues per virtual interface");
81 #define NETFRONT_SKB_CB(skb) ((struct netfront_cb *)((skb)->cb))
91 /* Queue name is interface name with "-qNNN" appended */
94 /* IRQ name is queue name with "-tx" or "-rx" appended */
108 unsigned int id; /* Queue ID, 0-based */
109 char name[QUEUE_NAME_SIZE]; /* DEVNAME-qN */
122 char tx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-tx */
123 char rx_irq_name[IRQ_NAME_SIZE]; /* DEVNAME-qN-rx */
126 struct xen_netif_tx_front_ring tx; member
130 * {tx,rx}_skbs store outstanding skbuffs. Free tx_skb entries
166 /* Multi-queue support */
167 struct netfront_queue *queues; member
188 struct xen_netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1];
215 return idx & (NET_RX_RING_SIZE - 1); in xennet_rxidx()
222 struct sk_buff *skb = queue->rx_skbs[i]; in xennet_get_rx_skb()
223 queue->rx_skbs[i] = NULL; in xennet_get_rx_skb()
231 grant_ref_t ref = queue->grant_rx_ref[i]; in xennet_get_rx_ref()
232 queue->grant_rx_ref[i] = INVALID_GRANT_REF; in xennet_get_rx_ref()
242 return dev->features & NETIF_F_SG; in xennet_can_sg()
250 napi_schedule(&queue->napi); in rx_refill_timeout()
255 return (queue->tx.req_prod_pvt - queue->tx.rsp_cons) < in netfront_tx_slot_available()
256 (NET_TX_RING_SIZE - XEN_NETIF_NR_SLOTS_MIN - 1); in netfront_tx_slot_available()
261 struct net_device *dev = queue->info->netdev; in xennet_maybe_wake_tx()
262 struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, queue->id); in xennet_maybe_wake_tx()
267 netif_tx_wake_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_maybe_wake_tx()
276 skb = __netdev_alloc_skb(queue->info->netdev, in xennet_alloc_one_rx_buffer()
282 page = page_pool_alloc_pages(queue->page_pool, in xennet_alloc_one_rx_buffer()
291 /* Align ip header to a 16 bytes boundary */ in xennet_alloc_one_rx_buffer()
293 skb->dev = queue->info->netdev; in xennet_alloc_one_rx_buffer()
301 RING_IDX req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
305 if (unlikely(!netif_carrier_ok(queue->info->netdev))) in xennet_alloc_rx_buffers()
308 for (req_prod = queue->rx.req_prod_pvt; in xennet_alloc_rx_buffers()
309 req_prod - queue->rx.rsp_cons < NET_RX_RING_SIZE; in xennet_alloc_rx_buffers()
319 err = -ENOMEM; in xennet_alloc_rx_buffers()
325 BUG_ON(queue->rx_skbs[id]); in xennet_alloc_rx_buffers()
326 queue->rx_skbs[id] = skb; in xennet_alloc_rx_buffers()
328 ref = gnttab_claim_grant_reference(&queue->gref_rx_head); in xennet_alloc_rx_buffers()
330 queue->grant_rx_ref[id] = ref; in xennet_alloc_rx_buffers()
332 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); in xennet_alloc_rx_buffers()
334 req = RING_GET_REQUEST(&queue->rx, req_prod); in xennet_alloc_rx_buffers()
336 queue->info->xbdev->otherend_id, in xennet_alloc_rx_buffers()
339 req->id = id; in xennet_alloc_rx_buffers()
340 req->gref = ref; in xennet_alloc_rx_buffers()
343 queue->rx.req_prod_pvt = req_prod; in xennet_alloc_rx_buffers()
350 if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || in xennet_alloc_rx_buffers()
352 mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); in xennet_alloc_rx_buffers()
356 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->rx, notify); in xennet_alloc_rx_buffers()
358 notify_remote_via_irq(queue->rx_irq); in xennet_alloc_rx_buffers()
364 unsigned int num_queues = dev->real_num_tx_queues; in xennet_open()
368 if (!np->queues || np->broken) in xennet_open()
369 return -ENODEV; in xennet_open()
372 queue = &np->queues[i]; in xennet_open()
373 napi_enable(&queue->napi); in xennet_open()
375 spin_lock_bh(&queue->rx_lock); in xennet_open()
378 queue->rx.sring->rsp_event = queue->rx.rsp_cons + 1; in xennet_open()
379 if (RING_HAS_UNCONSUMED_RESPONSES(&queue->rx)) in xennet_open()
380 napi_schedule(&queue->napi); in xennet_open()
382 spin_unlock_bh(&queue->rx_lock); in xennet_open()
397 const struct device *dev = &queue->info->netdev->dev; in xennet_tx_buf_gc()
399 BUG_ON(!netif_carrier_ok(queue->info->netdev)); in xennet_tx_buf_gc()
402 prod = queue->tx.sring->rsp_prod; in xennet_tx_buf_gc()
403 if (RING_RESPONSE_PROD_OVERFLOW(&queue->tx, prod)) { in xennet_tx_buf_gc()
405 prod - queue->tx.rsp_cons); in xennet_tx_buf_gc()
408 rmb(); /* Ensure we see responses up to 'rp'. */ in xennet_tx_buf_gc()
410 for (cons = queue->tx.rsp_cons; cons != prod; cons++) { in xennet_tx_buf_gc()
415 RING_COPY_RESPONSE(&queue->tx, cons, &txrsp); in xennet_tx_buf_gc()
420 if (id >= RING_SIZE(&queue->tx)) { in xennet_tx_buf_gc()
426 if (queue->tx_link[id] != TX_PENDING) { in xennet_tx_buf_gc()
432 queue->tx_link[id] = TX_LINK_NONE; in xennet_tx_buf_gc()
433 skb = queue->tx_skbs[id]; in xennet_tx_buf_gc()
434 queue->tx_skbs[id] = NULL; in xennet_tx_buf_gc()
436 queue->grant_tx_ref[id]))) { in xennet_tx_buf_gc()
438 "Grant still in use by backend domain\n"); in xennet_tx_buf_gc()
442 &queue->gref_tx_head, queue->grant_tx_ref[id]); in xennet_tx_buf_gc()
443 queue->grant_tx_ref[id] = INVALID_GRANT_REF; in xennet_tx_buf_gc()
444 queue->grant_tx_page[id] = NULL; in xennet_tx_buf_gc()
445 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, id); in xennet_tx_buf_gc()
449 queue->tx.rsp_cons = prod; in xennet_tx_buf_gc()
451 RING_FINAL_CHECK_FOR_RESPONSES(&queue->tx, more_to_do); in xennet_tx_buf_gc()
459 queue->info->broken = true; in xennet_tx_buf_gc()
460 dev_alert(dev, "Disabled for further use\n"); in xennet_tx_buf_gc()
469 struct xen_netif_tx_request *tx; /* Last request on ring page */ member
479 struct xen_netif_tx_request *tx; in xennet_tx_setup_grant() local
482 struct page *page = info->page; in xennet_tx_setup_grant()
483 struct netfront_queue *queue = info->queue; in xennet_tx_setup_grant()
484 struct sk_buff *skb = info->skb; in xennet_tx_setup_grant()
486 id = get_id_from_list(&queue->tx_skb_freelist, queue->tx_link); in xennet_tx_setup_grant()
487 tx = RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_tx_setup_grant()
488 ref = gnttab_claim_grant_reference(&queue->gref_tx_head); in xennet_tx_setup_grant()
491 gnttab_grant_foreign_access_ref(ref, queue->info->xbdev->otherend_id, in xennet_tx_setup_grant()
494 queue->tx_skbs[id] = skb; in xennet_tx_setup_grant()
495 queue->grant_tx_page[id] = page; in xennet_tx_setup_grant()
496 queue->grant_tx_ref[id] = ref; in xennet_tx_setup_grant()
498 info->tx_local.id = id; in xennet_tx_setup_grant()
499 info->tx_local.gref = ref; in xennet_tx_setup_grant()
500 info->tx_local.offset = offset; in xennet_tx_setup_grant()
501 info->tx_local.size = len; in xennet_tx_setup_grant()
502 info->tx_local.flags = 0; in xennet_tx_setup_grant()
504 *tx = info->tx_local; in xennet_tx_setup_grant()
507 * Put the request in the pending queue, it will be set to be pending in xennet_tx_setup_grant()
508 * when the producer index is about to be raised. in xennet_tx_setup_grant()
510 add_id_to_list(&queue->tx_pend_queue, queue->tx_link, id); in xennet_tx_setup_grant()
512 info->tx = tx; in xennet_tx_setup_grant()
513 info->size += info->tx_local.size; in xennet_tx_setup_grant()
520 info->size = 0; in xennet_make_first_txreq()
522 gnttab_for_one_grant(info->page, offset, len, xennet_tx_setup_grant, info); in xennet_make_first_txreq()
524 return info->tx; in xennet_make_first_txreq()
532 info->tx->flags |= XEN_NETTXF_more_data; in xennet_make_one_txreq()
533 skb_get(info->skb); in xennet_make_one_txreq()
547 info->page = page; in xennet_make_txreqs()
548 info->size = 0; in xennet_make_txreqs()
556 len -= info->size; in xennet_make_txreqs()
561 * Count how many ring slots are required to send this skb. Each frag
566 int i, frags = skb_shinfo(skb)->nr_frags; in xennet_count_skb_slots()
569 slots = gnttab_count_grant(offset_in_page(skb->data), in xennet_count_skb_slots()
573 skb_frag_t *frag = skb_shinfo(skb)->frags + i; in xennet_count_skb_slots()
589 unsigned int num_queues = dev->real_num_tx_queues; in xennet_select_queue()
608 while ((i = get_id_from_list(&queue->tx_pend_queue, queue->tx_link)) != in xennet_mark_tx_pending()
610 queue->tx_link[i] = TX_PENDING; in xennet_mark_tx_pending()
618 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); in xennet_xdp_xmit_one()
622 .page = virt_to_page(xdpf->data), in xennet_xdp_xmit_one()
627 offset_in_page(xdpf->data), in xennet_xdp_xmit_one()
628 xdpf->len); in xennet_xdp_xmit_one()
632 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_xdp_xmit_one()
634 notify_remote_via_irq(queue->tx_irq); in xennet_xdp_xmit_one()
636 u64_stats_update_begin(&tx_stats->syncp); in xennet_xdp_xmit_one()
637 tx_stats->bytes += xdpf->len; in xennet_xdp_xmit_one()
638 tx_stats->packets++; in xennet_xdp_xmit_one()
639 u64_stats_update_end(&tx_stats->syncp); in xennet_xdp_xmit_one()
647 unsigned int num_queues = dev->real_num_tx_queues; in xennet_xdp_xmit()
654 if (unlikely(np->broken)) in xennet_xdp_xmit()
655 return -ENODEV; in xennet_xdp_xmit()
657 return -EINVAL; in xennet_xdp_xmit()
659 queue = &np->queues[smp_processor_id() % num_queues]; in xennet_xdp_xmit()
661 spin_lock_irqsave(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
671 spin_unlock_irqrestore(&queue->tx_lock, irq_flags); in xennet_xdp_xmit()
679 /* Align size to allocate full pages and avoid contiguous data leaks */ in bounce_skb()
680 unsigned int size = ALIGN(skb_end_offset(skb) + skb->data_len, in bounce_skb()
687 if (!IS_ALIGNED((uintptr_t)n->head, XEN_PAGE_SIZE)) { in bounce_skb()
696 skb_put(n, skb->len); in bounce_skb()
698 BUG_ON(skb_copy_bits(skb, -headerlen, n->head, headerlen + skb->len)); in bounce_skb()
709 struct netfront_stats *tx_stats = this_cpu_ptr(np->tx_stats); in xennet_start_xmit()
720 unsigned int num_queues = dev->real_num_tx_queues; in xennet_start_xmit()
724 /* Drop the packet if no queues are set up */ in xennet_start_xmit()
727 if (unlikely(np->broken)) in xennet_start_xmit()
729 /* Determine which queue to transmit this SKB on */ in xennet_start_xmit()
731 queue = &np->queues[queue_index]; in xennet_start_xmit()
733 /* If skb->len is too big for wire format, drop skb and alert in xennet_start_xmit()
736 if (unlikely(skb->len > XEN_NETIF_MAX_TX_SIZE)) { in xennet_start_xmit()
738 "xennet: skb->len = %u, too big for wire format\n", in xennet_start_xmit()
739 skb->len); in xennet_start_xmit()
746 slots, skb->len); in xennet_start_xmit()
751 page = virt_to_page(skb->data); in xennet_start_xmit()
752 offset = offset_in_page(skb->data); in xennet_start_xmit()
757 * If the backend is not trusted bounce all data to zeroed pages to in xennet_start_xmit()
758 * avoid exposing contiguous data on the granted page not belonging to in xennet_start_xmit()
761 if (np->bounce || unlikely(PAGE_SIZE - offset < ETH_HLEN)) { in xennet_start_xmit()
767 page = virt_to_page(skb->data); in xennet_start_xmit()
768 offset = offset_in_page(skb->data); in xennet_start_xmit()
773 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_start_xmit()
778 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
792 len -= info.tx_local.size; in xennet_start_xmit()
794 if (skb->ip_summed == CHECKSUM_PARTIAL) in xennet_start_xmit()
796 first_tx->flags |= XEN_NETTXF_csum_blank | in xennet_start_xmit()
798 else if (skb->ip_summed == CHECKSUM_UNNECESSARY) in xennet_start_xmit()
800 first_tx->flags |= XEN_NETTXF_data_validated; in xennet_start_xmit()
803 if (skb_shinfo(skb)->gso_size) { in xennet_start_xmit()
807 RING_GET_REQUEST(&queue->tx, queue->tx.req_prod_pvt++); in xennet_start_xmit()
809 first_tx->flags |= XEN_NETTXF_extra_info; in xennet_start_xmit()
811 gso->u.gso.size = skb_shinfo(skb)->gso_size; in xennet_start_xmit()
812 gso->u.gso.type = (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) ? in xennet_start_xmit()
815 gso->u.gso.pad = 0; in xennet_start_xmit()
816 gso->u.gso.features = 0; in xennet_start_xmit()
818 gso->type = XEN_NETIF_EXTRA_TYPE_GSO; in xennet_start_xmit()
819 gso->flags = 0; in xennet_start_xmit()
826 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in xennet_start_xmit()
827 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in xennet_start_xmit()
834 first_tx->size = skb->len; in xennet_start_xmit()
841 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&queue->tx, notify); in xennet_start_xmit()
843 notify_remote_via_irq(queue->tx_irq); in xennet_start_xmit()
845 u64_stats_update_begin(&tx_stats->syncp); in xennet_start_xmit()
846 tx_stats->bytes += skb->len; in xennet_start_xmit()
847 tx_stats->packets++; in xennet_start_xmit()
848 u64_stats_update_end(&tx_stats->syncp); in xennet_start_xmit()
851 netif_tx_stop_queue(netdev_get_tx_queue(dev, queue->id)); in xennet_start_xmit()
853 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_start_xmit()
858 dev->stats.tx_dropped++; in xennet_start_xmit()
866 unsigned int num_queues = np->queues ? dev->real_num_tx_queues : 0; in xennet_close()
869 netif_tx_stop_all_queues(np->netdev); in xennet_close()
871 queue = &np->queues[i]; in xennet_close()
872 napi_disable(&queue->napi); in xennet_close()
881 if (!info->queues) in xennet_destroy_queues()
884 for (i = 0; i < info->netdev->real_num_tx_queues; i++) { in xennet_destroy_queues()
885 struct netfront_queue *queue = &info->queues[i]; in xennet_destroy_queues()
887 if (netif_running(info->netdev)) in xennet_destroy_queues()
888 napi_disable(&queue->napi); in xennet_destroy_queues()
889 netif_napi_del(&queue->napi); in xennet_destroy_queues()
892 kfree(info->queues); in xennet_destroy_queues()
893 info->queues = NULL; in xennet_destroy_queues()
906 spin_lock_irqsave(&queue->rx_cons_lock, flags); in xennet_set_rx_rsp_cons()
907 queue->rx.rsp_cons = val; in xennet_set_rx_rsp_cons()
908 queue->rx_rsp_unconsumed = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx); in xennet_set_rx_rsp_cons()
909 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); in xennet_set_rx_rsp_cons()
915 int new = xennet_rxidx(queue->rx.req_prod_pvt); in xennet_move_rx_slot()
917 BUG_ON(queue->rx_skbs[new]); in xennet_move_rx_slot()
918 queue->rx_skbs[new] = skb; in xennet_move_rx_slot()
919 queue->grant_rx_ref[new] = ref; in xennet_move_rx_slot()
920 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->id = new; in xennet_move_rx_slot()
921 RING_GET_REQUEST(&queue->rx, queue->rx.req_prod_pvt)->gref = ref; in xennet_move_rx_slot()
922 queue->rx.req_prod_pvt++; in xennet_move_rx_slot()
931 struct device *dev = &queue->info->netdev->dev; in xennet_get_extras()
932 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_extras()
942 err = -EBADR; in xennet_get_extras()
946 RING_COPY_RESPONSE(&queue->rx, ++cons, &extra); in xennet_get_extras()
953 err = -EINVAL; in xennet_get_extras()
955 extras[extra.type - 1] = extra; in xennet_get_extras()
972 u32 len = rx->status; in xennet_run_xdp()
976 xdp_init_buff(xdp, XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, in xennet_run_xdp()
977 &queue->xdp_rxq); in xennet_run_xdp()
986 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
990 err = xennet_xdp_xmit(queue->info->netdev, 1, &xdpf, 0); in xennet_run_xdp()
993 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
999 err = xdp_do_redirect(queue->info->netdev, xdp, prog); in xennet_run_xdp()
1002 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
1011 trace_xdp_exception(queue->info->netdev, prog, act); in xennet_run_xdp()
1015 bpf_warn_invalid_xdp_action(queue->info->netdev, prog, act); in xennet_run_xdp()
1026 struct xen_netif_rx_response *rx = &rinfo->rx, rx_local; in xennet_get_responses()
1027 int max = XEN_NETIF_NR_SLOTS_MIN + (rx->status <= RX_COPY_THRESHOLD); in xennet_get_responses()
1028 RING_IDX cons = queue->rx.rsp_cons; in xennet_get_responses()
1030 struct xen_netif_extra_info *extras = rinfo->extras; in xennet_get_responses()
1032 struct device *dev = &queue->info->netdev->dev; in xennet_get_responses()
1039 if (rx->flags & XEN_NETRXF_extra_info) { in xennet_get_responses()
1042 if (extras[XEN_NETIF_EXTRA_TYPE_XDP - 1].type) { in xennet_get_responses()
1045 xdp = &extras[XEN_NETIF_EXTRA_TYPE_XDP - 1]; in xennet_get_responses()
1046 rx->offset = xdp->u.xdp.headroom; in xennet_get_responses()
1049 cons = queue->rx.rsp_cons; in xennet_get_responses()
1056 * situation to the system controller to reboot the backend. in xennet_get_responses()
1061 rx->id); in xennet_get_responses()
1062 err = -EINVAL; in xennet_get_responses()
1066 if (unlikely(rx->status < 0 || in xennet_get_responses()
1067 rx->offset + rx->status > XEN_PAGE_SIZE)) { in xennet_get_responses()
1069 dev_warn(dev, "rx->offset: %u, size: %d\n", in xennet_get_responses()
1070 rx->offset, rx->status); in xennet_get_responses()
1072 err = -EINVAL; in xennet_get_responses()
1078 "Grant still in use by backend domain\n"); in xennet_get_responses()
1079 queue->info->broken = true; in xennet_get_responses()
1080 dev_alert(dev, "Disabled for further use\n"); in xennet_get_responses()
1081 return -EINVAL; in xennet_get_responses()
1084 gnttab_release_grant_reference(&queue->gref_rx_head, ref); in xennet_get_responses()
1087 xdp_prog = rcu_dereference(queue->xdp_prog); in xennet_get_responses()
1089 if (!(rx->flags & XEN_NETRXF_more_data)) { in xennet_get_responses()
1092 skb_frag_page(&skb_shinfo(skb)->frags[0]), in xennet_get_responses()
1095 err = -EINVAL; in xennet_get_responses()
1098 err = -EINVAL; in xennet_get_responses()
1106 if (!(rx->flags & XEN_NETRXF_more_data)) in xennet_get_responses()
1112 err = -ENOENT; in xennet_get_responses()
1116 RING_COPY_RESPONSE(&queue->rx, cons + slots, &rx_local); in xennet_get_responses()
1126 err = -E2BIG; in xennet_get_responses()
1138 if (!gso->u.gso.size) { in xennet_set_skb_gso()
1141 return -EINVAL; in xennet_set_skb_gso()
1144 if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4 && in xennet_set_skb_gso()
1145 gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV6) { in xennet_set_skb_gso()
1147 pr_warn("Bad GSO type %d\n", gso->u.gso.type); in xennet_set_skb_gso()
1148 return -EINVAL; in xennet_set_skb_gso()
1151 skb_shinfo(skb)->gso_size = gso->u.gso.size; in xennet_set_skb_gso()
1152 skb_shinfo(skb)->gso_type = in xennet_set_skb_gso()
1153 (gso->u.gso.type == XEN_NETIF_GSO_TYPE_TCPV4) ? in xennet_set_skb_gso()
1158 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; in xennet_set_skb_gso()
1159 skb_shinfo(skb)->gso_segs = 0; in xennet_set_skb_gso()
1168 RING_IDX cons = queue->rx.rsp_cons; in xennet_fill_frags()
1173 skb_frag_t *nfrag = &skb_shinfo(nskb)->frags[0]; in xennet_fill_frags()
1175 RING_COPY_RESPONSE(&queue->rx, ++cons, &rx); in xennet_fill_frags()
1177 if (skb_shinfo(skb)->nr_frags == MAX_SKB_FRAGS) { in xennet_fill_frags()
1178 unsigned int pull_to = NETFRONT_SKB_CB(skb)->pull_to; in xennet_fill_frags()
1181 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); in xennet_fill_frags()
1183 if (unlikely(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS)) { in xennet_fill_frags()
1187 return -ENOENT; in xennet_fill_frags()
1190 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, in xennet_fill_frags()
1194 skb_shinfo(nskb)->nr_frags = 0; in xennet_fill_frags()
1209 * peers can fail to set NETRXF_csum_blank when sending a GSO in checksum_setup()
1210 * frame. In this case force the SKB to CHECKSUM_PARTIAL and in checksum_setup()
1213 if (skb->ip_summed != CHECKSUM_PARTIAL && skb_is_gso(skb)) { in checksum_setup()
1215 atomic_inc(&np->rx_gso_checksum_fixup); in checksum_setup()
1216 skb->ip_summed = CHECKSUM_PARTIAL; in checksum_setup()
1220 /* A non-CHECKSUM_PARTIAL SKB does not require setup. */ in checksum_setup()
1221 if (skb->ip_summed != CHECKSUM_PARTIAL) in checksum_setup()
1230 struct netfront_stats *rx_stats = this_cpu_ptr(queue->info->rx_stats); in handle_incoming_queue()
1235 int pull_to = NETFRONT_SKB_CB(skb)->pull_to; in handle_incoming_queue()
1238 __pskb_pull_tail(skb, pull_to - skb_headlen(skb)); in handle_incoming_queue()
1240 /* Ethernet work: Delayed to here as it peeks the header. */ in handle_incoming_queue()
1241 skb->protocol = eth_type_trans(skb, queue->info->netdev); in handle_incoming_queue()
1244 if (checksum_setup(queue->info->netdev, skb)) { in handle_incoming_queue()
1247 queue->info->netdev->stats.rx_errors++; in handle_incoming_queue()
1251 u64_stats_update_begin(&rx_stats->syncp); in handle_incoming_queue()
1252 rx_stats->packets++; in handle_incoming_queue()
1253 rx_stats->bytes += skb->len; in handle_incoming_queue()
1254 u64_stats_update_end(&rx_stats->syncp); in handle_incoming_queue()
1257 napi_gro_receive(&queue->napi, skb); in handle_incoming_queue()
1266 struct net_device *dev = queue->info->netdev; in xennet_poll()
1279 spin_lock(&queue->rx_lock); in xennet_poll()
1285 rp = queue->rx.sring->rsp_prod; in xennet_poll()
1286 if (RING_RESPONSE_PROD_OVERFLOW(&queue->rx, rp)) { in xennet_poll()
1287 dev_alert(&dev->dev, "Illegal number of responses %u\n", in xennet_poll()
1288 rp - queue->rx.rsp_cons); in xennet_poll()
1289 queue->info->broken = true; in xennet_poll()
1290 spin_unlock(&queue->rx_lock); in xennet_poll()
1293 rmb(); /* Ensure we see queued responses up to 'rp'. */ in xennet_poll()
1295 i = queue->rx.rsp_cons; in xennet_poll()
1298 RING_COPY_RESPONSE(&queue->rx, i, rx); in xennet_poll()
1305 if (queue->info->broken) { in xennet_poll()
1306 spin_unlock(&queue->rx_lock); in xennet_poll()
1312 dev->stats.rx_errors++; in xennet_poll()
1313 i = queue->rx.rsp_cons; in xennet_poll()
1319 if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { in xennet_poll()
1321 gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; in xennet_poll()
1326 queue->rx.rsp_cons + in xennet_poll()
1332 NETFRONT_SKB_CB(skb)->pull_to = rx->status; in xennet_poll()
1333 if (NETFRONT_SKB_CB(skb)->pull_to > RX_COPY_THRESHOLD) in xennet_poll()
1334 NETFRONT_SKB_CB(skb)->pull_to = RX_COPY_THRESHOLD; in xennet_poll()
1336 skb_frag_off_set(&skb_shinfo(skb)->frags[0], rx->offset); in xennet_poll()
1337 skb_frag_size_set(&skb_shinfo(skb)->frags[0], rx->status); in xennet_poll()
1338 skb->data_len = rx->status; in xennet_poll()
1339 skb->len += rx->status; in xennet_poll()
1344 if (rx->flags & XEN_NETRXF_csum_blank) in xennet_poll()
1345 skb->ip_summed = CHECKSUM_PARTIAL; in xennet_poll()
1346 else if (rx->flags & XEN_NETRXF_data_validated) in xennet_poll()
1347 skb->ip_summed = CHECKSUM_UNNECESSARY; in xennet_poll()
1351 i = queue->rx.rsp_cons + 1; in xennet_poll()
1360 work_done -= handle_incoming_queue(queue, &rxq); in xennet_poll()
1369 RING_FINAL_CHECK_FOR_RESPONSES(&queue->rx, more_to_do); in xennet_poll()
1374 spin_unlock(&queue->rx_lock); in xennet_poll()
1384 return -EINVAL; in xennet_change_mtu()
1385 WRITE_ONCE(dev->mtu, mtu); in xennet_change_mtu()
1396 struct netfront_stats *rx_stats = per_cpu_ptr(np->rx_stats, cpu); in xennet_get_stats64()
1397 struct netfront_stats *tx_stats = per_cpu_ptr(np->tx_stats, cpu); in xennet_get_stats64()
1402 start = u64_stats_fetch_begin(&tx_stats->syncp); in xennet_get_stats64()
1403 tx_packets = tx_stats->packets; in xennet_get_stats64()
1404 tx_bytes = tx_stats->bytes; in xennet_get_stats64()
1405 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); in xennet_get_stats64()
1408 start = u64_stats_fetch_begin(&rx_stats->syncp); in xennet_get_stats64()
1409 rx_packets = rx_stats->packets; in xennet_get_stats64()
1410 rx_bytes = rx_stats->bytes; in xennet_get_stats64()
1411 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); in xennet_get_stats64()
1413 tot->rx_packets += rx_packets; in xennet_get_stats64()
1414 tot->tx_packets += tx_packets; in xennet_get_stats64()
1415 tot->rx_bytes += rx_bytes; in xennet_get_stats64()
1416 tot->tx_bytes += tx_bytes; in xennet_get_stats64()
1419 tot->rx_errors = dev->stats.rx_errors; in xennet_get_stats64()
1420 tot->tx_dropped = dev->stats.tx_dropped; in xennet_get_stats64()
1430 if (!queue->tx_skbs[i]) in xennet_release_tx_bufs()
1433 skb = queue->tx_skbs[i]; in xennet_release_tx_bufs()
1434 queue->tx_skbs[i] = NULL; in xennet_release_tx_bufs()
1435 get_page(queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1436 gnttab_end_foreign_access(queue->grant_tx_ref[i], in xennet_release_tx_bufs()
1437 queue->grant_tx_page[i]); in xennet_release_tx_bufs()
1438 queue->grant_tx_page[i] = NULL; in xennet_release_tx_bufs()
1439 queue->grant_tx_ref[i] = INVALID_GRANT_REF; in xennet_release_tx_bufs()
1440 add_id_to_list(&queue->tx_skb_freelist, queue->tx_link, i); in xennet_release_tx_bufs()
1449 spin_lock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1455 skb = queue->rx_skbs[id]; in xennet_release_rx_bufs()
1459 ref = queue->grant_rx_ref[id]; in xennet_release_rx_bufs()
1463 page = skb_frag_page(&skb_shinfo(skb)->frags[0]); in xennet_release_rx_bufs()
1470 queue->grant_rx_ref[id] = INVALID_GRANT_REF; in xennet_release_rx_bufs()
1475 spin_unlock_bh(&queue->rx_lock); in xennet_release_rx_bufs()
1484 !xenbus_read_unsigned(np->xbdev->otherend, "feature-sg", 0)) in xennet_fix_features()
1488 !xenbus_read_unsigned(np->xbdev->otherend, in xennet_fix_features()
1489 "feature-ipv6-csum-offload", 0)) in xennet_fix_features()
1493 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv4", 0)) in xennet_fix_features()
1497 !xenbus_read_unsigned(np->xbdev->otherend, "feature-gso-tcpv6", 0)) in xennet_fix_features()
1506 if (!(features & NETIF_F_SG) && dev->mtu > ETH_DATA_LEN) { in xennet_set_features()
1508 dev->mtu = ETH_DATA_LEN; in xennet_set_features()
1518 if (unlikely(queue->info->broken)) in xennet_handle_tx()
1521 spin_lock_irqsave(&queue->tx_lock, flags); in xennet_handle_tx()
1524 spin_unlock_irqrestore(&queue->tx_lock, flags); in xennet_handle_tx()
1544 if (unlikely(queue->info->broken)) in xennet_handle_rx()
1547 spin_lock_irqsave(&queue->rx_cons_lock, flags); in xennet_handle_rx()
1548 work_queued = XEN_RING_NR_UNCONSUMED_RESPONSES(&queue->rx); in xennet_handle_rx()
1549 if (work_queued > queue->rx_rsp_unconsumed) { in xennet_handle_rx()
1550 queue->rx_rsp_unconsumed = work_queued; in xennet_handle_rx()
1552 } else if (unlikely(work_queued < queue->rx_rsp_unconsumed)) { in xennet_handle_rx()
1553 const struct device *dev = &queue->info->netdev->dev; in xennet_handle_rx()
1555 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); in xennet_handle_rx()
1557 dev_alert(dev, "Disabled for further use\n"); in xennet_handle_rx()
1558 queue->info->broken = true; in xennet_handle_rx()
1561 spin_unlock_irqrestore(&queue->rx_cons_lock, flags); in xennet_handle_rx()
1563 if (likely(netif_carrier_ok(queue->info->netdev) && work_queued)) in xennet_handle_rx()
1564 napi_schedule(&queue->napi); in xennet_handle_rx()
1595 unsigned int num_queues = dev->real_num_tx_queues; in xennet_poll_controller()
1598 if (info->broken) in xennet_poll_controller()
1602 xennet_interrupt(0, &info->queues[i]); in xennet_poll_controller()
1615 err = xenbus_printf(XBT_NIL, np->xbdev->nodename, in talk_to_netback_xdp()
1616 "xdp-headroom", "%hu", in talk_to_netback_xdp()
1619 pr_warn("Error writing xdp-headroom\n"); in talk_to_netback_xdp()
1627 unsigned long max_mtu = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM; in xennet_xdp_set()
1632 if (dev->mtu > max_mtu) { in xennet_xdp_set()
1634 return -EINVAL; in xennet_xdp_set()
1637 if (!np->netback_has_xdp_headroom) in xennet_xdp_set()
1640 xenbus_switch_state(np->xbdev, XenbusStateReconfiguring); in xennet_xdp_set()
1649 xenbus_read_driver_state(np->xbdev->otherend) == in xennet_xdp_set()
1651 np->netfront_xdp_enabled = true; in xennet_xdp_set()
1653 old_prog = rtnl_dereference(np->queues[0].xdp_prog); in xennet_xdp_set()
1656 bpf_prog_add(prog, dev->real_num_tx_queues); in xennet_xdp_set()
1658 for (i = 0; i < dev->real_num_tx_queues; ++i) in xennet_xdp_set()
1659 rcu_assign_pointer(np->queues[i].xdp_prog, prog); in xennet_xdp_set()
1662 for (i = 0; i < dev->real_num_tx_queues; ++i) in xennet_xdp_set()
1665 xenbus_switch_state(np->xbdev, XenbusStateConnected); in xennet_xdp_set()
1674 if (np->broken) in xennet_xdp()
1675 return -ENODEV; in xennet_xdp()
1677 switch (xdp->command) { in xennet_xdp()
1679 return xennet_xdp_set(dev, xdp->prog, xdp->extack); in xennet_xdp()
1681 return -EINVAL; in xennet_xdp()
1708 free_percpu(np->rx_stats); in xennet_free_netdev()
1709 free_percpu(np->tx_stats); in xennet_free_netdev()
1721 return ERR_PTR(-ENOMEM); in xennet_create_dev()
1724 np->xbdev = dev; in xennet_create_dev()
1726 np->queues = NULL; in xennet_create_dev()
1728 err = -ENOMEM; in xennet_create_dev()
1729 np->rx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); in xennet_create_dev()
1730 if (np->rx_stats == NULL) in xennet_create_dev()
1732 np->tx_stats = netdev_alloc_pcpu_stats(struct netfront_stats); in xennet_create_dev()
1733 if (np->tx_stats == NULL) in xennet_create_dev()
1736 netdev->netdev_ops = &xennet_netdev_ops; in xennet_create_dev()
1738 netdev->features = NETIF_F_IP_CSUM | NETIF_F_RXCSUM | in xennet_create_dev()
1740 netdev->hw_features = NETIF_F_SG | in xennet_create_dev()
1746 * will be adjusted by the call to netdev_update_features() in in xennet_create_dev()
1750 netdev->features |= netdev->hw_features; in xennet_create_dev()
1751 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | in xennet_create_dev()
1754 netdev->ethtool_ops = &xennet_ethtool_ops; in xennet_create_dev()
1755 netdev->min_mtu = ETH_MIN_MTU; in xennet_create_dev()
1756 netdev->max_mtu = XEN_NETIF_MAX_TX_SIZE; in xennet_create_dev()
1757 SET_NETDEV_DEV(netdev, &dev->dev); in xennet_create_dev()
1759 np->netdev = netdev; in xennet_create_dev()
1760 np->netfront_xdp_enabled = false; in xennet_create_dev()
1767 xenbus_read_driver_state(dev->otherend) != in xennet_create_dev()
1769 xenbus_read_driver_state(dev->otherend) != in xennet_create_dev()
1781 * Entry point to this code when a new device is created. Allocate the basic
1800 dev_set_drvdata(&dev->dev, info); in netfront_probe()
1802 info->netdev->sysfs_groups[0] = &xennet_dev_group; in netfront_probe()
1810 /* This frees the page as a side-effect */ in xennet_end_access()
1818 unsigned int num_queues = info->netdev->real_num_tx_queues; in xennet_disconnect_backend()
1820 netif_carrier_off(info->netdev); in xennet_disconnect_backend()
1822 for (i = 0; i < num_queues && info->queues; ++i) { in xennet_disconnect_backend()
1823 struct netfront_queue *queue = &info->queues[i]; in xennet_disconnect_backend()
1825 timer_delete_sync(&queue->rx_refill_timer); in xennet_disconnect_backend()
1827 if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) in xennet_disconnect_backend()
1828 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1829 if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { in xennet_disconnect_backend()
1830 unbind_from_irqhandler(queue->tx_irq, queue); in xennet_disconnect_backend()
1831 unbind_from_irqhandler(queue->rx_irq, queue); in xennet_disconnect_backend()
1833 queue->tx_evtchn = queue->rx_evtchn = 0; in xennet_disconnect_backend()
1834 queue->tx_irq = queue->rx_irq = 0; in xennet_disconnect_backend()
1836 if (netif_running(info->netdev)) in xennet_disconnect_backend()
1837 napi_synchronize(&queue->napi); in xennet_disconnect_backend()
1841 gnttab_free_grant_references(queue->gref_tx_head); in xennet_disconnect_backend()
1842 gnttab_free_grant_references(queue->gref_rx_head); in xennet_disconnect_backend()
1845 xennet_end_access(queue->tx_ring_ref, queue->tx.sring); in xennet_disconnect_backend()
1846 xennet_end_access(queue->rx_ring_ref, queue->rx.sring); in xennet_disconnect_backend()
1848 queue->tx_ring_ref = INVALID_GRANT_REF; in xennet_disconnect_backend()
1849 queue->rx_ring_ref = INVALID_GRANT_REF; in xennet_disconnect_backend()
1850 queue->tx.sring = NULL; in xennet_disconnect_backend()
1851 queue->rx.sring = NULL; in xennet_disconnect_backend()
1853 page_pool_destroy(queue->page_pool); in xennet_disconnect_backend()
1858 * We are reconnecting to the backend, due to a suspend/resume, or a backend
1860 * leave the device-layer structures intact so that this is transparent to the
1865 struct netfront_info *info = dev_get_drvdata(&dev->dev); in netfront_resume()
1867 dev_dbg(&dev->dev, "%s\n", dev->nodename); in netfront_resume()
1869 netif_tx_lock_bh(info->netdev); in netfront_resume()
1870 netif_device_detach(info->netdev); in netfront_resume()
1871 netif_tx_unlock_bh(info->netdev); in netfront_resume()
1876 if (info->queues) in netfront_resume()
1888 macstr = s = xenbus_read(XBT_NIL, dev->nodename, "mac", NULL); in xen_net_read_mac()
1894 if ((s == e) || (*e != ((i == ETH_ALEN-1) ? '\0' : ':'))) { in xen_net_read_mac()
1896 return -ENOENT; in xen_net_read_mac()
1909 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_single()
1913 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, in setup_netfront_single()
1915 queue->info->netdev->name, in setup_netfront_single()
1919 queue->rx_evtchn = queue->tx_evtchn; in setup_netfront_single()
1920 queue->rx_irq = queue->tx_irq = err; in setup_netfront_single()
1925 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_single()
1926 queue->tx_evtchn = 0; in setup_netfront_single()
1935 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->tx_evtchn); in setup_netfront_split()
1938 err = xenbus_alloc_evtchn(queue->info->xbdev, &queue->rx_evtchn); in setup_netfront_split()
1942 snprintf(queue->tx_irq_name, sizeof(queue->tx_irq_name), in setup_netfront_split()
1943 "%s-tx", queue->name); in setup_netfront_split()
1944 err = bind_evtchn_to_irqhandler_lateeoi(queue->tx_evtchn, in setup_netfront_split()
1946 queue->tx_irq_name, queue); in setup_netfront_split()
1949 queue->tx_irq = err; in setup_netfront_split()
1951 snprintf(queue->rx_irq_name, sizeof(queue->rx_irq_name), in setup_netfront_split()
1952 "%s-rx", queue->name); in setup_netfront_split()
1953 err = bind_evtchn_to_irqhandler_lateeoi(queue->rx_evtchn, in setup_netfront_split()
1955 queue->rx_irq_name, queue); in setup_netfront_split()
1958 queue->rx_irq = err; in setup_netfront_split()
1963 unbind_from_irqhandler(queue->tx_irq, queue); in setup_netfront_split()
1964 queue->tx_irq = 0; in setup_netfront_split()
1966 xenbus_free_evtchn(queue->info->xbdev, queue->rx_evtchn); in setup_netfront_split()
1967 queue->rx_evtchn = 0; in setup_netfront_split()
1969 xenbus_free_evtchn(queue->info->xbdev, queue->tx_evtchn); in setup_netfront_split()
1970 queue->tx_evtchn = 0; in setup_netfront_split()
1982 queue->tx_ring_ref = INVALID_GRANT_REF; in setup_netfront()
1983 queue->rx_ring_ref = INVALID_GRANT_REF; in setup_netfront()
1984 queue->rx.sring = NULL; in setup_netfront()
1985 queue->tx.sring = NULL; in setup_netfront()
1988 1, &queue->tx_ring_ref); in setup_netfront()
1992 XEN_FRONT_RING_INIT(&queue->tx, txs, XEN_PAGE_SIZE); in setup_netfront()
1995 1, &queue->rx_ring_ref); in setup_netfront()
1999 XEN_FRONT_RING_INIT(&queue->rx, rxs, XEN_PAGE_SIZE); in setup_netfront()
2004 * a) feature-split-event-channels == 0 in setup_netfront()
2005 * b) feature-split-event-channels == 1 but failed to setup in setup_netfront()
2016 xenbus_teardown_ring((void **)&queue->rx.sring, 1, &queue->rx_ring_ref); in setup_netfront()
2017 xenbus_teardown_ring((void **)&queue->tx.sring, 1, &queue->tx_ring_ref); in setup_netfront()
2022 /* Queue-specific initialisation
2023 * This used to be done in xennet_create_dev() but must now
2024 * be run per-queue.
2032 spin_lock_init(&queue->tx_lock); in xennet_init_queue()
2033 spin_lock_init(&queue->rx_lock); in xennet_init_queue()
2034 spin_lock_init(&queue->rx_cons_lock); in xennet_init_queue()
2036 timer_setup(&queue->rx_refill_timer, rx_refill_timeout, 0); in xennet_init_queue()
2038 devid = strrchr(queue->info->xbdev->nodename, '/') + 1; in xennet_init_queue()
2039 snprintf(queue->name, sizeof(queue->name), "vif%s-q%u", in xennet_init_queue()
2040 devid, queue->id); in xennet_init_queue()
2043 queue->tx_skb_freelist = 0; in xennet_init_queue()
2044 queue->tx_pend_queue = TX_LINK_NONE; in xennet_init_queue()
2046 queue->tx_link[i] = i + 1; in xennet_init_queue()
2047 queue->grant_tx_ref[i] = INVALID_GRANT_REF; in xennet_init_queue()
2048 queue->grant_tx_page[i] = NULL; in xennet_init_queue()
2050 queue->tx_link[NET_TX_RING_SIZE - 1] = TX_LINK_NONE; in xennet_init_queue()
2054 queue->rx_skbs[i] = NULL; in xennet_init_queue()
2055 queue->grant_rx_ref[i] = INVALID_GRANT_REF; in xennet_init_queue()
2058 /* A grant for every tx ring slot */ in xennet_init_queue()
2060 &queue->gref_tx_head) < 0) { in xennet_init_queue()
2061 pr_alert("can't alloc tx grant refs\n"); in xennet_init_queue()
2062 err = -ENOMEM; in xennet_init_queue()
2068 &queue->gref_rx_head) < 0) { in xennet_init_queue()
2070 err = -ENOMEM; in xennet_init_queue()
2077 gnttab_free_grant_references(queue->gref_tx_head); in xennet_init_queue()
2085 /* Write the queue-specific keys into XenStore in the traditional in write_queue_xenstore_keys()
2087 * queues. in write_queue_xenstore_keys()
2089 struct xenbus_device *dev = queue->info->xbdev; in write_queue_xenstore_keys()
2095 /* Choose the correct place to write the keys */ in write_queue_xenstore_keys()
2097 pathsize = strlen(dev->nodename) + 10; in write_queue_xenstore_keys()
2100 err = -ENOMEM; in write_queue_xenstore_keys()
2104 snprintf(path, pathsize, "%s/queue-%u", in write_queue_xenstore_keys()
2105 dev->nodename, queue->id); in write_queue_xenstore_keys()
2107 path = (char *)dev->nodename; in write_queue_xenstore_keys()
2111 err = xenbus_printf(*xbt, path, "tx-ring-ref", "%u", in write_queue_xenstore_keys()
2112 queue->tx_ring_ref); in write_queue_xenstore_keys()
2114 message = "writing tx-ring-ref"; in write_queue_xenstore_keys()
2118 err = xenbus_printf(*xbt, path, "rx-ring-ref", "%u", in write_queue_xenstore_keys()
2119 queue->rx_ring_ref); in write_queue_xenstore_keys()
2121 message = "writing rx-ring-ref"; in write_queue_xenstore_keys()
2128 if (queue->tx_evtchn == queue->rx_evtchn) { in write_queue_xenstore_keys()
2131 "event-channel", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
2133 message = "writing event-channel"; in write_queue_xenstore_keys()
2139 "event-channel-tx", "%u", queue->tx_evtchn); in write_queue_xenstore_keys()
2141 message = "writing event-channel-tx"; in write_queue_xenstore_keys()
2146 "event-channel-rx", "%u", queue->rx_evtchn); in write_queue_xenstore_keys()
2148 message = "writing event-channel-rx"; in write_queue_xenstore_keys()
2174 .dev = &queue->info->netdev->dev, in xennet_create_page_pool()
2176 .max_len = XEN_PAGE_SIZE - XDP_PACKET_HEADROOM, in xennet_create_page_pool()
2179 queue->page_pool = page_pool_create(&pp_params); in xennet_create_page_pool()
2180 if (IS_ERR(queue->page_pool)) { in xennet_create_page_pool()
2181 err = PTR_ERR(queue->page_pool); in xennet_create_page_pool()
2182 queue->page_pool = NULL; in xennet_create_page_pool()
2186 err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, in xennet_create_page_pool()
2187 queue->id, 0); in xennet_create_page_pool()
2189 netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); in xennet_create_page_pool()
2193 err = xdp_rxq_info_reg_mem_model(&queue->xdp_rxq, in xennet_create_page_pool()
2194 MEM_TYPE_PAGE_POOL, queue->page_pool); in xennet_create_page_pool()
2196 netdev_err(queue->info->netdev, "xdp_rxq_info_reg_mem_model failed\n"); in xennet_create_page_pool()
2202 xdp_rxq_info_unreg(&queue->xdp_rxq); in xennet_create_page_pool()
2204 page_pool_destroy(queue->page_pool); in xennet_create_page_pool()
2205 queue->page_pool = NULL; in xennet_create_page_pool()
2215 info->queues = kcalloc(*num_queues, sizeof(struct netfront_queue), in xennet_create_queues()
2217 if (!info->queues) in xennet_create_queues()
2218 return -ENOMEM; in xennet_create_queues()
2221 struct netfront_queue *queue = &info->queues[i]; in xennet_create_queues()
2223 queue->id = i; in xennet_create_queues()
2224 queue->info = info; in xennet_create_queues()
2228 dev_warn(&info->xbdev->dev, in xennet_create_queues()
2229 "only created %d queues\n", i); in xennet_create_queues()
2234 /* use page pool recycling instead of buddy allocator */ in xennet_create_queues()
2237 dev_err(&info->xbdev->dev, "can't allocate page pool\n"); in xennet_create_queues()
2242 netif_napi_add(queue->info->netdev, &queue->napi, xennet_poll); in xennet_create_queues()
2243 if (netif_running(info->netdev)) in xennet_create_queues()
2244 napi_enable(&queue->napi); in xennet_create_queues()
2247 netif_set_real_num_tx_queues(info->netdev, *num_queues); in xennet_create_queues()
2250 dev_err(&info->xbdev->dev, "no queues\n"); in xennet_create_queues()
2251 return -EINVAL; in xennet_create_queues()
2270 info->netdev->irq = 0; in talk_to_netback()
2273 info->bounce = !xennet_trusted || in talk_to_netback()
2274 !xenbus_read_unsigned(dev->nodename, "trusted", 1); in talk_to_netback()
2276 /* Check if backend supports multiple queues */ in talk_to_netback()
2277 max_queues = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
2278 "multi-queue-max-queues", 1); in talk_to_netback()
2281 /* Check feature-split-event-channels */ in talk_to_netback()
2282 feature_split_evtchn = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
2283 "feature-split-event-channels", 0); in talk_to_netback()
2288 xenbus_dev_fatal(dev, err, "parsing %s/mac", dev->nodename); in talk_to_netback()
2291 eth_hw_addr_set(info->netdev, addr); in talk_to_netback()
2293 info->netback_has_xdp_headroom = xenbus_read_unsigned(info->xbdev->otherend, in talk_to_netback()
2294 "feature-xdp-headroom", 0); in talk_to_netback()
2295 if (info->netback_has_xdp_headroom) { in talk_to_netback()
2296 /* set the current xen-netfront xdp state */ in talk_to_netback()
2297 err = talk_to_netback_xdp(info, info->netfront_xdp_enabled ? in talk_to_netback()
2305 if (info->queues) in talk_to_netback()
2309 info->broken = false; in talk_to_netback()
2313 xenbus_dev_fatal(dev, err, "creating queues"); in talk_to_netback()
2314 kfree(info->queues); in talk_to_netback()
2315 info->queues = NULL; in talk_to_netback()
2320 /* Create shared ring, alloc event channel -- for each queue */ in talk_to_netback()
2322 queue = &info->queues[i]; in talk_to_netback()
2336 info->xbdev->otherend, "multi-queue-max-queues")) { in talk_to_netback()
2337 /* Write the number of queues */ in talk_to_netback()
2338 err = xenbus_printf(xbt, dev->nodename, in talk_to_netback()
2339 "multi-queue-num-queues", "%u", num_queues); in talk_to_netback()
2341 message = "writing multi-queue-num-queues"; in talk_to_netback()
2347 err = write_queue_xenstore_keys(&info->queues[0], &xbt, 0); /* flat */ in talk_to_netback()
2353 queue = &info->queues[i]; in talk_to_netback()
2360 /* The remaining keys are not queue-specific */ in talk_to_netback()
2361 err = xenbus_printf(xbt, dev->nodename, "request-rx-copy", "%u", in talk_to_netback()
2364 message = "writing request-rx-copy"; in talk_to_netback()
2368 err = xenbus_printf(xbt, dev->nodename, "feature-rx-notify", "%d", 1); in talk_to_netback()
2370 message = "writing feature-rx-notify"; in talk_to_netback()
2374 err = xenbus_printf(xbt, dev->nodename, "feature-sg", "%d", 1); in talk_to_netback()
2376 message = "writing feature-sg"; in talk_to_netback()
2380 err = xenbus_printf(xbt, dev->nodename, "feature-gso-tcpv4", "%d", 1); in talk_to_netback()
2382 message = "writing feature-gso-tcpv4"; in talk_to_netback()
2386 err = xenbus_write(xbt, dev->nodename, "feature-gso-tcpv6", "1"); in talk_to_netback()
2388 message = "writing feature-gso-tcpv6"; in talk_to_netback()
2392 err = xenbus_write(xbt, dev->nodename, "feature-ipv6-csum-offload", in talk_to_netback()
2395 message = "writing feature-ipv6-csum-offload"; in talk_to_netback()
2401 if (err == -EAGAIN) in talk_to_netback()
2420 device_unregister(&dev->dev); in talk_to_netback()
2432 if (!xenbus_read_unsigned(np->xbdev->otherend, "feature-rx-copy", 0)) { in xennet_connect()
2433 dev_info(&dev->dev, in xennet_connect()
2435 return -ENODEV; in xennet_connect()
2438 err = talk_to_netback(np->xbdev, np); in xennet_connect()
2441 if (np->netback_has_xdp_headroom) in xennet_connect()
2443 if (np->bounce) in xennet_connect()
2444 dev_info(&np->xbdev->dev, in xennet_connect()
2445 "bouncing transmitted data to zeroed pages\n"); in xennet_connect()
2447 /* talk_to_netback() sets the correct number of queues */ in xennet_connect()
2448 num_queues = dev->real_num_tx_queues; in xennet_connect()
2450 if (dev->reg_state == NETREG_UNINITIALIZED) { in xennet_connect()
2454 device_unregister(&np->xbdev->dev); in xennet_connect()
2465 * ready to start sending and receiving packets and give the driver in xennet_connect()
2469 netif_tx_lock_bh(np->netdev); in xennet_connect()
2470 netif_device_attach(np->netdev); in xennet_connect()
2471 netif_tx_unlock_bh(np->netdev); in xennet_connect()
2473 netif_carrier_on(np->netdev); in xennet_connect()
2475 queue = &np->queues[j]; in xennet_connect()
2477 notify_remote_via_irq(queue->tx_irq); in xennet_connect()
2478 if (queue->tx_irq != queue->rx_irq) in xennet_connect()
2479 notify_remote_via_irq(queue->rx_irq); in xennet_connect()
2481 spin_lock_bh(&queue->rx_lock); in xennet_connect()
2483 spin_unlock_bh(&queue->rx_lock); in xennet_connect()
2495 struct netfront_info *np = dev_get_drvdata(&dev->dev); in netback_changed()
2496 struct net_device *netdev = np->netdev; in netback_changed()
2498 dev_dbg(&dev->dev, "%s\n", xenbus_strstate(backend_state)); in netback_changed()
2511 if (dev->state != XenbusStateInitialising) in netback_changed()
2523 if (dev->state == XenbusStateClosed) in netback_changed()
2548 return -EINVAL; in xennet_get_sset_count()
2599 return -EPERM; in store_rxbuf()
2603 return -EBADMSG; in store_rxbuf()
2630 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) in xennet_bus_close()
2635 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2637 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2639 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2644 if (xenbus_read_driver_state(dev->otherend) == XenbusStateClosed) in xennet_bus_close()
2650 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2652 xenbus_read_driver_state(dev->otherend) == in xennet_bus_close()
2660 struct netfront_info *info = dev_get_drvdata(&dev->dev); in xennet_remove()
2665 if (info->netdev->reg_state == NETREG_REGISTERED) in xennet_remove()
2666 unregister_netdev(info->netdev); in xennet_remove()
2668 if (info->queues) { in xennet_remove()
2673 xennet_free_netdev(info->netdev); in xennet_remove()
2692 return -ENODEV; in netif_init()
2695 return -ENODEV; in netif_init()
2699 /* Allow as many queues as there are CPUs inut max. 8 if user has not in netif_init()