1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2874aeea5SJeff Kirsher /**************************************************************************** 3f7a6d2c4SBen Hutchings * Driver for Solarflare network controllers and boards 4874aeea5SJeff Kirsher * Copyright 2005-2006 Fen Systems Ltd. 5f7a6d2c4SBen Hutchings * Copyright 2005-2013 Solarflare Communications Inc. 6874aeea5SJeff Kirsher */ 7874aeea5SJeff Kirsher 8874aeea5SJeff Kirsher #include <linux/socket.h> 9874aeea5SJeff Kirsher #include <linux/in.h> 10874aeea5SJeff Kirsher #include <linux/slab.h> 11874aeea5SJeff Kirsher #include <linux/ip.h> 12c47b2d9dSBen Hutchings #include <linux/ipv6.h> 13874aeea5SJeff Kirsher #include <linux/tcp.h> 14874aeea5SJeff Kirsher #include <linux/udp.h> 15874aeea5SJeff Kirsher #include <linux/prefetch.h> 166eb07cafSPaul Gortmaker #include <linux/moduleparam.h> 172768935aSDaniel Pieczko #include <linux/iommu.h> 18874aeea5SJeff Kirsher #include <net/ip.h> 19874aeea5SJeff Kirsher #include <net/checksum.h> 20eb9a36beSCharles McLachlan #include <net/xdp.h> 21eb9a36beSCharles McLachlan #include <linux/bpf_trace.h> 22874aeea5SJeff Kirsher #include "net_driver.h" 23874aeea5SJeff Kirsher #include "efx.h" 24e1253f39SAlex Maftei (amaftei) #include "rx_common.h" 25add72477SBen Hutchings #include "filter.h" 26874aeea5SJeff Kirsher #include "nic.h" 27874aeea5SJeff Kirsher #include "selftest.h" 28874aeea5SJeff Kirsher #include "workarounds.h" 29874aeea5SJeff Kirsher 301648a23fSDaniel Pieczko /* Preferred number of descriptors to fill at once */ 311648a23fSDaniel Pieczko #define EFX_RX_PREFERRED_BATCH 8U 32874aeea5SJeff Kirsher 33eb9a36beSCharles McLachlan /* Maximum rx prefix used by any architecture. */ 34eb9a36beSCharles McLachlan #define EFX_MAX_RX_PREFIX_SIZE 16 35eb9a36beSCharles McLachlan 36874aeea5SJeff Kirsher /* Size of buffer allocated for skb header area. */ 37d4ef5b6fSJon Cooper #define EFX_SKB_HEADERS 128u 38874aeea5SJeff Kirsher 3985740cdfSBen Hutchings /* Each packet can consume up to ceil(max_frame_len / buffer_size) buffers */ 4085740cdfSBen Hutchings #define EFX_RX_MAX_FRAGS DIV_ROUND_UP(EFX_MAX_FRAME_LEN(EFX_MAX_MTU), \ 4185740cdfSBen Hutchings EFX_RX_USR_BUF_SIZE) 4285740cdfSBen Hutchings 432768935aSDaniel Pieczko static inline void efx_sync_rx_buffer(struct efx_nic *efx, 442768935aSDaniel Pieczko struct efx_rx_buffer *rx_buf, 452768935aSDaniel Pieczko unsigned int len) 462768935aSDaniel Pieczko { 472768935aSDaniel Pieczko dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, len, 482768935aSDaniel Pieczko DMA_FROM_DEVICE); 492768935aSDaniel Pieczko } 502768935aSDaniel Pieczko 51874aeea5SJeff Kirsher static void efx_rx_packet__check_len(struct efx_rx_queue *rx_queue, 52874aeea5SJeff Kirsher struct efx_rx_buffer *rx_buf, 5397d48a10SAlexandre Rames int len) 54874aeea5SJeff Kirsher { 55874aeea5SJeff Kirsher struct efx_nic *efx = rx_queue->efx; 56874aeea5SJeff Kirsher unsigned max_len = rx_buf->len - efx->type->rx_buffer_padding; 57874aeea5SJeff Kirsher 58874aeea5SJeff Kirsher if (likely(len <= max_len)) 59874aeea5SJeff Kirsher return; 60874aeea5SJeff Kirsher 61874aeea5SJeff Kirsher /* The packet must be discarded, but this is only a fatal error 62874aeea5SJeff Kirsher * if the caller indicated it was 63874aeea5SJeff Kirsher */ 64db339569SBen Hutchings rx_buf->flags |= EFX_RX_PKT_DISCARD; 65874aeea5SJeff Kirsher 66874aeea5SJeff Kirsher if (net_ratelimit()) 67874aeea5SJeff Kirsher netif_err(efx, rx_err, efx->net_dev, 685a6681e2SEdward Cree "RX queue %d overlength RX event (%#x > %#x)\n", 69874aeea5SJeff Kirsher efx_rx_queue_index(rx_queue), len, max_len); 70874aeea5SJeff Kirsher 71874aeea5SJeff Kirsher efx_rx_queue_channel(rx_queue)->n_rx_overlength++; 72874aeea5SJeff Kirsher } 73874aeea5SJeff Kirsher 7485740cdfSBen Hutchings /* Allocate and construct an SKB around page fragments */ 7597d48a10SAlexandre Rames static struct sk_buff *efx_rx_mk_skb(struct efx_channel *channel, 7697d48a10SAlexandre Rames struct efx_rx_buffer *rx_buf, 7785740cdfSBen Hutchings unsigned int n_frags, 7897d48a10SAlexandre Rames u8 *eh, int hdr_len) 7997d48a10SAlexandre Rames { 8097d48a10SAlexandre Rames struct efx_nic *efx = channel->efx; 8197d48a10SAlexandre Rames struct sk_buff *skb; 8297d48a10SAlexandre Rames 8397d48a10SAlexandre Rames /* Allocate an SKB to store the headers */ 842ccd0b19SBen Hutchings skb = netdev_alloc_skb(efx->net_dev, 852ccd0b19SBen Hutchings efx->rx_ip_align + efx->rx_prefix_size + 862ccd0b19SBen Hutchings hdr_len); 87e4d112e4SEdward Cree if (unlikely(skb == NULL)) { 88e4d112e4SEdward Cree atomic_inc(&efx->n_rx_noskb_drops); 8997d48a10SAlexandre Rames return NULL; 90e4d112e4SEdward Cree } 9197d48a10SAlexandre Rames 92e01b16a7SEdward Cree EFX_WARN_ON_ONCE_PARANOID(rx_buf->len < hdr_len); 9397d48a10SAlexandre Rames 942ccd0b19SBen Hutchings memcpy(skb->data + efx->rx_ip_align, eh - efx->rx_prefix_size, 952ccd0b19SBen Hutchings efx->rx_prefix_size + hdr_len); 962ccd0b19SBen Hutchings skb_reserve(skb, efx->rx_ip_align + efx->rx_prefix_size); 972ccd0b19SBen Hutchings __skb_put(skb, hdr_len); 9897d48a10SAlexandre Rames 9985740cdfSBen Hutchings /* Append the remaining page(s) onto the frag list */ 10097d48a10SAlexandre Rames if (rx_buf->len > hdr_len) { 10185740cdfSBen Hutchings rx_buf->page_offset += hdr_len; 10285740cdfSBen Hutchings rx_buf->len -= hdr_len; 10385740cdfSBen Hutchings 10485740cdfSBen Hutchings for (;;) { 10585740cdfSBen Hutchings skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 10685740cdfSBen Hutchings rx_buf->page, rx_buf->page_offset, 10785740cdfSBen Hutchings rx_buf->len); 10885740cdfSBen Hutchings rx_buf->page = NULL; 10985740cdfSBen Hutchings skb->len += rx_buf->len; 11085740cdfSBen Hutchings skb->data_len += rx_buf->len; 11185740cdfSBen Hutchings if (skb_shinfo(skb)->nr_frags == n_frags) 11285740cdfSBen Hutchings break; 11385740cdfSBen Hutchings 11485740cdfSBen Hutchings rx_buf = efx_rx_buf_next(&channel->rx_queue, rx_buf); 11585740cdfSBen Hutchings } 11697d48a10SAlexandre Rames } else { 11797d48a10SAlexandre Rames __free_pages(rx_buf->page, efx->rx_buffer_order); 11885740cdfSBen Hutchings rx_buf->page = NULL; 11985740cdfSBen Hutchings n_frags = 0; 12097d48a10SAlexandre Rames } 12197d48a10SAlexandre Rames 12285740cdfSBen Hutchings skb->truesize += n_frags * efx->rx_buffer_truesize; 12397d48a10SAlexandre Rames 12497d48a10SAlexandre Rames /* Move past the ethernet header */ 12597d48a10SAlexandre Rames skb->protocol = eth_type_trans(skb, efx->net_dev); 12697d48a10SAlexandre Rames 12736763266SAlexandre Rames skb_mark_napi_id(skb, &channel->napi_str); 12836763266SAlexandre Rames 12997d48a10SAlexandre Rames return skb; 130874aeea5SJeff Kirsher } 131874aeea5SJeff Kirsher 132874aeea5SJeff Kirsher void efx_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index, 13385740cdfSBen Hutchings unsigned int n_frags, unsigned int len, u16 flags) 134874aeea5SJeff Kirsher { 135874aeea5SJeff Kirsher struct efx_nic *efx = rx_queue->efx; 136874aeea5SJeff Kirsher struct efx_channel *channel = efx_rx_queue_channel(rx_queue); 137874aeea5SJeff Kirsher struct efx_rx_buffer *rx_buf; 138874aeea5SJeff Kirsher 1398ccf3800SAndrew Rybchenko rx_queue->rx_packets++; 1408ccf3800SAndrew Rybchenko 141874aeea5SJeff Kirsher rx_buf = efx_rx_buffer(rx_queue, index); 142179ea7f0SBen Hutchings rx_buf->flags |= flags; 143874aeea5SJeff Kirsher 14485740cdfSBen Hutchings /* Validate the number of fragments and completed length */ 14585740cdfSBen Hutchings if (n_frags == 1) { 1463dced740SBen Hutchings if (!(flags & EFX_RX_PKT_PREFIX_LEN)) 14797d48a10SAlexandre Rames efx_rx_packet__check_len(rx_queue, rx_buf, len); 14885740cdfSBen Hutchings } else if (unlikely(n_frags > EFX_RX_MAX_FRAGS) || 149e8c68c0aSJon Cooper unlikely(len <= (n_frags - 1) * efx->rx_dma_len) || 150e8c68c0aSJon Cooper unlikely(len > n_frags * efx->rx_dma_len) || 15185740cdfSBen Hutchings unlikely(!efx->rx_scatter)) { 15285740cdfSBen Hutchings /* If this isn't an explicit discard request, either 15385740cdfSBen Hutchings * the hardware or the driver is broken. 15485740cdfSBen Hutchings */ 15585740cdfSBen Hutchings WARN_ON(!(len == 0 && rx_buf->flags & EFX_RX_PKT_DISCARD)); 15685740cdfSBen Hutchings rx_buf->flags |= EFX_RX_PKT_DISCARD; 15785740cdfSBen Hutchings } 158874aeea5SJeff Kirsher 159874aeea5SJeff Kirsher netif_vdbg(efx, rx_status, efx->net_dev, 16085740cdfSBen Hutchings "RX queue %d received ids %x-%x len %d %s%s\n", 161874aeea5SJeff Kirsher efx_rx_queue_index(rx_queue), index, 16285740cdfSBen Hutchings (index + n_frags - 1) & rx_queue->ptr_mask, len, 163db339569SBen Hutchings (rx_buf->flags & EFX_RX_PKT_CSUMMED) ? " [SUMMED]" : "", 164db339569SBen Hutchings (rx_buf->flags & EFX_RX_PKT_DISCARD) ? " [DISCARD]" : ""); 165874aeea5SJeff Kirsher 16685740cdfSBen Hutchings /* Discard packet, if instructed to do so. Process the 16785740cdfSBen Hutchings * previous receive first. 16885740cdfSBen Hutchings */ 169db339569SBen Hutchings if (unlikely(rx_buf->flags & EFX_RX_PKT_DISCARD)) { 17085740cdfSBen Hutchings efx_rx_flush_packet(channel); 171734d4e15SBen Hutchings efx_discard_rx_packet(channel, rx_buf, n_frags); 17285740cdfSBen Hutchings return; 173874aeea5SJeff Kirsher } 174874aeea5SJeff Kirsher 1753dced740SBen Hutchings if (n_frags == 1 && !(flags & EFX_RX_PKT_PREFIX_LEN)) 17685740cdfSBen Hutchings rx_buf->len = len; 17785740cdfSBen Hutchings 1782768935aSDaniel Pieczko /* Release and/or sync the DMA mapping - assumes all RX buffers 1792768935aSDaniel Pieczko * consumed in-order per RX queue. 180874aeea5SJeff Kirsher */ 1812768935aSDaniel Pieczko efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); 182874aeea5SJeff Kirsher 183874aeea5SJeff Kirsher /* Prefetch nice and early so data will (hopefully) be in cache by 184874aeea5SJeff Kirsher * the time we look at it. 185874aeea5SJeff Kirsher */ 1865036b7c7SBen Hutchings prefetch(efx_rx_buf_va(rx_buf)); 187874aeea5SJeff Kirsher 18843a3739dSJon Cooper rx_buf->page_offset += efx->rx_prefix_size; 18943a3739dSJon Cooper rx_buf->len -= efx->rx_prefix_size; 19085740cdfSBen Hutchings 19185740cdfSBen Hutchings if (n_frags > 1) { 19285740cdfSBen Hutchings /* Release/sync DMA mapping for additional fragments. 19385740cdfSBen Hutchings * Fix length for last fragment. 19485740cdfSBen Hutchings */ 19585740cdfSBen Hutchings unsigned int tail_frags = n_frags - 1; 19685740cdfSBen Hutchings 19785740cdfSBen Hutchings for (;;) { 19885740cdfSBen Hutchings rx_buf = efx_rx_buf_next(rx_queue, rx_buf); 19985740cdfSBen Hutchings if (--tail_frags == 0) 20085740cdfSBen Hutchings break; 201e8c68c0aSJon Cooper efx_sync_rx_buffer(efx, rx_buf, efx->rx_dma_len); 20285740cdfSBen Hutchings } 203e8c68c0aSJon Cooper rx_buf->len = len - (n_frags - 1) * efx->rx_dma_len; 2042768935aSDaniel Pieczko efx_sync_rx_buffer(efx, rx_buf, rx_buf->len); 20585740cdfSBen Hutchings } 206b74e3e8cSBen Hutchings 207734d4e15SBen Hutchings /* All fragments have been DMA-synced, so recycle pages. */ 2082768935aSDaniel Pieczko rx_buf = efx_rx_buffer(rx_queue, index); 209734d4e15SBen Hutchings efx_recycle_rx_pages(channel, rx_buf, n_frags); 2102768935aSDaniel Pieczko 211874aeea5SJeff Kirsher /* Pipeline receives so that we give time for packet headers to be 212874aeea5SJeff Kirsher * prefetched into cache. 213874aeea5SJeff Kirsher */ 214ff734ef4SBen Hutchings efx_rx_flush_packet(channel); 21585740cdfSBen Hutchings channel->rx_pkt_n_frags = n_frags; 21685740cdfSBen Hutchings channel->rx_pkt_index = index; 217874aeea5SJeff Kirsher } 218874aeea5SJeff Kirsher 21997d48a10SAlexandre Rames static void efx_rx_deliver(struct efx_channel *channel, u8 *eh, 22085740cdfSBen Hutchings struct efx_rx_buffer *rx_buf, 22185740cdfSBen Hutchings unsigned int n_frags) 2221ddceb4cSBen Hutchings { 2231ddceb4cSBen Hutchings struct sk_buff *skb; 22497d48a10SAlexandre Rames u16 hdr_len = min_t(u16, rx_buf->len, EFX_SKB_HEADERS); 2251ddceb4cSBen Hutchings 22685740cdfSBen Hutchings skb = efx_rx_mk_skb(channel, rx_buf, n_frags, eh, hdr_len); 22797d48a10SAlexandre Rames if (unlikely(skb == NULL)) { 2289eb0a5d1SDaniel Pieczko struct efx_rx_queue *rx_queue; 2299eb0a5d1SDaniel Pieczko 2309eb0a5d1SDaniel Pieczko rx_queue = efx_channel_get_rx_queue(channel); 2319eb0a5d1SDaniel Pieczko efx_free_rx_buffers(rx_queue, rx_buf, n_frags); 23297d48a10SAlexandre Rames return; 23397d48a10SAlexandre Rames } 23497d48a10SAlexandre Rames skb_record_rx_queue(skb, channel->rx_queue.core_index); 2351ddceb4cSBen Hutchings 2361ddceb4cSBen Hutchings /* Set the SKB flags */ 2371ddceb4cSBen Hutchings skb_checksum_none_assert(skb); 238da50ae2eSJon Cooper if (likely(rx_buf->flags & EFX_RX_PKT_CSUMMED)) { 239c99dffc4SJon Cooper skb->ip_summed = CHECKSUM_UNNECESSARY; 240da50ae2eSJon Cooper skb->csum_level = !!(rx_buf->flags & EFX_RX_PKT_CSUM_LEVEL); 241da50ae2eSJon Cooper } 2421ddceb4cSBen Hutchings 243bd9a265dSJon Cooper efx_rx_skb_attach_timestamp(channel, skb); 244bd9a265dSJon Cooper 245c31e5f9fSStuart Hodgson if (channel->type->receive_skb) 2464a74dc65SBen Hutchings if (channel->type->receive_skb(channel, skb)) 24797d48a10SAlexandre Rames return; 2484a74dc65SBen Hutchings 2494a74dc65SBen Hutchings /* Pass the packet up */ 250e090bfb9SEdward Cree if (channel->rx_list != NULL) 251e090bfb9SEdward Cree /* Add to list, will pass up later */ 252e090bfb9SEdward Cree list_add_tail(&skb->list, channel->rx_list); 253e090bfb9SEdward Cree else 254e090bfb9SEdward Cree /* No list, so pass it up now */ 2551ddceb4cSBen Hutchings netif_receive_skb(skb); 2561ddceb4cSBen Hutchings } 2571ddceb4cSBen Hutchings 258eb9a36beSCharles McLachlan /** efx_do_xdp: perform XDP processing on a received packet 259eb9a36beSCharles McLachlan * 260eb9a36beSCharles McLachlan * Returns true if packet should still be delivered. 261eb9a36beSCharles McLachlan */ 262eb9a36beSCharles McLachlan static bool efx_do_xdp(struct efx_nic *efx, struct efx_channel *channel, 263eb9a36beSCharles McLachlan struct efx_rx_buffer *rx_buf, u8 **ehp) 264eb9a36beSCharles McLachlan { 265eb9a36beSCharles McLachlan u8 rx_prefix[EFX_MAX_RX_PREFIX_SIZE]; 266eb9a36beSCharles McLachlan struct efx_rx_queue *rx_queue; 267eb9a36beSCharles McLachlan struct bpf_prog *xdp_prog; 268dfe44c1fSCharles McLachlan struct xdp_frame *xdpf; 269eb9a36beSCharles McLachlan struct xdp_buff xdp; 270eb9a36beSCharles McLachlan u32 xdp_act; 271eb9a36beSCharles McLachlan s16 offset; 272eb9a36beSCharles McLachlan int err; 273eb9a36beSCharles McLachlan 274eb9a36beSCharles McLachlan rcu_read_lock(); 275eb9a36beSCharles McLachlan xdp_prog = rcu_dereference(efx->xdp_prog); 276eb9a36beSCharles McLachlan if (!xdp_prog) { 277eb9a36beSCharles McLachlan rcu_read_unlock(); 278eb9a36beSCharles McLachlan return true; 279eb9a36beSCharles McLachlan } 280eb9a36beSCharles McLachlan 281eb9a36beSCharles McLachlan rx_queue = efx_channel_get_rx_queue(channel); 282eb9a36beSCharles McLachlan 283eb9a36beSCharles McLachlan if (unlikely(channel->rx_pkt_n_frags > 1)) { 284eb9a36beSCharles McLachlan /* We can't do XDP on fragmented packets - drop. */ 285eb9a36beSCharles McLachlan rcu_read_unlock(); 286eb9a36beSCharles McLachlan efx_free_rx_buffers(rx_queue, rx_buf, 287eb9a36beSCharles McLachlan channel->rx_pkt_n_frags); 288eb9a36beSCharles McLachlan if (net_ratelimit()) 289eb9a36beSCharles McLachlan netif_err(efx, rx_err, efx->net_dev, 290eb9a36beSCharles McLachlan "XDP is not possible with multiple receive fragments (%d)\n", 291eb9a36beSCharles McLachlan channel->rx_pkt_n_frags); 292cd846befSCharles McLachlan channel->n_rx_xdp_bad_drops++; 293eb9a36beSCharles McLachlan return false; 294eb9a36beSCharles McLachlan } 295eb9a36beSCharles McLachlan 296eb9a36beSCharles McLachlan dma_sync_single_for_cpu(&efx->pci_dev->dev, rx_buf->dma_addr, 297eb9a36beSCharles McLachlan rx_buf->len, DMA_FROM_DEVICE); 298eb9a36beSCharles McLachlan 299eb9a36beSCharles McLachlan /* Save the rx prefix. */ 300eb9a36beSCharles McLachlan EFX_WARN_ON_PARANOID(efx->rx_prefix_size > EFX_MAX_RX_PREFIX_SIZE); 301eb9a36beSCharles McLachlan memcpy(rx_prefix, *ehp - efx->rx_prefix_size, 302eb9a36beSCharles McLachlan efx->rx_prefix_size); 303eb9a36beSCharles McLachlan 304eb9a36beSCharles McLachlan xdp.data = *ehp; 30586e85bf6SJesper Dangaard Brouer xdp.data_hard_start = xdp.data - EFX_XDP_HEADROOM; 306eb9a36beSCharles McLachlan 307eb9a36beSCharles McLachlan /* No support yet for XDP metadata */ 308eb9a36beSCharles McLachlan xdp_set_data_meta_invalid(&xdp); 309eb9a36beSCharles McLachlan xdp.data_end = xdp.data + rx_buf->len; 310eb9a36beSCharles McLachlan xdp.rxq = &rx_queue->xdp_rxq_info; 311983e4345SJesper Dangaard Brouer xdp.frame_sz = efx->rx_page_buf_step; 312eb9a36beSCharles McLachlan 313eb9a36beSCharles McLachlan xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp); 314eb9a36beSCharles McLachlan rcu_read_unlock(); 315eb9a36beSCharles McLachlan 316eb9a36beSCharles McLachlan offset = (u8 *)xdp.data - *ehp; 317eb9a36beSCharles McLachlan 318eb9a36beSCharles McLachlan switch (xdp_act) { 319eb9a36beSCharles McLachlan case XDP_PASS: 320eb9a36beSCharles McLachlan /* Fix up rx prefix. */ 321eb9a36beSCharles McLachlan if (offset) { 322eb9a36beSCharles McLachlan *ehp += offset; 323eb9a36beSCharles McLachlan rx_buf->page_offset += offset; 324eb9a36beSCharles McLachlan rx_buf->len -= offset; 325eb9a36beSCharles McLachlan memcpy(*ehp - efx->rx_prefix_size, rx_prefix, 326eb9a36beSCharles McLachlan efx->rx_prefix_size); 327eb9a36beSCharles McLachlan } 328eb9a36beSCharles McLachlan break; 329eb9a36beSCharles McLachlan 330eb9a36beSCharles McLachlan case XDP_TX: 331dfe44c1fSCharles McLachlan /* Buffer ownership passes to tx on success. */ 332*1b698fa5SLorenzo Bianconi xdpf = xdp_convert_buff_to_frame(&xdp); 333dfe44c1fSCharles McLachlan err = efx_xdp_tx_buffers(efx, 1, &xdpf, true); 334dfe44c1fSCharles McLachlan if (unlikely(err != 1)) { 335dfe44c1fSCharles McLachlan efx_free_rx_buffers(rx_queue, rx_buf, 1); 336dfe44c1fSCharles McLachlan if (net_ratelimit()) 337dfe44c1fSCharles McLachlan netif_err(efx, rx_err, efx->net_dev, 338dfe44c1fSCharles McLachlan "XDP TX failed (%d)\n", err); 339cd846befSCharles McLachlan channel->n_rx_xdp_bad_drops++; 3409440a875SArthur Fabre trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); 341cd846befSCharles McLachlan } else { 342cd846befSCharles McLachlan channel->n_rx_xdp_tx++; 343dfe44c1fSCharles McLachlan } 344dfe44c1fSCharles McLachlan break; 345eb9a36beSCharles McLachlan 346eb9a36beSCharles McLachlan case XDP_REDIRECT: 347eb9a36beSCharles McLachlan err = xdp_do_redirect(efx->net_dev, &xdp, xdp_prog); 348eb9a36beSCharles McLachlan if (unlikely(err)) { 349eb9a36beSCharles McLachlan efx_free_rx_buffers(rx_queue, rx_buf, 1); 350eb9a36beSCharles McLachlan if (net_ratelimit()) 351eb9a36beSCharles McLachlan netif_err(efx, rx_err, efx->net_dev, 352eb9a36beSCharles McLachlan "XDP redirect failed (%d)\n", err); 353cd846befSCharles McLachlan channel->n_rx_xdp_bad_drops++; 3549440a875SArthur Fabre trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); 355cd846befSCharles McLachlan } else { 356cd846befSCharles McLachlan channel->n_rx_xdp_redirect++; 357eb9a36beSCharles McLachlan } 358eb9a36beSCharles McLachlan break; 359eb9a36beSCharles McLachlan 360eb9a36beSCharles McLachlan default: 361eb9a36beSCharles McLachlan bpf_warn_invalid_xdp_action(xdp_act); 362eb9a36beSCharles McLachlan efx_free_rx_buffers(rx_queue, rx_buf, 1); 363cd846befSCharles McLachlan channel->n_rx_xdp_bad_drops++; 3649440a875SArthur Fabre trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); 365eb9a36beSCharles McLachlan break; 366eb9a36beSCharles McLachlan 367eb9a36beSCharles McLachlan case XDP_ABORTED: 368eb9a36beSCharles McLachlan trace_xdp_exception(efx->net_dev, xdp_prog, xdp_act); 369eb9a36beSCharles McLachlan /* Fall through */ 370eb9a36beSCharles McLachlan case XDP_DROP: 371eb9a36beSCharles McLachlan efx_free_rx_buffers(rx_queue, rx_buf, 1); 372cd846befSCharles McLachlan channel->n_rx_xdp_drops++; 373eb9a36beSCharles McLachlan break; 374eb9a36beSCharles McLachlan } 375eb9a36beSCharles McLachlan 376eb9a36beSCharles McLachlan return xdp_act == XDP_PASS; 377eb9a36beSCharles McLachlan } 378eb9a36beSCharles McLachlan 379874aeea5SJeff Kirsher /* Handle a received packet. Second half: Touches packet payload. */ 38085740cdfSBen Hutchings void __efx_rx_packet(struct efx_channel *channel) 381874aeea5SJeff Kirsher { 382874aeea5SJeff Kirsher struct efx_nic *efx = channel->efx; 38385740cdfSBen Hutchings struct efx_rx_buffer *rx_buf = 38485740cdfSBen Hutchings efx_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); 385b74e3e8cSBen Hutchings u8 *eh = efx_rx_buf_va(rx_buf); 386874aeea5SJeff Kirsher 3873dced740SBen Hutchings /* Read length from the prefix if necessary. This already 3883dced740SBen Hutchings * excludes the length of the prefix itself. 3893dced740SBen Hutchings */ 3903dced740SBen Hutchings if (rx_buf->flags & EFX_RX_PKT_PREFIX_LEN) 3913dced740SBen Hutchings rx_buf->len = le16_to_cpup((__le16 *) 3923dced740SBen Hutchings (eh + efx->rx_packet_len_offset)); 3933dced740SBen Hutchings 394874aeea5SJeff Kirsher /* If we're in loopback test, then pass the packet directly to the 395874aeea5SJeff Kirsher * loopback layer, and free the rx_buf here 396874aeea5SJeff Kirsher */ 397874aeea5SJeff Kirsher if (unlikely(efx->loopback_selftest)) { 3989eb0a5d1SDaniel Pieczko struct efx_rx_queue *rx_queue; 3999eb0a5d1SDaniel Pieczko 400874aeea5SJeff Kirsher efx_loopback_rx_packet(efx, eh, rx_buf->len); 4019eb0a5d1SDaniel Pieczko rx_queue = efx_channel_get_rx_queue(channel); 4029eb0a5d1SDaniel Pieczko efx_free_rx_buffers(rx_queue, rx_buf, 4039eb0a5d1SDaniel Pieczko channel->rx_pkt_n_frags); 40485740cdfSBen Hutchings goto out; 405874aeea5SJeff Kirsher } 406874aeea5SJeff Kirsher 407eb9a36beSCharles McLachlan if (!efx_do_xdp(efx, channel, rx_buf, &eh)) 408eb9a36beSCharles McLachlan goto out; 409eb9a36beSCharles McLachlan 410874aeea5SJeff Kirsher if (unlikely(!(efx->net_dev->features & NETIF_F_RXCSUM))) 411db339569SBen Hutchings rx_buf->flags &= ~EFX_RX_PKT_CSUMMED; 412874aeea5SJeff Kirsher 413e7fe9491SEric Dumazet if ((rx_buf->flags & EFX_RX_PKT_TCP) && !channel->type->receive_skb) 41485740cdfSBen Hutchings efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh); 4151ddceb4cSBen Hutchings else 41685740cdfSBen Hutchings efx_rx_deliver(channel, eh, rx_buf, channel->rx_pkt_n_frags); 41785740cdfSBen Hutchings out: 41885740cdfSBen Hutchings channel->rx_pkt_n_frags = 0; 419874aeea5SJeff Kirsher } 420874aeea5SJeff Kirsher 421add72477SBen Hutchings #ifdef CONFIG_RFS_ACCEL 422add72477SBen Hutchings 4233af0f342SEdward Cree static void efx_filter_rfs_work(struct work_struct *data) 4243af0f342SEdward Cree { 4253af0f342SEdward Cree struct efx_async_filter_insertion *req = container_of(data, struct efx_async_filter_insertion, 4263af0f342SEdward Cree work); 4273af0f342SEdward Cree struct efx_nic *efx = netdev_priv(req->net_dev); 4283af0f342SEdward Cree struct efx_channel *channel = efx_get_channel(efx, req->rxq_index); 429f993740eSEdward Cree int slot_idx = req - efx->rps_slot; 430f8d62037SEdward Cree struct efx_arfs_rule *rule; 431f8d62037SEdward Cree u16 arfs_id = 0; 4323af0f342SEdward Cree int rc; 4333af0f342SEdward Cree 434494bef4cSEdward Cree rc = efx->type->filter_insert(efx, &req->spec, true); 435ded8b9c7SEdward Cree if (rc >= 0) 4368490e75cSEdward Cree /* Discard 'priority' part of EF10+ filter ID (mcdi_filters) */ 437ded8b9c7SEdward Cree rc %= efx->type->max_rx_ip_filters; 438f8d62037SEdward Cree if (efx->rps_hash_table) { 439f8d62037SEdward Cree spin_lock_bh(&efx->rps_hash_lock); 440f8d62037SEdward Cree rule = efx_rps_hash_find(efx, &req->spec); 441f8d62037SEdward Cree /* The rule might have already gone, if someone else's request 442f8d62037SEdward Cree * for the same spec was already worked and then expired before 443f8d62037SEdward Cree * we got around to our work. In that case we have nothing 444f8d62037SEdward Cree * tying us to an arfs_id, meaning that as soon as the filter 445f8d62037SEdward Cree * is considered for expiry it will be removed. 446f8d62037SEdward Cree */ 447f8d62037SEdward Cree if (rule) { 448f8d62037SEdward Cree if (rc < 0) 449f8d62037SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_ERROR; 450f8d62037SEdward Cree else 451f8d62037SEdward Cree rule->filter_id = rc; 452f8d62037SEdward Cree arfs_id = rule->arfs_id; 453f8d62037SEdward Cree } 454f8d62037SEdward Cree spin_unlock_bh(&efx->rps_hash_lock); 455f8d62037SEdward Cree } 4563af0f342SEdward Cree if (rc >= 0) { 4573af0f342SEdward Cree /* Remember this so we can check whether to expire the filter 4583af0f342SEdward Cree * later. 4593af0f342SEdward Cree */ 4603af0f342SEdward Cree mutex_lock(&efx->rps_mutex); 4618490e75cSEdward Cree if (channel->rps_flow_id[rc] == RPS_FLOW_ID_INVALID) 4628490e75cSEdward Cree channel->rfs_filter_count++; 4633af0f342SEdward Cree channel->rps_flow_id[rc] = req->flow_id; 4643af0f342SEdward Cree mutex_unlock(&efx->rps_mutex); 4653af0f342SEdward Cree 4663af0f342SEdward Cree if (req->spec.ether_type == htons(ETH_P_IP)) 4673af0f342SEdward Cree netif_info(efx, rx_status, efx->net_dev, 468f8d62037SEdward Cree "steering %s %pI4:%u:%pI4:%u to queue %u [flow %u filter %d id %u]\n", 4693af0f342SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", 4703af0f342SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port), 4713af0f342SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port), 472f8d62037SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id); 4733af0f342SEdward Cree else 4743af0f342SEdward Cree netif_info(efx, rx_status, efx->net_dev, 475f8d62037SEdward Cree "steering %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u filter %d id %u]\n", 4763af0f342SEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", 4773af0f342SEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port), 4783af0f342SEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port), 479f8d62037SEdward Cree req->rxq_index, req->flow_id, rc, arfs_id); 480ca70bd42SEdward Cree channel->n_rfs_succeeded++; 4810aa6608dSEdward Cree } else { 4820aa6608dSEdward Cree if (req->spec.ether_type == htons(ETH_P_IP)) 4830aa6608dSEdward Cree netif_dbg(efx, rx_status, efx->net_dev, 4840aa6608dSEdward Cree "failed to steer %s %pI4:%u:%pI4:%u to queue %u [flow %u rc %d id %u]\n", 4850aa6608dSEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", 4860aa6608dSEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port), 4870aa6608dSEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port), 4880aa6608dSEdward Cree req->rxq_index, req->flow_id, rc, arfs_id); 4890aa6608dSEdward Cree else 4900aa6608dSEdward Cree netif_dbg(efx, rx_status, efx->net_dev, 4910aa6608dSEdward Cree "failed to steer %s [%pI6]:%u:[%pI6]:%u to queue %u [flow %u rc %d id %u]\n", 4920aa6608dSEdward Cree (req->spec.ip_proto == IPPROTO_TCP) ? "TCP" : "UDP", 4930aa6608dSEdward Cree req->spec.rem_host, ntohs(req->spec.rem_port), 4940aa6608dSEdward Cree req->spec.loc_host, ntohs(req->spec.loc_port), 4950aa6608dSEdward Cree req->rxq_index, req->flow_id, rc, arfs_id); 496ca70bd42SEdward Cree channel->n_rfs_failed++; 4970aa6608dSEdward Cree /* We're overloading the NIC's filter tables, so let's do a 4980aa6608dSEdward Cree * chunk of extra expiry work. 4990aa6608dSEdward Cree */ 5000aa6608dSEdward Cree __efx_filter_rfs_expire(channel, min(channel->rfs_filter_count, 5010aa6608dSEdward Cree 100u)); 5023af0f342SEdward Cree } 5033af0f342SEdward Cree 5043af0f342SEdward Cree /* Release references */ 505f993740eSEdward Cree clear_bit(slot_idx, &efx->rps_slot_map); 5063af0f342SEdward Cree dev_put(req->net_dev); 5073af0f342SEdward Cree } 5083af0f342SEdward Cree 509add72477SBen Hutchings int efx_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, 510add72477SBen Hutchings u16 rxq_index, u32 flow_id) 511add72477SBen Hutchings { 512add72477SBen Hutchings struct efx_nic *efx = netdev_priv(net_dev); 5133af0f342SEdward Cree struct efx_async_filter_insertion *req; 514f8d62037SEdward Cree struct efx_arfs_rule *rule; 51568bb399eSEdward Cree struct flow_keys fk; 516f993740eSEdward Cree int slot_idx; 517f8d62037SEdward Cree bool new; 518f993740eSEdward Cree int rc; 519add72477SBen Hutchings 520f993740eSEdward Cree /* find a free slot */ 521f993740eSEdward Cree for (slot_idx = 0; slot_idx < EFX_RPS_MAX_IN_FLIGHT; slot_idx++) 522f993740eSEdward Cree if (!test_and_set_bit(slot_idx, &efx->rps_slot_map)) 523f993740eSEdward Cree break; 524f993740eSEdward Cree if (slot_idx >= EFX_RPS_MAX_IN_FLIGHT) 525f993740eSEdward Cree return -EBUSY; 526faf8dcc1SJon Cooper 527f993740eSEdward Cree if (flow_id == RPS_FLOW_ID_INVALID) { 528f993740eSEdward Cree rc = -EINVAL; 529f993740eSEdward Cree goto out_clear; 530f993740eSEdward Cree } 531add72477SBen Hutchings 532f993740eSEdward Cree if (!skb_flow_dissect_flow_keys(skb, &fk, 0)) { 533f993740eSEdward Cree rc = -EPROTONOSUPPORT; 534f993740eSEdward Cree goto out_clear; 535f993740eSEdward Cree } 536add72477SBen Hutchings 537f993740eSEdward Cree if (fk.basic.n_proto != htons(ETH_P_IP) && fk.basic.n_proto != htons(ETH_P_IPV6)) { 538f993740eSEdward Cree rc = -EPROTONOSUPPORT; 539f993740eSEdward Cree goto out_clear; 540f993740eSEdward Cree } 541f993740eSEdward Cree if (fk.control.flags & FLOW_DIS_IS_FRAGMENT) { 542f993740eSEdward Cree rc = -EPROTONOSUPPORT; 543f993740eSEdward Cree goto out_clear; 544f993740eSEdward Cree } 5453af0f342SEdward Cree 546f993740eSEdward Cree req = efx->rps_slot + slot_idx; 5473af0f342SEdward Cree efx_filter_init_rx(&req->spec, EFX_FILTER_PRI_HINT, 548add72477SBen Hutchings efx->rx_scatter ? EFX_FILTER_FLAG_RX_SCATTER : 0, 549add72477SBen Hutchings rxq_index); 5503af0f342SEdward Cree req->spec.match_flags = 551c47b2d9dSBen Hutchings EFX_FILTER_MATCH_ETHER_TYPE | EFX_FILTER_MATCH_IP_PROTO | 552c47b2d9dSBen Hutchings EFX_FILTER_MATCH_LOC_HOST | EFX_FILTER_MATCH_LOC_PORT | 553c47b2d9dSBen Hutchings EFX_FILTER_MATCH_REM_HOST | EFX_FILTER_MATCH_REM_PORT; 5543af0f342SEdward Cree req->spec.ether_type = fk.basic.n_proto; 5553af0f342SEdward Cree req->spec.ip_proto = fk.basic.ip_proto; 556c47b2d9dSBen Hutchings 55768bb399eSEdward Cree if (fk.basic.n_proto == htons(ETH_P_IP)) { 5583af0f342SEdward Cree req->spec.rem_host[0] = fk.addrs.v4addrs.src; 5593af0f342SEdward Cree req->spec.loc_host[0] = fk.addrs.v4addrs.dst; 560c47b2d9dSBen Hutchings } else { 5613af0f342SEdward Cree memcpy(req->spec.rem_host, &fk.addrs.v6addrs.src, 5623af0f342SEdward Cree sizeof(struct in6_addr)); 5633af0f342SEdward Cree memcpy(req->spec.loc_host, &fk.addrs.v6addrs.dst, 5643af0f342SEdward Cree sizeof(struct in6_addr)); 565c47b2d9dSBen Hutchings } 566c47b2d9dSBen Hutchings 5673af0f342SEdward Cree req->spec.rem_port = fk.ports.src; 5683af0f342SEdward Cree req->spec.loc_port = fk.ports.dst; 569add72477SBen Hutchings 570f8d62037SEdward Cree if (efx->rps_hash_table) { 571f8d62037SEdward Cree /* Add it to ARFS hash table */ 572f8d62037SEdward Cree spin_lock(&efx->rps_hash_lock); 573f8d62037SEdward Cree rule = efx_rps_hash_add(efx, &req->spec, &new); 574f8d62037SEdward Cree if (!rule) { 575f8d62037SEdward Cree rc = -ENOMEM; 576f8d62037SEdward Cree goto out_unlock; 577f8d62037SEdward Cree } 578f8d62037SEdward Cree if (new) 579f8d62037SEdward Cree rule->arfs_id = efx->rps_next_id++ % RPS_NO_FILTER; 580f8d62037SEdward Cree rc = rule->arfs_id; 581f8d62037SEdward Cree /* Skip if existing or pending filter already does the right thing */ 582f8d62037SEdward Cree if (!new && rule->rxq_index == rxq_index && 583f8d62037SEdward Cree rule->filter_id >= EFX_ARFS_FILTER_ID_PENDING) 584f8d62037SEdward Cree goto out_unlock; 585f8d62037SEdward Cree rule->rxq_index = rxq_index; 586f8d62037SEdward Cree rule->filter_id = EFX_ARFS_FILTER_ID_PENDING; 587f8d62037SEdward Cree spin_unlock(&efx->rps_hash_lock); 588f8d62037SEdward Cree } else { 589f8d62037SEdward Cree /* Without an ARFS hash table, we just use arfs_id 0 for all 590f8d62037SEdward Cree * filters. This means if multiple flows hash to the same 591f8d62037SEdward Cree * flow_id, all but the most recently touched will be eligible 592f8d62037SEdward Cree * for expiry. 593f8d62037SEdward Cree */ 594f8d62037SEdward Cree rc = 0; 595f8d62037SEdward Cree } 596f8d62037SEdward Cree 597f8d62037SEdward Cree /* Queue the request */ 5983af0f342SEdward Cree dev_hold(req->net_dev = net_dev); 5993af0f342SEdward Cree INIT_WORK(&req->work, efx_filter_rfs_work); 6003af0f342SEdward Cree req->rxq_index = rxq_index; 6013af0f342SEdward Cree req->flow_id = flow_id; 6023af0f342SEdward Cree schedule_work(&req->work); 603f8d62037SEdward Cree return rc; 604f8d62037SEdward Cree out_unlock: 605f8d62037SEdward Cree spin_unlock(&efx->rps_hash_lock); 606f993740eSEdward Cree out_clear: 607f993740eSEdward Cree clear_bit(slot_idx, &efx->rps_slot_map); 608f993740eSEdward Cree return rc; 609add72477SBen Hutchings } 610add72477SBen Hutchings 6118490e75cSEdward Cree bool __efx_filter_rfs_expire(struct efx_channel *channel, unsigned int quota) 612add72477SBen Hutchings { 613add72477SBen Hutchings bool (*expire_one)(struct efx_nic *efx, u32 flow_id, unsigned int index); 6148490e75cSEdward Cree struct efx_nic *efx = channel->efx; 6158490e75cSEdward Cree unsigned int index, size, start; 616add72477SBen Hutchings u32 flow_id; 617add72477SBen Hutchings 6183af0f342SEdward Cree if (!mutex_trylock(&efx->rps_mutex)) 619add72477SBen Hutchings return false; 620add72477SBen Hutchings expire_one = efx->type->filter_rfs_expire_one; 6218490e75cSEdward Cree index = channel->rfs_expire_index; 6228490e75cSEdward Cree start = index; 623add72477SBen Hutchings size = efx->type->max_rx_ip_filters; 6248490e75cSEdward Cree while (quota) { 625faf8dcc1SJon Cooper flow_id = channel->rps_flow_id[index]; 626faf8dcc1SJon Cooper 6278490e75cSEdward Cree if (flow_id != RPS_FLOW_ID_INVALID) { 6288490e75cSEdward Cree quota--; 6298490e75cSEdward Cree if (expire_one(efx, flow_id, index)) { 630add72477SBen Hutchings netif_info(efx, rx_status, efx->net_dev, 6318490e75cSEdward Cree "expired filter %d [channel %u flow %u]\n", 6328490e75cSEdward Cree index, channel->channel, flow_id); 633faf8dcc1SJon Cooper channel->rps_flow_id[index] = RPS_FLOW_ID_INVALID; 6348490e75cSEdward Cree channel->rfs_filter_count--; 635faf8dcc1SJon Cooper } 6368490e75cSEdward Cree } 6378490e75cSEdward Cree if (++index == size) 638add72477SBen Hutchings index = 0; 6398490e75cSEdward Cree /* If we were called with a quota that exceeds the total number 6400aa6608dSEdward Cree * of filters in the table (which shouldn't happen, but could 6410aa6608dSEdward Cree * if two callers race), ensure that we don't loop forever - 6420aa6608dSEdward Cree * stop when we've examined every row of the table. 6438490e75cSEdward Cree */ 6440aa6608dSEdward Cree if (index == start) 6458490e75cSEdward Cree break; 646add72477SBen Hutchings } 647add72477SBen Hutchings 6488490e75cSEdward Cree channel->rfs_expire_index = index; 6493af0f342SEdward Cree mutex_unlock(&efx->rps_mutex); 650add72477SBen Hutchings return true; 651add72477SBen Hutchings } 652add72477SBen Hutchings 653add72477SBen Hutchings #endif /* CONFIG_RFS_ACCEL */ 654