1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2016-2017 Broadcom Limited 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 */ 9 #include <linux/kernel.h> 10 #include <linux/errno.h> 11 #include <linux/pci.h> 12 #include <linux/netdevice.h> 13 #include <linux/etherdevice.h> 14 #include <linux/if_vlan.h> 15 #include <linux/bpf.h> 16 #include <linux/bpf_trace.h> 17 #include <linux/filter.h> 18 #include <net/netdev_lock.h> 19 #include <net/page_pool/helpers.h> 20 #include <linux/bnxt/hsi.h> 21 #include "bnxt.h" 22 #include "bnxt_xdp.h" 23 24 DEFINE_STATIC_KEY_FALSE(bnxt_xdp_locking_key); 25 26 struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, 27 struct bnxt_tx_ring_info *txr, 28 dma_addr_t mapping, u32 len, 29 struct xdp_buff *xdp) 30 { 31 struct skb_shared_info *sinfo; 32 struct bnxt_sw_tx_bd *tx_buf; 33 struct tx_bd *txbd; 34 int num_frags = 0; 35 u32 flags; 36 u16 prod; 37 int i; 38 39 if (xdp && xdp_buff_has_frags(xdp)) { 40 sinfo = xdp_get_shared_info_from_buff(xdp); 41 num_frags = sinfo->nr_frags; 42 } 43 44 /* fill up the first buffer */ 45 prod = txr->tx_prod; 46 tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 47 tx_buf->nr_frags = num_frags; 48 if (xdp) 49 tx_buf->page = virt_to_head_page(xdp->data); 50 51 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 52 flags = (len << TX_BD_LEN_SHIFT) | TX_BD_CNT(num_frags + 1) | 53 bnxt_lhint_arr[len >> 9]; 54 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 55 txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 1 + num_frags); 56 txbd->tx_bd_haddr = cpu_to_le64(mapping); 57 58 /* now let us fill up the frags into the next buffers */ 59 for (i = 0; i < num_frags ; i++) { 60 skb_frag_t *frag = &sinfo->frags[i]; 61 struct bnxt_sw_tx_bd *frag_tx_buf; 62 dma_addr_t frag_mapping; 63 int frag_len; 64 65 prod = NEXT_TX(prod); 66 WRITE_ONCE(txr->tx_prod, prod); 67 68 /* first fill up the first buffer */ 69 frag_tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; 70 frag_tx_buf->page = skb_frag_page(frag); 71 72 txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; 73 74 frag_len = skb_frag_size(frag); 75 flags = frag_len << TX_BD_LEN_SHIFT; 76 txbd->tx_bd_len_flags_type = cpu_to_le32(flags); 77 frag_mapping = page_pool_get_dma_addr(skb_frag_page(frag)) + 78 skb_frag_off(frag); 79 txbd->tx_bd_haddr = cpu_to_le64(frag_mapping); 80 81 len = frag_len; 82 } 83 84 flags &= ~TX_BD_LEN; 85 txbd->tx_bd_len_flags_type = cpu_to_le32(((len) << TX_BD_LEN_SHIFT) | flags | 86 TX_BD_FLAGS_PACKET_END); 87 /* Sync TX BD */ 88 wmb(); 89 prod = NEXT_TX(prod); 90 WRITE_ONCE(txr->tx_prod, prod); 91 92 return tx_buf; 93 } 94 95 static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, 96 dma_addr_t mapping, u32 len, u16 rx_prod, 97 struct xdp_buff *xdp) 98 { 99 struct bnxt_sw_tx_bd *tx_buf; 100 101 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, xdp); 102 tx_buf->rx_prod = rx_prod; 103 tx_buf->action = XDP_TX; 104 105 } 106 107 static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, 108 struct bnxt_tx_ring_info *txr, 109 dma_addr_t mapping, u32 len, 110 struct xdp_frame *xdpf) 111 { 112 struct bnxt_sw_tx_bd *tx_buf; 113 114 tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL); 115 tx_buf->action = XDP_REDIRECT; 116 tx_buf->xdpf = xdpf; 117 dma_unmap_addr_set(tx_buf, mapping, mapping); 118 dma_unmap_len_set(tx_buf, len, len); 119 } 120 121 void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) 122 { 123 struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0]; 124 struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; 125 u16 tx_hw_cons = txr->tx_hw_cons; 126 bool rx_doorbell_needed = false; 127 struct bnxt_sw_tx_bd *tx_buf; 128 u16 tx_cons = txr->tx_cons; 129 u16 last_tx_cons = tx_cons; 130 int j, frags; 131 132 if (!budget) 133 return; 134 135 while (RING_TX(bp, tx_cons) != tx_hw_cons) { 136 tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)]; 137 138 if (tx_buf->action == XDP_REDIRECT) { 139 struct pci_dev *pdev = bp->pdev; 140 141 dma_unmap_single(&pdev->dev, 142 dma_unmap_addr(tx_buf, mapping), 143 dma_unmap_len(tx_buf, len), 144 DMA_TO_DEVICE); 145 xdp_return_frame(tx_buf->xdpf); 146 tx_buf->action = 0; 147 tx_buf->xdpf = NULL; 148 } else if (tx_buf->action == XDP_TX) { 149 tx_buf->action = 0; 150 rx_doorbell_needed = true; 151 last_tx_cons = tx_cons; 152 153 frags = tx_buf->nr_frags; 154 for (j = 0; j < frags; j++) { 155 tx_cons = NEXT_TX(tx_cons); 156 tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)]; 157 page_pool_recycle_direct(rxr->page_pool, tx_buf->page); 158 } 159 } else { 160 bnxt_sched_reset_txr(bp, txr, tx_cons); 161 return; 162 } 163 tx_cons = NEXT_TX(tx_cons); 164 } 165 166 bnapi->events &= ~BNXT_TX_CMP_EVENT; 167 WRITE_ONCE(txr->tx_cons, tx_cons); 168 if (rx_doorbell_needed) { 169 tx_buf = &txr->tx_buf_ring[RING_TX(bp, last_tx_cons)]; 170 bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod); 171 172 } 173 } 174 175 bool bnxt_xdp_attached(struct bnxt *bp, struct bnxt_rx_ring_info *rxr) 176 { 177 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); 178 179 return !!xdp_prog; 180 } 181 182 void bnxt_xdp_buff_init(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, 183 u16 cons, u8 *data_ptr, unsigned int len, 184 struct xdp_buff *xdp) 185 { 186 u32 buflen = rxr->rx_page_size; 187 struct bnxt_sw_rx_bd *rx_buf; 188 struct pci_dev *pdev; 189 dma_addr_t mapping; 190 u32 offset; 191 192 pdev = bp->pdev; 193 rx_buf = &rxr->rx_buf_ring[cons]; 194 offset = bp->rx_offset; 195 196 mapping = rx_buf->mapping - bp->rx_dma_offset; 197 dma_sync_single_for_cpu(&pdev->dev, mapping + offset, len, bp->rx_dir); 198 199 xdp_init_buff(xdp, buflen, &rxr->xdp_rxq); 200 xdp_prepare_buff(xdp, data_ptr - offset, offset, len, true); 201 } 202 203 void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, 204 struct xdp_buff *xdp) 205 { 206 struct skb_shared_info *shinfo; 207 int i; 208 209 if (!xdp || !xdp_buff_has_frags(xdp)) 210 return; 211 shinfo = xdp_get_shared_info_from_buff(xdp); 212 for (i = 0; i < shinfo->nr_frags; i++) { 213 struct page *page = skb_frag_page(&shinfo->frags[i]); 214 215 page_pool_recycle_direct(rxr->page_pool, page); 216 } 217 shinfo->nr_frags = 0; 218 } 219 220 /* returns the following: 221 * true - packet consumed by XDP and new buffer is allocated. 222 * false - packet should be passed to the stack. 223 */ 224 bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, 225 struct xdp_buff *xdp, struct page *page, u8 **data_ptr, 226 unsigned int *len, u8 *event) 227 { 228 struct bpf_prog *xdp_prog = READ_ONCE(rxr->xdp_prog); 229 struct bnxt_tx_ring_info *txr; 230 struct bnxt_sw_rx_bd *rx_buf; 231 struct pci_dev *pdev; 232 dma_addr_t mapping; 233 u32 tx_needed = 1; 234 void *orig_data; 235 u32 tx_avail; 236 u32 offset; 237 u32 act; 238 239 if (!xdp_prog) 240 return false; 241 242 pdev = bp->pdev; 243 offset = bp->rx_offset; 244 245 txr = rxr->bnapi->tx_ring[0]; 246 /* BNXT_RX_PAGE_MODE(bp) when XDP enabled */ 247 orig_data = xdp->data; 248 249 act = bpf_prog_run_xdp(xdp_prog, xdp); 250 251 tx_avail = bnxt_tx_avail(bp, txr); 252 /* If the tx ring is not full, we must not update the rx producer yet 253 * because we may still be transmitting on some BDs. 254 */ 255 if (tx_avail != bp->tx_ring_size) 256 *event &= ~BNXT_RX_EVENT; 257 258 *len = xdp->data_end - xdp->data; 259 if (orig_data != xdp->data) { 260 offset = xdp->data - xdp->data_hard_start; 261 *data_ptr = xdp->data_hard_start + offset; 262 } 263 264 switch (act) { 265 case XDP_PASS: 266 return false; 267 268 case XDP_TX: 269 rx_buf = &rxr->rx_buf_ring[cons]; 270 mapping = rx_buf->mapping - bp->rx_dma_offset; 271 272 if (unlikely(xdp_buff_has_frags(xdp))) { 273 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 274 275 tx_needed += sinfo->nr_frags; 276 } 277 278 if (tx_avail < tx_needed) { 279 trace_xdp_exception(bp->dev, xdp_prog, act); 280 bnxt_xdp_buff_frags_free(rxr, xdp); 281 bnxt_reuse_rx_data(rxr, cons, page); 282 return true; 283 } 284 285 dma_sync_single_for_device(&pdev->dev, mapping + offset, *len, 286 bp->rx_dir); 287 288 *event &= ~BNXT_RX_EVENT; 289 *event |= BNXT_TX_EVENT; 290 __bnxt_xmit_xdp(bp, txr, mapping + offset, *len, 291 NEXT_RX(rxr->rx_prod), xdp); 292 bnxt_reuse_rx_data(rxr, cons, page); 293 return true; 294 case XDP_REDIRECT: 295 /* if we are calling this here then we know that the 296 * redirect is coming from a frame received by the 297 * bnxt_en driver. 298 */ 299 300 /* if we are unable to allocate a new buffer, abort and reuse */ 301 if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) { 302 trace_xdp_exception(bp->dev, xdp_prog, act); 303 bnxt_xdp_buff_frags_free(rxr, xdp); 304 bnxt_reuse_rx_data(rxr, cons, page); 305 return true; 306 } 307 308 if (xdp_do_redirect(bp->dev, xdp, xdp_prog)) { 309 trace_xdp_exception(bp->dev, xdp_prog, act); 310 page_pool_recycle_direct(rxr->page_pool, page); 311 return true; 312 } 313 314 *event |= BNXT_REDIRECT_EVENT; 315 break; 316 default: 317 bpf_warn_invalid_xdp_action(bp->dev, xdp_prog, act); 318 fallthrough; 319 case XDP_ABORTED: 320 trace_xdp_exception(bp->dev, xdp_prog, act); 321 fallthrough; 322 case XDP_DROP: 323 bnxt_xdp_buff_frags_free(rxr, xdp); 324 bnxt_reuse_rx_data(rxr, cons, page); 325 break; 326 } 327 return true; 328 } 329 330 int bnxt_xdp_xmit(struct net_device *dev, int num_frames, 331 struct xdp_frame **frames, u32 flags) 332 { 333 struct bnxt *bp = netdev_priv(dev); 334 struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog); 335 struct pci_dev *pdev = bp->pdev; 336 struct bnxt_tx_ring_info *txr; 337 dma_addr_t mapping; 338 int nxmit = 0; 339 int ring; 340 int i; 341 342 if (!test_bit(BNXT_STATE_OPEN, &bp->state) || 343 !bp->tx_nr_rings_xdp || 344 !xdp_prog) 345 return -EINVAL; 346 347 ring = smp_processor_id() % bp->tx_nr_rings_xdp; 348 txr = &bp->tx_ring[ring]; 349 350 if (READ_ONCE(txr->dev_state) == BNXT_DEV_STATE_CLOSING) 351 return -EINVAL; 352 353 if (static_branch_unlikely(&bnxt_xdp_locking_key)) 354 spin_lock(&txr->xdp_tx_lock); 355 356 for (i = 0; i < num_frames; i++) { 357 struct xdp_frame *xdp = frames[i]; 358 359 if (!bnxt_tx_avail(bp, txr)) 360 break; 361 362 mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, 363 DMA_TO_DEVICE); 364 365 if (dma_mapping_error(&pdev->dev, mapping)) 366 break; 367 368 __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp); 369 nxmit++; 370 } 371 372 if (flags & XDP_XMIT_FLUSH) { 373 /* Sync BD data before updating doorbell */ 374 wmb(); 375 bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); 376 } 377 378 if (static_branch_unlikely(&bnxt_xdp_locking_key)) 379 spin_unlock(&txr->xdp_tx_lock); 380 381 return nxmit; 382 } 383 384 static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) 385 { 386 struct net_device *dev = bp->dev; 387 int tx_xdp = 0, tx_cp, rc, tc; 388 struct bpf_prog *old; 389 390 netdev_assert_locked(dev); 391 392 if (prog && !prog->aux->xdp_has_frags && 393 bp->dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { 394 netdev_warn(dev, "MTU %d larger than %d without XDP frag support.\n", 395 bp->dev->mtu, BNXT_MAX_PAGE_MODE_MTU); 396 return -EOPNOTSUPP; 397 } 398 if (prog && bp->flags & BNXT_FLAG_HDS) { 399 netdev_warn(dev, "XDP is disallowed when HDS is enabled.\n"); 400 return -EOPNOTSUPP; 401 } 402 if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) { 403 netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n"); 404 return -EOPNOTSUPP; 405 } 406 if (prog) 407 tx_xdp = bp->rx_nr_rings; 408 409 tc = bp->num_tc; 410 if (!tc) 411 tc = 1; 412 rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, 413 true, tc, tx_xdp); 414 if (rc) { 415 netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n"); 416 return rc; 417 } 418 if (netif_running(dev)) 419 bnxt_close_nic(bp, true, false); 420 421 old = xchg(&bp->xdp_prog, prog); 422 if (old) 423 bpf_prog_put(old); 424 425 if (prog) { 426 bnxt_set_rx_skb_mode(bp, true); 427 xdp_features_set_redirect_target_locked(dev, true); 428 } else { 429 xdp_features_clear_redirect_target_locked(dev); 430 bnxt_set_rx_skb_mode(bp, false); 431 } 432 bp->tx_nr_rings_xdp = tx_xdp; 433 bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp; 434 tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); 435 bp->cp_nr_rings = max_t(int, tx_cp, bp->rx_nr_rings); 436 bnxt_set_tpa_flags(bp); 437 bnxt_set_ring_params(bp); 438 439 if (netif_running(dev)) 440 return bnxt_open_nic(bp, true, false); 441 442 return 0; 443 } 444 445 int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) 446 { 447 struct bnxt *bp = netdev_priv(dev); 448 int rc; 449 450 switch (xdp->command) { 451 case XDP_SETUP_PROG: 452 rc = bnxt_xdp_set(bp, xdp->prog); 453 break; 454 default: 455 rc = -EINVAL; 456 break; 457 } 458 return rc; 459 } 460 461 struct sk_buff * 462 bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, 463 struct bnxt_rx_ring_info *rxr, struct xdp_buff *xdp) 464 { 465 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 466 467 if (!skb) 468 return NULL; 469 470 xdp_update_skb_frags_info(skb, num_frags, sinfo->xdp_frags_size, 471 rxr->rx_page_size * num_frags, 472 xdp_buff_get_skb_flags(xdp)); 473 return skb; 474 } 475