1 // SPDX-License-Identifier: ISC 2 /* 3 * Copyright (c) 2012-2017 Qualcomm Atheros, Inc. 4 * Copyright (c) 2018-2019, The Linux Foundation. All rights reserved. 5 */ 6 7 #include <linux/etherdevice.h> 8 #include <net/ieee80211_radiotap.h> 9 #include <linux/if_arp.h> 10 #include <linux/moduleparam.h> 11 #include <linux/ip.h> 12 #include <linux/ipv6.h> 13 #include <linux/if_vlan.h> 14 #include <net/ipv6.h> 15 #include <linux/prefetch.h> 16 17 #include "wil6210.h" 18 #include "wmi.h" 19 #include "txrx.h" 20 #include "trace.h" 21 #include "txrx_edma.h" 22 23 bool rx_align_2; 24 module_param(rx_align_2, bool, 0444); 25 MODULE_PARM_DESC(rx_align_2, " align Rx buffers on 4*n+2, default - no"); 26 27 bool rx_large_buf; 28 module_param(rx_large_buf, bool, 0444); 29 MODULE_PARM_DESC(rx_large_buf, " allocate 8KB RX buffers, default - no"); 30 31 /* Drop Tx packets in case Tx ring is full */ 32 bool drop_if_ring_full; 33 34 static inline uint wil_rx_snaplen(void) 35 { 36 return rx_align_2 ? 6 : 0; 37 } 38 39 /* wil_ring_wmark_low - low watermark for available descriptor space */ 40 static inline int wil_ring_wmark_low(struct wil_ring *ring) 41 { 42 return ring->size / 8; 43 } 44 45 /* wil_ring_wmark_high - high watermark for available descriptor space */ 46 static inline int wil_ring_wmark_high(struct wil_ring *ring) 47 { 48 return ring->size / 4; 49 } 50 51 /* returns true if num avail descriptors is lower than wmark_low */ 52 static inline int wil_ring_avail_low(struct wil_ring *ring) 53 { 54 return wil_ring_avail_tx(ring) < wil_ring_wmark_low(ring); 55 } 56 57 /* returns true if num avail descriptors is higher than wmark_high */ 58 static inline int wil_ring_avail_high(struct wil_ring *ring) 59 { 60 return wil_ring_avail_tx(ring) > wil_ring_wmark_high(ring); 61 } 62 63 /* returns true when all tx vrings are empty */ 64 bool wil_is_tx_idle(struct wil6210_priv *wil) 65 { 66 int i; 67 unsigned long data_comp_to; 68 int min_ring_id = wil_get_min_tx_ring_id(wil); 69 70 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 71 struct wil_ring *vring = &wil->ring_tx[i]; 72 int vring_index = vring - wil->ring_tx; 73 struct wil_ring_tx_data *txdata = 74 &wil->ring_tx_data[vring_index]; 75 76 spin_lock(&txdata->lock); 77 78 if (!vring->va || !txdata->enabled) { 79 spin_unlock(&txdata->lock); 80 continue; 81 } 82 83 data_comp_to = jiffies + msecs_to_jiffies( 84 WIL_DATA_COMPLETION_TO_MS); 85 if (test_bit(wil_status_napi_en, wil->status)) { 86 while (!wil_ring_is_empty(vring)) { 87 if (time_after(jiffies, data_comp_to)) { 88 wil_dbg_pm(wil, 89 "TO waiting for idle tx\n"); 90 spin_unlock(&txdata->lock); 91 return false; 92 } 93 wil_dbg_ratelimited(wil, 94 "tx vring is not empty -> NAPI\n"); 95 spin_unlock(&txdata->lock); 96 napi_synchronize(&wil->napi_tx); 97 msleep(20); 98 spin_lock(&txdata->lock); 99 if (!vring->va || !txdata->enabled) 100 break; 101 } 102 } 103 104 spin_unlock(&txdata->lock); 105 } 106 107 return true; 108 } 109 110 static int wil_vring_alloc(struct wil6210_priv *wil, struct wil_ring *vring) 111 { 112 struct device *dev = wil_to_dev(wil); 113 size_t sz = vring->size * sizeof(vring->va[0]); 114 uint i; 115 116 wil_dbg_misc(wil, "vring_alloc:\n"); 117 118 BUILD_BUG_ON(sizeof(vring->va[0]) != 32); 119 120 vring->swhead = 0; 121 vring->swtail = 0; 122 vring->ctx = kcalloc(vring->size, sizeof(vring->ctx[0]), GFP_KERNEL); 123 if (!vring->ctx) { 124 vring->va = NULL; 125 return -ENOMEM; 126 } 127 128 /* vring->va should be aligned on its size rounded up to power of 2 129 * This is granted by the dma_alloc_coherent. 130 * 131 * HW has limitation that all vrings addresses must share the same 132 * upper 16 msb bits part of 48 bits address. To workaround that, 133 * if we are using more than 32 bit addresses switch to 32 bit 134 * allocation before allocating vring memory. 135 * 136 * There's no check for the return value of dma_set_mask_and_coherent, 137 * since we assume if we were able to set the mask during 138 * initialization in this system it will not fail if we set it again 139 */ 140 if (wil->dma_addr_size > 32) 141 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 142 143 vring->va = dma_alloc_coherent(dev, sz, &vring->pa, GFP_KERNEL); 144 if (!vring->va) { 145 kfree(vring->ctx); 146 vring->ctx = NULL; 147 return -ENOMEM; 148 } 149 150 if (wil->dma_addr_size > 32) 151 dma_set_mask_and_coherent(dev, 152 DMA_BIT_MASK(wil->dma_addr_size)); 153 154 /* initially, all descriptors are SW owned 155 * For Tx and Rx, ownership bit is at the same location, thus 156 * we can use any 157 */ 158 for (i = 0; i < vring->size; i++) { 159 volatile struct vring_tx_desc *_d = 160 &vring->va[i].tx.legacy; 161 162 _d->dma.status = TX_DMA_STATUS_DU; 163 } 164 165 wil_dbg_misc(wil, "vring[%d] 0x%p:%pad 0x%p\n", vring->size, 166 vring->va, &vring->pa, vring->ctx); 167 168 return 0; 169 } 170 171 static void wil_txdesc_unmap(struct device *dev, union wil_tx_desc *desc, 172 struct wil_ctx *ctx) 173 { 174 struct vring_tx_desc *d = &desc->legacy; 175 dma_addr_t pa = wil_desc_addr(&d->dma.addr); 176 u16 dmalen = le16_to_cpu(d->dma.length); 177 178 switch (ctx->mapped_as) { 179 case wil_mapped_as_single: 180 dma_unmap_single(dev, pa, dmalen, DMA_TO_DEVICE); 181 break; 182 case wil_mapped_as_page: 183 dma_unmap_page(dev, pa, dmalen, DMA_TO_DEVICE); 184 break; 185 default: 186 break; 187 } 188 } 189 190 static void wil_vring_free(struct wil6210_priv *wil, struct wil_ring *vring) 191 { 192 struct device *dev = wil_to_dev(wil); 193 size_t sz = vring->size * sizeof(vring->va[0]); 194 195 lockdep_assert_held(&wil->mutex); 196 if (!vring->is_rx) { 197 int vring_index = vring - wil->ring_tx; 198 199 wil_dbg_misc(wil, "free Tx vring %d [%d] 0x%p:%pad 0x%p\n", 200 vring_index, vring->size, vring->va, 201 &vring->pa, vring->ctx); 202 } else { 203 wil_dbg_misc(wil, "free Rx vring [%d] 0x%p:%pad 0x%p\n", 204 vring->size, vring->va, 205 &vring->pa, vring->ctx); 206 } 207 208 while (!wil_ring_is_empty(vring)) { 209 dma_addr_t pa; 210 u16 dmalen; 211 struct wil_ctx *ctx; 212 213 if (!vring->is_rx) { 214 struct vring_tx_desc dd, *d = ⅆ 215 volatile struct vring_tx_desc *_d = 216 &vring->va[vring->swtail].tx.legacy; 217 218 ctx = &vring->ctx[vring->swtail]; 219 if (!ctx) { 220 wil_dbg_txrx(wil, 221 "ctx(%d) was already completed\n", 222 vring->swtail); 223 vring->swtail = wil_ring_next_tail(vring); 224 continue; 225 } 226 *d = *_d; 227 wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx); 228 if (ctx->skb) 229 dev_kfree_skb_any(ctx->skb); 230 vring->swtail = wil_ring_next_tail(vring); 231 } else { /* rx */ 232 struct vring_rx_desc dd, *d = ⅆ 233 volatile struct vring_rx_desc *_d = 234 &vring->va[vring->swhead].rx.legacy; 235 236 ctx = &vring->ctx[vring->swhead]; 237 *d = *_d; 238 pa = wil_desc_addr(&d->dma.addr); 239 dmalen = le16_to_cpu(d->dma.length); 240 dma_unmap_single(dev, pa, dmalen, DMA_FROM_DEVICE); 241 kfree_skb(ctx->skb); 242 wil_ring_advance_head(vring, 1); 243 } 244 } 245 dma_free_coherent(dev, sz, (void *)vring->va, vring->pa); 246 kfree(vring->ctx); 247 vring->pa = 0; 248 vring->va = NULL; 249 vring->ctx = NULL; 250 } 251 252 /** 253 * Allocate one skb for Rx VRING 254 * 255 * Safe to call from IRQ 256 */ 257 static int wil_vring_alloc_skb(struct wil6210_priv *wil, struct wil_ring *vring, 258 u32 i, int headroom) 259 { 260 struct device *dev = wil_to_dev(wil); 261 unsigned int sz = wil->rx_buf_len + ETH_HLEN + wil_rx_snaplen(); 262 struct vring_rx_desc dd, *d = ⅆ 263 volatile struct vring_rx_desc *_d = &vring->va[i].rx.legacy; 264 dma_addr_t pa; 265 struct sk_buff *skb = dev_alloc_skb(sz + headroom); 266 267 if (unlikely(!skb)) 268 return -ENOMEM; 269 270 skb_reserve(skb, headroom); 271 skb_put(skb, sz); 272 273 /** 274 * Make sure that the network stack calculates checksum for packets 275 * which failed the HW checksum calculation 276 */ 277 skb->ip_summed = CHECKSUM_NONE; 278 279 pa = dma_map_single(dev, skb->data, skb->len, DMA_FROM_DEVICE); 280 if (unlikely(dma_mapping_error(dev, pa))) { 281 kfree_skb(skb); 282 return -ENOMEM; 283 } 284 285 d->dma.d0 = RX_DMA_D0_CMD_DMA_RT | RX_DMA_D0_CMD_DMA_IT; 286 wil_desc_addr_set(&d->dma.addr, pa); 287 /* ip_length don't care */ 288 /* b11 don't care */ 289 /* error don't care */ 290 d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */ 291 d->dma.length = cpu_to_le16(sz); 292 *_d = *d; 293 vring->ctx[i].skb = skb; 294 295 return 0; 296 } 297 298 /** 299 * Adds radiotap header 300 * 301 * Any error indicated as "Bad FCS" 302 * 303 * Vendor data for 04:ce:14-1 (Wilocity-1) consists of: 304 * - Rx descriptor: 32 bytes 305 * - Phy info 306 */ 307 static void wil_rx_add_radiotap_header(struct wil6210_priv *wil, 308 struct sk_buff *skb) 309 { 310 struct wil6210_rtap { 311 struct ieee80211_radiotap_header rthdr; 312 /* fields should be in the order of bits in rthdr.it_present */ 313 /* flags */ 314 u8 flags; 315 /* channel */ 316 __le16 chnl_freq __aligned(2); 317 __le16 chnl_flags; 318 /* MCS */ 319 u8 mcs_present; 320 u8 mcs_flags; 321 u8 mcs_index; 322 } __packed; 323 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 324 struct wil6210_rtap *rtap; 325 int rtap_len = sizeof(struct wil6210_rtap); 326 struct ieee80211_channel *ch = wil->monitor_chandef.chan; 327 328 if (skb_headroom(skb) < rtap_len && 329 pskb_expand_head(skb, rtap_len, 0, GFP_ATOMIC)) { 330 wil_err(wil, "Unable to expand headroom to %d\n", rtap_len); 331 return; 332 } 333 334 rtap = skb_push(skb, rtap_len); 335 memset(rtap, 0, rtap_len); 336 337 rtap->rthdr.it_version = PKTHDR_RADIOTAP_VERSION; 338 rtap->rthdr.it_len = cpu_to_le16(rtap_len); 339 rtap->rthdr.it_present = cpu_to_le32((1 << IEEE80211_RADIOTAP_FLAGS) | 340 (1 << IEEE80211_RADIOTAP_CHANNEL) | 341 (1 << IEEE80211_RADIOTAP_MCS)); 342 if (d->dma.status & RX_DMA_STATUS_ERROR) 343 rtap->flags |= IEEE80211_RADIOTAP_F_BADFCS; 344 345 rtap->chnl_freq = cpu_to_le16(ch ? ch->center_freq : 58320); 346 rtap->chnl_flags = cpu_to_le16(0); 347 348 rtap->mcs_present = IEEE80211_RADIOTAP_MCS_HAVE_MCS; 349 rtap->mcs_flags = 0; 350 rtap->mcs_index = wil_rxdesc_mcs(d); 351 } 352 353 static bool wil_is_rx_idle(struct wil6210_priv *wil) 354 { 355 struct vring_rx_desc *_d; 356 struct wil_ring *ring = &wil->ring_rx; 357 358 _d = (struct vring_rx_desc *)&ring->va[ring->swhead].rx.legacy; 359 if (_d->dma.status & RX_DMA_STATUS_DU) 360 return false; 361 362 return true; 363 } 364 365 static int wil_rx_get_cid_by_skb(struct wil6210_priv *wil, struct sk_buff *skb) 366 { 367 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 368 int mid = wil_rxdesc_mid(d); 369 struct wil6210_vif *vif = wil->vifs[mid]; 370 /* cid from DMA descriptor is limited to 3 bits. 371 * In case of cid>=8, the value would be cid modulo 8 and we need to 372 * find real cid by locating the transmitter (ta) inside sta array 373 */ 374 int cid = wil_rxdesc_cid(d); 375 unsigned int snaplen = wil_rx_snaplen(); 376 struct ieee80211_hdr_3addr *hdr; 377 int i; 378 unsigned char *ta; 379 u8 ftype; 380 381 /* in monitor mode there are no connections */ 382 if (vif->wdev.iftype == NL80211_IFTYPE_MONITOR) 383 return cid; 384 385 ftype = wil_rxdesc_ftype(d) << 2; 386 if (likely(ftype == IEEE80211_FTYPE_DATA)) { 387 if (unlikely(skb->len < ETH_HLEN + snaplen)) { 388 wil_err_ratelimited(wil, 389 "Short data frame, len = %d\n", 390 skb->len); 391 return -ENOENT; 392 } 393 ta = wil_skb_get_sa(skb); 394 } else { 395 if (unlikely(skb->len < sizeof(struct ieee80211_hdr_3addr))) { 396 wil_err_ratelimited(wil, "Short frame, len = %d\n", 397 skb->len); 398 return -ENOENT; 399 } 400 hdr = (void *)skb->data; 401 ta = hdr->addr2; 402 } 403 404 if (wil->max_assoc_sta <= WIL6210_RX_DESC_MAX_CID) 405 return cid; 406 407 /* assuming no concurrency between AP interfaces and STA interfaces. 408 * multista is used only in P2P_GO or AP mode. In other modes return 409 * cid from the rx descriptor 410 */ 411 if (vif->wdev.iftype != NL80211_IFTYPE_P2P_GO && 412 vif->wdev.iftype != NL80211_IFTYPE_AP) 413 return cid; 414 415 /* For Rx packets cid from rx descriptor is limited to 3 bits (0..7), 416 * to find the real cid, compare transmitter address with the stored 417 * stations mac address in the driver sta array 418 */ 419 for (i = cid; i < wil->max_assoc_sta; i += WIL6210_RX_DESC_MAX_CID) { 420 if (wil->sta[i].status != wil_sta_unused && 421 ether_addr_equal(wil->sta[i].addr, ta)) { 422 cid = i; 423 break; 424 } 425 } 426 if (i >= wil->max_assoc_sta) { 427 wil_err_ratelimited(wil, "Could not find cid for frame with transmit addr = %pM, iftype = %d, frametype = %d, len = %d\n", 428 ta, vif->wdev.iftype, ftype, skb->len); 429 cid = -ENOENT; 430 } 431 432 return cid; 433 } 434 435 /** 436 * reap 1 frame from @swhead 437 * 438 * Rx descriptor copied to skb->cb 439 * 440 * Safe to call from IRQ 441 */ 442 static struct sk_buff *wil_vring_reap_rx(struct wil6210_priv *wil, 443 struct wil_ring *vring) 444 { 445 struct device *dev = wil_to_dev(wil); 446 struct wil6210_vif *vif; 447 struct net_device *ndev; 448 volatile struct vring_rx_desc *_d; 449 struct vring_rx_desc *d; 450 struct sk_buff *skb; 451 dma_addr_t pa; 452 unsigned int snaplen = wil_rx_snaplen(); 453 unsigned int sz = wil->rx_buf_len + ETH_HLEN + snaplen; 454 u16 dmalen; 455 u8 ftype; 456 int cid, mid; 457 int i; 458 struct wil_net_stats *stats; 459 460 BUILD_BUG_ON(sizeof(struct skb_rx_info) > sizeof(skb->cb)); 461 462 again: 463 if (unlikely(wil_ring_is_empty(vring))) 464 return NULL; 465 466 i = (int)vring->swhead; 467 _d = &vring->va[i].rx.legacy; 468 if (unlikely(!(_d->dma.status & RX_DMA_STATUS_DU))) { 469 /* it is not error, we just reached end of Rx done area */ 470 return NULL; 471 } 472 473 skb = vring->ctx[i].skb; 474 vring->ctx[i].skb = NULL; 475 wil_ring_advance_head(vring, 1); 476 if (!skb) { 477 wil_err(wil, "No Rx skb at [%d]\n", i); 478 goto again; 479 } 480 d = wil_skb_rxdesc(skb); 481 *d = *_d; 482 pa = wil_desc_addr(&d->dma.addr); 483 484 dma_unmap_single(dev, pa, sz, DMA_FROM_DEVICE); 485 dmalen = le16_to_cpu(d->dma.length); 486 487 trace_wil6210_rx(i, d); 488 wil_dbg_txrx(wil, "Rx[%3d] : %d bytes\n", i, dmalen); 489 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4, 490 (const void *)d, sizeof(*d), false); 491 492 mid = wil_rxdesc_mid(d); 493 vif = wil->vifs[mid]; 494 495 if (unlikely(!vif)) { 496 wil_dbg_txrx(wil, "skipped RX descriptor with invalid mid %d", 497 mid); 498 kfree_skb(skb); 499 goto again; 500 } 501 ndev = vif_to_ndev(vif); 502 if (unlikely(dmalen > sz)) { 503 wil_err_ratelimited(wil, "Rx size too large: %d bytes!\n", 504 dmalen); 505 kfree_skb(skb); 506 goto again; 507 } 508 skb_trim(skb, dmalen); 509 510 prefetch(skb->data); 511 512 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 513 skb->data, skb_headlen(skb), false); 514 515 cid = wil_rx_get_cid_by_skb(wil, skb); 516 if (cid == -ENOENT) { 517 kfree_skb(skb); 518 goto again; 519 } 520 wil_skb_set_cid(skb, (u8)cid); 521 stats = &wil->sta[cid].stats; 522 523 stats->last_mcs_rx = wil_rxdesc_mcs(d); 524 if (stats->last_mcs_rx < ARRAY_SIZE(stats->rx_per_mcs)) 525 stats->rx_per_mcs[stats->last_mcs_rx]++; 526 527 /* use radiotap header only if required */ 528 if (ndev->type == ARPHRD_IEEE80211_RADIOTAP) 529 wil_rx_add_radiotap_header(wil, skb); 530 531 /* no extra checks if in sniffer mode */ 532 if (ndev->type != ARPHRD_ETHER) 533 return skb; 534 /* Non-data frames may be delivered through Rx DMA channel (ex: BAR) 535 * Driver should recognize it by frame type, that is found 536 * in Rx descriptor. If type is not data, it is 802.11 frame as is 537 */ 538 ftype = wil_rxdesc_ftype(d) << 2; 539 if (unlikely(ftype != IEEE80211_FTYPE_DATA)) { 540 u8 fc1 = wil_rxdesc_fc1(d); 541 int tid = wil_rxdesc_tid(d); 542 u16 seq = wil_rxdesc_seq(d); 543 544 wil_dbg_txrx(wil, 545 "Non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 546 fc1, mid, cid, tid, seq); 547 stats->rx_non_data_frame++; 548 if (wil_is_back_req(fc1)) { 549 wil_dbg_txrx(wil, 550 "BAR: MID %d CID %d TID %d Seq 0x%03x\n", 551 mid, cid, tid, seq); 552 wil_rx_bar(wil, vif, cid, tid, seq); 553 } else { 554 /* print again all info. One can enable only this 555 * without overhead for printing every Rx frame 556 */ 557 wil_dbg_txrx(wil, 558 "Unhandled non-data frame FC[7:0] 0x%02x MID %d CID %d TID %d Seq 0x%03x\n", 559 fc1, mid, cid, tid, seq); 560 wil_hex_dump_txrx("RxD ", DUMP_PREFIX_NONE, 32, 4, 561 (const void *)d, sizeof(*d), false); 562 wil_hex_dump_txrx("Rx ", DUMP_PREFIX_OFFSET, 16, 1, 563 skb->data, skb_headlen(skb), false); 564 } 565 kfree_skb(skb); 566 goto again; 567 } 568 569 /* L4 IDENT is on when HW calculated checksum, check status 570 * and in case of error drop the packet 571 * higher stack layers will handle retransmission (if required) 572 */ 573 if (likely(d->dma.status & RX_DMA_STATUS_L4I)) { 574 /* L4 protocol identified, csum calculated */ 575 if (likely((d->dma.error & RX_DMA_ERROR_L4_ERR) == 0)) 576 skb->ip_summed = CHECKSUM_UNNECESSARY; 577 /* If HW reports bad checksum, let IP stack re-check it 578 * For example, HW don't understand Microsoft IP stack that 579 * mis-calculates TCP checksum - if it should be 0x0, 580 * it writes 0xffff in violation of RFC 1624 581 */ 582 else 583 stats->rx_csum_err++; 584 } 585 586 if (snaplen) { 587 /* Packet layout 588 * +-------+-------+---------+------------+------+ 589 * | SA(6) | DA(6) | SNAP(6) | ETHTYPE(2) | DATA | 590 * +-------+-------+---------+------------+------+ 591 * Need to remove SNAP, shifting SA and DA forward 592 */ 593 memmove(skb->data + snaplen, skb->data, 2 * ETH_ALEN); 594 skb_pull(skb, snaplen); 595 } 596 597 return skb; 598 } 599 600 /** 601 * allocate and fill up to @count buffers in rx ring 602 * buffers posted at @swtail 603 * Note: we have a single RX queue for servicing all VIFs, but we 604 * allocate skbs with headroom according to main interface only. This 605 * means it will not work with monitor interface together with other VIFs. 606 * Currently we only support monitor interface on its own without other VIFs, 607 * and we will need to fix this code once we add support. 608 */ 609 static int wil_rx_refill(struct wil6210_priv *wil, int count) 610 { 611 struct net_device *ndev = wil->main_ndev; 612 struct wil_ring *v = &wil->ring_rx; 613 u32 next_tail; 614 int rc = 0; 615 int headroom = ndev->type == ARPHRD_IEEE80211_RADIOTAP ? 616 WIL6210_RTAP_SIZE : 0; 617 618 for (; next_tail = wil_ring_next_tail(v), 619 (next_tail != v->swhead) && (count-- > 0); 620 v->swtail = next_tail) { 621 rc = wil_vring_alloc_skb(wil, v, v->swtail, headroom); 622 if (unlikely(rc)) { 623 wil_err_ratelimited(wil, "Error %d in rx refill[%d]\n", 624 rc, v->swtail); 625 break; 626 } 627 } 628 629 /* make sure all writes to descriptors (shared memory) are done before 630 * committing them to HW 631 */ 632 wmb(); 633 634 wil_w(wil, v->hwtail, v->swtail); 635 636 return rc; 637 } 638 639 /** 640 * reverse_memcmp - Compare two areas of memory, in reverse order 641 * @cs: One area of memory 642 * @ct: Another area of memory 643 * @count: The size of the area. 644 * 645 * Cut'n'paste from original memcmp (see lib/string.c) 646 * with minimal modifications 647 */ 648 int reverse_memcmp(const void *cs, const void *ct, size_t count) 649 { 650 const unsigned char *su1, *su2; 651 int res = 0; 652 653 for (su1 = cs + count - 1, su2 = ct + count - 1; count > 0; 654 --su1, --su2, count--) { 655 res = *su1 - *su2; 656 if (res) 657 break; 658 } 659 return res; 660 } 661 662 static int wil_rx_crypto_check(struct wil6210_priv *wil, struct sk_buff *skb) 663 { 664 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 665 int cid = wil_skb_get_cid(skb); 666 int tid = wil_rxdesc_tid(d); 667 int key_id = wil_rxdesc_key_id(d); 668 int mc = wil_rxdesc_mcast(d); 669 struct wil_sta_info *s = &wil->sta[cid]; 670 struct wil_tid_crypto_rx *c = mc ? &s->group_crypto_rx : 671 &s->tid_crypto_rx[tid]; 672 struct wil_tid_crypto_rx_single *cc = &c->key_id[key_id]; 673 const u8 *pn = (u8 *)&d->mac.pn_15_0; 674 675 if (!cc->key_set) { 676 wil_err_ratelimited(wil, 677 "Key missing. CID %d TID %d MCast %d KEY_ID %d\n", 678 cid, tid, mc, key_id); 679 return -EINVAL; 680 } 681 682 if (reverse_memcmp(pn, cc->pn, IEEE80211_GCMP_PN_LEN) <= 0) { 683 wil_err_ratelimited(wil, 684 "Replay attack. CID %d TID %d MCast %d KEY_ID %d PN %6phN last %6phN\n", 685 cid, tid, mc, key_id, pn, cc->pn); 686 return -EINVAL; 687 } 688 memcpy(cc->pn, pn, IEEE80211_GCMP_PN_LEN); 689 690 return 0; 691 } 692 693 static int wil_rx_error_check(struct wil6210_priv *wil, struct sk_buff *skb, 694 struct wil_net_stats *stats) 695 { 696 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 697 698 if ((d->dma.status & RX_DMA_STATUS_ERROR) && 699 (d->dma.error & RX_DMA_ERROR_MIC)) { 700 stats->rx_mic_error++; 701 wil_dbg_txrx(wil, "MIC error, dropping packet\n"); 702 return -EFAULT; 703 } 704 705 return 0; 706 } 707 708 static void wil_get_netif_rx_params(struct sk_buff *skb, int *cid, 709 int *security) 710 { 711 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 712 713 *cid = wil_skb_get_cid(skb); 714 *security = wil_rxdesc_security(d); 715 } 716 717 /* 718 * Check if skb is ptk eapol key message 719 * 720 * returns a pointer to the start of the eapol key structure, NULL 721 * if frame is not PTK eapol key 722 */ 723 static struct wil_eapol_key *wil_is_ptk_eapol_key(struct wil6210_priv *wil, 724 struct sk_buff *skb) 725 { 726 u8 *buf; 727 const struct wil_1x_hdr *hdr; 728 struct wil_eapol_key *key; 729 u16 key_info; 730 int len = skb->len; 731 732 if (!skb_mac_header_was_set(skb)) { 733 wil_err(wil, "mac header was not set\n"); 734 return NULL; 735 } 736 737 len -= skb_mac_offset(skb); 738 739 if (len < sizeof(struct ethhdr) + sizeof(struct wil_1x_hdr) + 740 sizeof(struct wil_eapol_key)) 741 return NULL; 742 743 buf = skb_mac_header(skb) + sizeof(struct ethhdr); 744 745 hdr = (const struct wil_1x_hdr *)buf; 746 if (hdr->type != WIL_1X_TYPE_EAPOL_KEY) 747 return NULL; 748 749 key = (struct wil_eapol_key *)(buf + sizeof(struct wil_1x_hdr)); 750 if (key->type != WIL_EAPOL_KEY_TYPE_WPA && 751 key->type != WIL_EAPOL_KEY_TYPE_RSN) 752 return NULL; 753 754 key_info = be16_to_cpu(key->key_info); 755 if (!(key_info & WIL_KEY_INFO_KEY_TYPE)) /* check if pairwise */ 756 return NULL; 757 758 return key; 759 } 760 761 static bool wil_skb_is_eap_3(struct wil6210_priv *wil, struct sk_buff *skb) 762 { 763 struct wil_eapol_key *key; 764 u16 key_info; 765 766 key = wil_is_ptk_eapol_key(wil, skb); 767 if (!key) 768 return false; 769 770 key_info = be16_to_cpu(key->key_info); 771 if (key_info & (WIL_KEY_INFO_MIC | 772 WIL_KEY_INFO_ENCR_KEY_DATA)) { 773 /* 3/4 of 4-Way Handshake */ 774 wil_dbg_misc(wil, "EAPOL key message 3\n"); 775 return true; 776 } 777 /* 1/4 of 4-Way Handshake */ 778 wil_dbg_misc(wil, "EAPOL key message 1\n"); 779 780 return false; 781 } 782 783 static bool wil_skb_is_eap_4(struct wil6210_priv *wil, struct sk_buff *skb) 784 { 785 struct wil_eapol_key *key; 786 u32 *nonce, i; 787 788 key = wil_is_ptk_eapol_key(wil, skb); 789 if (!key) 790 return false; 791 792 nonce = (u32 *)key->key_nonce; 793 for (i = 0; i < WIL_EAP_NONCE_LEN / sizeof(u32); i++, nonce++) { 794 if (*nonce != 0) { 795 /* message 2/4 */ 796 wil_dbg_misc(wil, "EAPOL key message 2\n"); 797 return false; 798 } 799 } 800 wil_dbg_misc(wil, "EAPOL key message 4\n"); 801 802 return true; 803 } 804 805 void wil_enable_tx_key_worker(struct work_struct *work) 806 { 807 struct wil6210_vif *vif = container_of(work, 808 struct wil6210_vif, enable_tx_key_worker); 809 struct wil6210_priv *wil = vif_to_wil(vif); 810 int rc, cid; 811 812 rtnl_lock(); 813 if (vif->ptk_rekey_state != WIL_REKEY_WAIT_M4_SENT) { 814 wil_dbg_misc(wil, "Invalid rekey state = %d\n", 815 vif->ptk_rekey_state); 816 rtnl_unlock(); 817 return; 818 } 819 820 cid = wil_find_cid_by_idx(wil, vif->mid, 0); 821 if (!wil_cid_valid(wil, cid)) { 822 wil_err(wil, "Invalid cid = %d\n", cid); 823 rtnl_unlock(); 824 return; 825 } 826 827 wil_dbg_misc(wil, "Apply PTK key after eapol was sent out\n"); 828 rc = wmi_add_cipher_key(vif, 0, wil->sta[cid].addr, 0, NULL, 829 WMI_KEY_USE_APPLY_PTK); 830 831 vif->ptk_rekey_state = WIL_REKEY_IDLE; 832 rtnl_unlock(); 833 834 if (rc) 835 wil_err(wil, "Apply PTK key failed %d\n", rc); 836 } 837 838 void wil_tx_complete_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb) 839 { 840 struct wil6210_priv *wil = vif_to_wil(vif); 841 struct wireless_dev *wdev = vif_to_wdev(vif); 842 bool q = false; 843 844 if (wdev->iftype != NL80211_IFTYPE_STATION || 845 !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities)) 846 return; 847 848 /* check if skb is an EAP message 4/4 */ 849 if (!wil_skb_is_eap_4(wil, skb)) 850 return; 851 852 spin_lock_bh(&wil->eap_lock); 853 switch (vif->ptk_rekey_state) { 854 case WIL_REKEY_IDLE: 855 /* ignore idle state, can happen due to M4 retransmission */ 856 break; 857 case WIL_REKEY_M3_RECEIVED: 858 vif->ptk_rekey_state = WIL_REKEY_IDLE; 859 break; 860 case WIL_REKEY_WAIT_M4_SENT: 861 q = true; 862 break; 863 default: 864 wil_err(wil, "Unknown rekey state = %d", 865 vif->ptk_rekey_state); 866 } 867 spin_unlock_bh(&wil->eap_lock); 868 869 if (q) { 870 q = queue_work(wil->wmi_wq, &vif->enable_tx_key_worker); 871 wil_dbg_misc(wil, "queue_work of enable_tx_key_worker -> %d\n", 872 q); 873 } 874 } 875 876 static void wil_rx_handle_eapol(struct wil6210_vif *vif, struct sk_buff *skb) 877 { 878 struct wil6210_priv *wil = vif_to_wil(vif); 879 struct wireless_dev *wdev = vif_to_wdev(vif); 880 881 if (wdev->iftype != NL80211_IFTYPE_STATION || 882 !test_bit(WMI_FW_CAPABILITY_SPLIT_REKEY, wil->fw_capabilities)) 883 return; 884 885 /* check if skb is a EAP message 3/4 */ 886 if (!wil_skb_is_eap_3(wil, skb)) 887 return; 888 889 if (vif->ptk_rekey_state == WIL_REKEY_IDLE) 890 vif->ptk_rekey_state = WIL_REKEY_M3_RECEIVED; 891 } 892 893 /* 894 * Pass Rx packet to the netif. Update statistics. 895 * Called in softirq context (NAPI poll). 896 */ 897 void wil_netif_rx(struct sk_buff *skb, struct net_device *ndev, int cid, 898 struct wil_net_stats *stats, bool gro) 899 { 900 struct wil6210_vif *vif = ndev_to_vif(ndev); 901 struct wil6210_priv *wil = ndev_to_wil(ndev); 902 struct wireless_dev *wdev = vif_to_wdev(vif); 903 unsigned int len = skb->len; 904 u8 *sa, *da = wil_skb_get_da(skb); 905 /* here looking for DA, not A1, thus Rxdesc's 'mcast' indication 906 * is not suitable, need to look at data 907 */ 908 int mcast = is_multicast_ether_addr(da); 909 struct sk_buff *xmit_skb = NULL; 910 911 if (wdev->iftype == NL80211_IFTYPE_STATION) { 912 sa = wil_skb_get_sa(skb); 913 if (mcast && ether_addr_equal(sa, ndev->dev_addr)) { 914 /* mcast packet looped back to us */ 915 dev_kfree_skb(skb); 916 ndev->stats.rx_dropped++; 917 stats->rx_dropped++; 918 wil_dbg_txrx(wil, "Rx drop %d bytes\n", len); 919 return; 920 } 921 } else if (wdev->iftype == NL80211_IFTYPE_AP && !vif->ap_isolate) { 922 if (mcast) { 923 /* send multicast frames both to higher layers in 924 * local net stack and back to the wireless medium 925 */ 926 xmit_skb = skb_copy(skb, GFP_ATOMIC); 927 } else { 928 int xmit_cid = wil_find_cid(wil, vif->mid, da); 929 930 if (xmit_cid >= 0) { 931 /* The destination station is associated to 932 * this AP (in this VLAN), so send the frame 933 * directly to it and do not pass it to local 934 * net stack. 935 */ 936 xmit_skb = skb; 937 skb = NULL; 938 } 939 } 940 } 941 if (xmit_skb) { 942 /* Send to wireless media and increase priority by 256 to 943 * keep the received priority instead of reclassifying 944 * the frame (see cfg80211_classify8021d). 945 */ 946 xmit_skb->dev = ndev; 947 xmit_skb->priority += 256; 948 xmit_skb->protocol = htons(ETH_P_802_3); 949 skb_reset_network_header(xmit_skb); 950 skb_reset_mac_header(xmit_skb); 951 wil_dbg_txrx(wil, "Rx -> Tx %d bytes\n", len); 952 dev_queue_xmit(xmit_skb); 953 } 954 955 if (skb) { /* deliver to local stack */ 956 skb->protocol = eth_type_trans(skb, ndev); 957 skb->dev = ndev; 958 959 if (skb->protocol == cpu_to_be16(ETH_P_PAE)) 960 wil_rx_handle_eapol(vif, skb); 961 962 if (gro) 963 napi_gro_receive(&wil->napi_rx, skb); 964 else 965 netif_rx_ni(skb); 966 } 967 ndev->stats.rx_packets++; 968 stats->rx_packets++; 969 ndev->stats.rx_bytes += len; 970 stats->rx_bytes += len; 971 if (mcast) 972 ndev->stats.multicast++; 973 } 974 975 void wil_netif_rx_any(struct sk_buff *skb, struct net_device *ndev) 976 { 977 int cid, security; 978 struct wil6210_priv *wil = ndev_to_wil(ndev); 979 struct wil_net_stats *stats; 980 981 wil->txrx_ops.get_netif_rx_params(skb, &cid, &security); 982 983 stats = &wil->sta[cid].stats; 984 985 skb_orphan(skb); 986 987 if (security && (wil->txrx_ops.rx_crypto_check(wil, skb) != 0)) { 988 wil_dbg_txrx(wil, "Rx drop %d bytes\n", skb->len); 989 dev_kfree_skb(skb); 990 ndev->stats.rx_dropped++; 991 stats->rx_replay++; 992 stats->rx_dropped++; 993 return; 994 } 995 996 /* check errors reported by HW and update statistics */ 997 if (unlikely(wil->txrx_ops.rx_error_check(wil, skb, stats))) { 998 dev_kfree_skb(skb); 999 return; 1000 } 1001 1002 wil_netif_rx(skb, ndev, cid, stats, true); 1003 } 1004 1005 /** 1006 * Proceed all completed skb's from Rx VRING 1007 * 1008 * Safe to call from NAPI poll, i.e. softirq with interrupts enabled 1009 */ 1010 void wil_rx_handle(struct wil6210_priv *wil, int *quota) 1011 { 1012 struct net_device *ndev = wil->main_ndev; 1013 struct wireless_dev *wdev = ndev->ieee80211_ptr; 1014 struct wil_ring *v = &wil->ring_rx; 1015 struct sk_buff *skb; 1016 1017 if (unlikely(!v->va)) { 1018 wil_err(wil, "Rx IRQ while Rx not yet initialized\n"); 1019 return; 1020 } 1021 wil_dbg_txrx(wil, "rx_handle\n"); 1022 while ((*quota > 0) && (NULL != (skb = wil_vring_reap_rx(wil, v)))) { 1023 (*quota)--; 1024 1025 /* monitor is currently supported on main interface only */ 1026 if (wdev->iftype == NL80211_IFTYPE_MONITOR) { 1027 skb->dev = ndev; 1028 skb_reset_mac_header(skb); 1029 skb->ip_summed = CHECKSUM_UNNECESSARY; 1030 skb->pkt_type = PACKET_OTHERHOST; 1031 skb->protocol = htons(ETH_P_802_2); 1032 wil_netif_rx_any(skb, ndev); 1033 } else { 1034 wil_rx_reorder(wil, skb); 1035 } 1036 } 1037 wil_rx_refill(wil, v->size); 1038 } 1039 1040 static void wil_rx_buf_len_init(struct wil6210_priv *wil) 1041 { 1042 wil->rx_buf_len = rx_large_buf ? 1043 WIL_MAX_ETH_MTU : TXRX_BUF_LEN_DEFAULT - WIL_MAX_MPDU_OVERHEAD; 1044 if (mtu_max > wil->rx_buf_len) { 1045 /* do not allow RX buffers to be smaller than mtu_max, for 1046 * backward compatibility (mtu_max parameter was also used 1047 * to support receiving large packets) 1048 */ 1049 wil_info(wil, "Override RX buffer to mtu_max(%d)\n", mtu_max); 1050 wil->rx_buf_len = mtu_max; 1051 } 1052 } 1053 1054 static int wil_rx_init(struct wil6210_priv *wil, uint order) 1055 { 1056 struct wil_ring *vring = &wil->ring_rx; 1057 int rc; 1058 1059 wil_dbg_misc(wil, "rx_init\n"); 1060 1061 if (vring->va) { 1062 wil_err(wil, "Rx ring already allocated\n"); 1063 return -EINVAL; 1064 } 1065 1066 wil_rx_buf_len_init(wil); 1067 1068 vring->size = 1 << order; 1069 vring->is_rx = true; 1070 rc = wil_vring_alloc(wil, vring); 1071 if (rc) 1072 return rc; 1073 1074 rc = wmi_rx_chain_add(wil, vring); 1075 if (rc) 1076 goto err_free; 1077 1078 rc = wil_rx_refill(wil, vring->size); 1079 if (rc) 1080 goto err_free; 1081 1082 return 0; 1083 err_free: 1084 wil_vring_free(wil, vring); 1085 1086 return rc; 1087 } 1088 1089 static void wil_rx_fini(struct wil6210_priv *wil) 1090 { 1091 struct wil_ring *vring = &wil->ring_rx; 1092 1093 wil_dbg_misc(wil, "rx_fini\n"); 1094 1095 if (vring->va) 1096 wil_vring_free(wil, vring); 1097 } 1098 1099 static int wil_tx_desc_map(union wil_tx_desc *desc, dma_addr_t pa, 1100 u32 len, int vring_index) 1101 { 1102 struct vring_tx_desc *d = &desc->legacy; 1103 1104 wil_desc_addr_set(&d->dma.addr, pa); 1105 d->dma.ip_length = 0; 1106 /* 0..6: mac_length; 7:ip_version 0-IP6 1-IP4*/ 1107 d->dma.b11 = 0/*14 | BIT(7)*/; 1108 d->dma.error = 0; 1109 d->dma.status = 0; /* BIT(0) should be 0 for HW_OWNED */ 1110 d->dma.length = cpu_to_le16((u16)len); 1111 d->dma.d0 = (vring_index << DMA_CFG_DESC_TX_0_QID_POS); 1112 d->mac.d[0] = 0; 1113 d->mac.d[1] = 0; 1114 d->mac.d[2] = 0; 1115 d->mac.ucode_cmd = 0; 1116 /* translation type: 0 - bypass; 1 - 802.3; 2 - native wifi */ 1117 d->mac.d[2] = BIT(MAC_CFG_DESC_TX_2_SNAP_HDR_INSERTION_EN_POS) | 1118 (1 << MAC_CFG_DESC_TX_2_L2_TRANSLATION_TYPE_POS); 1119 1120 return 0; 1121 } 1122 1123 void wil_tx_data_init(struct wil_ring_tx_data *txdata) 1124 { 1125 spin_lock_bh(&txdata->lock); 1126 txdata->dot1x_open = false; 1127 txdata->enabled = 0; 1128 txdata->idle = 0; 1129 txdata->last_idle = 0; 1130 txdata->begin = 0; 1131 txdata->agg_wsize = 0; 1132 txdata->agg_timeout = 0; 1133 txdata->agg_amsdu = 0; 1134 txdata->addba_in_progress = false; 1135 txdata->mid = U8_MAX; 1136 spin_unlock_bh(&txdata->lock); 1137 } 1138 1139 static int wil_vring_init_tx(struct wil6210_vif *vif, int id, int size, 1140 int cid, int tid) 1141 { 1142 struct wil6210_priv *wil = vif_to_wil(vif); 1143 int rc; 1144 struct wmi_vring_cfg_cmd cmd = { 1145 .action = cpu_to_le32(WMI_VRING_CMD_ADD), 1146 .vring_cfg = { 1147 .tx_sw_ring = { 1148 .max_mpdu_size = 1149 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 1150 .ring_size = cpu_to_le16(size), 1151 }, 1152 .ringid = id, 1153 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 1154 .mac_ctrl = 0, 1155 .to_resolution = 0, 1156 .agg_max_wsize = 0, 1157 .schd_params = { 1158 .priority = cpu_to_le16(0), 1159 .timeslot_us = cpu_to_le16(0xfff), 1160 }, 1161 }, 1162 }; 1163 struct { 1164 struct wmi_cmd_hdr wmi; 1165 struct wmi_vring_cfg_done_event cmd; 1166 } __packed reply = { 1167 .cmd = {.status = WMI_FW_STATUS_FAILURE}, 1168 }; 1169 struct wil_ring *vring = &wil->ring_tx[id]; 1170 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id]; 1171 1172 if (cid >= WIL6210_RX_DESC_MAX_CID) { 1173 cmd.vring_cfg.cidxtid = CIDXTID_EXTENDED_CID_TID; 1174 cmd.vring_cfg.cid = cid; 1175 cmd.vring_cfg.tid = tid; 1176 } else { 1177 cmd.vring_cfg.cidxtid = mk_cidxtid(cid, tid); 1178 } 1179 1180 wil_dbg_misc(wil, "vring_init_tx: max_mpdu_size %d\n", 1181 cmd.vring_cfg.tx_sw_ring.max_mpdu_size); 1182 lockdep_assert_held(&wil->mutex); 1183 1184 if (vring->va) { 1185 wil_err(wil, "Tx ring [%d] already allocated\n", id); 1186 rc = -EINVAL; 1187 goto out; 1188 } 1189 1190 wil_tx_data_init(txdata); 1191 vring->is_rx = false; 1192 vring->size = size; 1193 rc = wil_vring_alloc(wil, vring); 1194 if (rc) 1195 goto out; 1196 1197 wil->ring2cid_tid[id][0] = cid; 1198 wil->ring2cid_tid[id][1] = tid; 1199 1200 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 1201 1202 if (!vif->privacy) 1203 txdata->dot1x_open = true; 1204 rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd), 1205 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 1206 WIL_WMI_CALL_GENERAL_TO_MS); 1207 if (rc) 1208 goto out_free; 1209 1210 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 1211 wil_err(wil, "Tx config failed, status 0x%02x\n", 1212 reply.cmd.status); 1213 rc = -EINVAL; 1214 goto out_free; 1215 } 1216 1217 spin_lock_bh(&txdata->lock); 1218 vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr); 1219 txdata->mid = vif->mid; 1220 txdata->enabled = 1; 1221 spin_unlock_bh(&txdata->lock); 1222 1223 if (txdata->dot1x_open && (agg_wsize >= 0)) 1224 wil_addba_tx_request(wil, id, agg_wsize); 1225 1226 return 0; 1227 out_free: 1228 spin_lock_bh(&txdata->lock); 1229 txdata->dot1x_open = false; 1230 txdata->enabled = 0; 1231 spin_unlock_bh(&txdata->lock); 1232 wil_vring_free(wil, vring); 1233 wil->ring2cid_tid[id][0] = wil->max_assoc_sta; 1234 wil->ring2cid_tid[id][1] = 0; 1235 1236 out: 1237 1238 return rc; 1239 } 1240 1241 static int wil_tx_vring_modify(struct wil6210_vif *vif, int ring_id, int cid, 1242 int tid) 1243 { 1244 struct wil6210_priv *wil = vif_to_wil(vif); 1245 int rc; 1246 struct wmi_vring_cfg_cmd cmd = { 1247 .action = cpu_to_le32(WMI_VRING_CMD_MODIFY), 1248 .vring_cfg = { 1249 .tx_sw_ring = { 1250 .max_mpdu_size = 1251 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 1252 .ring_size = 0, 1253 }, 1254 .ringid = ring_id, 1255 .cidxtid = mk_cidxtid(cid, tid), 1256 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 1257 .mac_ctrl = 0, 1258 .to_resolution = 0, 1259 .agg_max_wsize = 0, 1260 .schd_params = { 1261 .priority = cpu_to_le16(0), 1262 .timeslot_us = cpu_to_le16(0xfff), 1263 }, 1264 }, 1265 }; 1266 struct { 1267 struct wmi_cmd_hdr wmi; 1268 struct wmi_vring_cfg_done_event cmd; 1269 } __packed reply = { 1270 .cmd = {.status = WMI_FW_STATUS_FAILURE}, 1271 }; 1272 struct wil_ring *vring = &wil->ring_tx[ring_id]; 1273 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_id]; 1274 1275 wil_dbg_misc(wil, "vring_modify: ring %d cid %d tid %d\n", ring_id, 1276 cid, tid); 1277 lockdep_assert_held(&wil->mutex); 1278 1279 if (!vring->va) { 1280 wil_err(wil, "Tx ring [%d] not allocated\n", ring_id); 1281 return -EINVAL; 1282 } 1283 1284 if (wil->ring2cid_tid[ring_id][0] != cid || 1285 wil->ring2cid_tid[ring_id][1] != tid) { 1286 wil_err(wil, "ring info does not match cid=%u tid=%u\n", 1287 wil->ring2cid_tid[ring_id][0], 1288 wil->ring2cid_tid[ring_id][1]); 1289 } 1290 1291 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 1292 1293 rc = wmi_call(wil, WMI_VRING_CFG_CMDID, vif->mid, &cmd, sizeof(cmd), 1294 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 1295 WIL_WMI_CALL_GENERAL_TO_MS); 1296 if (rc) 1297 goto fail; 1298 1299 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 1300 wil_err(wil, "Tx modify failed, status 0x%02x\n", 1301 reply.cmd.status); 1302 rc = -EINVAL; 1303 goto fail; 1304 } 1305 1306 /* set BA aggregation window size to 0 to force a new BA with the 1307 * new AP 1308 */ 1309 txdata->agg_wsize = 0; 1310 if (txdata->dot1x_open && agg_wsize >= 0) 1311 wil_addba_tx_request(wil, ring_id, agg_wsize); 1312 1313 return 0; 1314 fail: 1315 spin_lock_bh(&txdata->lock); 1316 txdata->dot1x_open = false; 1317 txdata->enabled = 0; 1318 spin_unlock_bh(&txdata->lock); 1319 wil->ring2cid_tid[ring_id][0] = wil->max_assoc_sta; 1320 wil->ring2cid_tid[ring_id][1] = 0; 1321 return rc; 1322 } 1323 1324 int wil_vring_init_bcast(struct wil6210_vif *vif, int id, int size) 1325 { 1326 struct wil6210_priv *wil = vif_to_wil(vif); 1327 int rc; 1328 struct wmi_bcast_vring_cfg_cmd cmd = { 1329 .action = cpu_to_le32(WMI_VRING_CMD_ADD), 1330 .vring_cfg = { 1331 .tx_sw_ring = { 1332 .max_mpdu_size = 1333 cpu_to_le16(wil_mtu2macbuf(mtu_max)), 1334 .ring_size = cpu_to_le16(size), 1335 }, 1336 .ringid = id, 1337 .encap_trans_type = WMI_VRING_ENC_TYPE_802_3, 1338 }, 1339 }; 1340 struct { 1341 struct wmi_cmd_hdr wmi; 1342 struct wmi_vring_cfg_done_event cmd; 1343 } __packed reply = { 1344 .cmd = {.status = WMI_FW_STATUS_FAILURE}, 1345 }; 1346 struct wil_ring *vring = &wil->ring_tx[id]; 1347 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[id]; 1348 1349 wil_dbg_misc(wil, "vring_init_bcast: max_mpdu_size %d\n", 1350 cmd.vring_cfg.tx_sw_ring.max_mpdu_size); 1351 lockdep_assert_held(&wil->mutex); 1352 1353 if (vring->va) { 1354 wil_err(wil, "Tx ring [%d] already allocated\n", id); 1355 rc = -EINVAL; 1356 goto out; 1357 } 1358 1359 wil_tx_data_init(txdata); 1360 vring->is_rx = false; 1361 vring->size = size; 1362 rc = wil_vring_alloc(wil, vring); 1363 if (rc) 1364 goto out; 1365 1366 wil->ring2cid_tid[id][0] = wil->max_assoc_sta; /* CID */ 1367 wil->ring2cid_tid[id][1] = 0; /* TID */ 1368 1369 cmd.vring_cfg.tx_sw_ring.ring_mem_base = cpu_to_le64(vring->pa); 1370 1371 if (!vif->privacy) 1372 txdata->dot1x_open = true; 1373 rc = wmi_call(wil, WMI_BCAST_VRING_CFG_CMDID, vif->mid, 1374 &cmd, sizeof(cmd), 1375 WMI_VRING_CFG_DONE_EVENTID, &reply, sizeof(reply), 1376 WIL_WMI_CALL_GENERAL_TO_MS); 1377 if (rc) 1378 goto out_free; 1379 1380 if (reply.cmd.status != WMI_FW_STATUS_SUCCESS) { 1381 wil_err(wil, "Tx config failed, status 0x%02x\n", 1382 reply.cmd.status); 1383 rc = -EINVAL; 1384 goto out_free; 1385 } 1386 1387 spin_lock_bh(&txdata->lock); 1388 vring->hwtail = le32_to_cpu(reply.cmd.tx_vring_tail_ptr); 1389 txdata->mid = vif->mid; 1390 txdata->enabled = 1; 1391 spin_unlock_bh(&txdata->lock); 1392 1393 return 0; 1394 out_free: 1395 spin_lock_bh(&txdata->lock); 1396 txdata->enabled = 0; 1397 txdata->dot1x_open = false; 1398 spin_unlock_bh(&txdata->lock); 1399 wil_vring_free(wil, vring); 1400 out: 1401 1402 return rc; 1403 } 1404 1405 static struct wil_ring *wil_find_tx_ucast(struct wil6210_priv *wil, 1406 struct wil6210_vif *vif, 1407 struct sk_buff *skb) 1408 { 1409 int i, cid; 1410 const u8 *da = wil_skb_get_da(skb); 1411 int min_ring_id = wil_get_min_tx_ring_id(wil); 1412 1413 cid = wil_find_cid(wil, vif->mid, da); 1414 1415 if (cid < 0 || cid >= wil->max_assoc_sta) 1416 return NULL; 1417 1418 /* TODO: fix for multiple TID */ 1419 for (i = min_ring_id; i < ARRAY_SIZE(wil->ring2cid_tid); i++) { 1420 if (!wil->ring_tx_data[i].dot1x_open && 1421 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1422 continue; 1423 if (wil->ring2cid_tid[i][0] == cid) { 1424 struct wil_ring *v = &wil->ring_tx[i]; 1425 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i]; 1426 1427 wil_dbg_txrx(wil, "find_tx_ucast: (%pM) -> [%d]\n", 1428 da, i); 1429 if (v->va && txdata->enabled) { 1430 return v; 1431 } else { 1432 wil_dbg_txrx(wil, 1433 "find_tx_ucast: vring[%d] not valid\n", 1434 i); 1435 return NULL; 1436 } 1437 } 1438 } 1439 1440 return NULL; 1441 } 1442 1443 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, 1444 struct wil_ring *ring, struct sk_buff *skb); 1445 1446 static struct wil_ring *wil_find_tx_ring_sta(struct wil6210_priv *wil, 1447 struct wil6210_vif *vif, 1448 struct sk_buff *skb) 1449 { 1450 struct wil_ring *ring; 1451 int i; 1452 u8 cid; 1453 struct wil_ring_tx_data *txdata; 1454 int min_ring_id = wil_get_min_tx_ring_id(wil); 1455 1456 /* In the STA mode, it is expected to have only 1 VRING 1457 * for the AP we connected to. 1458 * find 1-st vring eligible for this skb and use it. 1459 */ 1460 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 1461 ring = &wil->ring_tx[i]; 1462 txdata = &wil->ring_tx_data[i]; 1463 if (!ring->va || !txdata->enabled || txdata->mid != vif->mid) 1464 continue; 1465 1466 cid = wil->ring2cid_tid[i][0]; 1467 if (cid >= wil->max_assoc_sta) /* skip BCAST */ 1468 continue; 1469 1470 if (!wil->ring_tx_data[i].dot1x_open && 1471 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1472 continue; 1473 1474 wil_dbg_txrx(wil, "Tx -> ring %d\n", i); 1475 1476 return ring; 1477 } 1478 1479 wil_dbg_txrx(wil, "Tx while no rings active?\n"); 1480 1481 return NULL; 1482 } 1483 1484 /* Use one of 2 strategies: 1485 * 1486 * 1. New (real broadcast): 1487 * use dedicated broadcast vring 1488 * 2. Old (pseudo-DMS): 1489 * Find 1-st vring and return it; 1490 * duplicate skb and send it to other active vrings; 1491 * in all cases override dest address to unicast peer's address 1492 * Use old strategy when new is not supported yet: 1493 * - for PBSS 1494 */ 1495 static struct wil_ring *wil_find_tx_bcast_1(struct wil6210_priv *wil, 1496 struct wil6210_vif *vif, 1497 struct sk_buff *skb) 1498 { 1499 struct wil_ring *v; 1500 struct wil_ring_tx_data *txdata; 1501 int i = vif->bcast_ring; 1502 1503 if (i < 0) 1504 return NULL; 1505 v = &wil->ring_tx[i]; 1506 txdata = &wil->ring_tx_data[i]; 1507 if (!v->va || !txdata->enabled) 1508 return NULL; 1509 if (!wil->ring_tx_data[i].dot1x_open && 1510 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1511 return NULL; 1512 1513 return v; 1514 } 1515 1516 /* apply multicast to unicast only for ARP and IP packets 1517 * (see NL80211_CMD_SET_MULTICAST_TO_UNICAST for more info) 1518 */ 1519 static bool wil_check_multicast_to_unicast(struct wil6210_priv *wil, 1520 struct sk_buff *skb) 1521 { 1522 const struct ethhdr *eth = (void *)skb->data; 1523 const struct vlan_ethhdr *ethvlan = (void *)skb->data; 1524 __be16 ethertype; 1525 1526 if (!wil->multicast_to_unicast) 1527 return false; 1528 1529 /* multicast to unicast conversion only for some payload */ 1530 ethertype = eth->h_proto; 1531 if (ethertype == htons(ETH_P_8021Q) && skb->len >= VLAN_ETH_HLEN) 1532 ethertype = ethvlan->h_vlan_encapsulated_proto; 1533 switch (ethertype) { 1534 case htons(ETH_P_ARP): 1535 case htons(ETH_P_IP): 1536 case htons(ETH_P_IPV6): 1537 break; 1538 default: 1539 return false; 1540 } 1541 1542 return true; 1543 } 1544 1545 static void wil_set_da_for_vring(struct wil6210_priv *wil, 1546 struct sk_buff *skb, int vring_index) 1547 { 1548 u8 *da = wil_skb_get_da(skb); 1549 int cid = wil->ring2cid_tid[vring_index][0]; 1550 1551 ether_addr_copy(da, wil->sta[cid].addr); 1552 } 1553 1554 static struct wil_ring *wil_find_tx_bcast_2(struct wil6210_priv *wil, 1555 struct wil6210_vif *vif, 1556 struct sk_buff *skb) 1557 { 1558 struct wil_ring *v, *v2; 1559 struct sk_buff *skb2; 1560 int i; 1561 u8 cid; 1562 const u8 *src = wil_skb_get_sa(skb); 1563 struct wil_ring_tx_data *txdata, *txdata2; 1564 int min_ring_id = wil_get_min_tx_ring_id(wil); 1565 1566 /* find 1-st vring eligible for data */ 1567 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 1568 v = &wil->ring_tx[i]; 1569 txdata = &wil->ring_tx_data[i]; 1570 if (!v->va || !txdata->enabled || txdata->mid != vif->mid) 1571 continue; 1572 1573 cid = wil->ring2cid_tid[i][0]; 1574 if (cid >= wil->max_assoc_sta) /* skip BCAST */ 1575 continue; 1576 if (!wil->ring_tx_data[i].dot1x_open && 1577 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1578 continue; 1579 1580 /* don't Tx back to source when re-routing Rx->Tx at the AP */ 1581 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN)) 1582 continue; 1583 1584 goto found; 1585 } 1586 1587 wil_dbg_txrx(wil, "Tx while no vrings active?\n"); 1588 1589 return NULL; 1590 1591 found: 1592 wil_dbg_txrx(wil, "BCAST -> ring %d\n", i); 1593 wil_set_da_for_vring(wil, skb, i); 1594 1595 /* find other active vrings and duplicate skb for each */ 1596 for (i++; i < WIL6210_MAX_TX_RINGS; i++) { 1597 v2 = &wil->ring_tx[i]; 1598 txdata2 = &wil->ring_tx_data[i]; 1599 if (!v2->va || txdata2->mid != vif->mid) 1600 continue; 1601 cid = wil->ring2cid_tid[i][0]; 1602 if (cid >= wil->max_assoc_sta) /* skip BCAST */ 1603 continue; 1604 if (!wil->ring_tx_data[i].dot1x_open && 1605 skb->protocol != cpu_to_be16(ETH_P_PAE)) 1606 continue; 1607 1608 if (0 == memcmp(wil->sta[cid].addr, src, ETH_ALEN)) 1609 continue; 1610 1611 skb2 = skb_copy(skb, GFP_ATOMIC); 1612 if (skb2) { 1613 wil_dbg_txrx(wil, "BCAST DUP -> ring %d\n", i); 1614 wil_set_da_for_vring(wil, skb2, i); 1615 wil_tx_ring(wil, vif, v2, skb2); 1616 /* successful call to wil_tx_ring takes skb2 ref */ 1617 dev_kfree_skb_any(skb2); 1618 } else { 1619 wil_err(wil, "skb_copy failed\n"); 1620 } 1621 } 1622 1623 return v; 1624 } 1625 1626 static inline 1627 void wil_tx_desc_set_nr_frags(struct vring_tx_desc *d, int nr_frags) 1628 { 1629 d->mac.d[2] |= (nr_frags << MAC_CFG_DESC_TX_2_NUM_OF_DESCRIPTORS_POS); 1630 } 1631 1632 /** 1633 * Sets the descriptor @d up for csum and/or TSO offloading. The corresponding 1634 * @skb is used to obtain the protocol and headers length. 1635 * @tso_desc_type is a descriptor type for TSO: 0 - a header, 1 - first data, 1636 * 2 - middle, 3 - last descriptor. 1637 */ 1638 1639 static void wil_tx_desc_offload_setup_tso(struct vring_tx_desc *d, 1640 struct sk_buff *skb, 1641 int tso_desc_type, bool is_ipv4, 1642 int tcp_hdr_len, int skb_net_hdr_len) 1643 { 1644 d->dma.b11 = ETH_HLEN; /* MAC header length */ 1645 d->dma.b11 |= is_ipv4 << DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS; 1646 1647 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS); 1648 /* L4 header len: TCP header length */ 1649 d->dma.d0 |= (tcp_hdr_len & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1650 1651 /* Setup TSO: bit and desc type */ 1652 d->dma.d0 |= (BIT(DMA_CFG_DESC_TX_0_TCP_SEG_EN_POS)) | 1653 (tso_desc_type << DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS); 1654 d->dma.d0 |= (is_ipv4 << DMA_CFG_DESC_TX_0_IPV4_CHECKSUM_EN_POS); 1655 1656 d->dma.ip_length = skb_net_hdr_len; 1657 /* Enable TCP/UDP checksum */ 1658 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS); 1659 /* Calculate pseudo-header */ 1660 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS); 1661 } 1662 1663 /** 1664 * Sets the descriptor @d up for csum. The corresponding 1665 * @skb is used to obtain the protocol and headers length. 1666 * Returns the protocol: 0 - not TCP, 1 - TCPv4, 2 - TCPv6. 1667 * Note, if d==NULL, the function only returns the protocol result. 1668 * 1669 * It is very similar to previous wil_tx_desc_offload_setup_tso. This 1670 * is "if unrolling" to optimize the critical path. 1671 */ 1672 1673 static int wil_tx_desc_offload_setup(struct vring_tx_desc *d, 1674 struct sk_buff *skb){ 1675 int protocol; 1676 1677 if (skb->ip_summed != CHECKSUM_PARTIAL) 1678 return 0; 1679 1680 d->dma.b11 = ETH_HLEN; /* MAC header length */ 1681 1682 switch (skb->protocol) { 1683 case cpu_to_be16(ETH_P_IP): 1684 protocol = ip_hdr(skb)->protocol; 1685 d->dma.b11 |= BIT(DMA_CFG_DESC_TX_OFFLOAD_CFG_L3T_IPV4_POS); 1686 break; 1687 case cpu_to_be16(ETH_P_IPV6): 1688 protocol = ipv6_hdr(skb)->nexthdr; 1689 break; 1690 default: 1691 return -EINVAL; 1692 } 1693 1694 switch (protocol) { 1695 case IPPROTO_TCP: 1696 d->dma.d0 |= (2 << DMA_CFG_DESC_TX_0_L4_TYPE_POS); 1697 /* L4 header len: TCP header length */ 1698 d->dma.d0 |= 1699 (tcp_hdrlen(skb) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1700 break; 1701 case IPPROTO_UDP: 1702 /* L4 header len: UDP header length */ 1703 d->dma.d0 |= 1704 (sizeof(struct udphdr) & DMA_CFG_DESC_TX_0_L4_LENGTH_MSK); 1705 break; 1706 default: 1707 return -EINVAL; 1708 } 1709 1710 d->dma.ip_length = skb_network_header_len(skb); 1711 /* Enable TCP/UDP checksum */ 1712 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_TCP_UDP_CHECKSUM_EN_POS); 1713 /* Calculate pseudo-header */ 1714 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_PSEUDO_HEADER_CALC_EN_POS); 1715 1716 return 0; 1717 } 1718 1719 static inline void wil_tx_last_desc(struct vring_tx_desc *d) 1720 { 1721 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS) | 1722 BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS) | 1723 BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS); 1724 } 1725 1726 static inline void wil_set_tx_desc_last_tso(volatile struct vring_tx_desc *d) 1727 { 1728 d->dma.d0 |= wil_tso_type_lst << 1729 DMA_CFG_DESC_TX_0_SEGMENT_BUF_DETAILS_POS; 1730 } 1731 1732 static int __wil_tx_vring_tso(struct wil6210_priv *wil, struct wil6210_vif *vif, 1733 struct wil_ring *vring, struct sk_buff *skb) 1734 { 1735 struct device *dev = wil_to_dev(wil); 1736 1737 /* point to descriptors in shared memory */ 1738 volatile struct vring_tx_desc *_desc = NULL, *_hdr_desc, 1739 *_first_desc = NULL; 1740 1741 /* pointers to shadow descriptors */ 1742 struct vring_tx_desc desc_mem, hdr_desc_mem, first_desc_mem, 1743 *d = &hdr_desc_mem, *hdr_desc = &hdr_desc_mem, 1744 *first_desc = &first_desc_mem; 1745 1746 /* pointer to shadow descriptors' context */ 1747 struct wil_ctx *hdr_ctx, *first_ctx = NULL; 1748 1749 int descs_used = 0; /* total number of used descriptors */ 1750 int sg_desc_cnt = 0; /* number of descriptors for current mss*/ 1751 1752 u32 swhead = vring->swhead; 1753 int used, avail = wil_ring_avail_tx(vring); 1754 int nr_frags = skb_shinfo(skb)->nr_frags; 1755 int min_desc_required = nr_frags + 1; 1756 int mss = skb_shinfo(skb)->gso_size; /* payload size w/o headers */ 1757 int f, len, hdrlen, headlen; 1758 int vring_index = vring - wil->ring_tx; 1759 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[vring_index]; 1760 uint i = swhead; 1761 dma_addr_t pa; 1762 const skb_frag_t *frag = NULL; 1763 int rem_data = mss; 1764 int lenmss; 1765 int hdr_compensation_need = true; 1766 int desc_tso_type = wil_tso_type_first; 1767 bool is_ipv4; 1768 int tcp_hdr_len; 1769 int skb_net_hdr_len; 1770 int gso_type; 1771 int rc = -EINVAL; 1772 1773 wil_dbg_txrx(wil, "tx_vring_tso: %d bytes to vring %d\n", skb->len, 1774 vring_index); 1775 1776 if (unlikely(!txdata->enabled)) 1777 return -EINVAL; 1778 1779 /* A typical page 4K is 3-4 payloads, we assume each fragment 1780 * is a full payload, that's how min_desc_required has been 1781 * calculated. In real we might need more or less descriptors, 1782 * this is the initial check only. 1783 */ 1784 if (unlikely(avail < min_desc_required)) { 1785 wil_err_ratelimited(wil, 1786 "TSO: Tx ring[%2d] full. No space for %d fragments\n", 1787 vring_index, min_desc_required); 1788 return -ENOMEM; 1789 } 1790 1791 /* Header Length = MAC header len + IP header len + TCP header len*/ 1792 hdrlen = ETH_HLEN + 1793 (int)skb_network_header_len(skb) + 1794 tcp_hdrlen(skb); 1795 1796 gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV6 | SKB_GSO_TCPV4); 1797 switch (gso_type) { 1798 case SKB_GSO_TCPV4: 1799 /* TCP v4, zero out the IP length and IPv4 checksum fields 1800 * as required by the offloading doc 1801 */ 1802 ip_hdr(skb)->tot_len = 0; 1803 ip_hdr(skb)->check = 0; 1804 is_ipv4 = true; 1805 break; 1806 case SKB_GSO_TCPV6: 1807 /* TCP v6, zero out the payload length */ 1808 ipv6_hdr(skb)->payload_len = 0; 1809 is_ipv4 = false; 1810 break; 1811 default: 1812 /* other than TCPv4 or TCPv6 types are not supported for TSO. 1813 * It is also illegal for both to be set simultaneously 1814 */ 1815 return -EINVAL; 1816 } 1817 1818 if (skb->ip_summed != CHECKSUM_PARTIAL) 1819 return -EINVAL; 1820 1821 /* tcp header length and skb network header length are fixed for all 1822 * packet's descriptors - read then once here 1823 */ 1824 tcp_hdr_len = tcp_hdrlen(skb); 1825 skb_net_hdr_len = skb_network_header_len(skb); 1826 1827 _hdr_desc = &vring->va[i].tx.legacy; 1828 1829 pa = dma_map_single(dev, skb->data, hdrlen, DMA_TO_DEVICE); 1830 if (unlikely(dma_mapping_error(dev, pa))) { 1831 wil_err(wil, "TSO: Skb head DMA map error\n"); 1832 goto err_exit; 1833 } 1834 1835 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)hdr_desc, pa, 1836 hdrlen, vring_index); 1837 wil_tx_desc_offload_setup_tso(hdr_desc, skb, wil_tso_type_hdr, is_ipv4, 1838 tcp_hdr_len, skb_net_hdr_len); 1839 wil_tx_last_desc(hdr_desc); 1840 1841 vring->ctx[i].mapped_as = wil_mapped_as_single; 1842 hdr_ctx = &vring->ctx[i]; 1843 1844 descs_used++; 1845 headlen = skb_headlen(skb) - hdrlen; 1846 1847 for (f = headlen ? -1 : 0; f < nr_frags; f++) { 1848 if (headlen) { 1849 len = headlen; 1850 wil_dbg_txrx(wil, "TSO: process skb head, len %u\n", 1851 len); 1852 } else { 1853 frag = &skb_shinfo(skb)->frags[f]; 1854 len = skb_frag_size(frag); 1855 wil_dbg_txrx(wil, "TSO: frag[%d]: len %u\n", f, len); 1856 } 1857 1858 while (len) { 1859 wil_dbg_txrx(wil, 1860 "TSO: len %d, rem_data %d, descs_used %d\n", 1861 len, rem_data, descs_used); 1862 1863 if (descs_used == avail) { 1864 wil_err_ratelimited(wil, "TSO: ring overflow\n"); 1865 rc = -ENOMEM; 1866 goto mem_error; 1867 } 1868 1869 lenmss = min_t(int, rem_data, len); 1870 i = (swhead + descs_used) % vring->size; 1871 wil_dbg_txrx(wil, "TSO: lenmss %d, i %d\n", lenmss, i); 1872 1873 if (!headlen) { 1874 pa = skb_frag_dma_map(dev, frag, 1875 skb_frag_size(frag) - len, 1876 lenmss, DMA_TO_DEVICE); 1877 vring->ctx[i].mapped_as = wil_mapped_as_page; 1878 } else { 1879 pa = dma_map_single(dev, 1880 skb->data + 1881 skb_headlen(skb) - headlen, 1882 lenmss, 1883 DMA_TO_DEVICE); 1884 vring->ctx[i].mapped_as = wil_mapped_as_single; 1885 headlen -= lenmss; 1886 } 1887 1888 if (unlikely(dma_mapping_error(dev, pa))) { 1889 wil_err(wil, "TSO: DMA map page error\n"); 1890 goto mem_error; 1891 } 1892 1893 _desc = &vring->va[i].tx.legacy; 1894 1895 if (!_first_desc) { 1896 _first_desc = _desc; 1897 first_ctx = &vring->ctx[i]; 1898 d = first_desc; 1899 } else { 1900 d = &desc_mem; 1901 } 1902 1903 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, 1904 pa, lenmss, vring_index); 1905 wil_tx_desc_offload_setup_tso(d, skb, desc_tso_type, 1906 is_ipv4, tcp_hdr_len, 1907 skb_net_hdr_len); 1908 1909 /* use tso_type_first only once */ 1910 desc_tso_type = wil_tso_type_mid; 1911 1912 descs_used++; /* desc used so far */ 1913 sg_desc_cnt++; /* desc used for this segment */ 1914 len -= lenmss; 1915 rem_data -= lenmss; 1916 1917 wil_dbg_txrx(wil, 1918 "TSO: len %d, rem_data %d, descs_used %d, sg_desc_cnt %d,\n", 1919 len, rem_data, descs_used, sg_desc_cnt); 1920 1921 /* Close the segment if reached mss size or last frag*/ 1922 if (rem_data == 0 || (f == nr_frags - 1 && len == 0)) { 1923 if (hdr_compensation_need) { 1924 /* first segment include hdr desc for 1925 * release 1926 */ 1927 hdr_ctx->nr_frags = sg_desc_cnt; 1928 wil_tx_desc_set_nr_frags(first_desc, 1929 sg_desc_cnt + 1930 1); 1931 hdr_compensation_need = false; 1932 } else { 1933 wil_tx_desc_set_nr_frags(first_desc, 1934 sg_desc_cnt); 1935 } 1936 first_ctx->nr_frags = sg_desc_cnt - 1; 1937 1938 wil_tx_last_desc(d); 1939 1940 /* first descriptor may also be the last 1941 * for this mss - make sure not to copy 1942 * it twice 1943 */ 1944 if (first_desc != d) 1945 *_first_desc = *first_desc; 1946 1947 /*last descriptor will be copied at the end 1948 * of this TS processing 1949 */ 1950 if (f < nr_frags - 1 || len > 0) 1951 *_desc = *d; 1952 1953 rem_data = mss; 1954 _first_desc = NULL; 1955 sg_desc_cnt = 0; 1956 } else if (first_desc != d) /* update mid descriptor */ 1957 *_desc = *d; 1958 } 1959 } 1960 1961 if (!_desc) 1962 goto mem_error; 1963 1964 /* first descriptor may also be the last. 1965 * in this case d pointer is invalid 1966 */ 1967 if (_first_desc == _desc) 1968 d = first_desc; 1969 1970 /* Last data descriptor */ 1971 wil_set_tx_desc_last_tso(d); 1972 *_desc = *d; 1973 1974 /* Fill the total number of descriptors in first desc (hdr)*/ 1975 wil_tx_desc_set_nr_frags(hdr_desc, descs_used); 1976 *_hdr_desc = *hdr_desc; 1977 1978 /* hold reference to skb 1979 * to prevent skb release before accounting 1980 * in case of immediate "tx done" 1981 */ 1982 vring->ctx[i].skb = skb_get(skb); 1983 1984 /* performance monitoring */ 1985 used = wil_ring_used_tx(vring); 1986 if (wil_val_in_range(wil->ring_idle_trsh, 1987 used, used + descs_used)) { 1988 txdata->idle += get_cycles() - txdata->last_idle; 1989 wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 1990 vring_index, used, used + descs_used); 1991 } 1992 1993 /* Make sure to advance the head only after descriptor update is done. 1994 * This will prevent a race condition where the completion thread 1995 * will see the DU bit set from previous run and will handle the 1996 * skb before it was completed. 1997 */ 1998 wmb(); 1999 2000 /* advance swhead */ 2001 wil_ring_advance_head(vring, descs_used); 2002 wil_dbg_txrx(wil, "TSO: Tx swhead %d -> %d\n", swhead, vring->swhead); 2003 2004 /* make sure all writes to descriptors (shared memory) are done before 2005 * committing them to HW 2006 */ 2007 wmb(); 2008 2009 if (wil->tx_latency) 2010 *(ktime_t *)&skb->cb = ktime_get(); 2011 else 2012 memset(skb->cb, 0, sizeof(ktime_t)); 2013 2014 wil_w(wil, vring->hwtail, vring->swhead); 2015 return 0; 2016 2017 mem_error: 2018 while (descs_used > 0) { 2019 struct wil_ctx *ctx; 2020 2021 i = (swhead + descs_used - 1) % vring->size; 2022 d = (struct vring_tx_desc *)&vring->va[i].tx.legacy; 2023 _desc = &vring->va[i].tx.legacy; 2024 *d = *_desc; 2025 _desc->dma.status = TX_DMA_STATUS_DU; 2026 ctx = &vring->ctx[i]; 2027 wil_txdesc_unmap(dev, (union wil_tx_desc *)d, ctx); 2028 memset(ctx, 0, sizeof(*ctx)); 2029 descs_used--; 2030 } 2031 err_exit: 2032 return rc; 2033 } 2034 2035 static int __wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, 2036 struct wil_ring *ring, struct sk_buff *skb) 2037 { 2038 struct device *dev = wil_to_dev(wil); 2039 struct vring_tx_desc dd, *d = ⅆ 2040 volatile struct vring_tx_desc *_d; 2041 u32 swhead = ring->swhead; 2042 int avail = wil_ring_avail_tx(ring); 2043 int nr_frags = skb_shinfo(skb)->nr_frags; 2044 uint f = 0; 2045 int ring_index = ring - wil->ring_tx; 2046 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index]; 2047 uint i = swhead; 2048 dma_addr_t pa; 2049 int used; 2050 bool mcast = (ring_index == vif->bcast_ring); 2051 uint len = skb_headlen(skb); 2052 2053 wil_dbg_txrx(wil, "tx_ring: %d bytes to ring %d, nr_frags %d\n", 2054 skb->len, ring_index, nr_frags); 2055 2056 if (unlikely(!txdata->enabled)) 2057 return -EINVAL; 2058 2059 if (unlikely(avail < 1 + nr_frags)) { 2060 wil_err_ratelimited(wil, 2061 "Tx ring[%2d] full. No space for %d fragments\n", 2062 ring_index, 1 + nr_frags); 2063 return -ENOMEM; 2064 } 2065 _d = &ring->va[i].tx.legacy; 2066 2067 pa = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 2068 2069 wil_dbg_txrx(wil, "Tx[%2d] skb %d bytes 0x%p -> %pad\n", ring_index, 2070 skb_headlen(skb), skb->data, &pa); 2071 wil_hex_dump_txrx("Tx ", DUMP_PREFIX_OFFSET, 16, 1, 2072 skb->data, skb_headlen(skb), false); 2073 2074 if (unlikely(dma_mapping_error(dev, pa))) 2075 return -EINVAL; 2076 ring->ctx[i].mapped_as = wil_mapped_as_single; 2077 /* 1-st segment */ 2078 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, pa, len, 2079 ring_index); 2080 if (unlikely(mcast)) { 2081 d->mac.d[0] |= BIT(MAC_CFG_DESC_TX_0_MCS_EN_POS); /* MCS 0 */ 2082 if (unlikely(len > WIL_BCAST_MCS0_LIMIT)) /* set MCS 1 */ 2083 d->mac.d[0] |= (1 << MAC_CFG_DESC_TX_0_MCS_INDEX_POS); 2084 } 2085 /* Process TCP/UDP checksum offloading */ 2086 if (unlikely(wil_tx_desc_offload_setup(d, skb))) { 2087 wil_err(wil, "Tx[%2d] Failed to set cksum, drop packet\n", 2088 ring_index); 2089 goto dma_error; 2090 } 2091 2092 ring->ctx[i].nr_frags = nr_frags; 2093 wil_tx_desc_set_nr_frags(d, nr_frags + 1); 2094 2095 /* middle segments */ 2096 for (; f < nr_frags; f++) { 2097 const skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; 2098 int len = skb_frag_size(frag); 2099 2100 *_d = *d; 2101 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i); 2102 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 2103 (const void *)d, sizeof(*d), false); 2104 i = (swhead + f + 1) % ring->size; 2105 _d = &ring->va[i].tx.legacy; 2106 pa = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 2107 DMA_TO_DEVICE); 2108 if (unlikely(dma_mapping_error(dev, pa))) { 2109 wil_err(wil, "Tx[%2d] failed to map fragment\n", 2110 ring_index); 2111 goto dma_error; 2112 } 2113 ring->ctx[i].mapped_as = wil_mapped_as_page; 2114 wil->txrx_ops.tx_desc_map((union wil_tx_desc *)d, 2115 pa, len, ring_index); 2116 /* no need to check return code - 2117 * if it succeeded for 1-st descriptor, 2118 * it will succeed here too 2119 */ 2120 wil_tx_desc_offload_setup(d, skb); 2121 } 2122 /* for the last seg only */ 2123 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_EOP_POS); 2124 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_MARK_WB_POS); 2125 d->dma.d0 |= BIT(DMA_CFG_DESC_TX_0_CMD_DMA_IT_POS); 2126 *_d = *d; 2127 wil_dbg_txrx(wil, "Tx[%2d] desc[%4d]\n", ring_index, i); 2128 wil_hex_dump_txrx("TxD ", DUMP_PREFIX_NONE, 32, 4, 2129 (const void *)d, sizeof(*d), false); 2130 2131 /* hold reference to skb 2132 * to prevent skb release before accounting 2133 * in case of immediate "tx done" 2134 */ 2135 ring->ctx[i].skb = skb_get(skb); 2136 2137 /* performance monitoring */ 2138 used = wil_ring_used_tx(ring); 2139 if (wil_val_in_range(wil->ring_idle_trsh, 2140 used, used + nr_frags + 1)) { 2141 txdata->idle += get_cycles() - txdata->last_idle; 2142 wil_dbg_txrx(wil, "Ring[%2d] not idle %d -> %d\n", 2143 ring_index, used, used + nr_frags + 1); 2144 } 2145 2146 /* Make sure to advance the head only after descriptor update is done. 2147 * This will prevent a race condition where the completion thread 2148 * will see the DU bit set from previous run and will handle the 2149 * skb before it was completed. 2150 */ 2151 wmb(); 2152 2153 /* advance swhead */ 2154 wil_ring_advance_head(ring, nr_frags + 1); 2155 wil_dbg_txrx(wil, "Tx[%2d] swhead %d -> %d\n", ring_index, swhead, 2156 ring->swhead); 2157 trace_wil6210_tx(ring_index, swhead, skb->len, nr_frags); 2158 2159 /* make sure all writes to descriptors (shared memory) are done before 2160 * committing them to HW 2161 */ 2162 wmb(); 2163 2164 if (wil->tx_latency) 2165 *(ktime_t *)&skb->cb = ktime_get(); 2166 else 2167 memset(skb->cb, 0, sizeof(ktime_t)); 2168 2169 wil_w(wil, ring->hwtail, ring->swhead); 2170 2171 return 0; 2172 dma_error: 2173 /* unmap what we have mapped */ 2174 nr_frags = f + 1; /* frags mapped + one for skb head */ 2175 for (f = 0; f < nr_frags; f++) { 2176 struct wil_ctx *ctx; 2177 2178 i = (swhead + f) % ring->size; 2179 ctx = &ring->ctx[i]; 2180 _d = &ring->va[i].tx.legacy; 2181 *d = *_d; 2182 _d->dma.status = TX_DMA_STATUS_DU; 2183 wil->txrx_ops.tx_desc_unmap(dev, 2184 (union wil_tx_desc *)d, 2185 ctx); 2186 2187 memset(ctx, 0, sizeof(*ctx)); 2188 } 2189 2190 return -EINVAL; 2191 } 2192 2193 static int wil_tx_ring(struct wil6210_priv *wil, struct wil6210_vif *vif, 2194 struct wil_ring *ring, struct sk_buff *skb) 2195 { 2196 int ring_index = ring - wil->ring_tx; 2197 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ring_index]; 2198 int rc; 2199 2200 spin_lock(&txdata->lock); 2201 2202 if (test_bit(wil_status_suspending, wil->status) || 2203 test_bit(wil_status_suspended, wil->status) || 2204 test_bit(wil_status_resuming, wil->status)) { 2205 wil_dbg_txrx(wil, 2206 "suspend/resume in progress. drop packet\n"); 2207 spin_unlock(&txdata->lock); 2208 return -EINVAL; 2209 } 2210 2211 rc = (skb_is_gso(skb) ? wil->txrx_ops.tx_ring_tso : __wil_tx_ring) 2212 (wil, vif, ring, skb); 2213 2214 spin_unlock(&txdata->lock); 2215 2216 return rc; 2217 } 2218 2219 /** 2220 * Check status of tx vrings and stop/wake net queues if needed 2221 * It will start/stop net queues of a specific VIF net_device. 2222 * 2223 * This function does one of two checks: 2224 * In case check_stop is true, will check if net queues need to be stopped. If 2225 * the conditions for stopping are met, netif_tx_stop_all_queues() is called. 2226 * In case check_stop is false, will check if net queues need to be waked. If 2227 * the conditions for waking are met, netif_tx_wake_all_queues() is called. 2228 * vring is the vring which is currently being modified by either adding 2229 * descriptors (tx) into it or removing descriptors (tx complete) from it. Can 2230 * be null when irrelevant (e.g. connect/disconnect events). 2231 * 2232 * The implementation is to stop net queues if modified vring has low 2233 * descriptor availability. Wake if all vrings are not in low descriptor 2234 * availability and modified vring has high descriptor availability. 2235 */ 2236 static inline void __wil_update_net_queues(struct wil6210_priv *wil, 2237 struct wil6210_vif *vif, 2238 struct wil_ring *ring, 2239 bool check_stop) 2240 { 2241 int i; 2242 int min_ring_id = wil_get_min_tx_ring_id(wil); 2243 2244 if (unlikely(!vif)) 2245 return; 2246 2247 if (ring) 2248 wil_dbg_txrx(wil, "vring %d, mid %d, check_stop=%d, stopped=%d", 2249 (int)(ring - wil->ring_tx), vif->mid, check_stop, 2250 vif->net_queue_stopped); 2251 else 2252 wil_dbg_txrx(wil, "check_stop=%d, mid=%d, stopped=%d", 2253 check_stop, vif->mid, vif->net_queue_stopped); 2254 2255 if (ring && drop_if_ring_full) 2256 /* no need to stop/wake net queues */ 2257 return; 2258 2259 if (check_stop == vif->net_queue_stopped) 2260 /* net queues already in desired state */ 2261 return; 2262 2263 if (check_stop) { 2264 if (!ring || unlikely(wil_ring_avail_low(ring))) { 2265 /* not enough room in the vring */ 2266 netif_tx_stop_all_queues(vif_to_ndev(vif)); 2267 vif->net_queue_stopped = true; 2268 wil_dbg_txrx(wil, "netif_tx_stop called\n"); 2269 } 2270 return; 2271 } 2272 2273 /* Do not wake the queues in suspend flow */ 2274 if (test_bit(wil_status_suspending, wil->status) || 2275 test_bit(wil_status_suspended, wil->status)) 2276 return; 2277 2278 /* check wake */ 2279 for (i = min_ring_id; i < WIL6210_MAX_TX_RINGS; i++) { 2280 struct wil_ring *cur_ring = &wil->ring_tx[i]; 2281 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[i]; 2282 2283 if (txdata->mid != vif->mid || !cur_ring->va || 2284 !txdata->enabled || cur_ring == ring) 2285 continue; 2286 2287 if (wil_ring_avail_low(cur_ring)) { 2288 wil_dbg_txrx(wil, "ring %d full, can't wake\n", 2289 (int)(cur_ring - wil->ring_tx)); 2290 return; 2291 } 2292 } 2293 2294 if (!ring || wil_ring_avail_high(ring)) { 2295 /* enough room in the ring */ 2296 wil_dbg_txrx(wil, "calling netif_tx_wake\n"); 2297 netif_tx_wake_all_queues(vif_to_ndev(vif)); 2298 vif->net_queue_stopped = false; 2299 } 2300 } 2301 2302 void wil_update_net_queues(struct wil6210_priv *wil, struct wil6210_vif *vif, 2303 struct wil_ring *ring, bool check_stop) 2304 { 2305 spin_lock(&wil->net_queue_lock); 2306 __wil_update_net_queues(wil, vif, ring, check_stop); 2307 spin_unlock(&wil->net_queue_lock); 2308 } 2309 2310 void wil_update_net_queues_bh(struct wil6210_priv *wil, struct wil6210_vif *vif, 2311 struct wil_ring *ring, bool check_stop) 2312 { 2313 spin_lock_bh(&wil->net_queue_lock); 2314 __wil_update_net_queues(wil, vif, ring, check_stop); 2315 spin_unlock_bh(&wil->net_queue_lock); 2316 } 2317 2318 netdev_tx_t wil_start_xmit(struct sk_buff *skb, struct net_device *ndev) 2319 { 2320 struct wil6210_vif *vif = ndev_to_vif(ndev); 2321 struct wil6210_priv *wil = vif_to_wil(vif); 2322 const u8 *da = wil_skb_get_da(skb); 2323 bool bcast = is_multicast_ether_addr(da); 2324 struct wil_ring *ring; 2325 static bool pr_once_fw; 2326 int rc; 2327 2328 wil_dbg_txrx(wil, "start_xmit\n"); 2329 if (unlikely(!test_bit(wil_status_fwready, wil->status))) { 2330 if (!pr_once_fw) { 2331 wil_err(wil, "FW not ready\n"); 2332 pr_once_fw = true; 2333 } 2334 goto drop; 2335 } 2336 if (unlikely(!test_bit(wil_vif_fwconnected, vif->status))) { 2337 wil_dbg_ratelimited(wil, 2338 "VIF not connected, packet dropped\n"); 2339 goto drop; 2340 } 2341 if (unlikely(vif->wdev.iftype == NL80211_IFTYPE_MONITOR)) { 2342 wil_err(wil, "Xmit in monitor mode not supported\n"); 2343 goto drop; 2344 } 2345 pr_once_fw = false; 2346 2347 /* find vring */ 2348 if (vif->wdev.iftype == NL80211_IFTYPE_STATION && !vif->pbss) { 2349 /* in STA mode (ESS), all to same VRING (to AP) */ 2350 ring = wil_find_tx_ring_sta(wil, vif, skb); 2351 } else if (bcast) { 2352 if (vif->pbss || wil_check_multicast_to_unicast(wil, skb)) 2353 /* in pbss, no bcast VRING - duplicate skb in 2354 * all stations VRINGs 2355 */ 2356 ring = wil_find_tx_bcast_2(wil, vif, skb); 2357 else if (vif->wdev.iftype == NL80211_IFTYPE_AP) 2358 /* AP has a dedicated bcast VRING */ 2359 ring = wil_find_tx_bcast_1(wil, vif, skb); 2360 else 2361 /* unexpected combination, fallback to duplicating 2362 * the skb in all stations VRINGs 2363 */ 2364 ring = wil_find_tx_bcast_2(wil, vif, skb); 2365 } else { 2366 /* unicast, find specific VRING by dest. address */ 2367 ring = wil_find_tx_ucast(wil, vif, skb); 2368 } 2369 if (unlikely(!ring)) { 2370 wil_dbg_txrx(wil, "No Tx RING found for %pM\n", da); 2371 goto drop; 2372 } 2373 /* set up vring entry */ 2374 rc = wil_tx_ring(wil, vif, ring, skb); 2375 2376 switch (rc) { 2377 case 0: 2378 /* shall we stop net queues? */ 2379 wil_update_net_queues_bh(wil, vif, ring, true); 2380 /* statistics will be updated on the tx_complete */ 2381 dev_kfree_skb_any(skb); 2382 return NETDEV_TX_OK; 2383 case -ENOMEM: 2384 if (drop_if_ring_full) 2385 goto drop; 2386 return NETDEV_TX_BUSY; 2387 default: 2388 break; /* goto drop; */ 2389 } 2390 drop: 2391 ndev->stats.tx_dropped++; 2392 dev_kfree_skb_any(skb); 2393 2394 return NET_XMIT_DROP; 2395 } 2396 2397 void wil_tx_latency_calc(struct wil6210_priv *wil, struct sk_buff *skb, 2398 struct wil_sta_info *sta) 2399 { 2400 int skb_time_us; 2401 int bin; 2402 2403 if (!wil->tx_latency) 2404 return; 2405 2406 if (ktime_to_ms(*(ktime_t *)&skb->cb) == 0) 2407 return; 2408 2409 skb_time_us = ktime_us_delta(ktime_get(), *(ktime_t *)&skb->cb); 2410 bin = skb_time_us / wil->tx_latency_res; 2411 bin = min_t(int, bin, WIL_NUM_LATENCY_BINS - 1); 2412 2413 wil_dbg_txrx(wil, "skb time %dus => bin %d\n", skb_time_us, bin); 2414 sta->tx_latency_bins[bin]++; 2415 sta->stats.tx_latency_total_us += skb_time_us; 2416 if (skb_time_us < sta->stats.tx_latency_min_us) 2417 sta->stats.tx_latency_min_us = skb_time_us; 2418 if (skb_time_us > sta->stats.tx_latency_max_us) 2419 sta->stats.tx_latency_max_us = skb_time_us; 2420 } 2421 2422 /** 2423 * Clean up transmitted skb's from the Tx VRING 2424 * 2425 * Return number of descriptors cleared 2426 * 2427 * Safe to call from IRQ 2428 */ 2429 int wil_tx_complete(struct wil6210_vif *vif, int ringid) 2430 { 2431 struct wil6210_priv *wil = vif_to_wil(vif); 2432 struct net_device *ndev = vif_to_ndev(vif); 2433 struct device *dev = wil_to_dev(wil); 2434 struct wil_ring *vring = &wil->ring_tx[ringid]; 2435 struct wil_ring_tx_data *txdata = &wil->ring_tx_data[ringid]; 2436 int done = 0; 2437 int cid = wil->ring2cid_tid[ringid][0]; 2438 struct wil_net_stats *stats = NULL; 2439 volatile struct vring_tx_desc *_d; 2440 int used_before_complete; 2441 int used_new; 2442 2443 if (unlikely(!vring->va)) { 2444 wil_err(wil, "Tx irq[%d]: vring not initialized\n", ringid); 2445 return 0; 2446 } 2447 2448 if (unlikely(!txdata->enabled)) { 2449 wil_info(wil, "Tx irq[%d]: vring disabled\n", ringid); 2450 return 0; 2451 } 2452 2453 wil_dbg_txrx(wil, "tx_complete: (%d)\n", ringid); 2454 2455 used_before_complete = wil_ring_used_tx(vring); 2456 2457 if (cid < wil->max_assoc_sta) 2458 stats = &wil->sta[cid].stats; 2459 2460 while (!wil_ring_is_empty(vring)) { 2461 int new_swtail; 2462 struct wil_ctx *ctx = &vring->ctx[vring->swtail]; 2463 /** 2464 * For the fragmented skb, HW will set DU bit only for the 2465 * last fragment. look for it. 2466 * In TSO the first DU will include hdr desc 2467 */ 2468 int lf = (vring->swtail + ctx->nr_frags) % vring->size; 2469 /* TODO: check we are not past head */ 2470 2471 _d = &vring->va[lf].tx.legacy; 2472 if (unlikely(!(_d->dma.status & TX_DMA_STATUS_DU))) 2473 break; 2474 2475 new_swtail = (lf + 1) % vring->size; 2476 while (vring->swtail != new_swtail) { 2477 struct vring_tx_desc dd, *d = ⅆ 2478 u16 dmalen; 2479 struct sk_buff *skb; 2480 2481 ctx = &vring->ctx[vring->swtail]; 2482 skb = ctx->skb; 2483 _d = &vring->va[vring->swtail].tx.legacy; 2484 2485 *d = *_d; 2486 2487 dmalen = le16_to_cpu(d->dma.length); 2488 trace_wil6210_tx_done(ringid, vring->swtail, dmalen, 2489 d->dma.error); 2490 wil_dbg_txrx(wil, 2491 "TxC[%2d][%3d] : %d bytes, status 0x%02x err 0x%02x\n", 2492 ringid, vring->swtail, dmalen, 2493 d->dma.status, d->dma.error); 2494 wil_hex_dump_txrx("TxCD ", DUMP_PREFIX_NONE, 32, 4, 2495 (const void *)d, sizeof(*d), false); 2496 2497 wil->txrx_ops.tx_desc_unmap(dev, 2498 (union wil_tx_desc *)d, 2499 ctx); 2500 2501 if (skb) { 2502 if (likely(d->dma.error == 0)) { 2503 ndev->stats.tx_packets++; 2504 ndev->stats.tx_bytes += skb->len; 2505 if (stats) { 2506 stats->tx_packets++; 2507 stats->tx_bytes += skb->len; 2508 2509 wil_tx_latency_calc(wil, skb, 2510 &wil->sta[cid]); 2511 } 2512 } else { 2513 ndev->stats.tx_errors++; 2514 if (stats) 2515 stats->tx_errors++; 2516 } 2517 2518 if (skb->protocol == cpu_to_be16(ETH_P_PAE)) 2519 wil_tx_complete_handle_eapol(vif, skb); 2520 2521 wil_consume_skb(skb, d->dma.error == 0); 2522 } 2523 memset(ctx, 0, sizeof(*ctx)); 2524 /* Make sure the ctx is zeroed before updating the tail 2525 * to prevent a case where wil_tx_ring will see 2526 * this descriptor as used and handle it before ctx zero 2527 * is completed. 2528 */ 2529 wmb(); 2530 /* There is no need to touch HW descriptor: 2531 * - ststus bit TX_DMA_STATUS_DU is set by design, 2532 * so hardware will not try to process this desc., 2533 * - rest of descriptor will be initialized on Tx. 2534 */ 2535 vring->swtail = wil_ring_next_tail(vring); 2536 done++; 2537 } 2538 } 2539 2540 /* performance monitoring */ 2541 used_new = wil_ring_used_tx(vring); 2542 if (wil_val_in_range(wil->ring_idle_trsh, 2543 used_new, used_before_complete)) { 2544 wil_dbg_txrx(wil, "Ring[%2d] idle %d -> %d\n", 2545 ringid, used_before_complete, used_new); 2546 txdata->last_idle = get_cycles(); 2547 } 2548 2549 /* shall we wake net queues? */ 2550 if (done) 2551 wil_update_net_queues(wil, vif, vring, false); 2552 2553 return done; 2554 } 2555 2556 static inline int wil_tx_init(struct wil6210_priv *wil) 2557 { 2558 return 0; 2559 } 2560 2561 static inline void wil_tx_fini(struct wil6210_priv *wil) {} 2562 2563 static void wil_get_reorder_params(struct wil6210_priv *wil, 2564 struct sk_buff *skb, int *tid, int *cid, 2565 int *mid, u16 *seq, int *mcast, int *retry) 2566 { 2567 struct vring_rx_desc *d = wil_skb_rxdesc(skb); 2568 2569 *tid = wil_rxdesc_tid(d); 2570 *cid = wil_skb_get_cid(skb); 2571 *mid = wil_rxdesc_mid(d); 2572 *seq = wil_rxdesc_seq(d); 2573 *mcast = wil_rxdesc_mcast(d); 2574 *retry = wil_rxdesc_retry(d); 2575 } 2576 2577 void wil_init_txrx_ops_legacy_dma(struct wil6210_priv *wil) 2578 { 2579 wil->txrx_ops.configure_interrupt_moderation = 2580 wil_configure_interrupt_moderation; 2581 /* TX ops */ 2582 wil->txrx_ops.tx_desc_map = wil_tx_desc_map; 2583 wil->txrx_ops.tx_desc_unmap = wil_txdesc_unmap; 2584 wil->txrx_ops.tx_ring_tso = __wil_tx_vring_tso; 2585 wil->txrx_ops.ring_init_tx = wil_vring_init_tx; 2586 wil->txrx_ops.ring_fini_tx = wil_vring_free; 2587 wil->txrx_ops.ring_init_bcast = wil_vring_init_bcast; 2588 wil->txrx_ops.tx_init = wil_tx_init; 2589 wil->txrx_ops.tx_fini = wil_tx_fini; 2590 wil->txrx_ops.tx_ring_modify = wil_tx_vring_modify; 2591 /* RX ops */ 2592 wil->txrx_ops.rx_init = wil_rx_init; 2593 wil->txrx_ops.wmi_addba_rx_resp = wmi_addba_rx_resp; 2594 wil->txrx_ops.get_reorder_params = wil_get_reorder_params; 2595 wil->txrx_ops.get_netif_rx_params = 2596 wil_get_netif_rx_params; 2597 wil->txrx_ops.rx_crypto_check = wil_rx_crypto_check; 2598 wil->txrx_ops.rx_error_check = wil_rx_error_check; 2599 wil->txrx_ops.is_rx_idle = wil_is_rx_idle; 2600 wil->txrx_ops.rx_fini = wil_rx_fini; 2601 } 2602