1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2015 Jakub Kicinski <kubakici@wp.pl> 4 */ 5 6 #include "mt7601u.h" 7 #include "dma.h" 8 #include "usb.h" 9 #include "trace.h" 10 11 static int mt7601u_submit_rx_buf(struct mt7601u_dev *dev, 12 struct mt7601u_dma_buf_rx *e, gfp_t gfp); 13 14 static unsigned int ieee80211_get_hdrlen_from_buf(const u8 *data, unsigned len) 15 { 16 const struct ieee80211_hdr *hdr = (const struct ieee80211_hdr *)data; 17 unsigned int hdrlen; 18 19 if (unlikely(len < 10)) 20 return 0; 21 hdrlen = ieee80211_hdrlen(hdr->frame_control); 22 if (unlikely(hdrlen > len)) 23 return 0; 24 return hdrlen; 25 } 26 27 static struct sk_buff * 28 mt7601u_rx_skb_from_seg(struct mt7601u_dev *dev, struct mt7601u_rxwi *rxwi, 29 void *data, u32 seg_len, u32 truesize, struct page *p) 30 { 31 struct sk_buff *skb; 32 u32 true_len, hdr_len = 0, copy, frag; 33 34 skb = alloc_skb(p ? 128 : seg_len, GFP_ATOMIC); 35 if (!skb) 36 return NULL; 37 38 true_len = mt76_mac_process_rx(dev, skb, data, rxwi); 39 if (!true_len || true_len > seg_len) 40 goto bad_frame; 41 42 hdr_len = ieee80211_get_hdrlen_from_buf(data, true_len); 43 if (!hdr_len) 44 goto bad_frame; 45 46 if (rxwi->rxinfo & cpu_to_le32(MT_RXINFO_L2PAD)) { 47 skb_put_data(skb, data, hdr_len); 48 49 data += hdr_len + 2; 50 true_len -= hdr_len; 51 hdr_len = 0; 52 } 53 54 /* If not doing paged RX allocated skb will always have enough space */ 55 copy = (true_len <= skb_tailroom(skb)) ? true_len : hdr_len + 8; 56 frag = true_len - copy; 57 58 skb_put_data(skb, data, copy); 59 data += copy; 60 61 if (frag) { 62 skb_add_rx_frag(skb, 0, p, data - page_address(p), 63 frag, truesize); 64 get_page(p); 65 } 66 67 return skb; 68 69 bad_frame: 70 dev_err_ratelimited(dev->dev, "Error: incorrect frame len:%u hdr:%u\n", 71 true_len, hdr_len); 72 dev_kfree_skb(skb); 73 return NULL; 74 } 75 76 static void mt7601u_rx_process_seg(struct mt7601u_dev *dev, u8 *data, 77 u32 seg_len, struct page *p, 78 struct list_head *list) 79 { 80 struct sk_buff *skb; 81 struct mt7601u_rxwi *rxwi; 82 u32 fce_info, truesize = seg_len; 83 84 /* DMA_INFO field at the beginning of the segment contains only some of 85 * the information, we need to read the FCE descriptor from the end. 86 */ 87 fce_info = get_unaligned_le32(data + seg_len - MT_FCE_INFO_LEN); 88 seg_len -= MT_FCE_INFO_LEN; 89 90 data += MT_DMA_HDR_LEN; 91 seg_len -= MT_DMA_HDR_LEN; 92 93 rxwi = (struct mt7601u_rxwi *) data; 94 data += sizeof(struct mt7601u_rxwi); 95 seg_len -= sizeof(struct mt7601u_rxwi); 96 97 if (unlikely(rxwi->zero[0] || rxwi->zero[1] || rxwi->zero[2])) 98 dev_err_once(dev->dev, "Error: RXWI zero fields are set\n"); 99 if (unlikely(FIELD_GET(MT_RXD_INFO_TYPE, fce_info))) 100 dev_err_once(dev->dev, "Error: RX path seen a non-pkt urb\n"); 101 102 trace_mt_rx(dev, rxwi, fce_info); 103 104 skb = mt7601u_rx_skb_from_seg(dev, rxwi, data, seg_len, truesize, p); 105 if (!skb) 106 return; 107 108 local_bh_disable(); 109 rcu_read_lock(); 110 111 ieee80211_rx_list(dev->hw, NULL, skb, list); 112 113 rcu_read_unlock(); 114 local_bh_enable(); 115 } 116 117 static u16 mt7601u_rx_next_seg_len(u8 *data, u32 data_len) 118 { 119 u32 min_seg_len = MT_DMA_HDR_LEN + MT_RX_INFO_LEN + 120 sizeof(struct mt7601u_rxwi) + MT_FCE_INFO_LEN; 121 u16 dma_len = get_unaligned_le16(data); 122 123 if (data_len < min_seg_len || 124 WARN_ON_ONCE(!dma_len) || 125 WARN_ON_ONCE(dma_len + MT_DMA_HDRS > data_len) || 126 WARN_ON_ONCE(dma_len & 0x3)) 127 return 0; 128 129 return MT_DMA_HDRS + dma_len; 130 } 131 132 static void 133 mt7601u_rx_process_entry(struct mt7601u_dev *dev, struct mt7601u_dma_buf_rx *e) 134 { 135 u32 seg_len, data_len = e->urb->actual_length; 136 u8 *data = page_address(e->p); 137 struct page *new_p = NULL; 138 LIST_HEAD(list); 139 int cnt = 0; 140 141 if (!test_bit(MT7601U_STATE_INITIALIZED, &dev->state)) 142 return; 143 144 /* Copy if there is very little data in the buffer. */ 145 if (data_len > 512) 146 new_p = dev_alloc_pages(MT_RX_ORDER); 147 148 while ((seg_len = mt7601u_rx_next_seg_len(data, data_len))) { 149 mt7601u_rx_process_seg(dev, data, seg_len, 150 new_p ? e->p : NULL, &list); 151 152 data_len -= seg_len; 153 data += seg_len; 154 cnt++; 155 } 156 157 if (cnt > 1) 158 trace_mt_rx_dma_aggr(dev, cnt, !!new_p); 159 160 netif_receive_skb_list(&list); 161 162 if (new_p) { 163 /* we have one extra ref from the allocator */ 164 put_page(e->p); 165 e->p = new_p; 166 } 167 } 168 169 static struct mt7601u_dma_buf_rx * 170 mt7601u_rx_get_pending_entry(struct mt7601u_dev *dev) 171 { 172 struct mt7601u_rx_queue *q = &dev->rx_q; 173 struct mt7601u_dma_buf_rx *buf = NULL; 174 unsigned long flags; 175 176 spin_lock_irqsave(&dev->rx_lock, flags); 177 178 if (!q->pending) 179 goto out; 180 181 buf = &q->e[q->start]; 182 q->pending--; 183 q->start = (q->start + 1) % q->entries; 184 out: 185 spin_unlock_irqrestore(&dev->rx_lock, flags); 186 187 return buf; 188 } 189 190 static void mt7601u_complete_rx(struct urb *urb) 191 { 192 struct mt7601u_dev *dev = urb->context; 193 struct mt7601u_rx_queue *q = &dev->rx_q; 194 unsigned long flags; 195 196 /* do no schedule rx tasklet if urb has been unlinked 197 * or the device has been removed 198 */ 199 switch (urb->status) { 200 case -ECONNRESET: 201 case -ESHUTDOWN: 202 case -ENOENT: 203 case -EPROTO: 204 return; 205 default: 206 dev_err_ratelimited(dev->dev, "rx urb failed: %d\n", 207 urb->status); 208 fallthrough; 209 case 0: 210 break; 211 } 212 213 spin_lock_irqsave(&dev->rx_lock, flags); 214 if (WARN_ONCE(q->e[q->end].urb != urb, "RX urb mismatch")) 215 goto out; 216 217 q->end = (q->end + 1) % q->entries; 218 q->pending++; 219 tasklet_schedule(&dev->rx_tasklet); 220 out: 221 spin_unlock_irqrestore(&dev->rx_lock, flags); 222 } 223 224 static void mt7601u_rx_tasklet(struct tasklet_struct *t) 225 { 226 struct mt7601u_dev *dev = from_tasklet(dev, t, rx_tasklet); 227 struct mt7601u_dma_buf_rx *e; 228 229 while ((e = mt7601u_rx_get_pending_entry(dev))) { 230 if (e->urb->status) 231 continue; 232 233 mt7601u_rx_process_entry(dev, e); 234 mt7601u_submit_rx_buf(dev, e, GFP_ATOMIC); 235 } 236 } 237 238 static void mt7601u_complete_tx(struct urb *urb) 239 { 240 struct mt7601u_tx_queue *q = urb->context; 241 struct mt7601u_dev *dev = q->dev; 242 struct sk_buff *skb; 243 unsigned long flags; 244 245 switch (urb->status) { 246 case -ECONNRESET: 247 case -ESHUTDOWN: 248 case -ENOENT: 249 case -EPROTO: 250 return; 251 default: 252 dev_err_ratelimited(dev->dev, "tx urb failed: %d\n", 253 urb->status); 254 fallthrough; 255 case 0: 256 break; 257 } 258 259 spin_lock_irqsave(&dev->tx_lock, flags); 260 if (WARN_ONCE(q->e[q->start].urb != urb, "TX urb mismatch")) 261 goto out; 262 263 skb = q->e[q->start].skb; 264 q->e[q->start].skb = NULL; 265 trace_mt_tx_dma_done(dev, skb); 266 267 __skb_queue_tail(&dev->tx_skb_done, skb); 268 tasklet_schedule(&dev->tx_tasklet); 269 270 if (q->used == q->entries - q->entries / 8) 271 ieee80211_wake_queue(dev->hw, skb_get_queue_mapping(skb)); 272 273 q->start = (q->start + 1) % q->entries; 274 q->used--; 275 out: 276 spin_unlock_irqrestore(&dev->tx_lock, flags); 277 } 278 279 static void mt7601u_tx_tasklet(struct tasklet_struct *t) 280 { 281 struct mt7601u_dev *dev = from_tasklet(dev, t, tx_tasklet); 282 struct sk_buff_head skbs; 283 unsigned long flags; 284 285 __skb_queue_head_init(&skbs); 286 287 spin_lock_irqsave(&dev->tx_lock, flags); 288 289 set_bit(MT7601U_STATE_MORE_STATS, &dev->state); 290 if (!test_and_set_bit(MT7601U_STATE_READING_STATS, &dev->state)) 291 queue_delayed_work(dev->stat_wq, &dev->stat_work, 292 msecs_to_jiffies(10)); 293 294 skb_queue_splice_init(&dev->tx_skb_done, &skbs); 295 296 spin_unlock_irqrestore(&dev->tx_lock, flags); 297 298 while (!skb_queue_empty(&skbs)) { 299 struct sk_buff *skb = __skb_dequeue(&skbs); 300 301 mt7601u_tx_status(dev, skb); 302 } 303 } 304 305 static int mt7601u_dma_submit_tx(struct mt7601u_dev *dev, 306 struct sk_buff *skb, u8 ep) 307 { 308 struct usb_device *usb_dev = mt7601u_to_usb_dev(dev); 309 unsigned snd_pipe = usb_sndbulkpipe(usb_dev, dev->out_eps[ep]); 310 struct mt7601u_dma_buf_tx *e; 311 struct mt7601u_tx_queue *q = &dev->tx_q[ep]; 312 unsigned long flags; 313 int ret; 314 315 spin_lock_irqsave(&dev->tx_lock, flags); 316 317 if (WARN_ON(q->entries <= q->used)) { 318 ret = -ENOSPC; 319 goto out; 320 } 321 322 e = &q->e[q->end]; 323 usb_fill_bulk_urb(e->urb, usb_dev, snd_pipe, skb->data, skb->len, 324 mt7601u_complete_tx, q); 325 ret = usb_submit_urb(e->urb, GFP_ATOMIC); 326 if (ret) { 327 /* Special-handle ENODEV from TX urb submission because it will 328 * often be the first ENODEV we see after device is removed. 329 */ 330 if (ret == -ENODEV) 331 set_bit(MT7601U_STATE_REMOVED, &dev->state); 332 else 333 dev_err(dev->dev, "Error: TX urb submit failed:%d\n", 334 ret); 335 goto out; 336 } 337 338 q->end = (q->end + 1) % q->entries; 339 q->used++; 340 e->skb = skb; 341 342 if (q->used >= q->entries) 343 ieee80211_stop_queue(dev->hw, skb_get_queue_mapping(skb)); 344 out: 345 spin_unlock_irqrestore(&dev->tx_lock, flags); 346 347 return ret; 348 } 349 350 /* Map hardware Q to USB endpoint number */ 351 static u8 q2ep(u8 qid) 352 { 353 /* TODO: take management packets to queue 5 */ 354 return qid + 1; 355 } 356 357 /* Map USB endpoint number to Q id in the DMA engine */ 358 static enum mt76_qsel ep2dmaq(u8 ep) 359 { 360 if (ep == 5) 361 return MT_QSEL_MGMT; 362 return MT_QSEL_EDCA; 363 } 364 365 int mt7601u_dma_enqueue_tx(struct mt7601u_dev *dev, struct sk_buff *skb, 366 struct mt76_wcid *wcid, int hw_q) 367 { 368 u8 ep = q2ep(hw_q); 369 u32 dma_flags; 370 int ret; 371 372 dma_flags = MT_TXD_PKT_INFO_80211; 373 if (wcid->hw_key_idx == 0xff) 374 dma_flags |= MT_TXD_PKT_INFO_WIV; 375 376 ret = mt7601u_dma_skb_wrap_pkt(skb, ep2dmaq(ep), dma_flags); 377 if (ret) 378 return ret; 379 380 ret = mt7601u_dma_submit_tx(dev, skb, ep); 381 if (ret) { 382 ieee80211_free_txskb(dev->hw, skb); 383 return ret; 384 } 385 386 return 0; 387 } 388 389 static void mt7601u_kill_rx(struct mt7601u_dev *dev) 390 { 391 int i; 392 393 for (i = 0; i < dev->rx_q.entries; i++) 394 usb_poison_urb(dev->rx_q.e[i].urb); 395 } 396 397 static int mt7601u_submit_rx_buf(struct mt7601u_dev *dev, 398 struct mt7601u_dma_buf_rx *e, gfp_t gfp) 399 { 400 struct usb_device *usb_dev = mt7601u_to_usb_dev(dev); 401 u8 *buf = page_address(e->p); 402 unsigned pipe; 403 int ret; 404 405 pipe = usb_rcvbulkpipe(usb_dev, dev->in_eps[MT_EP_IN_PKT_RX]); 406 407 usb_fill_bulk_urb(e->urb, usb_dev, pipe, buf, MT_RX_URB_SIZE, 408 mt7601u_complete_rx, dev); 409 410 trace_mt_submit_urb(dev, e->urb); 411 ret = usb_submit_urb(e->urb, gfp); 412 if (ret) 413 dev_err(dev->dev, "Error: submit RX URB failed:%d\n", ret); 414 415 return ret; 416 } 417 418 static int mt7601u_submit_rx(struct mt7601u_dev *dev) 419 { 420 int i, ret; 421 422 for (i = 0; i < dev->rx_q.entries; i++) { 423 ret = mt7601u_submit_rx_buf(dev, &dev->rx_q.e[i], GFP_KERNEL); 424 if (ret) 425 return ret; 426 } 427 428 return 0; 429 } 430 431 static void mt7601u_free_rx(struct mt7601u_dev *dev) 432 { 433 int i; 434 435 for (i = 0; i < dev->rx_q.entries; i++) { 436 __free_pages(dev->rx_q.e[i].p, MT_RX_ORDER); 437 usb_free_urb(dev->rx_q.e[i].urb); 438 } 439 } 440 441 static int mt7601u_alloc_rx(struct mt7601u_dev *dev) 442 { 443 int i; 444 445 memset(&dev->rx_q, 0, sizeof(dev->rx_q)); 446 dev->rx_q.dev = dev; 447 dev->rx_q.entries = N_RX_ENTRIES; 448 449 for (i = 0; i < N_RX_ENTRIES; i++) { 450 dev->rx_q.e[i].urb = usb_alloc_urb(0, GFP_KERNEL); 451 dev->rx_q.e[i].p = dev_alloc_pages(MT_RX_ORDER); 452 453 if (!dev->rx_q.e[i].urb || !dev->rx_q.e[i].p) 454 return -ENOMEM; 455 } 456 457 return 0; 458 } 459 460 static void mt7601u_free_tx_queue(struct mt7601u_tx_queue *q) 461 { 462 int i; 463 464 for (i = 0; i < q->entries; i++) { 465 usb_poison_urb(q->e[i].urb); 466 if (q->e[i].skb) 467 mt7601u_tx_status(q->dev, q->e[i].skb); 468 usb_free_urb(q->e[i].urb); 469 } 470 } 471 472 static void mt7601u_free_tx(struct mt7601u_dev *dev) 473 { 474 int i; 475 476 if (!dev->tx_q) 477 return; 478 479 for (i = 0; i < __MT_EP_OUT_MAX; i++) 480 mt7601u_free_tx_queue(&dev->tx_q[i]); 481 } 482 483 static int mt7601u_alloc_tx_queue(struct mt7601u_dev *dev, 484 struct mt7601u_tx_queue *q) 485 { 486 int i; 487 488 q->dev = dev; 489 q->entries = N_TX_ENTRIES; 490 491 for (i = 0; i < N_TX_ENTRIES; i++) { 492 q->e[i].urb = usb_alloc_urb(0, GFP_KERNEL); 493 if (!q->e[i].urb) 494 return -ENOMEM; 495 } 496 497 return 0; 498 } 499 500 static int mt7601u_alloc_tx(struct mt7601u_dev *dev) 501 { 502 int i; 503 504 dev->tx_q = devm_kcalloc(dev->dev, __MT_EP_OUT_MAX, 505 sizeof(*dev->tx_q), GFP_KERNEL); 506 if (!dev->tx_q) 507 return -ENOMEM; 508 509 for (i = 0; i < __MT_EP_OUT_MAX; i++) 510 if (mt7601u_alloc_tx_queue(dev, &dev->tx_q[i])) 511 return -ENOMEM; 512 513 return 0; 514 } 515 516 int mt7601u_dma_init(struct mt7601u_dev *dev) 517 { 518 int ret = -ENOMEM; 519 520 tasklet_setup(&dev->tx_tasklet, mt7601u_tx_tasklet); 521 tasklet_setup(&dev->rx_tasklet, mt7601u_rx_tasklet); 522 523 ret = mt7601u_alloc_tx(dev); 524 if (ret) 525 goto err; 526 ret = mt7601u_alloc_rx(dev); 527 if (ret) 528 goto err; 529 530 ret = mt7601u_submit_rx(dev); 531 if (ret) 532 goto err; 533 534 return 0; 535 err: 536 mt7601u_dma_cleanup(dev); 537 return ret; 538 } 539 540 void mt7601u_dma_cleanup(struct mt7601u_dev *dev) 541 { 542 mt7601u_kill_rx(dev); 543 544 tasklet_kill(&dev->rx_tasklet); 545 546 mt7601u_free_rx(dev); 547 mt7601u_free_tx(dev); 548 549 tasklet_kill(&dev->tx_tasklet); 550 } 551