1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> 4 */ 5 6 #include <linux/module.h> 7 #include "mt76.h" 8 #include "usb_trace.h" 9 #include "dma.h" 10 11 #define MT_VEND_REQ_MAX_RETRY 10 12 #define MT_VEND_REQ_TOUT_MS 300 13 14 static bool disable_usb_sg; 15 module_param_named(disable_usb_sg, disable_usb_sg, bool, 0644); 16 MODULE_PARM_DESC(disable_usb_sg, "Disable usb scatter-gather support"); 17 18 int __mt76u_vendor_request(struct mt76_dev *dev, u8 req, u8 req_type, 19 u16 val, u16 offset, void *buf, size_t len) 20 { 21 struct usb_interface *uintf = to_usb_interface(dev->dev); 22 struct usb_device *udev = interface_to_usbdev(uintf); 23 unsigned int pipe; 24 int i, ret; 25 26 lockdep_assert_held(&dev->usb.usb_ctrl_mtx); 27 28 pipe = (req_type & USB_DIR_IN) ? usb_rcvctrlpipe(udev, 0) 29 : usb_sndctrlpipe(udev, 0); 30 for (i = 0; i < MT_VEND_REQ_MAX_RETRY; i++) { 31 if (test_bit(MT76_REMOVED, &dev->phy.state)) 32 return -EIO; 33 if (dev->usb.ctrl_timeout && atomic_read(&dev->bus_hung)) 34 return -EIO; 35 36 ret = usb_control_msg(udev, pipe, req, req_type, val, 37 offset, buf, len, MT_VEND_REQ_TOUT_MS); 38 if (ret == -ENODEV || ret == -EPROTO) 39 set_bit(MT76_REMOVED, &dev->phy.state); 40 if (ret >= 0 || ret == -ENODEV || ret == -EPROTO) 41 return ret; 42 usleep_range(5000, 10000); 43 } 44 45 dev_err(dev->dev, "vendor request req:%02x off:%04x failed:%d\n", 46 req, offset, ret); 47 48 if (dev->usb.ctrl_timeout) { 49 atomic_set(&dev->bus_hung, true); 50 dev_err(dev->dev, "vendor request req:%02x off:%04x timed out, marking bus hung\n", 51 req, offset); 52 dev->usb.ctrl_timeout(dev, ret); 53 return ret; 54 } 55 56 return ret; 57 } 58 EXPORT_SYMBOL_GPL(__mt76u_vendor_request); 59 60 int mt76u_vendor_request(struct mt76_dev *dev, u8 req, 61 u8 req_type, u16 val, u16 offset, 62 void *buf, size_t len) 63 { 64 int ret; 65 66 mutex_lock(&dev->usb.usb_ctrl_mtx); 67 ret = __mt76u_vendor_request(dev, req, req_type, 68 val, offset, buf, len); 69 trace_usb_reg_wr(dev, offset, val); 70 mutex_unlock(&dev->usb.usb_ctrl_mtx); 71 72 return ret; 73 } 74 EXPORT_SYMBOL_GPL(mt76u_vendor_request); 75 76 u32 ___mt76u_rr(struct mt76_dev *dev, u8 req, u8 req_type, u32 addr) 77 { 78 struct mt76_usb *usb = &dev->usb; 79 u32 data = ~0; 80 int ret; 81 82 ret = __mt76u_vendor_request(dev, req, req_type, addr >> 16, 83 addr, usb->data, sizeof(__le32)); 84 if (ret == sizeof(__le32)) 85 data = get_unaligned_le32(usb->data); 86 trace_usb_reg_rr(dev, addr, data); 87 88 return data; 89 } 90 EXPORT_SYMBOL_GPL(___mt76u_rr); 91 92 static u32 __mt76u_rr(struct mt76_dev *dev, u32 addr) 93 { 94 u8 req; 95 96 switch (addr & MT_VEND_TYPE_MASK) { 97 case MT_VEND_TYPE_EEPROM: 98 req = MT_VEND_READ_EEPROM; 99 break; 100 case MT_VEND_TYPE_CFG: 101 req = MT_VEND_READ_CFG; 102 break; 103 default: 104 req = MT_VEND_MULTI_READ; 105 break; 106 } 107 108 return ___mt76u_rr(dev, req, USB_DIR_IN | USB_TYPE_VENDOR, 109 addr & ~MT_VEND_TYPE_MASK); 110 } 111 112 static u32 mt76u_rr(struct mt76_dev *dev, u32 addr) 113 { 114 u32 ret; 115 116 mutex_lock(&dev->usb.usb_ctrl_mtx); 117 ret = __mt76u_rr(dev, addr); 118 mutex_unlock(&dev->usb.usb_ctrl_mtx); 119 120 return ret; 121 } 122 123 void ___mt76u_wr(struct mt76_dev *dev, u8 req, u8 req_type, 124 u32 addr, u32 val) 125 { 126 struct mt76_usb *usb = &dev->usb; 127 128 put_unaligned_le32(val, usb->data); 129 __mt76u_vendor_request(dev, req, req_type, addr >> 16, 130 addr, usb->data, sizeof(__le32)); 131 trace_usb_reg_wr(dev, addr, val); 132 } 133 EXPORT_SYMBOL_GPL(___mt76u_wr); 134 135 static void __mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val) 136 { 137 u8 req; 138 139 switch (addr & MT_VEND_TYPE_MASK) { 140 case MT_VEND_TYPE_CFG: 141 req = MT_VEND_WRITE_CFG; 142 break; 143 default: 144 req = MT_VEND_MULTI_WRITE; 145 break; 146 } 147 ___mt76u_wr(dev, req, USB_DIR_OUT | USB_TYPE_VENDOR, 148 addr & ~MT_VEND_TYPE_MASK, val); 149 } 150 151 static void mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val) 152 { 153 mutex_lock(&dev->usb.usb_ctrl_mtx); 154 __mt76u_wr(dev, addr, val); 155 mutex_unlock(&dev->usb.usb_ctrl_mtx); 156 } 157 158 static u32 mt76u_rmw(struct mt76_dev *dev, u32 addr, 159 u32 mask, u32 val) 160 { 161 mutex_lock(&dev->usb.usb_ctrl_mtx); 162 val |= __mt76u_rr(dev, addr) & ~mask; 163 __mt76u_wr(dev, addr, val); 164 mutex_unlock(&dev->usb.usb_ctrl_mtx); 165 166 return val; 167 } 168 169 static void mt76u_copy(struct mt76_dev *dev, u32 offset, 170 const void *data, int len) 171 { 172 struct mt76_usb *usb = &dev->usb; 173 const u8 *val = data; 174 int current_batch_size; 175 int len_aligned; 176 int copy_len; 177 int ret; 178 int i = 0; 179 180 /* Assure that always a multiple of 4 bytes are copied, 181 * otherwise beacons can be corrupted. 182 * See: "mt76: round up length on mt76_wr_copy" 183 * Commit 850e8f6fbd5d0003b0 184 */ 185 len_aligned = round_up(len, 4); 186 187 mutex_lock(&usb->usb_ctrl_mtx); 188 while (i < len_aligned) { 189 current_batch_size = min_t(int, usb->data_len, len_aligned - i); 190 copy_len = min_t(int, current_batch_size, len - i); 191 memcpy(usb->data, val + i, copy_len); 192 if (copy_len < current_batch_size) 193 memset(usb->data + copy_len, 0, 194 current_batch_size - copy_len); 195 ret = __mt76u_vendor_request(dev, MT_VEND_MULTI_WRITE, 196 USB_DIR_OUT | USB_TYPE_VENDOR, 197 0, offset + i, usb->data, 198 current_batch_size); 199 if (ret < 0) 200 break; 201 202 i += current_batch_size; 203 } 204 mutex_unlock(&usb->usb_ctrl_mtx); 205 } 206 207 void mt76u_read_copy(struct mt76_dev *dev, u32 offset, 208 void *data, int len) 209 { 210 struct mt76_usb *usb = &dev->usb; 211 int i = 0, batch_len, ret; 212 u8 *val = data; 213 214 len = round_up(len, 4); 215 mutex_lock(&usb->usb_ctrl_mtx); 216 while (i < len) { 217 batch_len = min_t(int, usb->data_len, len - i); 218 ret = __mt76u_vendor_request(dev, MT_VEND_READ_EXT, 219 USB_DIR_IN | USB_TYPE_VENDOR, 220 (offset + i) >> 16, offset + i, 221 usb->data, batch_len); 222 if (ret < 0) 223 break; 224 225 memcpy(val + i, usb->data, batch_len); 226 i += batch_len; 227 } 228 mutex_unlock(&usb->usb_ctrl_mtx); 229 } 230 EXPORT_SYMBOL_GPL(mt76u_read_copy); 231 232 void mt76u_single_wr(struct mt76_dev *dev, const u8 req, 233 const u16 offset, const u32 val) 234 { 235 mutex_lock(&dev->usb.usb_ctrl_mtx); 236 __mt76u_vendor_request(dev, req, 237 USB_DIR_OUT | USB_TYPE_VENDOR, 238 val & 0xffff, offset, NULL, 0); 239 __mt76u_vendor_request(dev, req, 240 USB_DIR_OUT | USB_TYPE_VENDOR, 241 val >> 16, offset + 2, NULL, 0); 242 mutex_unlock(&dev->usb.usb_ctrl_mtx); 243 } 244 EXPORT_SYMBOL_GPL(mt76u_single_wr); 245 246 static int 247 mt76u_req_wr_rp(struct mt76_dev *dev, u32 base, 248 const struct mt76_reg_pair *data, int len) 249 { 250 struct mt76_usb *usb = &dev->usb; 251 252 mutex_lock(&usb->usb_ctrl_mtx); 253 while (len > 0) { 254 __mt76u_wr(dev, base + data->reg, data->value); 255 len--; 256 data++; 257 } 258 mutex_unlock(&usb->usb_ctrl_mtx); 259 260 return 0; 261 } 262 263 static int 264 mt76u_wr_rp(struct mt76_dev *dev, u32 base, 265 const struct mt76_reg_pair *data, int n) 266 { 267 if (test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state)) 268 return dev->mcu_ops->mcu_wr_rp(dev, base, data, n); 269 else 270 return mt76u_req_wr_rp(dev, base, data, n); 271 } 272 273 static int 274 mt76u_req_rd_rp(struct mt76_dev *dev, u32 base, struct mt76_reg_pair *data, 275 int len) 276 { 277 struct mt76_usb *usb = &dev->usb; 278 279 mutex_lock(&usb->usb_ctrl_mtx); 280 while (len > 0) { 281 data->value = __mt76u_rr(dev, base + data->reg); 282 len--; 283 data++; 284 } 285 mutex_unlock(&usb->usb_ctrl_mtx); 286 287 return 0; 288 } 289 290 static int 291 mt76u_rd_rp(struct mt76_dev *dev, u32 base, 292 struct mt76_reg_pair *data, int n) 293 { 294 if (test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state)) 295 return dev->mcu_ops->mcu_rd_rp(dev, base, data, n); 296 else 297 return mt76u_req_rd_rp(dev, base, data, n); 298 } 299 300 static bool mt76u_check_sg(struct mt76_dev *dev) 301 { 302 struct usb_interface *uintf = to_usb_interface(dev->dev); 303 struct usb_device *udev = interface_to_usbdev(uintf); 304 305 return (!disable_usb_sg && udev->bus->sg_tablesize > 0 && 306 udev->bus->no_sg_constraint); 307 } 308 309 static int 310 mt76u_set_endpoints(struct usb_interface *intf, 311 struct mt76_usb *usb) 312 { 313 struct usb_host_interface *intf_desc = intf->cur_altsetting; 314 struct usb_endpoint_descriptor *ep_desc; 315 int i, in_ep = 0, out_ep = 0; 316 317 for (i = 0; i < intf_desc->desc.bNumEndpoints; i++) { 318 ep_desc = &intf_desc->endpoint[i].desc; 319 320 if (usb_endpoint_is_bulk_in(ep_desc) && 321 in_ep < __MT_EP_IN_MAX) { 322 usb->in_ep[in_ep] = usb_endpoint_num(ep_desc); 323 in_ep++; 324 } else if (usb_endpoint_is_bulk_out(ep_desc) && 325 out_ep < __MT_EP_OUT_MAX) { 326 usb->out_ep[out_ep] = usb_endpoint_num(ep_desc); 327 out_ep++; 328 } 329 } 330 331 if (in_ep != __MT_EP_IN_MAX || out_ep != __MT_EP_OUT_MAX) 332 return -EINVAL; 333 return 0; 334 } 335 336 static int 337 mt76u_fill_rx_sg(struct mt76_dev *dev, struct mt76_queue *q, struct urb *urb, 338 int nsgs) 339 { 340 int i; 341 342 for (i = 0; i < nsgs; i++) { 343 void *data; 344 int offset; 345 346 data = mt76_get_page_pool_buf(q, &offset, q->buf_size); 347 if (!data) 348 break; 349 350 sg_set_page(&urb->sg[i], virt_to_head_page(data), q->buf_size, 351 offset); 352 } 353 354 if (i < nsgs) { 355 int j; 356 357 for (j = nsgs; j < urb->num_sgs; j++) 358 mt76_put_page_pool_buf(sg_virt(&urb->sg[j]), false); 359 urb->num_sgs = i; 360 } 361 362 urb->num_sgs = max_t(int, i, urb->num_sgs); 363 urb->transfer_buffer_length = urb->num_sgs * q->buf_size; 364 sg_init_marker(urb->sg, urb->num_sgs); 365 366 return i ? : -ENOMEM; 367 } 368 369 static int 370 mt76u_refill_rx(struct mt76_dev *dev, struct mt76_queue *q, 371 struct urb *urb, int nsgs) 372 { 373 enum mt76_rxq_id qid = q - &dev->q_rx[MT_RXQ_MAIN]; 374 int offset; 375 376 if (qid == MT_RXQ_MAIN && dev->usb.sg_en) 377 return mt76u_fill_rx_sg(dev, q, urb, nsgs); 378 379 urb->transfer_buffer_length = q->buf_size; 380 urb->transfer_buffer = mt76_get_page_pool_buf(q, &offset, q->buf_size); 381 382 return urb->transfer_buffer ? 0 : -ENOMEM; 383 } 384 385 static int 386 mt76u_urb_alloc(struct mt76_dev *dev, struct mt76_queue_entry *e, 387 int sg_max_size) 388 { 389 unsigned int size = sizeof(struct urb); 390 391 if (dev->usb.sg_en) 392 size += sg_max_size * sizeof(struct scatterlist); 393 394 e->urb = kzalloc(size, GFP_KERNEL); 395 if (!e->urb) 396 return -ENOMEM; 397 398 usb_init_urb(e->urb); 399 400 if (dev->usb.sg_en && sg_max_size > 0) 401 e->urb->sg = (struct scatterlist *)(e->urb + 1); 402 403 return 0; 404 } 405 406 static int 407 mt76u_rx_urb_alloc(struct mt76_dev *dev, struct mt76_queue *q, 408 struct mt76_queue_entry *e) 409 { 410 enum mt76_rxq_id qid = q - &dev->q_rx[MT_RXQ_MAIN]; 411 int err, sg_size; 412 413 sg_size = qid == MT_RXQ_MAIN ? MT_RX_SG_MAX_SIZE : 0; 414 err = mt76u_urb_alloc(dev, e, sg_size); 415 if (err) 416 return err; 417 418 return mt76u_refill_rx(dev, q, e->urb, sg_size); 419 } 420 421 static void mt76u_urb_free(struct urb *urb) 422 { 423 int i; 424 425 for (i = 0; i < urb->num_sgs; i++) 426 mt76_put_page_pool_buf(sg_virt(&urb->sg[i]), false); 427 428 if (urb->transfer_buffer) 429 mt76_put_page_pool_buf(urb->transfer_buffer, false); 430 431 usb_free_urb(urb); 432 } 433 434 static void 435 mt76u_fill_bulk_urb(struct mt76_dev *dev, int dir, int index, 436 struct urb *urb, usb_complete_t complete_fn, 437 void *context) 438 { 439 struct usb_interface *uintf = to_usb_interface(dev->dev); 440 struct usb_device *udev = interface_to_usbdev(uintf); 441 unsigned int pipe; 442 443 if (dir == USB_DIR_IN) 444 pipe = usb_rcvbulkpipe(udev, dev->usb.in_ep[index]); 445 else 446 pipe = usb_sndbulkpipe(udev, dev->usb.out_ep[index]); 447 448 urb->dev = udev; 449 urb->pipe = pipe; 450 urb->complete = complete_fn; 451 urb->context = context; 452 } 453 454 static struct urb * 455 mt76u_get_next_rx_entry(struct mt76_queue *q) 456 { 457 struct urb *urb = NULL; 458 unsigned long flags; 459 460 spin_lock_irqsave(&q->lock, flags); 461 if (q->queued > 0) { 462 urb = q->entry[q->tail].urb; 463 q->tail = (q->tail + 1) % q->ndesc; 464 q->queued--; 465 } 466 spin_unlock_irqrestore(&q->lock, flags); 467 468 return urb; 469 } 470 471 static int 472 mt76u_get_rx_entry_len(struct mt76_dev *dev, u8 *data, 473 u32 data_len) 474 { 475 u16 dma_len, min_len; 476 477 dma_len = get_unaligned_le16(data); 478 if (dev->drv->drv_flags & MT_DRV_RX_DMA_HDR) 479 return dma_len; 480 481 min_len = MT_DMA_HDR_LEN + MT_RX_RXWI_LEN + MT_FCE_INFO_LEN; 482 if (data_len < min_len || !dma_len || 483 dma_len + MT_DMA_HDR_LEN > data_len || 484 (dma_len & 0x3)) 485 return -EINVAL; 486 return dma_len; 487 } 488 489 static struct sk_buff * 490 mt76u_build_rx_skb(struct mt76_dev *dev, void *data, 491 int len, int buf_size) 492 { 493 int head_room, drv_flags = dev->drv->drv_flags; 494 struct sk_buff *skb; 495 496 head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN; 497 if (SKB_WITH_OVERHEAD(buf_size) < head_room + len) { 498 struct page *page; 499 500 /* slow path, not enough space for data and 501 * skb_shared_info 502 */ 503 skb = alloc_skb(MT_SKB_HEAD_LEN, GFP_ATOMIC); 504 if (!skb) 505 return NULL; 506 507 skb_put_data(skb, data + head_room, MT_SKB_HEAD_LEN); 508 data += head_room + MT_SKB_HEAD_LEN; 509 page = virt_to_head_page(data); 510 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 511 page, data - page_address(page), 512 len - MT_SKB_HEAD_LEN, buf_size); 513 514 return skb; 515 } 516 517 /* fast path */ 518 skb = build_skb(data, buf_size); 519 if (!skb) 520 return NULL; 521 522 skb_reserve(skb, head_room); 523 __skb_put(skb, len); 524 525 return skb; 526 } 527 528 static int 529 mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb, 530 int buf_size) 531 { 532 u8 *data = urb->num_sgs ? sg_virt(&urb->sg[0]) : urb->transfer_buffer; 533 int data_len = urb->num_sgs ? urb->sg[0].length : urb->actual_length; 534 int len, nsgs = 1, head_room, drv_flags = dev->drv->drv_flags; 535 struct sk_buff *skb; 536 537 if (!test_bit(MT76_STATE_INITIALIZED, &dev->phy.state)) 538 return 0; 539 540 len = mt76u_get_rx_entry_len(dev, data, urb->actual_length); 541 if (len < 0) 542 return 0; 543 544 head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN; 545 data_len = min_t(int, len, data_len - head_room); 546 547 if (len == data_len && 548 dev->drv->rx_check && !dev->drv->rx_check(dev, data, data_len)) 549 return 0; 550 551 skb = mt76u_build_rx_skb(dev, data, data_len, buf_size); 552 if (!skb) 553 return 0; 554 555 len -= data_len; 556 while (len > 0 && nsgs < urb->num_sgs) { 557 data_len = min_t(int, len, urb->sg[nsgs].length); 558 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 559 sg_page(&urb->sg[nsgs]), 560 urb->sg[nsgs].offset, data_len, 561 buf_size); 562 len -= data_len; 563 nsgs++; 564 } 565 566 skb_mark_for_recycle(skb); 567 dev->drv->rx_skb(dev, MT_RXQ_MAIN, skb, NULL); 568 569 return nsgs; 570 } 571 572 static void mt76u_complete_rx(struct urb *urb) 573 { 574 struct mt76_dev *dev = dev_get_drvdata(&urb->dev->dev); 575 struct mt76_queue *q = urb->context; 576 unsigned long flags; 577 578 trace_rx_urb(dev, urb); 579 580 switch (urb->status) { 581 case -ECONNRESET: 582 case -ESHUTDOWN: 583 case -ENOENT: 584 case -EPROTO: 585 return; 586 default: 587 dev_err_ratelimited(dev->dev, "rx urb failed: %d\n", 588 urb->status); 589 fallthrough; 590 case 0: 591 break; 592 } 593 594 spin_lock_irqsave(&q->lock, flags); 595 if (WARN_ONCE(q->entry[q->head].urb != urb, "rx urb mismatch")) 596 goto out; 597 598 q->head = (q->head + 1) % q->ndesc; 599 q->queued++; 600 601 if (q == &dev->q_rx[MT_RXQ_MAIN]) 602 napi_schedule(&dev->napi[MT_RXQ_MAIN]); 603 else 604 mt76_worker_schedule(&dev->usb.rx_worker); 605 out: 606 spin_unlock_irqrestore(&q->lock, flags); 607 } 608 609 static int 610 mt76u_submit_rx_buf(struct mt76_dev *dev, enum mt76_rxq_id qid, 611 struct urb *urb) 612 { 613 int ep = qid == MT_RXQ_MAIN ? MT_EP_IN_PKT_RX : MT_EP_IN_CMD_RESP; 614 615 mt76u_fill_bulk_urb(dev, USB_DIR_IN, ep, urb, 616 mt76u_complete_rx, &dev->q_rx[qid]); 617 trace_submit_urb(dev, urb); 618 619 return usb_submit_urb(urb, GFP_ATOMIC); 620 } 621 622 static void 623 mt76u_process_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) 624 { 625 int qid = q - &dev->q_rx[MT_RXQ_MAIN]; 626 struct urb *urb; 627 int err, count; 628 629 while (true) { 630 urb = mt76u_get_next_rx_entry(q); 631 if (!urb) 632 break; 633 634 count = mt76u_process_rx_entry(dev, urb, q->buf_size); 635 if (count > 0) { 636 err = mt76u_refill_rx(dev, q, urb, count); 637 if (err < 0) 638 break; 639 } 640 mt76u_submit_rx_buf(dev, qid, urb); 641 } 642 } 643 644 /* Threaded NAPI poll for the MAIN RX queue: drain URBs, build skbs, resubmit, 645 * then deliver through napi_gro_receive() and let napi_complete() flush GRO. 646 */ 647 static int mt76u_napi_poll(struct napi_struct *napi, int budget) 648 { 649 struct mt76_dev *dev = mt76_priv(napi->dev); 650 651 rcu_read_lock(); 652 mt76u_process_rx_queue(dev, &dev->q_rx[MT_RXQ_MAIN]); 653 mt76_rx_poll_complete(dev, MT_RXQ_MAIN, napi); 654 rcu_read_unlock(); 655 656 napi_complete(napi); 657 658 return 0; 659 } 660 661 static void mt76u_rx_worker(struct mt76_worker *w) 662 { 663 struct mt76_usb *usb = container_of(w, struct mt76_usb, rx_worker); 664 struct mt76_dev *dev = container_of(usb, struct mt76_dev, usb); 665 int i; 666 667 rcu_read_lock(); 668 mt76_for_each_q_rx(dev, i) { 669 /* MT_RXQ_MAIN is serviced by the threaded NAPI poll */ 670 if (i == MT_RXQ_MAIN) 671 continue; 672 673 mt76u_process_rx_queue(dev, &dev->q_rx[i]); 674 } 675 rcu_read_unlock(); 676 } 677 678 static int 679 mt76u_submit_rx_buffers(struct mt76_dev *dev, enum mt76_rxq_id qid) 680 { 681 struct mt76_queue *q = &dev->q_rx[qid]; 682 unsigned long flags; 683 int i, err = 0; 684 685 spin_lock_irqsave(&q->lock, flags); 686 for (i = 0; i < q->ndesc; i++) { 687 err = mt76u_submit_rx_buf(dev, qid, q->entry[i].urb); 688 if (err < 0) 689 break; 690 } 691 q->head = q->tail = 0; 692 q->queued = 0; 693 spin_unlock_irqrestore(&q->lock, flags); 694 695 return err; 696 } 697 698 static int 699 mt76u_alloc_rx_queue(struct mt76_dev *dev, enum mt76_rxq_id qid) 700 { 701 struct mt76_queue *q = &dev->q_rx[qid]; 702 int i, err; 703 704 err = mt76_create_page_pool(dev, q); 705 if (err) 706 return err; 707 708 spin_lock_init(&q->lock); 709 q->entry = devm_kcalloc(dev->dev, 710 MT_NUM_RX_ENTRIES, sizeof(*q->entry), 711 GFP_KERNEL); 712 if (!q->entry) 713 return -ENOMEM; 714 715 q->ndesc = MT_NUM_RX_ENTRIES; 716 q->buf_size = PAGE_SIZE; 717 718 for (i = 0; i < q->ndesc; i++) { 719 err = mt76u_rx_urb_alloc(dev, q, &q->entry[i]); 720 if (err < 0) 721 return err; 722 } 723 724 return mt76u_submit_rx_buffers(dev, qid); 725 } 726 727 int mt76u_alloc_mcu_queue(struct mt76_dev *dev) 728 { 729 return mt76u_alloc_rx_queue(dev, MT_RXQ_MCU); 730 } 731 EXPORT_SYMBOL_GPL(mt76u_alloc_mcu_queue); 732 733 static void 734 mt76u_free_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) 735 { 736 int i; 737 738 for (i = 0; i < q->ndesc; i++) { 739 if (!q->entry[i].urb) 740 continue; 741 742 mt76u_urb_free(q->entry[i].urb); 743 q->entry[i].urb = NULL; 744 } 745 page_pool_destroy(q->page_pool); 746 q->page_pool = NULL; 747 } 748 749 static void mt76u_free_rx(struct mt76_dev *dev) 750 { 751 int i; 752 753 mt76_worker_teardown(&dev->usb.rx_worker); 754 755 mt76_for_each_q_rx(dev, i) 756 mt76u_free_rx_queue(dev, &dev->q_rx[i]); 757 } 758 759 void mt76u_stop_rx(struct mt76_dev *dev) 760 { 761 int i; 762 763 mt76_worker_disable(&dev->usb.rx_worker); 764 765 mt76_for_each_q_rx(dev, i) { 766 struct mt76_queue *q = &dev->q_rx[i]; 767 int j; 768 769 for (j = 0; j < q->ndesc; j++) 770 usb_poison_urb(q->entry[j].urb); 771 } 772 773 /* The MAIN queue napi stays enabled for the device lifetime. The URBs 774 * are now poisoned, so mt76u_complete_rx() can no longer reschedule it; 775 * just drain any in-flight poll before the caller frees or resets. 776 */ 777 if (dev->napi_dev) 778 napi_synchronize(&dev->napi[MT_RXQ_MAIN]); 779 } 780 EXPORT_SYMBOL_GPL(mt76u_stop_rx); 781 782 int mt76u_resume_rx(struct mt76_dev *dev) 783 { 784 int i; 785 786 mt76_for_each_q_rx(dev, i) { 787 struct mt76_queue *q = &dev->q_rx[i]; 788 int err, j; 789 790 for (j = 0; j < q->ndesc; j++) 791 usb_unpoison_urb(q->entry[j].urb); 792 793 err = mt76u_submit_rx_buffers(dev, i); 794 if (err < 0) 795 return err; 796 } 797 798 mt76_worker_enable(&dev->usb.rx_worker); 799 800 return 0; 801 } 802 EXPORT_SYMBOL_GPL(mt76u_resume_rx); 803 804 static void mt76u_status_worker(struct mt76_worker *w) 805 { 806 struct mt76_usb *usb = container_of(w, struct mt76_usb, status_worker); 807 struct mt76_dev *dev = container_of(usb, struct mt76_dev, usb); 808 struct mt76_queue_entry entry; 809 struct mt76_queue *q; 810 int i; 811 812 if (!test_bit(MT76_STATE_RUNNING, &dev->phy.state)) 813 return; 814 815 for (i = 0; i <= MT_TXQ_PSD; i++) { 816 q = dev->phy.q_tx[i]; 817 if (!q) 818 continue; 819 820 while (q->queued > 0) { 821 if (!q->entry[q->tail].done) 822 break; 823 824 entry = q->entry[q->tail]; 825 q->entry[q->tail].done = false; 826 827 mt76_queue_tx_complete(dev, q, &entry); 828 } 829 830 if (!q->queued) 831 wake_up(&dev->tx_wait); 832 833 mt76_worker_schedule(&dev->tx_worker); 834 } 835 836 if (dev->drv->tx_status_data && 837 !test_and_set_bit(MT76_READING_STATS, &dev->phy.state)) 838 queue_work(dev->wq, &dev->usb.stat_work); 839 } 840 841 static void mt76u_tx_status_data(struct work_struct *work) 842 { 843 struct mt76_usb *usb; 844 struct mt76_dev *dev; 845 u8 update = 1; 846 u16 count = 0; 847 848 usb = container_of(work, struct mt76_usb, stat_work); 849 dev = container_of(usb, struct mt76_dev, usb); 850 851 while (true) { 852 if (test_bit(MT76_REMOVED, &dev->phy.state)) 853 break; 854 855 if (!dev->drv->tx_status_data(dev, &update)) 856 break; 857 count++; 858 } 859 860 if (count && test_bit(MT76_STATE_RUNNING, &dev->phy.state)) 861 queue_work(dev->wq, &usb->stat_work); 862 else 863 clear_bit(MT76_READING_STATS, &dev->phy.state); 864 } 865 866 static void mt76u_complete_tx(struct urb *urb) 867 { 868 struct mt76_dev *dev = dev_get_drvdata(&urb->dev->dev); 869 struct mt76_queue_entry *e = urb->context; 870 871 if (mt76u_urb_error(urb)) 872 dev_err(dev->dev, "tx urb failed: %d\n", urb->status); 873 e->done = true; 874 875 mt76_worker_schedule(&dev->usb.status_worker); 876 } 877 878 static int 879 mt76u_tx_setup_buffers(struct mt76_dev *dev, struct sk_buff *skb, 880 struct urb *urb) 881 { 882 urb->transfer_buffer_length = skb->len; 883 884 if (!dev->usb.sg_en) { 885 urb->transfer_buffer = skb->data; 886 return 0; 887 } 888 889 sg_init_table(urb->sg, MT_TX_SG_MAX_SIZE); 890 urb->num_sgs = skb_to_sgvec(skb, urb->sg, 0, skb->len); 891 if (!urb->num_sgs) 892 return -ENOMEM; 893 894 return urb->num_sgs; 895 } 896 897 static int 898 mt76u_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, 899 enum mt76_txq_id qid, struct sk_buff *skb, 900 struct mt76_wcid *wcid, struct ieee80211_sta *sta) 901 { 902 struct ieee80211_tx_status status = { 903 .sta = sta, 904 }; 905 struct mt76_tx_info tx_info = { 906 .skb = skb, 907 }; 908 struct mt76_dev *dev = phy->dev; 909 u16 idx = q->head; 910 int err; 911 912 if (q->queued == q->ndesc) { 913 err = -ENOSPC; 914 goto err_free_skb; 915 } 916 917 skb->prev = skb->next = NULL; 918 err = dev->drv->tx_prepare_skb(dev, NULL, qid, wcid, sta, &tx_info); 919 if (err < 0) 920 goto err_free_skb; 921 922 err = mt76u_tx_setup_buffers(dev, tx_info.skb, q->entry[idx].urb); 923 if (err < 0) { 924 /* 925 * mt76_tx_status_skb_get() walks the idr and dereferences 926 * a freed skb. This SKB is not counted in non-AQL counter 927 * due to error return, so using 0xffff as wcid to keep 928 * balanced. 929 */ 930 mt76_tx_complete_skb(dev, 0xffff, tx_info.skb); 931 goto err_ret; 932 } 933 934 mt76u_fill_bulk_urb(dev, USB_DIR_OUT, q->ep, q->entry[idx].urb, 935 mt76u_complete_tx, &q->entry[idx]); 936 937 q->head = (q->head + 1) % q->ndesc; 938 q->entry[idx].skb = tx_info.skb; 939 q->entry[idx].wcid = 0xffff; 940 q->queued++; 941 942 return idx; 943 944 err_free_skb: 945 status.skb = tx_info.skb; 946 spin_lock_bh(&dev->rx_lock); 947 ieee80211_tx_status_ext(dev->hw, &status); 948 spin_unlock_bh(&dev->rx_lock); 949 err_ret: 950 return err; 951 } 952 953 static void mt76u_tx_kick(struct mt76_dev *dev, struct mt76_queue *q) 954 { 955 struct urb *urb; 956 int err; 957 958 while (q->first != q->head) { 959 urb = q->entry[q->first].urb; 960 961 trace_submit_urb(dev, urb); 962 err = usb_submit_urb(urb, GFP_ATOMIC); 963 if (err < 0) { 964 if (err == -ENODEV) 965 set_bit(MT76_REMOVED, &dev->phy.state); 966 else 967 dev_err(dev->dev, "tx urb submit failed:%d\n", 968 err); 969 break; 970 } 971 q->first = (q->first + 1) % q->ndesc; 972 } 973 } 974 975 static void 976 mt76u_ac_to_hwq(struct mt76_dev *dev, struct mt76_queue *q, u8 qid) 977 { 978 u8 ac = qid < IEEE80211_NUM_ACS ? qid : IEEE80211_AC_BE; 979 980 switch (mt76_chip(dev)) { 981 case 0x7663: { 982 static const u8 lmac_queue_map[] = { 983 /* ac to lmac mapping */ 984 [IEEE80211_AC_BK] = 0, 985 [IEEE80211_AC_BE] = 1, 986 [IEEE80211_AC_VI] = 2, 987 [IEEE80211_AC_VO] = 4, 988 }; 989 990 q->hw_idx = lmac_queue_map[ac]; 991 q->ep = q->hw_idx + 1; 992 break; 993 } 994 case 0x6639: 995 case 0x7902: 996 case 0x7961: 997 case 0x7925: 998 q->hw_idx = mt76_ac_to_hwq(ac); 999 q->ep = qid == MT_TXQ_PSD ? MT_EP_OUT_HCCA : q->hw_idx + 1; 1000 break; 1001 default: 1002 q->hw_idx = mt76_ac_to_hwq(ac); 1003 q->ep = q->hw_idx + 1; 1004 break; 1005 } 1006 } 1007 1008 static int mt76u_alloc_tx(struct mt76_dev *dev) 1009 { 1010 int i; 1011 1012 for (i = 0; i <= MT_TXQ_PSD; i++) { 1013 struct mt76_queue *q; 1014 int j, err; 1015 1016 q = devm_kzalloc(dev->dev, sizeof(*q), GFP_KERNEL); 1017 if (!q) 1018 return -ENOMEM; 1019 1020 spin_lock_init(&q->lock); 1021 mt76u_ac_to_hwq(dev, q, i); 1022 dev->phy.q_tx[i] = q; 1023 1024 q->entry = devm_kcalloc(dev->dev, 1025 MT_NUM_TX_ENTRIES, sizeof(*q->entry), 1026 GFP_KERNEL); 1027 if (!q->entry) 1028 return -ENOMEM; 1029 1030 q->ndesc = MT_NUM_TX_ENTRIES; 1031 for (j = 0; j < q->ndesc; j++) { 1032 err = mt76u_urb_alloc(dev, &q->entry[j], 1033 MT_TX_SG_MAX_SIZE); 1034 if (err < 0) 1035 return err; 1036 } 1037 } 1038 return 0; 1039 } 1040 1041 static void mt76u_free_tx(struct mt76_dev *dev) 1042 { 1043 int i; 1044 1045 mt76_worker_teardown(&dev->usb.status_worker); 1046 1047 for (i = 0; i <= MT_TXQ_PSD; i++) { 1048 struct mt76_queue *q; 1049 int j; 1050 1051 q = dev->phy.q_tx[i]; 1052 if (!q) 1053 continue; 1054 1055 for (j = 0; j < q->ndesc; j++) { 1056 usb_free_urb(q->entry[j].urb); 1057 q->entry[j].urb = NULL; 1058 } 1059 } 1060 } 1061 1062 void mt76u_stop_tx(struct mt76_dev *dev) 1063 { 1064 int ret; 1065 1066 mt76_worker_disable(&dev->usb.status_worker); 1067 1068 ret = wait_event_timeout(dev->tx_wait, !mt76_has_tx_pending(&dev->phy), 1069 HZ / 5); 1070 if (!ret) { 1071 struct mt76_queue_entry entry; 1072 struct mt76_queue *q; 1073 int i, j; 1074 1075 dev_err(dev->dev, "timed out waiting for pending tx\n"); 1076 1077 for (i = 0; i <= MT_TXQ_PSD; i++) { 1078 q = dev->phy.q_tx[i]; 1079 if (!q) 1080 continue; 1081 1082 for (j = 0; j < q->ndesc; j++) 1083 usb_kill_urb(q->entry[j].urb); 1084 } 1085 1086 mt76_worker_disable(&dev->tx_worker); 1087 1088 /* On device removal we maight queue skb's, but mt76u_tx_kick() 1089 * will fail to submit urb, cleanup those skb's manually. 1090 */ 1091 for (i = 0; i <= MT_TXQ_PSD; i++) { 1092 q = dev->phy.q_tx[i]; 1093 if (!q) 1094 continue; 1095 1096 while (q->queued > 0) { 1097 entry = q->entry[q->tail]; 1098 q->entry[q->tail].done = false; 1099 mt76_queue_tx_complete(dev, q, &entry); 1100 } 1101 } 1102 1103 mt76_worker_enable(&dev->tx_worker); 1104 } 1105 1106 cancel_work_sync(&dev->usb.stat_work); 1107 clear_bit(MT76_READING_STATS, &dev->phy.state); 1108 1109 mt76_worker_enable(&dev->usb.status_worker); 1110 1111 mt76_tx_status_check(dev, true); 1112 } 1113 EXPORT_SYMBOL_GPL(mt76u_stop_tx); 1114 1115 void mt76u_queues_deinit(struct mt76_dev *dev) 1116 { 1117 mt76u_stop_rx(dev); 1118 mt76u_stop_tx(dev); 1119 1120 if (dev->napi_dev) { 1121 napi_disable(&dev->napi[MT_RXQ_MAIN]); 1122 netif_napi_del(&dev->napi[MT_RXQ_MAIN]); 1123 free_netdev(dev->napi_dev); 1124 dev->napi_dev = NULL; 1125 } 1126 1127 mt76u_free_rx(dev); 1128 mt76u_free_tx(dev); 1129 } 1130 EXPORT_SYMBOL_GPL(mt76u_queues_deinit); 1131 1132 int mt76u_alloc_queues(struct mt76_dev *dev) 1133 { 1134 int err; 1135 1136 err = mt76u_alloc_rx_queue(dev, MT_RXQ_MAIN); 1137 if (err < 0) 1138 return err; 1139 1140 return mt76u_alloc_tx(dev); 1141 } 1142 EXPORT_SYMBOL_GPL(mt76u_alloc_queues); 1143 1144 static const struct mt76_queue_ops usb_queue_ops = { 1145 .tx_queue_skb = mt76u_tx_queue_skb, 1146 .kick = mt76u_tx_kick, 1147 }; 1148 1149 int __mt76u_init(struct mt76_dev *dev, struct usb_interface *intf, 1150 struct mt76_bus_ops *ops) 1151 { 1152 struct usb_device *udev = interface_to_usbdev(intf); 1153 struct mt76_usb *usb = &dev->usb; 1154 struct mt76_dev **priv; 1155 int err; 1156 1157 INIT_WORK(&usb->stat_work, mt76u_tx_status_data); 1158 1159 usb->data_len = usb_maxpacket(udev, usb_sndctrlpipe(udev, 0)); 1160 if (usb->data_len < 32) 1161 usb->data_len = 32; 1162 1163 usb->data = devm_kmalloc(dev->dev, usb->data_len, GFP_KERNEL); 1164 if (!usb->data) 1165 return -ENOMEM; 1166 1167 mutex_init(&usb->usb_ctrl_mtx); 1168 dev->bus = ops; 1169 dev->queue_ops = &usb_queue_ops; 1170 1171 dev_set_drvdata(&udev->dev, dev); 1172 1173 usb->sg_en = mt76u_check_sg(dev); 1174 1175 err = mt76u_set_endpoints(intf, usb); 1176 if (err < 0) 1177 return err; 1178 1179 err = mt76_worker_setup(dev->hw, &usb->rx_worker, mt76u_rx_worker, 1180 "usb-rx"); 1181 if (err) 1182 return err; 1183 1184 err = mt76_worker_setup(dev->hw, &usb->status_worker, 1185 mt76u_status_worker, "usb-status"); 1186 if (err) 1187 return err; 1188 1189 sched_set_fifo_low(usb->rx_worker.task); 1190 sched_set_fifo_low(usb->status_worker.task); 1191 1192 /* threaded NAPI on a dummy netdev (reusing mt76_dev's napi_dev/napi[]) 1193 * services the MAIN RX queue and gives the RX path GRO 1194 */ 1195 dev->napi_dev = alloc_netdev_dummy(sizeof(struct mt76_dev *)); 1196 if (!dev->napi_dev) 1197 return -ENOMEM; 1198 1199 priv = netdev_priv(dev->napi_dev); 1200 *priv = dev; 1201 strscpy(dev->napi_dev->name, "mt76u-rx", sizeof(dev->napi_dev->name)); 1202 dev->napi_dev->threaded = 1; 1203 netif_napi_add(dev->napi_dev, &dev->napi[MT_RXQ_MAIN], mt76u_napi_poll); 1204 napi_enable(&dev->napi[MT_RXQ_MAIN]); 1205 1206 return 0; 1207 } 1208 EXPORT_SYMBOL_GPL(__mt76u_init); 1209 1210 int mt76u_init(struct mt76_dev *dev, struct usb_interface *intf) 1211 { 1212 static struct mt76_bus_ops bus_ops = { 1213 .rr = mt76u_rr, 1214 .wr = mt76u_wr, 1215 .rmw = mt76u_rmw, 1216 .read_copy = mt76u_read_copy, 1217 .write_copy = mt76u_copy, 1218 .wr_rp = mt76u_wr_rp, 1219 .rd_rp = mt76u_rd_rp, 1220 .type = MT76_BUS_USB, 1221 }; 1222 1223 return __mt76u_init(dev, intf, &bus_ops); 1224 } 1225 EXPORT_SYMBOL_GPL(mt76u_init); 1226 1227 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>"); 1228 MODULE_DESCRIPTION("MediaTek MT76x USB helpers"); 1229 MODULE_LICENSE("Dual BSD/GPL"); 1230