1 // SPDX-License-Identifier: BSD-3-Clause-Clear 2 /* 3 * Copyright (C) 2018 Lorenzo Bianconi <lorenzo.bianconi83@gmail.com> 4 */ 5 6 #include <linux/module.h> 7 #include "mt76.h" 8 #include "usb_trace.h" 9 #include "dma.h" 10 11 #define MT_VEND_REQ_MAX_RETRY 10 12 #define MT_VEND_REQ_TOUT_MS 300 13 14 static bool disable_usb_sg; 15 module_param_named(disable_usb_sg, disable_usb_sg, bool, 0644); 16 MODULE_PARM_DESC(disable_usb_sg, "Disable usb scatter-gather support"); 17 18 int __mt76u_vendor_request(struct mt76_dev *dev, u8 req, u8 req_type, 19 u16 val, u16 offset, void *buf, size_t len) 20 { 21 struct usb_interface *uintf = to_usb_interface(dev->dev); 22 struct usb_device *udev = interface_to_usbdev(uintf); 23 unsigned int pipe; 24 int i, ret; 25 26 lockdep_assert_held(&dev->usb.usb_ctrl_mtx); 27 28 pipe = (req_type & USB_DIR_IN) ? usb_rcvctrlpipe(udev, 0) 29 : usb_sndctrlpipe(udev, 0); 30 for (i = 0; i < MT_VEND_REQ_MAX_RETRY; i++) { 31 if (test_bit(MT76_REMOVED, &dev->phy.state)) 32 return -EIO; 33 if (dev->usb.ctrl_timeout && atomic_read(&dev->bus_hung)) 34 return -EIO; 35 36 ret = usb_control_msg(udev, pipe, req, req_type, val, 37 offset, buf, len, MT_VEND_REQ_TOUT_MS); 38 if (ret == -ENODEV || ret == -EPROTO) 39 set_bit(MT76_REMOVED, &dev->phy.state); 40 if (ret >= 0 || ret == -ENODEV || ret == -EPROTO) 41 return ret; 42 usleep_range(5000, 10000); 43 } 44 45 dev_err(dev->dev, "vendor request req:%02x off:%04x failed:%d\n", 46 req, offset, ret); 47 48 if (dev->usb.ctrl_timeout) { 49 atomic_set(&dev->bus_hung, true); 50 dev_err(dev->dev, "vendor request req:%02x off:%04x timed out, marking bus hung\n", 51 req, offset); 52 dev->usb.ctrl_timeout(dev, ret); 53 return ret; 54 } 55 56 return ret; 57 } 58 EXPORT_SYMBOL_GPL(__mt76u_vendor_request); 59 60 int mt76u_vendor_request(struct mt76_dev *dev, u8 req, 61 u8 req_type, u16 val, u16 offset, 62 void *buf, size_t len) 63 { 64 int ret; 65 66 mutex_lock(&dev->usb.usb_ctrl_mtx); 67 ret = __mt76u_vendor_request(dev, req, req_type, 68 val, offset, buf, len); 69 trace_usb_reg_wr(dev, offset, val); 70 mutex_unlock(&dev->usb.usb_ctrl_mtx); 71 72 return ret; 73 } 74 EXPORT_SYMBOL_GPL(mt76u_vendor_request); 75 76 u32 ___mt76u_rr(struct mt76_dev *dev, u8 req, u8 req_type, u32 addr) 77 { 78 struct mt76_usb *usb = &dev->usb; 79 u32 data = ~0; 80 int ret; 81 82 ret = __mt76u_vendor_request(dev, req, req_type, addr >> 16, 83 addr, usb->data, sizeof(__le32)); 84 if (ret == sizeof(__le32)) 85 data = get_unaligned_le32(usb->data); 86 trace_usb_reg_rr(dev, addr, data); 87 88 return data; 89 } 90 EXPORT_SYMBOL_GPL(___mt76u_rr); 91 92 static u32 __mt76u_rr(struct mt76_dev *dev, u32 addr) 93 { 94 u8 req; 95 96 switch (addr & MT_VEND_TYPE_MASK) { 97 case MT_VEND_TYPE_EEPROM: 98 req = MT_VEND_READ_EEPROM; 99 break; 100 case MT_VEND_TYPE_CFG: 101 req = MT_VEND_READ_CFG; 102 break; 103 default: 104 req = MT_VEND_MULTI_READ; 105 break; 106 } 107 108 return ___mt76u_rr(dev, req, USB_DIR_IN | USB_TYPE_VENDOR, 109 addr & ~MT_VEND_TYPE_MASK); 110 } 111 112 static u32 mt76u_rr(struct mt76_dev *dev, u32 addr) 113 { 114 u32 ret; 115 116 mutex_lock(&dev->usb.usb_ctrl_mtx); 117 ret = __mt76u_rr(dev, addr); 118 mutex_unlock(&dev->usb.usb_ctrl_mtx); 119 120 return ret; 121 } 122 123 void ___mt76u_wr(struct mt76_dev *dev, u8 req, u8 req_type, 124 u32 addr, u32 val) 125 { 126 struct mt76_usb *usb = &dev->usb; 127 128 put_unaligned_le32(val, usb->data); 129 __mt76u_vendor_request(dev, req, req_type, addr >> 16, 130 addr, usb->data, sizeof(__le32)); 131 trace_usb_reg_wr(dev, addr, val); 132 } 133 EXPORT_SYMBOL_GPL(___mt76u_wr); 134 135 static void __mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val) 136 { 137 u8 req; 138 139 switch (addr & MT_VEND_TYPE_MASK) { 140 case MT_VEND_TYPE_CFG: 141 req = MT_VEND_WRITE_CFG; 142 break; 143 default: 144 req = MT_VEND_MULTI_WRITE; 145 break; 146 } 147 ___mt76u_wr(dev, req, USB_DIR_OUT | USB_TYPE_VENDOR, 148 addr & ~MT_VEND_TYPE_MASK, val); 149 } 150 151 static void mt76u_wr(struct mt76_dev *dev, u32 addr, u32 val) 152 { 153 mutex_lock(&dev->usb.usb_ctrl_mtx); 154 __mt76u_wr(dev, addr, val); 155 mutex_unlock(&dev->usb.usb_ctrl_mtx); 156 } 157 158 static u32 mt76u_rmw(struct mt76_dev *dev, u32 addr, 159 u32 mask, u32 val) 160 { 161 mutex_lock(&dev->usb.usb_ctrl_mtx); 162 val |= __mt76u_rr(dev, addr) & ~mask; 163 __mt76u_wr(dev, addr, val); 164 mutex_unlock(&dev->usb.usb_ctrl_mtx); 165 166 return val; 167 } 168 169 static void mt76u_copy(struct mt76_dev *dev, u32 offset, 170 const void *data, int len) 171 { 172 struct mt76_usb *usb = &dev->usb; 173 const u8 *val = data; 174 int ret; 175 int current_batch_size; 176 int i = 0; 177 178 /* Assure that always a multiple of 4 bytes are copied, 179 * otherwise beacons can be corrupted. 180 * See: "mt76: round up length on mt76_wr_copy" 181 * Commit 850e8f6fbd5d0003b0 182 */ 183 len = round_up(len, 4); 184 185 mutex_lock(&usb->usb_ctrl_mtx); 186 while (i < len) { 187 current_batch_size = min_t(int, usb->data_len, len - i); 188 memcpy(usb->data, val + i, current_batch_size); 189 ret = __mt76u_vendor_request(dev, MT_VEND_MULTI_WRITE, 190 USB_DIR_OUT | USB_TYPE_VENDOR, 191 0, offset + i, usb->data, 192 current_batch_size); 193 if (ret < 0) 194 break; 195 196 i += current_batch_size; 197 } 198 mutex_unlock(&usb->usb_ctrl_mtx); 199 } 200 201 void mt76u_read_copy(struct mt76_dev *dev, u32 offset, 202 void *data, int len) 203 { 204 struct mt76_usb *usb = &dev->usb; 205 int i = 0, batch_len, ret; 206 u8 *val = data; 207 208 len = round_up(len, 4); 209 mutex_lock(&usb->usb_ctrl_mtx); 210 while (i < len) { 211 batch_len = min_t(int, usb->data_len, len - i); 212 ret = __mt76u_vendor_request(dev, MT_VEND_READ_EXT, 213 USB_DIR_IN | USB_TYPE_VENDOR, 214 (offset + i) >> 16, offset + i, 215 usb->data, batch_len); 216 if (ret < 0) 217 break; 218 219 memcpy(val + i, usb->data, batch_len); 220 i += batch_len; 221 } 222 mutex_unlock(&usb->usb_ctrl_mtx); 223 } 224 EXPORT_SYMBOL_GPL(mt76u_read_copy); 225 226 void mt76u_single_wr(struct mt76_dev *dev, const u8 req, 227 const u16 offset, const u32 val) 228 { 229 mutex_lock(&dev->usb.usb_ctrl_mtx); 230 __mt76u_vendor_request(dev, req, 231 USB_DIR_OUT | USB_TYPE_VENDOR, 232 val & 0xffff, offset, NULL, 0); 233 __mt76u_vendor_request(dev, req, 234 USB_DIR_OUT | USB_TYPE_VENDOR, 235 val >> 16, offset + 2, NULL, 0); 236 mutex_unlock(&dev->usb.usb_ctrl_mtx); 237 } 238 EXPORT_SYMBOL_GPL(mt76u_single_wr); 239 240 static int 241 mt76u_req_wr_rp(struct mt76_dev *dev, u32 base, 242 const struct mt76_reg_pair *data, int len) 243 { 244 struct mt76_usb *usb = &dev->usb; 245 246 mutex_lock(&usb->usb_ctrl_mtx); 247 while (len > 0) { 248 __mt76u_wr(dev, base + data->reg, data->value); 249 len--; 250 data++; 251 } 252 mutex_unlock(&usb->usb_ctrl_mtx); 253 254 return 0; 255 } 256 257 static int 258 mt76u_wr_rp(struct mt76_dev *dev, u32 base, 259 const struct mt76_reg_pair *data, int n) 260 { 261 if (test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state)) 262 return dev->mcu_ops->mcu_wr_rp(dev, base, data, n); 263 else 264 return mt76u_req_wr_rp(dev, base, data, n); 265 } 266 267 static int 268 mt76u_req_rd_rp(struct mt76_dev *dev, u32 base, struct mt76_reg_pair *data, 269 int len) 270 { 271 struct mt76_usb *usb = &dev->usb; 272 273 mutex_lock(&usb->usb_ctrl_mtx); 274 while (len > 0) { 275 data->value = __mt76u_rr(dev, base + data->reg); 276 len--; 277 data++; 278 } 279 mutex_unlock(&usb->usb_ctrl_mtx); 280 281 return 0; 282 } 283 284 static int 285 mt76u_rd_rp(struct mt76_dev *dev, u32 base, 286 struct mt76_reg_pair *data, int n) 287 { 288 if (test_bit(MT76_STATE_MCU_RUNNING, &dev->phy.state)) 289 return dev->mcu_ops->mcu_rd_rp(dev, base, data, n); 290 else 291 return mt76u_req_rd_rp(dev, base, data, n); 292 } 293 294 static bool mt76u_check_sg(struct mt76_dev *dev) 295 { 296 struct usb_interface *uintf = to_usb_interface(dev->dev); 297 struct usb_device *udev = interface_to_usbdev(uintf); 298 299 return (!disable_usb_sg && udev->bus->sg_tablesize > 0 && 300 udev->bus->no_sg_constraint); 301 } 302 303 static int 304 mt76u_set_endpoints(struct usb_interface *intf, 305 struct mt76_usb *usb) 306 { 307 struct usb_host_interface *intf_desc = intf->cur_altsetting; 308 struct usb_endpoint_descriptor *ep_desc; 309 int i, in_ep = 0, out_ep = 0; 310 311 for (i = 0; i < intf_desc->desc.bNumEndpoints; i++) { 312 ep_desc = &intf_desc->endpoint[i].desc; 313 314 if (usb_endpoint_is_bulk_in(ep_desc) && 315 in_ep < __MT_EP_IN_MAX) { 316 usb->in_ep[in_ep] = usb_endpoint_num(ep_desc); 317 in_ep++; 318 } else if (usb_endpoint_is_bulk_out(ep_desc) && 319 out_ep < __MT_EP_OUT_MAX) { 320 usb->out_ep[out_ep] = usb_endpoint_num(ep_desc); 321 out_ep++; 322 } 323 } 324 325 if (in_ep != __MT_EP_IN_MAX || out_ep != __MT_EP_OUT_MAX) 326 return -EINVAL; 327 return 0; 328 } 329 330 static int 331 mt76u_fill_rx_sg(struct mt76_dev *dev, struct mt76_queue *q, struct urb *urb, 332 int nsgs) 333 { 334 int i; 335 336 for (i = 0; i < nsgs; i++) { 337 void *data; 338 int offset; 339 340 data = mt76_get_page_pool_buf(q, &offset, q->buf_size); 341 if (!data) 342 break; 343 344 sg_set_page(&urb->sg[i], virt_to_head_page(data), q->buf_size, 345 offset); 346 } 347 348 if (i < nsgs) { 349 int j; 350 351 for (j = nsgs; j < urb->num_sgs; j++) 352 mt76_put_page_pool_buf(sg_virt(&urb->sg[j]), false); 353 urb->num_sgs = i; 354 } 355 356 urb->num_sgs = max_t(int, i, urb->num_sgs); 357 urb->transfer_buffer_length = urb->num_sgs * q->buf_size; 358 sg_init_marker(urb->sg, urb->num_sgs); 359 360 return i ? : -ENOMEM; 361 } 362 363 static int 364 mt76u_refill_rx(struct mt76_dev *dev, struct mt76_queue *q, 365 struct urb *urb, int nsgs) 366 { 367 enum mt76_rxq_id qid = q - &dev->q_rx[MT_RXQ_MAIN]; 368 int offset; 369 370 if (qid == MT_RXQ_MAIN && dev->usb.sg_en) 371 return mt76u_fill_rx_sg(dev, q, urb, nsgs); 372 373 urb->transfer_buffer_length = q->buf_size; 374 urb->transfer_buffer = mt76_get_page_pool_buf(q, &offset, q->buf_size); 375 376 return urb->transfer_buffer ? 0 : -ENOMEM; 377 } 378 379 static int 380 mt76u_urb_alloc(struct mt76_dev *dev, struct mt76_queue_entry *e, 381 int sg_max_size) 382 { 383 unsigned int size = sizeof(struct urb); 384 385 if (dev->usb.sg_en) 386 size += sg_max_size * sizeof(struct scatterlist); 387 388 e->urb = kzalloc(size, GFP_KERNEL); 389 if (!e->urb) 390 return -ENOMEM; 391 392 usb_init_urb(e->urb); 393 394 if (dev->usb.sg_en && sg_max_size > 0) 395 e->urb->sg = (struct scatterlist *)(e->urb + 1); 396 397 return 0; 398 } 399 400 static int 401 mt76u_rx_urb_alloc(struct mt76_dev *dev, struct mt76_queue *q, 402 struct mt76_queue_entry *e) 403 { 404 enum mt76_rxq_id qid = q - &dev->q_rx[MT_RXQ_MAIN]; 405 int err, sg_size; 406 407 sg_size = qid == MT_RXQ_MAIN ? MT_RX_SG_MAX_SIZE : 0; 408 err = mt76u_urb_alloc(dev, e, sg_size); 409 if (err) 410 return err; 411 412 return mt76u_refill_rx(dev, q, e->urb, sg_size); 413 } 414 415 static void mt76u_urb_free(struct urb *urb) 416 { 417 int i; 418 419 for (i = 0; i < urb->num_sgs; i++) 420 mt76_put_page_pool_buf(sg_virt(&urb->sg[i]), false); 421 422 if (urb->transfer_buffer) 423 mt76_put_page_pool_buf(urb->transfer_buffer, false); 424 425 usb_free_urb(urb); 426 } 427 428 static void 429 mt76u_fill_bulk_urb(struct mt76_dev *dev, int dir, int index, 430 struct urb *urb, usb_complete_t complete_fn, 431 void *context) 432 { 433 struct usb_interface *uintf = to_usb_interface(dev->dev); 434 struct usb_device *udev = interface_to_usbdev(uintf); 435 unsigned int pipe; 436 437 if (dir == USB_DIR_IN) 438 pipe = usb_rcvbulkpipe(udev, dev->usb.in_ep[index]); 439 else 440 pipe = usb_sndbulkpipe(udev, dev->usb.out_ep[index]); 441 442 urb->dev = udev; 443 urb->pipe = pipe; 444 urb->complete = complete_fn; 445 urb->context = context; 446 } 447 448 static struct urb * 449 mt76u_get_next_rx_entry(struct mt76_queue *q) 450 { 451 struct urb *urb = NULL; 452 unsigned long flags; 453 454 spin_lock_irqsave(&q->lock, flags); 455 if (q->queued > 0) { 456 urb = q->entry[q->tail].urb; 457 q->tail = (q->tail + 1) % q->ndesc; 458 q->queued--; 459 } 460 spin_unlock_irqrestore(&q->lock, flags); 461 462 return urb; 463 } 464 465 static int 466 mt76u_get_rx_entry_len(struct mt76_dev *dev, u8 *data, 467 u32 data_len) 468 { 469 u16 dma_len, min_len; 470 471 dma_len = get_unaligned_le16(data); 472 if (dev->drv->drv_flags & MT_DRV_RX_DMA_HDR) 473 return dma_len; 474 475 min_len = MT_DMA_HDR_LEN + MT_RX_RXWI_LEN + MT_FCE_INFO_LEN; 476 if (data_len < min_len || !dma_len || 477 dma_len + MT_DMA_HDR_LEN > data_len || 478 (dma_len & 0x3)) 479 return -EINVAL; 480 return dma_len; 481 } 482 483 static struct sk_buff * 484 mt76u_build_rx_skb(struct mt76_dev *dev, void *data, 485 int len, int buf_size) 486 { 487 int head_room, drv_flags = dev->drv->drv_flags; 488 struct sk_buff *skb; 489 490 head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN; 491 if (SKB_WITH_OVERHEAD(buf_size) < head_room + len) { 492 struct page *page; 493 494 /* slow path, not enough space for data and 495 * skb_shared_info 496 */ 497 skb = alloc_skb(MT_SKB_HEAD_LEN, GFP_ATOMIC); 498 if (!skb) 499 return NULL; 500 501 skb_put_data(skb, data + head_room, MT_SKB_HEAD_LEN); 502 data += head_room + MT_SKB_HEAD_LEN; 503 page = virt_to_head_page(data); 504 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 505 page, data - page_address(page), 506 len - MT_SKB_HEAD_LEN, buf_size); 507 508 return skb; 509 } 510 511 /* fast path */ 512 skb = build_skb(data, buf_size); 513 if (!skb) 514 return NULL; 515 516 skb_reserve(skb, head_room); 517 __skb_put(skb, len); 518 519 return skb; 520 } 521 522 static int 523 mt76u_process_rx_entry(struct mt76_dev *dev, struct urb *urb, 524 int buf_size) 525 { 526 u8 *data = urb->num_sgs ? sg_virt(&urb->sg[0]) : urb->transfer_buffer; 527 int data_len = urb->num_sgs ? urb->sg[0].length : urb->actual_length; 528 int len, nsgs = 1, head_room, drv_flags = dev->drv->drv_flags; 529 struct sk_buff *skb; 530 531 if (!test_bit(MT76_STATE_INITIALIZED, &dev->phy.state)) 532 return 0; 533 534 len = mt76u_get_rx_entry_len(dev, data, urb->actual_length); 535 if (len < 0) 536 return 0; 537 538 head_room = drv_flags & MT_DRV_RX_DMA_HDR ? 0 : MT_DMA_HDR_LEN; 539 data_len = min_t(int, len, data_len - head_room); 540 541 if (len == data_len && 542 dev->drv->rx_check && !dev->drv->rx_check(dev, data, data_len)) 543 return 0; 544 545 skb = mt76u_build_rx_skb(dev, data, data_len, buf_size); 546 if (!skb) 547 return 0; 548 549 len -= data_len; 550 while (len > 0 && nsgs < urb->num_sgs) { 551 data_len = min_t(int, len, urb->sg[nsgs].length); 552 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, 553 sg_page(&urb->sg[nsgs]), 554 urb->sg[nsgs].offset, data_len, 555 buf_size); 556 len -= data_len; 557 nsgs++; 558 } 559 560 skb_mark_for_recycle(skb); 561 dev->drv->rx_skb(dev, MT_RXQ_MAIN, skb, NULL); 562 563 return nsgs; 564 } 565 566 static void mt76u_complete_rx(struct urb *urb) 567 { 568 struct mt76_dev *dev = dev_get_drvdata(&urb->dev->dev); 569 struct mt76_queue *q = urb->context; 570 unsigned long flags; 571 572 trace_rx_urb(dev, urb); 573 574 switch (urb->status) { 575 case -ECONNRESET: 576 case -ESHUTDOWN: 577 case -ENOENT: 578 case -EPROTO: 579 return; 580 default: 581 dev_err_ratelimited(dev->dev, "rx urb failed: %d\n", 582 urb->status); 583 fallthrough; 584 case 0: 585 break; 586 } 587 588 spin_lock_irqsave(&q->lock, flags); 589 if (WARN_ONCE(q->entry[q->head].urb != urb, "rx urb mismatch")) 590 goto out; 591 592 q->head = (q->head + 1) % q->ndesc; 593 q->queued++; 594 595 if (q == &dev->q_rx[MT_RXQ_MAIN]) 596 napi_schedule(&dev->napi[MT_RXQ_MAIN]); 597 else 598 mt76_worker_schedule(&dev->usb.rx_worker); 599 out: 600 spin_unlock_irqrestore(&q->lock, flags); 601 } 602 603 static int 604 mt76u_submit_rx_buf(struct mt76_dev *dev, enum mt76_rxq_id qid, 605 struct urb *urb) 606 { 607 int ep = qid == MT_RXQ_MAIN ? MT_EP_IN_PKT_RX : MT_EP_IN_CMD_RESP; 608 609 mt76u_fill_bulk_urb(dev, USB_DIR_IN, ep, urb, 610 mt76u_complete_rx, &dev->q_rx[qid]); 611 trace_submit_urb(dev, urb); 612 613 return usb_submit_urb(urb, GFP_ATOMIC); 614 } 615 616 static void 617 mt76u_process_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) 618 { 619 int qid = q - &dev->q_rx[MT_RXQ_MAIN]; 620 struct urb *urb; 621 int err, count; 622 623 while (true) { 624 urb = mt76u_get_next_rx_entry(q); 625 if (!urb) 626 break; 627 628 count = mt76u_process_rx_entry(dev, urb, q->buf_size); 629 if (count > 0) { 630 err = mt76u_refill_rx(dev, q, urb, count); 631 if (err < 0) 632 break; 633 } 634 mt76u_submit_rx_buf(dev, qid, urb); 635 } 636 } 637 638 /* Threaded NAPI poll for the MAIN RX queue: drain URBs, build skbs, resubmit, 639 * then deliver through napi_gro_receive() and let napi_complete() flush GRO. 640 */ 641 static int mt76u_napi_poll(struct napi_struct *napi, int budget) 642 { 643 struct mt76_dev *dev = mt76_priv(napi->dev); 644 645 rcu_read_lock(); 646 mt76u_process_rx_queue(dev, &dev->q_rx[MT_RXQ_MAIN]); 647 mt76_rx_poll_complete(dev, MT_RXQ_MAIN, napi); 648 rcu_read_unlock(); 649 650 napi_complete(napi); 651 652 return 0; 653 } 654 655 static void mt76u_rx_worker(struct mt76_worker *w) 656 { 657 struct mt76_usb *usb = container_of(w, struct mt76_usb, rx_worker); 658 struct mt76_dev *dev = container_of(usb, struct mt76_dev, usb); 659 int i; 660 661 rcu_read_lock(); 662 mt76_for_each_q_rx(dev, i) { 663 /* MT_RXQ_MAIN is serviced by the threaded NAPI poll */ 664 if (i == MT_RXQ_MAIN) 665 continue; 666 667 mt76u_process_rx_queue(dev, &dev->q_rx[i]); 668 } 669 rcu_read_unlock(); 670 } 671 672 static int 673 mt76u_submit_rx_buffers(struct mt76_dev *dev, enum mt76_rxq_id qid) 674 { 675 struct mt76_queue *q = &dev->q_rx[qid]; 676 unsigned long flags; 677 int i, err = 0; 678 679 spin_lock_irqsave(&q->lock, flags); 680 for (i = 0; i < q->ndesc; i++) { 681 err = mt76u_submit_rx_buf(dev, qid, q->entry[i].urb); 682 if (err < 0) 683 break; 684 } 685 q->head = q->tail = 0; 686 q->queued = 0; 687 spin_unlock_irqrestore(&q->lock, flags); 688 689 return err; 690 } 691 692 static int 693 mt76u_alloc_rx_queue(struct mt76_dev *dev, enum mt76_rxq_id qid) 694 { 695 struct mt76_queue *q = &dev->q_rx[qid]; 696 int i, err; 697 698 err = mt76_create_page_pool(dev, q); 699 if (err) 700 return err; 701 702 spin_lock_init(&q->lock); 703 q->entry = devm_kcalloc(dev->dev, 704 MT_NUM_RX_ENTRIES, sizeof(*q->entry), 705 GFP_KERNEL); 706 if (!q->entry) 707 return -ENOMEM; 708 709 q->ndesc = MT_NUM_RX_ENTRIES; 710 q->buf_size = PAGE_SIZE; 711 712 for (i = 0; i < q->ndesc; i++) { 713 err = mt76u_rx_urb_alloc(dev, q, &q->entry[i]); 714 if (err < 0) 715 return err; 716 } 717 718 return mt76u_submit_rx_buffers(dev, qid); 719 } 720 721 int mt76u_alloc_mcu_queue(struct mt76_dev *dev) 722 { 723 return mt76u_alloc_rx_queue(dev, MT_RXQ_MCU); 724 } 725 EXPORT_SYMBOL_GPL(mt76u_alloc_mcu_queue); 726 727 static void 728 mt76u_free_rx_queue(struct mt76_dev *dev, struct mt76_queue *q) 729 { 730 int i; 731 732 for (i = 0; i < q->ndesc; i++) { 733 if (!q->entry[i].urb) 734 continue; 735 736 mt76u_urb_free(q->entry[i].urb); 737 q->entry[i].urb = NULL; 738 } 739 page_pool_destroy(q->page_pool); 740 q->page_pool = NULL; 741 } 742 743 static void mt76u_free_rx(struct mt76_dev *dev) 744 { 745 int i; 746 747 mt76_worker_teardown(&dev->usb.rx_worker); 748 749 mt76_for_each_q_rx(dev, i) 750 mt76u_free_rx_queue(dev, &dev->q_rx[i]); 751 } 752 753 void mt76u_stop_rx(struct mt76_dev *dev) 754 { 755 int i; 756 757 mt76_worker_disable(&dev->usb.rx_worker); 758 759 mt76_for_each_q_rx(dev, i) { 760 struct mt76_queue *q = &dev->q_rx[i]; 761 int j; 762 763 for (j = 0; j < q->ndesc; j++) 764 usb_poison_urb(q->entry[j].urb); 765 } 766 767 /* The MAIN queue napi stays enabled for the device lifetime. The URBs 768 * are now poisoned, so mt76u_complete_rx() can no longer reschedule it; 769 * just drain any in-flight poll before the caller frees or resets. 770 */ 771 if (dev->napi_dev) 772 napi_synchronize(&dev->napi[MT_RXQ_MAIN]); 773 } 774 EXPORT_SYMBOL_GPL(mt76u_stop_rx); 775 776 int mt76u_resume_rx(struct mt76_dev *dev) 777 { 778 int i; 779 780 mt76_for_each_q_rx(dev, i) { 781 struct mt76_queue *q = &dev->q_rx[i]; 782 int err, j; 783 784 for (j = 0; j < q->ndesc; j++) 785 usb_unpoison_urb(q->entry[j].urb); 786 787 err = mt76u_submit_rx_buffers(dev, i); 788 if (err < 0) 789 return err; 790 } 791 792 mt76_worker_enable(&dev->usb.rx_worker); 793 794 return 0; 795 } 796 EXPORT_SYMBOL_GPL(mt76u_resume_rx); 797 798 static void mt76u_status_worker(struct mt76_worker *w) 799 { 800 struct mt76_usb *usb = container_of(w, struct mt76_usb, status_worker); 801 struct mt76_dev *dev = container_of(usb, struct mt76_dev, usb); 802 struct mt76_queue_entry entry; 803 struct mt76_queue *q; 804 int i; 805 806 if (!test_bit(MT76_STATE_RUNNING, &dev->phy.state)) 807 return; 808 809 for (i = 0; i <= MT_TXQ_PSD; i++) { 810 q = dev->phy.q_tx[i]; 811 if (!q) 812 continue; 813 814 while (q->queued > 0) { 815 if (!q->entry[q->tail].done) 816 break; 817 818 entry = q->entry[q->tail]; 819 q->entry[q->tail].done = false; 820 821 mt76_queue_tx_complete(dev, q, &entry); 822 } 823 824 if (!q->queued) 825 wake_up(&dev->tx_wait); 826 827 mt76_worker_schedule(&dev->tx_worker); 828 } 829 830 if (dev->drv->tx_status_data && 831 !test_and_set_bit(MT76_READING_STATS, &dev->phy.state)) 832 queue_work(dev->wq, &dev->usb.stat_work); 833 } 834 835 static void mt76u_tx_status_data(struct work_struct *work) 836 { 837 struct mt76_usb *usb; 838 struct mt76_dev *dev; 839 u8 update = 1; 840 u16 count = 0; 841 842 usb = container_of(work, struct mt76_usb, stat_work); 843 dev = container_of(usb, struct mt76_dev, usb); 844 845 while (true) { 846 if (test_bit(MT76_REMOVED, &dev->phy.state)) 847 break; 848 849 if (!dev->drv->tx_status_data(dev, &update)) 850 break; 851 count++; 852 } 853 854 if (count && test_bit(MT76_STATE_RUNNING, &dev->phy.state)) 855 queue_work(dev->wq, &usb->stat_work); 856 else 857 clear_bit(MT76_READING_STATS, &dev->phy.state); 858 } 859 860 static void mt76u_complete_tx(struct urb *urb) 861 { 862 struct mt76_dev *dev = dev_get_drvdata(&urb->dev->dev); 863 struct mt76_queue_entry *e = urb->context; 864 865 if (mt76u_urb_error(urb)) 866 dev_err(dev->dev, "tx urb failed: %d\n", urb->status); 867 e->done = true; 868 869 mt76_worker_schedule(&dev->usb.status_worker); 870 } 871 872 static int 873 mt76u_tx_setup_buffers(struct mt76_dev *dev, struct sk_buff *skb, 874 struct urb *urb) 875 { 876 urb->transfer_buffer_length = skb->len; 877 878 if (!dev->usb.sg_en) { 879 urb->transfer_buffer = skb->data; 880 return 0; 881 } 882 883 sg_init_table(urb->sg, MT_TX_SG_MAX_SIZE); 884 urb->num_sgs = skb_to_sgvec(skb, urb->sg, 0, skb->len); 885 if (!urb->num_sgs) 886 return -ENOMEM; 887 888 return urb->num_sgs; 889 } 890 891 static int 892 mt76u_tx_queue_skb(struct mt76_phy *phy, struct mt76_queue *q, 893 enum mt76_txq_id qid, struct sk_buff *skb, 894 struct mt76_wcid *wcid, struct ieee80211_sta *sta) 895 { 896 struct mt76_tx_info tx_info = { 897 .skb = skb, 898 }; 899 struct mt76_dev *dev = phy->dev; 900 u16 idx = q->head; 901 int err; 902 903 if (q->queued == q->ndesc) 904 return -ENOSPC; 905 906 skb->prev = skb->next = NULL; 907 err = dev->drv->tx_prepare_skb(dev, NULL, qid, wcid, sta, &tx_info); 908 if (err < 0) 909 return err; 910 911 err = mt76u_tx_setup_buffers(dev, tx_info.skb, q->entry[idx].urb); 912 if (err < 0) 913 return err; 914 915 mt76u_fill_bulk_urb(dev, USB_DIR_OUT, q->ep, q->entry[idx].urb, 916 mt76u_complete_tx, &q->entry[idx]); 917 918 q->head = (q->head + 1) % q->ndesc; 919 q->entry[idx].skb = tx_info.skb; 920 q->entry[idx].wcid = 0xffff; 921 q->queued++; 922 923 return idx; 924 } 925 926 static void mt76u_tx_kick(struct mt76_dev *dev, struct mt76_queue *q) 927 { 928 struct urb *urb; 929 int err; 930 931 while (q->first != q->head) { 932 urb = q->entry[q->first].urb; 933 934 trace_submit_urb(dev, urb); 935 err = usb_submit_urb(urb, GFP_ATOMIC); 936 if (err < 0) { 937 if (err == -ENODEV) 938 set_bit(MT76_REMOVED, &dev->phy.state); 939 else 940 dev_err(dev->dev, "tx urb submit failed:%d\n", 941 err); 942 break; 943 } 944 q->first = (q->first + 1) % q->ndesc; 945 } 946 } 947 948 static void 949 mt76u_ac_to_hwq(struct mt76_dev *dev, struct mt76_queue *q, u8 qid) 950 { 951 u8 ac = qid < IEEE80211_NUM_ACS ? qid : IEEE80211_AC_BE; 952 953 switch (mt76_chip(dev)) { 954 case 0x7663: { 955 static const u8 lmac_queue_map[] = { 956 /* ac to lmac mapping */ 957 [IEEE80211_AC_BK] = 0, 958 [IEEE80211_AC_BE] = 1, 959 [IEEE80211_AC_VI] = 2, 960 [IEEE80211_AC_VO] = 4, 961 }; 962 963 q->hw_idx = lmac_queue_map[ac]; 964 q->ep = q->hw_idx + 1; 965 break; 966 } 967 case 0x6639: 968 case 0x7902: 969 case 0x7961: 970 case 0x7925: 971 q->hw_idx = mt76_ac_to_hwq(ac); 972 q->ep = qid == MT_TXQ_PSD ? MT_EP_OUT_HCCA : q->hw_idx + 1; 973 break; 974 default: 975 q->hw_idx = mt76_ac_to_hwq(ac); 976 q->ep = q->hw_idx + 1; 977 break; 978 } 979 } 980 981 static int mt76u_alloc_tx(struct mt76_dev *dev) 982 { 983 int i; 984 985 for (i = 0; i <= MT_TXQ_PSD; i++) { 986 struct mt76_queue *q; 987 int j, err; 988 989 q = devm_kzalloc(dev->dev, sizeof(*q), GFP_KERNEL); 990 if (!q) 991 return -ENOMEM; 992 993 spin_lock_init(&q->lock); 994 mt76u_ac_to_hwq(dev, q, i); 995 dev->phy.q_tx[i] = q; 996 997 q->entry = devm_kcalloc(dev->dev, 998 MT_NUM_TX_ENTRIES, sizeof(*q->entry), 999 GFP_KERNEL); 1000 if (!q->entry) 1001 return -ENOMEM; 1002 1003 q->ndesc = MT_NUM_TX_ENTRIES; 1004 for (j = 0; j < q->ndesc; j++) { 1005 err = mt76u_urb_alloc(dev, &q->entry[j], 1006 MT_TX_SG_MAX_SIZE); 1007 if (err < 0) 1008 return err; 1009 } 1010 } 1011 return 0; 1012 } 1013 1014 static void mt76u_free_tx(struct mt76_dev *dev) 1015 { 1016 int i; 1017 1018 mt76_worker_teardown(&dev->usb.status_worker); 1019 1020 for (i = 0; i <= MT_TXQ_PSD; i++) { 1021 struct mt76_queue *q; 1022 int j; 1023 1024 q = dev->phy.q_tx[i]; 1025 if (!q) 1026 continue; 1027 1028 for (j = 0; j < q->ndesc; j++) { 1029 usb_free_urb(q->entry[j].urb); 1030 q->entry[j].urb = NULL; 1031 } 1032 } 1033 } 1034 1035 void mt76u_stop_tx(struct mt76_dev *dev) 1036 { 1037 int ret; 1038 1039 mt76_worker_disable(&dev->usb.status_worker); 1040 1041 ret = wait_event_timeout(dev->tx_wait, !mt76_has_tx_pending(&dev->phy), 1042 HZ / 5); 1043 if (!ret) { 1044 struct mt76_queue_entry entry; 1045 struct mt76_queue *q; 1046 int i, j; 1047 1048 dev_err(dev->dev, "timed out waiting for pending tx\n"); 1049 1050 for (i = 0; i <= MT_TXQ_PSD; i++) { 1051 q = dev->phy.q_tx[i]; 1052 if (!q) 1053 continue; 1054 1055 for (j = 0; j < q->ndesc; j++) 1056 usb_kill_urb(q->entry[j].urb); 1057 } 1058 1059 mt76_worker_disable(&dev->tx_worker); 1060 1061 /* On device removal we maight queue skb's, but mt76u_tx_kick() 1062 * will fail to submit urb, cleanup those skb's manually. 1063 */ 1064 for (i = 0; i <= MT_TXQ_PSD; i++) { 1065 q = dev->phy.q_tx[i]; 1066 if (!q) 1067 continue; 1068 1069 while (q->queued > 0) { 1070 entry = q->entry[q->tail]; 1071 q->entry[q->tail].done = false; 1072 mt76_queue_tx_complete(dev, q, &entry); 1073 } 1074 } 1075 1076 mt76_worker_enable(&dev->tx_worker); 1077 } 1078 1079 cancel_work_sync(&dev->usb.stat_work); 1080 clear_bit(MT76_READING_STATS, &dev->phy.state); 1081 1082 mt76_worker_enable(&dev->usb.status_worker); 1083 1084 mt76_tx_status_check(dev, true); 1085 } 1086 EXPORT_SYMBOL_GPL(mt76u_stop_tx); 1087 1088 void mt76u_queues_deinit(struct mt76_dev *dev) 1089 { 1090 mt76u_stop_rx(dev); 1091 mt76u_stop_tx(dev); 1092 1093 if (dev->napi_dev) { 1094 napi_disable(&dev->napi[MT_RXQ_MAIN]); 1095 netif_napi_del(&dev->napi[MT_RXQ_MAIN]); 1096 free_netdev(dev->napi_dev); 1097 dev->napi_dev = NULL; 1098 } 1099 1100 mt76u_free_rx(dev); 1101 mt76u_free_tx(dev); 1102 } 1103 EXPORT_SYMBOL_GPL(mt76u_queues_deinit); 1104 1105 int mt76u_alloc_queues(struct mt76_dev *dev) 1106 { 1107 int err; 1108 1109 err = mt76u_alloc_rx_queue(dev, MT_RXQ_MAIN); 1110 if (err < 0) 1111 return err; 1112 1113 return mt76u_alloc_tx(dev); 1114 } 1115 EXPORT_SYMBOL_GPL(mt76u_alloc_queues); 1116 1117 static const struct mt76_queue_ops usb_queue_ops = { 1118 .tx_queue_skb = mt76u_tx_queue_skb, 1119 .kick = mt76u_tx_kick, 1120 }; 1121 1122 int __mt76u_init(struct mt76_dev *dev, struct usb_interface *intf, 1123 struct mt76_bus_ops *ops) 1124 { 1125 struct usb_device *udev = interface_to_usbdev(intf); 1126 struct mt76_usb *usb = &dev->usb; 1127 struct mt76_dev **priv; 1128 int err; 1129 1130 INIT_WORK(&usb->stat_work, mt76u_tx_status_data); 1131 1132 usb->data_len = usb_maxpacket(udev, usb_sndctrlpipe(udev, 0)); 1133 if (usb->data_len < 32) 1134 usb->data_len = 32; 1135 1136 usb->data = devm_kmalloc(dev->dev, usb->data_len, GFP_KERNEL); 1137 if (!usb->data) 1138 return -ENOMEM; 1139 1140 mutex_init(&usb->usb_ctrl_mtx); 1141 dev->bus = ops; 1142 dev->queue_ops = &usb_queue_ops; 1143 1144 dev_set_drvdata(&udev->dev, dev); 1145 1146 usb->sg_en = mt76u_check_sg(dev); 1147 1148 err = mt76u_set_endpoints(intf, usb); 1149 if (err < 0) 1150 return err; 1151 1152 err = mt76_worker_setup(dev->hw, &usb->rx_worker, mt76u_rx_worker, 1153 "usb-rx"); 1154 if (err) 1155 return err; 1156 1157 err = mt76_worker_setup(dev->hw, &usb->status_worker, 1158 mt76u_status_worker, "usb-status"); 1159 if (err) 1160 return err; 1161 1162 sched_set_fifo_low(usb->rx_worker.task); 1163 sched_set_fifo_low(usb->status_worker.task); 1164 1165 /* threaded NAPI on a dummy netdev (reusing mt76_dev's napi_dev/napi[]) 1166 * services the MAIN RX queue and gives the RX path GRO 1167 */ 1168 dev->napi_dev = alloc_netdev_dummy(sizeof(struct mt76_dev *)); 1169 if (!dev->napi_dev) 1170 return -ENOMEM; 1171 1172 priv = netdev_priv(dev->napi_dev); 1173 *priv = dev; 1174 strscpy(dev->napi_dev->name, "mt76u-rx", sizeof(dev->napi_dev->name)); 1175 dev->napi_dev->threaded = 1; 1176 netif_napi_add(dev->napi_dev, &dev->napi[MT_RXQ_MAIN], mt76u_napi_poll); 1177 napi_enable(&dev->napi[MT_RXQ_MAIN]); 1178 1179 return 0; 1180 } 1181 EXPORT_SYMBOL_GPL(__mt76u_init); 1182 1183 int mt76u_init(struct mt76_dev *dev, struct usb_interface *intf) 1184 { 1185 static struct mt76_bus_ops bus_ops = { 1186 .rr = mt76u_rr, 1187 .wr = mt76u_wr, 1188 .rmw = mt76u_rmw, 1189 .read_copy = mt76u_read_copy, 1190 .write_copy = mt76u_copy, 1191 .wr_rp = mt76u_wr_rp, 1192 .rd_rp = mt76u_rd_rp, 1193 .type = MT76_BUS_USB, 1194 }; 1195 1196 return __mt76u_init(dev, intf, &bus_ops); 1197 } 1198 EXPORT_SYMBOL_GPL(mt76u_init); 1199 1200 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo.bianconi83@gmail.com>"); 1201 MODULE_DESCRIPTION("MediaTek MT76x USB helpers"); 1202 MODULE_LICENSE("Dual BSD/GPL"); 1203