1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cadence USBSS DRD Driver - gadget side. 4 * 5 * Copyright (C) 2018-2019 Cadence Design Systems. 6 * Copyright (C) 2017-2018 NXP 7 * 8 * Authors: Pawel Jez <pjez@cadence.com>, 9 * Pawel Laszczak <pawell@cadence.com> 10 * Peter Chen <peter.chen@nxp.com> 11 */ 12 13 /* 14 * Work around 1: 15 * At some situations, the controller may get stale data address in TRB 16 * at below sequences: 17 * 1. Controller read TRB includes data address 18 * 2. Software updates TRBs includes data address and Cycle bit 19 * 3. Controller read TRB which includes Cycle bit 20 * 4. DMA run with stale data address 21 * 22 * To fix this problem, driver needs to make the first TRB in TD as invalid. 23 * After preparing all TRBs driver needs to check the position of DMA and 24 * if the DMA point to the first just added TRB and doorbell is 1, 25 * then driver must defer making this TRB as valid. This TRB will be make 26 * as valid during adding next TRB only if DMA is stopped or at TRBERR 27 * interrupt. 28 * 29 * Issue has been fixed in DEV_VER_V3 version of controller. 30 * 31 * Work around 2: 32 * Controller for OUT endpoints has shared on-chip buffers for all incoming 33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA 34 * in correct order. If the first packet in the buffer will not be handled, 35 * then the following packets directed for other endpoints and functions 36 * will be blocked. 37 * Additionally the packets directed to one endpoint can block entire on-chip 38 * buffers. In this case transfer to other endpoints also will blocked. 39 * 40 * To resolve this issue after raising the descriptor missing interrupt 41 * driver prepares internal usb_request object and use it to arm DMA transfer. 42 * 43 * The problematic situation was observed in case when endpoint has been enabled 44 * but no usb_request were queued. Driver try detects such endpoints and will 45 * use this workaround only for these endpoint. 46 * 47 * Driver use limited number of buffer. This number can be set by macro 48 * CDNS3_WA2_NUM_BUFFERS. 49 * 50 * Such blocking situation was observed on ACM gadget. For this function 51 * host send OUT data packet but ACM function is not prepared for this packet. 52 * It's cause that buffer placed in on chip memory block transfer to other 53 * endpoints. 54 * 55 * Issue has been fixed in DEV_VER_V2 version of controller. 56 * 57 */ 58 59 #include <linux/dma-mapping.h> 60 #include <linux/usb/gadget.h> 61 #include <linux/module.h> 62 #include <linux/dmapool.h> 63 #include <linux/iopoll.h> 64 65 #include "core.h" 66 #include "gadget-export.h" 67 #include "cdns3-gadget.h" 68 #include "cdns3-trace.h" 69 #include "drd.h" 70 71 static int __cdns3_gadget_ep_queue(struct usb_ep *ep, 72 struct usb_request *request, 73 gfp_t gfp_flags); 74 75 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, 76 struct usb_request *request); 77 78 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep, 79 struct usb_request *request); 80 81 /** 82 * cdns3_clear_register_bit - clear bit in given register. 83 * @ptr: address of device controller register to be read and changed 84 * @mask: bits requested to clar 85 */ 86 static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask) 87 { 88 mask = readl(ptr) & ~mask; 89 writel(mask, ptr); 90 } 91 92 /** 93 * cdns3_set_register_bit - set bit in given register. 94 * @ptr: address of device controller register to be read and changed 95 * @mask: bits requested to set 96 */ 97 void cdns3_set_register_bit(void __iomem *ptr, u32 mask) 98 { 99 mask = readl(ptr) | mask; 100 writel(mask, ptr); 101 } 102 103 /** 104 * cdns3_ep_addr_to_index - Macro converts endpoint address to 105 * index of endpoint object in cdns3_device.eps[] container 106 * @ep_addr: endpoint address for which endpoint object is required 107 * 108 */ 109 u8 cdns3_ep_addr_to_index(u8 ep_addr) 110 { 111 return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0)); 112 } 113 114 static int cdns3_get_dma_pos(struct cdns3_device *priv_dev, 115 struct cdns3_endpoint *priv_ep) 116 { 117 int dma_index; 118 119 dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma; 120 121 return dma_index / TRB_SIZE; 122 } 123 124 /** 125 * cdns3_next_request - returns next request from list 126 * @list: list containing requests 127 * 128 * Returns request or NULL if no requests in list 129 */ 130 struct usb_request *cdns3_next_request(struct list_head *list) 131 { 132 return list_first_entry_or_null(list, struct usb_request, list); 133 } 134 135 /** 136 * cdns3_next_align_buf - returns next buffer from list 137 * @list: list containing buffers 138 * 139 * Returns buffer or NULL if no buffers in list 140 */ 141 static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list) 142 { 143 return list_first_entry_or_null(list, struct cdns3_aligned_buf, list); 144 } 145 146 /** 147 * cdns3_next_priv_request - returns next request from list 148 * @list: list containing requests 149 * 150 * Returns request or NULL if no requests in list 151 */ 152 static struct cdns3_request *cdns3_next_priv_request(struct list_head *list) 153 { 154 return list_first_entry_or_null(list, struct cdns3_request, list); 155 } 156 157 /** 158 * cdns3_select_ep - selects endpoint 159 * @priv_dev: extended gadget object 160 * @ep: endpoint address 161 */ 162 void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep) 163 { 164 if (priv_dev->selected_ep == ep) 165 return; 166 167 priv_dev->selected_ep = ep; 168 writel(ep, &priv_dev->regs->ep_sel); 169 } 170 171 /** 172 * cdns3_get_tdl - gets current tdl for selected endpoint. 173 * @priv_dev: extended gadget object 174 * 175 * Before calling this function the appropriate endpoint must 176 * be selected by means of cdns3_select_ep function. 177 */ 178 static int cdns3_get_tdl(struct cdns3_device *priv_dev) 179 { 180 if (priv_dev->dev_ver < DEV_VER_V3) 181 return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); 182 else 183 return readl(&priv_dev->regs->ep_tdl); 184 } 185 186 dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep, 187 struct cdns3_trb *trb) 188 { 189 u32 offset = (char *)trb - (char *)priv_ep->trb_pool; 190 191 return priv_ep->trb_pool_dma + offset; 192 } 193 194 static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep) 195 { 196 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 197 198 if (priv_ep->trb_pool) { 199 dma_pool_free(priv_dev->eps_dma_pool, 200 priv_ep->trb_pool, priv_ep->trb_pool_dma); 201 priv_ep->trb_pool = NULL; 202 } 203 } 204 205 /** 206 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint 207 * @priv_ep: endpoint object 208 * 209 * Function will return 0 on success or -ENOMEM on allocation error 210 */ 211 int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep) 212 { 213 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 214 int ring_size = TRB_RING_SIZE; 215 int num_trbs = ring_size / TRB_SIZE; 216 struct cdns3_trb *link_trb; 217 218 if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size) 219 cdns3_free_trb_pool(priv_ep); 220 221 if (!priv_ep->trb_pool) { 222 priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool, 223 GFP_DMA32 | GFP_ATOMIC, 224 &priv_ep->trb_pool_dma); 225 226 if (!priv_ep->trb_pool) 227 return -ENOMEM; 228 229 priv_ep->alloc_ring_size = ring_size; 230 } 231 232 memset(priv_ep->trb_pool, 0, ring_size); 233 234 priv_ep->num_trbs = num_trbs; 235 236 if (!priv_ep->num) 237 return 0; 238 239 /* Initialize the last TRB as Link TRB */ 240 link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1)); 241 242 if (priv_ep->use_streams) { 243 /* 244 * For stream capable endpoints driver use single correct TRB. 245 * The last trb has zeroed cycle bit 246 */ 247 link_trb->control = 0; 248 } else { 249 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma)); 250 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE); 251 } 252 return 0; 253 } 254 255 /** 256 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint 257 * @priv_ep: endpoint object 258 * 259 * Endpoint must be selected before call to this function 260 */ 261 static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep) 262 { 263 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 264 int val; 265 266 trace_cdns3_halt(priv_ep, 1, 1); 267 268 writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL, 269 &priv_dev->regs->ep_cmd); 270 271 /* wait for DFLUSH cleared */ 272 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, 273 !(val & EP_CMD_DFLUSH), 1, 1000); 274 priv_ep->flags |= EP_STALLED; 275 priv_ep->flags &= ~EP_STALL_PENDING; 276 } 277 278 /** 279 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller. 280 * @priv_dev: extended gadget object 281 */ 282 void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev) 283 { 284 int i; 285 286 writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf); 287 288 cdns3_allow_enable_l1(priv_dev, 0); 289 priv_dev->hw_configured_flag = 0; 290 priv_dev->onchip_used_size = 0; 291 priv_dev->out_mem_is_allocated = 0; 292 priv_dev->wait_for_setup = 0; 293 priv_dev->using_streams = 0; 294 295 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) 296 if (priv_dev->eps[i]) 297 priv_dev->eps[i]->flags &= ~EP_CONFIGURED; 298 } 299 300 /** 301 * cdns3_ep_inc_trb - increment a trb index. 302 * @index: Pointer to the TRB index to increment. 303 * @cs: Cycle state 304 * @trb_in_seg: number of TRBs in segment 305 * 306 * The index should never point to the link TRB. After incrementing, 307 * if it is point to the link TRB, wrap around to the beginning and revert 308 * cycle state bit The 309 * link TRB is always at the last TRB entry. 310 */ 311 static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg) 312 { 313 (*index)++; 314 if (*index == (trb_in_seg - 1)) { 315 *index = 0; 316 *cs ^= 1; 317 } 318 } 319 320 /** 321 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer 322 * @priv_ep: The endpoint whose enqueue pointer we're incrementing 323 */ 324 static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep) 325 { 326 priv_ep->free_trbs--; 327 cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs); 328 } 329 330 /** 331 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer 332 * @priv_ep: The endpoint whose dequeue pointer we're incrementing 333 */ 334 static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep) 335 { 336 priv_ep->free_trbs++; 337 cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs); 338 } 339 340 static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req) 341 { 342 struct cdns3_endpoint *priv_ep = priv_req->priv_ep; 343 int current_trb = priv_req->start_trb; 344 345 while (current_trb != priv_req->end_trb) { 346 cdns3_ep_inc_deq(priv_ep); 347 current_trb = priv_ep->dequeue; 348 } 349 350 cdns3_ep_inc_deq(priv_ep); 351 } 352 353 /** 354 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1. 355 * @priv_dev: Extended gadget object 356 * @enable: Enable/disable permit to transition to L1. 357 * 358 * If bit USB_CONF_L1EN is set and device receive Extended Token packet, 359 * then controller answer with ACK handshake. 360 * If bit USB_CONF_L1DS is set and device receive Extended Token packet, 361 * then controller answer with NYET handshake. 362 */ 363 void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable) 364 { 365 if (enable) 366 writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf); 367 else 368 writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf); 369 } 370 371 enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev) 372 { 373 u32 reg; 374 375 reg = readl(&priv_dev->regs->usb_sts); 376 377 if (DEV_SUPERSPEED(reg)) 378 return USB_SPEED_SUPER; 379 else if (DEV_HIGHSPEED(reg)) 380 return USB_SPEED_HIGH; 381 else if (DEV_FULLSPEED(reg)) 382 return USB_SPEED_FULL; 383 else if (DEV_LOWSPEED(reg)) 384 return USB_SPEED_LOW; 385 return USB_SPEED_UNKNOWN; 386 } 387 388 /** 389 * cdns3_start_all_request - add to ring all request not started 390 * @priv_dev: Extended gadget object 391 * @priv_ep: The endpoint for whom request will be started. 392 * 393 * Returns return ENOMEM if transfer ring i not enough TRBs to start 394 * all requests. 395 */ 396 static int cdns3_start_all_request(struct cdns3_device *priv_dev, 397 struct cdns3_endpoint *priv_ep) 398 { 399 struct usb_request *request; 400 int ret = 0; 401 u8 pending_empty = list_empty(&priv_ep->pending_req_list); 402 403 /* 404 * If the last pending transfer is INTERNAL 405 * OR streams are enabled for this endpoint 406 * do NOT start new transfer till the last one is pending 407 */ 408 if (!pending_empty) { 409 struct cdns3_request *priv_req; 410 411 request = cdns3_next_request(&priv_ep->pending_req_list); 412 priv_req = to_cdns3_request(request); 413 if ((priv_req->flags & REQUEST_INTERNAL) || 414 (priv_ep->flags & EP_TDLCHK_EN) || 415 priv_ep->use_streams) { 416 dev_dbg(priv_dev->dev, "Blocking external request\n"); 417 return ret; 418 } 419 } 420 421 while (!list_empty(&priv_ep->deferred_req_list)) { 422 request = cdns3_next_request(&priv_ep->deferred_req_list); 423 424 if (!priv_ep->use_streams) { 425 ret = cdns3_ep_run_transfer(priv_ep, request); 426 } else { 427 priv_ep->stream_sg_idx = 0; 428 ret = cdns3_ep_run_stream_transfer(priv_ep, request); 429 } 430 if (ret) 431 return ret; 432 433 list_move_tail(&request->list, &priv_ep->pending_req_list); 434 if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN)) 435 break; 436 } 437 438 priv_ep->flags &= ~EP_RING_FULL; 439 return ret; 440 } 441 442 /* 443 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set 444 * driver try to detect whether endpoint need additional internal 445 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared 446 * if before first DESCMISS interrupt the DMA will be armed. 447 */ 448 #define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \ 449 if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \ 450 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \ 451 (reg) |= EP_STS_EN_DESCMISEN; \ 452 } } while (0) 453 454 static void __cdns3_descmiss_copy_data(struct usb_request *request, 455 struct usb_request *descmiss_req) 456 { 457 int length = request->actual + descmiss_req->actual; 458 struct scatterlist *s = request->sg; 459 460 if (!s) { 461 if (length <= request->length) { 462 memcpy(&((u8 *)request->buf)[request->actual], 463 descmiss_req->buf, 464 descmiss_req->actual); 465 request->actual = length; 466 } else { 467 /* It should never occures */ 468 request->status = -ENOMEM; 469 } 470 } else { 471 if (length <= sg_dma_len(s)) { 472 void *p = phys_to_virt(sg_dma_address(s)); 473 474 memcpy(&((u8 *)p)[request->actual], 475 descmiss_req->buf, 476 descmiss_req->actual); 477 request->actual = length; 478 } else { 479 request->status = -ENOMEM; 480 } 481 } 482 } 483 484 /** 485 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to 486 * request queued by class driver. 487 * @priv_ep: extended endpoint object 488 * @request: request object 489 */ 490 static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep, 491 struct usb_request *request) 492 { 493 struct usb_request *descmiss_req; 494 struct cdns3_request *descmiss_priv_req; 495 496 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { 497 int chunk_end; 498 499 descmiss_priv_req = 500 cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); 501 descmiss_req = &descmiss_priv_req->request; 502 503 /* driver can't touch pending request */ 504 if (descmiss_priv_req->flags & REQUEST_PENDING) 505 break; 506 507 chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH; 508 request->status = descmiss_req->status; 509 __cdns3_descmiss_copy_data(request, descmiss_req); 510 list_del_init(&descmiss_priv_req->list); 511 kfree(descmiss_req->buf); 512 cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req); 513 --priv_ep->wa2_counter; 514 515 if (!chunk_end) 516 break; 517 } 518 } 519 520 static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev, 521 struct cdns3_endpoint *priv_ep, 522 struct cdns3_request *priv_req) 523 { 524 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN && 525 priv_req->flags & REQUEST_INTERNAL) { 526 struct usb_request *req; 527 528 req = cdns3_next_request(&priv_ep->deferred_req_list); 529 530 priv_ep->descmis_req = NULL; 531 532 if (!req) 533 return NULL; 534 535 /* unmap the gadget request before copying data */ 536 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req, 537 priv_ep->dir); 538 539 cdns3_wa2_descmiss_copy_data(priv_ep, req); 540 if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) && 541 req->length != req->actual) { 542 /* wait for next part of transfer */ 543 /* re-map the gadget request buffer*/ 544 usb_gadget_map_request_by_dev(priv_dev->sysdev, req, 545 usb_endpoint_dir_in(priv_ep->endpoint.desc)); 546 return NULL; 547 } 548 549 if (req->status == -EINPROGRESS) 550 req->status = 0; 551 552 list_del_init(&req->list); 553 cdns3_start_all_request(priv_dev, priv_ep); 554 return req; 555 } 556 557 return &priv_req->request; 558 } 559 560 static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev, 561 struct cdns3_endpoint *priv_ep, 562 struct cdns3_request *priv_req) 563 { 564 int deferred = 0; 565 566 /* 567 * If transfer was queued before DESCMISS appear than we 568 * can disable handling of DESCMISS interrupt. Driver assumes that it 569 * can disable special treatment for this endpoint. 570 */ 571 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { 572 u32 reg; 573 574 cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir); 575 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; 576 reg = readl(&priv_dev->regs->ep_sts_en); 577 reg &= ~EP_STS_EN_DESCMISEN; 578 trace_cdns3_wa2(priv_ep, "workaround disabled\n"); 579 writel(reg, &priv_dev->regs->ep_sts_en); 580 } 581 582 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { 583 u8 pending_empty = list_empty(&priv_ep->pending_req_list); 584 u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list); 585 586 /* 587 * DESCMISS transfer has been finished, so data will be 588 * directly copied from internal allocated usb_request 589 * objects. 590 */ 591 if (pending_empty && !descmiss_empty && 592 !(priv_req->flags & REQUEST_INTERNAL)) { 593 cdns3_wa2_descmiss_copy_data(priv_ep, 594 &priv_req->request); 595 596 trace_cdns3_wa2(priv_ep, "get internal stored data"); 597 598 list_add_tail(&priv_req->request.list, 599 &priv_ep->pending_req_list); 600 cdns3_gadget_giveback(priv_ep, priv_req, 601 priv_req->request.status); 602 603 /* 604 * Intentionally driver returns positive value as 605 * correct value. It informs that transfer has 606 * been finished. 607 */ 608 return EINPROGRESS; 609 } 610 611 /* 612 * Driver will wait for completion DESCMISS transfer, 613 * before starts new, not DESCMISS transfer. 614 */ 615 if (!pending_empty && !descmiss_empty) { 616 trace_cdns3_wa2(priv_ep, "wait for pending transfer\n"); 617 deferred = 1; 618 } 619 620 if (priv_req->flags & REQUEST_INTERNAL) 621 list_add_tail(&priv_req->list, 622 &priv_ep->wa2_descmiss_req_list); 623 } 624 625 return deferred; 626 } 627 628 static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep) 629 { 630 struct cdns3_request *priv_req; 631 632 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { 633 u8 chain; 634 635 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); 636 chain = !!(priv_req->flags & REQUEST_INTERNAL_CH); 637 638 trace_cdns3_wa2(priv_ep, "removes eldest request"); 639 640 kfree(priv_req->request.buf); 641 cdns3_gadget_ep_free_request(&priv_ep->endpoint, 642 &priv_req->request); 643 list_del_init(&priv_req->list); 644 --priv_ep->wa2_counter; 645 646 if (!chain) 647 break; 648 } 649 } 650 651 /** 652 * cdns3_wa2_descmissing_packet - handles descriptor missing event. 653 * @priv_ep: extended gadget object 654 * 655 * This function is used only for WA2. For more information see Work around 2 656 * description. 657 */ 658 static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep) 659 { 660 struct cdns3_request *priv_req; 661 struct usb_request *request; 662 u8 pending_empty = list_empty(&priv_ep->pending_req_list); 663 664 /* check for pending transfer */ 665 if (!pending_empty) { 666 trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n"); 667 return; 668 } 669 670 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) { 671 priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET; 672 priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN; 673 } 674 675 trace_cdns3_wa2(priv_ep, "Description Missing detected\n"); 676 677 if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) { 678 trace_cdns3_wa2(priv_ep, "WA2 overflow\n"); 679 cdns3_wa2_remove_old_request(priv_ep); 680 } 681 682 request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint, 683 GFP_ATOMIC); 684 if (!request) 685 goto err; 686 687 priv_req = to_cdns3_request(request); 688 priv_req->flags |= REQUEST_INTERNAL; 689 690 /* if this field is still assigned it indicate that transfer related 691 * with this request has not been finished yet. Driver in this 692 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH 693 * flag to previous one. It will indicate that current request is 694 * part of the previous one. 695 */ 696 if (priv_ep->descmis_req) 697 priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH; 698 699 priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE, 700 GFP_ATOMIC); 701 priv_ep->wa2_counter++; 702 703 if (!priv_req->request.buf) { 704 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); 705 goto err; 706 } 707 708 priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE; 709 priv_ep->descmis_req = priv_req; 710 711 __cdns3_gadget_ep_queue(&priv_ep->endpoint, 712 &priv_ep->descmis_req->request, 713 GFP_ATOMIC); 714 715 return; 716 717 err: 718 dev_err(priv_ep->cdns3_dev->dev, 719 "Failed: No sufficient memory for DESCMIS\n"); 720 } 721 722 static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev) 723 { 724 u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); 725 726 if (tdl) { 727 u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl; 728 729 writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL, 730 &priv_dev->regs->ep_cmd); 731 } 732 } 733 734 static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev) 735 { 736 u32 ep_sts_reg; 737 738 /* select EP0-out */ 739 cdns3_select_ep(priv_dev, 0); 740 741 ep_sts_reg = readl(&priv_dev->regs->ep_sts); 742 743 if (EP_STS_OUTQ_VAL(ep_sts_reg)) { 744 u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg); 745 struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num]; 746 747 if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) && 748 outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) { 749 u8 pending_empty = list_empty(&outq_ep->pending_req_list); 750 751 if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) || 752 (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) || 753 !pending_empty) { 754 } else { 755 u32 ep_sts_en_reg; 756 u32 ep_cmd_reg; 757 758 cdns3_select_ep(priv_dev, outq_ep->num | 759 outq_ep->dir); 760 ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en); 761 ep_cmd_reg = readl(&priv_dev->regs->ep_cmd); 762 763 outq_ep->flags |= EP_TDLCHK_EN; 764 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, 765 EP_CFG_TDL_CHK); 766 767 cdns3_wa2_enable_detection(priv_dev, outq_ep, 768 ep_sts_en_reg); 769 writel(ep_sts_en_reg, 770 &priv_dev->regs->ep_sts_en); 771 /* reset tdl value to zero */ 772 cdns3_wa2_reset_tdl(priv_dev); 773 /* 774 * Memory barrier - Reset tdl before ringing the 775 * doorbell. 776 */ 777 wmb(); 778 if (EP_CMD_DRDY & ep_cmd_reg) { 779 trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n"); 780 781 } else { 782 trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n"); 783 /* 784 * ring doorbell to generate DESCMIS irq 785 */ 786 writel(EP_CMD_DRDY, 787 &priv_dev->regs->ep_cmd); 788 } 789 } 790 } 791 } 792 } 793 794 /** 795 * cdns3_gadget_giveback - call struct usb_request's ->complete callback 796 * @priv_ep: The endpoint to whom the request belongs to 797 * @priv_req: The request we're giving back 798 * @status: completion code for the request 799 * 800 * Must be called with controller's lock held and interrupts disabled. This 801 * function will unmap @req and call its ->complete() callback to notify upper 802 * layers that it has completed. 803 */ 804 void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep, 805 struct cdns3_request *priv_req, 806 int status) 807 { 808 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 809 struct usb_request *request = &priv_req->request; 810 811 list_del_init(&request->list); 812 813 if (request->status == -EINPROGRESS) 814 request->status = status; 815 816 usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request, 817 priv_ep->dir); 818 819 if ((priv_req->flags & REQUEST_UNALIGNED) && 820 priv_ep->dir == USB_DIR_OUT && !request->status) { 821 /* Make DMA buffer CPU accessible */ 822 dma_sync_single_for_cpu(priv_dev->sysdev, 823 priv_req->aligned_buf->dma, 824 priv_req->aligned_buf->size, 825 priv_req->aligned_buf->dir); 826 memcpy(request->buf, priv_req->aligned_buf->buf, 827 request->length); 828 } 829 830 priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED); 831 /* All TRBs have finished, clear the counter */ 832 priv_req->finished_trb = 0; 833 trace_cdns3_gadget_giveback(priv_req); 834 835 if (priv_dev->dev_ver < DEV_VER_V2) { 836 request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep, 837 priv_req); 838 if (!request) 839 return; 840 } 841 842 if (request->complete) { 843 spin_unlock(&priv_dev->lock); 844 usb_gadget_giveback_request(&priv_ep->endpoint, 845 request); 846 spin_lock(&priv_dev->lock); 847 } 848 849 if (request->buf == priv_dev->zlp_buf) 850 cdns3_gadget_ep_free_request(&priv_ep->endpoint, request); 851 } 852 853 static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep) 854 { 855 /* Work around for stale data address in TRB*/ 856 if (priv_ep->wa1_set) { 857 trace_cdns3_wa1(priv_ep, "restore cycle bit"); 858 859 priv_ep->wa1_set = 0; 860 priv_ep->wa1_trb_index = 0xFFFF; 861 if (priv_ep->wa1_cycle_bit) { 862 priv_ep->wa1_trb->control = 863 priv_ep->wa1_trb->control | cpu_to_le32(0x1); 864 } else { 865 priv_ep->wa1_trb->control = 866 priv_ep->wa1_trb->control & cpu_to_le32(~0x1); 867 } 868 } 869 } 870 871 static void cdns3_free_aligned_request_buf(struct work_struct *work) 872 { 873 struct cdns3_device *priv_dev = container_of(work, struct cdns3_device, 874 aligned_buf_wq); 875 struct cdns3_aligned_buf *buf, *tmp; 876 unsigned long flags; 877 878 spin_lock_irqsave(&priv_dev->lock, flags); 879 880 list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) { 881 if (!buf->in_use) { 882 list_del(&buf->list); 883 884 /* 885 * Re-enable interrupts to free DMA capable memory. 886 * Driver can't free this memory with disabled 887 * interrupts. 888 */ 889 spin_unlock_irqrestore(&priv_dev->lock, flags); 890 dma_free_noncoherent(priv_dev->sysdev, buf->size, 891 buf->buf, buf->dma, buf->dir); 892 kfree(buf); 893 spin_lock_irqsave(&priv_dev->lock, flags); 894 } 895 } 896 897 spin_unlock_irqrestore(&priv_dev->lock, flags); 898 } 899 900 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req) 901 { 902 struct cdns3_endpoint *priv_ep = priv_req->priv_ep; 903 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 904 struct cdns3_aligned_buf *buf; 905 906 /* check if buffer is aligned to 8. */ 907 if (!((uintptr_t)priv_req->request.buf & 0x7)) 908 return 0; 909 910 buf = priv_req->aligned_buf; 911 912 if (!buf || priv_req->request.length > buf->size) { 913 buf = kzalloc(sizeof(*buf), GFP_ATOMIC); 914 if (!buf) 915 return -ENOMEM; 916 917 buf->size = priv_req->request.length; 918 buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ? 919 DMA_TO_DEVICE : DMA_FROM_DEVICE; 920 921 buf->buf = dma_alloc_noncoherent(priv_dev->sysdev, 922 buf->size, 923 &buf->dma, 924 buf->dir, 925 GFP_ATOMIC); 926 if (!buf->buf) { 927 kfree(buf); 928 return -ENOMEM; 929 } 930 931 if (priv_req->aligned_buf) { 932 trace_cdns3_free_aligned_request(priv_req); 933 priv_req->aligned_buf->in_use = 0; 934 queue_work(system_freezable_wq, 935 &priv_dev->aligned_buf_wq); 936 } 937 938 buf->in_use = 1; 939 priv_req->aligned_buf = buf; 940 941 list_add_tail(&buf->list, 942 &priv_dev->aligned_buf_list); 943 } 944 945 if (priv_ep->dir == USB_DIR_IN) { 946 /* Make DMA buffer CPU accessible */ 947 dma_sync_single_for_cpu(priv_dev->sysdev, 948 buf->dma, buf->size, buf->dir); 949 memcpy(buf->buf, priv_req->request.buf, 950 priv_req->request.length); 951 } 952 953 /* Transfer DMA buffer ownership back to device */ 954 dma_sync_single_for_device(priv_dev->sysdev, 955 buf->dma, buf->size, buf->dir); 956 957 priv_req->flags |= REQUEST_UNALIGNED; 958 trace_cdns3_prepare_aligned_request(priv_req); 959 960 return 0; 961 } 962 963 static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep, 964 struct cdns3_trb *trb) 965 { 966 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 967 968 if (!priv_ep->wa1_set) { 969 u32 doorbell; 970 971 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); 972 973 if (doorbell) { 974 priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0; 975 priv_ep->wa1_set = 1; 976 priv_ep->wa1_trb = trb; 977 priv_ep->wa1_trb_index = priv_ep->enqueue; 978 trace_cdns3_wa1(priv_ep, "set guard"); 979 return 0; 980 } 981 } 982 return 1; 983 } 984 985 static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev, 986 struct cdns3_endpoint *priv_ep) 987 { 988 int dma_index; 989 u32 doorbell; 990 991 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); 992 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep); 993 994 if (!doorbell || dma_index != priv_ep->wa1_trb_index) 995 cdns3_wa1_restore_cycle_bit(priv_ep); 996 } 997 998 static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep, 999 struct usb_request *request) 1000 { 1001 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 1002 struct cdns3_request *priv_req; 1003 struct cdns3_trb *trb; 1004 dma_addr_t trb_dma; 1005 int address; 1006 u32 control; 1007 u32 length; 1008 u32 tdl; 1009 unsigned int sg_idx = priv_ep->stream_sg_idx; 1010 1011 priv_req = to_cdns3_request(request); 1012 address = priv_ep->endpoint.desc->bEndpointAddress; 1013 1014 priv_ep->flags |= EP_PENDING_REQUEST; 1015 1016 /* must allocate buffer aligned to 8 */ 1017 if (priv_req->flags & REQUEST_UNALIGNED) 1018 trb_dma = priv_req->aligned_buf->dma; 1019 else 1020 trb_dma = request->dma; 1021 1022 /* For stream capable endpoints driver use only single TD. */ 1023 trb = priv_ep->trb_pool + priv_ep->enqueue; 1024 priv_req->start_trb = priv_ep->enqueue; 1025 priv_req->end_trb = priv_req->start_trb; 1026 priv_req->trb = trb; 1027 1028 cdns3_select_ep(priv_ep->cdns3_dev, address); 1029 1030 control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE | 1031 TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP; 1032 1033 if (!request->num_sgs) { 1034 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); 1035 length = request->length; 1036 } else { 1037 trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address)); 1038 length = request->sg[sg_idx].length; 1039 } 1040 1041 tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket); 1042 1043 trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length)); 1044 1045 /* 1046 * For DEV_VER_V2 controller version we have enabled 1047 * USB_CONF2_EN_TDL_TRB in DMULT configuration. 1048 * This enables TDL calculation based on TRB, hence setting TDL in TRB. 1049 */ 1050 if (priv_dev->dev_ver >= DEV_VER_V2) { 1051 if (priv_dev->gadget.speed == USB_SPEED_SUPER) 1052 trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl)); 1053 } 1054 priv_req->flags |= REQUEST_PENDING; 1055 1056 trb->control = cpu_to_le32(control); 1057 1058 trace_cdns3_prepare_trb(priv_ep, priv_req->trb); 1059 1060 /* 1061 * Memory barrier - Cycle Bit must be set before trb->length and 1062 * trb->buffer fields. 1063 */ 1064 wmb(); 1065 1066 /* always first element */ 1067 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma), 1068 &priv_dev->regs->ep_traddr); 1069 1070 if (!(priv_ep->flags & EP_STALLED)) { 1071 trace_cdns3_ring(priv_ep); 1072 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/ 1073 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); 1074 1075 priv_ep->prime_flag = false; 1076 1077 /* 1078 * Controller version DEV_VER_V2 tdl calculation 1079 * is based on TRB 1080 */ 1081 1082 if (priv_dev->dev_ver < DEV_VER_V2) 1083 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, 1084 &priv_dev->regs->ep_cmd); 1085 else if (priv_dev->dev_ver > DEV_VER_V2) 1086 writel(tdl, &priv_dev->regs->ep_tdl); 1087 1088 priv_ep->last_stream_id = priv_req->request.stream_id; 1089 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); 1090 writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) | 1091 EP_CMD_ERDY, &priv_dev->regs->ep_cmd); 1092 1093 trace_cdns3_doorbell_epx(priv_ep->name, 1094 readl(&priv_dev->regs->ep_traddr)); 1095 } 1096 1097 /* WORKAROUND for transition to L0 */ 1098 __cdns3_gadget_wakeup(priv_dev); 1099 1100 return 0; 1101 } 1102 1103 /** 1104 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware 1105 * @priv_ep: endpoint object 1106 * @request: request object 1107 * 1108 * Returns zero on success or negative value on failure 1109 */ 1110 static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep, 1111 struct usb_request *request) 1112 { 1113 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 1114 struct cdns3_request *priv_req; 1115 struct cdns3_trb *trb; 1116 struct cdns3_trb *link_trb = NULL; 1117 dma_addr_t trb_dma; 1118 u32 togle_pcs = 1; 1119 int sg_iter = 0; 1120 int num_trb; 1121 int address; 1122 u32 control; 1123 int pcs; 1124 u16 total_tdl = 0; 1125 struct scatterlist *s = NULL; 1126 bool sg_supported = !!(request->num_mapped_sgs); 1127 1128 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) 1129 num_trb = priv_ep->interval; 1130 else 1131 num_trb = sg_supported ? request->num_mapped_sgs : 1; 1132 1133 if (num_trb > priv_ep->free_trbs) { 1134 priv_ep->flags |= EP_RING_FULL; 1135 return -ENOBUFS; 1136 } 1137 1138 priv_req = to_cdns3_request(request); 1139 address = priv_ep->endpoint.desc->bEndpointAddress; 1140 1141 priv_ep->flags |= EP_PENDING_REQUEST; 1142 1143 /* must allocate buffer aligned to 8 */ 1144 if (priv_req->flags & REQUEST_UNALIGNED) 1145 trb_dma = priv_req->aligned_buf->dma; 1146 else 1147 trb_dma = request->dma; 1148 1149 trb = priv_ep->trb_pool + priv_ep->enqueue; 1150 priv_req->start_trb = priv_ep->enqueue; 1151 priv_req->trb = trb; 1152 1153 cdns3_select_ep(priv_ep->cdns3_dev, address); 1154 1155 /* prepare ring */ 1156 if ((priv_ep->enqueue + num_trb) >= (priv_ep->num_trbs - 1)) { 1157 int doorbell, dma_index; 1158 u32 ch_bit = 0; 1159 1160 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); 1161 dma_index = cdns3_get_dma_pos(priv_dev, priv_ep); 1162 1163 /* Driver can't update LINK TRB if it is current processed. */ 1164 if (doorbell && dma_index == priv_ep->num_trbs - 1) { 1165 priv_ep->flags |= EP_DEFERRED_DRDY; 1166 return -ENOBUFS; 1167 } 1168 1169 /*updating C bt in Link TRB before starting DMA*/ 1170 link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1); 1171 /* 1172 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes 1173 * that DMA stuck at the LINK TRB. 1174 * On the other hand, removing TRB_CHAIN for longer TRs for 1175 * epXout cause that DMA stuck after handling LINK TRB. 1176 * To eliminate this strange behavioral driver set TRB_CHAIN 1177 * bit only for TR size > 2. 1178 */ 1179 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC || 1180 TRBS_PER_SEGMENT > 2) 1181 ch_bit = TRB_CHAIN; 1182 1183 link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) | 1184 TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit); 1185 } 1186 1187 if (priv_dev->dev_ver <= DEV_VER_V2) 1188 togle_pcs = cdns3_wa1_update_guard(priv_ep, trb); 1189 1190 if (sg_supported) 1191 s = request->sg; 1192 1193 /* set incorrect Cycle Bit for first trb*/ 1194 control = priv_ep->pcs ? 0 : TRB_CYCLE; 1195 trb->length = 0; 1196 if (priv_dev->dev_ver >= DEV_VER_V2) { 1197 u16 td_size; 1198 1199 td_size = DIV_ROUND_UP(request->length, 1200 priv_ep->endpoint.maxpacket); 1201 if (priv_dev->gadget.speed == USB_SPEED_SUPER) 1202 trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size)); 1203 else 1204 control |= TRB_TDL_HS_SIZE(td_size); 1205 } 1206 1207 do { 1208 u32 length; 1209 1210 /* fill TRB */ 1211 control |= TRB_TYPE(TRB_NORMAL); 1212 if (sg_supported) { 1213 trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s))); 1214 length = sg_dma_len(s); 1215 } else { 1216 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); 1217 length = request->length; 1218 } 1219 1220 if (priv_ep->flags & EP_TDLCHK_EN) 1221 total_tdl += DIV_ROUND_UP(length, 1222 priv_ep->endpoint.maxpacket); 1223 1224 trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) | 1225 TRB_LEN(length)); 1226 pcs = priv_ep->pcs ? TRB_CYCLE : 0; 1227 1228 /* 1229 * first trb should be prepared as last to avoid processing 1230 * transfer to early 1231 */ 1232 if (sg_iter != 0) 1233 control |= pcs; 1234 1235 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { 1236 control |= TRB_IOC | TRB_ISP; 1237 } else { 1238 /* for last element in TD or in SG list */ 1239 if (sg_iter == (num_trb - 1) && sg_iter != 0) 1240 control |= pcs | TRB_IOC | TRB_ISP; 1241 } 1242 1243 if (sg_iter) 1244 trb->control = cpu_to_le32(control); 1245 else 1246 priv_req->trb->control = cpu_to_le32(control); 1247 1248 if (sg_supported) { 1249 trb->control |= cpu_to_le32(TRB_ISP); 1250 /* Don't set chain bit for last TRB */ 1251 if (sg_iter < num_trb - 1) 1252 trb->control |= cpu_to_le32(TRB_CHAIN); 1253 1254 s = sg_next(s); 1255 } 1256 1257 control = 0; 1258 ++sg_iter; 1259 priv_req->end_trb = priv_ep->enqueue; 1260 cdns3_ep_inc_enq(priv_ep); 1261 trb = priv_ep->trb_pool + priv_ep->enqueue; 1262 trb->length = 0; 1263 } while (sg_iter < num_trb); 1264 1265 trb = priv_req->trb; 1266 1267 priv_req->flags |= REQUEST_PENDING; 1268 priv_req->num_of_trb = num_trb; 1269 1270 if (sg_iter == 1) 1271 trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP); 1272 1273 if (priv_dev->dev_ver < DEV_VER_V2 && 1274 (priv_ep->flags & EP_TDLCHK_EN)) { 1275 u16 tdl = total_tdl; 1276 u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd)); 1277 1278 if (tdl > EP_CMD_TDL_MAX) { 1279 tdl = EP_CMD_TDL_MAX; 1280 priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX; 1281 } 1282 1283 if (old_tdl < tdl) { 1284 tdl -= old_tdl; 1285 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, 1286 &priv_dev->regs->ep_cmd); 1287 } 1288 } 1289 1290 /* 1291 * Memory barrier - cycle bit must be set before other filds in trb. 1292 */ 1293 wmb(); 1294 1295 /* give the TD to the consumer*/ 1296 if (togle_pcs) 1297 trb->control = trb->control ^ cpu_to_le32(1); 1298 1299 if (priv_dev->dev_ver <= DEV_VER_V2) 1300 cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep); 1301 1302 if (num_trb > 1) { 1303 int i = 0; 1304 1305 while (i < num_trb) { 1306 trace_cdns3_prepare_trb(priv_ep, trb + i); 1307 if (trb + i == link_trb) { 1308 trb = priv_ep->trb_pool; 1309 num_trb = num_trb - i; 1310 i = 0; 1311 } else { 1312 i++; 1313 } 1314 } 1315 } else { 1316 trace_cdns3_prepare_trb(priv_ep, priv_req->trb); 1317 } 1318 1319 /* 1320 * Memory barrier - Cycle Bit must be set before trb->length and 1321 * trb->buffer fields. 1322 */ 1323 wmb(); 1324 1325 /* 1326 * For DMULT mode we can set address to transfer ring only once after 1327 * enabling endpoint. 1328 */ 1329 if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) { 1330 /* 1331 * Until SW is not ready to handle the OUT transfer the ISO OUT 1332 * Endpoint should be disabled (EP_CFG.ENABLE = 0). 1333 * EP_CFG_ENABLE must be set before updating ep_traddr. 1334 */ 1335 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir && 1336 !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) { 1337 priv_ep->flags |= EP_QUIRK_ISO_OUT_EN; 1338 cdns3_set_register_bit(&priv_dev->regs->ep_cfg, 1339 EP_CFG_ENABLE); 1340 } 1341 1342 writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma + 1343 priv_req->start_trb * TRB_SIZE), 1344 &priv_dev->regs->ep_traddr); 1345 1346 priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR; 1347 } 1348 1349 if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) { 1350 trace_cdns3_ring(priv_ep); 1351 /*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/ 1352 writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts); 1353 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); 1354 trace_cdns3_doorbell_epx(priv_ep->name, 1355 readl(&priv_dev->regs->ep_traddr)); 1356 } 1357 1358 /* WORKAROUND for transition to L0 */ 1359 __cdns3_gadget_wakeup(priv_dev); 1360 1361 return 0; 1362 } 1363 1364 void cdns3_set_hw_configuration(struct cdns3_device *priv_dev) 1365 { 1366 struct cdns3_endpoint *priv_ep; 1367 struct usb_ep *ep; 1368 1369 if (priv_dev->hw_configured_flag) 1370 return; 1371 1372 writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf); 1373 1374 cdns3_set_register_bit(&priv_dev->regs->usb_conf, 1375 USB_CONF_U1EN | USB_CONF_U2EN); 1376 1377 priv_dev->hw_configured_flag = 1; 1378 1379 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { 1380 if (ep->enabled) { 1381 priv_ep = ep_to_cdns3_ep(ep); 1382 cdns3_start_all_request(priv_dev, priv_ep); 1383 } 1384 } 1385 1386 cdns3_allow_enable_l1(priv_dev, 1); 1387 } 1388 1389 /** 1390 * cdns3_trb_handled - check whether trb has been handled by DMA 1391 * 1392 * @priv_ep: extended endpoint object. 1393 * @priv_req: request object for checking 1394 * 1395 * Endpoint must be selected before invoking this function. 1396 * 1397 * Returns false if request has not been handled by DMA, else returns true. 1398 * 1399 * SR - start ring 1400 * ER - end ring 1401 * DQ = priv_ep->dequeue - dequeue position 1402 * EQ = priv_ep->enqueue - enqueue position 1403 * ST = priv_req->start_trb - index of first TRB in transfer ring 1404 * ET = priv_req->end_trb - index of last TRB in transfer ring 1405 * CI = current_index - index of processed TRB by DMA. 1406 * 1407 * As first step, we check if the TRB between the ST and ET. 1408 * Then, we check if cycle bit for index priv_ep->dequeue 1409 * is correct. 1410 * 1411 * some rules: 1412 * 1. priv_ep->dequeue never equals to current_index. 1413 * 2 priv_ep->enqueue never exceed priv_ep->dequeue 1414 * 3. exception: priv_ep->enqueue == priv_ep->dequeue 1415 * and priv_ep->free_trbs is zero. 1416 * This case indicate that TR is full. 1417 * 1418 * At below two cases, the request have been handled. 1419 * Case 1 - priv_ep->dequeue < current_index 1420 * SR ... EQ ... DQ ... CI ... ER 1421 * SR ... DQ ... CI ... EQ ... ER 1422 * 1423 * Case 2 - priv_ep->dequeue > current_index 1424 * This situation takes place when CI go through the LINK TRB at the end of 1425 * transfer ring. 1426 * SR ... CI ... EQ ... DQ ... ER 1427 */ 1428 static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep, 1429 struct cdns3_request *priv_req) 1430 { 1431 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 1432 struct cdns3_trb *trb; 1433 int current_index = 0; 1434 int handled = 0; 1435 int doorbell; 1436 1437 current_index = cdns3_get_dma_pos(priv_dev, priv_ep); 1438 doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY); 1439 1440 /* current trb doesn't belong to this request */ 1441 if (priv_req->start_trb < priv_req->end_trb) { 1442 if (priv_ep->dequeue > priv_req->end_trb) 1443 goto finish; 1444 1445 if (priv_ep->dequeue < priv_req->start_trb) 1446 goto finish; 1447 } 1448 1449 if ((priv_req->start_trb > priv_req->end_trb) && 1450 (priv_ep->dequeue > priv_req->end_trb) && 1451 (priv_ep->dequeue < priv_req->start_trb)) 1452 goto finish; 1453 1454 if ((priv_req->start_trb == priv_req->end_trb) && 1455 (priv_ep->dequeue != priv_req->end_trb)) 1456 goto finish; 1457 1458 trb = &priv_ep->trb_pool[priv_ep->dequeue]; 1459 1460 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs) 1461 goto finish; 1462 1463 if (doorbell == 1 && current_index == priv_ep->dequeue) 1464 goto finish; 1465 1466 /* The corner case for TRBS_PER_SEGMENT equal 2). */ 1467 if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { 1468 handled = 1; 1469 goto finish; 1470 } 1471 1472 if (priv_ep->enqueue == priv_ep->dequeue && 1473 priv_ep->free_trbs == 0) { 1474 handled = 1; 1475 } else if (priv_ep->dequeue < current_index) { 1476 if ((current_index == (priv_ep->num_trbs - 1)) && 1477 !priv_ep->dequeue) 1478 goto finish; 1479 1480 handled = 1; 1481 } else if (priv_ep->dequeue > current_index) { 1482 handled = 1; 1483 } 1484 1485 finish: 1486 trace_cdns3_request_handled(priv_req, current_index, handled); 1487 1488 return handled; 1489 } 1490 1491 static void cdns3_transfer_completed(struct cdns3_device *priv_dev, 1492 struct cdns3_endpoint *priv_ep) 1493 { 1494 struct cdns3_request *priv_req; 1495 struct usb_request *request; 1496 struct cdns3_trb *trb; 1497 bool request_handled = false; 1498 bool transfer_end = false; 1499 1500 while (!list_empty(&priv_ep->pending_req_list)) { 1501 request = cdns3_next_request(&priv_ep->pending_req_list); 1502 priv_req = to_cdns3_request(request); 1503 1504 trb = priv_ep->trb_pool + priv_ep->dequeue; 1505 1506 /* Request was dequeued and TRB was changed to TRB_LINK. */ 1507 if (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) { 1508 trace_cdns3_complete_trb(priv_ep, trb); 1509 cdns3_move_deq_to_next_trb(priv_req); 1510 } 1511 1512 if (!request->stream_id) { 1513 /* Re-select endpoint. It could be changed by other CPU 1514 * during handling usb_gadget_giveback_request. 1515 */ 1516 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); 1517 1518 while (cdns3_trb_handled(priv_ep, priv_req)) { 1519 priv_req->finished_trb++; 1520 if (priv_req->finished_trb >= priv_req->num_of_trb) 1521 request_handled = true; 1522 1523 trb = priv_ep->trb_pool + priv_ep->dequeue; 1524 trace_cdns3_complete_trb(priv_ep, trb); 1525 1526 if (!transfer_end) 1527 request->actual += 1528 TRB_LEN(le32_to_cpu(trb->length)); 1529 1530 if (priv_req->num_of_trb > 1 && 1531 le32_to_cpu(trb->control) & TRB_SMM) 1532 transfer_end = true; 1533 1534 cdns3_ep_inc_deq(priv_ep); 1535 } 1536 1537 if (request_handled) { 1538 cdns3_gadget_giveback(priv_ep, priv_req, 0); 1539 request_handled = false; 1540 transfer_end = false; 1541 } else { 1542 goto prepare_next_td; 1543 } 1544 1545 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && 1546 TRBS_PER_SEGMENT == 2) 1547 break; 1548 } else { 1549 /* Re-select endpoint. It could be changed by other CPU 1550 * during handling usb_gadget_giveback_request. 1551 */ 1552 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); 1553 1554 trb = priv_ep->trb_pool; 1555 trace_cdns3_complete_trb(priv_ep, trb); 1556 1557 if (trb != priv_req->trb) 1558 dev_warn(priv_dev->dev, 1559 "request_trb=0x%p, queue_trb=0x%p\n", 1560 priv_req->trb, trb); 1561 1562 request->actual += TRB_LEN(le32_to_cpu(trb->length)); 1563 1564 if (!request->num_sgs || 1565 (request->num_sgs == (priv_ep->stream_sg_idx + 1))) { 1566 priv_ep->stream_sg_idx = 0; 1567 cdns3_gadget_giveback(priv_ep, priv_req, 0); 1568 } else { 1569 priv_ep->stream_sg_idx++; 1570 cdns3_ep_run_stream_transfer(priv_ep, request); 1571 } 1572 break; 1573 } 1574 } 1575 priv_ep->flags &= ~EP_PENDING_REQUEST; 1576 1577 prepare_next_td: 1578 if (!(priv_ep->flags & EP_STALLED) && 1579 !(priv_ep->flags & EP_STALL_PENDING)) 1580 cdns3_start_all_request(priv_dev, priv_ep); 1581 } 1582 1583 void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm) 1584 { 1585 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 1586 1587 cdns3_wa1_restore_cycle_bit(priv_ep); 1588 1589 if (rearm) { 1590 trace_cdns3_ring(priv_ep); 1591 1592 /* Cycle Bit must be updated before arming DMA. */ 1593 wmb(); 1594 writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd); 1595 1596 __cdns3_gadget_wakeup(priv_dev); 1597 1598 trace_cdns3_doorbell_epx(priv_ep->name, 1599 readl(&priv_dev->regs->ep_traddr)); 1600 } 1601 } 1602 1603 static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep) 1604 { 1605 u16 tdl = priv_ep->pending_tdl; 1606 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 1607 1608 if (tdl > EP_CMD_TDL_MAX) { 1609 tdl = EP_CMD_TDL_MAX; 1610 priv_ep->pending_tdl -= EP_CMD_TDL_MAX; 1611 } else { 1612 priv_ep->pending_tdl = 0; 1613 } 1614 1615 writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd); 1616 } 1617 1618 /** 1619 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint 1620 * @priv_ep: endpoint object 1621 * 1622 * Returns 0 1623 */ 1624 static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep) 1625 { 1626 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 1627 u32 ep_sts_reg; 1628 struct usb_request *deferred_request; 1629 struct usb_request *pending_request; 1630 u32 tdl = 0; 1631 1632 cdns3_select_ep(priv_dev, priv_ep->endpoint.address); 1633 1634 trace_cdns3_epx_irq(priv_dev, priv_ep); 1635 1636 ep_sts_reg = readl(&priv_dev->regs->ep_sts); 1637 writel(ep_sts_reg, &priv_dev->regs->ep_sts); 1638 1639 if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) { 1640 bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY); 1641 1642 tdl = cdns3_get_tdl(priv_dev); 1643 1644 /* 1645 * Continue the previous transfer: 1646 * There is some racing between ERDY and PRIME. The device send 1647 * ERDY and almost in the same time Host send PRIME. It cause 1648 * that host ignore the ERDY packet and driver has to send it 1649 * again. 1650 */ 1651 if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) || 1652 EP_STS_HOSTPP(ep_sts_reg))) { 1653 writel(EP_CMD_ERDY | 1654 EP_CMD_ERDY_SID(priv_ep->last_stream_id), 1655 &priv_dev->regs->ep_cmd); 1656 ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC); 1657 } else { 1658 priv_ep->prime_flag = true; 1659 1660 pending_request = cdns3_next_request(&priv_ep->pending_req_list); 1661 deferred_request = cdns3_next_request(&priv_ep->deferred_req_list); 1662 1663 if (deferred_request && !pending_request) { 1664 cdns3_start_all_request(priv_dev, priv_ep); 1665 } 1666 } 1667 } 1668 1669 if (ep_sts_reg & EP_STS_TRBERR) { 1670 if (priv_ep->flags & EP_STALL_PENDING && 1671 !(ep_sts_reg & EP_STS_DESCMIS && 1672 priv_dev->dev_ver < DEV_VER_V2)) { 1673 cdns3_ep_stall_flush(priv_ep); 1674 } 1675 1676 /* 1677 * For isochronous transfer driver completes request on 1678 * IOC or on TRBERR. IOC appears only when device receive 1679 * OUT data packet. If host disable stream or lost some packet 1680 * then the only way to finish all queued transfer is to do it 1681 * on TRBERR event. 1682 */ 1683 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && 1684 !priv_ep->wa1_set) { 1685 if (!priv_ep->dir) { 1686 u32 ep_cfg = readl(&priv_dev->regs->ep_cfg); 1687 1688 ep_cfg &= ~EP_CFG_ENABLE; 1689 writel(ep_cfg, &priv_dev->regs->ep_cfg); 1690 priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN; 1691 } 1692 cdns3_transfer_completed(priv_dev, priv_ep); 1693 } else if (!(priv_ep->flags & EP_STALLED) && 1694 !(priv_ep->flags & EP_STALL_PENDING)) { 1695 if (priv_ep->flags & EP_DEFERRED_DRDY) { 1696 priv_ep->flags &= ~EP_DEFERRED_DRDY; 1697 cdns3_start_all_request(priv_dev, priv_ep); 1698 } else { 1699 cdns3_rearm_transfer(priv_ep, 1700 priv_ep->wa1_set); 1701 } 1702 } 1703 } 1704 1705 if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) || 1706 (ep_sts_reg & EP_STS_IOT)) { 1707 if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) { 1708 if (ep_sts_reg & EP_STS_ISP) 1709 priv_ep->flags |= EP_QUIRK_END_TRANSFER; 1710 else 1711 priv_ep->flags &= ~EP_QUIRK_END_TRANSFER; 1712 } 1713 1714 if (!priv_ep->use_streams) { 1715 if ((ep_sts_reg & EP_STS_IOC) || 1716 (ep_sts_reg & EP_STS_ISP)) { 1717 cdns3_transfer_completed(priv_dev, priv_ep); 1718 } else if ((priv_ep->flags & EP_TDLCHK_EN) & 1719 priv_ep->pending_tdl) { 1720 /* handle IOT with pending tdl */ 1721 cdns3_reprogram_tdl(priv_ep); 1722 } 1723 } else if (priv_ep->dir == USB_DIR_OUT) { 1724 priv_ep->ep_sts_pending |= ep_sts_reg; 1725 } else if (ep_sts_reg & EP_STS_IOT) { 1726 cdns3_transfer_completed(priv_dev, priv_ep); 1727 } 1728 } 1729 1730 /* 1731 * MD_EXIT interrupt sets when stream capable endpoint exits 1732 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine 1733 */ 1734 if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) && 1735 (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) { 1736 priv_ep->ep_sts_pending = 0; 1737 cdns3_transfer_completed(priv_dev, priv_ep); 1738 } 1739 1740 /* 1741 * WA2: this condition should only be meet when 1742 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or 1743 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN. 1744 * In other cases this interrupt will be disabled. 1745 */ 1746 if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 && 1747 !(priv_ep->flags & EP_STALLED)) 1748 cdns3_wa2_descmissing_packet(priv_ep); 1749 1750 return 0; 1751 } 1752 1753 static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev) 1754 { 1755 if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) 1756 priv_dev->gadget_driver->disconnect(&priv_dev->gadget); 1757 } 1758 1759 /** 1760 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device 1761 * @priv_dev: extended gadget object 1762 * @usb_ists: bitmap representation of device's reported interrupts 1763 * (usb_ists register value) 1764 */ 1765 static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev, 1766 u32 usb_ists) 1767 __must_hold(&priv_dev->lock) 1768 { 1769 int speed = 0; 1770 1771 trace_cdns3_usb_irq(priv_dev, usb_ists); 1772 if (usb_ists & USB_ISTS_L1ENTI) { 1773 /* 1774 * WORKAROUND: CDNS3 controller has issue with hardware resuming 1775 * from L1. To fix it, if any DMA transfer is pending driver 1776 * must starts driving resume signal immediately. 1777 */ 1778 if (readl(&priv_dev->regs->drbl)) 1779 __cdns3_gadget_wakeup(priv_dev); 1780 } 1781 1782 /* Connection detected */ 1783 if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) { 1784 speed = cdns3_get_speed(priv_dev); 1785 priv_dev->gadget.speed = speed; 1786 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED); 1787 cdns3_ep0_config(priv_dev); 1788 } 1789 1790 /* Disconnection detected */ 1791 if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) { 1792 spin_unlock(&priv_dev->lock); 1793 cdns3_disconnect_gadget(priv_dev); 1794 spin_lock(&priv_dev->lock); 1795 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; 1796 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); 1797 cdns3_hw_reset_eps_config(priv_dev); 1798 } 1799 1800 if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) { 1801 if (priv_dev->gadget_driver && 1802 priv_dev->gadget_driver->suspend) { 1803 spin_unlock(&priv_dev->lock); 1804 priv_dev->gadget_driver->suspend(&priv_dev->gadget); 1805 spin_lock(&priv_dev->lock); 1806 } 1807 } 1808 1809 if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) { 1810 if (priv_dev->gadget_driver && 1811 priv_dev->gadget_driver->resume) { 1812 spin_unlock(&priv_dev->lock); 1813 priv_dev->gadget_driver->resume(&priv_dev->gadget); 1814 spin_lock(&priv_dev->lock); 1815 } 1816 } 1817 1818 /* reset*/ 1819 if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) { 1820 if (priv_dev->gadget_driver) { 1821 spin_unlock(&priv_dev->lock); 1822 usb_gadget_udc_reset(&priv_dev->gadget, 1823 priv_dev->gadget_driver); 1824 spin_lock(&priv_dev->lock); 1825 1826 /*read again to check the actual speed*/ 1827 speed = cdns3_get_speed(priv_dev); 1828 priv_dev->gadget.speed = speed; 1829 cdns3_hw_reset_eps_config(priv_dev); 1830 cdns3_ep0_config(priv_dev); 1831 } 1832 } 1833 } 1834 1835 /** 1836 * cdns3_device_irq_handler - interrupt handler for device part of controller 1837 * 1838 * @irq: irq number for cdns3 core device 1839 * @data: structure of cdns3 1840 * 1841 * Returns IRQ_HANDLED or IRQ_NONE 1842 */ 1843 static irqreturn_t cdns3_device_irq_handler(int irq, void *data) 1844 { 1845 struct cdns3_device *priv_dev = data; 1846 struct cdns *cdns = dev_get_drvdata(priv_dev->dev); 1847 irqreturn_t ret = IRQ_NONE; 1848 u32 reg; 1849 1850 if (cdns->in_lpm) 1851 return ret; 1852 1853 /* check USB device interrupt */ 1854 reg = readl(&priv_dev->regs->usb_ists); 1855 if (reg) { 1856 /* After masking interrupts the new interrupts won't be 1857 * reported in usb_ists/ep_ists. In order to not lose some 1858 * of them driver disables only detected interrupts. 1859 * They will be enabled ASAP after clearing source of 1860 * interrupt. This an unusual behavior only applies to 1861 * usb_ists register. 1862 */ 1863 reg = ~reg & readl(&priv_dev->regs->usb_ien); 1864 /* mask deferred interrupt. */ 1865 writel(reg, &priv_dev->regs->usb_ien); 1866 ret = IRQ_WAKE_THREAD; 1867 } 1868 1869 /* check endpoint interrupt */ 1870 reg = readl(&priv_dev->regs->ep_ists); 1871 if (reg) { 1872 writel(0, &priv_dev->regs->ep_ien); 1873 ret = IRQ_WAKE_THREAD; 1874 } 1875 1876 return ret; 1877 } 1878 1879 /** 1880 * cdns3_device_thread_irq_handler - interrupt handler for device part 1881 * of controller 1882 * 1883 * @irq: irq number for cdns3 core device 1884 * @data: structure of cdns3 1885 * 1886 * Returns IRQ_HANDLED or IRQ_NONE 1887 */ 1888 static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data) 1889 { 1890 struct cdns3_device *priv_dev = data; 1891 irqreturn_t ret = IRQ_NONE; 1892 unsigned long flags; 1893 unsigned int bit; 1894 unsigned long reg; 1895 1896 spin_lock_irqsave(&priv_dev->lock, flags); 1897 1898 reg = readl(&priv_dev->regs->usb_ists); 1899 if (reg) { 1900 writel(reg, &priv_dev->regs->usb_ists); 1901 writel(USB_IEN_INIT, &priv_dev->regs->usb_ien); 1902 cdns3_check_usb_interrupt_proceed(priv_dev, reg); 1903 ret = IRQ_HANDLED; 1904 } 1905 1906 reg = readl(&priv_dev->regs->ep_ists); 1907 1908 /* handle default endpoint OUT */ 1909 if (reg & EP_ISTS_EP_OUT0) { 1910 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT); 1911 ret = IRQ_HANDLED; 1912 } 1913 1914 /* handle default endpoint IN */ 1915 if (reg & EP_ISTS_EP_IN0) { 1916 cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN); 1917 ret = IRQ_HANDLED; 1918 } 1919 1920 /* check if interrupt from non default endpoint, if no exit */ 1921 reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0); 1922 if (!reg) 1923 goto irqend; 1924 1925 for_each_set_bit(bit, ®, 1926 sizeof(u32) * BITS_PER_BYTE) { 1927 cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]); 1928 ret = IRQ_HANDLED; 1929 } 1930 1931 if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams) 1932 cdns3_wa2_check_outq_status(priv_dev); 1933 1934 irqend: 1935 writel(~0, &priv_dev->regs->ep_ien); 1936 spin_unlock_irqrestore(&priv_dev->lock, flags); 1937 1938 return ret; 1939 } 1940 1941 /** 1942 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP 1943 * 1944 * The real reservation will occur during write to EP_CFG register, 1945 * this function is used to check if the 'size' reservation is allowed. 1946 * 1947 * @priv_dev: extended gadget object 1948 * @size: the size (KB) for EP would like to allocate 1949 * @is_in: endpoint direction 1950 * 1951 * Return 0 if the required size can met or negative value on failure 1952 */ 1953 static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev, 1954 int size, int is_in) 1955 { 1956 int remained; 1957 1958 /* 2KB are reserved for EP0*/ 1959 remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2; 1960 1961 if (is_in) { 1962 if (remained < size) 1963 return -EPERM; 1964 1965 priv_dev->onchip_used_size += size; 1966 } else { 1967 int required; 1968 1969 /** 1970 * ALL OUT EPs are shared the same chunk onchip memory, so 1971 * driver checks if it already has assigned enough buffers 1972 */ 1973 if (priv_dev->out_mem_is_allocated >= size) 1974 return 0; 1975 1976 required = size - priv_dev->out_mem_is_allocated; 1977 1978 if (required > remained) 1979 return -EPERM; 1980 1981 priv_dev->out_mem_is_allocated += required; 1982 priv_dev->onchip_used_size += required; 1983 } 1984 1985 return 0; 1986 } 1987 1988 static void cdns3_configure_dmult(struct cdns3_device *priv_dev, 1989 struct cdns3_endpoint *priv_ep) 1990 { 1991 struct cdns3_usb_regs __iomem *regs = priv_dev->regs; 1992 1993 /* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */ 1994 if (priv_dev->dev_ver <= DEV_VER_V2) 1995 writel(USB_CONF_DMULT, ®s->usb_conf); 1996 1997 if (priv_dev->dev_ver == DEV_VER_V2) 1998 writel(USB_CONF2_EN_TDL_TRB, ®s->usb_conf2); 1999 2000 if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) { 2001 u32 mask; 2002 2003 if (priv_ep->dir) 2004 mask = BIT(priv_ep->num + 16); 2005 else 2006 mask = BIT(priv_ep->num); 2007 2008 if (priv_ep->type != USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) { 2009 cdns3_set_register_bit(®s->tdl_from_trb, mask); 2010 cdns3_set_register_bit(®s->tdl_beh, mask); 2011 cdns3_set_register_bit(®s->tdl_beh2, mask); 2012 cdns3_set_register_bit(®s->dma_adv_td, mask); 2013 } 2014 2015 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) 2016 cdns3_set_register_bit(®s->tdl_from_trb, mask); 2017 2018 cdns3_set_register_bit(®s->dtrans, mask); 2019 } 2020 } 2021 2022 /** 2023 * cdns3_ep_config - Configure hardware endpoint 2024 * @priv_ep: extended endpoint object 2025 * @enable: set EP_CFG_ENABLE bit in ep_cfg register. 2026 */ 2027 int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable) 2028 { 2029 bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC); 2030 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 2031 u32 bEndpointAddress = priv_ep->num | priv_ep->dir; 2032 u32 max_packet_size = 0; 2033 u8 maxburst = 0; 2034 u32 ep_cfg = 0; 2035 u8 buffering; 2036 u8 mult = 0; 2037 int ret; 2038 2039 buffering = CDNS3_EP_BUF_SIZE - 1; 2040 2041 cdns3_configure_dmult(priv_dev, priv_ep); 2042 2043 switch (priv_ep->type) { 2044 case USB_ENDPOINT_XFER_INT: 2045 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT); 2046 2047 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) 2048 ep_cfg |= EP_CFG_TDL_CHK; 2049 break; 2050 case USB_ENDPOINT_XFER_BULK: 2051 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK); 2052 2053 if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir) 2054 ep_cfg |= EP_CFG_TDL_CHK; 2055 break; 2056 default: 2057 ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC); 2058 mult = CDNS3_EP_ISO_HS_MULT - 1; 2059 buffering = mult + 1; 2060 } 2061 2062 switch (priv_dev->gadget.speed) { 2063 case USB_SPEED_FULL: 2064 max_packet_size = is_iso_ep ? 1023 : 64; 2065 break; 2066 case USB_SPEED_HIGH: 2067 max_packet_size = is_iso_ep ? 1024 : 512; 2068 break; 2069 case USB_SPEED_SUPER: 2070 /* It's limitation that driver assumes in driver. */ 2071 mult = 0; 2072 max_packet_size = 1024; 2073 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) { 2074 maxburst = CDNS3_EP_ISO_SS_BURST - 1; 2075 buffering = (mult + 1) * 2076 (maxburst + 1); 2077 2078 if (priv_ep->interval > 1) 2079 buffering++; 2080 } else { 2081 maxburst = CDNS3_EP_BUF_SIZE - 1; 2082 } 2083 break; 2084 default: 2085 /* all other speed are not supported */ 2086 return -EINVAL; 2087 } 2088 2089 if (max_packet_size == 1024) 2090 priv_ep->trb_burst_size = 128; 2091 else if (max_packet_size >= 512) 2092 priv_ep->trb_burst_size = 64; 2093 else 2094 priv_ep->trb_burst_size = 16; 2095 2096 /* onchip buffer is only allocated before configuration */ 2097 if (!priv_dev->hw_configured_flag) { 2098 ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1, 2099 !!priv_ep->dir); 2100 if (ret) { 2101 dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n"); 2102 return ret; 2103 } 2104 } 2105 2106 if (enable) 2107 ep_cfg |= EP_CFG_ENABLE; 2108 2109 if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) { 2110 if (priv_dev->dev_ver >= DEV_VER_V3) { 2111 u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0)); 2112 2113 /* 2114 * Stream capable endpoints are handled by using ep_tdl 2115 * register. Other endpoints use TDL from TRB feature. 2116 */ 2117 cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, 2118 mask); 2119 } 2120 2121 /* Enable Stream Bit TDL chk and SID chk */ 2122 ep_cfg |= EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK; 2123 } 2124 2125 ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) | 2126 EP_CFG_MULT(mult) | 2127 EP_CFG_BUFFERING(buffering) | 2128 EP_CFG_MAXBURST(maxburst); 2129 2130 cdns3_select_ep(priv_dev, bEndpointAddress); 2131 writel(ep_cfg, &priv_dev->regs->ep_cfg); 2132 priv_ep->flags |= EP_CONFIGURED; 2133 2134 dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n", 2135 priv_ep->name, ep_cfg); 2136 2137 return 0; 2138 } 2139 2140 /* Find correct direction for HW endpoint according to description */ 2141 static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc, 2142 struct cdns3_endpoint *priv_ep) 2143 { 2144 return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) || 2145 (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc)); 2146 } 2147 2148 static struct 2149 cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev, 2150 struct usb_endpoint_descriptor *desc) 2151 { 2152 struct usb_ep *ep; 2153 struct cdns3_endpoint *priv_ep; 2154 2155 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { 2156 unsigned long num; 2157 int ret; 2158 /* ep name pattern likes epXin or epXout */ 2159 char c[2] = {ep->name[2], '\0'}; 2160 2161 ret = kstrtoul(c, 10, &num); 2162 if (ret) 2163 return ERR_PTR(ret); 2164 2165 priv_ep = ep_to_cdns3_ep(ep); 2166 if (cdns3_ep_dir_is_correct(desc, priv_ep)) { 2167 if (!(priv_ep->flags & EP_CLAIMED)) { 2168 priv_ep->num = num; 2169 return priv_ep; 2170 } 2171 } 2172 } 2173 2174 return ERR_PTR(-ENOENT); 2175 } 2176 2177 /* 2178 * Cadence IP has one limitation that all endpoints must be configured 2179 * (Type & MaxPacketSize) before setting configuration through hardware 2180 * register, it means we can't change endpoints configuration after 2181 * set_configuration. 2182 * 2183 * This function set EP_CLAIMED flag which is added when the gadget driver 2184 * uses usb_ep_autoconfig to configure specific endpoint; 2185 * When the udc driver receives set_configurion request, 2186 * it goes through all claimed endpoints, and configure all endpoints 2187 * accordingly. 2188 * 2189 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through 2190 * ep_cfg register which can be changed after set_configuration, and do 2191 * some software operation accordingly. 2192 */ 2193 static struct 2194 usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget, 2195 struct usb_endpoint_descriptor *desc, 2196 struct usb_ss_ep_comp_descriptor *comp_desc) 2197 { 2198 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2199 struct cdns3_endpoint *priv_ep; 2200 unsigned long flags; 2201 2202 priv_ep = cdns3_find_available_ep(priv_dev, desc); 2203 if (IS_ERR(priv_ep)) { 2204 dev_err(priv_dev->dev, "no available ep\n"); 2205 return NULL; 2206 } 2207 2208 dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name); 2209 2210 spin_lock_irqsave(&priv_dev->lock, flags); 2211 priv_ep->endpoint.desc = desc; 2212 priv_ep->dir = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT; 2213 priv_ep->type = usb_endpoint_type(desc); 2214 priv_ep->flags |= EP_CLAIMED; 2215 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; 2216 2217 spin_unlock_irqrestore(&priv_dev->lock, flags); 2218 return &priv_ep->endpoint; 2219 } 2220 2221 /** 2222 * cdns3_gadget_ep_alloc_request - Allocates request 2223 * @ep: endpoint object associated with request 2224 * @gfp_flags: gfp flags 2225 * 2226 * Returns allocated request address, NULL on allocation error 2227 */ 2228 struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep, 2229 gfp_t gfp_flags) 2230 { 2231 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); 2232 struct cdns3_request *priv_req; 2233 2234 priv_req = kzalloc(sizeof(*priv_req), gfp_flags); 2235 if (!priv_req) 2236 return NULL; 2237 2238 priv_req->priv_ep = priv_ep; 2239 2240 trace_cdns3_alloc_request(priv_req); 2241 return &priv_req->request; 2242 } 2243 2244 /** 2245 * cdns3_gadget_ep_free_request - Free memory occupied by request 2246 * @ep: endpoint object associated with request 2247 * @request: request to free memory 2248 */ 2249 void cdns3_gadget_ep_free_request(struct usb_ep *ep, 2250 struct usb_request *request) 2251 { 2252 struct cdns3_request *priv_req = to_cdns3_request(request); 2253 2254 if (priv_req->aligned_buf) 2255 priv_req->aligned_buf->in_use = 0; 2256 2257 trace_cdns3_free_request(priv_req); 2258 kfree(priv_req); 2259 } 2260 2261 /** 2262 * cdns3_gadget_ep_enable - Enable endpoint 2263 * @ep: endpoint object 2264 * @desc: endpoint descriptor 2265 * 2266 * Returns 0 on success, error code elsewhere 2267 */ 2268 static int cdns3_gadget_ep_enable(struct usb_ep *ep, 2269 const struct usb_endpoint_descriptor *desc) 2270 { 2271 struct cdns3_endpoint *priv_ep; 2272 struct cdns3_device *priv_dev; 2273 const struct usb_ss_ep_comp_descriptor *comp_desc; 2274 u32 reg = EP_STS_EN_TRBERREN; 2275 u32 bEndpointAddress; 2276 unsigned long flags; 2277 int enable = 1; 2278 int ret = 0; 2279 int val; 2280 2281 priv_ep = ep_to_cdns3_ep(ep); 2282 priv_dev = priv_ep->cdns3_dev; 2283 comp_desc = priv_ep->endpoint.comp_desc; 2284 2285 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 2286 dev_dbg(priv_dev->dev, "usbss: invalid parameters\n"); 2287 return -EINVAL; 2288 } 2289 2290 if (!desc->wMaxPacketSize) { 2291 dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n"); 2292 return -EINVAL; 2293 } 2294 2295 if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED, 2296 "%s is already enabled\n", priv_ep->name)) 2297 return 0; 2298 2299 spin_lock_irqsave(&priv_dev->lock, flags); 2300 2301 priv_ep->endpoint.desc = desc; 2302 priv_ep->type = usb_endpoint_type(desc); 2303 priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; 2304 2305 if (priv_ep->interval > ISO_MAX_INTERVAL && 2306 priv_ep->type == USB_ENDPOINT_XFER_ISOC) { 2307 dev_err(priv_dev->dev, "Driver is limited to %d period\n", 2308 ISO_MAX_INTERVAL); 2309 2310 ret = -EINVAL; 2311 goto exit; 2312 } 2313 2314 bEndpointAddress = priv_ep->num | priv_ep->dir; 2315 cdns3_select_ep(priv_dev, bEndpointAddress); 2316 2317 /* 2318 * For some versions of controller at some point during ISO OUT traffic 2319 * DMA reads Transfer Ring for the EP which has never got doorbell. 2320 * This issue was detected only on simulation, but to avoid this issue 2321 * driver add protection against it. To fix it driver enable ISO OUT 2322 * endpoint before setting DRBL. This special treatment of ISO OUT 2323 * endpoints are recommended by controller specification. 2324 */ 2325 if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir) 2326 enable = 0; 2327 2328 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 2329 /* 2330 * Enable stream support (SS mode) related interrupts 2331 * in EP_STS_EN Register 2332 */ 2333 if (priv_dev->gadget.speed >= USB_SPEED_SUPER) { 2334 reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN | 2335 EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN | 2336 EP_STS_EN_STREAMREN; 2337 priv_ep->use_streams = true; 2338 ret = cdns3_ep_config(priv_ep, enable); 2339 priv_dev->using_streams |= true; 2340 } 2341 } else { 2342 ret = cdns3_ep_config(priv_ep, enable); 2343 } 2344 2345 if (ret) 2346 goto exit; 2347 2348 ret = cdns3_allocate_trb_pool(priv_ep); 2349 if (ret) 2350 goto exit; 2351 2352 bEndpointAddress = priv_ep->num | priv_ep->dir; 2353 cdns3_select_ep(priv_dev, bEndpointAddress); 2354 2355 trace_cdns3_gadget_ep_enable(priv_ep); 2356 2357 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); 2358 2359 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, 2360 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)), 2361 1, 1000); 2362 2363 if (unlikely(ret)) { 2364 cdns3_free_trb_pool(priv_ep); 2365 ret = -EINVAL; 2366 goto exit; 2367 } 2368 2369 /* enable interrupt for selected endpoint */ 2370 cdns3_set_register_bit(&priv_dev->regs->ep_ien, 2371 BIT(cdns3_ep_addr_to_index(bEndpointAddress))); 2372 2373 if (priv_dev->dev_ver < DEV_VER_V2) 2374 cdns3_wa2_enable_detection(priv_dev, priv_ep, reg); 2375 2376 writel(reg, &priv_dev->regs->ep_sts_en); 2377 2378 ep->desc = desc; 2379 priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING | 2380 EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN); 2381 priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR; 2382 priv_ep->wa1_set = 0; 2383 priv_ep->enqueue = 0; 2384 priv_ep->dequeue = 0; 2385 reg = readl(&priv_dev->regs->ep_sts); 2386 priv_ep->pcs = !!EP_STS_CCS(reg); 2387 priv_ep->ccs = !!EP_STS_CCS(reg); 2388 /* one TRB is reserved for link TRB used in DMULT mode*/ 2389 priv_ep->free_trbs = priv_ep->num_trbs - 1; 2390 exit: 2391 spin_unlock_irqrestore(&priv_dev->lock, flags); 2392 2393 return ret; 2394 } 2395 2396 /** 2397 * cdns3_gadget_ep_disable - Disable endpoint 2398 * @ep: endpoint object 2399 * 2400 * Returns 0 on success, error code elsewhere 2401 */ 2402 static int cdns3_gadget_ep_disable(struct usb_ep *ep) 2403 { 2404 struct cdns3_endpoint *priv_ep; 2405 struct cdns3_request *priv_req; 2406 struct cdns3_device *priv_dev; 2407 struct usb_request *request; 2408 unsigned long flags; 2409 int ret = 0; 2410 u32 ep_cfg; 2411 int val; 2412 2413 if (!ep) { 2414 pr_err("usbss: invalid parameters\n"); 2415 return -EINVAL; 2416 } 2417 2418 priv_ep = ep_to_cdns3_ep(ep); 2419 priv_dev = priv_ep->cdns3_dev; 2420 2421 if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED), 2422 "%s is already disabled\n", priv_ep->name)) 2423 return 0; 2424 2425 spin_lock_irqsave(&priv_dev->lock, flags); 2426 2427 trace_cdns3_gadget_ep_disable(priv_ep); 2428 2429 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); 2430 2431 ep_cfg = readl(&priv_dev->regs->ep_cfg); 2432 ep_cfg &= ~EP_CFG_ENABLE; 2433 writel(ep_cfg, &priv_dev->regs->ep_cfg); 2434 2435 /** 2436 * Driver needs some time before resetting endpoint. 2437 * It need waits for clearing DBUSY bit or for timeout expired. 2438 * 10us is enough time for controller to stop transfer. 2439 */ 2440 readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val, 2441 !(val & EP_STS_DBUSY), 1, 10); 2442 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); 2443 2444 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, 2445 !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)), 2446 1, 1000); 2447 if (unlikely(ret)) 2448 dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n", 2449 priv_ep->name); 2450 2451 while (!list_empty(&priv_ep->pending_req_list)) { 2452 request = cdns3_next_request(&priv_ep->pending_req_list); 2453 2454 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), 2455 -ESHUTDOWN); 2456 } 2457 2458 while (!list_empty(&priv_ep->wa2_descmiss_req_list)) { 2459 priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list); 2460 2461 kfree(priv_req->request.buf); 2462 cdns3_gadget_ep_free_request(&priv_ep->endpoint, 2463 &priv_req->request); 2464 list_del_init(&priv_req->list); 2465 --priv_ep->wa2_counter; 2466 } 2467 2468 while (!list_empty(&priv_ep->deferred_req_list)) { 2469 request = cdns3_next_request(&priv_ep->deferred_req_list); 2470 2471 cdns3_gadget_giveback(priv_ep, to_cdns3_request(request), 2472 -ESHUTDOWN); 2473 } 2474 2475 priv_ep->descmis_req = NULL; 2476 2477 ep->desc = NULL; 2478 priv_ep->flags &= ~EP_ENABLED; 2479 priv_ep->use_streams = false; 2480 2481 spin_unlock_irqrestore(&priv_dev->lock, flags); 2482 2483 return ret; 2484 } 2485 2486 /** 2487 * __cdns3_gadget_ep_queue - Transfer data on endpoint 2488 * @ep: endpoint object 2489 * @request: request object 2490 * @gfp_flags: gfp flags 2491 * 2492 * Returns 0 on success, error code elsewhere 2493 */ 2494 static int __cdns3_gadget_ep_queue(struct usb_ep *ep, 2495 struct usb_request *request, 2496 gfp_t gfp_flags) 2497 { 2498 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); 2499 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 2500 struct cdns3_request *priv_req; 2501 int ret = 0; 2502 2503 request->actual = 0; 2504 request->status = -EINPROGRESS; 2505 priv_req = to_cdns3_request(request); 2506 trace_cdns3_ep_queue(priv_req); 2507 2508 if (priv_dev->dev_ver < DEV_VER_V2) { 2509 ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep, 2510 priv_req); 2511 2512 if (ret == EINPROGRESS) 2513 return 0; 2514 } 2515 2516 ret = cdns3_prepare_aligned_request_buf(priv_req); 2517 if (ret < 0) 2518 return ret; 2519 2520 ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request, 2521 usb_endpoint_dir_in(ep->desc)); 2522 if (ret) 2523 return ret; 2524 2525 list_add_tail(&request->list, &priv_ep->deferred_req_list); 2526 2527 /* 2528 * For stream capable endpoint if prime irq flag is set then only start 2529 * request. 2530 * If hardware endpoint configuration has not been set yet then 2531 * just queue request in deferred list. Transfer will be started in 2532 * cdns3_set_hw_configuration. 2533 */ 2534 if (!request->stream_id) { 2535 if (priv_dev->hw_configured_flag && 2536 !(priv_ep->flags & EP_STALLED) && 2537 !(priv_ep->flags & EP_STALL_PENDING)) 2538 cdns3_start_all_request(priv_dev, priv_ep); 2539 } else { 2540 if (priv_dev->hw_configured_flag && priv_ep->prime_flag) 2541 cdns3_start_all_request(priv_dev, priv_ep); 2542 } 2543 2544 return 0; 2545 } 2546 2547 static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 2548 gfp_t gfp_flags) 2549 { 2550 struct usb_request *zlp_request; 2551 struct cdns3_endpoint *priv_ep; 2552 struct cdns3_device *priv_dev; 2553 unsigned long flags; 2554 int ret; 2555 2556 if (!request || !ep) 2557 return -EINVAL; 2558 2559 priv_ep = ep_to_cdns3_ep(ep); 2560 priv_dev = priv_ep->cdns3_dev; 2561 2562 spin_lock_irqsave(&priv_dev->lock, flags); 2563 2564 ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags); 2565 2566 if (ret == 0 && request->zero && request->length && 2567 (request->length % ep->maxpacket == 0)) { 2568 struct cdns3_request *priv_req; 2569 2570 zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC); 2571 zlp_request->buf = priv_dev->zlp_buf; 2572 zlp_request->length = 0; 2573 2574 priv_req = to_cdns3_request(zlp_request); 2575 priv_req->flags |= REQUEST_ZLP; 2576 2577 dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n", 2578 priv_ep->name); 2579 ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags); 2580 } 2581 2582 spin_unlock_irqrestore(&priv_dev->lock, flags); 2583 return ret; 2584 } 2585 2586 /** 2587 * cdns3_gadget_ep_dequeue - Remove request from transfer queue 2588 * @ep: endpoint object associated with request 2589 * @request: request object 2590 * 2591 * Returns 0 on success, error code elsewhere 2592 */ 2593 int cdns3_gadget_ep_dequeue(struct usb_ep *ep, 2594 struct usb_request *request) 2595 { 2596 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); 2597 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 2598 struct usb_request *req, *req_temp; 2599 struct cdns3_request *priv_req; 2600 struct cdns3_trb *link_trb; 2601 u8 req_on_hw_ring = 0; 2602 unsigned long flags; 2603 int ret = 0; 2604 2605 if (!ep || !request || !ep->desc) 2606 return -EINVAL; 2607 2608 spin_lock_irqsave(&priv_dev->lock, flags); 2609 2610 priv_req = to_cdns3_request(request); 2611 2612 trace_cdns3_ep_dequeue(priv_req); 2613 2614 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); 2615 2616 list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list, 2617 list) { 2618 if (request == req) { 2619 req_on_hw_ring = 1; 2620 goto found; 2621 } 2622 } 2623 2624 list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list, 2625 list) { 2626 if (request == req) 2627 goto found; 2628 } 2629 2630 goto not_found; 2631 2632 found: 2633 link_trb = priv_req->trb; 2634 2635 /* Update ring only if removed request is on pending_req_list list */ 2636 if (req_on_hw_ring && link_trb) { 2637 link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma + 2638 ((priv_req->end_trb + 1) * TRB_SIZE))); 2639 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) | 2640 TRB_TYPE(TRB_LINK) | TRB_CHAIN); 2641 2642 if (priv_ep->wa1_trb == priv_req->trb) 2643 cdns3_wa1_restore_cycle_bit(priv_ep); 2644 } 2645 2646 cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET); 2647 2648 not_found: 2649 spin_unlock_irqrestore(&priv_dev->lock, flags); 2650 return ret; 2651 } 2652 2653 /** 2654 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint 2655 * Should be called after acquiring spin_lock and selecting ep 2656 * @priv_ep: endpoint object to set stall on. 2657 */ 2658 void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep) 2659 { 2660 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 2661 2662 trace_cdns3_halt(priv_ep, 1, 0); 2663 2664 if (!(priv_ep->flags & EP_STALLED)) { 2665 u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts); 2666 2667 if (!(ep_sts_reg & EP_STS_DBUSY)) 2668 cdns3_ep_stall_flush(priv_ep); 2669 else 2670 priv_ep->flags |= EP_STALL_PENDING; 2671 } 2672 } 2673 2674 /** 2675 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint 2676 * Should be called after acquiring spin_lock and selecting ep 2677 * @priv_ep: endpoint object to clear stall on 2678 */ 2679 int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep) 2680 { 2681 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 2682 struct usb_request *request; 2683 struct cdns3_request *priv_req; 2684 struct cdns3_trb *trb = NULL; 2685 int ret; 2686 int val; 2687 2688 trace_cdns3_halt(priv_ep, 0, 0); 2689 2690 request = cdns3_next_request(&priv_ep->pending_req_list); 2691 if (request) { 2692 priv_req = to_cdns3_request(request); 2693 trb = priv_req->trb; 2694 if (trb) 2695 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); 2696 } 2697 2698 writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd); 2699 2700 /* wait for EPRST cleared */ 2701 ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, 2702 !(val & EP_CMD_EPRST), 1, 100); 2703 if (ret) 2704 return -EINVAL; 2705 2706 priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING); 2707 2708 if (request) { 2709 if (trb) 2710 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); 2711 2712 cdns3_rearm_transfer(priv_ep, 1); 2713 } 2714 2715 cdns3_start_all_request(priv_dev, priv_ep); 2716 return ret; 2717 } 2718 2719 /** 2720 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint 2721 * @ep: endpoint object to set/clear stall on 2722 * @value: 1 for set stall, 0 for clear stall 2723 * 2724 * Returns 0 on success, error code elsewhere 2725 */ 2726 int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value) 2727 { 2728 struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep); 2729 struct cdns3_device *priv_dev = priv_ep->cdns3_dev; 2730 unsigned long flags; 2731 int ret = 0; 2732 2733 if (!(priv_ep->flags & EP_ENABLED)) 2734 return -EPERM; 2735 2736 spin_lock_irqsave(&priv_dev->lock, flags); 2737 2738 cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress); 2739 2740 if (!value) { 2741 priv_ep->flags &= ~EP_WEDGE; 2742 ret = __cdns3_gadget_ep_clear_halt(priv_ep); 2743 } else { 2744 __cdns3_gadget_ep_set_halt(priv_ep); 2745 } 2746 2747 spin_unlock_irqrestore(&priv_dev->lock, flags); 2748 2749 return ret; 2750 } 2751 2752 extern const struct usb_ep_ops cdns3_gadget_ep0_ops; 2753 2754 static const struct usb_ep_ops cdns3_gadget_ep_ops = { 2755 .enable = cdns3_gadget_ep_enable, 2756 .disable = cdns3_gadget_ep_disable, 2757 .alloc_request = cdns3_gadget_ep_alloc_request, 2758 .free_request = cdns3_gadget_ep_free_request, 2759 .queue = cdns3_gadget_ep_queue, 2760 .dequeue = cdns3_gadget_ep_dequeue, 2761 .set_halt = cdns3_gadget_ep_set_halt, 2762 .set_wedge = cdns3_gadget_ep_set_wedge, 2763 }; 2764 2765 /** 2766 * cdns3_gadget_get_frame - Returns number of actual ITP frame 2767 * @gadget: gadget object 2768 * 2769 * Returns number of actual ITP frame 2770 */ 2771 static int cdns3_gadget_get_frame(struct usb_gadget *gadget) 2772 { 2773 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2774 2775 return readl(&priv_dev->regs->usb_itpn); 2776 } 2777 2778 int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev) 2779 { 2780 enum usb_device_speed speed; 2781 2782 speed = cdns3_get_speed(priv_dev); 2783 2784 if (speed >= USB_SPEED_SUPER) 2785 return 0; 2786 2787 /* Start driving resume signaling to indicate remote wakeup. */ 2788 writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf); 2789 2790 return 0; 2791 } 2792 2793 static int cdns3_gadget_wakeup(struct usb_gadget *gadget) 2794 { 2795 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2796 unsigned long flags; 2797 int ret = 0; 2798 2799 spin_lock_irqsave(&priv_dev->lock, flags); 2800 ret = __cdns3_gadget_wakeup(priv_dev); 2801 spin_unlock_irqrestore(&priv_dev->lock, flags); 2802 return ret; 2803 } 2804 2805 static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget, 2806 int is_selfpowered) 2807 { 2808 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2809 unsigned long flags; 2810 2811 spin_lock_irqsave(&priv_dev->lock, flags); 2812 priv_dev->is_selfpowered = !!is_selfpowered; 2813 spin_unlock_irqrestore(&priv_dev->lock, flags); 2814 return 0; 2815 } 2816 2817 static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on) 2818 { 2819 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2820 2821 if (is_on) { 2822 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); 2823 } else { 2824 writel(~0, &priv_dev->regs->ep_ists); 2825 writel(~0, &priv_dev->regs->usb_ists); 2826 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); 2827 } 2828 2829 return 0; 2830 } 2831 2832 static void cdns3_gadget_config(struct cdns3_device *priv_dev) 2833 { 2834 struct cdns3_usb_regs __iomem *regs = priv_dev->regs; 2835 u32 reg; 2836 2837 cdns3_ep0_config(priv_dev); 2838 2839 /* enable interrupts for endpoint 0 (in and out) */ 2840 writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, ®s->ep_ien); 2841 2842 /* 2843 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1 2844 * revision of controller. 2845 */ 2846 if (priv_dev->dev_ver == DEV_VER_TI_V1) { 2847 reg = readl(®s->dbg_link1); 2848 2849 reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK; 2850 reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) | 2851 DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET; 2852 writel(reg, ®s->dbg_link1); 2853 } 2854 2855 /* 2856 * By default some platforms has set protected access to memory. 2857 * This cause problem with cache, so driver restore non-secure 2858 * access to memory. 2859 */ 2860 reg = readl(®s->dma_axi_ctrl); 2861 reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) | 2862 DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE); 2863 writel(reg, ®s->dma_axi_ctrl); 2864 2865 /* enable generic interrupt*/ 2866 writel(USB_IEN_INIT, ®s->usb_ien); 2867 writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, ®s->usb_conf); 2868 /* keep Fast Access bit */ 2869 writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr); 2870 2871 cdns3_configure_dmult(priv_dev, NULL); 2872 } 2873 2874 /** 2875 * cdns3_gadget_udc_start - Gadget start 2876 * @gadget: gadget object 2877 * @driver: driver which operates on this gadget 2878 * 2879 * Returns 0 on success, error code elsewhere 2880 */ 2881 static int cdns3_gadget_udc_start(struct usb_gadget *gadget, 2882 struct usb_gadget_driver *driver) 2883 { 2884 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2885 unsigned long flags; 2886 enum usb_device_speed max_speed = driver->max_speed; 2887 2888 spin_lock_irqsave(&priv_dev->lock, flags); 2889 priv_dev->gadget_driver = driver; 2890 2891 /* limit speed if necessary */ 2892 max_speed = min(driver->max_speed, gadget->max_speed); 2893 2894 switch (max_speed) { 2895 case USB_SPEED_FULL: 2896 writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf); 2897 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); 2898 break; 2899 case USB_SPEED_HIGH: 2900 writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf); 2901 break; 2902 case USB_SPEED_SUPER: 2903 break; 2904 default: 2905 dev_err(priv_dev->dev, 2906 "invalid maximum_speed parameter %d\n", 2907 max_speed); 2908 fallthrough; 2909 case USB_SPEED_UNKNOWN: 2910 /* default to superspeed */ 2911 max_speed = USB_SPEED_SUPER; 2912 break; 2913 } 2914 2915 cdns3_gadget_config(priv_dev); 2916 spin_unlock_irqrestore(&priv_dev->lock, flags); 2917 return 0; 2918 } 2919 2920 /** 2921 * cdns3_gadget_udc_stop - Stops gadget 2922 * @gadget: gadget object 2923 * 2924 * Returns 0 2925 */ 2926 static int cdns3_gadget_udc_stop(struct usb_gadget *gadget) 2927 { 2928 struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget); 2929 struct cdns3_endpoint *priv_ep; 2930 u32 bEndpointAddress; 2931 struct usb_ep *ep; 2932 int val; 2933 2934 priv_dev->gadget_driver = NULL; 2935 2936 priv_dev->onchip_used_size = 0; 2937 priv_dev->out_mem_is_allocated = 0; 2938 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; 2939 2940 list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) { 2941 priv_ep = ep_to_cdns3_ep(ep); 2942 bEndpointAddress = priv_ep->num | priv_ep->dir; 2943 cdns3_select_ep(priv_dev, bEndpointAddress); 2944 writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd); 2945 readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val, 2946 !(val & EP_CMD_EPRST), 1, 100); 2947 2948 priv_ep->flags &= ~EP_CLAIMED; 2949 } 2950 2951 /* disable interrupt for device */ 2952 writel(0, &priv_dev->regs->usb_ien); 2953 writel(0, &priv_dev->regs->usb_pwr); 2954 writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf); 2955 2956 return 0; 2957 } 2958 2959 static const struct usb_gadget_ops cdns3_gadget_ops = { 2960 .get_frame = cdns3_gadget_get_frame, 2961 .wakeup = cdns3_gadget_wakeup, 2962 .set_selfpowered = cdns3_gadget_set_selfpowered, 2963 .pullup = cdns3_gadget_pullup, 2964 .udc_start = cdns3_gadget_udc_start, 2965 .udc_stop = cdns3_gadget_udc_stop, 2966 .match_ep = cdns3_gadget_match_ep, 2967 }; 2968 2969 static void cdns3_free_all_eps(struct cdns3_device *priv_dev) 2970 { 2971 int i; 2972 2973 /* ep0 OUT point to ep0 IN. */ 2974 priv_dev->eps[16] = NULL; 2975 2976 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) 2977 if (priv_dev->eps[i]) { 2978 cdns3_free_trb_pool(priv_dev->eps[i]); 2979 devm_kfree(priv_dev->dev, priv_dev->eps[i]); 2980 } 2981 } 2982 2983 /** 2984 * cdns3_init_eps - Initializes software endpoints of gadget 2985 * @priv_dev: extended gadget object 2986 * 2987 * Returns 0 on success, error code elsewhere 2988 */ 2989 static int cdns3_init_eps(struct cdns3_device *priv_dev) 2990 { 2991 u32 ep_enabled_reg, iso_ep_reg; 2992 struct cdns3_endpoint *priv_ep; 2993 int ep_dir, ep_number; 2994 u32 ep_mask; 2995 int ret = 0; 2996 int i; 2997 2998 /* Read it from USB_CAP3 to USB_CAP5 */ 2999 ep_enabled_reg = readl(&priv_dev->regs->usb_cap3); 3000 iso_ep_reg = readl(&priv_dev->regs->usb_cap4); 3001 3002 dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n"); 3003 3004 for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) { 3005 ep_dir = i >> 4; /* i div 16 */ 3006 ep_number = i & 0xF; /* i % 16 */ 3007 ep_mask = BIT(i); 3008 3009 if (!(ep_enabled_reg & ep_mask)) 3010 continue; 3011 3012 if (ep_dir && !ep_number) { 3013 priv_dev->eps[i] = priv_dev->eps[0]; 3014 continue; 3015 } 3016 3017 priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep), 3018 GFP_KERNEL); 3019 if (!priv_ep) 3020 goto err; 3021 3022 /* set parent of endpoint object */ 3023 priv_ep->cdns3_dev = priv_dev; 3024 priv_dev->eps[i] = priv_ep; 3025 priv_ep->num = ep_number; 3026 priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT; 3027 3028 if (!ep_number) { 3029 ret = cdns3_init_ep0(priv_dev, priv_ep); 3030 if (ret) { 3031 dev_err(priv_dev->dev, "Failed to init ep0\n"); 3032 goto err; 3033 } 3034 } else { 3035 snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s", 3036 ep_number, !!ep_dir ? "in" : "out"); 3037 priv_ep->endpoint.name = priv_ep->name; 3038 3039 usb_ep_set_maxpacket_limit(&priv_ep->endpoint, 3040 CDNS3_EP_MAX_PACKET_LIMIT); 3041 priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS; 3042 priv_ep->endpoint.ops = &cdns3_gadget_ep_ops; 3043 if (ep_dir) 3044 priv_ep->endpoint.caps.dir_in = 1; 3045 else 3046 priv_ep->endpoint.caps.dir_out = 1; 3047 3048 if (iso_ep_reg & ep_mask) 3049 priv_ep->endpoint.caps.type_iso = 1; 3050 3051 priv_ep->endpoint.caps.type_bulk = 1; 3052 priv_ep->endpoint.caps.type_int = 1; 3053 3054 list_add_tail(&priv_ep->endpoint.ep_list, 3055 &priv_dev->gadget.ep_list); 3056 } 3057 3058 priv_ep->flags = 0; 3059 3060 dev_dbg(priv_dev->dev, "Initialized %s support: %s %s\n", 3061 priv_ep->name, 3062 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "", 3063 priv_ep->endpoint.caps.type_iso ? "ISO" : ""); 3064 3065 INIT_LIST_HEAD(&priv_ep->pending_req_list); 3066 INIT_LIST_HEAD(&priv_ep->deferred_req_list); 3067 INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list); 3068 } 3069 3070 return 0; 3071 err: 3072 cdns3_free_all_eps(priv_dev); 3073 return -ENOMEM; 3074 } 3075 3076 static void cdns3_gadget_release(struct device *dev) 3077 { 3078 struct cdns3_device *priv_dev = container_of(dev, 3079 struct cdns3_device, gadget.dev); 3080 3081 kfree(priv_dev); 3082 } 3083 3084 static void cdns3_gadget_exit(struct cdns *cdns) 3085 { 3086 struct cdns3_device *priv_dev; 3087 3088 priv_dev = cdns->gadget_dev; 3089 3090 3091 pm_runtime_mark_last_busy(cdns->dev); 3092 pm_runtime_put_autosuspend(cdns->dev); 3093 3094 usb_del_gadget(&priv_dev->gadget); 3095 devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev); 3096 3097 cdns3_free_all_eps(priv_dev); 3098 3099 while (!list_empty(&priv_dev->aligned_buf_list)) { 3100 struct cdns3_aligned_buf *buf; 3101 3102 buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list); 3103 dma_free_noncoherent(priv_dev->sysdev, buf->size, 3104 buf->buf, 3105 buf->dma, 3106 buf->dir); 3107 3108 list_del(&buf->list); 3109 kfree(buf); 3110 } 3111 3112 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, 3113 priv_dev->setup_dma); 3114 dma_pool_destroy(priv_dev->eps_dma_pool); 3115 3116 kfree(priv_dev->zlp_buf); 3117 usb_put_gadget(&priv_dev->gadget); 3118 cdns->gadget_dev = NULL; 3119 cdns_drd_gadget_off(cdns); 3120 } 3121 3122 static int cdns3_gadget_start(struct cdns *cdns) 3123 { 3124 struct cdns3_device *priv_dev; 3125 u32 max_speed; 3126 int ret; 3127 3128 priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL); 3129 if (!priv_dev) 3130 return -ENOMEM; 3131 3132 usb_initialize_gadget(cdns->dev, &priv_dev->gadget, 3133 cdns3_gadget_release); 3134 cdns->gadget_dev = priv_dev; 3135 priv_dev->sysdev = cdns->dev; 3136 priv_dev->dev = cdns->dev; 3137 priv_dev->regs = cdns->dev_regs; 3138 3139 device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size", 3140 &priv_dev->onchip_buffers); 3141 3142 if (priv_dev->onchip_buffers <= 0) { 3143 u32 reg = readl(&priv_dev->regs->usb_cap2); 3144 3145 priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg); 3146 } 3147 3148 if (!priv_dev->onchip_buffers) 3149 priv_dev->onchip_buffers = 256; 3150 3151 max_speed = usb_get_maximum_speed(cdns->dev); 3152 3153 /* Check the maximum_speed parameter */ 3154 switch (max_speed) { 3155 case USB_SPEED_FULL: 3156 case USB_SPEED_HIGH: 3157 case USB_SPEED_SUPER: 3158 break; 3159 default: 3160 dev_err(cdns->dev, "invalid maximum_speed parameter %d\n", 3161 max_speed); 3162 fallthrough; 3163 case USB_SPEED_UNKNOWN: 3164 /* default to superspeed */ 3165 max_speed = USB_SPEED_SUPER; 3166 break; 3167 } 3168 3169 /* fill gadget fields */ 3170 priv_dev->gadget.max_speed = max_speed; 3171 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; 3172 priv_dev->gadget.ops = &cdns3_gadget_ops; 3173 priv_dev->gadget.name = "usb-ss-gadget"; 3174 priv_dev->gadget.quirk_avoids_skb_reserve = 1; 3175 priv_dev->gadget.irq = cdns->dev_irq; 3176 3177 spin_lock_init(&priv_dev->lock); 3178 INIT_WORK(&priv_dev->pending_status_wq, 3179 cdns3_pending_setup_status_handler); 3180 3181 INIT_WORK(&priv_dev->aligned_buf_wq, 3182 cdns3_free_aligned_request_buf); 3183 3184 /* initialize endpoint container */ 3185 INIT_LIST_HEAD(&priv_dev->gadget.ep_list); 3186 INIT_LIST_HEAD(&priv_dev->aligned_buf_list); 3187 priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool", 3188 priv_dev->sysdev, 3189 TRB_RING_SIZE, 8, 0); 3190 if (!priv_dev->eps_dma_pool) { 3191 dev_err(priv_dev->dev, "Failed to create TRB dma pool\n"); 3192 ret = -ENOMEM; 3193 goto err1; 3194 } 3195 3196 ret = cdns3_init_eps(priv_dev); 3197 if (ret) { 3198 dev_err(priv_dev->dev, "Failed to create endpoints\n"); 3199 goto err1; 3200 } 3201 3202 /* allocate memory for setup packet buffer */ 3203 priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8, 3204 &priv_dev->setup_dma, GFP_DMA); 3205 if (!priv_dev->setup_buf) { 3206 ret = -ENOMEM; 3207 goto err2; 3208 } 3209 3210 priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6); 3211 3212 dev_dbg(priv_dev->dev, "Device Controller version: %08x\n", 3213 readl(&priv_dev->regs->usb_cap6)); 3214 dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n", 3215 readl(&priv_dev->regs->usb_cap1)); 3216 dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n", 3217 readl(&priv_dev->regs->usb_cap2)); 3218 3219 priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver); 3220 if (priv_dev->dev_ver >= DEV_VER_V2) 3221 priv_dev->gadget.sg_supported = 1; 3222 3223 priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL); 3224 if (!priv_dev->zlp_buf) { 3225 ret = -ENOMEM; 3226 goto err3; 3227 } 3228 3229 /* add USB gadget device */ 3230 ret = usb_add_gadget(&priv_dev->gadget); 3231 if (ret < 0) { 3232 dev_err(priv_dev->dev, "Failed to add gadget\n"); 3233 goto err4; 3234 } 3235 3236 return 0; 3237 err4: 3238 kfree(priv_dev->zlp_buf); 3239 err3: 3240 dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf, 3241 priv_dev->setup_dma); 3242 err2: 3243 cdns3_free_all_eps(priv_dev); 3244 err1: 3245 dma_pool_destroy(priv_dev->eps_dma_pool); 3246 3247 usb_put_gadget(&priv_dev->gadget); 3248 cdns->gadget_dev = NULL; 3249 return ret; 3250 } 3251 3252 static int __cdns3_gadget_init(struct cdns *cdns) 3253 { 3254 int ret = 0; 3255 3256 /* Ensure 32-bit DMA Mask in case we switched back from Host mode */ 3257 ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32)); 3258 if (ret) { 3259 dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret); 3260 return ret; 3261 } 3262 3263 cdns_drd_gadget_on(cdns); 3264 pm_runtime_get_sync(cdns->dev); 3265 3266 ret = cdns3_gadget_start(cdns); 3267 if (ret) { 3268 pm_runtime_put_sync(cdns->dev); 3269 return ret; 3270 } 3271 3272 /* 3273 * Because interrupt line can be shared with other components in 3274 * driver it can't use IRQF_ONESHOT flag here. 3275 */ 3276 ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq, 3277 cdns3_device_irq_handler, 3278 cdns3_device_thread_irq_handler, 3279 IRQF_SHARED, dev_name(cdns->dev), 3280 cdns->gadget_dev); 3281 3282 if (ret) 3283 goto err0; 3284 3285 return 0; 3286 err0: 3287 cdns3_gadget_exit(cdns); 3288 return ret; 3289 } 3290 3291 static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup) 3292 __must_hold(&cdns->lock) 3293 { 3294 struct cdns3_device *priv_dev = cdns->gadget_dev; 3295 3296 spin_unlock(&cdns->lock); 3297 cdns3_disconnect_gadget(priv_dev); 3298 spin_lock(&cdns->lock); 3299 3300 priv_dev->gadget.speed = USB_SPEED_UNKNOWN; 3301 usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED); 3302 cdns3_hw_reset_eps_config(priv_dev); 3303 3304 /* disable interrupt for device */ 3305 writel(0, &priv_dev->regs->usb_ien); 3306 3307 return 0; 3308 } 3309 3310 static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated) 3311 { 3312 struct cdns3_device *priv_dev = cdns->gadget_dev; 3313 3314 if (!priv_dev->gadget_driver) 3315 return 0; 3316 3317 cdns3_gadget_config(priv_dev); 3318 if (hibernated) 3319 writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf); 3320 3321 return 0; 3322 } 3323 3324 /** 3325 * cdns3_gadget_init - initialize device structure 3326 * 3327 * @cdns: cdns instance 3328 * 3329 * This function initializes the gadget. 3330 */ 3331 int cdns3_gadget_init(struct cdns *cdns) 3332 { 3333 struct cdns_role_driver *rdrv; 3334 3335 rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL); 3336 if (!rdrv) 3337 return -ENOMEM; 3338 3339 rdrv->start = __cdns3_gadget_init; 3340 rdrv->stop = cdns3_gadget_exit; 3341 rdrv->suspend = cdns3_gadget_suspend; 3342 rdrv->resume = cdns3_gadget_resume; 3343 rdrv->state = CDNS_ROLE_STATE_INACTIVE; 3344 rdrv->name = "gadget"; 3345 cdns->roles[USB_ROLE_DEVICE] = rdrv; 3346 3347 return 0; 3348 } 3349