1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Altera TSE SGDMA and MSGDMA Linux driver 3 * Copyright (C) 2014 Altera Corporation. All rights reserved 4 */ 5 6 #include <linux/list.h> 7 #include "altera_utils.h" 8 #include "altera_tse.h" 9 #include "altera_sgdmahw.h" 10 #include "altera_sgdma.h" 11 12 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc, 13 struct sgdma_descrip __iomem *ndesc, 14 dma_addr_t ndesc_phys, 15 dma_addr_t raddr, 16 dma_addr_t waddr, 17 u16 length, 18 int generate_eop, 19 int rfixed, 20 int wfixed); 21 22 static int sgdma_async_write(struct altera_tse_private *priv, 23 struct sgdma_descrip __iomem *desc); 24 25 static int sgdma_async_read(struct altera_tse_private *priv); 26 27 static dma_addr_t 28 sgdma_txphysaddr(struct altera_tse_private *priv, 29 struct sgdma_descrip __iomem *desc); 30 31 static dma_addr_t 32 sgdma_rxphysaddr(struct altera_tse_private *priv, 33 struct sgdma_descrip __iomem *desc); 34 35 static int sgdma_txbusy(struct altera_tse_private *priv); 36 37 static int sgdma_rxbusy(struct altera_tse_private *priv); 38 39 static void 40 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer); 41 42 static void 43 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer); 44 45 static struct tse_buffer * 46 dequeue_tx(struct altera_tse_private *priv); 47 48 static struct tse_buffer * 49 dequeue_rx(struct altera_tse_private *priv); 50 51 static struct tse_buffer * 52 queue_rx_peekhead(struct altera_tse_private *priv); 53 54 int sgdma_initialize(struct altera_tse_private *priv) 55 { 56 priv->txctrlreg = SGDMA_CTRLREG_ILASTD | 57 SGDMA_CTRLREG_INTEN; 58 59 priv->rxctrlreg = SGDMA_CTRLREG_IDESCRIP | 60 SGDMA_CTRLREG_INTEN | 61 SGDMA_CTRLREG_ILASTD; 62 63 INIT_LIST_HEAD(&priv->txlisthd); 64 INIT_LIST_HEAD(&priv->rxlisthd); 65 66 priv->rxdescphys = (dma_addr_t) 0; 67 priv->txdescphys = (dma_addr_t) 0; 68 69 priv->rxdescphys = dma_map_single(priv->device, 70 (void __force *)priv->rx_dma_desc, 71 priv->rxdescmem, DMA_BIDIRECTIONAL); 72 73 if (dma_mapping_error(priv->device, priv->rxdescphys)) { 74 sgdma_uninitialize(priv); 75 netdev_err(priv->dev, "error mapping rx descriptor memory\n"); 76 return -EINVAL; 77 } 78 79 priv->txdescphys = dma_map_single(priv->device, 80 (void __force *)priv->tx_dma_desc, 81 priv->txdescmem, DMA_TO_DEVICE); 82 83 if (dma_mapping_error(priv->device, priv->txdescphys)) { 84 sgdma_uninitialize(priv); 85 netdev_err(priv->dev, "error mapping tx descriptor memory\n"); 86 return -EINVAL; 87 } 88 89 /* Initialize descriptor memory to all 0's, sync memory to cache */ 90 memset_io(priv->tx_dma_desc, 0, priv->txdescmem); 91 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem); 92 93 dma_sync_single_for_device(priv->device, priv->txdescphys, 94 priv->txdescmem, DMA_TO_DEVICE); 95 96 dma_sync_single_for_device(priv->device, priv->rxdescphys, 97 priv->rxdescmem, DMA_TO_DEVICE); 98 99 return 0; 100 } 101 102 void sgdma_uninitialize(struct altera_tse_private *priv) 103 { 104 if (priv->rxdescphys) 105 dma_unmap_single(priv->device, priv->rxdescphys, 106 priv->rxdescmem, DMA_BIDIRECTIONAL); 107 108 if (priv->txdescphys) 109 dma_unmap_single(priv->device, priv->txdescphys, 110 priv->txdescmem, DMA_TO_DEVICE); 111 } 112 113 /* This function resets the SGDMA controller and clears the 114 * descriptor memory used for transmits and receives. 115 */ 116 void sgdma_reset(struct altera_tse_private *priv) 117 { 118 /* Initialize descriptor memory to 0 */ 119 memset_io(priv->tx_dma_desc, 0, priv->txdescmem); 120 memset_io(priv->rx_dma_desc, 0, priv->rxdescmem); 121 122 csrwr32(SGDMA_CTRLREG_RESET, priv->tx_dma_csr, sgdma_csroffs(control)); 123 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control)); 124 125 csrwr32(SGDMA_CTRLREG_RESET, priv->rx_dma_csr, sgdma_csroffs(control)); 126 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control)); 127 } 128 129 /* For SGDMA, interrupts remain enabled after initially enabling, 130 * so no need to provide implementations for abstract enable 131 * and disable 132 */ 133 134 void sgdma_enable_rxirq(struct altera_tse_private *priv) 135 { 136 } 137 138 void sgdma_enable_txirq(struct altera_tse_private *priv) 139 { 140 } 141 142 void sgdma_disable_rxirq(struct altera_tse_private *priv) 143 { 144 } 145 146 void sgdma_disable_txirq(struct altera_tse_private *priv) 147 { 148 } 149 150 void sgdma_clear_rxirq(struct altera_tse_private *priv) 151 { 152 tse_set_bit(priv->rx_dma_csr, sgdma_csroffs(control), 153 SGDMA_CTRLREG_CLRINT); 154 } 155 156 void sgdma_clear_txirq(struct altera_tse_private *priv) 157 { 158 tse_set_bit(priv->tx_dma_csr, sgdma_csroffs(control), 159 SGDMA_CTRLREG_CLRINT); 160 } 161 162 /* transmits buffer through SGDMA. Returns number of buffers 163 * transmitted, 0 if not possible. 164 * 165 * tx_lock is held by the caller 166 */ 167 int sgdma_tx_buffer(struct altera_tse_private *priv, struct tse_buffer *buffer) 168 { 169 struct sgdma_descrip __iomem *descbase = 170 (struct sgdma_descrip __iomem *)priv->tx_dma_desc; 171 172 struct sgdma_descrip __iomem *cdesc = &descbase[0]; 173 struct sgdma_descrip __iomem *ndesc = &descbase[1]; 174 175 /* wait 'til the tx sgdma is ready for the next transmit request */ 176 if (sgdma_txbusy(priv)) 177 return 0; 178 179 sgdma_setup_descrip(cdesc, /* current descriptor */ 180 ndesc, /* next descriptor */ 181 sgdma_txphysaddr(priv, ndesc), 182 buffer->dma_addr, /* address of packet to xmit */ 183 0, /* write addr 0 for tx dma */ 184 buffer->len, /* length of packet */ 185 SGDMA_CONTROL_EOP, /* Generate EOP */ 186 0, /* read fixed */ 187 SGDMA_CONTROL_WR_FIXED); /* Generate SOP */ 188 189 sgdma_async_write(priv, cdesc); 190 191 /* enqueue the request to the pending transmit queue */ 192 queue_tx(priv, buffer); 193 194 return 1; 195 } 196 197 198 /* tx_lock held to protect access to queued tx list 199 */ 200 u32 sgdma_tx_completions(struct altera_tse_private *priv) 201 { 202 u32 ready = 0; 203 204 if (!sgdma_txbusy(priv) && 205 ((csrrd8(priv->tx_dma_desc, sgdma_descroffs(control)) 206 & SGDMA_CONTROL_HW_OWNED) == 0) && 207 (dequeue_tx(priv))) { 208 ready = 1; 209 } 210 211 return ready; 212 } 213 214 void sgdma_start_rxdma(struct altera_tse_private *priv) 215 { 216 sgdma_async_read(priv); 217 } 218 219 void sgdma_add_rx_desc(struct altera_tse_private *priv, 220 struct tse_buffer *rxbuffer) 221 { 222 queue_rx(priv, rxbuffer); 223 } 224 225 /* status is returned on upper 16 bits, 226 * length is returned in lower 16 bits 227 */ 228 u32 sgdma_rx_status(struct altera_tse_private *priv) 229 { 230 struct sgdma_descrip __iomem *base = 231 (struct sgdma_descrip __iomem *)priv->rx_dma_desc; 232 struct sgdma_descrip __iomem *desc = NULL; 233 struct tse_buffer *rxbuffer = NULL; 234 unsigned int rxstatus = 0; 235 236 u32 sts = csrrd32(priv->rx_dma_csr, sgdma_csroffs(status)); 237 238 desc = &base[0]; 239 if (sts & SGDMA_STSREG_EOP) { 240 unsigned int pktlength = 0; 241 unsigned int pktstatus = 0; 242 dma_sync_single_for_cpu(priv->device, 243 priv->rxdescphys, 244 SGDMA_DESC_LEN, 245 DMA_FROM_DEVICE); 246 247 pktlength = csrrd16(desc, sgdma_descroffs(bytes_xferred)); 248 pktstatus = csrrd8(desc, sgdma_descroffs(status)); 249 rxstatus = pktstatus & ~SGDMA_STATUS_EOP; 250 rxstatus = rxstatus << 16; 251 rxstatus |= (pktlength & 0xffff); 252 253 if (rxstatus) { 254 csrwr8(0, desc, sgdma_descroffs(status)); 255 256 rxbuffer = dequeue_rx(priv); 257 if (rxbuffer == NULL) 258 netdev_info(priv->dev, 259 "sgdma rx and rx queue empty!\n"); 260 261 /* Clear control */ 262 csrwr32(0, priv->rx_dma_csr, sgdma_csroffs(control)); 263 /* clear status */ 264 csrwr32(0xf, priv->rx_dma_csr, sgdma_csroffs(status)); 265 266 /* kick the rx sgdma after reaping this descriptor */ 267 sgdma_async_read(priv); 268 269 } else { 270 /* If the SGDMA indicated an end of packet on recv, 271 * then it's expected that the rxstatus from the 272 * descriptor is non-zero - meaning a valid packet 273 * with a nonzero length, or an error has been 274 * indicated. if not, then all we can do is signal 275 * an error and return no packet received. Most likely 276 * there is a system design error, or an error in the 277 * underlying kernel (cache or cache management problem) 278 */ 279 netdev_err(priv->dev, 280 "SGDMA RX Error Info: %x, %x, %x\n", 281 sts, csrrd8(desc, sgdma_descroffs(status)), 282 rxstatus); 283 } 284 } else if (sts == 0) { 285 sgdma_async_read(priv); 286 } 287 288 return rxstatus; 289 } 290 291 292 /* Private functions */ 293 static void sgdma_setup_descrip(struct sgdma_descrip __iomem *desc, 294 struct sgdma_descrip __iomem *ndesc, 295 dma_addr_t ndesc_phys, 296 dma_addr_t raddr, 297 dma_addr_t waddr, 298 u16 length, 299 int generate_eop, 300 int rfixed, 301 int wfixed) 302 { 303 /* Clear the next descriptor as not owned by hardware */ 304 305 u32 ctrl = csrrd8(ndesc, sgdma_descroffs(control)); 306 ctrl &= ~SGDMA_CONTROL_HW_OWNED; 307 csrwr8(ctrl, ndesc, sgdma_descroffs(control)); 308 309 ctrl = SGDMA_CONTROL_HW_OWNED; 310 ctrl |= generate_eop; 311 ctrl |= rfixed; 312 ctrl |= wfixed; 313 314 /* Channel is implicitly zero, initialized to 0 by default */ 315 csrwr32(lower_32_bits(raddr), desc, sgdma_descroffs(raddr)); 316 csrwr32(lower_32_bits(waddr), desc, sgdma_descroffs(waddr)); 317 318 csrwr32(0, desc, sgdma_descroffs(pad1)); 319 csrwr32(0, desc, sgdma_descroffs(pad2)); 320 csrwr32(lower_32_bits(ndesc_phys), desc, sgdma_descroffs(next)); 321 322 csrwr8(ctrl, desc, sgdma_descroffs(control)); 323 csrwr8(0, desc, sgdma_descroffs(status)); 324 csrwr8(0, desc, sgdma_descroffs(wburst)); 325 csrwr8(0, desc, sgdma_descroffs(rburst)); 326 csrwr16(length, desc, sgdma_descroffs(bytes)); 327 csrwr16(0, desc, sgdma_descroffs(bytes_xferred)); 328 } 329 330 /* If hardware is busy, don't restart async read. 331 * if status register is 0 - meaning initial state, restart async read, 332 * probably for the first time when populating a receive buffer. 333 * If read status indicate not busy and a status, restart the async 334 * DMA read. 335 */ 336 static int sgdma_async_read(struct altera_tse_private *priv) 337 { 338 struct sgdma_descrip __iomem *descbase = 339 (struct sgdma_descrip __iomem *)priv->rx_dma_desc; 340 341 struct sgdma_descrip __iomem *cdesc = &descbase[0]; 342 struct sgdma_descrip __iomem *ndesc = &descbase[1]; 343 struct tse_buffer *rxbuffer = NULL; 344 345 if (!sgdma_rxbusy(priv)) { 346 rxbuffer = queue_rx_peekhead(priv); 347 if (rxbuffer == NULL) { 348 netdev_err(priv->dev, "no rx buffers available\n"); 349 return 0; 350 } 351 352 sgdma_setup_descrip(cdesc, /* current descriptor */ 353 ndesc, /* next descriptor */ 354 sgdma_rxphysaddr(priv, ndesc), 355 0, /* read addr 0 for rx dma */ 356 rxbuffer->dma_addr, /* write addr for rx dma */ 357 0, /* read 'til EOP */ 358 0, /* EOP: NA for rx dma */ 359 0, /* read fixed: NA for rx dma */ 360 0); /* SOP: NA for rx DMA */ 361 362 dma_sync_single_for_device(priv->device, 363 priv->rxdescphys, 364 SGDMA_DESC_LEN, 365 DMA_TO_DEVICE); 366 367 csrwr32(lower_32_bits(sgdma_rxphysaddr(priv, cdesc)), 368 priv->rx_dma_csr, 369 sgdma_csroffs(next_descrip)); 370 371 csrwr32((priv->rxctrlreg | SGDMA_CTRLREG_START), 372 priv->rx_dma_csr, 373 sgdma_csroffs(control)); 374 375 return 1; 376 } 377 378 return 0; 379 } 380 381 static int sgdma_async_write(struct altera_tse_private *priv, 382 struct sgdma_descrip __iomem *desc) 383 { 384 if (sgdma_txbusy(priv)) 385 return 0; 386 387 /* clear control and status */ 388 csrwr32(0, priv->tx_dma_csr, sgdma_csroffs(control)); 389 csrwr32(0x1f, priv->tx_dma_csr, sgdma_csroffs(status)); 390 391 dma_sync_single_for_device(priv->device, priv->txdescphys, 392 SGDMA_DESC_LEN, DMA_TO_DEVICE); 393 394 csrwr32(lower_32_bits(sgdma_txphysaddr(priv, desc)), 395 priv->tx_dma_csr, 396 sgdma_csroffs(next_descrip)); 397 398 csrwr32((priv->txctrlreg | SGDMA_CTRLREG_START), 399 priv->tx_dma_csr, 400 sgdma_csroffs(control)); 401 402 return 1; 403 } 404 405 static dma_addr_t 406 sgdma_txphysaddr(struct altera_tse_private *priv, 407 struct sgdma_descrip __iomem *desc) 408 { 409 dma_addr_t paddr = priv->txdescmem_busaddr; 410 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->tx_dma_desc; 411 return (dma_addr_t)((uintptr_t)paddr + offs); 412 } 413 414 static dma_addr_t 415 sgdma_rxphysaddr(struct altera_tse_private *priv, 416 struct sgdma_descrip __iomem *desc) 417 { 418 dma_addr_t paddr = priv->rxdescmem_busaddr; 419 uintptr_t offs = (uintptr_t)desc - (uintptr_t)priv->rx_dma_desc; 420 return (dma_addr_t)((uintptr_t)paddr + offs); 421 } 422 423 #define list_remove_head(list, entry, type, member) \ 424 do { \ 425 entry = NULL; \ 426 if (!list_empty(list)) { \ 427 entry = list_entry((list)->next, type, member); \ 428 list_del_init(&entry->member); \ 429 } \ 430 } while (0) 431 432 #define list_peek_head(list, entry, type, member) \ 433 do { \ 434 entry = NULL; \ 435 if (!list_empty(list)) { \ 436 entry = list_entry((list)->next, type, member); \ 437 } \ 438 } while (0) 439 440 /* adds a tse_buffer to the tail of a tx buffer list. 441 * assumes the caller is managing and holding a mutual exclusion 442 * primitive to avoid simultaneous pushes/pops to the list. 443 */ 444 static void 445 queue_tx(struct altera_tse_private *priv, struct tse_buffer *buffer) 446 { 447 list_add_tail(&buffer->lh, &priv->txlisthd); 448 } 449 450 451 /* adds a tse_buffer to the tail of a rx buffer list 452 * assumes the caller is managing and holding a mutual exclusion 453 * primitive to avoid simultaneous pushes/pops to the list. 454 */ 455 static void 456 queue_rx(struct altera_tse_private *priv, struct tse_buffer *buffer) 457 { 458 list_add_tail(&buffer->lh, &priv->rxlisthd); 459 } 460 461 /* dequeues a tse_buffer from the transmit buffer list, otherwise 462 * returns NULL if empty. 463 * assumes the caller is managing and holding a mutual exclusion 464 * primitive to avoid simultaneous pushes/pops to the list. 465 */ 466 static struct tse_buffer * 467 dequeue_tx(struct altera_tse_private *priv) 468 { 469 struct tse_buffer *buffer = NULL; 470 list_remove_head(&priv->txlisthd, buffer, struct tse_buffer, lh); 471 return buffer; 472 } 473 474 /* dequeues a tse_buffer from the receive buffer list, otherwise 475 * returns NULL if empty 476 * assumes the caller is managing and holding a mutual exclusion 477 * primitive to avoid simultaneous pushes/pops to the list. 478 */ 479 static struct tse_buffer * 480 dequeue_rx(struct altera_tse_private *priv) 481 { 482 struct tse_buffer *buffer = NULL; 483 list_remove_head(&priv->rxlisthd, buffer, struct tse_buffer, lh); 484 return buffer; 485 } 486 487 /* dequeues a tse_buffer from the receive buffer list, otherwise 488 * returns NULL if empty 489 * assumes the caller is managing and holding a mutual exclusion 490 * primitive to avoid simultaneous pushes/pops to the list while the 491 * head is being examined. 492 */ 493 static struct tse_buffer * 494 queue_rx_peekhead(struct altera_tse_private *priv) 495 { 496 struct tse_buffer *buffer = NULL; 497 list_peek_head(&priv->rxlisthd, buffer, struct tse_buffer, lh); 498 return buffer; 499 } 500 501 /* check and return rx sgdma status without polling 502 */ 503 static int sgdma_rxbusy(struct altera_tse_private *priv) 504 { 505 return csrrd32(priv->rx_dma_csr, sgdma_csroffs(status)) 506 & SGDMA_STSREG_BUSY; 507 } 508 509 /* waits for the tx sgdma to finish it's current operation, returns 0 510 * when it transitions to nonbusy, returns 1 if the operation times out 511 */ 512 static int sgdma_txbusy(struct altera_tse_private *priv) 513 { 514 int delay = 0; 515 516 /* if DMA is busy, wait for current transaction to finish */ 517 while ((csrrd32(priv->tx_dma_csr, sgdma_csroffs(status)) 518 & SGDMA_STSREG_BUSY) && (delay++ < 100)) 519 udelay(1); 520 521 if (csrrd32(priv->tx_dma_csr, sgdma_csroffs(status)) 522 & SGDMA_STSREG_BUSY) { 523 netdev_err(priv->dev, "timeout waiting for tx dma\n"); 524 return 1; 525 } 526 return 0; 527 } 528