1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * RapidIO mport driver for Tsi721 PCIExpress-to-SRIO bridge 4 * 5 * Copyright 2011 Integrated Device Technology, Inc. 6 * Alexandre Bounine <alexandre.bounine@idt.com> 7 * Chul Kim <chul.kim@idt.com> 8 */ 9 10 #include <linux/io.h> 11 #include <linux/errno.h> 12 #include <linux/init.h> 13 #include <linux/ioport.h> 14 #include <linux/kernel.h> 15 #include <linux/module.h> 16 #include <linux/pci.h> 17 #include <linux/rio.h> 18 #include <linux/rio_drv.h> 19 #include <linux/dma-mapping.h> 20 #include <linux/interrupt.h> 21 #include <linux/kfifo.h> 22 #include <linux/delay.h> 23 24 #include "tsi721.h" 25 26 #ifdef DEBUG 27 u32 tsi_dbg_level; 28 module_param_named(dbg_level, tsi_dbg_level, uint, S_IWUSR | S_IRUGO); 29 MODULE_PARM_DESC(dbg_level, "Debugging output level (default 0 = none)"); 30 #endif 31 32 static int pcie_mrrs = -1; 33 module_param(pcie_mrrs, int, S_IRUGO); 34 MODULE_PARM_DESC(pcie_mrrs, "PCIe MRRS override value (0...5)"); 35 36 static u8 mbox_sel = 0x0f; 37 module_param(mbox_sel, byte, S_IRUGO); 38 MODULE_PARM_DESC(mbox_sel, 39 "RIO Messaging MBOX Selection Mask (default: 0x0f = all)"); 40 41 static DEFINE_SPINLOCK(tsi721_maint_lock); 42 43 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch); 44 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch); 45 46 /** 47 * tsi721_lcread - read from local SREP config space 48 * @mport: RapidIO master port info 49 * @index: ID of RapdiIO interface 50 * @offset: Offset into configuration space 51 * @len: Length (in bytes) of the maintenance transaction 52 * @data: Value to be read into 53 * 54 * Generates a local SREP space read. 55 * 56 * Returns: %0 on success or %-EINVAL on failure. 57 */ 58 static int tsi721_lcread(struct rio_mport *mport, int index, u32 offset, 59 int len, u32 *data) 60 { 61 struct tsi721_device *priv = mport->priv; 62 63 if (len != sizeof(u32)) 64 return -EINVAL; /* only 32-bit access is supported */ 65 66 *data = ioread32(priv->regs + offset); 67 68 return 0; 69 } 70 71 /** 72 * tsi721_lcwrite - write into local SREP config space 73 * @mport: RapidIO master port info 74 * @index: ID of RapdiIO interface 75 * @offset: Offset into configuration space 76 * @len: Length (in bytes) of the maintenance transaction 77 * @data: Value to be written 78 * 79 * Generates a local write into SREP configuration space. 80 * 81 * Returns: %0 on success or %-EINVAL on failure. 82 */ 83 static int tsi721_lcwrite(struct rio_mport *mport, int index, u32 offset, 84 int len, u32 data) 85 { 86 struct tsi721_device *priv = mport->priv; 87 88 if (len != sizeof(u32)) 89 return -EINVAL; /* only 32-bit access is supported */ 90 91 iowrite32(data, priv->regs + offset); 92 93 return 0; 94 } 95 96 /** 97 * tsi721_maint_dma - Helper function to generate RapidIO maintenance 98 * transactions using designated Tsi721 DMA channel. 99 * @priv: pointer to tsi721 private data 100 * @sys_size: RapdiIO transport system size 101 * @destid: Destination ID of transaction 102 * @hopcount: Number of hops to target device 103 * @offset: Offset into configuration space 104 * @len: Length (in bytes) of the maintenance transaction 105 * @data: Location to be read from or write into 106 * @do_wr: Operation flag (1 == MAINT_WR) 107 * 108 * Generates a RapidIO maintenance transaction (Read or Write). 109 * Returns: %0 on success and %-EINVAL or %-EFAULT on failure. 110 */ 111 static int tsi721_maint_dma(struct tsi721_device *priv, u32 sys_size, 112 u16 destid, u8 hopcount, u32 offset, int len, 113 u32 *data, int do_wr) 114 { 115 void __iomem *regs = priv->regs + TSI721_DMAC_BASE(priv->mdma.ch_id); 116 struct tsi721_dma_desc *bd_ptr; 117 u32 rd_count, swr_ptr, ch_stat; 118 unsigned long flags; 119 int i, err = 0; 120 u32 op = do_wr ? MAINT_WR : MAINT_RD; 121 122 if (offset > (RIO_MAINT_SPACE_SZ - len) || (len != sizeof(u32))) 123 return -EINVAL; 124 125 spin_lock_irqsave(&tsi721_maint_lock, flags); 126 127 bd_ptr = priv->mdma.bd_base; 128 129 rd_count = ioread32(regs + TSI721_DMAC_DRDCNT); 130 131 /* Initialize DMA descriptor */ 132 bd_ptr[0].type_id = cpu_to_le32((DTYPE2 << 29) | (op << 19) | destid); 133 bd_ptr[0].bcount = cpu_to_le32((sys_size << 26) | 0x04); 134 bd_ptr[0].raddr_lo = cpu_to_le32((hopcount << 24) | offset); 135 bd_ptr[0].raddr_hi = 0; 136 if (do_wr) 137 bd_ptr[0].data[0] = cpu_to_be32p(data); 138 else 139 bd_ptr[0].data[0] = 0xffffffff; 140 141 mb(); 142 143 /* Start DMA operation */ 144 iowrite32(rd_count + 2, regs + TSI721_DMAC_DWRCNT); 145 ioread32(regs + TSI721_DMAC_DWRCNT); 146 i = 0; 147 148 /* Wait until DMA transfer is finished */ 149 while ((ch_stat = ioread32(regs + TSI721_DMAC_STS)) 150 & TSI721_DMAC_STS_RUN) { 151 udelay(1); 152 if (++i >= 5000000) { 153 tsi_debug(MAINT, &priv->pdev->dev, 154 "DMA[%d] read timeout ch_status=%x", 155 priv->mdma.ch_id, ch_stat); 156 if (!do_wr) 157 *data = 0xffffffff; 158 err = -EIO; 159 goto err_out; 160 } 161 } 162 163 if (ch_stat & TSI721_DMAC_STS_ABORT) { 164 /* If DMA operation aborted due to error, 165 * reinitialize DMA channel 166 */ 167 tsi_debug(MAINT, &priv->pdev->dev, "DMA ABORT ch_stat=%x", 168 ch_stat); 169 tsi_debug(MAINT, &priv->pdev->dev, 170 "OP=%d : destid=%x hc=%x off=%x", 171 do_wr ? MAINT_WR : MAINT_RD, 172 destid, hopcount, offset); 173 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT); 174 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL); 175 udelay(10); 176 iowrite32(0, regs + TSI721_DMAC_DWRCNT); 177 udelay(1); 178 if (!do_wr) 179 *data = 0xffffffff; 180 err = -EIO; 181 goto err_out; 182 } 183 184 if (!do_wr) 185 *data = be32_to_cpu(bd_ptr[0].data[0]); 186 187 /* 188 * Update descriptor status FIFO RD pointer. 189 * NOTE: Skipping check and clear FIFO entries because we are waiting 190 * for transfer to be completed. 191 */ 192 swr_ptr = ioread32(regs + TSI721_DMAC_DSWP); 193 iowrite32(swr_ptr, regs + TSI721_DMAC_DSRP); 194 195 err_out: 196 spin_unlock_irqrestore(&tsi721_maint_lock, flags); 197 198 return err; 199 } 200 201 /** 202 * tsi721_cread_dma - Generate a RapidIO maintenance read transaction 203 * using Tsi721 BDMA engine. 204 * @mport: RapidIO master port control structure 205 * @index: ID of RapdiIO interface 206 * @destid: Destination ID of transaction 207 * @hopcount: Number of hops to target device 208 * @offset: Offset into configuration space 209 * @len: Length (in bytes) of the maintenance transaction 210 * @data: Location to be read into 211 * 212 * Generates a RapidIO maintenance read transaction. 213 * Returns: %0 on success and %-EINVAL or %-EFAULT on failure. 214 */ 215 static int tsi721_cread_dma(struct rio_mport *mport, int index, u16 destid, 216 u8 hopcount, u32 offset, int len, u32 *data) 217 { 218 struct tsi721_device *priv = mport->priv; 219 220 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount, 221 offset, len, data, 0); 222 } 223 224 /** 225 * tsi721_cwrite_dma - Generate a RapidIO maintenance write transaction 226 * using Tsi721 BDMA engine 227 * @mport: RapidIO master port control structure 228 * @index: ID of RapdiIO interface 229 * @destid: Destination ID of transaction 230 * @hopcount: Number of hops to target device 231 * @offset: Offset into configuration space 232 * @len: Length (in bytes) of the maintenance transaction 233 * @data: Value to be written 234 * 235 * Generates a RapidIO maintenance write transaction. 236 * Returns: %0 on success and %-EINVAL or %-EFAULT on failure. 237 */ 238 static int tsi721_cwrite_dma(struct rio_mport *mport, int index, u16 destid, 239 u8 hopcount, u32 offset, int len, u32 data) 240 { 241 struct tsi721_device *priv = mport->priv; 242 u32 temp = data; 243 244 return tsi721_maint_dma(priv, mport->sys_size, destid, hopcount, 245 offset, len, &temp, 1); 246 } 247 248 /** 249 * tsi721_pw_handler - Tsi721 inbound port-write interrupt handler 250 * @priv: tsi721 device private structure 251 * 252 * Handles inbound port-write interrupts. Copies PW message from an internal 253 * buffer into PW message FIFO and schedules deferred routine to process 254 * queued messages. 255 * 256 * Returns: %0 257 */ 258 static int 259 tsi721_pw_handler(struct tsi721_device *priv) 260 { 261 u32 pw_stat; 262 u32 pw_buf[TSI721_RIO_PW_MSG_SIZE/sizeof(u32)]; 263 264 265 pw_stat = ioread32(priv->regs + TSI721_RIO_PW_RX_STAT); 266 267 if (pw_stat & TSI721_RIO_PW_RX_STAT_PW_VAL) { 268 pw_buf[0] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(0)); 269 pw_buf[1] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(1)); 270 pw_buf[2] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(2)); 271 pw_buf[3] = ioread32(priv->regs + TSI721_RIO_PW_RX_CAPT(3)); 272 273 /* Queue PW message (if there is room in FIFO), 274 * otherwise discard it. 275 */ 276 spin_lock(&priv->pw_fifo_lock); 277 if (kfifo_avail(&priv->pw_fifo) >= TSI721_RIO_PW_MSG_SIZE) 278 kfifo_in(&priv->pw_fifo, pw_buf, 279 TSI721_RIO_PW_MSG_SIZE); 280 else 281 priv->pw_discard_count++; 282 spin_unlock(&priv->pw_fifo_lock); 283 } 284 285 /* Clear pending PW interrupts */ 286 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL, 287 priv->regs + TSI721_RIO_PW_RX_STAT); 288 289 schedule_work(&priv->pw_work); 290 291 return 0; 292 } 293 294 static void tsi721_pw_dpc(struct work_struct *work) 295 { 296 struct tsi721_device *priv = container_of(work, struct tsi721_device, 297 pw_work); 298 union rio_pw_msg pwmsg; 299 300 /* 301 * Process port-write messages 302 */ 303 while (kfifo_out_spinlocked(&priv->pw_fifo, (unsigned char *)&pwmsg, 304 TSI721_RIO_PW_MSG_SIZE, &priv->pw_fifo_lock)) { 305 /* Pass the port-write message to RIO core for processing */ 306 rio_inb_pwrite_handler(&priv->mport, &pwmsg); 307 } 308 } 309 310 /** 311 * tsi721_pw_enable - enable/disable port-write interface init 312 * @mport: Master port implementing the port write unit 313 * @enable: 1=enable; 0=disable port-write message handling 314 * 315 * Returns: %0 316 */ 317 static int tsi721_pw_enable(struct rio_mport *mport, int enable) 318 { 319 struct tsi721_device *priv = mport->priv; 320 u32 rval; 321 322 rval = ioread32(priv->regs + TSI721_RIO_EM_INT_ENABLE); 323 324 if (enable) 325 rval |= TSI721_RIO_EM_INT_ENABLE_PW_RX; 326 else 327 rval &= ~TSI721_RIO_EM_INT_ENABLE_PW_RX; 328 329 /* Clear pending PW interrupts */ 330 iowrite32(TSI721_RIO_PW_RX_STAT_PW_DISC | TSI721_RIO_PW_RX_STAT_PW_VAL, 331 priv->regs + TSI721_RIO_PW_RX_STAT); 332 /* Update enable bits */ 333 iowrite32(rval, priv->regs + TSI721_RIO_EM_INT_ENABLE); 334 335 return 0; 336 } 337 338 /** 339 * tsi721_dsend - Send a RapidIO doorbell 340 * @mport: RapidIO master port info 341 * @index: ID of RapidIO interface 342 * @destid: Destination ID of target device 343 * @data: 16-bit info field of RapidIO doorbell 344 * 345 * Sends a RapidIO doorbell message. 346 * 347 * Returns: %0 348 */ 349 static int tsi721_dsend(struct rio_mport *mport, int index, 350 u16 destid, u16 data) 351 { 352 struct tsi721_device *priv = mport->priv; 353 u32 offset; 354 355 offset = (((mport->sys_size) ? RIO_TT_CODE_16 : RIO_TT_CODE_8) << 18) | 356 (destid << 2); 357 358 tsi_debug(DBELL, &priv->pdev->dev, 359 "Send Doorbell 0x%04x to destID 0x%x", data, destid); 360 iowrite16be(data, priv->odb_base + offset); 361 362 return 0; 363 } 364 365 /** 366 * tsi721_dbell_handler - Tsi721 doorbell interrupt handler 367 * @priv: tsi721 device-specific data structure 368 * 369 * Handles inbound doorbell interrupts. Copies doorbell entry from an internal 370 * buffer into DB message FIFO and schedules deferred routine to process 371 * queued DBs. 372 * 373 * Returns: %0 374 */ 375 static int 376 tsi721_dbell_handler(struct tsi721_device *priv) 377 { 378 u32 regval; 379 380 /* Disable IDB interrupts */ 381 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE)); 382 regval &= ~TSI721_SR_CHINT_IDBQRCV; 383 iowrite32(regval, 384 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE)); 385 386 schedule_work(&priv->idb_work); 387 388 return 0; 389 } 390 391 static void tsi721_db_dpc(struct work_struct *work) 392 { 393 struct tsi721_device *priv = container_of(work, struct tsi721_device, 394 idb_work); 395 struct rio_mport *mport; 396 struct rio_dbell *dbell; 397 u32 wr_ptr, rd_ptr; 398 u64 *idb_entry; 399 u32 regval; 400 union { 401 u64 msg; 402 u8 bytes[8]; 403 } idb; 404 405 /* 406 * Process queued inbound doorbells 407 */ 408 mport = &priv->mport; 409 410 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE; 411 rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE; 412 413 while (wr_ptr != rd_ptr) { 414 int found = 0; 415 416 idb_entry = (u64 *)(priv->idb_base + 417 (TSI721_IDB_ENTRY_SIZE * rd_ptr)); 418 rd_ptr++; 419 rd_ptr %= IDB_QSIZE; 420 idb.msg = *idb_entry; 421 *idb_entry = 0; 422 423 /* Process one doorbell */ 424 list_for_each_entry(dbell, &mport->dbells, node) { 425 if ((dbell->res->start <= DBELL_INF(idb.bytes)) && 426 (dbell->res->end >= DBELL_INF(idb.bytes))) { 427 found = 1; 428 break; 429 } 430 } 431 432 if (found) { 433 dbell->dinb(mport, dbell->dev_id, DBELL_SID(idb.bytes), 434 DBELL_TID(idb.bytes), DBELL_INF(idb.bytes)); 435 } else { 436 tsi_debug(DBELL, &priv->pdev->dev, 437 "spurious IDB sid %2.2x tid %2.2x info %4.4x", 438 DBELL_SID(idb.bytes), DBELL_TID(idb.bytes), 439 DBELL_INF(idb.bytes)); 440 } 441 442 wr_ptr = ioread32(priv->regs + 443 TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE; 444 } 445 446 iowrite32(rd_ptr & (IDB_QSIZE - 1), 447 priv->regs + TSI721_IDQ_RP(IDB_QUEUE)); 448 449 /* Re-enable IDB interrupts */ 450 regval = ioread32(priv->regs + TSI721_SR_CHINTE(IDB_QUEUE)); 451 regval |= TSI721_SR_CHINT_IDBQRCV; 452 iowrite32(regval, 453 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE)); 454 455 wr_ptr = ioread32(priv->regs + TSI721_IDQ_WP(IDB_QUEUE)) % IDB_QSIZE; 456 if (wr_ptr != rd_ptr) 457 schedule_work(&priv->idb_work); 458 } 459 460 /** 461 * tsi721_irqhandler - Tsi721 interrupt handler 462 * @irq: Linux interrupt number 463 * @ptr: Pointer to interrupt-specific data (tsi721_device structure) 464 * 465 * Handles Tsi721 interrupts signaled using MSI and INTA. Checks reported 466 * interrupt events and calls an event-specific handler(s). 467 * 468 * Returns: %IRQ_HANDLED or %IRQ_NONE 469 */ 470 static irqreturn_t tsi721_irqhandler(int irq, void *ptr) 471 { 472 struct tsi721_device *priv = (struct tsi721_device *)ptr; 473 u32 dev_int; 474 u32 dev_ch_int; 475 u32 intval; 476 u32 ch_inte; 477 478 /* For MSI mode disable all device-level interrupts */ 479 if (priv->flags & TSI721_USING_MSI) 480 iowrite32(0, priv->regs + TSI721_DEV_INTE); 481 482 dev_int = ioread32(priv->regs + TSI721_DEV_INT); 483 if (!dev_int) 484 return IRQ_NONE; 485 486 dev_ch_int = ioread32(priv->regs + TSI721_DEV_CHAN_INT); 487 488 if (dev_int & TSI721_DEV_INT_SR2PC_CH) { 489 /* Service SR2PC Channel interrupts */ 490 if (dev_ch_int & TSI721_INT_SR2PC_CHAN(IDB_QUEUE)) { 491 /* Service Inbound Doorbell interrupt */ 492 intval = ioread32(priv->regs + 493 TSI721_SR_CHINT(IDB_QUEUE)); 494 if (intval & TSI721_SR_CHINT_IDBQRCV) 495 tsi721_dbell_handler(priv); 496 else 497 tsi_info(&priv->pdev->dev, 498 "Unsupported SR_CH_INT %x", intval); 499 500 /* Clear interrupts */ 501 iowrite32(intval, 502 priv->regs + TSI721_SR_CHINT(IDB_QUEUE)); 503 ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE)); 504 } 505 } 506 507 if (dev_int & TSI721_DEV_INT_SMSG_CH) { 508 int ch; 509 510 /* 511 * Service channel interrupts from Messaging Engine 512 */ 513 514 if (dev_ch_int & TSI721_INT_IMSG_CHAN_M) { /* Inbound Msg */ 515 /* Disable signaled OB MSG Channel interrupts */ 516 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 517 ch_inte &= ~(dev_ch_int & TSI721_INT_IMSG_CHAN_M); 518 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE); 519 520 /* 521 * Process Inbound Message interrupt for each MBOX 522 */ 523 for (ch = 4; ch < RIO_MAX_MBOX + 4; ch++) { 524 if (!(dev_ch_int & TSI721_INT_IMSG_CHAN(ch))) 525 continue; 526 tsi721_imsg_handler(priv, ch); 527 } 528 } 529 530 if (dev_ch_int & TSI721_INT_OMSG_CHAN_M) { /* Outbound Msg */ 531 /* Disable signaled OB MSG Channel interrupts */ 532 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 533 ch_inte &= ~(dev_ch_int & TSI721_INT_OMSG_CHAN_M); 534 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE); 535 536 /* 537 * Process Outbound Message interrupts for each MBOX 538 */ 539 540 for (ch = 0; ch < RIO_MAX_MBOX; ch++) { 541 if (!(dev_ch_int & TSI721_INT_OMSG_CHAN(ch))) 542 continue; 543 tsi721_omsg_handler(priv, ch); 544 } 545 } 546 } 547 548 if (dev_int & TSI721_DEV_INT_SRIO) { 549 /* Service SRIO MAC interrupts */ 550 intval = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT); 551 if (intval & TSI721_RIO_EM_INT_STAT_PW_RX) 552 tsi721_pw_handler(priv); 553 } 554 555 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 556 if (dev_int & TSI721_DEV_INT_BDMA_CH) { 557 int ch; 558 559 if (dev_ch_int & TSI721_INT_BDMA_CHAN_M) { 560 tsi_debug(DMA, &priv->pdev->dev, 561 "IRQ from DMA channel 0x%08x", dev_ch_int); 562 563 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) { 564 if (!(dev_ch_int & TSI721_INT_BDMA_CHAN(ch))) 565 continue; 566 tsi721_bdma_handler(&priv->bdma[ch]); 567 } 568 } 569 } 570 #endif 571 572 /* For MSI mode re-enable device-level interrupts */ 573 if (priv->flags & TSI721_USING_MSI) { 574 dev_int = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO | 575 TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH; 576 iowrite32(dev_int, priv->regs + TSI721_DEV_INTE); 577 } 578 579 return IRQ_HANDLED; 580 } 581 582 static void tsi721_interrupts_init(struct tsi721_device *priv) 583 { 584 u32 intr; 585 586 /* Enable IDB interrupts */ 587 iowrite32(TSI721_SR_CHINT_ALL, 588 priv->regs + TSI721_SR_CHINT(IDB_QUEUE)); 589 iowrite32(TSI721_SR_CHINT_IDBQRCV, 590 priv->regs + TSI721_SR_CHINTE(IDB_QUEUE)); 591 592 /* Enable SRIO MAC interrupts */ 593 iowrite32(TSI721_RIO_EM_DEV_INT_EN_INT, 594 priv->regs + TSI721_RIO_EM_DEV_INT_EN); 595 596 /* Enable interrupts from channels in use */ 597 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 598 intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE) | 599 (TSI721_INT_BDMA_CHAN_M & 600 ~TSI721_INT_BDMA_CHAN(TSI721_DMACH_MAINT)); 601 #else 602 intr = TSI721_INT_SR2PC_CHAN(IDB_QUEUE); 603 #endif 604 iowrite32(intr, priv->regs + TSI721_DEV_CHAN_INTE); 605 606 if (priv->flags & TSI721_USING_MSIX) 607 intr = TSI721_DEV_INT_SRIO; 608 else 609 intr = TSI721_DEV_INT_SR2PC_CH | TSI721_DEV_INT_SRIO | 610 TSI721_DEV_INT_SMSG_CH | TSI721_DEV_INT_BDMA_CH; 611 612 iowrite32(intr, priv->regs + TSI721_DEV_INTE); 613 ioread32(priv->regs + TSI721_DEV_INTE); 614 } 615 616 #ifdef CONFIG_PCI_MSI 617 /** 618 * tsi721_omsg_msix - MSI-X interrupt handler for outbound messaging 619 * @irq: Linux interrupt number 620 * @ptr: Pointer to interrupt-specific data (tsi721_device structure) 621 * 622 * Handles outbound messaging interrupts signaled using MSI-X. 623 * 624 * Returns: %IRQ_HANDLED 625 */ 626 static irqreturn_t tsi721_omsg_msix(int irq, void *ptr) 627 { 628 struct tsi721_device *priv = (struct tsi721_device *)ptr; 629 int mbox; 630 631 mbox = (irq - priv->msix[TSI721_VECT_OMB0_DONE].vector) % RIO_MAX_MBOX; 632 tsi721_omsg_handler(priv, mbox); 633 return IRQ_HANDLED; 634 } 635 636 /** 637 * tsi721_imsg_msix - MSI-X interrupt handler for inbound messaging 638 * @irq: Linux interrupt number 639 * @ptr: Pointer to interrupt-specific data (tsi721_device structure) 640 * 641 * Handles inbound messaging interrupts signaled using MSI-X. 642 * 643 * Returns: %IRQ_HANDLED 644 */ 645 static irqreturn_t tsi721_imsg_msix(int irq, void *ptr) 646 { 647 struct tsi721_device *priv = (struct tsi721_device *)ptr; 648 int mbox; 649 650 mbox = (irq - priv->msix[TSI721_VECT_IMB0_RCV].vector) % RIO_MAX_MBOX; 651 tsi721_imsg_handler(priv, mbox + 4); 652 return IRQ_HANDLED; 653 } 654 655 /** 656 * tsi721_srio_msix - Tsi721 MSI-X SRIO MAC interrupt handler 657 * @irq: Linux interrupt number 658 * @ptr: Pointer to interrupt-specific data (tsi721_device structure) 659 * 660 * Handles Tsi721 interrupts from SRIO MAC. 661 * 662 * Returns: %IRQ_HANDLED 663 */ 664 static irqreturn_t tsi721_srio_msix(int irq, void *ptr) 665 { 666 struct tsi721_device *priv = (struct tsi721_device *)ptr; 667 u32 srio_int; 668 669 /* Service SRIO MAC interrupts */ 670 srio_int = ioread32(priv->regs + TSI721_RIO_EM_INT_STAT); 671 if (srio_int & TSI721_RIO_EM_INT_STAT_PW_RX) 672 tsi721_pw_handler(priv); 673 674 return IRQ_HANDLED; 675 } 676 677 /** 678 * tsi721_sr2pc_ch_msix - Tsi721 MSI-X SR2PC Channel interrupt handler 679 * @irq: Linux interrupt number 680 * @ptr: Pointer to interrupt-specific data (tsi721_device structure) 681 * 682 * Handles Tsi721 interrupts from SR2PC Channel. 683 * NOTE: At this moment services only one SR2PC channel associated with inbound 684 * doorbells. 685 * 686 * Returns: %IRQ_HANDLED 687 */ 688 static irqreturn_t tsi721_sr2pc_ch_msix(int irq, void *ptr) 689 { 690 struct tsi721_device *priv = (struct tsi721_device *)ptr; 691 u32 sr_ch_int; 692 693 /* Service Inbound DB interrupt from SR2PC channel */ 694 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE)); 695 if (sr_ch_int & TSI721_SR_CHINT_IDBQRCV) 696 tsi721_dbell_handler(priv); 697 698 /* Clear interrupts */ 699 iowrite32(sr_ch_int, priv->regs + TSI721_SR_CHINT(IDB_QUEUE)); 700 /* Read back to ensure that interrupt was cleared */ 701 sr_ch_int = ioread32(priv->regs + TSI721_SR_CHINT(IDB_QUEUE)); 702 703 return IRQ_HANDLED; 704 } 705 706 /** 707 * tsi721_request_msix - register interrupt service for MSI-X mode. 708 * @priv: tsi721 device-specific data structure 709 * 710 * Registers MSI-X interrupt service routines for interrupts that are active 711 * immediately after mport initialization. Messaging interrupt service routines 712 * should be registered during corresponding open requests. 713 * 714 * Returns: %0 on success or -errno value on failure. 715 */ 716 static int tsi721_request_msix(struct tsi721_device *priv) 717 { 718 int err = 0; 719 720 err = request_irq(priv->msix[TSI721_VECT_IDB].vector, 721 tsi721_sr2pc_ch_msix, 0, 722 priv->msix[TSI721_VECT_IDB].irq_name, (void *)priv); 723 if (err) 724 return err; 725 726 err = request_irq(priv->msix[TSI721_VECT_PWRX].vector, 727 tsi721_srio_msix, 0, 728 priv->msix[TSI721_VECT_PWRX].irq_name, (void *)priv); 729 if (err) { 730 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv); 731 return err; 732 } 733 734 return 0; 735 } 736 737 /** 738 * tsi721_enable_msix - Attempts to enable MSI-X support for Tsi721. 739 * @priv: pointer to tsi721 private data 740 * 741 * Configures MSI-X support for Tsi721. Supports only an exact number 742 * of requested vectors. 743 * 744 * Returns: %0 on success or -errno value on failure. 745 */ 746 static int tsi721_enable_msix(struct tsi721_device *priv) 747 { 748 struct msix_entry entries[TSI721_VECT_MAX]; 749 int err; 750 int i; 751 752 entries[TSI721_VECT_IDB].entry = TSI721_MSIX_SR2PC_IDBQ_RCV(IDB_QUEUE); 753 entries[TSI721_VECT_PWRX].entry = TSI721_MSIX_SRIO_MAC_INT; 754 755 /* 756 * Initialize MSI-X entries for Messaging Engine: 757 * this driver supports four RIO mailboxes (inbound and outbound) 758 * NOTE: Inbound message MBOX 0...4 use IB channels 4...7. Therefore 759 * offset +4 is added to IB MBOX number. 760 */ 761 for (i = 0; i < RIO_MAX_MBOX; i++) { 762 entries[TSI721_VECT_IMB0_RCV + i].entry = 763 TSI721_MSIX_IMSG_DQ_RCV(i + 4); 764 entries[TSI721_VECT_IMB0_INT + i].entry = 765 TSI721_MSIX_IMSG_INT(i + 4); 766 entries[TSI721_VECT_OMB0_DONE + i].entry = 767 TSI721_MSIX_OMSG_DONE(i); 768 entries[TSI721_VECT_OMB0_INT + i].entry = 769 TSI721_MSIX_OMSG_INT(i); 770 } 771 772 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 773 /* 774 * Initialize MSI-X entries for Block DMA Engine: 775 * this driver supports XXX DMA channels 776 * (one is reserved for SRIO maintenance transactions) 777 */ 778 for (i = 0; i < TSI721_DMA_CHNUM; i++) { 779 entries[TSI721_VECT_DMA0_DONE + i].entry = 780 TSI721_MSIX_DMACH_DONE(i); 781 entries[TSI721_VECT_DMA0_INT + i].entry = 782 TSI721_MSIX_DMACH_INT(i); 783 } 784 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */ 785 786 err = pci_enable_msix_exact(priv->pdev, entries, ARRAY_SIZE(entries)); 787 if (err) { 788 tsi_err(&priv->pdev->dev, 789 "Failed to enable MSI-X (err=%d)", err); 790 return err; 791 } 792 793 /* 794 * Copy MSI-X vector information into tsi721 private structure 795 */ 796 priv->msix[TSI721_VECT_IDB].vector = entries[TSI721_VECT_IDB].vector; 797 snprintf(priv->msix[TSI721_VECT_IDB].irq_name, IRQ_DEVICE_NAME_MAX, 798 DRV_NAME "-idb@pci:%s", pci_name(priv->pdev)); 799 priv->msix[TSI721_VECT_PWRX].vector = entries[TSI721_VECT_PWRX].vector; 800 snprintf(priv->msix[TSI721_VECT_PWRX].irq_name, IRQ_DEVICE_NAME_MAX, 801 DRV_NAME "-pwrx@pci:%s", pci_name(priv->pdev)); 802 803 for (i = 0; i < RIO_MAX_MBOX; i++) { 804 priv->msix[TSI721_VECT_IMB0_RCV + i].vector = 805 entries[TSI721_VECT_IMB0_RCV + i].vector; 806 snprintf(priv->msix[TSI721_VECT_IMB0_RCV + i].irq_name, 807 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbr%d@pci:%s", 808 i, pci_name(priv->pdev)); 809 810 priv->msix[TSI721_VECT_IMB0_INT + i].vector = 811 entries[TSI721_VECT_IMB0_INT + i].vector; 812 snprintf(priv->msix[TSI721_VECT_IMB0_INT + i].irq_name, 813 IRQ_DEVICE_NAME_MAX, DRV_NAME "-imbi%d@pci:%s", 814 i, pci_name(priv->pdev)); 815 816 priv->msix[TSI721_VECT_OMB0_DONE + i].vector = 817 entries[TSI721_VECT_OMB0_DONE + i].vector; 818 snprintf(priv->msix[TSI721_VECT_OMB0_DONE + i].irq_name, 819 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombd%d@pci:%s", 820 i, pci_name(priv->pdev)); 821 822 priv->msix[TSI721_VECT_OMB0_INT + i].vector = 823 entries[TSI721_VECT_OMB0_INT + i].vector; 824 snprintf(priv->msix[TSI721_VECT_OMB0_INT + i].irq_name, 825 IRQ_DEVICE_NAME_MAX, DRV_NAME "-ombi%d@pci:%s", 826 i, pci_name(priv->pdev)); 827 } 828 829 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 830 for (i = 0; i < TSI721_DMA_CHNUM; i++) { 831 priv->msix[TSI721_VECT_DMA0_DONE + i].vector = 832 entries[TSI721_VECT_DMA0_DONE + i].vector; 833 snprintf(priv->msix[TSI721_VECT_DMA0_DONE + i].irq_name, 834 IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmad%d@pci:%s", 835 i, pci_name(priv->pdev)); 836 837 priv->msix[TSI721_VECT_DMA0_INT + i].vector = 838 entries[TSI721_VECT_DMA0_INT + i].vector; 839 snprintf(priv->msix[TSI721_VECT_DMA0_INT + i].irq_name, 840 IRQ_DEVICE_NAME_MAX, DRV_NAME "-dmai%d@pci:%s", 841 i, pci_name(priv->pdev)); 842 } 843 #endif /* CONFIG_RAPIDIO_DMA_ENGINE */ 844 845 return 0; 846 } 847 #endif /* CONFIG_PCI_MSI */ 848 849 static int tsi721_request_irq(struct tsi721_device *priv) 850 { 851 int err; 852 853 #ifdef CONFIG_PCI_MSI 854 if (priv->flags & TSI721_USING_MSIX) 855 err = tsi721_request_msix(priv); 856 else 857 #endif 858 err = request_irq(priv->pdev->irq, tsi721_irqhandler, 859 (priv->flags & TSI721_USING_MSI) ? 0 : IRQF_SHARED, 860 DRV_NAME, (void *)priv); 861 862 if (err) 863 tsi_err(&priv->pdev->dev, 864 "Unable to allocate interrupt, err=%d", err); 865 866 return err; 867 } 868 869 static void tsi721_free_irq(struct tsi721_device *priv) 870 { 871 #ifdef CONFIG_PCI_MSI 872 if (priv->flags & TSI721_USING_MSIX) { 873 free_irq(priv->msix[TSI721_VECT_IDB].vector, (void *)priv); 874 free_irq(priv->msix[TSI721_VECT_PWRX].vector, (void *)priv); 875 } else 876 #endif 877 free_irq(priv->pdev->irq, (void *)priv); 878 } 879 880 static int 881 tsi721_obw_alloc(struct tsi721_device *priv, struct tsi721_obw_bar *pbar, 882 u32 size, int *win_id) 883 { 884 u64 win_base; 885 u64 bar_base; 886 u64 bar_end; 887 u32 align; 888 struct tsi721_ob_win *win; 889 struct tsi721_ob_win *new_win = NULL; 890 int new_win_idx = -1; 891 int i = 0; 892 893 bar_base = pbar->base; 894 bar_end = bar_base + pbar->size; 895 win_base = bar_base; 896 align = size/TSI721_PC2SR_ZONES; 897 898 while (i < TSI721_IBWIN_NUM) { 899 for (i = 0; i < TSI721_IBWIN_NUM; i++) { 900 if (!priv->ob_win[i].active) { 901 if (new_win == NULL) { 902 new_win = &priv->ob_win[i]; 903 new_win_idx = i; 904 } 905 continue; 906 } 907 908 /* 909 * If this window belongs to the current BAR check it 910 * for overlap 911 */ 912 win = &priv->ob_win[i]; 913 914 if (win->base >= bar_base && win->base < bar_end) { 915 if (win_base < (win->base + win->size) && 916 (win_base + size) > win->base) { 917 /* Overlap detected */ 918 win_base = win->base + win->size; 919 win_base = ALIGN(win_base, align); 920 break; 921 } 922 } 923 } 924 } 925 926 if (win_base + size > bar_end) 927 return -ENOMEM; 928 929 if (!new_win) { 930 tsi_err(&priv->pdev->dev, "OBW count tracking failed"); 931 return -EIO; 932 } 933 934 new_win->active = true; 935 new_win->base = win_base; 936 new_win->size = size; 937 new_win->pbar = pbar; 938 priv->obwin_cnt--; 939 pbar->free -= size; 940 *win_id = new_win_idx; 941 return 0; 942 } 943 944 static int tsi721_map_outb_win(struct rio_mport *mport, u16 destid, u64 rstart, 945 u32 size, u32 flags, dma_addr_t *laddr) 946 { 947 struct tsi721_device *priv = mport->priv; 948 int i; 949 struct tsi721_obw_bar *pbar; 950 struct tsi721_ob_win *ob_win; 951 int obw = -1; 952 u32 rval; 953 u64 rio_addr; 954 u32 zsize; 955 int ret = -ENOMEM; 956 957 tsi_debug(OBW, &priv->pdev->dev, 958 "did=%d ra=0x%llx sz=0x%x", destid, rstart, size); 959 960 if (!is_power_of_2(size) || (size < 0x8000) || (rstart & (size - 1))) 961 return -EINVAL; 962 963 if (priv->obwin_cnt == 0) 964 return -EBUSY; 965 966 for (i = 0; i < 2; i++) { 967 if (priv->p2r_bar[i].free >= size) { 968 pbar = &priv->p2r_bar[i]; 969 ret = tsi721_obw_alloc(priv, pbar, size, &obw); 970 if (!ret) 971 break; 972 } 973 } 974 975 if (ret) 976 return ret; 977 978 WARN_ON(obw == -1); 979 ob_win = &priv->ob_win[obw]; 980 ob_win->destid = destid; 981 ob_win->rstart = rstart; 982 tsi_debug(OBW, &priv->pdev->dev, 983 "allocated OBW%d @%llx", obw, ob_win->base); 984 985 /* 986 * Configure Outbound Window 987 */ 988 989 zsize = size/TSI721_PC2SR_ZONES; 990 rio_addr = rstart; 991 992 /* 993 * Program Address Translation Zones: 994 * This implementation uses all 8 zones associated wit window. 995 */ 996 for (i = 0; i < TSI721_PC2SR_ZONES; i++) { 997 998 while (ioread32(priv->regs + TSI721_ZONE_SEL) & 999 TSI721_ZONE_SEL_GO) { 1000 udelay(1); 1001 } 1002 1003 rval = (u32)(rio_addr & TSI721_LUT_DATA0_ADD) | 1004 TSI721_LUT_DATA0_NREAD | TSI721_LUT_DATA0_NWR; 1005 iowrite32(rval, priv->regs + TSI721_LUT_DATA0); 1006 rval = (u32)(rio_addr >> 32); 1007 iowrite32(rval, priv->regs + TSI721_LUT_DATA1); 1008 rval = destid; 1009 iowrite32(rval, priv->regs + TSI721_LUT_DATA2); 1010 1011 rval = TSI721_ZONE_SEL_GO | (obw << 3) | i; 1012 iowrite32(rval, priv->regs + TSI721_ZONE_SEL); 1013 1014 rio_addr += zsize; 1015 } 1016 1017 iowrite32(TSI721_OBWIN_SIZE(size) << 8, 1018 priv->regs + TSI721_OBWINSZ(obw)); 1019 iowrite32((u32)(ob_win->base >> 32), priv->regs + TSI721_OBWINUB(obw)); 1020 iowrite32((u32)(ob_win->base & TSI721_OBWINLB_BA) | TSI721_OBWINLB_WEN, 1021 priv->regs + TSI721_OBWINLB(obw)); 1022 1023 *laddr = ob_win->base; 1024 return 0; 1025 } 1026 1027 static void tsi721_unmap_outb_win(struct rio_mport *mport, 1028 u16 destid, u64 rstart) 1029 { 1030 struct tsi721_device *priv = mport->priv; 1031 struct tsi721_ob_win *ob_win; 1032 int i; 1033 1034 tsi_debug(OBW, &priv->pdev->dev, "did=%d ra=0x%llx", destid, rstart); 1035 1036 for (i = 0; i < TSI721_OBWIN_NUM; i++) { 1037 ob_win = &priv->ob_win[i]; 1038 1039 if (ob_win->active && 1040 ob_win->destid == destid && ob_win->rstart == rstart) { 1041 tsi_debug(OBW, &priv->pdev->dev, 1042 "free OBW%d @%llx", i, ob_win->base); 1043 ob_win->active = false; 1044 iowrite32(0, priv->regs + TSI721_OBWINLB(i)); 1045 ob_win->pbar->free += ob_win->size; 1046 priv->obwin_cnt++; 1047 break; 1048 } 1049 } 1050 } 1051 1052 /** 1053 * tsi721_init_pc2sr_mapping - initializes outbound (PCIe->SRIO) 1054 * translation regions. 1055 * @priv: pointer to tsi721 private data 1056 * 1057 * Disables SREP translation regions. 1058 */ 1059 static void tsi721_init_pc2sr_mapping(struct tsi721_device *priv) 1060 { 1061 int i, z; 1062 u32 rval; 1063 1064 /* Disable all PC2SR translation windows */ 1065 for (i = 0; i < TSI721_OBWIN_NUM; i++) 1066 iowrite32(0, priv->regs + TSI721_OBWINLB(i)); 1067 1068 /* Initialize zone lookup tables to avoid ECC errors on reads */ 1069 iowrite32(0, priv->regs + TSI721_LUT_DATA0); 1070 iowrite32(0, priv->regs + TSI721_LUT_DATA1); 1071 iowrite32(0, priv->regs + TSI721_LUT_DATA2); 1072 1073 for (i = 0; i < TSI721_OBWIN_NUM; i++) { 1074 for (z = 0; z < TSI721_PC2SR_ZONES; z++) { 1075 while (ioread32(priv->regs + TSI721_ZONE_SEL) & 1076 TSI721_ZONE_SEL_GO) { 1077 udelay(1); 1078 } 1079 rval = TSI721_ZONE_SEL_GO | (i << 3) | z; 1080 iowrite32(rval, priv->regs + TSI721_ZONE_SEL); 1081 } 1082 } 1083 1084 if (priv->p2r_bar[0].size == 0 && priv->p2r_bar[1].size == 0) { 1085 priv->obwin_cnt = 0; 1086 return; 1087 } 1088 1089 priv->p2r_bar[0].free = priv->p2r_bar[0].size; 1090 priv->p2r_bar[1].free = priv->p2r_bar[1].size; 1091 1092 for (i = 0; i < TSI721_OBWIN_NUM; i++) 1093 priv->ob_win[i].active = false; 1094 1095 priv->obwin_cnt = TSI721_OBWIN_NUM; 1096 } 1097 1098 /** 1099 * tsi721_rio_map_inb_mem -- Mapping inbound memory region. 1100 * @mport: RapidIO master port 1101 * @lstart: Local memory space start address. 1102 * @rstart: RapidIO space start address. 1103 * @size: The mapping region size. 1104 * @flags: Flags for mapping. 0 for using default flags. 1105 * 1106 * Return: 0 -- Success. 1107 * 1108 * This function will create the inbound mapping 1109 * from rstart to lstart. 1110 */ 1111 static int tsi721_rio_map_inb_mem(struct rio_mport *mport, dma_addr_t lstart, 1112 u64 rstart, u64 size, u32 flags) 1113 { 1114 struct tsi721_device *priv = mport->priv; 1115 int i, avail = -1; 1116 u32 regval; 1117 struct tsi721_ib_win *ib_win; 1118 bool direct = (lstart == rstart); 1119 u64 ibw_size; 1120 dma_addr_t loc_start; 1121 u64 ibw_start; 1122 struct tsi721_ib_win_mapping *map = NULL; 1123 int ret = -EBUSY; 1124 1125 /* Max IBW size supported by HW is 16GB */ 1126 if (size > 0x400000000UL) 1127 return -EINVAL; 1128 1129 if (direct) { 1130 /* Calculate minimal acceptable window size and base address */ 1131 1132 ibw_size = roundup_pow_of_two(size); 1133 ibw_start = lstart & ~(ibw_size - 1); 1134 1135 tsi_debug(IBW, &priv->pdev->dev, 1136 "Direct (RIO_0x%llx -> PCIe_%pad), size=0x%llx, ibw_start = 0x%llx", 1137 rstart, &lstart, size, ibw_start); 1138 1139 while ((lstart + size) > (ibw_start + ibw_size)) { 1140 ibw_size *= 2; 1141 ibw_start = lstart & ~(ibw_size - 1); 1142 /* Check for crossing IBW max size 16GB */ 1143 if (ibw_size > 0x400000000UL) 1144 return -EBUSY; 1145 } 1146 1147 loc_start = ibw_start; 1148 1149 map = kzalloc_obj(struct tsi721_ib_win_mapping, GFP_ATOMIC); 1150 if (map == NULL) 1151 return -ENOMEM; 1152 1153 } else { 1154 tsi_debug(IBW, &priv->pdev->dev, 1155 "Translated (RIO_0x%llx -> PCIe_%pad), size=0x%llx", 1156 rstart, &lstart, size); 1157 1158 if (!is_power_of_2(size) || size < 0x1000 || 1159 ((u64)lstart & (size - 1)) || (rstart & (size - 1))) 1160 return -EINVAL; 1161 if (priv->ibwin_cnt == 0) 1162 return -EBUSY; 1163 ibw_start = rstart; 1164 ibw_size = size; 1165 loc_start = lstart; 1166 } 1167 1168 /* 1169 * Scan for overlapping with active regions and mark the first available 1170 * IB window at the same time. 1171 */ 1172 for (i = 0; i < TSI721_IBWIN_NUM; i++) { 1173 ib_win = &priv->ib_win[i]; 1174 1175 if (!ib_win->active) { 1176 if (avail == -1) { 1177 avail = i; 1178 ret = 0; 1179 } 1180 } else if (ibw_start < (ib_win->rstart + ib_win->size) && 1181 (ibw_start + ibw_size) > ib_win->rstart) { 1182 /* Return error if address translation involved */ 1183 if (!direct || ib_win->xlat) { 1184 ret = -EFAULT; 1185 break; 1186 } 1187 1188 /* 1189 * Direct mappings usually are larger than originally 1190 * requested fragments - check if this new request fits 1191 * into it. 1192 */ 1193 if (rstart >= ib_win->rstart && 1194 (rstart + size) <= (ib_win->rstart + 1195 ib_win->size)) { 1196 /* We are in - no further mapping required */ 1197 map->lstart = lstart; 1198 list_add_tail(&map->node, &ib_win->mappings); 1199 return 0; 1200 } 1201 1202 ret = -EFAULT; 1203 break; 1204 } 1205 } 1206 1207 if (ret) 1208 goto out; 1209 i = avail; 1210 1211 /* Sanity check: available IB window must be disabled at this point */ 1212 regval = ioread32(priv->regs + TSI721_IBWIN_LB(i)); 1213 if (WARN_ON(regval & TSI721_IBWIN_LB_WEN)) { 1214 ret = -EIO; 1215 goto out; 1216 } 1217 1218 ib_win = &priv->ib_win[i]; 1219 ib_win->active = true; 1220 ib_win->rstart = ibw_start; 1221 ib_win->lstart = loc_start; 1222 ib_win->size = ibw_size; 1223 ib_win->xlat = (lstart != rstart); 1224 INIT_LIST_HEAD(&ib_win->mappings); 1225 1226 /* 1227 * When using direct IBW mapping and have larger than requested IBW size 1228 * we can have multiple local memory blocks mapped through the same IBW 1229 * To handle this situation we maintain list of "clients" for such IBWs. 1230 */ 1231 if (direct) { 1232 map->lstart = lstart; 1233 list_add_tail(&map->node, &ib_win->mappings); 1234 } 1235 1236 iowrite32(TSI721_IBWIN_SIZE(ibw_size) << 8, 1237 priv->regs + TSI721_IBWIN_SZ(i)); 1238 1239 iowrite32(((u64)loc_start >> 32), priv->regs + TSI721_IBWIN_TUA(i)); 1240 iowrite32(((u64)loc_start & TSI721_IBWIN_TLA_ADD), 1241 priv->regs + TSI721_IBWIN_TLA(i)); 1242 1243 iowrite32(ibw_start >> 32, priv->regs + TSI721_IBWIN_UB(i)); 1244 iowrite32((ibw_start & TSI721_IBWIN_LB_BA) | TSI721_IBWIN_LB_WEN, 1245 priv->regs + TSI721_IBWIN_LB(i)); 1246 1247 priv->ibwin_cnt--; 1248 1249 tsi_debug(IBW, &priv->pdev->dev, 1250 "Configured IBWIN%d (RIO_0x%llx -> PCIe_%pad), size=0x%llx", 1251 i, ibw_start, &loc_start, ibw_size); 1252 1253 return 0; 1254 out: 1255 kfree(map); 1256 return ret; 1257 } 1258 1259 /** 1260 * tsi721_rio_unmap_inb_mem -- Unmapping inbound memory region. 1261 * @mport: RapidIO master port 1262 * @lstart: Local memory space start address. 1263 */ 1264 static void tsi721_rio_unmap_inb_mem(struct rio_mport *mport, 1265 dma_addr_t lstart) 1266 { 1267 struct tsi721_device *priv = mport->priv; 1268 struct tsi721_ib_win *ib_win; 1269 int i; 1270 1271 tsi_debug(IBW, &priv->pdev->dev, 1272 "Unmap IBW mapped to PCIe_%pad", &lstart); 1273 1274 /* Search for matching active inbound translation window */ 1275 for (i = 0; i < TSI721_IBWIN_NUM; i++) { 1276 ib_win = &priv->ib_win[i]; 1277 1278 /* Address translating IBWs must to be an exact march */ 1279 if (!ib_win->active || 1280 (ib_win->xlat && lstart != ib_win->lstart)) 1281 continue; 1282 1283 if (lstart >= ib_win->lstart && 1284 lstart < (ib_win->lstart + ib_win->size)) { 1285 1286 if (!ib_win->xlat) { 1287 struct tsi721_ib_win_mapping *map; 1288 int found = 0; 1289 1290 list_for_each_entry(map, 1291 &ib_win->mappings, node) { 1292 if (map->lstart == lstart) { 1293 list_del(&map->node); 1294 kfree(map); 1295 found = 1; 1296 break; 1297 } 1298 } 1299 1300 if (!found) 1301 continue; 1302 1303 if (!list_empty(&ib_win->mappings)) 1304 break; 1305 } 1306 1307 tsi_debug(IBW, &priv->pdev->dev, "Disable IBWIN_%d", i); 1308 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i)); 1309 ib_win->active = false; 1310 priv->ibwin_cnt++; 1311 break; 1312 } 1313 } 1314 1315 if (i == TSI721_IBWIN_NUM) 1316 tsi_debug(IBW, &priv->pdev->dev, 1317 "IB window mapped to %pad not found", &lstart); 1318 } 1319 1320 /** 1321 * tsi721_init_sr2pc_mapping - initializes inbound (SRIO->PCIe) 1322 * translation regions. 1323 * @priv: pointer to tsi721 private data 1324 * 1325 * Disables inbound windows. 1326 */ 1327 static void tsi721_init_sr2pc_mapping(struct tsi721_device *priv) 1328 { 1329 int i; 1330 1331 /* Disable all SR2PC inbound windows */ 1332 for (i = 0; i < TSI721_IBWIN_NUM; i++) 1333 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i)); 1334 priv->ibwin_cnt = TSI721_IBWIN_NUM; 1335 } 1336 1337 /* 1338 * tsi721_close_sr2pc_mapping - closes all active inbound (SRIO->PCIe) 1339 * translation regions. 1340 * @priv: pointer to tsi721 device private data 1341 */ 1342 static void tsi721_close_sr2pc_mapping(struct tsi721_device *priv) 1343 { 1344 struct tsi721_ib_win *ib_win; 1345 int i; 1346 1347 /* Disable all active SR2PC inbound windows */ 1348 for (i = 0; i < TSI721_IBWIN_NUM; i++) { 1349 ib_win = &priv->ib_win[i]; 1350 if (ib_win->active) { 1351 iowrite32(0, priv->regs + TSI721_IBWIN_LB(i)); 1352 ib_win->active = false; 1353 } 1354 } 1355 } 1356 1357 /** 1358 * tsi721_port_write_init - Inbound port write interface init 1359 * @priv: pointer to tsi721 private data 1360 * 1361 * Initializes inbound port write handler. 1362 * Returns: %0 on success or %-ENOMEM on failure. 1363 */ 1364 static int tsi721_port_write_init(struct tsi721_device *priv) 1365 { 1366 priv->pw_discard_count = 0; 1367 INIT_WORK(&priv->pw_work, tsi721_pw_dpc); 1368 spin_lock_init(&priv->pw_fifo_lock); 1369 if (kfifo_alloc(&priv->pw_fifo, 1370 TSI721_RIO_PW_MSG_SIZE * 32, GFP_KERNEL)) { 1371 tsi_err(&priv->pdev->dev, "PW FIFO allocation failed"); 1372 return -ENOMEM; 1373 } 1374 1375 /* Use reliable port-write capture mode */ 1376 iowrite32(TSI721_RIO_PW_CTL_PWC_REL, priv->regs + TSI721_RIO_PW_CTL); 1377 return 0; 1378 } 1379 1380 static void tsi721_port_write_free(struct tsi721_device *priv) 1381 { 1382 kfifo_free(&priv->pw_fifo); 1383 } 1384 1385 static int tsi721_doorbell_init(struct tsi721_device *priv) 1386 { 1387 /* Outbound Doorbells do not require any setup. 1388 * Tsi721 uses dedicated PCI BAR1 to generate doorbells. 1389 * That BAR1 was mapped during the probe routine. 1390 */ 1391 1392 /* Initialize Inbound Doorbell processing DPC and queue */ 1393 priv->db_discard_count = 0; 1394 INIT_WORK(&priv->idb_work, tsi721_db_dpc); 1395 1396 /* Allocate buffer for inbound doorbells queue */ 1397 priv->idb_base = dma_alloc_coherent(&priv->pdev->dev, 1398 IDB_QSIZE * TSI721_IDB_ENTRY_SIZE, 1399 &priv->idb_dma, GFP_KERNEL); 1400 if (!priv->idb_base) 1401 return -ENOMEM; 1402 1403 tsi_debug(DBELL, &priv->pdev->dev, 1404 "Allocated IDB buffer @ %p (phys = %pad)", 1405 priv->idb_base, &priv->idb_dma); 1406 1407 iowrite32(TSI721_IDQ_SIZE_VAL(IDB_QSIZE), 1408 priv->regs + TSI721_IDQ_SIZE(IDB_QUEUE)); 1409 iowrite32(((u64)priv->idb_dma >> 32), 1410 priv->regs + TSI721_IDQ_BASEU(IDB_QUEUE)); 1411 iowrite32(((u64)priv->idb_dma & TSI721_IDQ_BASEL_ADDR), 1412 priv->regs + TSI721_IDQ_BASEL(IDB_QUEUE)); 1413 /* Enable accepting all inbound doorbells */ 1414 iowrite32(0, priv->regs + TSI721_IDQ_MASK(IDB_QUEUE)); 1415 1416 iowrite32(TSI721_IDQ_INIT, priv->regs + TSI721_IDQ_CTL(IDB_QUEUE)); 1417 1418 iowrite32(0, priv->regs + TSI721_IDQ_RP(IDB_QUEUE)); 1419 1420 return 0; 1421 } 1422 1423 static void tsi721_doorbell_free(struct tsi721_device *priv) 1424 { 1425 if (priv->idb_base == NULL) 1426 return; 1427 1428 /* Free buffer allocated for inbound doorbell queue */ 1429 dma_free_coherent(&priv->pdev->dev, IDB_QSIZE * TSI721_IDB_ENTRY_SIZE, 1430 priv->idb_base, priv->idb_dma); 1431 priv->idb_base = NULL; 1432 } 1433 1434 /** 1435 * tsi721_bdma_maint_init - Initialize maintenance request BDMA channel. 1436 * @priv: pointer to tsi721 private data 1437 * 1438 * Initialize BDMA channel allocated for RapidIO maintenance read/write 1439 * request generation 1440 * 1441 * Returns: %0 on success or %-ENOMEM on failure. 1442 */ 1443 static int tsi721_bdma_maint_init(struct tsi721_device *priv) 1444 { 1445 struct tsi721_dma_desc *bd_ptr; 1446 u64 *sts_ptr; 1447 dma_addr_t bd_phys, sts_phys; 1448 int sts_size; 1449 int bd_num = 2; 1450 void __iomem *regs; 1451 1452 tsi_debug(MAINT, &priv->pdev->dev, 1453 "Init BDMA_%d Maintenance requests", TSI721_DMACH_MAINT); 1454 1455 /* 1456 * Initialize DMA channel for maintenance requests 1457 */ 1458 1459 priv->mdma.ch_id = TSI721_DMACH_MAINT; 1460 regs = priv->regs + TSI721_DMAC_BASE(TSI721_DMACH_MAINT); 1461 1462 /* Allocate space for DMA descriptors */ 1463 bd_ptr = dma_alloc_coherent(&priv->pdev->dev, 1464 bd_num * sizeof(struct tsi721_dma_desc), 1465 &bd_phys, GFP_KERNEL); 1466 if (!bd_ptr) 1467 return -ENOMEM; 1468 1469 priv->mdma.bd_num = bd_num; 1470 priv->mdma.bd_phys = bd_phys; 1471 priv->mdma.bd_base = bd_ptr; 1472 1473 tsi_debug(MAINT, &priv->pdev->dev, "DMA descriptors @ %p (phys = %pad)", 1474 bd_ptr, &bd_phys); 1475 1476 /* Allocate space for descriptor status FIFO */ 1477 sts_size = (bd_num >= TSI721_DMA_MINSTSSZ) ? 1478 bd_num : TSI721_DMA_MINSTSSZ; 1479 sts_size = roundup_pow_of_two(sts_size); 1480 sts_ptr = dma_alloc_coherent(&priv->pdev->dev, 1481 sts_size * sizeof(struct tsi721_dma_sts), 1482 &sts_phys, GFP_KERNEL); 1483 if (!sts_ptr) { 1484 /* Free space allocated for DMA descriptors */ 1485 dma_free_coherent(&priv->pdev->dev, 1486 bd_num * sizeof(struct tsi721_dma_desc), 1487 bd_ptr, bd_phys); 1488 priv->mdma.bd_base = NULL; 1489 return -ENOMEM; 1490 } 1491 1492 priv->mdma.sts_phys = sts_phys; 1493 priv->mdma.sts_base = sts_ptr; 1494 priv->mdma.sts_size = sts_size; 1495 1496 tsi_debug(MAINT, &priv->pdev->dev, 1497 "desc status FIFO @ %p (phys = %pad) size=0x%x", 1498 sts_ptr, &sts_phys, sts_size); 1499 1500 /* Initialize DMA descriptors ring */ 1501 bd_ptr[bd_num - 1].type_id = cpu_to_le32(DTYPE3 << 29); 1502 bd_ptr[bd_num - 1].next_lo = cpu_to_le32((u64)bd_phys & 1503 TSI721_DMAC_DPTRL_MASK); 1504 bd_ptr[bd_num - 1].next_hi = cpu_to_le32((u64)bd_phys >> 32); 1505 1506 /* Setup DMA descriptor pointers */ 1507 iowrite32(((u64)bd_phys >> 32), regs + TSI721_DMAC_DPTRH); 1508 iowrite32(((u64)bd_phys & TSI721_DMAC_DPTRL_MASK), 1509 regs + TSI721_DMAC_DPTRL); 1510 1511 /* Setup descriptor status FIFO */ 1512 iowrite32(((u64)sts_phys >> 32), regs + TSI721_DMAC_DSBH); 1513 iowrite32(((u64)sts_phys & TSI721_DMAC_DSBL_MASK), 1514 regs + TSI721_DMAC_DSBL); 1515 iowrite32(TSI721_DMAC_DSSZ_SIZE(sts_size), 1516 regs + TSI721_DMAC_DSSZ); 1517 1518 /* Clear interrupt bits */ 1519 iowrite32(TSI721_DMAC_INT_ALL, regs + TSI721_DMAC_INT); 1520 1521 ioread32(regs + TSI721_DMAC_INT); 1522 1523 /* Toggle DMA channel initialization */ 1524 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL); 1525 ioread32(regs + TSI721_DMAC_CTL); 1526 udelay(10); 1527 1528 return 0; 1529 } 1530 1531 static int tsi721_bdma_maint_free(struct tsi721_device *priv) 1532 { 1533 u32 ch_stat; 1534 struct tsi721_bdma_maint *mdma = &priv->mdma; 1535 void __iomem *regs = priv->regs + TSI721_DMAC_BASE(mdma->ch_id); 1536 1537 if (mdma->bd_base == NULL) 1538 return 0; 1539 1540 /* Check if DMA channel still running */ 1541 ch_stat = ioread32(regs + TSI721_DMAC_STS); 1542 if (ch_stat & TSI721_DMAC_STS_RUN) 1543 return -EFAULT; 1544 1545 /* Put DMA channel into init state */ 1546 iowrite32(TSI721_DMAC_CTL_INIT, regs + TSI721_DMAC_CTL); 1547 1548 /* Free space allocated for DMA descriptors */ 1549 dma_free_coherent(&priv->pdev->dev, 1550 mdma->bd_num * sizeof(struct tsi721_dma_desc), 1551 mdma->bd_base, mdma->bd_phys); 1552 mdma->bd_base = NULL; 1553 1554 /* Free space allocated for status FIFO */ 1555 dma_free_coherent(&priv->pdev->dev, 1556 mdma->sts_size * sizeof(struct tsi721_dma_sts), 1557 mdma->sts_base, mdma->sts_phys); 1558 mdma->sts_base = NULL; 1559 return 0; 1560 } 1561 1562 /* Enable Inbound Messaging Interrupts */ 1563 static void 1564 tsi721_imsg_interrupt_enable(struct tsi721_device *priv, int ch, 1565 u32 inte_mask) 1566 { 1567 u32 rval; 1568 1569 if (!inte_mask) 1570 return; 1571 1572 /* Clear pending Inbound Messaging interrupts */ 1573 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch)); 1574 1575 /* Enable Inbound Messaging interrupts */ 1576 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch)); 1577 iowrite32(rval | inte_mask, priv->regs + TSI721_IBDMAC_INTE(ch)); 1578 1579 if (priv->flags & TSI721_USING_MSIX) 1580 return; /* Finished if we are in MSI-X mode */ 1581 1582 /* 1583 * For MSI and INTA interrupt signalling we need to enable next levels 1584 */ 1585 1586 /* Enable Device Channel Interrupt */ 1587 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 1588 iowrite32(rval | TSI721_INT_IMSG_CHAN(ch), 1589 priv->regs + TSI721_DEV_CHAN_INTE); 1590 } 1591 1592 /* Disable Inbound Messaging Interrupts */ 1593 static void 1594 tsi721_imsg_interrupt_disable(struct tsi721_device *priv, int ch, 1595 u32 inte_mask) 1596 { 1597 u32 rval; 1598 1599 if (!inte_mask) 1600 return; 1601 1602 /* Clear pending Inbound Messaging interrupts */ 1603 iowrite32(inte_mask, priv->regs + TSI721_IBDMAC_INT(ch)); 1604 1605 /* Disable Inbound Messaging interrupts */ 1606 rval = ioread32(priv->regs + TSI721_IBDMAC_INTE(ch)); 1607 rval &= ~inte_mask; 1608 iowrite32(rval, priv->regs + TSI721_IBDMAC_INTE(ch)); 1609 1610 if (priv->flags & TSI721_USING_MSIX) 1611 return; /* Finished if we are in MSI-X mode */ 1612 1613 /* 1614 * For MSI and INTA interrupt signalling we need to disable next levels 1615 */ 1616 1617 /* Disable Device Channel Interrupt */ 1618 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 1619 rval &= ~TSI721_INT_IMSG_CHAN(ch); 1620 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE); 1621 } 1622 1623 /* Enable Outbound Messaging interrupts */ 1624 static void 1625 tsi721_omsg_interrupt_enable(struct tsi721_device *priv, int ch, 1626 u32 inte_mask) 1627 { 1628 u32 rval; 1629 1630 if (!inte_mask) 1631 return; 1632 1633 /* Clear pending Outbound Messaging interrupts */ 1634 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch)); 1635 1636 /* Enable Outbound Messaging channel interrupts */ 1637 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch)); 1638 iowrite32(rval | inte_mask, priv->regs + TSI721_OBDMAC_INTE(ch)); 1639 1640 if (priv->flags & TSI721_USING_MSIX) 1641 return; /* Finished if we are in MSI-X mode */ 1642 1643 /* 1644 * For MSI and INTA interrupt signalling we need to enable next levels 1645 */ 1646 1647 /* Enable Device Channel Interrupt */ 1648 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 1649 iowrite32(rval | TSI721_INT_OMSG_CHAN(ch), 1650 priv->regs + TSI721_DEV_CHAN_INTE); 1651 } 1652 1653 /* Disable Outbound Messaging interrupts */ 1654 static void 1655 tsi721_omsg_interrupt_disable(struct tsi721_device *priv, int ch, 1656 u32 inte_mask) 1657 { 1658 u32 rval; 1659 1660 if (!inte_mask) 1661 return; 1662 1663 /* Clear pending Outbound Messaging interrupts */ 1664 iowrite32(inte_mask, priv->regs + TSI721_OBDMAC_INT(ch)); 1665 1666 /* Disable Outbound Messaging interrupts */ 1667 rval = ioread32(priv->regs + TSI721_OBDMAC_INTE(ch)); 1668 rval &= ~inte_mask; 1669 iowrite32(rval, priv->regs + TSI721_OBDMAC_INTE(ch)); 1670 1671 if (priv->flags & TSI721_USING_MSIX) 1672 return; /* Finished if we are in MSI-X mode */ 1673 1674 /* 1675 * For MSI and INTA interrupt signalling we need to disable next levels 1676 */ 1677 1678 /* Disable Device Channel Interrupt */ 1679 rval = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 1680 rval &= ~TSI721_INT_OMSG_CHAN(ch); 1681 iowrite32(rval, priv->regs + TSI721_DEV_CHAN_INTE); 1682 } 1683 1684 /** 1685 * tsi721_add_outb_message - Add message to the Tsi721 outbound message queue 1686 * @mport: Master port with outbound message queue 1687 * @rdev: Target of outbound message 1688 * @mbox: Outbound mailbox 1689 * @buffer: Message to add to outbound queue 1690 * @len: Length of message 1691 * 1692 * Returns: %0 on success or -errno value on failure. 1693 */ 1694 static int 1695 tsi721_add_outb_message(struct rio_mport *mport, struct rio_dev *rdev, int mbox, 1696 void *buffer, size_t len) 1697 { 1698 struct tsi721_device *priv = mport->priv; 1699 struct tsi721_omsg_desc *desc; 1700 u32 tx_slot; 1701 unsigned long flags; 1702 1703 if (!priv->omsg_init[mbox] || 1704 len > TSI721_MSG_MAX_SIZE || len < 8) 1705 return -EINVAL; 1706 1707 spin_lock_irqsave(&priv->omsg_ring[mbox].lock, flags); 1708 1709 tx_slot = priv->omsg_ring[mbox].tx_slot; 1710 1711 /* Copy copy message into transfer buffer */ 1712 memcpy(priv->omsg_ring[mbox].omq_base[tx_slot], buffer, len); 1713 1714 if (len & 0x7) 1715 len += 8; 1716 1717 /* Build descriptor associated with buffer */ 1718 desc = priv->omsg_ring[mbox].omd_base; 1719 desc[tx_slot].type_id = cpu_to_le32((DTYPE4 << 29) | rdev->destid); 1720 #ifdef TSI721_OMSG_DESC_INT 1721 /* Request IOF_DONE interrupt generation for each N-th frame in queue */ 1722 if (tx_slot % 4 == 0) 1723 desc[tx_slot].type_id |= cpu_to_le32(TSI721_OMD_IOF); 1724 #endif 1725 desc[tx_slot].msg_info = 1726 cpu_to_le32((mport->sys_size << 26) | (mbox << 22) | 1727 (0xe << 12) | (len & 0xff8)); 1728 desc[tx_slot].bufptr_lo = 1729 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] & 1730 0xffffffff); 1731 desc[tx_slot].bufptr_hi = 1732 cpu_to_le32((u64)priv->omsg_ring[mbox].omq_phys[tx_slot] >> 32); 1733 1734 priv->omsg_ring[mbox].wr_count++; 1735 1736 /* Go to next descriptor */ 1737 if (++priv->omsg_ring[mbox].tx_slot == priv->omsg_ring[mbox].size) { 1738 priv->omsg_ring[mbox].tx_slot = 0; 1739 /* Move through the ring link descriptor at the end */ 1740 priv->omsg_ring[mbox].wr_count++; 1741 } 1742 1743 mb(); 1744 1745 /* Set new write count value */ 1746 iowrite32(priv->omsg_ring[mbox].wr_count, 1747 priv->regs + TSI721_OBDMAC_DWRCNT(mbox)); 1748 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox)); 1749 1750 spin_unlock_irqrestore(&priv->omsg_ring[mbox].lock, flags); 1751 1752 return 0; 1753 } 1754 1755 /** 1756 * tsi721_omsg_handler - Outbound Message Interrupt Handler 1757 * @priv: pointer to tsi721 private data 1758 * @ch: number of OB MSG channel to service 1759 * 1760 * Services channel interrupts from outbound messaging engine. 1761 */ 1762 static void tsi721_omsg_handler(struct tsi721_device *priv, int ch) 1763 { 1764 u32 omsg_int; 1765 struct rio_mport *mport = &priv->mport; 1766 void *dev_id = NULL; 1767 u32 tx_slot = 0xffffffff; 1768 int do_callback = 0; 1769 1770 spin_lock(&priv->omsg_ring[ch].lock); 1771 1772 omsg_int = ioread32(priv->regs + TSI721_OBDMAC_INT(ch)); 1773 1774 if (omsg_int & TSI721_OBDMAC_INT_ST_FULL) 1775 tsi_info(&priv->pdev->dev, 1776 "OB MBOX%d: Status FIFO is full", ch); 1777 1778 if (omsg_int & (TSI721_OBDMAC_INT_DONE | TSI721_OBDMAC_INT_IOF_DONE)) { 1779 u32 srd_ptr; 1780 u64 *sts_ptr, last_ptr = 0, prev_ptr = 0; 1781 int i, j; 1782 1783 /* 1784 * Find last successfully processed descriptor 1785 */ 1786 1787 /* Check and clear descriptor status FIFO entries */ 1788 srd_ptr = priv->omsg_ring[ch].sts_rdptr; 1789 sts_ptr = priv->omsg_ring[ch].sts_base; 1790 j = srd_ptr * 8; 1791 while (sts_ptr[j]) { 1792 for (i = 0; i < 8 && sts_ptr[j]; i++, j++) { 1793 prev_ptr = last_ptr; 1794 last_ptr = le64_to_cpu(sts_ptr[j]); 1795 sts_ptr[j] = 0; 1796 } 1797 1798 ++srd_ptr; 1799 srd_ptr %= priv->omsg_ring[ch].sts_size; 1800 j = srd_ptr * 8; 1801 } 1802 1803 if (last_ptr == 0) 1804 goto no_sts_update; 1805 1806 priv->omsg_ring[ch].sts_rdptr = srd_ptr; 1807 iowrite32(srd_ptr, priv->regs + TSI721_OBDMAC_DSRP(ch)); 1808 1809 if (!mport->outb_msg[ch].mcback) 1810 goto no_sts_update; 1811 1812 /* Inform upper layer about transfer completion */ 1813 1814 tx_slot = (last_ptr - (u64)priv->omsg_ring[ch].omd_phys)/ 1815 sizeof(struct tsi721_omsg_desc); 1816 1817 /* 1818 * Check if this is a Link Descriptor (LD). 1819 * If yes, ignore LD and use descriptor processed 1820 * before LD. 1821 */ 1822 if (tx_slot == priv->omsg_ring[ch].size) { 1823 if (prev_ptr) 1824 tx_slot = (prev_ptr - 1825 (u64)priv->omsg_ring[ch].omd_phys)/ 1826 sizeof(struct tsi721_omsg_desc); 1827 else 1828 goto no_sts_update; 1829 } 1830 1831 if (tx_slot >= priv->omsg_ring[ch].size) 1832 tsi_debug(OMSG, &priv->pdev->dev, 1833 "OB_MSG tx_slot=%x > size=%x", 1834 tx_slot, priv->omsg_ring[ch].size); 1835 WARN_ON(tx_slot >= priv->omsg_ring[ch].size); 1836 1837 /* Move slot index to the next message to be sent */ 1838 ++tx_slot; 1839 if (tx_slot == priv->omsg_ring[ch].size) 1840 tx_slot = 0; 1841 1842 dev_id = priv->omsg_ring[ch].dev_id; 1843 do_callback = 1; 1844 } 1845 1846 no_sts_update: 1847 1848 if (omsg_int & TSI721_OBDMAC_INT_ERROR) { 1849 /* 1850 * Outbound message operation aborted due to error, 1851 * reinitialize OB MSG channel 1852 */ 1853 1854 tsi_debug(OMSG, &priv->pdev->dev, "OB MSG ABORT ch_stat=%x", 1855 ioread32(priv->regs + TSI721_OBDMAC_STS(ch))); 1856 1857 iowrite32(TSI721_OBDMAC_INT_ERROR, 1858 priv->regs + TSI721_OBDMAC_INT(ch)); 1859 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT, 1860 priv->regs + TSI721_OBDMAC_CTL(ch)); 1861 ioread32(priv->regs + TSI721_OBDMAC_CTL(ch)); 1862 1863 /* Inform upper level to clear all pending tx slots */ 1864 dev_id = priv->omsg_ring[ch].dev_id; 1865 tx_slot = priv->omsg_ring[ch].tx_slot; 1866 do_callback = 1; 1867 1868 /* Synch tx_slot tracking */ 1869 iowrite32(priv->omsg_ring[ch].tx_slot, 1870 priv->regs + TSI721_OBDMAC_DRDCNT(ch)); 1871 ioread32(priv->regs + TSI721_OBDMAC_DRDCNT(ch)); 1872 priv->omsg_ring[ch].wr_count = priv->omsg_ring[ch].tx_slot; 1873 priv->omsg_ring[ch].sts_rdptr = 0; 1874 } 1875 1876 /* Clear channel interrupts */ 1877 iowrite32(omsg_int, priv->regs + TSI721_OBDMAC_INT(ch)); 1878 1879 if (!(priv->flags & TSI721_USING_MSIX)) { 1880 u32 ch_inte; 1881 1882 /* Re-enable channel interrupts */ 1883 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 1884 ch_inte |= TSI721_INT_OMSG_CHAN(ch); 1885 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE); 1886 } 1887 1888 spin_unlock(&priv->omsg_ring[ch].lock); 1889 1890 if (mport->outb_msg[ch].mcback && do_callback) 1891 mport->outb_msg[ch].mcback(mport, dev_id, ch, tx_slot); 1892 } 1893 1894 /** 1895 * tsi721_open_outb_mbox - Initialize Tsi721 outbound mailbox 1896 * @mport: Master port implementing Outbound Messaging Engine 1897 * @dev_id: Device specific pointer to pass on event 1898 * @mbox: Mailbox to open 1899 * @entries: Number of entries in the outbound mailbox ring 1900 * 1901 * Returns: %0 on success or -errno value on failure. 1902 */ 1903 static int tsi721_open_outb_mbox(struct rio_mport *mport, void *dev_id, 1904 int mbox, int entries) 1905 { 1906 struct tsi721_device *priv = mport->priv; 1907 struct tsi721_omsg_desc *bd_ptr; 1908 int i, rc = 0; 1909 1910 if ((entries < TSI721_OMSGD_MIN_RING_SIZE) || 1911 (entries > (TSI721_OMSGD_RING_SIZE)) || 1912 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) { 1913 rc = -EINVAL; 1914 goto out; 1915 } 1916 1917 if ((mbox_sel & (1 << mbox)) == 0) { 1918 rc = -ENODEV; 1919 goto out; 1920 } 1921 1922 priv->omsg_ring[mbox].dev_id = dev_id; 1923 priv->omsg_ring[mbox].size = entries; 1924 priv->omsg_ring[mbox].sts_rdptr = 0; 1925 spin_lock_init(&priv->omsg_ring[mbox].lock); 1926 1927 /* Outbound Msg Buffer allocation based on 1928 the number of maximum descriptor entries */ 1929 for (i = 0; i < entries; i++) { 1930 priv->omsg_ring[mbox].omq_base[i] = 1931 dma_alloc_coherent( 1932 &priv->pdev->dev, TSI721_MSG_BUFFER_SIZE, 1933 &priv->omsg_ring[mbox].omq_phys[i], 1934 GFP_KERNEL); 1935 if (priv->omsg_ring[mbox].omq_base[i] == NULL) { 1936 tsi_debug(OMSG, &priv->pdev->dev, 1937 "ENOMEM for OB_MSG_%d data buffer", mbox); 1938 rc = -ENOMEM; 1939 goto out_buf; 1940 } 1941 } 1942 1943 /* Outbound message descriptor allocation */ 1944 priv->omsg_ring[mbox].omd_base = dma_alloc_coherent( 1945 &priv->pdev->dev, 1946 (entries + 1) * sizeof(struct tsi721_omsg_desc), 1947 &priv->omsg_ring[mbox].omd_phys, GFP_KERNEL); 1948 if (priv->omsg_ring[mbox].omd_base == NULL) { 1949 tsi_debug(OMSG, &priv->pdev->dev, 1950 "ENOMEM for OB_MSG_%d descriptor memory", mbox); 1951 rc = -ENOMEM; 1952 goto out_buf; 1953 } 1954 1955 priv->omsg_ring[mbox].tx_slot = 0; 1956 1957 /* Outbound message descriptor status FIFO allocation */ 1958 priv->omsg_ring[mbox].sts_size = roundup_pow_of_two(entries + 1); 1959 priv->omsg_ring[mbox].sts_base = dma_alloc_coherent(&priv->pdev->dev, 1960 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts), 1961 &priv->omsg_ring[mbox].sts_phys, 1962 GFP_KERNEL); 1963 if (priv->omsg_ring[mbox].sts_base == NULL) { 1964 tsi_debug(OMSG, &priv->pdev->dev, 1965 "ENOMEM for OB_MSG_%d status FIFO", mbox); 1966 rc = -ENOMEM; 1967 goto out_desc; 1968 } 1969 1970 /* 1971 * Configure Outbound Messaging Engine 1972 */ 1973 1974 /* Setup Outbound Message descriptor pointer */ 1975 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys >> 32), 1976 priv->regs + TSI721_OBDMAC_DPTRH(mbox)); 1977 iowrite32(((u64)priv->omsg_ring[mbox].omd_phys & 1978 TSI721_OBDMAC_DPTRL_MASK), 1979 priv->regs + TSI721_OBDMAC_DPTRL(mbox)); 1980 1981 /* Setup Outbound Message descriptor status FIFO */ 1982 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys >> 32), 1983 priv->regs + TSI721_OBDMAC_DSBH(mbox)); 1984 iowrite32(((u64)priv->omsg_ring[mbox].sts_phys & 1985 TSI721_OBDMAC_DSBL_MASK), 1986 priv->regs + TSI721_OBDMAC_DSBL(mbox)); 1987 iowrite32(TSI721_DMAC_DSSZ_SIZE(priv->omsg_ring[mbox].sts_size), 1988 priv->regs + (u32)TSI721_OBDMAC_DSSZ(mbox)); 1989 1990 /* Enable interrupts */ 1991 1992 #ifdef CONFIG_PCI_MSI 1993 if (priv->flags & TSI721_USING_MSIX) { 1994 int idx = TSI721_VECT_OMB0_DONE + mbox; 1995 1996 /* Request interrupt service if we are in MSI-X mode */ 1997 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0, 1998 priv->msix[idx].irq_name, (void *)priv); 1999 2000 if (rc) { 2001 tsi_debug(OMSG, &priv->pdev->dev, 2002 "Unable to get MSI-X IRQ for OBOX%d-DONE", 2003 mbox); 2004 goto out_stat; 2005 } 2006 2007 idx = TSI721_VECT_OMB0_INT + mbox; 2008 rc = request_irq(priv->msix[idx].vector, tsi721_omsg_msix, 0, 2009 priv->msix[idx].irq_name, (void *)priv); 2010 2011 if (rc) { 2012 tsi_debug(OMSG, &priv->pdev->dev, 2013 "Unable to get MSI-X IRQ for MBOX%d-INT", mbox); 2014 idx = TSI721_VECT_OMB0_DONE + mbox; 2015 free_irq(priv->msix[idx].vector, (void *)priv); 2016 goto out_stat; 2017 } 2018 } 2019 #endif /* CONFIG_PCI_MSI */ 2020 2021 tsi721_omsg_interrupt_enable(priv, mbox, TSI721_OBDMAC_INT_ALL); 2022 2023 /* Initialize Outbound Message descriptors ring */ 2024 bd_ptr = priv->omsg_ring[mbox].omd_base; 2025 bd_ptr[entries].type_id = cpu_to_le32(DTYPE5 << 29); 2026 bd_ptr[entries].msg_info = 0; 2027 bd_ptr[entries].next_lo = 2028 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys & 2029 TSI721_OBDMAC_DPTRL_MASK); 2030 bd_ptr[entries].next_hi = 2031 cpu_to_le32((u64)priv->omsg_ring[mbox].omd_phys >> 32); 2032 priv->omsg_ring[mbox].wr_count = 0; 2033 mb(); 2034 2035 /* Initialize Outbound Message engine */ 2036 iowrite32(TSI721_OBDMAC_CTL_RETRY_THR | TSI721_OBDMAC_CTL_INIT, 2037 priv->regs + TSI721_OBDMAC_CTL(mbox)); 2038 ioread32(priv->regs + TSI721_OBDMAC_DWRCNT(mbox)); 2039 udelay(10); 2040 2041 priv->omsg_init[mbox] = 1; 2042 2043 return 0; 2044 2045 #ifdef CONFIG_PCI_MSI 2046 out_stat: 2047 dma_free_coherent(&priv->pdev->dev, 2048 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts), 2049 priv->omsg_ring[mbox].sts_base, 2050 priv->omsg_ring[mbox].sts_phys); 2051 2052 priv->omsg_ring[mbox].sts_base = NULL; 2053 #endif /* CONFIG_PCI_MSI */ 2054 2055 out_desc: 2056 dma_free_coherent(&priv->pdev->dev, 2057 (entries + 1) * sizeof(struct tsi721_omsg_desc), 2058 priv->omsg_ring[mbox].omd_base, 2059 priv->omsg_ring[mbox].omd_phys); 2060 2061 priv->omsg_ring[mbox].omd_base = NULL; 2062 2063 out_buf: 2064 for (i = 0; i < priv->omsg_ring[mbox].size; i++) { 2065 if (priv->omsg_ring[mbox].omq_base[i]) { 2066 dma_free_coherent(&priv->pdev->dev, 2067 TSI721_MSG_BUFFER_SIZE, 2068 priv->omsg_ring[mbox].omq_base[i], 2069 priv->omsg_ring[mbox].omq_phys[i]); 2070 2071 priv->omsg_ring[mbox].omq_base[i] = NULL; 2072 } 2073 } 2074 2075 out: 2076 return rc; 2077 } 2078 2079 /** 2080 * tsi721_close_outb_mbox - Close Tsi721 outbound mailbox 2081 * @mport: Master port implementing the outbound message unit 2082 * @mbox: Mailbox to close 2083 */ 2084 static void tsi721_close_outb_mbox(struct rio_mport *mport, int mbox) 2085 { 2086 struct tsi721_device *priv = mport->priv; 2087 u32 i; 2088 2089 if (!priv->omsg_init[mbox]) 2090 return; 2091 priv->omsg_init[mbox] = 0; 2092 2093 /* Disable Interrupts */ 2094 2095 tsi721_omsg_interrupt_disable(priv, mbox, TSI721_OBDMAC_INT_ALL); 2096 2097 #ifdef CONFIG_PCI_MSI 2098 if (priv->flags & TSI721_USING_MSIX) { 2099 free_irq(priv->msix[TSI721_VECT_OMB0_DONE + mbox].vector, 2100 (void *)priv); 2101 free_irq(priv->msix[TSI721_VECT_OMB0_INT + mbox].vector, 2102 (void *)priv); 2103 } 2104 #endif /* CONFIG_PCI_MSI */ 2105 2106 /* Free OMSG Descriptor Status FIFO */ 2107 dma_free_coherent(&priv->pdev->dev, 2108 priv->omsg_ring[mbox].sts_size * sizeof(struct tsi721_dma_sts), 2109 priv->omsg_ring[mbox].sts_base, 2110 priv->omsg_ring[mbox].sts_phys); 2111 2112 priv->omsg_ring[mbox].sts_base = NULL; 2113 2114 /* Free OMSG descriptors */ 2115 dma_free_coherent(&priv->pdev->dev, 2116 (priv->omsg_ring[mbox].size + 1) * 2117 sizeof(struct tsi721_omsg_desc), 2118 priv->omsg_ring[mbox].omd_base, 2119 priv->omsg_ring[mbox].omd_phys); 2120 2121 priv->omsg_ring[mbox].omd_base = NULL; 2122 2123 /* Free message buffers */ 2124 for (i = 0; i < priv->omsg_ring[mbox].size; i++) { 2125 if (priv->omsg_ring[mbox].omq_base[i]) { 2126 dma_free_coherent(&priv->pdev->dev, 2127 TSI721_MSG_BUFFER_SIZE, 2128 priv->omsg_ring[mbox].omq_base[i], 2129 priv->omsg_ring[mbox].omq_phys[i]); 2130 2131 priv->omsg_ring[mbox].omq_base[i] = NULL; 2132 } 2133 } 2134 } 2135 2136 /** 2137 * tsi721_imsg_handler - Inbound Message Interrupt Handler 2138 * @priv: pointer to tsi721 private data 2139 * @ch: inbound message channel number to service 2140 * 2141 * Services channel interrupts from inbound messaging engine. 2142 */ 2143 static void tsi721_imsg_handler(struct tsi721_device *priv, int ch) 2144 { 2145 u32 mbox = ch - 4; 2146 u32 imsg_int; 2147 struct rio_mport *mport = &priv->mport; 2148 2149 spin_lock(&priv->imsg_ring[mbox].lock); 2150 2151 imsg_int = ioread32(priv->regs + TSI721_IBDMAC_INT(ch)); 2152 2153 if (imsg_int & TSI721_IBDMAC_INT_SRTO) 2154 tsi_info(&priv->pdev->dev, "IB MBOX%d SRIO timeout", mbox); 2155 2156 if (imsg_int & TSI721_IBDMAC_INT_PC_ERROR) 2157 tsi_info(&priv->pdev->dev, "IB MBOX%d PCIe error", mbox); 2158 2159 if (imsg_int & TSI721_IBDMAC_INT_FQ_LOW) 2160 tsi_info(&priv->pdev->dev, "IB MBOX%d IB free queue low", mbox); 2161 2162 /* Clear IB channel interrupts */ 2163 iowrite32(imsg_int, priv->regs + TSI721_IBDMAC_INT(ch)); 2164 2165 /* If an IB Msg is received notify the upper layer */ 2166 if (imsg_int & TSI721_IBDMAC_INT_DQ_RCV && 2167 mport->inb_msg[mbox].mcback) 2168 mport->inb_msg[mbox].mcback(mport, 2169 priv->imsg_ring[mbox].dev_id, mbox, -1); 2170 2171 if (!(priv->flags & TSI721_USING_MSIX)) { 2172 u32 ch_inte; 2173 2174 /* Re-enable channel interrupts */ 2175 ch_inte = ioread32(priv->regs + TSI721_DEV_CHAN_INTE); 2176 ch_inte |= TSI721_INT_IMSG_CHAN(ch); 2177 iowrite32(ch_inte, priv->regs + TSI721_DEV_CHAN_INTE); 2178 } 2179 2180 spin_unlock(&priv->imsg_ring[mbox].lock); 2181 } 2182 2183 /** 2184 * tsi721_open_inb_mbox - Initialize Tsi721 inbound mailbox 2185 * @mport: Master port implementing the Inbound Messaging Engine 2186 * @dev_id: Device specific pointer to pass on event 2187 * @mbox: Mailbox to open 2188 * @entries: Number of entries in the inbound mailbox ring 2189 * 2190 * Returns: %0 on success or -errno value on failure. 2191 */ 2192 static int tsi721_open_inb_mbox(struct rio_mport *mport, void *dev_id, 2193 int mbox, int entries) 2194 { 2195 struct tsi721_device *priv = mport->priv; 2196 int ch = mbox + 4; 2197 int i; 2198 u64 *free_ptr; 2199 int rc = 0; 2200 2201 if ((entries < TSI721_IMSGD_MIN_RING_SIZE) || 2202 (entries > TSI721_IMSGD_RING_SIZE) || 2203 (!is_power_of_2(entries)) || mbox >= RIO_MAX_MBOX) { 2204 rc = -EINVAL; 2205 goto out; 2206 } 2207 2208 if ((mbox_sel & (1 << mbox)) == 0) { 2209 rc = -ENODEV; 2210 goto out; 2211 } 2212 2213 /* Initialize IB Messaging Ring */ 2214 priv->imsg_ring[mbox].dev_id = dev_id; 2215 priv->imsg_ring[mbox].size = entries; 2216 priv->imsg_ring[mbox].rx_slot = 0; 2217 priv->imsg_ring[mbox].desc_rdptr = 0; 2218 priv->imsg_ring[mbox].fq_wrptr = 0; 2219 for (i = 0; i < priv->imsg_ring[mbox].size; i++) 2220 priv->imsg_ring[mbox].imq_base[i] = NULL; 2221 spin_lock_init(&priv->imsg_ring[mbox].lock); 2222 2223 /* Allocate buffers for incoming messages */ 2224 priv->imsg_ring[mbox].buf_base = 2225 dma_alloc_coherent(&priv->pdev->dev, 2226 entries * TSI721_MSG_BUFFER_SIZE, 2227 &priv->imsg_ring[mbox].buf_phys, 2228 GFP_KERNEL); 2229 2230 if (priv->imsg_ring[mbox].buf_base == NULL) { 2231 tsi_err(&priv->pdev->dev, 2232 "Failed to allocate buffers for IB MBOX%d", mbox); 2233 rc = -ENOMEM; 2234 goto out; 2235 } 2236 2237 /* Allocate memory for circular free list */ 2238 priv->imsg_ring[mbox].imfq_base = 2239 dma_alloc_coherent(&priv->pdev->dev, 2240 entries * 8, 2241 &priv->imsg_ring[mbox].imfq_phys, 2242 GFP_KERNEL); 2243 2244 if (priv->imsg_ring[mbox].imfq_base == NULL) { 2245 tsi_err(&priv->pdev->dev, 2246 "Failed to allocate free queue for IB MBOX%d", mbox); 2247 rc = -ENOMEM; 2248 goto out_buf; 2249 } 2250 2251 /* Allocate memory for Inbound message descriptors */ 2252 priv->imsg_ring[mbox].imd_base = 2253 dma_alloc_coherent(&priv->pdev->dev, 2254 entries * sizeof(struct tsi721_imsg_desc), 2255 &priv->imsg_ring[mbox].imd_phys, GFP_KERNEL); 2256 2257 if (priv->imsg_ring[mbox].imd_base == NULL) { 2258 tsi_err(&priv->pdev->dev, 2259 "Failed to allocate descriptor memory for IB MBOX%d", 2260 mbox); 2261 rc = -ENOMEM; 2262 goto out_dma; 2263 } 2264 2265 /* Fill free buffer pointer list */ 2266 free_ptr = priv->imsg_ring[mbox].imfq_base; 2267 for (i = 0; i < entries; i++) 2268 free_ptr[i] = cpu_to_le64( 2269 (u64)(priv->imsg_ring[mbox].buf_phys) + 2270 i * 0x1000); 2271 2272 mb(); 2273 2274 /* 2275 * For mapping of inbound SRIO Messages into appropriate queues we need 2276 * to set Inbound Device ID register in the messaging engine. We do it 2277 * once when first inbound mailbox is requested. 2278 */ 2279 if (!(priv->flags & TSI721_IMSGID_SET)) { 2280 iowrite32((u32)priv->mport.host_deviceid, 2281 priv->regs + TSI721_IB_DEVID); 2282 priv->flags |= TSI721_IMSGID_SET; 2283 } 2284 2285 /* 2286 * Configure Inbound Messaging channel (ch = mbox + 4) 2287 */ 2288 2289 /* Setup Inbound Message free queue */ 2290 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys >> 32), 2291 priv->regs + TSI721_IBDMAC_FQBH(ch)); 2292 iowrite32(((u64)priv->imsg_ring[mbox].imfq_phys & 2293 TSI721_IBDMAC_FQBL_MASK), 2294 priv->regs+TSI721_IBDMAC_FQBL(ch)); 2295 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries), 2296 priv->regs + TSI721_IBDMAC_FQSZ(ch)); 2297 2298 /* Setup Inbound Message descriptor queue */ 2299 iowrite32(((u64)priv->imsg_ring[mbox].imd_phys >> 32), 2300 priv->regs + TSI721_IBDMAC_DQBH(ch)); 2301 iowrite32(((u32)priv->imsg_ring[mbox].imd_phys & 2302 (u32)TSI721_IBDMAC_DQBL_MASK), 2303 priv->regs+TSI721_IBDMAC_DQBL(ch)); 2304 iowrite32(TSI721_DMAC_DSSZ_SIZE(entries), 2305 priv->regs + TSI721_IBDMAC_DQSZ(ch)); 2306 2307 /* Enable interrupts */ 2308 2309 #ifdef CONFIG_PCI_MSI 2310 if (priv->flags & TSI721_USING_MSIX) { 2311 int idx = TSI721_VECT_IMB0_RCV + mbox; 2312 2313 /* Request interrupt service if we are in MSI-X mode */ 2314 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0, 2315 priv->msix[idx].irq_name, (void *)priv); 2316 2317 if (rc) { 2318 tsi_debug(IMSG, &priv->pdev->dev, 2319 "Unable to get MSI-X IRQ for IBOX%d-DONE", 2320 mbox); 2321 goto out_desc; 2322 } 2323 2324 idx = TSI721_VECT_IMB0_INT + mbox; 2325 rc = request_irq(priv->msix[idx].vector, tsi721_imsg_msix, 0, 2326 priv->msix[idx].irq_name, (void *)priv); 2327 2328 if (rc) { 2329 tsi_debug(IMSG, &priv->pdev->dev, 2330 "Unable to get MSI-X IRQ for IBOX%d-INT", mbox); 2331 free_irq( 2332 priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector, 2333 (void *)priv); 2334 goto out_desc; 2335 } 2336 } 2337 #endif /* CONFIG_PCI_MSI */ 2338 2339 tsi721_imsg_interrupt_enable(priv, ch, TSI721_IBDMAC_INT_ALL); 2340 2341 /* Initialize Inbound Message Engine */ 2342 iowrite32(TSI721_IBDMAC_CTL_INIT, priv->regs + TSI721_IBDMAC_CTL(ch)); 2343 ioread32(priv->regs + TSI721_IBDMAC_CTL(ch)); 2344 udelay(10); 2345 priv->imsg_ring[mbox].fq_wrptr = entries - 1; 2346 iowrite32(entries - 1, priv->regs + TSI721_IBDMAC_FQWP(ch)); 2347 2348 priv->imsg_init[mbox] = 1; 2349 return 0; 2350 2351 #ifdef CONFIG_PCI_MSI 2352 out_desc: 2353 dma_free_coherent(&priv->pdev->dev, 2354 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc), 2355 priv->imsg_ring[mbox].imd_base, 2356 priv->imsg_ring[mbox].imd_phys); 2357 2358 priv->imsg_ring[mbox].imd_base = NULL; 2359 #endif /* CONFIG_PCI_MSI */ 2360 2361 out_dma: 2362 dma_free_coherent(&priv->pdev->dev, 2363 priv->imsg_ring[mbox].size * 8, 2364 priv->imsg_ring[mbox].imfq_base, 2365 priv->imsg_ring[mbox].imfq_phys); 2366 2367 priv->imsg_ring[mbox].imfq_base = NULL; 2368 2369 out_buf: 2370 dma_free_coherent(&priv->pdev->dev, 2371 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE, 2372 priv->imsg_ring[mbox].buf_base, 2373 priv->imsg_ring[mbox].buf_phys); 2374 2375 priv->imsg_ring[mbox].buf_base = NULL; 2376 2377 out: 2378 return rc; 2379 } 2380 2381 /** 2382 * tsi721_close_inb_mbox - Shut down Tsi721 inbound mailbox 2383 * @mport: Master port implementing the Inbound Messaging Engine 2384 * @mbox: Mailbox to close 2385 */ 2386 static void tsi721_close_inb_mbox(struct rio_mport *mport, int mbox) 2387 { 2388 struct tsi721_device *priv = mport->priv; 2389 u32 rx_slot; 2390 int ch = mbox + 4; 2391 2392 if (!priv->imsg_init[mbox]) /* mbox isn't initialized yet */ 2393 return; 2394 priv->imsg_init[mbox] = 0; 2395 2396 /* Disable Inbound Messaging Engine */ 2397 2398 /* Disable Interrupts */ 2399 tsi721_imsg_interrupt_disable(priv, ch, TSI721_OBDMAC_INT_MASK); 2400 2401 #ifdef CONFIG_PCI_MSI 2402 if (priv->flags & TSI721_USING_MSIX) { 2403 free_irq(priv->msix[TSI721_VECT_IMB0_RCV + mbox].vector, 2404 (void *)priv); 2405 free_irq(priv->msix[TSI721_VECT_IMB0_INT + mbox].vector, 2406 (void *)priv); 2407 } 2408 #endif /* CONFIG_PCI_MSI */ 2409 2410 /* Clear Inbound Buffer Queue */ 2411 for (rx_slot = 0; rx_slot < priv->imsg_ring[mbox].size; rx_slot++) 2412 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL; 2413 2414 /* Free memory allocated for message buffers */ 2415 dma_free_coherent(&priv->pdev->dev, 2416 priv->imsg_ring[mbox].size * TSI721_MSG_BUFFER_SIZE, 2417 priv->imsg_ring[mbox].buf_base, 2418 priv->imsg_ring[mbox].buf_phys); 2419 2420 priv->imsg_ring[mbox].buf_base = NULL; 2421 2422 /* Free memory allocated for free pointr list */ 2423 dma_free_coherent(&priv->pdev->dev, 2424 priv->imsg_ring[mbox].size * 8, 2425 priv->imsg_ring[mbox].imfq_base, 2426 priv->imsg_ring[mbox].imfq_phys); 2427 2428 priv->imsg_ring[mbox].imfq_base = NULL; 2429 2430 /* Free memory allocated for RX descriptors */ 2431 dma_free_coherent(&priv->pdev->dev, 2432 priv->imsg_ring[mbox].size * sizeof(struct tsi721_imsg_desc), 2433 priv->imsg_ring[mbox].imd_base, 2434 priv->imsg_ring[mbox].imd_phys); 2435 2436 priv->imsg_ring[mbox].imd_base = NULL; 2437 } 2438 2439 /** 2440 * tsi721_add_inb_buffer - Add buffer to the Tsi721 inbound message queue 2441 * @mport: Master port implementing the Inbound Messaging Engine 2442 * @mbox: Inbound mailbox number 2443 * @buf: Buffer to add to inbound queue 2444 * 2445 * Returns: %0 on success or -errno value on failure. 2446 */ 2447 static int tsi721_add_inb_buffer(struct rio_mport *mport, int mbox, void *buf) 2448 { 2449 struct tsi721_device *priv = mport->priv; 2450 u32 rx_slot; 2451 int rc = 0; 2452 2453 rx_slot = priv->imsg_ring[mbox].rx_slot; 2454 if (priv->imsg_ring[mbox].imq_base[rx_slot]) { 2455 tsi_err(&priv->pdev->dev, 2456 "Error adding inbound buffer %d, buffer exists", 2457 rx_slot); 2458 rc = -EINVAL; 2459 goto out; 2460 } 2461 2462 priv->imsg_ring[mbox].imq_base[rx_slot] = buf; 2463 2464 if (++priv->imsg_ring[mbox].rx_slot == priv->imsg_ring[mbox].size) 2465 priv->imsg_ring[mbox].rx_slot = 0; 2466 2467 out: 2468 return rc; 2469 } 2470 2471 /** 2472 * tsi721_get_inb_message - Fetch inbound message from the Tsi721 MSG Queue 2473 * @mport: Master port implementing the Inbound Messaging Engine 2474 * @mbox: Inbound mailbox number 2475 * 2476 * Returns: pointer to the message on success or %NULL on failure. 2477 */ 2478 static void *tsi721_get_inb_message(struct rio_mport *mport, int mbox) 2479 { 2480 struct tsi721_device *priv = mport->priv; 2481 struct tsi721_imsg_desc *desc; 2482 u32 rx_slot; 2483 void *rx_virt = NULL; 2484 u64 rx_phys; 2485 void *buf = NULL; 2486 u64 *free_ptr; 2487 int ch = mbox + 4; 2488 int msg_size; 2489 2490 if (!priv->imsg_init[mbox]) 2491 return NULL; 2492 2493 desc = priv->imsg_ring[mbox].imd_base; 2494 desc += priv->imsg_ring[mbox].desc_rdptr; 2495 2496 if (!(le32_to_cpu(desc->msg_info) & TSI721_IMD_HO)) 2497 goto out; 2498 2499 rx_slot = priv->imsg_ring[mbox].rx_slot; 2500 while (priv->imsg_ring[mbox].imq_base[rx_slot] == NULL) { 2501 if (++rx_slot == priv->imsg_ring[mbox].size) 2502 rx_slot = 0; 2503 } 2504 2505 rx_phys = ((u64)le32_to_cpu(desc->bufptr_hi) << 32) | 2506 le32_to_cpu(desc->bufptr_lo); 2507 2508 rx_virt = priv->imsg_ring[mbox].buf_base + 2509 (rx_phys - (u64)priv->imsg_ring[mbox].buf_phys); 2510 2511 buf = priv->imsg_ring[mbox].imq_base[rx_slot]; 2512 msg_size = le32_to_cpu(desc->msg_info) & TSI721_IMD_BCOUNT; 2513 if (msg_size == 0) 2514 msg_size = RIO_MAX_MSG_SIZE; 2515 2516 memcpy(buf, rx_virt, msg_size); 2517 priv->imsg_ring[mbox].imq_base[rx_slot] = NULL; 2518 2519 desc->msg_info &= cpu_to_le32(~TSI721_IMD_HO); 2520 if (++priv->imsg_ring[mbox].desc_rdptr == priv->imsg_ring[mbox].size) 2521 priv->imsg_ring[mbox].desc_rdptr = 0; 2522 2523 iowrite32(priv->imsg_ring[mbox].desc_rdptr, 2524 priv->regs + TSI721_IBDMAC_DQRP(ch)); 2525 2526 /* Return free buffer into the pointer list */ 2527 free_ptr = priv->imsg_ring[mbox].imfq_base; 2528 free_ptr[priv->imsg_ring[mbox].fq_wrptr] = cpu_to_le64(rx_phys); 2529 2530 if (++priv->imsg_ring[mbox].fq_wrptr == priv->imsg_ring[mbox].size) 2531 priv->imsg_ring[mbox].fq_wrptr = 0; 2532 2533 iowrite32(priv->imsg_ring[mbox].fq_wrptr, 2534 priv->regs + TSI721_IBDMAC_FQWP(ch)); 2535 out: 2536 return buf; 2537 } 2538 2539 /** 2540 * tsi721_messages_init - Initialization of Messaging Engine 2541 * @priv: pointer to tsi721 private data 2542 * 2543 * Configures Tsi721 messaging engine. 2544 * 2545 * Returns: %0 2546 */ 2547 static int tsi721_messages_init(struct tsi721_device *priv) 2548 { 2549 int ch; 2550 2551 iowrite32(0, priv->regs + TSI721_SMSG_ECC_LOG); 2552 iowrite32(0, priv->regs + TSI721_RETRY_GEN_CNT); 2553 iowrite32(0, priv->regs + TSI721_RETRY_RX_CNT); 2554 2555 /* Set SRIO Message Request/Response Timeout */ 2556 iowrite32(TSI721_RQRPTO_VAL, priv->regs + TSI721_RQRPTO); 2557 2558 /* Initialize Inbound Messaging Engine Registers */ 2559 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) { 2560 /* Clear interrupt bits */ 2561 iowrite32(TSI721_IBDMAC_INT_MASK, 2562 priv->regs + TSI721_IBDMAC_INT(ch)); 2563 /* Clear Status */ 2564 iowrite32(0, priv->regs + TSI721_IBDMAC_STS(ch)); 2565 2566 iowrite32(TSI721_SMSG_ECC_COR_LOG_MASK, 2567 priv->regs + TSI721_SMSG_ECC_COR_LOG(ch)); 2568 iowrite32(TSI721_SMSG_ECC_NCOR_MASK, 2569 priv->regs + TSI721_SMSG_ECC_NCOR(ch)); 2570 } 2571 2572 return 0; 2573 } 2574 2575 /** 2576 * tsi721_query_mport - Fetch inbound message from the Tsi721 MSG Queue 2577 * @mport: Master port implementing the Inbound Messaging Engine 2578 * @attr: mport device attributes 2579 * 2580 * Returns: pointer to the message on success or %NULL on failure. 2581 */ 2582 static int tsi721_query_mport(struct rio_mport *mport, 2583 struct rio_mport_attr *attr) 2584 { 2585 struct tsi721_device *priv = mport->priv; 2586 u32 rval; 2587 2588 rval = ioread32(priv->regs + 0x100 + RIO_PORT_N_ERR_STS_CSR(0, 0)); 2589 if (rval & RIO_PORT_N_ERR_STS_PORT_OK) { 2590 rval = ioread32(priv->regs + 0x100 + RIO_PORT_N_CTL2_CSR(0, 0)); 2591 attr->link_speed = (rval & RIO_PORT_N_CTL2_SEL_BAUD) >> 28; 2592 rval = ioread32(priv->regs + 0x100 + RIO_PORT_N_CTL_CSR(0, 0)); 2593 attr->link_width = (rval & RIO_PORT_N_CTL_IPW) >> 27; 2594 } else 2595 attr->link_speed = RIO_LINK_DOWN; 2596 2597 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 2598 attr->flags = RIO_MPORT_DMA | RIO_MPORT_DMA_SG; 2599 attr->dma_max_sge = 0; 2600 attr->dma_max_size = TSI721_BDMA_MAX_BCOUNT; 2601 attr->dma_align = 0; 2602 #else 2603 attr->flags = 0; 2604 #endif 2605 return 0; 2606 } 2607 2608 /** 2609 * tsi721_disable_ints - disables all device interrupts 2610 * @priv: pointer to tsi721 private data 2611 */ 2612 static void tsi721_disable_ints(struct tsi721_device *priv) 2613 { 2614 int ch; 2615 2616 /* Disable all device level interrupts */ 2617 iowrite32(0, priv->regs + TSI721_DEV_INTE); 2618 2619 /* Disable all Device Channel interrupts */ 2620 iowrite32(0, priv->regs + TSI721_DEV_CHAN_INTE); 2621 2622 /* Disable all Inbound Msg Channel interrupts */ 2623 for (ch = 0; ch < TSI721_IMSG_CHNUM; ch++) 2624 iowrite32(0, priv->regs + TSI721_IBDMAC_INTE(ch)); 2625 2626 /* Disable all Outbound Msg Channel interrupts */ 2627 for (ch = 0; ch < TSI721_OMSG_CHNUM; ch++) 2628 iowrite32(0, priv->regs + TSI721_OBDMAC_INTE(ch)); 2629 2630 /* Disable all general messaging interrupts */ 2631 iowrite32(0, priv->regs + TSI721_SMSG_INTE); 2632 2633 /* Disable all BDMA Channel interrupts */ 2634 for (ch = 0; ch < TSI721_DMA_MAXCH; ch++) 2635 iowrite32(0, 2636 priv->regs + TSI721_DMAC_BASE(ch) + TSI721_DMAC_INTE); 2637 2638 /* Disable all general BDMA interrupts */ 2639 iowrite32(0, priv->regs + TSI721_BDMA_INTE); 2640 2641 /* Disable all SRIO Channel interrupts */ 2642 for (ch = 0; ch < TSI721_SRIO_MAXCH; ch++) 2643 iowrite32(0, priv->regs + TSI721_SR_CHINTE(ch)); 2644 2645 /* Disable all general SR2PC interrupts */ 2646 iowrite32(0, priv->regs + TSI721_SR2PC_GEN_INTE); 2647 2648 /* Disable all PC2SR interrupts */ 2649 iowrite32(0, priv->regs + TSI721_PC2SR_INTE); 2650 2651 /* Disable all I2C interrupts */ 2652 iowrite32(0, priv->regs + TSI721_I2C_INT_ENABLE); 2653 2654 /* Disable SRIO MAC interrupts */ 2655 iowrite32(0, priv->regs + TSI721_RIO_EM_INT_ENABLE); 2656 iowrite32(0, priv->regs + TSI721_RIO_EM_DEV_INT_EN); 2657 } 2658 2659 static struct rio_ops tsi721_rio_ops = { 2660 .lcread = tsi721_lcread, 2661 .lcwrite = tsi721_lcwrite, 2662 .cread = tsi721_cread_dma, 2663 .cwrite = tsi721_cwrite_dma, 2664 .dsend = tsi721_dsend, 2665 .open_inb_mbox = tsi721_open_inb_mbox, 2666 .close_inb_mbox = tsi721_close_inb_mbox, 2667 .open_outb_mbox = tsi721_open_outb_mbox, 2668 .close_outb_mbox = tsi721_close_outb_mbox, 2669 .add_outb_message = tsi721_add_outb_message, 2670 .add_inb_buffer = tsi721_add_inb_buffer, 2671 .get_inb_message = tsi721_get_inb_message, 2672 .map_inb = tsi721_rio_map_inb_mem, 2673 .unmap_inb = tsi721_rio_unmap_inb_mem, 2674 .pwenable = tsi721_pw_enable, 2675 .query_mport = tsi721_query_mport, 2676 .map_outb = tsi721_map_outb_win, 2677 .unmap_outb = tsi721_unmap_outb_win, 2678 }; 2679 2680 static void tsi721_mport_release(struct device *dev) 2681 { 2682 struct rio_mport *mport = to_rio_mport(dev); 2683 2684 tsi_debug(EXIT, dev, "%s id=%d", mport->name, mport->id); 2685 } 2686 2687 /** 2688 * tsi721_setup_mport - Setup Tsi721 as RapidIO subsystem master port 2689 * @priv: pointer to tsi721 private data 2690 * 2691 * Configures Tsi721 as RapidIO master port. 2692 * 2693 * Returns: %0 on success or -errno value on failure. 2694 */ 2695 static int tsi721_setup_mport(struct tsi721_device *priv) 2696 { 2697 struct pci_dev *pdev = priv->pdev; 2698 int err = 0; 2699 struct rio_mport *mport = &priv->mport; 2700 2701 err = rio_mport_initialize(mport); 2702 if (err) 2703 return err; 2704 2705 mport->ops = &tsi721_rio_ops; 2706 mport->index = 0; 2707 mport->sys_size = 0; /* small system */ 2708 mport->priv = (void *)priv; 2709 mport->phys_efptr = 0x100; 2710 mport->phys_rmap = 1; 2711 mport->dev.parent = &pdev->dev; 2712 mport->dev.release = tsi721_mport_release; 2713 2714 INIT_LIST_HEAD(&mport->dbells); 2715 2716 rio_init_dbell_res(&mport->riores[RIO_DOORBELL_RESOURCE], 0, 0xffff); 2717 rio_init_mbox_res(&mport->riores[RIO_INB_MBOX_RESOURCE], 0, 3); 2718 rio_init_mbox_res(&mport->riores[RIO_OUTB_MBOX_RESOURCE], 0, 3); 2719 snprintf(mport->name, RIO_MAX_MPORT_NAME, "%s(%s)", 2720 dev_driver_string(&pdev->dev), dev_name(&pdev->dev)); 2721 2722 /* Hook up interrupt handler */ 2723 2724 #ifdef CONFIG_PCI_MSI 2725 if (!tsi721_enable_msix(priv)) 2726 priv->flags |= TSI721_USING_MSIX; 2727 else if (!pci_enable_msi(pdev)) 2728 priv->flags |= TSI721_USING_MSI; 2729 else 2730 tsi_debug(MPORT, &pdev->dev, 2731 "MSI/MSI-X is not available. Using legacy INTx."); 2732 #endif /* CONFIG_PCI_MSI */ 2733 2734 err = tsi721_request_irq(priv); 2735 2736 if (err) { 2737 tsi_err(&pdev->dev, "Unable to get PCI IRQ %02X (err=0x%x)", 2738 pdev->irq, err); 2739 return err; 2740 } 2741 2742 #ifdef CONFIG_RAPIDIO_DMA_ENGINE 2743 err = tsi721_register_dma(priv); 2744 if (err) 2745 goto err_exit; 2746 #endif 2747 /* Enable SRIO link */ 2748 iowrite32(ioread32(priv->regs + TSI721_DEVCTL) | 2749 TSI721_DEVCTL_SRBOOT_CMPL, 2750 priv->regs + TSI721_DEVCTL); 2751 2752 if (mport->host_deviceid >= 0) 2753 iowrite32(RIO_PORT_GEN_HOST | RIO_PORT_GEN_MASTER | 2754 RIO_PORT_GEN_DISCOVERED, 2755 priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR)); 2756 else 2757 iowrite32(0, priv->regs + (0x100 + RIO_PORT_GEN_CTL_CSR)); 2758 2759 err = rio_register_mport(mport); 2760 if (err) { 2761 tsi721_unregister_dma(priv); 2762 goto err_exit; 2763 } 2764 2765 return 0; 2766 2767 err_exit: 2768 tsi721_free_irq(priv); 2769 return err; 2770 } 2771 2772 static int tsi721_probe(struct pci_dev *pdev, 2773 const struct pci_device_id *id) 2774 { 2775 struct tsi721_device *priv; 2776 int err; 2777 2778 priv = kzalloc_obj(struct tsi721_device); 2779 if (!priv) { 2780 err = -ENOMEM; 2781 goto err_exit; 2782 } 2783 2784 err = pci_enable_device(pdev); 2785 if (err) { 2786 tsi_err(&pdev->dev, "Failed to enable PCI device"); 2787 goto err_clean; 2788 } 2789 2790 priv->pdev = pdev; 2791 2792 #ifdef DEBUG 2793 { 2794 int i; 2795 2796 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 2797 tsi_debug(INIT, &pdev->dev, "res%d %pR", 2798 i, &pdev->resource[i]); 2799 } 2800 } 2801 #endif 2802 /* 2803 * Verify BAR configuration 2804 */ 2805 2806 /* BAR_0 (registers) must be 512KB+ in 32-bit address space */ 2807 if (!(pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM) || 2808 pci_resource_flags(pdev, BAR_0) & IORESOURCE_MEM_64 || 2809 pci_resource_len(pdev, BAR_0) < TSI721_REG_SPACE_SIZE) { 2810 tsi_err(&pdev->dev, "Missing or misconfigured CSR BAR0"); 2811 err = -ENODEV; 2812 goto err_disable_pdev; 2813 } 2814 2815 /* BAR_1 (outbound doorbells) must be 16MB+ in 32-bit address space */ 2816 if (!(pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM) || 2817 pci_resource_flags(pdev, BAR_1) & IORESOURCE_MEM_64 || 2818 pci_resource_len(pdev, BAR_1) < TSI721_DB_WIN_SIZE) { 2819 tsi_err(&pdev->dev, "Missing or misconfigured Doorbell BAR1"); 2820 err = -ENODEV; 2821 goto err_disable_pdev; 2822 } 2823 2824 /* 2825 * BAR_2 and BAR_4 (outbound translation) must be in 64-bit PCIe address 2826 * space. 2827 * NOTE: BAR_2 and BAR_4 are not used by this version of driver. 2828 * It may be a good idea to keep them disabled using HW configuration 2829 * to save PCI memory space. 2830 */ 2831 2832 priv->p2r_bar[0].size = priv->p2r_bar[1].size = 0; 2833 2834 if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_MEM_64) { 2835 if (pci_resource_flags(pdev, BAR_2) & IORESOURCE_PREFETCH) 2836 tsi_debug(INIT, &pdev->dev, 2837 "Prefetchable OBW BAR2 will not be used"); 2838 else { 2839 priv->p2r_bar[0].base = pci_resource_start(pdev, BAR_2); 2840 priv->p2r_bar[0].size = pci_resource_len(pdev, BAR_2); 2841 } 2842 } 2843 2844 if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_MEM_64) { 2845 if (pci_resource_flags(pdev, BAR_4) & IORESOURCE_PREFETCH) 2846 tsi_debug(INIT, &pdev->dev, 2847 "Prefetchable OBW BAR4 will not be used"); 2848 else { 2849 priv->p2r_bar[1].base = pci_resource_start(pdev, BAR_4); 2850 priv->p2r_bar[1].size = pci_resource_len(pdev, BAR_4); 2851 } 2852 } 2853 2854 err = pci_request_regions(pdev, DRV_NAME); 2855 if (err) { 2856 tsi_err(&pdev->dev, "Unable to obtain PCI resources"); 2857 goto err_disable_pdev; 2858 } 2859 2860 pci_set_master(pdev); 2861 2862 priv->regs = pci_ioremap_bar(pdev, BAR_0); 2863 if (!priv->regs) { 2864 tsi_err(&pdev->dev, "Unable to map device registers space"); 2865 err = -ENOMEM; 2866 goto err_free_res; 2867 } 2868 2869 priv->odb_base = pci_ioremap_bar(pdev, BAR_1); 2870 if (!priv->odb_base) { 2871 tsi_err(&pdev->dev, "Unable to map outbound doorbells space"); 2872 err = -ENOMEM; 2873 goto err_unmap_bars; 2874 } 2875 2876 /* Configure DMA attributes. */ 2877 if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) { 2878 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 2879 if (err) { 2880 tsi_err(&pdev->dev, "Unable to set DMA mask"); 2881 goto err_unmap_bars; 2882 } 2883 2884 if (dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32))) 2885 tsi_info(&pdev->dev, "Unable to set consistent DMA mask"); 2886 } else { 2887 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64)); 2888 if (err) 2889 tsi_info(&pdev->dev, "Unable to set consistent DMA mask"); 2890 } 2891 2892 BUG_ON(!pci_is_pcie(pdev)); 2893 2894 /* Clear "no snoop" and "relaxed ordering" bits. */ 2895 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL, 2896 PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN, 0); 2897 2898 /* Override PCIe Maximum Read Request Size setting if requested */ 2899 if (pcie_mrrs >= 0) { 2900 if (pcie_mrrs <= 5) 2901 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL, 2902 PCI_EXP_DEVCTL_READRQ, pcie_mrrs << 12); 2903 else 2904 tsi_info(&pdev->dev, 2905 "Invalid MRRS override value %d", pcie_mrrs); 2906 } 2907 2908 /* Set PCIe completion timeout to 1-10ms */ 2909 pcie_capability_clear_and_set_word(pdev, PCI_EXP_DEVCTL2, 2910 PCI_EXP_DEVCTL2_COMP_TIMEOUT, 0x2); 2911 2912 /* 2913 * FIXUP: correct offsets of MSI-X tables in the MSI-X Capability Block 2914 */ 2915 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0x01); 2916 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXTBL, 2917 TSI721_MSIXTBL_OFFSET); 2918 pci_write_config_dword(pdev, TSI721_PCIECFG_MSIXPBA, 2919 TSI721_MSIXPBA_OFFSET); 2920 pci_write_config_dword(pdev, TSI721_PCIECFG_EPCTL, 0); 2921 /* End of FIXUP */ 2922 2923 tsi721_disable_ints(priv); 2924 2925 tsi721_init_pc2sr_mapping(priv); 2926 tsi721_init_sr2pc_mapping(priv); 2927 2928 if (tsi721_bdma_maint_init(priv)) { 2929 tsi_err(&pdev->dev, "BDMA initialization failed"); 2930 err = -ENOMEM; 2931 goto err_unmap_bars; 2932 } 2933 2934 err = tsi721_doorbell_init(priv); 2935 if (err) 2936 goto err_free_bdma; 2937 2938 tsi721_port_write_init(priv); 2939 2940 err = tsi721_messages_init(priv); 2941 if (err) 2942 goto err_free_consistent; 2943 2944 err = tsi721_setup_mport(priv); 2945 if (err) 2946 goto err_free_consistent; 2947 2948 pci_set_drvdata(pdev, priv); 2949 tsi721_interrupts_init(priv); 2950 2951 return 0; 2952 2953 err_free_consistent: 2954 tsi721_port_write_free(priv); 2955 tsi721_doorbell_free(priv); 2956 err_free_bdma: 2957 tsi721_bdma_maint_free(priv); 2958 err_unmap_bars: 2959 if (priv->regs) 2960 iounmap(priv->regs); 2961 if (priv->odb_base) 2962 iounmap(priv->odb_base); 2963 err_free_res: 2964 pci_release_regions(pdev); 2965 err_disable_pdev: 2966 pci_disable_device(pdev); 2967 err_clean: 2968 kfree(priv); 2969 err_exit: 2970 return err; 2971 } 2972 2973 static void tsi721_remove(struct pci_dev *pdev) 2974 { 2975 struct tsi721_device *priv = pci_get_drvdata(pdev); 2976 2977 tsi_debug(EXIT, &pdev->dev, "enter"); 2978 2979 tsi721_disable_ints(priv); 2980 tsi721_free_irq(priv); 2981 flush_work(&priv->idb_work); 2982 flush_work(&priv->pw_work); 2983 rio_unregister_mport(&priv->mport); 2984 2985 tsi721_unregister_dma(priv); 2986 tsi721_bdma_maint_free(priv); 2987 tsi721_doorbell_free(priv); 2988 tsi721_port_write_free(priv); 2989 tsi721_close_sr2pc_mapping(priv); 2990 2991 if (priv->regs) 2992 iounmap(priv->regs); 2993 if (priv->odb_base) 2994 iounmap(priv->odb_base); 2995 #ifdef CONFIG_PCI_MSI 2996 if (priv->flags & TSI721_USING_MSIX) 2997 pci_disable_msix(priv->pdev); 2998 else if (priv->flags & TSI721_USING_MSI) 2999 pci_disable_msi(priv->pdev); 3000 #endif 3001 pci_release_regions(pdev); 3002 pci_disable_device(pdev); 3003 pci_set_drvdata(pdev, NULL); 3004 kfree(priv); 3005 tsi_debug(EXIT, &pdev->dev, "exit"); 3006 } 3007 3008 static void tsi721_shutdown(struct pci_dev *pdev) 3009 { 3010 struct tsi721_device *priv = pci_get_drvdata(pdev); 3011 3012 tsi_debug(EXIT, &pdev->dev, "enter"); 3013 3014 tsi721_disable_ints(priv); 3015 tsi721_dma_stop_all(priv); 3016 pci_disable_device(pdev); 3017 } 3018 3019 static const struct pci_device_id tsi721_pci_tbl[] = { 3020 { PCI_DEVICE(PCI_VENDOR_ID_IDT, PCI_DEVICE_ID_TSI721) }, 3021 { 0, } /* terminate list */ 3022 }; 3023 3024 MODULE_DEVICE_TABLE(pci, tsi721_pci_tbl); 3025 3026 static struct pci_driver tsi721_driver = { 3027 .name = "tsi721", 3028 .id_table = tsi721_pci_tbl, 3029 .probe = tsi721_probe, 3030 .remove = tsi721_remove, 3031 .shutdown = tsi721_shutdown, 3032 }; 3033 3034 module_pci_driver(tsi721_driver); 3035 3036 MODULE_DESCRIPTION("IDT Tsi721 PCIExpress-to-SRIO bridge driver"); 3037 MODULE_AUTHOR("Integrated Device Technology, Inc."); 3038 MODULE_LICENSE("GPL"); 3039