1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * sata_nv.c - NVIDIA nForce SATA 4 * 5 * Copyright 2004 NVIDIA Corp. All rights reserved. 6 * Copyright 2004 Andrew Chew 7 * 8 * libata documentation is available via 'make {ps|pdf}docs', 9 * as Documentation/driver-api/libata.rst 10 * 11 * No hardware documentation available outside of NVIDIA. 12 * This driver programs the NVIDIA SATA controller in a similar 13 * fashion as with other PCI IDE BMDMA controllers, with a few 14 * NV-specific details such as register offsets, SATA phy location, 15 * hotplug info, etc. 16 * 17 * CK804/MCP04 controllers support an alternate programming interface 18 * similar to the ADMA specification (with some modifications). 19 * This allows the use of NCQ. Non-DMA-mapped ATA commands are still 20 * sent through the legacy interface. 21 */ 22 23 #include <linux/kernel.h> 24 #include <linux/module.h> 25 #include <linux/gfp.h> 26 #include <linux/pci.h> 27 #include <linux/blkdev.h> 28 #include <linux/delay.h> 29 #include <linux/interrupt.h> 30 #include <linux/device.h> 31 #include <scsi/scsi_host.h> 32 #include <scsi/scsi_device.h> 33 #include <linux/libata.h> 34 #include <trace/events/libata.h> 35 36 #define DRV_NAME "sata_nv" 37 #define DRV_VERSION "3.5" 38 39 #define NV_ADMA_DMA_BOUNDARY 0xffffffffUL 40 41 enum { 42 NV_MMIO_BAR = 5, 43 44 NV_PORTS = 2, 45 NV_PIO_MASK = ATA_PIO4, 46 NV_MWDMA_MASK = ATA_MWDMA2, 47 NV_UDMA_MASK = ATA_UDMA6, 48 NV_PORT0_SCR_REG_OFFSET = 0x00, 49 NV_PORT1_SCR_REG_OFFSET = 0x40, 50 51 /* INT_STATUS/ENABLE */ 52 NV_INT_STATUS = 0x10, 53 NV_INT_ENABLE = 0x11, 54 NV_INT_STATUS_CK804 = 0x440, 55 NV_INT_ENABLE_CK804 = 0x441, 56 57 /* INT_STATUS/ENABLE bits */ 58 NV_INT_DEV = 0x01, 59 NV_INT_PM = 0x02, 60 NV_INT_ADDED = 0x04, 61 NV_INT_REMOVED = 0x08, 62 63 NV_INT_PORT_SHIFT = 4, /* each port occupies 4 bits */ 64 65 NV_INT_ALL = 0x0f, 66 NV_INT_MASK = NV_INT_DEV | 67 NV_INT_ADDED | NV_INT_REMOVED, 68 69 /* INT_CONFIG */ 70 NV_INT_CONFIG = 0x12, 71 NV_INT_CONFIG_METHD = 0x01, // 0 = INT, 1 = SMI 72 73 // For PCI config register 20 74 NV_MCP_SATA_CFG_20 = 0x50, 75 NV_MCP_SATA_CFG_20_SATA_SPACE_EN = 0x04, 76 NV_MCP_SATA_CFG_20_PORT0_EN = (1 << 17), 77 NV_MCP_SATA_CFG_20_PORT1_EN = (1 << 16), 78 NV_MCP_SATA_CFG_20_PORT0_PWB_EN = (1 << 14), 79 NV_MCP_SATA_CFG_20_PORT1_PWB_EN = (1 << 12), 80 81 NV_ADMA_MAX_CPBS = 32, 82 NV_ADMA_CPB_SZ = 128, 83 NV_ADMA_APRD_SZ = 16, 84 NV_ADMA_SGTBL_LEN = (1024 - NV_ADMA_CPB_SZ) / 85 NV_ADMA_APRD_SZ, 86 NV_ADMA_SGTBL_TOTAL_LEN = NV_ADMA_SGTBL_LEN + 5, 87 NV_ADMA_SGTBL_SZ = NV_ADMA_SGTBL_LEN * NV_ADMA_APRD_SZ, 88 NV_ADMA_PORT_PRIV_DMA_SZ = NV_ADMA_MAX_CPBS * 89 (NV_ADMA_CPB_SZ + NV_ADMA_SGTBL_SZ), 90 91 /* BAR5 offset to ADMA general registers */ 92 NV_ADMA_GEN = 0x400, 93 NV_ADMA_GEN_CTL = 0x00, 94 NV_ADMA_NOTIFIER_CLEAR = 0x30, 95 96 /* BAR5 offset to ADMA ports */ 97 NV_ADMA_PORT = 0x480, 98 99 /* size of ADMA port register space */ 100 NV_ADMA_PORT_SIZE = 0x100, 101 102 /* ADMA port registers */ 103 NV_ADMA_CTL = 0x40, 104 NV_ADMA_CPB_COUNT = 0x42, 105 NV_ADMA_NEXT_CPB_IDX = 0x43, 106 NV_ADMA_STAT = 0x44, 107 NV_ADMA_CPB_BASE_LOW = 0x48, 108 NV_ADMA_CPB_BASE_HIGH = 0x4C, 109 NV_ADMA_APPEND = 0x50, 110 NV_ADMA_NOTIFIER = 0x68, 111 NV_ADMA_NOTIFIER_ERROR = 0x6C, 112 113 /* NV_ADMA_CTL register bits */ 114 NV_ADMA_CTL_HOTPLUG_IEN = (1 << 0), 115 NV_ADMA_CTL_CHANNEL_RESET = (1 << 5), 116 NV_ADMA_CTL_GO = (1 << 7), 117 NV_ADMA_CTL_AIEN = (1 << 8), 118 NV_ADMA_CTL_READ_NON_COHERENT = (1 << 11), 119 NV_ADMA_CTL_WRITE_NON_COHERENT = (1 << 12), 120 121 /* CPB response flag bits */ 122 NV_CPB_RESP_DONE = (1 << 0), 123 NV_CPB_RESP_ATA_ERR = (1 << 3), 124 NV_CPB_RESP_CMD_ERR = (1 << 4), 125 NV_CPB_RESP_CPB_ERR = (1 << 7), 126 127 /* CPB control flag bits */ 128 NV_CPB_CTL_CPB_VALID = (1 << 0), 129 NV_CPB_CTL_QUEUE = (1 << 1), 130 NV_CPB_CTL_APRD_VALID = (1 << 2), 131 NV_CPB_CTL_IEN = (1 << 3), 132 NV_CPB_CTL_FPDMA = (1 << 4), 133 134 /* APRD flags */ 135 NV_APRD_WRITE = (1 << 1), 136 NV_APRD_END = (1 << 2), 137 NV_APRD_CONT = (1 << 3), 138 139 /* NV_ADMA_STAT flags */ 140 NV_ADMA_STAT_TIMEOUT = (1 << 0), 141 NV_ADMA_STAT_HOTUNPLUG = (1 << 1), 142 NV_ADMA_STAT_HOTPLUG = (1 << 2), 143 NV_ADMA_STAT_CPBERR = (1 << 4), 144 NV_ADMA_STAT_SERROR = (1 << 5), 145 NV_ADMA_STAT_CMD_COMPLETE = (1 << 6), 146 NV_ADMA_STAT_IDLE = (1 << 8), 147 NV_ADMA_STAT_LEGACY = (1 << 9), 148 NV_ADMA_STAT_STOPPED = (1 << 10), 149 NV_ADMA_STAT_DONE = (1 << 12), 150 NV_ADMA_STAT_ERR = NV_ADMA_STAT_CPBERR | 151 NV_ADMA_STAT_TIMEOUT, 152 153 /* port flags */ 154 NV_ADMA_PORT_REGISTER_MODE = (1 << 0), 155 NV_ADMA_ATAPI_SETUP_COMPLETE = (1 << 1), 156 157 /* MCP55 reg offset */ 158 NV_CTL_MCP55 = 0x400, 159 NV_INT_STATUS_MCP55 = 0x440, 160 NV_INT_ENABLE_MCP55 = 0x444, 161 NV_NCQ_REG_MCP55 = 0x448, 162 163 /* MCP55 */ 164 NV_INT_ALL_MCP55 = 0xffff, 165 NV_INT_PORT_SHIFT_MCP55 = 16, /* each port occupies 16 bits */ 166 NV_INT_MASK_MCP55 = NV_INT_ALL_MCP55 & 0xfffd, 167 168 /* SWNCQ ENABLE BITS*/ 169 NV_CTL_PRI_SWNCQ = 0x02, 170 NV_CTL_SEC_SWNCQ = 0x04, 171 172 /* SW NCQ status bits*/ 173 NV_SWNCQ_IRQ_DEV = (1 << 0), 174 NV_SWNCQ_IRQ_PM = (1 << 1), 175 NV_SWNCQ_IRQ_ADDED = (1 << 2), 176 NV_SWNCQ_IRQ_REMOVED = (1 << 3), 177 178 NV_SWNCQ_IRQ_BACKOUT = (1 << 4), 179 NV_SWNCQ_IRQ_SDBFIS = (1 << 5), 180 NV_SWNCQ_IRQ_DHREGFIS = (1 << 6), 181 NV_SWNCQ_IRQ_DMASETUP = (1 << 7), 182 183 NV_SWNCQ_IRQ_HOTPLUG = NV_SWNCQ_IRQ_ADDED | 184 NV_SWNCQ_IRQ_REMOVED, 185 186 }; 187 188 /* ADMA Physical Region Descriptor - one SG segment */ 189 struct nv_adma_prd { 190 __le64 addr; 191 __le32 len; 192 u8 flags; 193 u8 packet_len; 194 __le16 reserved; 195 }; 196 197 enum nv_adma_regbits { 198 CMDEND = (1 << 15), /* end of command list */ 199 WNB = (1 << 14), /* wait-not-BSY */ 200 IGN = (1 << 13), /* ignore this entry */ 201 CS1n = (1 << (4 + 8)), /* std. PATA signals follow... */ 202 DA2 = (1 << (2 + 8)), 203 DA1 = (1 << (1 + 8)), 204 DA0 = (1 << (0 + 8)), 205 }; 206 207 /* ADMA Command Parameter Block 208 The first 5 SG segments are stored inside the Command Parameter Block itself. 209 If there are more than 5 segments the remainder are stored in a separate 210 memory area indicated by next_aprd. */ 211 struct nv_adma_cpb { 212 u8 resp_flags; /* 0 */ 213 u8 reserved1; /* 1 */ 214 u8 ctl_flags; /* 2 */ 215 /* len is length of taskfile in 64 bit words */ 216 u8 len; /* 3 */ 217 u8 tag; /* 4 */ 218 u8 next_cpb_idx; /* 5 */ 219 __le16 reserved2; /* 6-7 */ 220 __le16 tf[12]; /* 8-31 */ 221 struct nv_adma_prd aprd[5]; /* 32-111 */ 222 __le64 next_aprd; /* 112-119 */ 223 __le64 reserved3; /* 120-127 */ 224 }; 225 226 227 struct nv_adma_port_priv { 228 struct nv_adma_cpb *cpb; 229 dma_addr_t cpb_dma; 230 struct nv_adma_prd *aprd; 231 dma_addr_t aprd_dma; 232 void __iomem *ctl_block; 233 void __iomem *gen_block; 234 void __iomem *notifier_clear_block; 235 u64 adma_dma_mask; 236 u8 flags; 237 int last_issue_ncq; 238 }; 239 240 struct nv_host_priv { 241 unsigned long type; 242 }; 243 244 struct defer_queue { 245 u32 defer_bits; 246 unsigned int head; 247 unsigned int tail; 248 unsigned int tag[ATA_MAX_QUEUE]; 249 }; 250 251 enum ncq_saw_flag_list { 252 ncq_saw_d2h = (1U << 0), 253 ncq_saw_dmas = (1U << 1), 254 ncq_saw_sdb = (1U << 2), 255 ncq_saw_backout = (1U << 3), 256 }; 257 258 struct nv_swncq_port_priv { 259 struct ata_bmdma_prd *prd; /* our SG list */ 260 dma_addr_t prd_dma; /* and its DMA mapping */ 261 void __iomem *sactive_block; 262 void __iomem *irq_block; 263 void __iomem *tag_block; 264 u32 qc_active; 265 266 unsigned int last_issue_tag; 267 268 /* fifo circular queue to store deferral command */ 269 struct defer_queue defer_queue; 270 271 /* for NCQ interrupt analysis */ 272 u32 dhfis_bits; 273 u32 dmafis_bits; 274 u32 sdbfis_bits; 275 276 unsigned int ncq_flags; 277 }; 278 279 280 #define NV_ADMA_CHECK_INTR(GCTL, PORT) ((GCTL) & (1 << (19 + (12 * (PORT))))) 281 282 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent); 283 #ifdef CONFIG_PM_SLEEP 284 static int nv_pci_device_resume(struct pci_dev *pdev); 285 #endif 286 static void nv_ck804_host_stop(struct ata_host *host); 287 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance); 288 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance); 289 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance); 290 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val); 291 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val); 292 293 static int nv_hardreset(struct ata_link *link, unsigned int *class, 294 unsigned long deadline); 295 static void nv_nf2_freeze(struct ata_port *ap); 296 static void nv_nf2_thaw(struct ata_port *ap); 297 static void nv_ck804_freeze(struct ata_port *ap); 298 static void nv_ck804_thaw(struct ata_port *ap); 299 static int nv_adma_sdev_configure(struct scsi_device *sdev, 300 struct queue_limits *lim); 301 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc); 302 static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc); 303 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc); 304 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance); 305 static void nv_adma_irq_clear(struct ata_port *ap); 306 static int nv_adma_port_start(struct ata_port *ap); 307 static void nv_adma_port_stop(struct ata_port *ap); 308 #ifdef CONFIG_PM 309 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg); 310 static int nv_adma_port_resume(struct ata_port *ap); 311 #endif 312 static void nv_adma_freeze(struct ata_port *ap); 313 static void nv_adma_thaw(struct ata_port *ap); 314 static void nv_adma_error_handler(struct ata_port *ap); 315 static void nv_adma_host_stop(struct ata_host *host); 316 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc); 317 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf); 318 319 static void nv_mcp55_thaw(struct ata_port *ap); 320 static void nv_mcp55_freeze(struct ata_port *ap); 321 static void nv_swncq_error_handler(struct ata_port *ap); 322 static int nv_swncq_sdev_configure(struct scsi_device *sdev, 323 struct queue_limits *lim); 324 static int nv_swncq_port_start(struct ata_port *ap); 325 static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc); 326 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc); 327 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc); 328 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis); 329 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance); 330 #ifdef CONFIG_PM 331 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg); 332 static int nv_swncq_port_resume(struct ata_port *ap); 333 #endif 334 335 enum nv_host_type 336 { 337 GENERIC, 338 NFORCE2, 339 NFORCE3 = NFORCE2, /* NF2 == NF3 as far as sata_nv is concerned */ 340 CK804, 341 ADMA, 342 MCP5x, 343 SWNCQ, 344 }; 345 346 static const struct pci_device_id nv_pci_tbl[] = { 347 { 348 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE2S_SATA), 349 .driver_data = NFORCE2, 350 }, { 351 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA), 352 .driver_data = NFORCE3, 353 }, { 354 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE3S_SATA2), 355 .driver_data = NFORCE3, 356 }, { 357 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA), 358 .driver_data = CK804, 359 }, { 360 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_CK804_SATA2), 361 .driver_data = CK804, 362 }, { 363 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA), 364 .driver_data = CK804, 365 }, { 366 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP04_SATA2), 367 .driver_data = CK804, 368 }, { 369 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA), 370 .driver_data = MCP5x, 371 }, { 372 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2), 373 .driver_data = MCP5x, 374 }, { 375 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA), 376 .driver_data = MCP5x, 377 }, { 378 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2), 379 .driver_data = MCP5x, 380 }, { 381 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA), 382 .driver_data = GENERIC, 383 }, { 384 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA2), 385 .driver_data = GENERIC, 386 }, { 387 PCI_VDEVICE(NVIDIA, PCI_DEVICE_ID_NVIDIA_NFORCE_MCP61_SATA3), 388 .driver_data = GENERIC, 389 }, 390 { } /* terminate list */ 391 }; 392 393 static struct pci_driver nv_pci_driver = { 394 .name = DRV_NAME, 395 .id_table = nv_pci_tbl, 396 .probe = nv_init_one, 397 #ifdef CONFIG_PM_SLEEP 398 .suspend = ata_pci_device_suspend, 399 .resume = nv_pci_device_resume, 400 #endif 401 .remove = ata_pci_remove_one, 402 }; 403 404 static const struct scsi_host_template nv_sht = { 405 ATA_BMDMA_SHT(DRV_NAME), 406 }; 407 408 static const struct scsi_host_template nv_adma_sht = { 409 __ATA_BASE_SHT(DRV_NAME), 410 .can_queue = NV_ADMA_MAX_CPBS, 411 .sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN, 412 .dma_boundary = NV_ADMA_DMA_BOUNDARY, 413 .sdev_configure = nv_adma_sdev_configure, 414 .sdev_groups = ata_ncq_sdev_groups, 415 .change_queue_depth = ata_scsi_change_queue_depth, 416 .tag_alloc_policy_rr = true, 417 }; 418 419 static const struct scsi_host_template nv_swncq_sht = { 420 __ATA_BASE_SHT(DRV_NAME), 421 .can_queue = ATA_MAX_QUEUE - 1, 422 .sg_tablesize = LIBATA_MAX_PRD, 423 .dma_boundary = ATA_DMA_BOUNDARY, 424 .sdev_configure = nv_swncq_sdev_configure, 425 .sdev_groups = ata_ncq_sdev_groups, 426 .change_queue_depth = ata_scsi_change_queue_depth, 427 .tag_alloc_policy_rr = true, 428 }; 429 430 /* 431 * NV SATA controllers have various different problems with hardreset 432 * protocol depending on the specific controller and device. 433 * 434 * GENERIC: 435 * 436 * bko11195 reports that link doesn't come online after hardreset on 437 * generic nv's and there have been several other similar reports on 438 * linux-ide. 439 * 440 * bko12351#c23 reports that warmplug on MCP61 doesn't work with 441 * softreset. 442 * 443 * NF2/3: 444 * 445 * bko3352 reports nf2/3 controllers can't determine device signature 446 * reliably after hardreset. The following thread reports detection 447 * failure on cold boot with the standard debouncing timing. 448 * 449 * http://thread.gmane.org/gmane.linux.ide/34098 450 * 451 * bko12176 reports that hardreset fails to bring up the link during 452 * boot on nf2. 453 * 454 * CK804: 455 * 456 * For initial probing after boot and hot plugging, hardreset mostly 457 * works fine on CK804 but curiously, reprobing on the initial port 458 * by rescanning or rmmod/insmod fails to acquire the initial D2H Reg 459 * FIS in somewhat undeterministic way. 460 * 461 * SWNCQ: 462 * 463 * bko12351 reports that when SWNCQ is enabled, for hotplug to work, 464 * hardreset should be used and hardreset can't report proper 465 * signature, which suggests that mcp5x is closer to nf2 as long as 466 * reset quirkiness is concerned. 467 * 468 * bko12703 reports that boot probing fails for intel SSD with 469 * hardreset. Link fails to come online. Softreset works fine. 470 * 471 * The failures are varied but the following patterns seem true for 472 * all flavors. 473 * 474 * - Softreset during boot always works. 475 * 476 * - Hardreset during boot sometimes fails to bring up the link on 477 * certain comibnations and device signature acquisition is 478 * unreliable. 479 * 480 * - Hardreset is often necessary after hotplug. 481 * 482 * So, preferring softreset for boot probing and error handling (as 483 * hardreset might bring down the link) but using hardreset for 484 * post-boot probing should work around the above issues in most 485 * cases. Define nv_hardreset() which only kicks in for post-boot 486 * probing and use it for all variants. 487 */ 488 static struct ata_port_operations nv_generic_ops = { 489 .inherits = &ata_bmdma_port_ops, 490 .lost_interrupt = ATA_OP_NULL, 491 .scr_read = nv_scr_read, 492 .scr_write = nv_scr_write, 493 .reset.hardreset = nv_hardreset, 494 }; 495 496 static struct ata_port_operations nv_nf2_ops = { 497 .inherits = &nv_generic_ops, 498 .freeze = nv_nf2_freeze, 499 .thaw = nv_nf2_thaw, 500 }; 501 502 static struct ata_port_operations nv_ck804_ops = { 503 .inherits = &nv_generic_ops, 504 .freeze = nv_ck804_freeze, 505 .thaw = nv_ck804_thaw, 506 .host_stop = nv_ck804_host_stop, 507 }; 508 509 static struct ata_port_operations nv_adma_ops = { 510 .inherits = &nv_ck804_ops, 511 512 .check_atapi_dma = nv_adma_check_atapi_dma, 513 .sff_tf_read = nv_adma_tf_read, 514 .qc_defer = ata_std_qc_defer, 515 .qc_prep = nv_adma_qc_prep, 516 .qc_issue = nv_adma_qc_issue, 517 .sff_irq_clear = nv_adma_irq_clear, 518 519 .freeze = nv_adma_freeze, 520 .thaw = nv_adma_thaw, 521 .error_handler = nv_adma_error_handler, 522 .post_internal_cmd = nv_adma_post_internal_cmd, 523 524 .port_start = nv_adma_port_start, 525 .port_stop = nv_adma_port_stop, 526 #ifdef CONFIG_PM 527 .port_suspend = nv_adma_port_suspend, 528 .port_resume = nv_adma_port_resume, 529 #endif 530 .host_stop = nv_adma_host_stop, 531 }; 532 533 static struct ata_port_operations nv_swncq_ops = { 534 .inherits = &nv_generic_ops, 535 536 .qc_defer = ata_std_qc_defer, 537 .qc_prep = nv_swncq_qc_prep, 538 .qc_issue = nv_swncq_qc_issue, 539 540 .freeze = nv_mcp55_freeze, 541 .thaw = nv_mcp55_thaw, 542 .error_handler = nv_swncq_error_handler, 543 544 #ifdef CONFIG_PM 545 .port_suspend = nv_swncq_port_suspend, 546 .port_resume = nv_swncq_port_resume, 547 #endif 548 .port_start = nv_swncq_port_start, 549 }; 550 551 struct nv_pi_priv { 552 irq_handler_t irq_handler; 553 const struct scsi_host_template *sht; 554 }; 555 556 #define NV_PI_PRIV(_irq_handler, _sht) \ 557 &(struct nv_pi_priv){ .irq_handler = _irq_handler, .sht = _sht } 558 559 static const struct ata_port_info nv_port_info[] = { 560 /* generic */ 561 { 562 .flags = ATA_FLAG_SATA, 563 .pio_mask = NV_PIO_MASK, 564 .mwdma_mask = NV_MWDMA_MASK, 565 .udma_mask = NV_UDMA_MASK, 566 .port_ops = &nv_generic_ops, 567 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht), 568 }, 569 /* nforce2/3 */ 570 { 571 .flags = ATA_FLAG_SATA, 572 .pio_mask = NV_PIO_MASK, 573 .mwdma_mask = NV_MWDMA_MASK, 574 .udma_mask = NV_UDMA_MASK, 575 .port_ops = &nv_nf2_ops, 576 .private_data = NV_PI_PRIV(nv_nf2_interrupt, &nv_sht), 577 }, 578 /* ck804 */ 579 { 580 .flags = ATA_FLAG_SATA, 581 .pio_mask = NV_PIO_MASK, 582 .mwdma_mask = NV_MWDMA_MASK, 583 .udma_mask = NV_UDMA_MASK, 584 .port_ops = &nv_ck804_ops, 585 .private_data = NV_PI_PRIV(nv_ck804_interrupt, &nv_sht), 586 }, 587 /* ADMA */ 588 { 589 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ, 590 .pio_mask = NV_PIO_MASK, 591 .mwdma_mask = NV_MWDMA_MASK, 592 .udma_mask = NV_UDMA_MASK, 593 .port_ops = &nv_adma_ops, 594 .private_data = NV_PI_PRIV(nv_adma_interrupt, &nv_adma_sht), 595 }, 596 /* MCP5x */ 597 { 598 .flags = ATA_FLAG_SATA, 599 .pio_mask = NV_PIO_MASK, 600 .mwdma_mask = NV_MWDMA_MASK, 601 .udma_mask = NV_UDMA_MASK, 602 .port_ops = &nv_generic_ops, 603 .private_data = NV_PI_PRIV(nv_generic_interrupt, &nv_sht), 604 }, 605 /* SWNCQ */ 606 { 607 .flags = ATA_FLAG_SATA | ATA_FLAG_NCQ, 608 .pio_mask = NV_PIO_MASK, 609 .mwdma_mask = NV_MWDMA_MASK, 610 .udma_mask = NV_UDMA_MASK, 611 .port_ops = &nv_swncq_ops, 612 .private_data = NV_PI_PRIV(nv_swncq_interrupt, &nv_swncq_sht), 613 }, 614 }; 615 616 MODULE_AUTHOR("NVIDIA"); 617 MODULE_DESCRIPTION("low-level driver for NVIDIA nForce SATA controller"); 618 MODULE_LICENSE("GPL"); 619 MODULE_DEVICE_TABLE(pci, nv_pci_tbl); 620 MODULE_VERSION(DRV_VERSION); 621 622 static bool adma_enabled; 623 static bool swncq_enabled = true; 624 static bool msi_enabled; 625 626 static void nv_adma_register_mode(struct ata_port *ap) 627 { 628 struct nv_adma_port_priv *pp = ap->private_data; 629 void __iomem *mmio = pp->ctl_block; 630 u16 tmp, status; 631 int count = 0; 632 633 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) 634 return; 635 636 status = readw(mmio + NV_ADMA_STAT); 637 while (!(status & NV_ADMA_STAT_IDLE) && count < 20) { 638 ndelay(50); 639 status = readw(mmio + NV_ADMA_STAT); 640 count++; 641 } 642 if (count == 20) 643 ata_port_warn(ap, "timeout waiting for ADMA IDLE, stat=0x%hx\n", 644 status); 645 646 tmp = readw(mmio + NV_ADMA_CTL); 647 writew(tmp & ~NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); 648 649 count = 0; 650 status = readw(mmio + NV_ADMA_STAT); 651 while (!(status & NV_ADMA_STAT_LEGACY) && count < 20) { 652 ndelay(50); 653 status = readw(mmio + NV_ADMA_STAT); 654 count++; 655 } 656 if (count == 20) 657 ata_port_warn(ap, 658 "timeout waiting for ADMA LEGACY, stat=0x%hx\n", 659 status); 660 661 pp->flags |= NV_ADMA_PORT_REGISTER_MODE; 662 } 663 664 static void nv_adma_mode(struct ata_port *ap) 665 { 666 struct nv_adma_port_priv *pp = ap->private_data; 667 void __iomem *mmio = pp->ctl_block; 668 u16 tmp, status; 669 int count = 0; 670 671 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) 672 return; 673 674 WARN_ON(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE); 675 676 tmp = readw(mmio + NV_ADMA_CTL); 677 writew(tmp | NV_ADMA_CTL_GO, mmio + NV_ADMA_CTL); 678 679 status = readw(mmio + NV_ADMA_STAT); 680 while (((status & NV_ADMA_STAT_LEGACY) || 681 !(status & NV_ADMA_STAT_IDLE)) && count < 20) { 682 ndelay(50); 683 status = readw(mmio + NV_ADMA_STAT); 684 count++; 685 } 686 if (count == 20) 687 ata_port_warn(ap, 688 "timeout waiting for ADMA LEGACY clear and IDLE, stat=0x%hx\n", 689 status); 690 691 pp->flags &= ~NV_ADMA_PORT_REGISTER_MODE; 692 } 693 694 static int nv_adma_sdev_configure(struct scsi_device *sdev, 695 struct queue_limits *lim) 696 { 697 struct ata_port *ap = ata_shost_to_port(sdev->host); 698 struct nv_adma_port_priv *pp = ap->private_data; 699 struct nv_adma_port_priv *port0, *port1; 700 struct pci_dev *pdev = to_pci_dev(ap->host->dev); 701 unsigned long segment_boundary, flags; 702 unsigned short sg_tablesize; 703 int rc; 704 int adma_enable; 705 u32 current_reg, new_reg, config_mask; 706 707 rc = ata_scsi_sdev_configure(sdev, lim); 708 709 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun) 710 /* Not a proper libata device, ignore */ 711 return rc; 712 713 spin_lock_irqsave(ap->lock, flags); 714 715 if (ap->link.device[sdev->id].class == ATA_DEV_ATAPI) { 716 /* 717 * NVIDIA reports that ADMA mode does not support ATAPI commands. 718 * Therefore ATAPI commands are sent through the legacy interface. 719 * However, the legacy interface only supports 32-bit DMA. 720 * Restrict DMA parameters as required by the legacy interface 721 * when an ATAPI device is connected. 722 */ 723 segment_boundary = ATA_DMA_BOUNDARY; 724 /* Subtract 1 since an extra entry may be needed for padding, see 725 libata-scsi.c */ 726 sg_tablesize = LIBATA_MAX_PRD - 1; 727 728 /* Since the legacy DMA engine is in use, we need to disable ADMA 729 on the port. */ 730 adma_enable = 0; 731 nv_adma_register_mode(ap); 732 } else { 733 segment_boundary = NV_ADMA_DMA_BOUNDARY; 734 sg_tablesize = NV_ADMA_SGTBL_TOTAL_LEN; 735 adma_enable = 1; 736 } 737 738 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, ¤t_reg); 739 740 if (ap->port_no == 1) 741 config_mask = NV_MCP_SATA_CFG_20_PORT1_EN | 742 NV_MCP_SATA_CFG_20_PORT1_PWB_EN; 743 else 744 config_mask = NV_MCP_SATA_CFG_20_PORT0_EN | 745 NV_MCP_SATA_CFG_20_PORT0_PWB_EN; 746 747 if (adma_enable) { 748 new_reg = current_reg | config_mask; 749 pp->flags &= ~NV_ADMA_ATAPI_SETUP_COMPLETE; 750 } else { 751 new_reg = current_reg & ~config_mask; 752 pp->flags |= NV_ADMA_ATAPI_SETUP_COMPLETE; 753 } 754 755 if (current_reg != new_reg) 756 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, new_reg); 757 758 port0 = ap->host->ports[0]->private_data; 759 port1 = ap->host->ports[1]->private_data; 760 if ((port0->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) || 761 (port1->flags & NV_ADMA_ATAPI_SETUP_COMPLETE)) { 762 /* 763 * We have to set the DMA mask to 32-bit if either port is in 764 * ATAPI mode, since they are on the same PCI device which is 765 * used for DMA mapping. If either SCSI device is not allocated 766 * yet, it's OK since that port will discover its correct 767 * setting when it does get allocated. 768 */ 769 rc = dma_set_mask(&pdev->dev, ATA_DMA_MASK); 770 } else { 771 rc = dma_set_mask(&pdev->dev, pp->adma_dma_mask); 772 } 773 774 lim->seg_boundary_mask = segment_boundary; 775 lim->max_segments = sg_tablesize; 776 ata_port_info(ap, 777 "DMA mask 0x%llX, segment boundary 0x%lX, hw segs %hu\n", 778 (unsigned long long)*ap->host->dev->dma_mask, 779 segment_boundary, sg_tablesize); 780 781 spin_unlock_irqrestore(ap->lock, flags); 782 783 return rc; 784 } 785 786 static int nv_adma_check_atapi_dma(struct ata_queued_cmd *qc) 787 { 788 struct nv_adma_port_priv *pp = qc->ap->private_data; 789 return !(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE); 790 } 791 792 static void nv_adma_tf_read(struct ata_port *ap, struct ata_taskfile *tf) 793 { 794 /* Other than when internal or pass-through commands are executed, 795 the only time this function will be called in ADMA mode will be 796 if a command fails. In the failure case we don't care about going 797 into register mode with ADMA commands pending, as the commands will 798 all shortly be aborted anyway. We assume that NCQ commands are not 799 issued via passthrough, which is the only way that switching into 800 ADMA mode could abort outstanding commands. */ 801 nv_adma_register_mode(ap); 802 803 ata_sff_tf_read(ap, tf); 804 } 805 806 static unsigned int nv_adma_tf_to_cpb(struct ata_taskfile *tf, __le16 *cpb) 807 { 808 unsigned int idx = 0; 809 810 if (tf->flags & ATA_TFLAG_ISADDR) { 811 if (tf->flags & ATA_TFLAG_LBA48) { 812 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->hob_feature | WNB); 813 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->hob_nsect); 814 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->hob_lbal); 815 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->hob_lbam); 816 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->hob_lbah); 817 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature); 818 } else 819 cpb[idx++] = cpu_to_le16((ATA_REG_ERR << 8) | tf->feature | WNB); 820 821 cpb[idx++] = cpu_to_le16((ATA_REG_NSECT << 8) | tf->nsect); 822 cpb[idx++] = cpu_to_le16((ATA_REG_LBAL << 8) | tf->lbal); 823 cpb[idx++] = cpu_to_le16((ATA_REG_LBAM << 8) | tf->lbam); 824 cpb[idx++] = cpu_to_le16((ATA_REG_LBAH << 8) | tf->lbah); 825 } 826 827 if (tf->flags & ATA_TFLAG_DEVICE) 828 cpb[idx++] = cpu_to_le16((ATA_REG_DEVICE << 8) | tf->device); 829 830 cpb[idx++] = cpu_to_le16((ATA_REG_CMD << 8) | tf->command | CMDEND); 831 832 while (idx < 12) 833 cpb[idx++] = cpu_to_le16(IGN); 834 835 return idx; 836 } 837 838 static int nv_adma_check_cpb(struct ata_port *ap, int cpb_num, int force_err) 839 { 840 struct nv_adma_port_priv *pp = ap->private_data; 841 u8 flags = pp->cpb[cpb_num].resp_flags; 842 843 ata_port_dbg(ap, "CPB %d, flags=0x%x\n", cpb_num, flags); 844 845 if (unlikely((force_err || 846 flags & (NV_CPB_RESP_ATA_ERR | 847 NV_CPB_RESP_CMD_ERR | 848 NV_CPB_RESP_CPB_ERR)))) { 849 struct ata_eh_info *ehi = &ap->link.eh_info; 850 int freeze = 0; 851 852 ata_ehi_clear_desc(ehi); 853 __ata_ehi_push_desc(ehi, "CPB resp_flags 0x%x: ", flags); 854 if (flags & NV_CPB_RESP_ATA_ERR) { 855 ata_ehi_push_desc(ehi, "ATA error"); 856 ehi->err_mask |= AC_ERR_DEV; 857 } else if (flags & NV_CPB_RESP_CMD_ERR) { 858 ata_ehi_push_desc(ehi, "CMD error"); 859 ehi->err_mask |= AC_ERR_DEV; 860 } else if (flags & NV_CPB_RESP_CPB_ERR) { 861 ata_ehi_push_desc(ehi, "CPB error"); 862 ehi->err_mask |= AC_ERR_SYSTEM; 863 freeze = 1; 864 } else { 865 /* notifier error, but no error in CPB flags? */ 866 ata_ehi_push_desc(ehi, "unknown"); 867 ehi->err_mask |= AC_ERR_OTHER; 868 freeze = 1; 869 } 870 /* Kill all commands. EH will determine what actually failed. */ 871 if (freeze) 872 ata_port_freeze(ap); 873 else 874 ata_port_abort(ap); 875 return -1; 876 } 877 878 if (likely(flags & NV_CPB_RESP_DONE)) 879 return 1; 880 return 0; 881 } 882 883 static int nv_host_intr(struct ata_port *ap, u8 irq_stat) 884 { 885 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag); 886 887 /* freeze if hotplugged */ 888 if (unlikely(irq_stat & (NV_INT_ADDED | NV_INT_REMOVED))) { 889 ata_port_freeze(ap); 890 return 1; 891 } 892 893 /* bail out if not our interrupt */ 894 if (!(irq_stat & NV_INT_DEV)) 895 return 0; 896 897 /* DEV interrupt w/ no active qc? */ 898 if (unlikely(!qc || (qc->tf.flags & ATA_TFLAG_POLLING))) { 899 ata_sff_check_status(ap); 900 return 1; 901 } 902 903 /* handle interrupt */ 904 return ata_bmdma_port_intr(ap, qc); 905 } 906 907 static irqreturn_t nv_adma_interrupt(int irq, void *dev_instance) 908 { 909 struct ata_host *host = dev_instance; 910 int i, handled = 0; 911 u32 notifier_clears[2]; 912 913 spin_lock(&host->lock); 914 915 for (i = 0; i < host->n_ports; i++) { 916 struct ata_port *ap = host->ports[i]; 917 struct nv_adma_port_priv *pp = ap->private_data; 918 void __iomem *mmio = pp->ctl_block; 919 u16 status; 920 u32 gen_ctl; 921 u32 notifier, notifier_error; 922 923 notifier_clears[i] = 0; 924 925 /* if ADMA is disabled, use standard ata interrupt handler */ 926 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) { 927 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804) 928 >> (NV_INT_PORT_SHIFT * i); 929 handled += nv_host_intr(ap, irq_stat); 930 continue; 931 } 932 933 /* if in ATA register mode, check for standard interrupts */ 934 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) { 935 u8 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804) 936 >> (NV_INT_PORT_SHIFT * i); 937 if (ata_tag_valid(ap->link.active_tag)) 938 /** NV_INT_DEV indication seems unreliable 939 at times at least in ADMA mode. Force it 940 on always when a command is active, to 941 prevent losing interrupts. */ 942 irq_stat |= NV_INT_DEV; 943 handled += nv_host_intr(ap, irq_stat); 944 } 945 946 notifier = readl(mmio + NV_ADMA_NOTIFIER); 947 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); 948 notifier_clears[i] = notifier | notifier_error; 949 950 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL); 951 952 if (!NV_ADMA_CHECK_INTR(gen_ctl, ap->port_no) && !notifier && 953 !notifier_error) 954 /* Nothing to do */ 955 continue; 956 957 status = readw(mmio + NV_ADMA_STAT); 958 959 /* 960 * Clear status. Ensure the controller sees the 961 * clearing before we start looking at any of the CPB 962 * statuses, so that any CPB completions after this 963 * point in the handler will raise another interrupt. 964 */ 965 writew(status, mmio + NV_ADMA_STAT); 966 readw(mmio + NV_ADMA_STAT); /* flush posted write */ 967 rmb(); 968 969 handled++; /* irq handled if we got here */ 970 971 /* freeze if hotplugged or controller error */ 972 if (unlikely(status & (NV_ADMA_STAT_HOTPLUG | 973 NV_ADMA_STAT_HOTUNPLUG | 974 NV_ADMA_STAT_TIMEOUT | 975 NV_ADMA_STAT_SERROR))) { 976 struct ata_eh_info *ehi = &ap->link.eh_info; 977 978 ata_ehi_clear_desc(ehi); 979 __ata_ehi_push_desc(ehi, "ADMA status 0x%08x: ", status); 980 if (status & NV_ADMA_STAT_TIMEOUT) { 981 ehi->err_mask |= AC_ERR_SYSTEM; 982 ata_ehi_push_desc(ehi, "timeout"); 983 } else if (status & NV_ADMA_STAT_HOTPLUG) { 984 ata_ehi_hotplugged(ehi); 985 ata_ehi_push_desc(ehi, "hotplug"); 986 } else if (status & NV_ADMA_STAT_HOTUNPLUG) { 987 ata_ehi_hotplugged(ehi); 988 ata_ehi_push_desc(ehi, "hot unplug"); 989 } else if (status & NV_ADMA_STAT_SERROR) { 990 /* let EH analyze SError and figure out cause */ 991 ata_ehi_push_desc(ehi, "SError"); 992 } else 993 ata_ehi_push_desc(ehi, "unknown"); 994 ata_port_freeze(ap); 995 continue; 996 } 997 998 if (status & (NV_ADMA_STAT_DONE | 999 NV_ADMA_STAT_CPBERR | 1000 NV_ADMA_STAT_CMD_COMPLETE)) { 1001 u32 check_commands = notifier_clears[i]; 1002 u32 done_mask = 0; 1003 int pos, rc; 1004 1005 if (status & NV_ADMA_STAT_CPBERR) { 1006 /* check all active commands */ 1007 if (ata_tag_valid(ap->link.active_tag)) 1008 check_commands = 1 << 1009 ap->link.active_tag; 1010 else 1011 check_commands = ap->link.sactive; 1012 } 1013 1014 /* check CPBs for completed commands */ 1015 while ((pos = ffs(check_commands))) { 1016 pos--; 1017 rc = nv_adma_check_cpb(ap, pos, 1018 notifier_error & (1 << pos)); 1019 if (rc > 0) 1020 done_mask |= 1 << pos; 1021 else if (unlikely(rc < 0)) 1022 check_commands = 0; 1023 check_commands &= ~(1 << pos); 1024 } 1025 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask); 1026 } 1027 } 1028 1029 if (notifier_clears[0] || notifier_clears[1]) { 1030 /* Note: Both notifier clear registers must be written 1031 if either is set, even if one is zero, according to NVIDIA. */ 1032 struct nv_adma_port_priv *pp = host->ports[0]->private_data; 1033 writel(notifier_clears[0], pp->notifier_clear_block); 1034 pp = host->ports[1]->private_data; 1035 writel(notifier_clears[1], pp->notifier_clear_block); 1036 } 1037 1038 spin_unlock(&host->lock); 1039 1040 return IRQ_RETVAL(handled); 1041 } 1042 1043 static void nv_adma_freeze(struct ata_port *ap) 1044 { 1045 struct nv_adma_port_priv *pp = ap->private_data; 1046 void __iomem *mmio = pp->ctl_block; 1047 u16 tmp; 1048 1049 nv_ck804_freeze(ap); 1050 1051 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 1052 return; 1053 1054 /* clear any outstanding CK804 notifications */ 1055 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT), 1056 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804); 1057 1058 /* Disable interrupt */ 1059 tmp = readw(mmio + NV_ADMA_CTL); 1060 writew(tmp & ~(NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN), 1061 mmio + NV_ADMA_CTL); 1062 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1063 } 1064 1065 static void nv_adma_thaw(struct ata_port *ap) 1066 { 1067 struct nv_adma_port_priv *pp = ap->private_data; 1068 void __iomem *mmio = pp->ctl_block; 1069 u16 tmp; 1070 1071 nv_ck804_thaw(ap); 1072 1073 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 1074 return; 1075 1076 /* Enable interrupt */ 1077 tmp = readw(mmio + NV_ADMA_CTL); 1078 writew(tmp | (NV_ADMA_CTL_AIEN | NV_ADMA_CTL_HOTPLUG_IEN), 1079 mmio + NV_ADMA_CTL); 1080 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1081 } 1082 1083 static void nv_adma_irq_clear(struct ata_port *ap) 1084 { 1085 struct nv_adma_port_priv *pp = ap->private_data; 1086 void __iomem *mmio = pp->ctl_block; 1087 u32 notifier_clears[2]; 1088 1089 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) { 1090 ata_bmdma_irq_clear(ap); 1091 return; 1092 } 1093 1094 /* clear any outstanding CK804 notifications */ 1095 writeb(NV_INT_ALL << (ap->port_no * NV_INT_PORT_SHIFT), 1096 ap->host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804); 1097 1098 /* clear ADMA status */ 1099 writew(0xffff, mmio + NV_ADMA_STAT); 1100 1101 /* clear notifiers - note both ports need to be written with 1102 something even though we are only clearing on one */ 1103 if (ap->port_no == 0) { 1104 notifier_clears[0] = 0xFFFFFFFF; 1105 notifier_clears[1] = 0; 1106 } else { 1107 notifier_clears[0] = 0; 1108 notifier_clears[1] = 0xFFFFFFFF; 1109 } 1110 pp = ap->host->ports[0]->private_data; 1111 writel(notifier_clears[0], pp->notifier_clear_block); 1112 pp = ap->host->ports[1]->private_data; 1113 writel(notifier_clears[1], pp->notifier_clear_block); 1114 } 1115 1116 static void nv_adma_post_internal_cmd(struct ata_queued_cmd *qc) 1117 { 1118 struct nv_adma_port_priv *pp = qc->ap->private_data; 1119 1120 if (pp->flags & NV_ADMA_PORT_REGISTER_MODE) 1121 ata_bmdma_post_internal_cmd(qc); 1122 } 1123 1124 static int nv_adma_port_start(struct ata_port *ap) 1125 { 1126 struct device *dev = ap->host->dev; 1127 struct nv_adma_port_priv *pp; 1128 int rc; 1129 void *mem; 1130 dma_addr_t mem_dma; 1131 void __iomem *mmio; 1132 struct pci_dev *pdev = to_pci_dev(dev); 1133 u16 tmp; 1134 1135 /* 1136 * Ensure DMA mask is set to 32-bit before allocating legacy PRD and 1137 * pad buffers. 1138 */ 1139 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1140 if (rc) 1141 return rc; 1142 1143 /* we might fallback to bmdma, allocate bmdma resources */ 1144 rc = ata_bmdma_port_start(ap); 1145 if (rc) 1146 return rc; 1147 1148 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1149 if (!pp) 1150 return -ENOMEM; 1151 1152 mmio = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_PORT + 1153 ap->port_no * NV_ADMA_PORT_SIZE; 1154 pp->ctl_block = mmio; 1155 pp->gen_block = ap->host->iomap[NV_MMIO_BAR] + NV_ADMA_GEN; 1156 pp->notifier_clear_block = pp->gen_block + 1157 NV_ADMA_NOTIFIER_CLEAR + (4 * ap->port_no); 1158 1159 /* 1160 * Now that the legacy PRD and padding buffer are allocated we can 1161 * raise the DMA mask to allocate the CPB/APRD table. 1162 */ 1163 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1164 1165 pp->adma_dma_mask = *dev->dma_mask; 1166 1167 mem = dmam_alloc_coherent(dev, NV_ADMA_PORT_PRIV_DMA_SZ, 1168 &mem_dma, GFP_KERNEL); 1169 if (!mem) 1170 return -ENOMEM; 1171 1172 /* 1173 * First item in chunk of DMA memory: 1174 * 128-byte command parameter block (CPB) 1175 * one for each command tag 1176 */ 1177 pp->cpb = mem; 1178 pp->cpb_dma = mem_dma; 1179 1180 writel(mem_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW); 1181 writel((mem_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH); 1182 1183 mem += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ; 1184 mem_dma += NV_ADMA_MAX_CPBS * NV_ADMA_CPB_SZ; 1185 1186 /* 1187 * Second item: block of ADMA_SGTBL_LEN s/g entries 1188 */ 1189 pp->aprd = mem; 1190 pp->aprd_dma = mem_dma; 1191 1192 ap->private_data = pp; 1193 1194 /* clear any outstanding interrupt conditions */ 1195 writew(0xffff, mmio + NV_ADMA_STAT); 1196 1197 /* initialize port variables */ 1198 pp->flags = NV_ADMA_PORT_REGISTER_MODE; 1199 1200 /* clear CPB fetch count */ 1201 writew(0, mmio + NV_ADMA_CPB_COUNT); 1202 1203 /* clear GO for register mode, enable interrupt */ 1204 tmp = readw(mmio + NV_ADMA_CTL); 1205 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN | 1206 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL); 1207 1208 tmp = readw(mmio + NV_ADMA_CTL); 1209 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1210 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1211 udelay(1); 1212 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1213 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1214 1215 return 0; 1216 } 1217 1218 static void nv_adma_port_stop(struct ata_port *ap) 1219 { 1220 struct nv_adma_port_priv *pp = ap->private_data; 1221 void __iomem *mmio = pp->ctl_block; 1222 1223 writew(0, mmio + NV_ADMA_CTL); 1224 } 1225 1226 #ifdef CONFIG_PM 1227 static int nv_adma_port_suspend(struct ata_port *ap, pm_message_t mesg) 1228 { 1229 struct nv_adma_port_priv *pp = ap->private_data; 1230 void __iomem *mmio = pp->ctl_block; 1231 1232 /* Go to register mode - clears GO */ 1233 nv_adma_register_mode(ap); 1234 1235 /* clear CPB fetch count */ 1236 writew(0, mmio + NV_ADMA_CPB_COUNT); 1237 1238 /* disable interrupt, shut down port */ 1239 writew(0, mmio + NV_ADMA_CTL); 1240 1241 return 0; 1242 } 1243 1244 static int nv_adma_port_resume(struct ata_port *ap) 1245 { 1246 struct nv_adma_port_priv *pp = ap->private_data; 1247 void __iomem *mmio = pp->ctl_block; 1248 u16 tmp; 1249 1250 /* set CPB block location */ 1251 writel(pp->cpb_dma & 0xFFFFFFFF, mmio + NV_ADMA_CPB_BASE_LOW); 1252 writel((pp->cpb_dma >> 16) >> 16, mmio + NV_ADMA_CPB_BASE_HIGH); 1253 1254 /* clear any outstanding interrupt conditions */ 1255 writew(0xffff, mmio + NV_ADMA_STAT); 1256 1257 /* initialize port variables */ 1258 pp->flags |= NV_ADMA_PORT_REGISTER_MODE; 1259 1260 /* clear CPB fetch count */ 1261 writew(0, mmio + NV_ADMA_CPB_COUNT); 1262 1263 /* clear GO for register mode, enable interrupt */ 1264 tmp = readw(mmio + NV_ADMA_CTL); 1265 writew((tmp & ~NV_ADMA_CTL_GO) | NV_ADMA_CTL_AIEN | 1266 NV_ADMA_CTL_HOTPLUG_IEN, mmio + NV_ADMA_CTL); 1267 1268 tmp = readw(mmio + NV_ADMA_CTL); 1269 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1270 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1271 udelay(1); 1272 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1273 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1274 1275 return 0; 1276 } 1277 #endif 1278 1279 static void nv_adma_setup_port(struct ata_port *ap) 1280 { 1281 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1282 struct ata_ioports *ioport = &ap->ioaddr; 1283 1284 mmio += NV_ADMA_PORT + ap->port_no * NV_ADMA_PORT_SIZE; 1285 1286 ioport->cmd_addr = mmio; 1287 ioport->data_addr = mmio + (ATA_REG_DATA * 4); 1288 ioport->error_addr = 1289 ioport->feature_addr = mmio + (ATA_REG_ERR * 4); 1290 ioport->nsect_addr = mmio + (ATA_REG_NSECT * 4); 1291 ioport->lbal_addr = mmio + (ATA_REG_LBAL * 4); 1292 ioport->lbam_addr = mmio + (ATA_REG_LBAM * 4); 1293 ioport->lbah_addr = mmio + (ATA_REG_LBAH * 4); 1294 ioport->device_addr = mmio + (ATA_REG_DEVICE * 4); 1295 ioport->status_addr = 1296 ioport->command_addr = mmio + (ATA_REG_STATUS * 4); 1297 ioport->altstatus_addr = 1298 ioport->ctl_addr = mmio + 0x20; 1299 } 1300 1301 static int nv_adma_host_init(struct ata_host *host) 1302 { 1303 struct pci_dev *pdev = to_pci_dev(host->dev); 1304 unsigned int i; 1305 u32 tmp32; 1306 1307 /* enable ADMA on the ports */ 1308 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); 1309 tmp32 |= NV_MCP_SATA_CFG_20_PORT0_EN | 1310 NV_MCP_SATA_CFG_20_PORT0_PWB_EN | 1311 NV_MCP_SATA_CFG_20_PORT1_EN | 1312 NV_MCP_SATA_CFG_20_PORT1_PWB_EN; 1313 1314 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); 1315 1316 for (i = 0; i < host->n_ports; i++) 1317 nv_adma_setup_port(host->ports[i]); 1318 1319 return 0; 1320 } 1321 1322 static void nv_adma_fill_aprd(struct ata_queued_cmd *qc, 1323 struct scatterlist *sg, 1324 int idx, 1325 struct nv_adma_prd *aprd) 1326 { 1327 u8 flags = 0; 1328 if (qc->tf.flags & ATA_TFLAG_WRITE) 1329 flags |= NV_APRD_WRITE; 1330 if (idx == qc->n_elem - 1) 1331 flags |= NV_APRD_END; 1332 else if (idx != 4) 1333 flags |= NV_APRD_CONT; 1334 1335 aprd->addr = cpu_to_le64(((u64)sg_dma_address(sg))); 1336 aprd->len = cpu_to_le32(((u32)sg_dma_len(sg))); /* len in bytes */ 1337 aprd->flags = flags; 1338 aprd->packet_len = 0; 1339 } 1340 1341 static void nv_adma_fill_sg(struct ata_queued_cmd *qc, struct nv_adma_cpb *cpb) 1342 { 1343 struct nv_adma_port_priv *pp = qc->ap->private_data; 1344 struct nv_adma_prd *aprd; 1345 struct scatterlist *sg; 1346 unsigned int si; 1347 1348 for_each_sg(qc->sg, sg, qc->n_elem, si) { 1349 aprd = (si < 5) ? &cpb->aprd[si] : 1350 &pp->aprd[NV_ADMA_SGTBL_LEN * qc->hw_tag + (si-5)]; 1351 nv_adma_fill_aprd(qc, sg, si, aprd); 1352 } 1353 if (si > 5) 1354 cpb->next_aprd = cpu_to_le64(((u64)(pp->aprd_dma + NV_ADMA_SGTBL_SZ * qc->hw_tag))); 1355 else 1356 cpb->next_aprd = cpu_to_le64(0); 1357 } 1358 1359 static int nv_adma_use_reg_mode(struct ata_queued_cmd *qc) 1360 { 1361 struct nv_adma_port_priv *pp = qc->ap->private_data; 1362 1363 /* ADMA engine can only be used for non-ATAPI DMA commands, 1364 or interrupt-driven no-data commands. */ 1365 if ((pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) || 1366 (qc->tf.flags & ATA_TFLAG_POLLING)) 1367 return 1; 1368 1369 if ((qc->flags & ATA_QCFLAG_DMAMAP) || 1370 (qc->tf.protocol == ATA_PROT_NODATA)) 1371 return 0; 1372 1373 return 1; 1374 } 1375 1376 static enum ata_completion_errors nv_adma_qc_prep(struct ata_queued_cmd *qc) 1377 { 1378 struct nv_adma_port_priv *pp = qc->ap->private_data; 1379 struct nv_adma_cpb *cpb = &pp->cpb[qc->hw_tag]; 1380 u8 ctl_flags = NV_CPB_CTL_CPB_VALID | 1381 NV_CPB_CTL_IEN; 1382 1383 if (nv_adma_use_reg_mode(qc)) { 1384 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) && 1385 (qc->flags & ATA_QCFLAG_DMAMAP)); 1386 nv_adma_register_mode(qc->ap); 1387 ata_bmdma_qc_prep(qc); 1388 return AC_ERR_OK; 1389 } 1390 1391 cpb->resp_flags = NV_CPB_RESP_DONE; 1392 wmb(); 1393 cpb->ctl_flags = 0; 1394 wmb(); 1395 1396 cpb->len = 3; 1397 cpb->tag = qc->hw_tag; 1398 cpb->next_cpb_idx = 0; 1399 1400 /* turn on NCQ flags for NCQ commands */ 1401 if (qc->tf.protocol == ATA_PROT_NCQ) 1402 ctl_flags |= NV_CPB_CTL_QUEUE | NV_CPB_CTL_FPDMA; 1403 1404 nv_adma_tf_to_cpb(&qc->tf, cpb->tf); 1405 1406 if (qc->flags & ATA_QCFLAG_DMAMAP) { 1407 nv_adma_fill_sg(qc, cpb); 1408 ctl_flags |= NV_CPB_CTL_APRD_VALID; 1409 } else 1410 memset(&cpb->aprd[0], 0, sizeof(struct nv_adma_prd) * 5); 1411 1412 /* Be paranoid and don't let the device see NV_CPB_CTL_CPB_VALID 1413 until we are finished filling in all of the contents */ 1414 wmb(); 1415 cpb->ctl_flags = ctl_flags; 1416 wmb(); 1417 cpb->resp_flags = 0; 1418 1419 return AC_ERR_OK; 1420 } 1421 1422 static unsigned int nv_adma_qc_issue(struct ata_queued_cmd *qc) 1423 { 1424 struct nv_adma_port_priv *pp = qc->ap->private_data; 1425 void __iomem *mmio = pp->ctl_block; 1426 int curr_ncq = (qc->tf.protocol == ATA_PROT_NCQ); 1427 1428 /* We can't handle result taskfile with NCQ commands, since 1429 retrieving the taskfile switches us out of ADMA mode and would abort 1430 existing commands. */ 1431 if (unlikely(qc->tf.protocol == ATA_PROT_NCQ && 1432 (qc->flags & ATA_QCFLAG_RESULT_TF))) { 1433 ata_dev_err(qc->dev, "NCQ w/ RESULT_TF not allowed\n"); 1434 return AC_ERR_SYSTEM; 1435 } 1436 1437 if (nv_adma_use_reg_mode(qc)) { 1438 /* use ATA register mode */ 1439 BUG_ON(!(pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) && 1440 (qc->flags & ATA_QCFLAG_DMAMAP)); 1441 nv_adma_register_mode(qc->ap); 1442 return ata_bmdma_qc_issue(qc); 1443 } else 1444 nv_adma_mode(qc->ap); 1445 1446 /* write append register, command tag in lower 8 bits 1447 and (number of cpbs to append -1) in top 8 bits */ 1448 wmb(); 1449 1450 if (curr_ncq != pp->last_issue_ncq) { 1451 /* Seems to need some delay before switching between NCQ and 1452 non-NCQ commands, else we get command timeouts and such. */ 1453 udelay(20); 1454 pp->last_issue_ncq = curr_ncq; 1455 } 1456 1457 writew(qc->hw_tag, mmio + NV_ADMA_APPEND); 1458 1459 return 0; 1460 } 1461 1462 static irqreturn_t nv_generic_interrupt(int irq, void *dev_instance) 1463 { 1464 struct ata_host *host = dev_instance; 1465 unsigned int i; 1466 unsigned int handled = 0; 1467 unsigned long flags; 1468 1469 spin_lock_irqsave(&host->lock, flags); 1470 1471 for (i = 0; i < host->n_ports; i++) { 1472 struct ata_port *ap = host->ports[i]; 1473 struct ata_queued_cmd *qc; 1474 1475 qc = ata_qc_from_tag(ap, ap->link.active_tag); 1476 if (qc && (!(qc->tf.flags & ATA_TFLAG_POLLING))) { 1477 handled += ata_bmdma_port_intr(ap, qc); 1478 } else { 1479 /* 1480 * No request pending? Clear interrupt status 1481 * anyway, in case there's one pending. 1482 */ 1483 ap->ops->sff_check_status(ap); 1484 } 1485 } 1486 1487 spin_unlock_irqrestore(&host->lock, flags); 1488 1489 return IRQ_RETVAL(handled); 1490 } 1491 1492 static irqreturn_t nv_do_interrupt(struct ata_host *host, u8 irq_stat) 1493 { 1494 int i, handled = 0; 1495 1496 for (i = 0; i < host->n_ports; i++) { 1497 handled += nv_host_intr(host->ports[i], irq_stat); 1498 irq_stat >>= NV_INT_PORT_SHIFT; 1499 } 1500 1501 return IRQ_RETVAL(handled); 1502 } 1503 1504 static irqreturn_t nv_nf2_interrupt(int irq, void *dev_instance) 1505 { 1506 struct ata_host *host = dev_instance; 1507 u8 irq_stat; 1508 irqreturn_t ret; 1509 1510 spin_lock(&host->lock); 1511 irq_stat = ioread8(host->ports[0]->ioaddr.scr_addr + NV_INT_STATUS); 1512 ret = nv_do_interrupt(host, irq_stat); 1513 spin_unlock(&host->lock); 1514 1515 return ret; 1516 } 1517 1518 static irqreturn_t nv_ck804_interrupt(int irq, void *dev_instance) 1519 { 1520 struct ata_host *host = dev_instance; 1521 u8 irq_stat; 1522 irqreturn_t ret; 1523 1524 spin_lock(&host->lock); 1525 irq_stat = readb(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_CK804); 1526 ret = nv_do_interrupt(host, irq_stat); 1527 spin_unlock(&host->lock); 1528 1529 return ret; 1530 } 1531 1532 static int nv_scr_read(struct ata_link *link, unsigned int sc_reg, u32 *val) 1533 { 1534 if (sc_reg > SCR_CONTROL) 1535 return -EINVAL; 1536 1537 *val = ioread32(link->ap->ioaddr.scr_addr + (sc_reg * 4)); 1538 return 0; 1539 } 1540 1541 static int nv_scr_write(struct ata_link *link, unsigned int sc_reg, u32 val) 1542 { 1543 if (sc_reg > SCR_CONTROL) 1544 return -EINVAL; 1545 1546 iowrite32(val, link->ap->ioaddr.scr_addr + (sc_reg * 4)); 1547 return 0; 1548 } 1549 1550 static int nv_hardreset(struct ata_link *link, unsigned int *class, 1551 unsigned long deadline) 1552 { 1553 struct ata_eh_context *ehc = &link->eh_context; 1554 1555 /* Do hardreset iff it's post-boot probing, please read the 1556 * comment above port ops for details. 1557 */ 1558 if (!(link->ap->pflags & ATA_PFLAG_LOADING) && 1559 !ata_dev_enabled(link->device)) 1560 sata_link_hardreset(link, sata_deb_timing_hotplug, deadline, 1561 NULL, NULL); 1562 else { 1563 const unsigned int *timing = sata_ehc_deb_timing(ehc); 1564 int rc; 1565 1566 if (!(ehc->i.flags & ATA_EHI_QUIET)) 1567 ata_link_info(link, 1568 "nv: skipping hardreset on occupied port\n"); 1569 1570 /* make sure the link is online */ 1571 rc = sata_link_resume(link, timing, deadline); 1572 /* whine about phy resume failure but proceed */ 1573 if (rc && rc != -EOPNOTSUPP) 1574 ata_link_warn(link, "failed to resume link (errno=%d)\n", 1575 rc); 1576 } 1577 1578 /* device signature acquisition is unreliable */ 1579 return -EAGAIN; 1580 } 1581 1582 static void nv_nf2_freeze(struct ata_port *ap) 1583 { 1584 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr; 1585 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1586 u8 mask; 1587 1588 mask = ioread8(scr_addr + NV_INT_ENABLE); 1589 mask &= ~(NV_INT_ALL << shift); 1590 iowrite8(mask, scr_addr + NV_INT_ENABLE); 1591 } 1592 1593 static void nv_nf2_thaw(struct ata_port *ap) 1594 { 1595 void __iomem *scr_addr = ap->host->ports[0]->ioaddr.scr_addr; 1596 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1597 u8 mask; 1598 1599 iowrite8(NV_INT_ALL << shift, scr_addr + NV_INT_STATUS); 1600 1601 mask = ioread8(scr_addr + NV_INT_ENABLE); 1602 mask |= (NV_INT_MASK << shift); 1603 iowrite8(mask, scr_addr + NV_INT_ENABLE); 1604 } 1605 1606 static void nv_ck804_freeze(struct ata_port *ap) 1607 { 1608 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1609 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1610 u8 mask; 1611 1612 mask = readb(mmio_base + NV_INT_ENABLE_CK804); 1613 mask &= ~(NV_INT_ALL << shift); 1614 writeb(mask, mmio_base + NV_INT_ENABLE_CK804); 1615 } 1616 1617 static void nv_ck804_thaw(struct ata_port *ap) 1618 { 1619 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1620 int shift = ap->port_no * NV_INT_PORT_SHIFT; 1621 u8 mask; 1622 1623 writeb(NV_INT_ALL << shift, mmio_base + NV_INT_STATUS_CK804); 1624 1625 mask = readb(mmio_base + NV_INT_ENABLE_CK804); 1626 mask |= (NV_INT_MASK << shift); 1627 writeb(mask, mmio_base + NV_INT_ENABLE_CK804); 1628 } 1629 1630 static void nv_mcp55_freeze(struct ata_port *ap) 1631 { 1632 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1633 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55; 1634 u32 mask; 1635 1636 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55); 1637 1638 mask = readl(mmio_base + NV_INT_ENABLE_MCP55); 1639 mask &= ~(NV_INT_ALL_MCP55 << shift); 1640 writel(mask, mmio_base + NV_INT_ENABLE_MCP55); 1641 } 1642 1643 static void nv_mcp55_thaw(struct ata_port *ap) 1644 { 1645 void __iomem *mmio_base = ap->host->iomap[NV_MMIO_BAR]; 1646 int shift = ap->port_no * NV_INT_PORT_SHIFT_MCP55; 1647 u32 mask; 1648 1649 writel(NV_INT_ALL_MCP55 << shift, mmio_base + NV_INT_STATUS_MCP55); 1650 1651 mask = readl(mmio_base + NV_INT_ENABLE_MCP55); 1652 mask |= (NV_INT_MASK_MCP55 << shift); 1653 writel(mask, mmio_base + NV_INT_ENABLE_MCP55); 1654 } 1655 1656 static void nv_adma_error_handler(struct ata_port *ap) 1657 __must_hold(&ap->host->eh_mutex) 1658 { 1659 struct nv_adma_port_priv *pp = ap->private_data; 1660 if (!(pp->flags & NV_ADMA_PORT_REGISTER_MODE)) { 1661 void __iomem *mmio = pp->ctl_block; 1662 int i; 1663 u16 tmp; 1664 1665 if (ata_tag_valid(ap->link.active_tag) || ap->link.sactive) { 1666 u32 notifier = readl(mmio + NV_ADMA_NOTIFIER); 1667 u32 notifier_error = readl(mmio + NV_ADMA_NOTIFIER_ERROR); 1668 u32 gen_ctl = readl(pp->gen_block + NV_ADMA_GEN_CTL); 1669 u32 status = readw(mmio + NV_ADMA_STAT); 1670 u8 cpb_count = readb(mmio + NV_ADMA_CPB_COUNT); 1671 u8 next_cpb_idx = readb(mmio + NV_ADMA_NEXT_CPB_IDX); 1672 1673 ata_port_err(ap, 1674 "EH in ADMA mode, notifier 0x%X " 1675 "notifier_error 0x%X gen_ctl 0x%X status 0x%X " 1676 "next cpb count 0x%X next cpb idx 0x%x\n", 1677 notifier, notifier_error, gen_ctl, status, 1678 cpb_count, next_cpb_idx); 1679 1680 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) { 1681 struct nv_adma_cpb *cpb = &pp->cpb[i]; 1682 if ((ata_tag_valid(ap->link.active_tag) && i == ap->link.active_tag) || 1683 ap->link.sactive & (1 << i)) 1684 ata_port_err(ap, 1685 "CPB %d: ctl_flags 0x%x, resp_flags 0x%x\n", 1686 i, cpb->ctl_flags, cpb->resp_flags); 1687 } 1688 } 1689 1690 /* Push us back into port register mode for error handling. */ 1691 nv_adma_register_mode(ap); 1692 1693 /* Mark all of the CPBs as invalid to prevent them from 1694 being executed */ 1695 for (i = 0; i < NV_ADMA_MAX_CPBS; i++) 1696 pp->cpb[i].ctl_flags &= ~NV_CPB_CTL_CPB_VALID; 1697 1698 /* clear CPB fetch count */ 1699 writew(0, mmio + NV_ADMA_CPB_COUNT); 1700 1701 /* Reset channel */ 1702 tmp = readw(mmio + NV_ADMA_CTL); 1703 writew(tmp | NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1704 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1705 udelay(1); 1706 writew(tmp & ~NV_ADMA_CTL_CHANNEL_RESET, mmio + NV_ADMA_CTL); 1707 readw(mmio + NV_ADMA_CTL); /* flush posted write */ 1708 } 1709 1710 ata_bmdma_error_handler(ap); 1711 } 1712 1713 static void nv_swncq_qc_to_dq(struct ata_port *ap, struct ata_queued_cmd *qc) 1714 { 1715 struct nv_swncq_port_priv *pp = ap->private_data; 1716 struct defer_queue *dq = &pp->defer_queue; 1717 1718 /* queue is full */ 1719 WARN_ON(dq->tail - dq->head == ATA_MAX_QUEUE); 1720 dq->defer_bits |= (1 << qc->hw_tag); 1721 dq->tag[dq->tail++ & (ATA_MAX_QUEUE - 1)] = qc->hw_tag; 1722 } 1723 1724 static struct ata_queued_cmd *nv_swncq_qc_from_dq(struct ata_port *ap) 1725 { 1726 struct nv_swncq_port_priv *pp = ap->private_data; 1727 struct defer_queue *dq = &pp->defer_queue; 1728 unsigned int tag; 1729 1730 if (dq->head == dq->tail) /* null queue */ 1731 return NULL; 1732 1733 tag = dq->tag[dq->head & (ATA_MAX_QUEUE - 1)]; 1734 dq->tag[dq->head++ & (ATA_MAX_QUEUE - 1)] = ATA_TAG_POISON; 1735 WARN_ON(!(dq->defer_bits & (1 << tag))); 1736 dq->defer_bits &= ~(1 << tag); 1737 1738 return ata_qc_from_tag(ap, tag); 1739 } 1740 1741 static void nv_swncq_fis_reinit(struct ata_port *ap) 1742 { 1743 struct nv_swncq_port_priv *pp = ap->private_data; 1744 1745 pp->dhfis_bits = 0; 1746 pp->dmafis_bits = 0; 1747 pp->sdbfis_bits = 0; 1748 pp->ncq_flags = 0; 1749 } 1750 1751 static void nv_swncq_pp_reinit(struct ata_port *ap) 1752 { 1753 struct nv_swncq_port_priv *pp = ap->private_data; 1754 struct defer_queue *dq = &pp->defer_queue; 1755 1756 dq->head = 0; 1757 dq->tail = 0; 1758 dq->defer_bits = 0; 1759 pp->qc_active = 0; 1760 pp->last_issue_tag = ATA_TAG_POISON; 1761 nv_swncq_fis_reinit(ap); 1762 } 1763 1764 static void nv_swncq_irq_clear(struct ata_port *ap, u16 fis) 1765 { 1766 struct nv_swncq_port_priv *pp = ap->private_data; 1767 1768 writew(fis, pp->irq_block); 1769 } 1770 1771 static void __ata_bmdma_stop(struct ata_port *ap) 1772 { 1773 struct ata_queued_cmd qc; 1774 1775 qc.ap = ap; 1776 ata_bmdma_stop(&qc); 1777 } 1778 1779 static void nv_swncq_ncq_stop(struct ata_port *ap) 1780 { 1781 struct nv_swncq_port_priv *pp = ap->private_data; 1782 unsigned int i; 1783 u32 sactive; 1784 u32 done_mask; 1785 1786 ata_port_err(ap, "EH in SWNCQ mode,QC:qc_active 0x%llX sactive 0x%X\n", 1787 ap->qc_active, ap->link.sactive); 1788 ata_port_err(ap, 1789 "SWNCQ:qc_active 0x%X defer_bits 0x%X last_issue_tag 0x%x\n " 1790 "dhfis 0x%X dmafis 0x%X sdbfis 0x%X\n", 1791 pp->qc_active, pp->defer_queue.defer_bits, pp->last_issue_tag, 1792 pp->dhfis_bits, pp->dmafis_bits, pp->sdbfis_bits); 1793 1794 ata_port_err(ap, "ATA_REG 0x%X ERR_REG 0x%X\n", 1795 ap->ops->sff_check_status(ap), 1796 ioread8(ap->ioaddr.error_addr)); 1797 1798 sactive = readl(pp->sactive_block); 1799 done_mask = pp->qc_active ^ sactive; 1800 1801 ata_port_err(ap, "tag : dhfis dmafis sdbfis sactive\n"); 1802 for (i = 0; i < ATA_MAX_QUEUE; i++) { 1803 u8 err = 0; 1804 if (pp->qc_active & (1 << i)) 1805 err = 0; 1806 else if (done_mask & (1 << i)) 1807 err = 1; 1808 else 1809 continue; 1810 1811 ata_port_err(ap, 1812 "tag 0x%x: %01x %01x %01x %01x %s\n", i, 1813 (pp->dhfis_bits >> i) & 0x1, 1814 (pp->dmafis_bits >> i) & 0x1, 1815 (pp->sdbfis_bits >> i) & 0x1, 1816 (sactive >> i) & 0x1, 1817 (err ? "error! tag doesn't exit" : " ")); 1818 } 1819 1820 nv_swncq_pp_reinit(ap); 1821 ap->ops->sff_irq_clear(ap); 1822 __ata_bmdma_stop(ap); 1823 nv_swncq_irq_clear(ap, 0xffff); 1824 } 1825 1826 static void nv_swncq_error_handler(struct ata_port *ap) 1827 __must_hold(&ap->host->eh_mutex) 1828 { 1829 struct ata_eh_context *ehc = &ap->link.eh_context; 1830 1831 if (ap->link.sactive) { 1832 nv_swncq_ncq_stop(ap); 1833 ehc->i.action |= ATA_EH_RESET; 1834 } 1835 1836 ata_bmdma_error_handler(ap); 1837 } 1838 1839 #ifdef CONFIG_PM 1840 static int nv_swncq_port_suspend(struct ata_port *ap, pm_message_t mesg) 1841 { 1842 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1843 u32 tmp; 1844 1845 /* clear irq */ 1846 writel(~0, mmio + NV_INT_STATUS_MCP55); 1847 1848 /* disable irq */ 1849 writel(0, mmio + NV_INT_ENABLE_MCP55); 1850 1851 /* disable swncq */ 1852 tmp = readl(mmio + NV_CTL_MCP55); 1853 tmp &= ~(NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ); 1854 writel(tmp, mmio + NV_CTL_MCP55); 1855 1856 return 0; 1857 } 1858 1859 static int nv_swncq_port_resume(struct ata_port *ap) 1860 { 1861 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1862 u32 tmp; 1863 1864 /* clear irq */ 1865 writel(~0, mmio + NV_INT_STATUS_MCP55); 1866 1867 /* enable irq */ 1868 writel(0x00fd00fd, mmio + NV_INT_ENABLE_MCP55); 1869 1870 /* enable swncq */ 1871 tmp = readl(mmio + NV_CTL_MCP55); 1872 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55); 1873 1874 return 0; 1875 } 1876 #endif 1877 1878 static void nv_swncq_host_init(struct ata_host *host) 1879 { 1880 u32 tmp; 1881 void __iomem *mmio = host->iomap[NV_MMIO_BAR]; 1882 struct pci_dev *pdev = to_pci_dev(host->dev); 1883 u8 regval; 1884 1885 /* disable ECO 398 */ 1886 pci_read_config_byte(pdev, 0x7f, ®val); 1887 regval &= ~(1 << 7); 1888 pci_write_config_byte(pdev, 0x7f, regval); 1889 1890 /* enable swncq */ 1891 tmp = readl(mmio + NV_CTL_MCP55); 1892 dev_dbg(&pdev->dev, "HOST_CTL:0x%X\n", tmp); 1893 writel(tmp | NV_CTL_PRI_SWNCQ | NV_CTL_SEC_SWNCQ, mmio + NV_CTL_MCP55); 1894 1895 /* enable irq intr */ 1896 tmp = readl(mmio + NV_INT_ENABLE_MCP55); 1897 dev_dbg(&pdev->dev, "HOST_ENABLE:0x%X\n", tmp); 1898 writel(tmp | 0x00fd00fd, mmio + NV_INT_ENABLE_MCP55); 1899 1900 /* clear port irq */ 1901 writel(~0x0, mmio + NV_INT_STATUS_MCP55); 1902 } 1903 1904 static int nv_swncq_sdev_configure(struct scsi_device *sdev, 1905 struct queue_limits *lim) 1906 { 1907 struct ata_port *ap = ata_shost_to_port(sdev->host); 1908 struct pci_dev *pdev = to_pci_dev(ap->host->dev); 1909 struct ata_device *dev; 1910 int rc; 1911 u8 rev; 1912 u8 check_maxtor = 0; 1913 unsigned char model_num[ATA_ID_PROD_LEN + 1]; 1914 1915 rc = ata_scsi_sdev_configure(sdev, lim); 1916 if (sdev->id >= ATA_MAX_DEVICES || sdev->channel || sdev->lun) 1917 /* Not a proper libata device, ignore */ 1918 return rc; 1919 1920 dev = &ap->link.device[sdev->id]; 1921 if (!(ap->flags & ATA_FLAG_NCQ) || dev->class == ATA_DEV_ATAPI) 1922 return rc; 1923 1924 /* if MCP51 and Maxtor, then disable ncq */ 1925 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA || 1926 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP51_SATA2) 1927 check_maxtor = 1; 1928 1929 /* if MCP55 and rev <= a2 and Maxtor, then disable ncq */ 1930 if (pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA || 1931 pdev->device == PCI_DEVICE_ID_NVIDIA_NFORCE_MCP55_SATA2) { 1932 pci_read_config_byte(pdev, 0x8, &rev); 1933 if (rev <= 0xa2) 1934 check_maxtor = 1; 1935 } 1936 1937 if (!check_maxtor) 1938 return rc; 1939 1940 ata_id_c_string(dev->id, model_num, ATA_ID_PROD, sizeof(model_num)); 1941 1942 if (strncmp(model_num, "Maxtor", 6) == 0) { 1943 ata_scsi_change_queue_depth(sdev, 1); 1944 ata_dev_notice(dev, "Disabling SWNCQ mode (depth %x)\n", 1945 sdev->queue_depth); 1946 } 1947 1948 return rc; 1949 } 1950 1951 static int nv_swncq_port_start(struct ata_port *ap) 1952 { 1953 struct device *dev = ap->host->dev; 1954 void __iomem *mmio = ap->host->iomap[NV_MMIO_BAR]; 1955 struct nv_swncq_port_priv *pp; 1956 int rc; 1957 1958 /* we might fallback to bmdma, allocate bmdma resources */ 1959 rc = ata_bmdma_port_start(ap); 1960 if (rc) 1961 return rc; 1962 1963 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1964 if (!pp) 1965 return -ENOMEM; 1966 1967 pp->prd = dmam_alloc_coherent(dev, ATA_PRD_TBL_SZ * ATA_MAX_QUEUE, 1968 &pp->prd_dma, GFP_KERNEL); 1969 if (!pp->prd) 1970 return -ENOMEM; 1971 1972 ap->private_data = pp; 1973 pp->sactive_block = ap->ioaddr.scr_addr + 4 * SCR_ACTIVE; 1974 pp->irq_block = mmio + NV_INT_STATUS_MCP55 + ap->port_no * 2; 1975 pp->tag_block = mmio + NV_NCQ_REG_MCP55 + ap->port_no * 2; 1976 1977 return 0; 1978 } 1979 1980 static enum ata_completion_errors nv_swncq_qc_prep(struct ata_queued_cmd *qc) 1981 { 1982 if (qc->tf.protocol != ATA_PROT_NCQ) { 1983 ata_bmdma_qc_prep(qc); 1984 return AC_ERR_OK; 1985 } 1986 1987 if (!(qc->flags & ATA_QCFLAG_DMAMAP)) 1988 return AC_ERR_OK; 1989 1990 nv_swncq_fill_sg(qc); 1991 1992 return AC_ERR_OK; 1993 } 1994 1995 static void nv_swncq_fill_sg(struct ata_queued_cmd *qc) 1996 { 1997 struct ata_port *ap = qc->ap; 1998 struct scatterlist *sg; 1999 struct nv_swncq_port_priv *pp = ap->private_data; 2000 struct ata_bmdma_prd *prd; 2001 unsigned int si, idx; 2002 2003 prd = pp->prd + ATA_MAX_PRD * qc->hw_tag; 2004 2005 idx = 0; 2006 for_each_sg(qc->sg, sg, qc->n_elem, si) { 2007 u32 addr, offset; 2008 u32 sg_len, len; 2009 2010 addr = (u32)sg_dma_address(sg); 2011 sg_len = sg_dma_len(sg); 2012 2013 while (sg_len) { 2014 offset = addr & 0xffff; 2015 len = sg_len; 2016 if ((offset + sg_len) > 0x10000) 2017 len = 0x10000 - offset; 2018 2019 prd[idx].addr = cpu_to_le32(addr); 2020 prd[idx].flags_len = cpu_to_le32(len & 0xffff); 2021 2022 idx++; 2023 sg_len -= len; 2024 addr += len; 2025 } 2026 } 2027 2028 prd[idx - 1].flags_len |= cpu_to_le32(ATA_PRD_EOT); 2029 } 2030 2031 static unsigned int nv_swncq_issue_atacmd(struct ata_port *ap, 2032 struct ata_queued_cmd *qc) 2033 { 2034 struct nv_swncq_port_priv *pp = ap->private_data; 2035 2036 if (qc == NULL) 2037 return 0; 2038 2039 writel((1 << qc->hw_tag), pp->sactive_block); 2040 pp->last_issue_tag = qc->hw_tag; 2041 pp->dhfis_bits &= ~(1 << qc->hw_tag); 2042 pp->dmafis_bits &= ~(1 << qc->hw_tag); 2043 pp->qc_active |= (0x1 << qc->hw_tag); 2044 2045 trace_ata_tf_load(ap, &qc->tf); 2046 ap->ops->sff_tf_load(ap, &qc->tf); /* load tf registers */ 2047 trace_ata_exec_command(ap, &qc->tf, qc->hw_tag); 2048 ap->ops->sff_exec_command(ap, &qc->tf); 2049 2050 return 0; 2051 } 2052 2053 static unsigned int nv_swncq_qc_issue(struct ata_queued_cmd *qc) 2054 { 2055 struct ata_port *ap = qc->ap; 2056 struct nv_swncq_port_priv *pp = ap->private_data; 2057 2058 if (qc->tf.protocol != ATA_PROT_NCQ) 2059 return ata_bmdma_qc_issue(qc); 2060 2061 if (!pp->qc_active) 2062 nv_swncq_issue_atacmd(ap, qc); 2063 else 2064 nv_swncq_qc_to_dq(ap, qc); /* add qc to defer queue */ 2065 2066 return 0; 2067 } 2068 2069 static void nv_swncq_hotplug(struct ata_port *ap, u32 fis) 2070 { 2071 u32 serror; 2072 struct ata_eh_info *ehi = &ap->link.eh_info; 2073 2074 ata_ehi_clear_desc(ehi); 2075 2076 /* AHCI needs SError cleared; otherwise, it might lock up */ 2077 sata_scr_read(&ap->link, SCR_ERROR, &serror); 2078 sata_scr_write(&ap->link, SCR_ERROR, serror); 2079 2080 /* analyze @irq_stat */ 2081 if (fis & NV_SWNCQ_IRQ_ADDED) 2082 ata_ehi_push_desc(ehi, "hot plug"); 2083 else if (fis & NV_SWNCQ_IRQ_REMOVED) 2084 ata_ehi_push_desc(ehi, "hot unplug"); 2085 2086 ata_ehi_hotplugged(ehi); 2087 2088 /* okay, let's hand over to EH */ 2089 ehi->serror |= serror; 2090 2091 ata_port_freeze(ap); 2092 } 2093 2094 static int nv_swncq_sdbfis(struct ata_port *ap) 2095 { 2096 struct ata_queued_cmd *qc; 2097 struct nv_swncq_port_priv *pp = ap->private_data; 2098 struct ata_eh_info *ehi = &ap->link.eh_info; 2099 u32 sactive; 2100 u32 done_mask; 2101 u8 host_stat; 2102 u8 lack_dhfis = 0; 2103 2104 host_stat = ap->ops->bmdma_status(ap); 2105 trace_ata_bmdma_status(ap, host_stat); 2106 if (unlikely(host_stat & ATA_DMA_ERR)) { 2107 /* error when transferring data to/from memory */ 2108 ata_ehi_clear_desc(ehi); 2109 ata_ehi_push_desc(ehi, "BMDMA stat 0x%x", host_stat); 2110 ehi->err_mask |= AC_ERR_HOST_BUS; 2111 ehi->action |= ATA_EH_RESET; 2112 return -EINVAL; 2113 } 2114 2115 ap->ops->sff_irq_clear(ap); 2116 __ata_bmdma_stop(ap); 2117 2118 sactive = readl(pp->sactive_block); 2119 done_mask = pp->qc_active ^ sactive; 2120 2121 pp->qc_active &= ~done_mask; 2122 pp->dhfis_bits &= ~done_mask; 2123 pp->dmafis_bits &= ~done_mask; 2124 pp->sdbfis_bits |= done_mask; 2125 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask); 2126 2127 if (!ap->qc_active) { 2128 ata_port_dbg(ap, "over\n"); 2129 nv_swncq_pp_reinit(ap); 2130 return 0; 2131 } 2132 2133 if (pp->qc_active & pp->dhfis_bits) 2134 return 0; 2135 2136 if ((pp->ncq_flags & ncq_saw_backout) || 2137 (pp->qc_active ^ pp->dhfis_bits)) 2138 /* if the controller can't get a device to host register FIS, 2139 * The driver needs to reissue the new command. 2140 */ 2141 lack_dhfis = 1; 2142 2143 ata_port_dbg(ap, "QC: qc_active 0x%llx," 2144 "SWNCQ:qc_active 0x%X defer_bits %X " 2145 "dhfis 0x%X dmafis 0x%X last_issue_tag %x\n", 2146 ap->qc_active, pp->qc_active, 2147 pp->defer_queue.defer_bits, pp->dhfis_bits, 2148 pp->dmafis_bits, pp->last_issue_tag); 2149 2150 nv_swncq_fis_reinit(ap); 2151 2152 if (lack_dhfis) { 2153 qc = ata_qc_from_tag(ap, pp->last_issue_tag); 2154 nv_swncq_issue_atacmd(ap, qc); 2155 return 0; 2156 } 2157 2158 if (pp->defer_queue.defer_bits) { 2159 /* send deferral queue command */ 2160 qc = nv_swncq_qc_from_dq(ap); 2161 WARN_ON(qc == NULL); 2162 nv_swncq_issue_atacmd(ap, qc); 2163 } 2164 2165 return 0; 2166 } 2167 2168 static inline u32 nv_swncq_tag(struct ata_port *ap) 2169 { 2170 struct nv_swncq_port_priv *pp = ap->private_data; 2171 u32 tag; 2172 2173 tag = readb(pp->tag_block) >> 2; 2174 return (tag & 0x1f); 2175 } 2176 2177 static void nv_swncq_dmafis(struct ata_port *ap) 2178 { 2179 struct ata_queued_cmd *qc; 2180 unsigned int rw; 2181 u8 dmactl; 2182 u32 tag; 2183 struct nv_swncq_port_priv *pp = ap->private_data; 2184 2185 __ata_bmdma_stop(ap); 2186 tag = nv_swncq_tag(ap); 2187 2188 ata_port_dbg(ap, "dma setup tag 0x%x\n", tag); 2189 qc = ata_qc_from_tag(ap, tag); 2190 2191 if (unlikely(!qc)) 2192 return; 2193 2194 rw = qc->tf.flags & ATA_TFLAG_WRITE; 2195 2196 /* load PRD table addr. */ 2197 iowrite32(pp->prd_dma + ATA_PRD_TBL_SZ * qc->hw_tag, 2198 ap->ioaddr.bmdma_addr + ATA_DMA_TABLE_OFS); 2199 2200 /* specify data direction, triple-check start bit is clear */ 2201 dmactl = ioread8(ap->ioaddr.bmdma_addr + ATA_DMA_CMD); 2202 dmactl &= ~ATA_DMA_WR; 2203 if (!rw) 2204 dmactl |= ATA_DMA_WR; 2205 2206 iowrite8(dmactl | ATA_DMA_START, ap->ioaddr.bmdma_addr + ATA_DMA_CMD); 2207 } 2208 2209 static void nv_swncq_host_interrupt(struct ata_port *ap, u16 fis) 2210 { 2211 struct nv_swncq_port_priv *pp = ap->private_data; 2212 struct ata_queued_cmd *qc; 2213 struct ata_eh_info *ehi = &ap->link.eh_info; 2214 u32 serror; 2215 u8 ata_stat; 2216 2217 ata_stat = ap->ops->sff_check_status(ap); 2218 nv_swncq_irq_clear(ap, fis); 2219 if (!fis) 2220 return; 2221 2222 if (ata_port_is_frozen(ap)) 2223 return; 2224 2225 if (fis & NV_SWNCQ_IRQ_HOTPLUG) { 2226 nv_swncq_hotplug(ap, fis); 2227 return; 2228 } 2229 2230 if (!pp->qc_active) 2231 return; 2232 2233 if (ap->ops->scr_read(&ap->link, SCR_ERROR, &serror)) 2234 return; 2235 ap->ops->scr_write(&ap->link, SCR_ERROR, serror); 2236 2237 if (ata_stat & ATA_ERR) { 2238 ata_ehi_clear_desc(ehi); 2239 ata_ehi_push_desc(ehi, "Ata error. fis:0x%X", fis); 2240 ehi->err_mask |= AC_ERR_DEV; 2241 ehi->serror |= serror; 2242 ehi->action |= ATA_EH_RESET; 2243 ata_port_freeze(ap); 2244 return; 2245 } 2246 2247 if (fis & NV_SWNCQ_IRQ_BACKOUT) { 2248 /* If the IRQ is backout, driver must issue 2249 * the new command again some time later. 2250 */ 2251 pp->ncq_flags |= ncq_saw_backout; 2252 } 2253 2254 if (fis & NV_SWNCQ_IRQ_SDBFIS) { 2255 pp->ncq_flags |= ncq_saw_sdb; 2256 ata_port_dbg(ap, "SWNCQ: qc_active 0x%X " 2257 "dhfis 0x%X dmafis 0x%X sactive 0x%X\n", 2258 pp->qc_active, pp->dhfis_bits, 2259 pp->dmafis_bits, readl(pp->sactive_block)); 2260 if (nv_swncq_sdbfis(ap) < 0) 2261 goto irq_error; 2262 } 2263 2264 if (fis & NV_SWNCQ_IRQ_DHREGFIS) { 2265 /* The interrupt indicates the new command 2266 * was transmitted correctly to the drive. 2267 */ 2268 pp->dhfis_bits |= (0x1 << pp->last_issue_tag); 2269 pp->ncq_flags |= ncq_saw_d2h; 2270 if (pp->ncq_flags & (ncq_saw_sdb | ncq_saw_backout)) { 2271 ata_ehi_push_desc(ehi, "illegal fis transaction"); 2272 ehi->err_mask |= AC_ERR_HSM; 2273 ehi->action |= ATA_EH_RESET; 2274 goto irq_error; 2275 } 2276 2277 if (!(fis & NV_SWNCQ_IRQ_DMASETUP) && 2278 !(pp->ncq_flags & ncq_saw_dmas)) { 2279 ata_stat = ap->ops->sff_check_status(ap); 2280 if (ata_stat & ATA_BUSY) 2281 goto irq_exit; 2282 2283 if (pp->defer_queue.defer_bits) { 2284 ata_port_dbg(ap, "send next command\n"); 2285 qc = nv_swncq_qc_from_dq(ap); 2286 nv_swncq_issue_atacmd(ap, qc); 2287 } 2288 } 2289 } 2290 2291 if (fis & NV_SWNCQ_IRQ_DMASETUP) { 2292 /* program the dma controller with appropriate PRD buffers 2293 * and start the DMA transfer for requested command. 2294 */ 2295 pp->dmafis_bits |= (0x1 << nv_swncq_tag(ap)); 2296 pp->ncq_flags |= ncq_saw_dmas; 2297 nv_swncq_dmafis(ap); 2298 } 2299 2300 irq_exit: 2301 return; 2302 irq_error: 2303 ata_ehi_push_desc(ehi, "fis:0x%x", fis); 2304 ata_port_freeze(ap); 2305 return; 2306 } 2307 2308 static irqreturn_t nv_swncq_interrupt(int irq, void *dev_instance) 2309 { 2310 struct ata_host *host = dev_instance; 2311 unsigned int i; 2312 unsigned int handled = 0; 2313 unsigned long flags; 2314 u32 irq_stat; 2315 2316 spin_lock_irqsave(&host->lock, flags); 2317 2318 irq_stat = readl(host->iomap[NV_MMIO_BAR] + NV_INT_STATUS_MCP55); 2319 2320 for (i = 0; i < host->n_ports; i++) { 2321 struct ata_port *ap = host->ports[i]; 2322 2323 if (ap->link.sactive) { 2324 nv_swncq_host_interrupt(ap, (u16)irq_stat); 2325 handled = 1; 2326 } else { 2327 if (irq_stat) /* reserve Hotplug */ 2328 nv_swncq_irq_clear(ap, 0xfff0); 2329 2330 handled += nv_host_intr(ap, (u8)irq_stat); 2331 } 2332 irq_stat >>= NV_INT_PORT_SHIFT_MCP55; 2333 } 2334 2335 spin_unlock_irqrestore(&host->lock, flags); 2336 2337 return IRQ_RETVAL(handled); 2338 } 2339 2340 static int nv_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 2341 { 2342 const struct ata_port_info *ppi[] = { NULL, NULL }; 2343 struct nv_pi_priv *ipriv; 2344 struct ata_host *host; 2345 struct nv_host_priv *hpriv; 2346 int rc; 2347 u32 bar; 2348 void __iomem *base; 2349 unsigned long type = ent->driver_data; 2350 2351 // Make sure this is a SATA controller by counting the number of bars 2352 // (NVIDIA SATA controllers will always have six bars). Otherwise, 2353 // it's an IDE controller and we ignore it. 2354 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) 2355 if (pci_resource_start(pdev, bar) == 0) 2356 return -ENODEV; 2357 2358 ata_print_version_once(&pdev->dev, DRV_VERSION); 2359 2360 rc = pcim_enable_device(pdev); 2361 if (rc) 2362 return rc; 2363 2364 /* determine type and allocate host */ 2365 if (type == CK804 && adma_enabled) { 2366 dev_notice(&pdev->dev, "Using ADMA mode\n"); 2367 type = ADMA; 2368 } else if (type == MCP5x && swncq_enabled) { 2369 dev_notice(&pdev->dev, "Using SWNCQ mode\n"); 2370 type = SWNCQ; 2371 } 2372 2373 ppi[0] = &nv_port_info[type]; 2374 ipriv = ppi[0]->private_data; 2375 rc = ata_pci_bmdma_prepare_host(pdev, ppi, &host); 2376 if (rc) 2377 return rc; 2378 2379 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL); 2380 if (!hpriv) 2381 return -ENOMEM; 2382 hpriv->type = type; 2383 host->private_data = hpriv; 2384 2385 /* request and iomap NV_MMIO_BAR */ 2386 rc = pcim_iomap_regions(pdev, 1 << NV_MMIO_BAR, DRV_NAME); 2387 if (rc) 2388 return rc; 2389 2390 /* configure SCR access */ 2391 base = host->iomap[NV_MMIO_BAR]; 2392 host->ports[0]->ioaddr.scr_addr = base + NV_PORT0_SCR_REG_OFFSET; 2393 host->ports[1]->ioaddr.scr_addr = base + NV_PORT1_SCR_REG_OFFSET; 2394 2395 /* enable SATA space for CK804 */ 2396 if (type >= CK804) { 2397 u8 regval; 2398 2399 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); 2400 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN; 2401 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); 2402 } 2403 2404 /* init ADMA */ 2405 if (type == ADMA) { 2406 rc = nv_adma_host_init(host); 2407 if (rc) 2408 return rc; 2409 } else if (type == SWNCQ) 2410 nv_swncq_host_init(host); 2411 2412 if (msi_enabled) { 2413 dev_notice(&pdev->dev, "Using MSI\n"); 2414 pci_enable_msi(pdev); 2415 } 2416 2417 pci_set_master(pdev); 2418 return ata_pci_sff_activate_host(host, ipriv->irq_handler, ipriv->sht); 2419 } 2420 2421 #ifdef CONFIG_PM_SLEEP 2422 static int nv_pci_device_resume(struct pci_dev *pdev) 2423 { 2424 struct ata_host *host = pci_get_drvdata(pdev); 2425 struct nv_host_priv *hpriv = host->private_data; 2426 int rc; 2427 2428 rc = ata_pci_device_do_resume(pdev); 2429 if (rc) 2430 return rc; 2431 2432 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) { 2433 if (hpriv->type >= CK804) { 2434 u8 regval; 2435 2436 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); 2437 regval |= NV_MCP_SATA_CFG_20_SATA_SPACE_EN; 2438 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); 2439 } 2440 if (hpriv->type == ADMA) { 2441 u32 tmp32; 2442 struct nv_adma_port_priv *pp; 2443 /* enable/disable ADMA on the ports appropriately */ 2444 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); 2445 2446 pp = host->ports[0]->private_data; 2447 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 2448 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN | 2449 NV_MCP_SATA_CFG_20_PORT0_PWB_EN); 2450 else 2451 tmp32 |= (NV_MCP_SATA_CFG_20_PORT0_EN | 2452 NV_MCP_SATA_CFG_20_PORT0_PWB_EN); 2453 pp = host->ports[1]->private_data; 2454 if (pp->flags & NV_ADMA_ATAPI_SETUP_COMPLETE) 2455 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT1_EN | 2456 NV_MCP_SATA_CFG_20_PORT1_PWB_EN); 2457 else 2458 tmp32 |= (NV_MCP_SATA_CFG_20_PORT1_EN | 2459 NV_MCP_SATA_CFG_20_PORT1_PWB_EN); 2460 2461 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); 2462 } 2463 } 2464 2465 ata_host_resume(host); 2466 2467 return 0; 2468 } 2469 #endif 2470 2471 static void nv_ck804_host_stop(struct ata_host *host) 2472 { 2473 struct pci_dev *pdev = to_pci_dev(host->dev); 2474 u8 regval; 2475 2476 /* disable SATA space for CK804 */ 2477 pci_read_config_byte(pdev, NV_MCP_SATA_CFG_20, ®val); 2478 regval &= ~NV_MCP_SATA_CFG_20_SATA_SPACE_EN; 2479 pci_write_config_byte(pdev, NV_MCP_SATA_CFG_20, regval); 2480 } 2481 2482 static void nv_adma_host_stop(struct ata_host *host) 2483 { 2484 struct pci_dev *pdev = to_pci_dev(host->dev); 2485 u32 tmp32; 2486 2487 /* disable ADMA on the ports */ 2488 pci_read_config_dword(pdev, NV_MCP_SATA_CFG_20, &tmp32); 2489 tmp32 &= ~(NV_MCP_SATA_CFG_20_PORT0_EN | 2490 NV_MCP_SATA_CFG_20_PORT0_PWB_EN | 2491 NV_MCP_SATA_CFG_20_PORT1_EN | 2492 NV_MCP_SATA_CFG_20_PORT1_PWB_EN); 2493 2494 pci_write_config_dword(pdev, NV_MCP_SATA_CFG_20, tmp32); 2495 2496 nv_ck804_host_stop(host); 2497 } 2498 2499 module_pci_driver(nv_pci_driver); 2500 2501 module_param_named(adma, adma_enabled, bool, 0444); 2502 MODULE_PARM_DESC(adma, "Enable use of ADMA (Default: false)"); 2503 module_param_named(swncq, swncq_enabled, bool, 0444); 2504 MODULE_PARM_DESC(swncq, "Enable use of SWNCQ (Default: true)"); 2505 module_param_named(msi, msi_enabled, bool, 0444); 2506 MODULE_PARM_DESC(msi, "Enable use of MSI (Default: false)"); 2507