1 /* 2 * ahci.c - AHCI SATA support 3 * 4 * Maintained by: Jeff Garzik <jgarzik@pobox.com> 5 * Please ALWAYS copy linux-ide@vger.kernel.org 6 * on emails. 7 * 8 * Copyright 2004-2005 Red Hat, Inc. 9 * 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License as published by 13 * the Free Software Foundation; either version 2, or (at your option) 14 * any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 19 * GNU General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public License 22 * along with this program; see the file COPYING. If not, write to 23 * the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. 24 * 25 * 26 * libata documentation is available via 'make {ps|pdf}docs', 27 * as Documentation/DocBook/libata.* 28 * 29 * AHCI hardware documentation: 30 * http://www.intel.com/technology/serialata/pdf/rev1_0.pdf 31 * http://www.intel.com/technology/serialata/pdf/rev1_1.pdf 32 * 33 */ 34 35 #include <linux/kernel.h> 36 #include <linux/module.h> 37 #include <linux/pci.h> 38 #include <linux/init.h> 39 #include <linux/blkdev.h> 40 #include <linux/delay.h> 41 #include <linux/interrupt.h> 42 #include <linux/dma-mapping.h> 43 #include <linux/device.h> 44 #include <scsi/scsi_host.h> 45 #include <scsi/scsi_cmnd.h> 46 #include <linux/libata.h> 47 48 #define DRV_NAME "ahci" 49 #define DRV_VERSION "2.1" 50 51 52 enum { 53 AHCI_PCI_BAR = 5, 54 AHCI_MAX_PORTS = 32, 55 AHCI_MAX_SG = 168, /* hardware max is 64K */ 56 AHCI_DMA_BOUNDARY = 0xffffffff, 57 AHCI_USE_CLUSTERING = 0, 58 AHCI_MAX_CMDS = 32, 59 AHCI_CMD_SZ = 32, 60 AHCI_CMD_SLOT_SZ = AHCI_MAX_CMDS * AHCI_CMD_SZ, 61 AHCI_RX_FIS_SZ = 256, 62 AHCI_CMD_TBL_CDB = 0x40, 63 AHCI_CMD_TBL_HDR_SZ = 0x80, 64 AHCI_CMD_TBL_SZ = AHCI_CMD_TBL_HDR_SZ + (AHCI_MAX_SG * 16), 65 AHCI_CMD_TBL_AR_SZ = AHCI_CMD_TBL_SZ * AHCI_MAX_CMDS, 66 AHCI_PORT_PRIV_DMA_SZ = AHCI_CMD_SLOT_SZ + AHCI_CMD_TBL_AR_SZ + 67 AHCI_RX_FIS_SZ, 68 AHCI_IRQ_ON_SG = (1 << 31), 69 AHCI_CMD_ATAPI = (1 << 5), 70 AHCI_CMD_WRITE = (1 << 6), 71 AHCI_CMD_PREFETCH = (1 << 7), 72 AHCI_CMD_RESET = (1 << 8), 73 AHCI_CMD_CLR_BUSY = (1 << 10), 74 75 RX_FIS_D2H_REG = 0x40, /* offset of D2H Register FIS data */ 76 RX_FIS_SDB = 0x58, /* offset of SDB FIS data */ 77 RX_FIS_UNK = 0x60, /* offset of Unknown FIS data */ 78 79 board_ahci = 0, 80 board_ahci_pi = 1, 81 board_ahci_vt8251 = 2, 82 board_ahci_ign_iferr = 3, 83 board_ahci_sb600 = 4, 84 85 /* global controller registers */ 86 HOST_CAP = 0x00, /* host capabilities */ 87 HOST_CTL = 0x04, /* global host control */ 88 HOST_IRQ_STAT = 0x08, /* interrupt status */ 89 HOST_PORTS_IMPL = 0x0c, /* bitmap of implemented ports */ 90 HOST_VERSION = 0x10, /* AHCI spec. version compliancy */ 91 92 /* HOST_CTL bits */ 93 HOST_RESET = (1 << 0), /* reset controller; self-clear */ 94 HOST_IRQ_EN = (1 << 1), /* global IRQ enable */ 95 HOST_AHCI_EN = (1 << 31), /* AHCI enabled */ 96 97 /* HOST_CAP bits */ 98 HOST_CAP_SSC = (1 << 14), /* Slumber capable */ 99 HOST_CAP_CLO = (1 << 24), /* Command List Override support */ 100 HOST_CAP_SSS = (1 << 27), /* Staggered Spin-up */ 101 HOST_CAP_NCQ = (1 << 30), /* Native Command Queueing */ 102 HOST_CAP_64 = (1 << 31), /* PCI DAC (64-bit DMA) support */ 103 104 /* registers for each SATA port */ 105 PORT_LST_ADDR = 0x00, /* command list DMA addr */ 106 PORT_LST_ADDR_HI = 0x04, /* command list DMA addr hi */ 107 PORT_FIS_ADDR = 0x08, /* FIS rx buf addr */ 108 PORT_FIS_ADDR_HI = 0x0c, /* FIS rx buf addr hi */ 109 PORT_IRQ_STAT = 0x10, /* interrupt status */ 110 PORT_IRQ_MASK = 0x14, /* interrupt enable/disable mask */ 111 PORT_CMD = 0x18, /* port command */ 112 PORT_TFDATA = 0x20, /* taskfile data */ 113 PORT_SIG = 0x24, /* device TF signature */ 114 PORT_CMD_ISSUE = 0x38, /* command issue */ 115 PORT_SCR = 0x28, /* SATA phy register block */ 116 PORT_SCR_STAT = 0x28, /* SATA phy register: SStatus */ 117 PORT_SCR_CTL = 0x2c, /* SATA phy register: SControl */ 118 PORT_SCR_ERR = 0x30, /* SATA phy register: SError */ 119 PORT_SCR_ACT = 0x34, /* SATA phy register: SActive */ 120 121 /* PORT_IRQ_{STAT,MASK} bits */ 122 PORT_IRQ_COLD_PRES = (1 << 31), /* cold presence detect */ 123 PORT_IRQ_TF_ERR = (1 << 30), /* task file error */ 124 PORT_IRQ_HBUS_ERR = (1 << 29), /* host bus fatal error */ 125 PORT_IRQ_HBUS_DATA_ERR = (1 << 28), /* host bus data error */ 126 PORT_IRQ_IF_ERR = (1 << 27), /* interface fatal error */ 127 PORT_IRQ_IF_NONFATAL = (1 << 26), /* interface non-fatal error */ 128 PORT_IRQ_OVERFLOW = (1 << 24), /* xfer exhausted available S/G */ 129 PORT_IRQ_BAD_PMP = (1 << 23), /* incorrect port multiplier */ 130 131 PORT_IRQ_PHYRDY = (1 << 22), /* PhyRdy changed */ 132 PORT_IRQ_DEV_ILCK = (1 << 7), /* device interlock */ 133 PORT_IRQ_CONNECT = (1 << 6), /* port connect change status */ 134 PORT_IRQ_SG_DONE = (1 << 5), /* descriptor processed */ 135 PORT_IRQ_UNK_FIS = (1 << 4), /* unknown FIS rx'd */ 136 PORT_IRQ_SDB_FIS = (1 << 3), /* Set Device Bits FIS rx'd */ 137 PORT_IRQ_DMAS_FIS = (1 << 2), /* DMA Setup FIS rx'd */ 138 PORT_IRQ_PIOS_FIS = (1 << 1), /* PIO Setup FIS rx'd */ 139 PORT_IRQ_D2H_REG_FIS = (1 << 0), /* D2H Register FIS rx'd */ 140 141 PORT_IRQ_FREEZE = PORT_IRQ_HBUS_ERR | 142 PORT_IRQ_IF_ERR | 143 PORT_IRQ_CONNECT | 144 PORT_IRQ_PHYRDY | 145 PORT_IRQ_UNK_FIS, 146 PORT_IRQ_ERROR = PORT_IRQ_FREEZE | 147 PORT_IRQ_TF_ERR | 148 PORT_IRQ_HBUS_DATA_ERR, 149 DEF_PORT_IRQ = PORT_IRQ_ERROR | PORT_IRQ_SG_DONE | 150 PORT_IRQ_SDB_FIS | PORT_IRQ_DMAS_FIS | 151 PORT_IRQ_PIOS_FIS | PORT_IRQ_D2H_REG_FIS, 152 153 /* PORT_CMD bits */ 154 PORT_CMD_ATAPI = (1 << 24), /* Device is ATAPI */ 155 PORT_CMD_LIST_ON = (1 << 15), /* cmd list DMA engine running */ 156 PORT_CMD_FIS_ON = (1 << 14), /* FIS DMA engine running */ 157 PORT_CMD_FIS_RX = (1 << 4), /* Enable FIS receive DMA engine */ 158 PORT_CMD_CLO = (1 << 3), /* Command list override */ 159 PORT_CMD_POWER_ON = (1 << 2), /* Power up device */ 160 PORT_CMD_SPIN_UP = (1 << 1), /* Spin up device */ 161 PORT_CMD_START = (1 << 0), /* Enable port DMA engine */ 162 163 PORT_CMD_ICC_MASK = (0xf << 28), /* i/f ICC state mask */ 164 PORT_CMD_ICC_ACTIVE = (0x1 << 28), /* Put i/f in active state */ 165 PORT_CMD_ICC_PARTIAL = (0x2 << 28), /* Put i/f in partial state */ 166 PORT_CMD_ICC_SLUMBER = (0x6 << 28), /* Put i/f in slumber state */ 167 168 /* ap->flags bits */ 169 AHCI_FLAG_NO_NCQ = (1 << 24), 170 AHCI_FLAG_IGN_IRQ_IF_ERR = (1 << 25), /* ignore IRQ_IF_ERR */ 171 AHCI_FLAG_HONOR_PI = (1 << 26), /* honor PORTS_IMPL */ 172 AHCI_FLAG_IGN_SERR_INTERNAL = (1 << 27), /* ignore SERR_INTERNAL */ 173 174 AHCI_FLAG_COMMON = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY | 175 ATA_FLAG_MMIO | ATA_FLAG_PIO_DMA | 176 ATA_FLAG_SKIP_D2H_BSY, 177 }; 178 179 struct ahci_cmd_hdr { 180 u32 opts; 181 u32 status; 182 u32 tbl_addr; 183 u32 tbl_addr_hi; 184 u32 reserved[4]; 185 }; 186 187 struct ahci_sg { 188 u32 addr; 189 u32 addr_hi; 190 u32 reserved; 191 u32 flags_size; 192 }; 193 194 struct ahci_host_priv { 195 u32 cap; /* cap to use */ 196 u32 port_map; /* port map to use */ 197 u32 saved_cap; /* saved initial cap */ 198 u32 saved_port_map; /* saved initial port_map */ 199 }; 200 201 struct ahci_port_priv { 202 struct ahci_cmd_hdr *cmd_slot; 203 dma_addr_t cmd_slot_dma; 204 void *cmd_tbl; 205 dma_addr_t cmd_tbl_dma; 206 void *rx_fis; 207 dma_addr_t rx_fis_dma; 208 /* for NCQ spurious interrupt analysis */ 209 unsigned int ncq_saw_d2h:1; 210 unsigned int ncq_saw_dmas:1; 211 unsigned int ncq_saw_sdb:1; 212 }; 213 214 static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg); 215 static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg, u32 val); 216 static int ahci_init_one (struct pci_dev *pdev, const struct pci_device_id *ent); 217 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc); 218 static void ahci_irq_clear(struct ata_port *ap); 219 static int ahci_port_start(struct ata_port *ap); 220 static void ahci_port_stop(struct ata_port *ap); 221 static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf); 222 static void ahci_qc_prep(struct ata_queued_cmd *qc); 223 static u8 ahci_check_status(struct ata_port *ap); 224 static void ahci_freeze(struct ata_port *ap); 225 static void ahci_thaw(struct ata_port *ap); 226 static void ahci_error_handler(struct ata_port *ap); 227 static void ahci_vt8251_error_handler(struct ata_port *ap); 228 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc); 229 #ifdef CONFIG_PM 230 static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg); 231 static int ahci_port_resume(struct ata_port *ap); 232 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg); 233 static int ahci_pci_device_resume(struct pci_dev *pdev); 234 #endif 235 236 static struct scsi_host_template ahci_sht = { 237 .module = THIS_MODULE, 238 .name = DRV_NAME, 239 .ioctl = ata_scsi_ioctl, 240 .queuecommand = ata_scsi_queuecmd, 241 .change_queue_depth = ata_scsi_change_queue_depth, 242 .can_queue = AHCI_MAX_CMDS - 1, 243 .this_id = ATA_SHT_THIS_ID, 244 .sg_tablesize = AHCI_MAX_SG, 245 .cmd_per_lun = ATA_SHT_CMD_PER_LUN, 246 .emulated = ATA_SHT_EMULATED, 247 .use_clustering = AHCI_USE_CLUSTERING, 248 .proc_name = DRV_NAME, 249 .dma_boundary = AHCI_DMA_BOUNDARY, 250 .slave_configure = ata_scsi_slave_config, 251 .slave_destroy = ata_scsi_slave_destroy, 252 .bios_param = ata_std_bios_param, 253 #ifdef CONFIG_PM 254 .suspend = ata_scsi_device_suspend, 255 .resume = ata_scsi_device_resume, 256 #endif 257 }; 258 259 static const struct ata_port_operations ahci_ops = { 260 .port_disable = ata_port_disable, 261 262 .check_status = ahci_check_status, 263 .check_altstatus = ahci_check_status, 264 .dev_select = ata_noop_dev_select, 265 266 .tf_read = ahci_tf_read, 267 268 .qc_prep = ahci_qc_prep, 269 .qc_issue = ahci_qc_issue, 270 271 .irq_clear = ahci_irq_clear, 272 .irq_on = ata_dummy_irq_on, 273 .irq_ack = ata_dummy_irq_ack, 274 275 .scr_read = ahci_scr_read, 276 .scr_write = ahci_scr_write, 277 278 .freeze = ahci_freeze, 279 .thaw = ahci_thaw, 280 281 .error_handler = ahci_error_handler, 282 .post_internal_cmd = ahci_post_internal_cmd, 283 284 #ifdef CONFIG_PM 285 .port_suspend = ahci_port_suspend, 286 .port_resume = ahci_port_resume, 287 #endif 288 289 .port_start = ahci_port_start, 290 .port_stop = ahci_port_stop, 291 }; 292 293 static const struct ata_port_operations ahci_vt8251_ops = { 294 .port_disable = ata_port_disable, 295 296 .check_status = ahci_check_status, 297 .check_altstatus = ahci_check_status, 298 .dev_select = ata_noop_dev_select, 299 300 .tf_read = ahci_tf_read, 301 302 .qc_prep = ahci_qc_prep, 303 .qc_issue = ahci_qc_issue, 304 305 .irq_clear = ahci_irq_clear, 306 .irq_on = ata_dummy_irq_on, 307 .irq_ack = ata_dummy_irq_ack, 308 309 .scr_read = ahci_scr_read, 310 .scr_write = ahci_scr_write, 311 312 .freeze = ahci_freeze, 313 .thaw = ahci_thaw, 314 315 .error_handler = ahci_vt8251_error_handler, 316 .post_internal_cmd = ahci_post_internal_cmd, 317 318 #ifdef CONFIG_PM 319 .port_suspend = ahci_port_suspend, 320 .port_resume = ahci_port_resume, 321 #endif 322 323 .port_start = ahci_port_start, 324 .port_stop = ahci_port_stop, 325 }; 326 327 static const struct ata_port_info ahci_port_info[] = { 328 /* board_ahci */ 329 { 330 .flags = AHCI_FLAG_COMMON, 331 .pio_mask = 0x1f, /* pio0-4 */ 332 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 333 .port_ops = &ahci_ops, 334 }, 335 /* board_ahci_pi */ 336 { 337 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_HONOR_PI, 338 .pio_mask = 0x1f, /* pio0-4 */ 339 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 340 .port_ops = &ahci_ops, 341 }, 342 /* board_ahci_vt8251 */ 343 { 344 .flags = AHCI_FLAG_COMMON | ATA_FLAG_HRST_TO_RESUME | 345 AHCI_FLAG_NO_NCQ, 346 .pio_mask = 0x1f, /* pio0-4 */ 347 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 348 .port_ops = &ahci_vt8251_ops, 349 }, 350 /* board_ahci_ign_iferr */ 351 { 352 .flags = AHCI_FLAG_COMMON | AHCI_FLAG_IGN_IRQ_IF_ERR, 353 .pio_mask = 0x1f, /* pio0-4 */ 354 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 355 .port_ops = &ahci_ops, 356 }, 357 /* board_ahci_sb600 */ 358 { 359 .flags = AHCI_FLAG_COMMON | 360 AHCI_FLAG_IGN_SERR_INTERNAL, 361 .pio_mask = 0x1f, /* pio0-4 */ 362 .udma_mask = 0x7f, /* udma0-6 ; FIXME */ 363 .port_ops = &ahci_ops, 364 }, 365 }; 366 367 static const struct pci_device_id ahci_pci_tbl[] = { 368 /* Intel */ 369 { PCI_VDEVICE(INTEL, 0x2652), board_ahci }, /* ICH6 */ 370 { PCI_VDEVICE(INTEL, 0x2653), board_ahci }, /* ICH6M */ 371 { PCI_VDEVICE(INTEL, 0x27c1), board_ahci }, /* ICH7 */ 372 { PCI_VDEVICE(INTEL, 0x27c5), board_ahci }, /* ICH7M */ 373 { PCI_VDEVICE(INTEL, 0x27c3), board_ahci }, /* ICH7R */ 374 { PCI_VDEVICE(AL, 0x5288), board_ahci_ign_iferr }, /* ULi M5288 */ 375 { PCI_VDEVICE(INTEL, 0x2681), board_ahci }, /* ESB2 */ 376 { PCI_VDEVICE(INTEL, 0x2682), board_ahci }, /* ESB2 */ 377 { PCI_VDEVICE(INTEL, 0x2683), board_ahci }, /* ESB2 */ 378 { PCI_VDEVICE(INTEL, 0x27c6), board_ahci }, /* ICH7-M DH */ 379 { PCI_VDEVICE(INTEL, 0x2821), board_ahci_pi }, /* ICH8 */ 380 { PCI_VDEVICE(INTEL, 0x2822), board_ahci_pi }, /* ICH8 */ 381 { PCI_VDEVICE(INTEL, 0x2824), board_ahci_pi }, /* ICH8 */ 382 { PCI_VDEVICE(INTEL, 0x2829), board_ahci_pi }, /* ICH8M */ 383 { PCI_VDEVICE(INTEL, 0x282a), board_ahci_pi }, /* ICH8M */ 384 { PCI_VDEVICE(INTEL, 0x2922), board_ahci_pi }, /* ICH9 */ 385 { PCI_VDEVICE(INTEL, 0x2923), board_ahci_pi }, /* ICH9 */ 386 { PCI_VDEVICE(INTEL, 0x2924), board_ahci_pi }, /* ICH9 */ 387 { PCI_VDEVICE(INTEL, 0x2925), board_ahci_pi }, /* ICH9 */ 388 { PCI_VDEVICE(INTEL, 0x2927), board_ahci_pi }, /* ICH9 */ 389 { PCI_VDEVICE(INTEL, 0x2929), board_ahci_pi }, /* ICH9M */ 390 { PCI_VDEVICE(INTEL, 0x292a), board_ahci_pi }, /* ICH9M */ 391 { PCI_VDEVICE(INTEL, 0x292b), board_ahci_pi }, /* ICH9M */ 392 { PCI_VDEVICE(INTEL, 0x292c), board_ahci_pi }, /* ICH9M */ 393 { PCI_VDEVICE(INTEL, 0x292f), board_ahci_pi }, /* ICH9M */ 394 { PCI_VDEVICE(INTEL, 0x294d), board_ahci_pi }, /* ICH9 */ 395 { PCI_VDEVICE(INTEL, 0x294e), board_ahci_pi }, /* ICH9M */ 396 397 /* JMicron 360/1/3/5/6, match class to avoid IDE function */ 398 { PCI_VENDOR_ID_JMICRON, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 399 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci_ign_iferr }, 400 401 /* ATI */ 402 { PCI_VDEVICE(ATI, 0x4380), board_ahci_sb600 }, /* ATI SB600 */ 403 404 /* VIA */ 405 { PCI_VDEVICE(VIA, 0x3349), board_ahci_vt8251 }, /* VIA VT8251 */ 406 { PCI_VDEVICE(VIA, 0x6287), board_ahci_vt8251 }, /* VIA VT8251 */ 407 408 /* NVIDIA */ 409 { PCI_VDEVICE(NVIDIA, 0x044c), board_ahci }, /* MCP65 */ 410 { PCI_VDEVICE(NVIDIA, 0x044d), board_ahci }, /* MCP65 */ 411 { PCI_VDEVICE(NVIDIA, 0x044e), board_ahci }, /* MCP65 */ 412 { PCI_VDEVICE(NVIDIA, 0x044f), board_ahci }, /* MCP65 */ 413 { PCI_VDEVICE(NVIDIA, 0x045c), board_ahci }, /* MCP65 */ 414 { PCI_VDEVICE(NVIDIA, 0x045d), board_ahci }, /* MCP65 */ 415 { PCI_VDEVICE(NVIDIA, 0x045e), board_ahci }, /* MCP65 */ 416 { PCI_VDEVICE(NVIDIA, 0x045f), board_ahci }, /* MCP65 */ 417 { PCI_VDEVICE(NVIDIA, 0x0550), board_ahci }, /* MCP67 */ 418 { PCI_VDEVICE(NVIDIA, 0x0551), board_ahci }, /* MCP67 */ 419 { PCI_VDEVICE(NVIDIA, 0x0552), board_ahci }, /* MCP67 */ 420 { PCI_VDEVICE(NVIDIA, 0x0553), board_ahci }, /* MCP67 */ 421 { PCI_VDEVICE(NVIDIA, 0x0554), board_ahci }, /* MCP67 */ 422 { PCI_VDEVICE(NVIDIA, 0x0555), board_ahci }, /* MCP67 */ 423 { PCI_VDEVICE(NVIDIA, 0x0556), board_ahci }, /* MCP67 */ 424 { PCI_VDEVICE(NVIDIA, 0x0557), board_ahci }, /* MCP67 */ 425 { PCI_VDEVICE(NVIDIA, 0x0558), board_ahci }, /* MCP67 */ 426 { PCI_VDEVICE(NVIDIA, 0x0559), board_ahci }, /* MCP67 */ 427 { PCI_VDEVICE(NVIDIA, 0x055a), board_ahci }, /* MCP67 */ 428 { PCI_VDEVICE(NVIDIA, 0x055b), board_ahci }, /* MCP67 */ 429 430 /* SiS */ 431 { PCI_VDEVICE(SI, 0x1184), board_ahci }, /* SiS 966 */ 432 { PCI_VDEVICE(SI, 0x1185), board_ahci }, /* SiS 966 */ 433 { PCI_VDEVICE(SI, 0x0186), board_ahci }, /* SiS 968 */ 434 435 /* Generic, PCI class code for AHCI */ 436 { PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, PCI_ANY_ID, 437 PCI_CLASS_STORAGE_SATA_AHCI, 0xffffff, board_ahci }, 438 439 { } /* terminate list */ 440 }; 441 442 443 static struct pci_driver ahci_pci_driver = { 444 .name = DRV_NAME, 445 .id_table = ahci_pci_tbl, 446 .probe = ahci_init_one, 447 .remove = ata_pci_remove_one, 448 #ifdef CONFIG_PM 449 .suspend = ahci_pci_device_suspend, 450 .resume = ahci_pci_device_resume, 451 #endif 452 }; 453 454 455 static inline int ahci_nr_ports(u32 cap) 456 { 457 return (cap & 0x1f) + 1; 458 } 459 460 static inline void __iomem *ahci_port_base(struct ata_port *ap) 461 { 462 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR]; 463 464 return mmio + 0x100 + (ap->port_no * 0x80); 465 } 466 467 /** 468 * ahci_save_initial_config - Save and fixup initial config values 469 * @pdev: target PCI device 470 * @pi: associated ATA port info 471 * @hpriv: host private area to store config values 472 * 473 * Some registers containing configuration info might be setup by 474 * BIOS and might be cleared on reset. This function saves the 475 * initial values of those registers into @hpriv such that they 476 * can be restored after controller reset. 477 * 478 * If inconsistent, config values are fixed up by this function. 479 * 480 * LOCKING: 481 * None. 482 */ 483 static void ahci_save_initial_config(struct pci_dev *pdev, 484 const struct ata_port_info *pi, 485 struct ahci_host_priv *hpriv) 486 { 487 void __iomem *mmio = pcim_iomap_table(pdev)[AHCI_PCI_BAR]; 488 u32 cap, port_map; 489 int i; 490 491 /* Values prefixed with saved_ are written back to host after 492 * reset. Values without are used for driver operation. 493 */ 494 hpriv->saved_cap = cap = readl(mmio + HOST_CAP); 495 hpriv->saved_port_map = port_map = readl(mmio + HOST_PORTS_IMPL); 496 497 /* fixup zero port_map */ 498 if (!port_map) { 499 port_map = (1 << ahci_nr_ports(hpriv->cap)) - 1; 500 dev_printk(KERN_WARNING, &pdev->dev, 501 "PORTS_IMPL is zero, forcing 0x%x\n", port_map); 502 503 /* write the fixed up value to the PI register */ 504 hpriv->saved_port_map = port_map; 505 } 506 507 /* cross check port_map and cap.n_ports */ 508 if (pi->flags & AHCI_FLAG_HONOR_PI) { 509 u32 tmp_port_map = port_map; 510 int n_ports = ahci_nr_ports(cap); 511 512 for (i = 0; i < AHCI_MAX_PORTS && n_ports; i++) { 513 if (tmp_port_map & (1 << i)) { 514 n_ports--; 515 tmp_port_map &= ~(1 << i); 516 } 517 } 518 519 /* Whine if inconsistent. No need to update cap. 520 * port_map is used to determine number of ports. 521 */ 522 if (n_ports || tmp_port_map) 523 dev_printk(KERN_WARNING, &pdev->dev, 524 "nr_ports (%u) and implemented port map " 525 "(0x%x) don't match\n", 526 ahci_nr_ports(cap), port_map); 527 } else { 528 /* fabricate port_map from cap.nr_ports */ 529 port_map = (1 << ahci_nr_ports(cap)) - 1; 530 } 531 532 /* record values to use during operation */ 533 hpriv->cap = cap; 534 hpriv->port_map = port_map; 535 } 536 537 /** 538 * ahci_restore_initial_config - Restore initial config 539 * @host: target ATA host 540 * 541 * Restore initial config stored by ahci_save_initial_config(). 542 * 543 * LOCKING: 544 * None. 545 */ 546 static void ahci_restore_initial_config(struct ata_host *host) 547 { 548 struct ahci_host_priv *hpriv = host->private_data; 549 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 550 551 writel(hpriv->saved_cap, mmio + HOST_CAP); 552 writel(hpriv->saved_port_map, mmio + HOST_PORTS_IMPL); 553 (void) readl(mmio + HOST_PORTS_IMPL); /* flush */ 554 } 555 556 static u32 ahci_scr_read (struct ata_port *ap, unsigned int sc_reg_in) 557 { 558 unsigned int sc_reg; 559 560 switch (sc_reg_in) { 561 case SCR_STATUS: sc_reg = 0; break; 562 case SCR_CONTROL: sc_reg = 1; break; 563 case SCR_ERROR: sc_reg = 2; break; 564 case SCR_ACTIVE: sc_reg = 3; break; 565 default: 566 return 0xffffffffU; 567 } 568 569 return readl(ap->ioaddr.scr_addr + (sc_reg * 4)); 570 } 571 572 573 static void ahci_scr_write (struct ata_port *ap, unsigned int sc_reg_in, 574 u32 val) 575 { 576 unsigned int sc_reg; 577 578 switch (sc_reg_in) { 579 case SCR_STATUS: sc_reg = 0; break; 580 case SCR_CONTROL: sc_reg = 1; break; 581 case SCR_ERROR: sc_reg = 2; break; 582 case SCR_ACTIVE: sc_reg = 3; break; 583 default: 584 return; 585 } 586 587 writel(val, ap->ioaddr.scr_addr + (sc_reg * 4)); 588 } 589 590 static void ahci_start_engine(struct ata_port *ap) 591 { 592 void __iomem *port_mmio = ahci_port_base(ap); 593 u32 tmp; 594 595 /* start DMA */ 596 tmp = readl(port_mmio + PORT_CMD); 597 tmp |= PORT_CMD_START; 598 writel(tmp, port_mmio + PORT_CMD); 599 readl(port_mmio + PORT_CMD); /* flush */ 600 } 601 602 static int ahci_stop_engine(struct ata_port *ap) 603 { 604 void __iomem *port_mmio = ahci_port_base(ap); 605 u32 tmp; 606 607 tmp = readl(port_mmio + PORT_CMD); 608 609 /* check if the HBA is idle */ 610 if ((tmp & (PORT_CMD_START | PORT_CMD_LIST_ON)) == 0) 611 return 0; 612 613 /* setting HBA to idle */ 614 tmp &= ~PORT_CMD_START; 615 writel(tmp, port_mmio + PORT_CMD); 616 617 /* wait for engine to stop. This could be as long as 500 msec */ 618 tmp = ata_wait_register(port_mmio + PORT_CMD, 619 PORT_CMD_LIST_ON, PORT_CMD_LIST_ON, 1, 500); 620 if (tmp & PORT_CMD_LIST_ON) 621 return -EIO; 622 623 return 0; 624 } 625 626 static void ahci_start_fis_rx(struct ata_port *ap) 627 { 628 void __iomem *port_mmio = ahci_port_base(ap); 629 struct ahci_host_priv *hpriv = ap->host->private_data; 630 struct ahci_port_priv *pp = ap->private_data; 631 u32 tmp; 632 633 /* set FIS registers */ 634 if (hpriv->cap & HOST_CAP_64) 635 writel((pp->cmd_slot_dma >> 16) >> 16, 636 port_mmio + PORT_LST_ADDR_HI); 637 writel(pp->cmd_slot_dma & 0xffffffff, port_mmio + PORT_LST_ADDR); 638 639 if (hpriv->cap & HOST_CAP_64) 640 writel((pp->rx_fis_dma >> 16) >> 16, 641 port_mmio + PORT_FIS_ADDR_HI); 642 writel(pp->rx_fis_dma & 0xffffffff, port_mmio + PORT_FIS_ADDR); 643 644 /* enable FIS reception */ 645 tmp = readl(port_mmio + PORT_CMD); 646 tmp |= PORT_CMD_FIS_RX; 647 writel(tmp, port_mmio + PORT_CMD); 648 649 /* flush */ 650 readl(port_mmio + PORT_CMD); 651 } 652 653 static int ahci_stop_fis_rx(struct ata_port *ap) 654 { 655 void __iomem *port_mmio = ahci_port_base(ap); 656 u32 tmp; 657 658 /* disable FIS reception */ 659 tmp = readl(port_mmio + PORT_CMD); 660 tmp &= ~PORT_CMD_FIS_RX; 661 writel(tmp, port_mmio + PORT_CMD); 662 663 /* wait for completion, spec says 500ms, give it 1000 */ 664 tmp = ata_wait_register(port_mmio + PORT_CMD, PORT_CMD_FIS_ON, 665 PORT_CMD_FIS_ON, 10, 1000); 666 if (tmp & PORT_CMD_FIS_ON) 667 return -EBUSY; 668 669 return 0; 670 } 671 672 static void ahci_power_up(struct ata_port *ap) 673 { 674 struct ahci_host_priv *hpriv = ap->host->private_data; 675 void __iomem *port_mmio = ahci_port_base(ap); 676 u32 cmd; 677 678 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK; 679 680 /* spin up device */ 681 if (hpriv->cap & HOST_CAP_SSS) { 682 cmd |= PORT_CMD_SPIN_UP; 683 writel(cmd, port_mmio + PORT_CMD); 684 } 685 686 /* wake up link */ 687 writel(cmd | PORT_CMD_ICC_ACTIVE, port_mmio + PORT_CMD); 688 } 689 690 #ifdef CONFIG_PM 691 static void ahci_power_down(struct ata_port *ap) 692 { 693 struct ahci_host_priv *hpriv = ap->host->private_data; 694 void __iomem *port_mmio = ahci_port_base(ap); 695 u32 cmd, scontrol; 696 697 if (!(hpriv->cap & HOST_CAP_SSS)) 698 return; 699 700 /* put device into listen mode, first set PxSCTL.DET to 0 */ 701 scontrol = readl(port_mmio + PORT_SCR_CTL); 702 scontrol &= ~0xf; 703 writel(scontrol, port_mmio + PORT_SCR_CTL); 704 705 /* then set PxCMD.SUD to 0 */ 706 cmd = readl(port_mmio + PORT_CMD) & ~PORT_CMD_ICC_MASK; 707 cmd &= ~PORT_CMD_SPIN_UP; 708 writel(cmd, port_mmio + PORT_CMD); 709 } 710 #endif 711 712 static void ahci_init_port(struct ata_port *ap) 713 { 714 /* enable FIS reception */ 715 ahci_start_fis_rx(ap); 716 717 /* enable DMA */ 718 ahci_start_engine(ap); 719 } 720 721 static int ahci_deinit_port(struct ata_port *ap, const char **emsg) 722 { 723 int rc; 724 725 /* disable DMA */ 726 rc = ahci_stop_engine(ap); 727 if (rc) { 728 *emsg = "failed to stop engine"; 729 return rc; 730 } 731 732 /* disable FIS reception */ 733 rc = ahci_stop_fis_rx(ap); 734 if (rc) { 735 *emsg = "failed stop FIS RX"; 736 return rc; 737 } 738 739 return 0; 740 } 741 742 static int ahci_reset_controller(struct ata_host *host) 743 { 744 struct pci_dev *pdev = to_pci_dev(host->dev); 745 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 746 u32 tmp; 747 748 /* global controller reset */ 749 tmp = readl(mmio + HOST_CTL); 750 if ((tmp & HOST_RESET) == 0) { 751 writel(tmp | HOST_RESET, mmio + HOST_CTL); 752 readl(mmio + HOST_CTL); /* flush */ 753 } 754 755 /* reset must complete within 1 second, or 756 * the hardware should be considered fried. 757 */ 758 ssleep(1); 759 760 tmp = readl(mmio + HOST_CTL); 761 if (tmp & HOST_RESET) { 762 dev_printk(KERN_ERR, host->dev, 763 "controller reset failed (0x%x)\n", tmp); 764 return -EIO; 765 } 766 767 /* turn on AHCI mode */ 768 writel(HOST_AHCI_EN, mmio + HOST_CTL); 769 (void) readl(mmio + HOST_CTL); /* flush */ 770 771 /* some registers might be cleared on reset. restore initial values */ 772 ahci_restore_initial_config(host); 773 774 if (pdev->vendor == PCI_VENDOR_ID_INTEL) { 775 u16 tmp16; 776 777 /* configure PCS */ 778 pci_read_config_word(pdev, 0x92, &tmp16); 779 tmp16 |= 0xf; 780 pci_write_config_word(pdev, 0x92, tmp16); 781 } 782 783 return 0; 784 } 785 786 static void ahci_init_controller(struct ata_host *host) 787 { 788 struct pci_dev *pdev = to_pci_dev(host->dev); 789 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 790 int i, rc; 791 u32 tmp; 792 793 for (i = 0; i < host->n_ports; i++) { 794 struct ata_port *ap = host->ports[i]; 795 void __iomem *port_mmio = ahci_port_base(ap); 796 const char *emsg = NULL; 797 798 if (ata_port_is_dummy(ap)) 799 continue; 800 801 /* make sure port is not active */ 802 rc = ahci_deinit_port(ap, &emsg); 803 if (rc) 804 dev_printk(KERN_WARNING, &pdev->dev, 805 "%s (%d)\n", emsg, rc); 806 807 /* clear SError */ 808 tmp = readl(port_mmio + PORT_SCR_ERR); 809 VPRINTK("PORT_SCR_ERR 0x%x\n", tmp); 810 writel(tmp, port_mmio + PORT_SCR_ERR); 811 812 /* clear port IRQ */ 813 tmp = readl(port_mmio + PORT_IRQ_STAT); 814 VPRINTK("PORT_IRQ_STAT 0x%x\n", tmp); 815 if (tmp) 816 writel(tmp, port_mmio + PORT_IRQ_STAT); 817 818 writel(1 << i, mmio + HOST_IRQ_STAT); 819 } 820 821 tmp = readl(mmio + HOST_CTL); 822 VPRINTK("HOST_CTL 0x%x\n", tmp); 823 writel(tmp | HOST_IRQ_EN, mmio + HOST_CTL); 824 tmp = readl(mmio + HOST_CTL); 825 VPRINTK("HOST_CTL 0x%x\n", tmp); 826 } 827 828 static unsigned int ahci_dev_classify(struct ata_port *ap) 829 { 830 void __iomem *port_mmio = ahci_port_base(ap); 831 struct ata_taskfile tf; 832 u32 tmp; 833 834 tmp = readl(port_mmio + PORT_SIG); 835 tf.lbah = (tmp >> 24) & 0xff; 836 tf.lbam = (tmp >> 16) & 0xff; 837 tf.lbal = (tmp >> 8) & 0xff; 838 tf.nsect = (tmp) & 0xff; 839 840 return ata_dev_classify(&tf); 841 } 842 843 static void ahci_fill_cmd_slot(struct ahci_port_priv *pp, unsigned int tag, 844 u32 opts) 845 { 846 dma_addr_t cmd_tbl_dma; 847 848 cmd_tbl_dma = pp->cmd_tbl_dma + tag * AHCI_CMD_TBL_SZ; 849 850 pp->cmd_slot[tag].opts = cpu_to_le32(opts); 851 pp->cmd_slot[tag].status = 0; 852 pp->cmd_slot[tag].tbl_addr = cpu_to_le32(cmd_tbl_dma & 0xffffffff); 853 pp->cmd_slot[tag].tbl_addr_hi = cpu_to_le32((cmd_tbl_dma >> 16) >> 16); 854 } 855 856 static int ahci_clo(struct ata_port *ap) 857 { 858 void __iomem *port_mmio = ap->ioaddr.cmd_addr; 859 struct ahci_host_priv *hpriv = ap->host->private_data; 860 u32 tmp; 861 862 if (!(hpriv->cap & HOST_CAP_CLO)) 863 return -EOPNOTSUPP; 864 865 tmp = readl(port_mmio + PORT_CMD); 866 tmp |= PORT_CMD_CLO; 867 writel(tmp, port_mmio + PORT_CMD); 868 869 tmp = ata_wait_register(port_mmio + PORT_CMD, 870 PORT_CMD_CLO, PORT_CMD_CLO, 1, 500); 871 if (tmp & PORT_CMD_CLO) 872 return -EIO; 873 874 return 0; 875 } 876 877 static int ahci_softreset(struct ata_port *ap, unsigned int *class) 878 { 879 struct ahci_port_priv *pp = ap->private_data; 880 void __iomem *port_mmio = ahci_port_base(ap); 881 const u32 cmd_fis_len = 5; /* five dwords */ 882 const char *reason = NULL; 883 struct ata_taskfile tf; 884 u32 tmp; 885 u8 *fis; 886 int rc; 887 888 DPRINTK("ENTER\n"); 889 890 if (ata_port_offline(ap)) { 891 DPRINTK("PHY reports no device\n"); 892 *class = ATA_DEV_NONE; 893 return 0; 894 } 895 896 /* prepare for SRST (AHCI-1.1 10.4.1) */ 897 rc = ahci_stop_engine(ap); 898 if (rc) { 899 reason = "failed to stop engine"; 900 goto fail_restart; 901 } 902 903 /* check BUSY/DRQ, perform Command List Override if necessary */ 904 if (ahci_check_status(ap) & (ATA_BUSY | ATA_DRQ)) { 905 rc = ahci_clo(ap); 906 907 if (rc == -EOPNOTSUPP) { 908 reason = "port busy but CLO unavailable"; 909 goto fail_restart; 910 } else if (rc) { 911 reason = "port busy but CLO failed"; 912 goto fail_restart; 913 } 914 } 915 916 /* restart engine */ 917 ahci_start_engine(ap); 918 919 ata_tf_init(ap->device, &tf); 920 fis = pp->cmd_tbl; 921 922 /* issue the first D2H Register FIS */ 923 ahci_fill_cmd_slot(pp, 0, 924 cmd_fis_len | AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY); 925 926 tf.ctl |= ATA_SRST; 927 ata_tf_to_fis(&tf, fis, 0); 928 fis[1] &= ~(1 << 7); /* turn off Command FIS bit */ 929 930 writel(1, port_mmio + PORT_CMD_ISSUE); 931 932 tmp = ata_wait_register(port_mmio + PORT_CMD_ISSUE, 0x1, 0x1, 1, 500); 933 if (tmp & 0x1) { 934 rc = -EIO; 935 reason = "1st FIS failed"; 936 goto fail; 937 } 938 939 /* spec says at least 5us, but be generous and sleep for 1ms */ 940 msleep(1); 941 942 /* issue the second D2H Register FIS */ 943 ahci_fill_cmd_slot(pp, 0, cmd_fis_len); 944 945 tf.ctl &= ~ATA_SRST; 946 ata_tf_to_fis(&tf, fis, 0); 947 fis[1] &= ~(1 << 7); /* turn off Command FIS bit */ 948 949 writel(1, port_mmio + PORT_CMD_ISSUE); 950 readl(port_mmio + PORT_CMD_ISSUE); /* flush */ 951 952 /* spec mandates ">= 2ms" before checking status. 953 * We wait 150ms, because that was the magic delay used for 954 * ATAPI devices in Hale Landis's ATADRVR, for the period of time 955 * between when the ATA command register is written, and then 956 * status is checked. Because waiting for "a while" before 957 * checking status is fine, post SRST, we perform this magic 958 * delay here as well. 959 */ 960 msleep(150); 961 962 *class = ATA_DEV_NONE; 963 if (ata_port_online(ap)) { 964 if (ata_busy_sleep(ap, ATA_TMOUT_BOOT_QUICK, ATA_TMOUT_BOOT)) { 965 rc = -EIO; 966 reason = "device not ready"; 967 goto fail; 968 } 969 *class = ahci_dev_classify(ap); 970 } 971 972 DPRINTK("EXIT, class=%u\n", *class); 973 return 0; 974 975 fail_restart: 976 ahci_start_engine(ap); 977 fail: 978 ata_port_printk(ap, KERN_ERR, "softreset failed (%s)\n", reason); 979 return rc; 980 } 981 982 static int ahci_hardreset(struct ata_port *ap, unsigned int *class) 983 { 984 struct ahci_port_priv *pp = ap->private_data; 985 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG; 986 struct ata_taskfile tf; 987 int rc; 988 989 DPRINTK("ENTER\n"); 990 991 ahci_stop_engine(ap); 992 993 /* clear D2H reception area to properly wait for D2H FIS */ 994 ata_tf_init(ap->device, &tf); 995 tf.command = 0x80; 996 ata_tf_to_fis(&tf, d2h_fis, 0); 997 998 rc = sata_std_hardreset(ap, class); 999 1000 ahci_start_engine(ap); 1001 1002 if (rc == 0 && ata_port_online(ap)) 1003 *class = ahci_dev_classify(ap); 1004 if (*class == ATA_DEV_UNKNOWN) 1005 *class = ATA_DEV_NONE; 1006 1007 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class); 1008 return rc; 1009 } 1010 1011 static int ahci_vt8251_hardreset(struct ata_port *ap, unsigned int *class) 1012 { 1013 int rc; 1014 1015 DPRINTK("ENTER\n"); 1016 1017 ahci_stop_engine(ap); 1018 1019 rc = sata_port_hardreset(ap, sata_ehc_deb_timing(&ap->eh_context)); 1020 1021 /* vt8251 needs SError cleared for the port to operate */ 1022 ahci_scr_write(ap, SCR_ERROR, ahci_scr_read(ap, SCR_ERROR)); 1023 1024 ahci_start_engine(ap); 1025 1026 DPRINTK("EXIT, rc=%d, class=%u\n", rc, *class); 1027 1028 /* vt8251 doesn't clear BSY on signature FIS reception, 1029 * request follow-up softreset. 1030 */ 1031 return rc ?: -EAGAIN; 1032 } 1033 1034 static void ahci_postreset(struct ata_port *ap, unsigned int *class) 1035 { 1036 void __iomem *port_mmio = ahci_port_base(ap); 1037 u32 new_tmp, tmp; 1038 1039 ata_std_postreset(ap, class); 1040 1041 /* Make sure port's ATAPI bit is set appropriately */ 1042 new_tmp = tmp = readl(port_mmio + PORT_CMD); 1043 if (*class == ATA_DEV_ATAPI) 1044 new_tmp |= PORT_CMD_ATAPI; 1045 else 1046 new_tmp &= ~PORT_CMD_ATAPI; 1047 if (new_tmp != tmp) { 1048 writel(new_tmp, port_mmio + PORT_CMD); 1049 readl(port_mmio + PORT_CMD); /* flush */ 1050 } 1051 } 1052 1053 static u8 ahci_check_status(struct ata_port *ap) 1054 { 1055 void __iomem *mmio = ap->ioaddr.cmd_addr; 1056 1057 return readl(mmio + PORT_TFDATA) & 0xFF; 1058 } 1059 1060 static void ahci_tf_read(struct ata_port *ap, struct ata_taskfile *tf) 1061 { 1062 struct ahci_port_priv *pp = ap->private_data; 1063 u8 *d2h_fis = pp->rx_fis + RX_FIS_D2H_REG; 1064 1065 ata_tf_from_fis(d2h_fis, tf); 1066 } 1067 1068 static unsigned int ahci_fill_sg(struct ata_queued_cmd *qc, void *cmd_tbl) 1069 { 1070 struct scatterlist *sg; 1071 struct ahci_sg *ahci_sg; 1072 unsigned int n_sg = 0; 1073 1074 VPRINTK("ENTER\n"); 1075 1076 /* 1077 * Next, the S/G list. 1078 */ 1079 ahci_sg = cmd_tbl + AHCI_CMD_TBL_HDR_SZ; 1080 ata_for_each_sg(sg, qc) { 1081 dma_addr_t addr = sg_dma_address(sg); 1082 u32 sg_len = sg_dma_len(sg); 1083 1084 ahci_sg->addr = cpu_to_le32(addr & 0xffffffff); 1085 ahci_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16); 1086 ahci_sg->flags_size = cpu_to_le32(sg_len - 1); 1087 1088 ahci_sg++; 1089 n_sg++; 1090 } 1091 1092 return n_sg; 1093 } 1094 1095 static void ahci_qc_prep(struct ata_queued_cmd *qc) 1096 { 1097 struct ata_port *ap = qc->ap; 1098 struct ahci_port_priv *pp = ap->private_data; 1099 int is_atapi = is_atapi_taskfile(&qc->tf); 1100 void *cmd_tbl; 1101 u32 opts; 1102 const u32 cmd_fis_len = 5; /* five dwords */ 1103 unsigned int n_elem; 1104 1105 /* 1106 * Fill in command table information. First, the header, 1107 * a SATA Register - Host to Device command FIS. 1108 */ 1109 cmd_tbl = pp->cmd_tbl + qc->tag * AHCI_CMD_TBL_SZ; 1110 1111 ata_tf_to_fis(&qc->tf, cmd_tbl, 0); 1112 if (is_atapi) { 1113 memset(cmd_tbl + AHCI_CMD_TBL_CDB, 0, 32); 1114 memcpy(cmd_tbl + AHCI_CMD_TBL_CDB, qc->cdb, qc->dev->cdb_len); 1115 } 1116 1117 n_elem = 0; 1118 if (qc->flags & ATA_QCFLAG_DMAMAP) 1119 n_elem = ahci_fill_sg(qc, cmd_tbl); 1120 1121 /* 1122 * Fill in command slot information. 1123 */ 1124 opts = cmd_fis_len | n_elem << 16; 1125 if (qc->tf.flags & ATA_TFLAG_WRITE) 1126 opts |= AHCI_CMD_WRITE; 1127 if (is_atapi) 1128 opts |= AHCI_CMD_ATAPI | AHCI_CMD_PREFETCH; 1129 1130 ahci_fill_cmd_slot(pp, qc->tag, opts); 1131 } 1132 1133 static void ahci_error_intr(struct ata_port *ap, u32 irq_stat) 1134 { 1135 struct ahci_port_priv *pp = ap->private_data; 1136 struct ata_eh_info *ehi = &ap->eh_info; 1137 unsigned int err_mask = 0, action = 0; 1138 struct ata_queued_cmd *qc; 1139 u32 serror; 1140 1141 ata_ehi_clear_desc(ehi); 1142 1143 /* AHCI needs SError cleared; otherwise, it might lock up */ 1144 serror = ahci_scr_read(ap, SCR_ERROR); 1145 ahci_scr_write(ap, SCR_ERROR, serror); 1146 1147 /* analyze @irq_stat */ 1148 ata_ehi_push_desc(ehi, "irq_stat 0x%08x", irq_stat); 1149 1150 /* some controllers set IRQ_IF_ERR on device errors, ignore it */ 1151 if (ap->flags & AHCI_FLAG_IGN_IRQ_IF_ERR) 1152 irq_stat &= ~PORT_IRQ_IF_ERR; 1153 1154 if (irq_stat & PORT_IRQ_TF_ERR) { 1155 err_mask |= AC_ERR_DEV; 1156 if (ap->flags & AHCI_FLAG_IGN_SERR_INTERNAL) 1157 serror &= ~SERR_INTERNAL; 1158 } 1159 1160 if (irq_stat & (PORT_IRQ_HBUS_ERR | PORT_IRQ_HBUS_DATA_ERR)) { 1161 err_mask |= AC_ERR_HOST_BUS; 1162 action |= ATA_EH_SOFTRESET; 1163 } 1164 1165 if (irq_stat & PORT_IRQ_IF_ERR) { 1166 err_mask |= AC_ERR_ATA_BUS; 1167 action |= ATA_EH_SOFTRESET; 1168 ata_ehi_push_desc(ehi, ", interface fatal error"); 1169 } 1170 1171 if (irq_stat & (PORT_IRQ_CONNECT | PORT_IRQ_PHYRDY)) { 1172 ata_ehi_hotplugged(ehi); 1173 ata_ehi_push_desc(ehi, ", %s", irq_stat & PORT_IRQ_CONNECT ? 1174 "connection status changed" : "PHY RDY changed"); 1175 } 1176 1177 if (irq_stat & PORT_IRQ_UNK_FIS) { 1178 u32 *unk = (u32 *)(pp->rx_fis + RX_FIS_UNK); 1179 1180 err_mask |= AC_ERR_HSM; 1181 action |= ATA_EH_SOFTRESET; 1182 ata_ehi_push_desc(ehi, ", unknown FIS %08x %08x %08x %08x", 1183 unk[0], unk[1], unk[2], unk[3]); 1184 } 1185 1186 /* okay, let's hand over to EH */ 1187 ehi->serror |= serror; 1188 ehi->action |= action; 1189 1190 qc = ata_qc_from_tag(ap, ap->active_tag); 1191 if (qc) 1192 qc->err_mask |= err_mask; 1193 else 1194 ehi->err_mask |= err_mask; 1195 1196 if (irq_stat & PORT_IRQ_FREEZE) 1197 ata_port_freeze(ap); 1198 else 1199 ata_port_abort(ap); 1200 } 1201 1202 static void ahci_host_intr(struct ata_port *ap) 1203 { 1204 void __iomem *port_mmio = ap->ioaddr.cmd_addr; 1205 struct ata_eh_info *ehi = &ap->eh_info; 1206 struct ahci_port_priv *pp = ap->private_data; 1207 u32 status, qc_active; 1208 int rc, known_irq = 0; 1209 1210 status = readl(port_mmio + PORT_IRQ_STAT); 1211 writel(status, port_mmio + PORT_IRQ_STAT); 1212 1213 if (unlikely(status & PORT_IRQ_ERROR)) { 1214 ahci_error_intr(ap, status); 1215 return; 1216 } 1217 1218 if (ap->sactive) 1219 qc_active = readl(port_mmio + PORT_SCR_ACT); 1220 else 1221 qc_active = readl(port_mmio + PORT_CMD_ISSUE); 1222 1223 rc = ata_qc_complete_multiple(ap, qc_active, NULL); 1224 if (rc > 0) 1225 return; 1226 if (rc < 0) { 1227 ehi->err_mask |= AC_ERR_HSM; 1228 ehi->action |= ATA_EH_SOFTRESET; 1229 ata_port_freeze(ap); 1230 return; 1231 } 1232 1233 /* hmmm... a spurious interupt */ 1234 1235 /* if !NCQ, ignore. No modern ATA device has broken HSM 1236 * implementation for non-NCQ commands. 1237 */ 1238 if (!ap->sactive) 1239 return; 1240 1241 if (status & PORT_IRQ_D2H_REG_FIS) { 1242 if (!pp->ncq_saw_d2h) 1243 ata_port_printk(ap, KERN_INFO, 1244 "D2H reg with I during NCQ, " 1245 "this message won't be printed again\n"); 1246 pp->ncq_saw_d2h = 1; 1247 known_irq = 1; 1248 } 1249 1250 if (status & PORT_IRQ_DMAS_FIS) { 1251 if (!pp->ncq_saw_dmas) 1252 ata_port_printk(ap, KERN_INFO, 1253 "DMAS FIS during NCQ, " 1254 "this message won't be printed again\n"); 1255 pp->ncq_saw_dmas = 1; 1256 known_irq = 1; 1257 } 1258 1259 if (status & PORT_IRQ_SDB_FIS) { 1260 const __le32 *f = pp->rx_fis + RX_FIS_SDB; 1261 1262 if (le32_to_cpu(f[1])) { 1263 /* SDB FIS containing spurious completions 1264 * might be dangerous, whine and fail commands 1265 * with HSM violation. EH will turn off NCQ 1266 * after several such failures. 1267 */ 1268 ata_ehi_push_desc(ehi, 1269 "spurious completions during NCQ " 1270 "issue=0x%x SAct=0x%x FIS=%08x:%08x", 1271 readl(port_mmio + PORT_CMD_ISSUE), 1272 readl(port_mmio + PORT_SCR_ACT), 1273 le32_to_cpu(f[0]), le32_to_cpu(f[1])); 1274 ehi->err_mask |= AC_ERR_HSM; 1275 ehi->action |= ATA_EH_SOFTRESET; 1276 ata_port_freeze(ap); 1277 } else { 1278 if (!pp->ncq_saw_sdb) 1279 ata_port_printk(ap, KERN_INFO, 1280 "spurious SDB FIS %08x:%08x during NCQ, " 1281 "this message won't be printed again\n", 1282 le32_to_cpu(f[0]), le32_to_cpu(f[1])); 1283 pp->ncq_saw_sdb = 1; 1284 } 1285 known_irq = 1; 1286 } 1287 1288 if (!known_irq) 1289 ata_port_printk(ap, KERN_INFO, "spurious interrupt " 1290 "(irq_stat 0x%x active_tag 0x%x sactive 0x%x)\n", 1291 status, ap->active_tag, ap->sactive); 1292 } 1293 1294 static void ahci_irq_clear(struct ata_port *ap) 1295 { 1296 /* TODO */ 1297 } 1298 1299 static irqreturn_t ahci_interrupt(int irq, void *dev_instance) 1300 { 1301 struct ata_host *host = dev_instance; 1302 struct ahci_host_priv *hpriv; 1303 unsigned int i, handled = 0; 1304 void __iomem *mmio; 1305 u32 irq_stat, irq_ack = 0; 1306 1307 VPRINTK("ENTER\n"); 1308 1309 hpriv = host->private_data; 1310 mmio = host->iomap[AHCI_PCI_BAR]; 1311 1312 /* sigh. 0xffffffff is a valid return from h/w */ 1313 irq_stat = readl(mmio + HOST_IRQ_STAT); 1314 irq_stat &= hpriv->port_map; 1315 if (!irq_stat) 1316 return IRQ_NONE; 1317 1318 spin_lock(&host->lock); 1319 1320 for (i = 0; i < host->n_ports; i++) { 1321 struct ata_port *ap; 1322 1323 if (!(irq_stat & (1 << i))) 1324 continue; 1325 1326 ap = host->ports[i]; 1327 if (ap) { 1328 ahci_host_intr(ap); 1329 VPRINTK("port %u\n", i); 1330 } else { 1331 VPRINTK("port %u (no irq)\n", i); 1332 if (ata_ratelimit()) 1333 dev_printk(KERN_WARNING, host->dev, 1334 "interrupt on disabled port %u\n", i); 1335 } 1336 1337 irq_ack |= (1 << i); 1338 } 1339 1340 if (irq_ack) { 1341 writel(irq_ack, mmio + HOST_IRQ_STAT); 1342 handled = 1; 1343 } 1344 1345 spin_unlock(&host->lock); 1346 1347 VPRINTK("EXIT\n"); 1348 1349 return IRQ_RETVAL(handled); 1350 } 1351 1352 static unsigned int ahci_qc_issue(struct ata_queued_cmd *qc) 1353 { 1354 struct ata_port *ap = qc->ap; 1355 void __iomem *port_mmio = ahci_port_base(ap); 1356 1357 if (qc->tf.protocol == ATA_PROT_NCQ) 1358 writel(1 << qc->tag, port_mmio + PORT_SCR_ACT); 1359 writel(1 << qc->tag, port_mmio + PORT_CMD_ISSUE); 1360 readl(port_mmio + PORT_CMD_ISSUE); /* flush */ 1361 1362 return 0; 1363 } 1364 1365 static void ahci_freeze(struct ata_port *ap) 1366 { 1367 void __iomem *port_mmio = ahci_port_base(ap); 1368 1369 /* turn IRQ off */ 1370 writel(0, port_mmio + PORT_IRQ_MASK); 1371 } 1372 1373 static void ahci_thaw(struct ata_port *ap) 1374 { 1375 void __iomem *mmio = ap->host->iomap[AHCI_PCI_BAR]; 1376 void __iomem *port_mmio = ahci_port_base(ap); 1377 u32 tmp; 1378 1379 /* clear IRQ */ 1380 tmp = readl(port_mmio + PORT_IRQ_STAT); 1381 writel(tmp, port_mmio + PORT_IRQ_STAT); 1382 writel(1 << ap->port_no, mmio + HOST_IRQ_STAT); 1383 1384 /* turn IRQ back on */ 1385 writel(DEF_PORT_IRQ, port_mmio + PORT_IRQ_MASK); 1386 } 1387 1388 static void ahci_error_handler(struct ata_port *ap) 1389 { 1390 if (!(ap->pflags & ATA_PFLAG_FROZEN)) { 1391 /* restart engine */ 1392 ahci_stop_engine(ap); 1393 ahci_start_engine(ap); 1394 } 1395 1396 /* perform recovery */ 1397 ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_hardreset, 1398 ahci_postreset); 1399 } 1400 1401 static void ahci_vt8251_error_handler(struct ata_port *ap) 1402 { 1403 if (!(ap->pflags & ATA_PFLAG_FROZEN)) { 1404 /* restart engine */ 1405 ahci_stop_engine(ap); 1406 ahci_start_engine(ap); 1407 } 1408 1409 /* perform recovery */ 1410 ata_do_eh(ap, ata_std_prereset, ahci_softreset, ahci_vt8251_hardreset, 1411 ahci_postreset); 1412 } 1413 1414 static void ahci_post_internal_cmd(struct ata_queued_cmd *qc) 1415 { 1416 struct ata_port *ap = qc->ap; 1417 1418 if (qc->flags & ATA_QCFLAG_FAILED) { 1419 /* make DMA engine forget about the failed command */ 1420 ahci_stop_engine(ap); 1421 ahci_start_engine(ap); 1422 } 1423 } 1424 1425 #ifdef CONFIG_PM 1426 static int ahci_port_suspend(struct ata_port *ap, pm_message_t mesg) 1427 { 1428 const char *emsg = NULL; 1429 int rc; 1430 1431 rc = ahci_deinit_port(ap, &emsg); 1432 if (rc == 0) 1433 ahci_power_down(ap); 1434 else { 1435 ata_port_printk(ap, KERN_ERR, "%s (%d)\n", emsg, rc); 1436 ahci_init_port(ap); 1437 } 1438 1439 return rc; 1440 } 1441 1442 static int ahci_port_resume(struct ata_port *ap) 1443 { 1444 ahci_power_up(ap); 1445 ahci_init_port(ap); 1446 1447 return 0; 1448 } 1449 1450 static int ahci_pci_device_suspend(struct pci_dev *pdev, pm_message_t mesg) 1451 { 1452 struct ata_host *host = dev_get_drvdata(&pdev->dev); 1453 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 1454 u32 ctl; 1455 1456 if (mesg.event == PM_EVENT_SUSPEND) { 1457 /* AHCI spec rev1.1 section 8.3.3: 1458 * Software must disable interrupts prior to requesting a 1459 * transition of the HBA to D3 state. 1460 */ 1461 ctl = readl(mmio + HOST_CTL); 1462 ctl &= ~HOST_IRQ_EN; 1463 writel(ctl, mmio + HOST_CTL); 1464 readl(mmio + HOST_CTL); /* flush */ 1465 } 1466 1467 return ata_pci_device_suspend(pdev, mesg); 1468 } 1469 1470 static int ahci_pci_device_resume(struct pci_dev *pdev) 1471 { 1472 struct ata_host *host = dev_get_drvdata(&pdev->dev); 1473 int rc; 1474 1475 rc = ata_pci_device_do_resume(pdev); 1476 if (rc) 1477 return rc; 1478 1479 if (pdev->dev.power.power_state.event == PM_EVENT_SUSPEND) { 1480 rc = ahci_reset_controller(host); 1481 if (rc) 1482 return rc; 1483 1484 ahci_init_controller(host); 1485 } 1486 1487 ata_host_resume(host); 1488 1489 return 0; 1490 } 1491 #endif 1492 1493 static int ahci_port_start(struct ata_port *ap) 1494 { 1495 struct device *dev = ap->host->dev; 1496 struct ahci_port_priv *pp; 1497 void *mem; 1498 dma_addr_t mem_dma; 1499 int rc; 1500 1501 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1502 if (!pp) 1503 return -ENOMEM; 1504 1505 rc = ata_pad_alloc(ap, dev); 1506 if (rc) 1507 return rc; 1508 1509 mem = dmam_alloc_coherent(dev, AHCI_PORT_PRIV_DMA_SZ, &mem_dma, 1510 GFP_KERNEL); 1511 if (!mem) 1512 return -ENOMEM; 1513 memset(mem, 0, AHCI_PORT_PRIV_DMA_SZ); 1514 1515 /* 1516 * First item in chunk of DMA memory: 32-slot command table, 1517 * 32 bytes each in size 1518 */ 1519 pp->cmd_slot = mem; 1520 pp->cmd_slot_dma = mem_dma; 1521 1522 mem += AHCI_CMD_SLOT_SZ; 1523 mem_dma += AHCI_CMD_SLOT_SZ; 1524 1525 /* 1526 * Second item: Received-FIS area 1527 */ 1528 pp->rx_fis = mem; 1529 pp->rx_fis_dma = mem_dma; 1530 1531 mem += AHCI_RX_FIS_SZ; 1532 mem_dma += AHCI_RX_FIS_SZ; 1533 1534 /* 1535 * Third item: data area for storing a single command 1536 * and its scatter-gather table 1537 */ 1538 pp->cmd_tbl = mem; 1539 pp->cmd_tbl_dma = mem_dma; 1540 1541 ap->private_data = pp; 1542 1543 /* power up port */ 1544 ahci_power_up(ap); 1545 1546 /* initialize port */ 1547 ahci_init_port(ap); 1548 1549 return 0; 1550 } 1551 1552 static void ahci_port_stop(struct ata_port *ap) 1553 { 1554 const char *emsg = NULL; 1555 int rc; 1556 1557 /* de-initialize port */ 1558 rc = ahci_deinit_port(ap, &emsg); 1559 if (rc) 1560 ata_port_printk(ap, KERN_WARNING, "%s (%d)\n", emsg, rc); 1561 } 1562 1563 static int ahci_configure_dma_masks(struct pci_dev *pdev, int using_dac) 1564 { 1565 int rc; 1566 1567 if (using_dac && 1568 !pci_set_dma_mask(pdev, DMA_64BIT_MASK)) { 1569 rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK); 1570 if (rc) { 1571 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); 1572 if (rc) { 1573 dev_printk(KERN_ERR, &pdev->dev, 1574 "64-bit DMA enable failed\n"); 1575 return rc; 1576 } 1577 } 1578 } else { 1579 rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK); 1580 if (rc) { 1581 dev_printk(KERN_ERR, &pdev->dev, 1582 "32-bit DMA enable failed\n"); 1583 return rc; 1584 } 1585 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK); 1586 if (rc) { 1587 dev_printk(KERN_ERR, &pdev->dev, 1588 "32-bit consistent DMA enable failed\n"); 1589 return rc; 1590 } 1591 } 1592 return 0; 1593 } 1594 1595 static void ahci_print_info(struct ata_host *host) 1596 { 1597 struct ahci_host_priv *hpriv = host->private_data; 1598 struct pci_dev *pdev = to_pci_dev(host->dev); 1599 void __iomem *mmio = host->iomap[AHCI_PCI_BAR]; 1600 u32 vers, cap, impl, speed; 1601 const char *speed_s; 1602 u16 cc; 1603 const char *scc_s; 1604 1605 vers = readl(mmio + HOST_VERSION); 1606 cap = hpriv->cap; 1607 impl = hpriv->port_map; 1608 1609 speed = (cap >> 20) & 0xf; 1610 if (speed == 1) 1611 speed_s = "1.5"; 1612 else if (speed == 2) 1613 speed_s = "3"; 1614 else 1615 speed_s = "?"; 1616 1617 pci_read_config_word(pdev, 0x0a, &cc); 1618 if (cc == PCI_CLASS_STORAGE_IDE) 1619 scc_s = "IDE"; 1620 else if (cc == PCI_CLASS_STORAGE_SATA) 1621 scc_s = "SATA"; 1622 else if (cc == PCI_CLASS_STORAGE_RAID) 1623 scc_s = "RAID"; 1624 else 1625 scc_s = "unknown"; 1626 1627 dev_printk(KERN_INFO, &pdev->dev, 1628 "AHCI %02x%02x.%02x%02x " 1629 "%u slots %u ports %s Gbps 0x%x impl %s mode\n" 1630 , 1631 1632 (vers >> 24) & 0xff, 1633 (vers >> 16) & 0xff, 1634 (vers >> 8) & 0xff, 1635 vers & 0xff, 1636 1637 ((cap >> 8) & 0x1f) + 1, 1638 (cap & 0x1f) + 1, 1639 speed_s, 1640 impl, 1641 scc_s); 1642 1643 dev_printk(KERN_INFO, &pdev->dev, 1644 "flags: " 1645 "%s%s%s%s%s%s" 1646 "%s%s%s%s%s%s%s\n" 1647 , 1648 1649 cap & (1 << 31) ? "64bit " : "", 1650 cap & (1 << 30) ? "ncq " : "", 1651 cap & (1 << 28) ? "ilck " : "", 1652 cap & (1 << 27) ? "stag " : "", 1653 cap & (1 << 26) ? "pm " : "", 1654 cap & (1 << 25) ? "led " : "", 1655 1656 cap & (1 << 24) ? "clo " : "", 1657 cap & (1 << 19) ? "nz " : "", 1658 cap & (1 << 18) ? "only " : "", 1659 cap & (1 << 17) ? "pmp " : "", 1660 cap & (1 << 15) ? "pio " : "", 1661 cap & (1 << 14) ? "slum " : "", 1662 cap & (1 << 13) ? "part " : "" 1663 ); 1664 } 1665 1666 static int ahci_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 1667 { 1668 static int printed_version; 1669 struct ata_port_info pi = ahci_port_info[ent->driver_data]; 1670 const struct ata_port_info *ppi[] = { &pi, NULL }; 1671 struct device *dev = &pdev->dev; 1672 struct ahci_host_priv *hpriv; 1673 struct ata_host *host; 1674 int i, rc; 1675 1676 VPRINTK("ENTER\n"); 1677 1678 WARN_ON(ATA_MAX_QUEUE > AHCI_MAX_CMDS); 1679 1680 if (!printed_version++) 1681 dev_printk(KERN_DEBUG, &pdev->dev, "version " DRV_VERSION "\n"); 1682 1683 /* acquire resources */ 1684 rc = pcim_enable_device(pdev); 1685 if (rc) 1686 return rc; 1687 1688 rc = pcim_iomap_regions(pdev, 1 << AHCI_PCI_BAR, DRV_NAME); 1689 if (rc == -EBUSY) 1690 pcim_pin_device(pdev); 1691 if (rc) 1692 return rc; 1693 1694 if (pci_enable_msi(pdev)) 1695 pci_intx(pdev, 1); 1696 1697 hpriv = devm_kzalloc(dev, sizeof(*hpriv), GFP_KERNEL); 1698 if (!hpriv) 1699 return -ENOMEM; 1700 1701 /* save initial config */ 1702 ahci_save_initial_config(pdev, &pi, hpriv); 1703 1704 /* prepare host */ 1705 if (!(pi.flags & AHCI_FLAG_NO_NCQ) && (hpriv->cap & HOST_CAP_NCQ)) 1706 pi.flags |= ATA_FLAG_NCQ; 1707 1708 host = ata_host_alloc_pinfo(&pdev->dev, ppi, fls(hpriv->port_map)); 1709 if (!host) 1710 return -ENOMEM; 1711 host->iomap = pcim_iomap_table(pdev); 1712 host->private_data = hpriv; 1713 1714 for (i = 0; i < host->n_ports; i++) { 1715 if (hpriv->port_map & (1 << i)) { 1716 struct ata_port *ap = host->ports[i]; 1717 void __iomem *port_mmio = ahci_port_base(ap); 1718 1719 ap->ioaddr.cmd_addr = port_mmio; 1720 ap->ioaddr.scr_addr = port_mmio + PORT_SCR; 1721 } else 1722 host->ports[i]->ops = &ata_dummy_port_ops; 1723 } 1724 1725 /* initialize adapter */ 1726 rc = ahci_configure_dma_masks(pdev, hpriv->cap & HOST_CAP_64); 1727 if (rc) 1728 return rc; 1729 1730 rc = ahci_reset_controller(host); 1731 if (rc) 1732 return rc; 1733 1734 ahci_init_controller(host); 1735 ahci_print_info(host); 1736 1737 pci_set_master(pdev); 1738 return ata_host_activate(host, pdev->irq, ahci_interrupt, IRQF_SHARED, 1739 &ahci_sht); 1740 } 1741 1742 static int __init ahci_init(void) 1743 { 1744 return pci_register_driver(&ahci_pci_driver); 1745 } 1746 1747 static void __exit ahci_exit(void) 1748 { 1749 pci_unregister_driver(&ahci_pci_driver); 1750 } 1751 1752 1753 MODULE_AUTHOR("Jeff Garzik"); 1754 MODULE_DESCRIPTION("AHCI SATA low-level driver"); 1755 MODULE_LICENSE("GPL"); 1756 MODULE_DEVICE_TABLE(pci, ahci_pci_tbl); 1757 MODULE_VERSION(DRV_VERSION); 1758 1759 module_init(ahci_init); 1760 module_exit(ahci_exit); 1761