1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * sata_mv.c - Marvell SATA support 4 * 5 * Copyright 2008-2009: Marvell Corporation, all rights reserved. 6 * Copyright 2005: EMC Corporation, all rights reserved. 7 * Copyright 2005 Red Hat, Inc. All rights reserved. 8 * 9 * Originally written by Brett Russ. 10 * Extensive overhaul and enhancement by Mark Lord <mlord@pobox.com>. 11 * 12 * Please ALWAYS copy linux-ide@vger.kernel.org on emails. 13 */ 14 15 /* 16 * sata_mv TODO list: 17 * 18 * --> Develop a low-power-consumption strategy, and implement it. 19 * 20 * --> Add sysfs attributes for per-chip / per-HC IRQ coalescing thresholds. 21 * 22 * --> [Experiment, Marvell value added] Is it possible to use target 23 * mode to cross-connect two Linux boxes with Marvell cards? If so, 24 * creating LibATA target mode support would be very interesting. 25 * 26 * Target mode, for those without docs, is the ability to directly 27 * connect two SATA ports. 28 */ 29 30 /* 31 * 80x1-B2 errata PCI#11: 32 * 33 * Users of the 6041/6081 Rev.B2 chips (current is C0) 34 * should be careful to insert those cards only onto PCI-X bus #0, 35 * and only in device slots 0..7, not higher. The chips may not 36 * work correctly otherwise (note: this is a pretty rare condition). 37 */ 38 39 #include <linux/kernel.h> 40 #include <linux/module.h> 41 #include <linux/pci.h> 42 #include <linux/init.h> 43 #include <linux/blkdev.h> 44 #include <linux/delay.h> 45 #include <linux/interrupt.h> 46 #include <linux/dmapool.h> 47 #include <linux/dma-mapping.h> 48 #include <linux/device.h> 49 #include <linux/clk.h> 50 #include <linux/phy/phy.h> 51 #include <linux/platform_device.h> 52 #include <linux/ata_platform.h> 53 #include <linux/mbus.h> 54 #include <linux/bitops.h> 55 #include <linux/gfp.h> 56 #include <linux/of.h> 57 #include <linux/of_irq.h> 58 #include <scsi/scsi_host.h> 59 #include <scsi/scsi_cmnd.h> 60 #include <scsi/scsi_device.h> 61 #include <linux/libata.h> 62 63 #define DRV_NAME "sata_mv" 64 #define DRV_VERSION "1.28" 65 66 /* 67 * module options 68 */ 69 70 #ifdef CONFIG_PCI 71 static int msi; 72 module_param(msi, int, S_IRUGO); 73 MODULE_PARM_DESC(msi, "Enable use of PCI MSI (0=off, 1=on)"); 74 #endif 75 76 static int irq_coalescing_io_count; 77 module_param(irq_coalescing_io_count, int, S_IRUGO); 78 MODULE_PARM_DESC(irq_coalescing_io_count, 79 "IRQ coalescing I/O count threshold (0..255)"); 80 81 static int irq_coalescing_usecs; 82 module_param(irq_coalescing_usecs, int, S_IRUGO); 83 MODULE_PARM_DESC(irq_coalescing_usecs, 84 "IRQ coalescing time threshold in usecs"); 85 86 enum { 87 /* BAR's are enumerated in terms of pci_resource_start() terms */ 88 MV_PRIMARY_BAR = 0, /* offset 0x10: memory space */ 89 MV_IO_BAR = 2, /* offset 0x18: IO space */ 90 MV_MISC_BAR = 3, /* offset 0x1c: FLASH, NVRAM, SRAM */ 91 92 MV_MAJOR_REG_AREA_SZ = 0x10000, /* 64KB */ 93 MV_MINOR_REG_AREA_SZ = 0x2000, /* 8KB */ 94 95 /* For use with both IRQ coalescing methods ("all ports" or "per-HC" */ 96 COAL_CLOCKS_PER_USEC = 150, /* for calculating COAL_TIMEs */ 97 MAX_COAL_TIME_THRESHOLD = ((1 << 24) - 1), /* internal clocks count */ 98 MAX_COAL_IO_COUNT = 255, /* completed I/O count */ 99 100 MV_PCI_REG_BASE = 0, 101 102 /* 103 * Per-chip ("all ports") interrupt coalescing feature. 104 * This is only for GEN_II / GEN_IIE hardware. 105 * 106 * Coalescing defers the interrupt until either the IO_THRESHOLD 107 * (count of completed I/Os) is met, or the TIME_THRESHOLD is met. 108 */ 109 COAL_REG_BASE = 0x18000, 110 IRQ_COAL_CAUSE = (COAL_REG_BASE + 0x08), 111 ALL_PORTS_COAL_IRQ = (1 << 4), /* all ports irq event */ 112 113 IRQ_COAL_IO_THRESHOLD = (COAL_REG_BASE + 0xcc), 114 IRQ_COAL_TIME_THRESHOLD = (COAL_REG_BASE + 0xd0), 115 116 /* 117 * Registers for the (unused here) transaction coalescing feature: 118 */ 119 TRAN_COAL_CAUSE_LO = (COAL_REG_BASE + 0x88), 120 TRAN_COAL_CAUSE_HI = (COAL_REG_BASE + 0x8c), 121 122 SATAHC0_REG_BASE = 0x20000, 123 FLASH_CTL = 0x1046c, 124 GPIO_PORT_CTL = 0x104f0, 125 RESET_CFG = 0x180d8, 126 127 MV_PCI_REG_SZ = MV_MAJOR_REG_AREA_SZ, 128 MV_SATAHC_REG_SZ = MV_MAJOR_REG_AREA_SZ, 129 MV_SATAHC_ARBTR_REG_SZ = MV_MINOR_REG_AREA_SZ, /* arbiter */ 130 MV_PORT_REG_SZ = MV_MINOR_REG_AREA_SZ, 131 132 MV_MAX_Q_DEPTH = 32, 133 MV_MAX_Q_DEPTH_MASK = MV_MAX_Q_DEPTH - 1, 134 135 /* CRQB needs alignment on a 1KB boundary. Size == 1KB 136 * CRPB needs alignment on a 256B boundary. Size == 256B 137 * ePRD (SG) entries need alignment on a 16B boundary. Size == 16B 138 */ 139 MV_CRQB_Q_SZ = (32 * MV_MAX_Q_DEPTH), 140 MV_CRPB_Q_SZ = (8 * MV_MAX_Q_DEPTH), 141 MV_MAX_SG_CT = 256, 142 MV_SG_TBL_SZ = (16 * MV_MAX_SG_CT), 143 144 /* Determine hc from 0-7 port: hc = port >> MV_PORT_HC_SHIFT */ 145 MV_PORT_HC_SHIFT = 2, 146 MV_PORTS_PER_HC = (1 << MV_PORT_HC_SHIFT), /* 4 */ 147 /* Determine hc port from 0-7 port: hardport = port & MV_PORT_MASK */ 148 MV_PORT_MASK = (MV_PORTS_PER_HC - 1), /* 3 */ 149 150 /* Host Flags */ 151 MV_FLAG_DUAL_HC = (1 << 30), /* two SATA Host Controllers */ 152 153 MV_COMMON_FLAGS = ATA_FLAG_SATA | ATA_FLAG_PIO_POLLING, 154 155 MV_GEN_I_FLAGS = MV_COMMON_FLAGS | ATA_FLAG_NO_ATAPI, 156 157 MV_GEN_II_FLAGS = MV_COMMON_FLAGS | ATA_FLAG_NCQ | 158 ATA_FLAG_PMP | ATA_FLAG_ACPI_SATA, 159 160 MV_GEN_IIE_FLAGS = MV_GEN_II_FLAGS | ATA_FLAG_AN, 161 162 CRQB_FLAG_READ = (1 << 0), 163 CRQB_TAG_SHIFT = 1, 164 CRQB_IOID_SHIFT = 6, /* CRQB Gen-II/IIE IO Id shift */ 165 CRQB_PMP_SHIFT = 12, /* CRQB Gen-II/IIE PMP shift */ 166 CRQB_HOSTQ_SHIFT = 17, /* CRQB Gen-II/IIE HostQueTag shift */ 167 CRQB_CMD_ADDR_SHIFT = 8, 168 CRQB_CMD_CS = (0x2 << 11), 169 CRQB_CMD_LAST = (1 << 15), 170 171 CRPB_FLAG_STATUS_SHIFT = 8, 172 CRPB_IOID_SHIFT_6 = 5, /* CRPB Gen-II IO Id shift */ 173 CRPB_IOID_SHIFT_7 = 7, /* CRPB Gen-IIE IO Id shift */ 174 175 EPRD_FLAG_END_OF_TBL = (1 << 31), 176 177 /* PCI interface registers */ 178 179 MV_PCI_COMMAND = 0xc00, 180 MV_PCI_COMMAND_MWRCOM = (1 << 4), /* PCI Master Write Combining */ 181 MV_PCI_COMMAND_MRDTRIG = (1 << 7), /* PCI Master Read Trigger */ 182 183 PCI_MAIN_CMD_STS = 0xd30, 184 STOP_PCI_MASTER = (1 << 2), 185 PCI_MASTER_EMPTY = (1 << 3), 186 GLOB_SFT_RST = (1 << 4), 187 188 MV_PCI_MODE = 0xd00, 189 MV_PCI_MODE_MASK = 0x30, 190 191 MV_PCI_EXP_ROM_BAR_CTL = 0xd2c, 192 MV_PCI_DISC_TIMER = 0xd04, 193 MV_PCI_MSI_TRIGGER = 0xc38, 194 MV_PCI_SERR_MASK = 0xc28, 195 MV_PCI_XBAR_TMOUT = 0x1d04, 196 MV_PCI_ERR_LOW_ADDRESS = 0x1d40, 197 MV_PCI_ERR_HIGH_ADDRESS = 0x1d44, 198 MV_PCI_ERR_ATTRIBUTE = 0x1d48, 199 MV_PCI_ERR_COMMAND = 0x1d50, 200 201 PCI_IRQ_CAUSE = 0x1d58, 202 PCI_IRQ_MASK = 0x1d5c, 203 PCI_UNMASK_ALL_IRQS = 0x7fffff, /* bits 22-0 */ 204 205 PCIE_IRQ_CAUSE = 0x1900, 206 PCIE_IRQ_MASK = 0x1910, 207 PCIE_UNMASK_ALL_IRQS = 0x40a, /* assorted bits */ 208 209 /* Host Controller Main Interrupt Cause/Mask registers (1 per-chip) */ 210 PCI_HC_MAIN_IRQ_CAUSE = 0x1d60, 211 PCI_HC_MAIN_IRQ_MASK = 0x1d64, 212 SOC_HC_MAIN_IRQ_CAUSE = 0x20020, 213 SOC_HC_MAIN_IRQ_MASK = 0x20024, 214 ERR_IRQ = (1 << 0), /* shift by (2 * port #) */ 215 DONE_IRQ = (1 << 1), /* shift by (2 * port #) */ 216 HC0_IRQ_PEND = 0x1ff, /* bits 0-8 = HC0's ports */ 217 HC_SHIFT = 9, /* bits 9-17 = HC1's ports */ 218 DONE_IRQ_0_3 = 0x000000aa, /* DONE_IRQ ports 0,1,2,3 */ 219 DONE_IRQ_4_7 = (DONE_IRQ_0_3 << HC_SHIFT), /* 4,5,6,7 */ 220 PCI_ERR = (1 << 18), 221 TRAN_COAL_LO_DONE = (1 << 19), /* transaction coalescing */ 222 TRAN_COAL_HI_DONE = (1 << 20), /* transaction coalescing */ 223 PORTS_0_3_COAL_DONE = (1 << 8), /* HC0 IRQ coalescing */ 224 PORTS_4_7_COAL_DONE = (1 << 17), /* HC1 IRQ coalescing */ 225 ALL_PORTS_COAL_DONE = (1 << 21), /* GEN_II(E) IRQ coalescing */ 226 GPIO_INT = (1 << 22), 227 SELF_INT = (1 << 23), 228 TWSI_INT = (1 << 24), 229 HC_MAIN_RSVD = (0x7f << 25), /* bits 31-25 */ 230 HC_MAIN_RSVD_5 = (0x1fff << 19), /* bits 31-19 */ 231 HC_MAIN_RSVD_SOC = (0x3fffffb << 6), /* bits 31-9, 7-6 */ 232 233 /* SATAHC registers */ 234 HC_CFG = 0x00, 235 236 HC_IRQ_CAUSE = 0x14, 237 DMA_IRQ = (1 << 0), /* shift by port # */ 238 HC_COAL_IRQ = (1 << 4), /* IRQ coalescing */ 239 DEV_IRQ = (1 << 8), /* shift by port # */ 240 241 /* 242 * Per-HC (Host-Controller) interrupt coalescing feature. 243 * This is present on all chip generations. 244 * 245 * Coalescing defers the interrupt until either the IO_THRESHOLD 246 * (count of completed I/Os) is met, or the TIME_THRESHOLD is met. 247 */ 248 HC_IRQ_COAL_IO_THRESHOLD = 0x000c, 249 HC_IRQ_COAL_TIME_THRESHOLD = 0x0010, 250 251 SOC_LED_CTRL = 0x2c, 252 SOC_LED_CTRL_BLINK = (1 << 0), /* Active LED blink */ 253 SOC_LED_CTRL_ACT_PRESENCE = (1 << 2), /* Multiplex dev presence */ 254 /* with dev activity LED */ 255 256 /* Shadow block registers */ 257 SHD_BLK = 0x100, 258 SHD_CTL_AST = 0x20, /* ofs from SHD_BLK */ 259 260 /* SATA registers */ 261 SATA_STATUS = 0x300, /* ctrl, err regs follow status */ 262 SATA_ACTIVE = 0x350, 263 FIS_IRQ_CAUSE = 0x364, 264 FIS_IRQ_CAUSE_AN = (1 << 9), /* async notification */ 265 266 LTMODE = 0x30c, /* requires read-after-write */ 267 LTMODE_BIT8 = (1 << 8), /* unknown, but necessary */ 268 269 PHY_MODE2 = 0x330, 270 PHY_MODE3 = 0x310, 271 272 PHY_MODE4 = 0x314, /* requires read-after-write */ 273 PHY_MODE4_CFG_MASK = 0x00000003, /* phy internal config field */ 274 PHY_MODE4_CFG_VALUE = 0x00000001, /* phy internal config field */ 275 PHY_MODE4_RSVD_ZEROS = 0x5de3fffa, /* Gen2e always write zeros */ 276 PHY_MODE4_RSVD_ONES = 0x00000005, /* Gen2e always write ones */ 277 278 SATA_IFCTL = 0x344, 279 SATA_TESTCTL = 0x348, 280 SATA_IFSTAT = 0x34c, 281 VENDOR_UNIQUE_FIS = 0x35c, 282 283 FISCFG = 0x360, 284 FISCFG_WAIT_DEV_ERR = (1 << 8), /* wait for host on DevErr */ 285 FISCFG_SINGLE_SYNC = (1 << 16), /* SYNC on DMA activation */ 286 287 PHY_MODE9_GEN2 = 0x398, 288 PHY_MODE9_GEN1 = 0x39c, 289 PHYCFG_OFS = 0x3a0, /* only in 65n devices */ 290 291 MV5_PHY_MODE = 0x74, 292 MV5_LTMODE = 0x30, 293 MV5_PHY_CTL = 0x0C, 294 SATA_IFCFG = 0x050, 295 LP_PHY_CTL = 0x058, 296 LP_PHY_CTL_PIN_PU_PLL = (1 << 0), 297 LP_PHY_CTL_PIN_PU_RX = (1 << 1), 298 LP_PHY_CTL_PIN_PU_TX = (1 << 2), 299 LP_PHY_CTL_GEN_TX_3G = (1 << 5), 300 LP_PHY_CTL_GEN_RX_3G = (1 << 9), 301 302 MV_M2_PREAMP_MASK = 0x7e0, 303 304 /* Port registers */ 305 EDMA_CFG = 0, 306 EDMA_CFG_Q_DEPTH = 0x1f, /* max device queue depth */ 307 EDMA_CFG_NCQ = (1 << 5), /* for R/W FPDMA queued */ 308 EDMA_CFG_NCQ_GO_ON_ERR = (1 << 14), /* continue on error */ 309 EDMA_CFG_RD_BRST_EXT = (1 << 11), /* read burst 512B */ 310 EDMA_CFG_WR_BUFF_LEN = (1 << 13), /* write buffer 512B */ 311 EDMA_CFG_EDMA_FBS = (1 << 16), /* EDMA FIS-Based Switching */ 312 EDMA_CFG_FBS = (1 << 26), /* FIS-Based Switching */ 313 314 EDMA_ERR_IRQ_CAUSE = 0x8, 315 EDMA_ERR_IRQ_MASK = 0xc, 316 EDMA_ERR_D_PAR = (1 << 0), /* UDMA data parity err */ 317 EDMA_ERR_PRD_PAR = (1 << 1), /* UDMA PRD parity err */ 318 EDMA_ERR_DEV = (1 << 2), /* device error */ 319 EDMA_ERR_DEV_DCON = (1 << 3), /* device disconnect */ 320 EDMA_ERR_DEV_CON = (1 << 4), /* device connected */ 321 EDMA_ERR_SERR = (1 << 5), /* SError bits [WBDST] raised */ 322 EDMA_ERR_SELF_DIS = (1 << 7), /* Gen II/IIE self-disable */ 323 EDMA_ERR_SELF_DIS_5 = (1 << 8), /* Gen I self-disable */ 324 EDMA_ERR_BIST_ASYNC = (1 << 8), /* BIST FIS or Async Notify */ 325 EDMA_ERR_TRANS_IRQ_7 = (1 << 8), /* Gen IIE transprt layer irq */ 326 EDMA_ERR_CRQB_PAR = (1 << 9), /* CRQB parity error */ 327 EDMA_ERR_CRPB_PAR = (1 << 10), /* CRPB parity error */ 328 EDMA_ERR_INTRL_PAR = (1 << 11), /* internal parity error */ 329 EDMA_ERR_IORDY = (1 << 12), /* IORdy timeout */ 330 331 EDMA_ERR_LNK_CTRL_RX = (0xf << 13), /* link ctrl rx error */ 332 EDMA_ERR_LNK_CTRL_RX_0 = (1 << 13), /* transient: CRC err */ 333 EDMA_ERR_LNK_CTRL_RX_1 = (1 << 14), /* transient: FIFO err */ 334 EDMA_ERR_LNK_CTRL_RX_2 = (1 << 15), /* fatal: caught SYNC */ 335 EDMA_ERR_LNK_CTRL_RX_3 = (1 << 16), /* transient: FIS rx err */ 336 337 EDMA_ERR_LNK_DATA_RX = (0xf << 17), /* link data rx error */ 338 339 EDMA_ERR_LNK_CTRL_TX = (0x1f << 21), /* link ctrl tx error */ 340 EDMA_ERR_LNK_CTRL_TX_0 = (1 << 21), /* transient: CRC err */ 341 EDMA_ERR_LNK_CTRL_TX_1 = (1 << 22), /* transient: FIFO err */ 342 EDMA_ERR_LNK_CTRL_TX_2 = (1 << 23), /* transient: caught SYNC */ 343 EDMA_ERR_LNK_CTRL_TX_3 = (1 << 24), /* transient: caught DMAT */ 344 EDMA_ERR_LNK_CTRL_TX_4 = (1 << 25), /* transient: FIS collision */ 345 346 EDMA_ERR_LNK_DATA_TX = (0x1f << 26), /* link data tx error */ 347 348 EDMA_ERR_TRANS_PROTO = (1 << 31), /* transport protocol error */ 349 EDMA_ERR_OVERRUN_5 = (1 << 5), 350 EDMA_ERR_UNDERRUN_5 = (1 << 6), 351 352 EDMA_ERR_IRQ_TRANSIENT = EDMA_ERR_LNK_CTRL_RX_0 | 353 EDMA_ERR_LNK_CTRL_RX_1 | 354 EDMA_ERR_LNK_CTRL_RX_3 | 355 EDMA_ERR_LNK_CTRL_TX, 356 357 EDMA_EH_FREEZE = EDMA_ERR_D_PAR | 358 EDMA_ERR_PRD_PAR | 359 EDMA_ERR_DEV_DCON | 360 EDMA_ERR_DEV_CON | 361 EDMA_ERR_SERR | 362 EDMA_ERR_SELF_DIS | 363 EDMA_ERR_CRQB_PAR | 364 EDMA_ERR_CRPB_PAR | 365 EDMA_ERR_INTRL_PAR | 366 EDMA_ERR_IORDY | 367 EDMA_ERR_LNK_CTRL_RX_2 | 368 EDMA_ERR_LNK_DATA_RX | 369 EDMA_ERR_LNK_DATA_TX | 370 EDMA_ERR_TRANS_PROTO, 371 372 EDMA_EH_FREEZE_5 = EDMA_ERR_D_PAR | 373 EDMA_ERR_PRD_PAR | 374 EDMA_ERR_DEV_DCON | 375 EDMA_ERR_DEV_CON | 376 EDMA_ERR_OVERRUN_5 | 377 EDMA_ERR_UNDERRUN_5 | 378 EDMA_ERR_SELF_DIS_5 | 379 EDMA_ERR_CRQB_PAR | 380 EDMA_ERR_CRPB_PAR | 381 EDMA_ERR_INTRL_PAR | 382 EDMA_ERR_IORDY, 383 384 EDMA_REQ_Q_BASE_HI = 0x10, 385 EDMA_REQ_Q_IN_PTR = 0x14, /* also contains BASE_LO */ 386 387 EDMA_REQ_Q_OUT_PTR = 0x18, 388 EDMA_REQ_Q_PTR_SHIFT = 5, 389 390 EDMA_RSP_Q_BASE_HI = 0x1c, 391 EDMA_RSP_Q_IN_PTR = 0x20, 392 EDMA_RSP_Q_OUT_PTR = 0x24, /* also contains BASE_LO */ 393 EDMA_RSP_Q_PTR_SHIFT = 3, 394 395 EDMA_CMD = 0x28, /* EDMA command register */ 396 EDMA_EN = (1 << 0), /* enable EDMA */ 397 EDMA_DS = (1 << 1), /* disable EDMA; self-negated */ 398 EDMA_RESET = (1 << 2), /* reset eng/trans/link/phy */ 399 400 EDMA_STATUS = 0x30, /* EDMA engine status */ 401 EDMA_STATUS_CACHE_EMPTY = (1 << 6), /* GenIIe command cache empty */ 402 EDMA_STATUS_IDLE = (1 << 7), /* GenIIe EDMA enabled/idle */ 403 404 EDMA_IORDY_TMOUT = 0x34, 405 EDMA_ARB_CFG = 0x38, 406 407 EDMA_HALTCOND = 0x60, /* GenIIe halt conditions */ 408 EDMA_UNKNOWN_RSVD = 0x6C, /* GenIIe unknown/reserved */ 409 410 BMDMA_CMD = 0x224, /* bmdma command register */ 411 BMDMA_STATUS = 0x228, /* bmdma status register */ 412 BMDMA_PRD_LOW = 0x22c, /* bmdma PRD addr 31:0 */ 413 BMDMA_PRD_HIGH = 0x230, /* bmdma PRD addr 63:32 */ 414 415 /* Host private flags (hp_flags) */ 416 MV_HP_FLAG_MSI = (1 << 0), 417 MV_HP_ERRATA_50XXB0 = (1 << 1), 418 MV_HP_ERRATA_50XXB2 = (1 << 2), 419 MV_HP_ERRATA_60X1B2 = (1 << 3), 420 MV_HP_ERRATA_60X1C0 = (1 << 4), 421 MV_HP_GEN_I = (1 << 6), /* Generation I: 50xx */ 422 MV_HP_GEN_II = (1 << 7), /* Generation II: 60xx */ 423 MV_HP_GEN_IIE = (1 << 8), /* Generation IIE: 6042/7042 */ 424 MV_HP_PCIE = (1 << 9), /* PCIe bus/regs: 7042 */ 425 MV_HP_CUT_THROUGH = (1 << 10), /* can use EDMA cut-through */ 426 MV_HP_FLAG_SOC = (1 << 11), /* SystemOnChip, no PCI */ 427 MV_HP_QUIRK_LED_BLINK_EN = (1 << 12), /* is led blinking enabled? */ 428 MV_HP_FIX_LP_PHY_CTL = (1 << 13), /* fix speed in LP_PHY_CTL ? */ 429 430 /* Port private flags (pp_flags) */ 431 MV_PP_FLAG_EDMA_EN = (1 << 0), /* is EDMA engine enabled? */ 432 MV_PP_FLAG_NCQ_EN = (1 << 1), /* is EDMA set up for NCQ? */ 433 MV_PP_FLAG_FBS_EN = (1 << 2), /* is EDMA set up for FBS? */ 434 MV_PP_FLAG_DELAYED_EH = (1 << 3), /* delayed dev err handling */ 435 MV_PP_FLAG_FAKE_ATA_BUSY = (1 << 4), /* ignore initial ATA_DRDY */ 436 }; 437 438 #define IS_GEN_I(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_I) 439 #define IS_GEN_II(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_II) 440 #define IS_GEN_IIE(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_IIE) 441 #define IS_PCIE(hpriv) ((hpriv)->hp_flags & MV_HP_PCIE) 442 #define IS_SOC(hpriv) ((hpriv)->hp_flags & MV_HP_FLAG_SOC) 443 444 #define WINDOW_CTRL(i) (0x20030 + ((i) << 4)) 445 #define WINDOW_BASE(i) (0x20034 + ((i) << 4)) 446 447 enum { 448 /* DMA boundary 0xffff is required by the s/g splitting 449 * we need on /length/ in mv_fill-sg(). 450 */ 451 MV_DMA_BOUNDARY = 0xffffU, 452 453 /* mask of register bits containing lower 32 bits 454 * of EDMA request queue DMA address 455 */ 456 EDMA_REQ_Q_BASE_LO_MASK = 0xfffffc00U, 457 458 /* ditto, for response queue */ 459 EDMA_RSP_Q_BASE_LO_MASK = 0xffffff00U, 460 }; 461 462 enum chip_type { 463 chip_504x, 464 chip_508x, 465 chip_5080, 466 chip_604x, 467 chip_608x, 468 chip_6042, 469 chip_7042, 470 chip_soc, 471 }; 472 473 /* Command ReQuest Block: 32B */ 474 struct mv_crqb { 475 __le32 sg_addr; 476 __le32 sg_addr_hi; 477 __le16 ctrl_flags; 478 __le16 ata_cmd[11]; 479 }; 480 481 struct mv_crqb_iie { 482 __le32 addr; 483 __le32 addr_hi; 484 __le32 flags; 485 __le32 len; 486 __le32 ata_cmd[4]; 487 }; 488 489 /* Command ResPonse Block: 8B */ 490 struct mv_crpb { 491 __le16 id; 492 __le16 flags; 493 __le32 tmstmp; 494 }; 495 496 /* EDMA Physical Region Descriptor (ePRD); A.K.A. SG */ 497 struct mv_sg { 498 __le32 addr; 499 __le32 flags_size; 500 __le32 addr_hi; 501 __le32 reserved; 502 }; 503 504 /* 505 * We keep a local cache of a few frequently accessed port 506 * registers here, to avoid having to read them (very slow) 507 * when switching between EDMA and non-EDMA modes. 508 */ 509 struct mv_cached_regs { 510 u32 fiscfg; 511 u32 ltmode; 512 u32 haltcond; 513 u32 unknown_rsvd; 514 }; 515 516 struct mv_port_priv { 517 struct mv_crqb *crqb; 518 dma_addr_t crqb_dma; 519 struct mv_crpb *crpb; 520 dma_addr_t crpb_dma; 521 struct mv_sg *sg_tbl[MV_MAX_Q_DEPTH]; 522 dma_addr_t sg_tbl_dma[MV_MAX_Q_DEPTH]; 523 524 unsigned int req_idx; 525 unsigned int resp_idx; 526 527 u32 pp_flags; 528 struct mv_cached_regs cached; 529 unsigned int delayed_eh_pmp_map; 530 }; 531 532 struct mv_port_signal { 533 u32 amps; 534 u32 pre; 535 }; 536 537 struct mv_host_priv { 538 u32 hp_flags; 539 unsigned int board_idx; 540 u32 main_irq_mask; 541 struct mv_port_signal signal[8]; 542 const struct mv_hw_ops *ops; 543 int n_ports; 544 void __iomem *base; 545 void __iomem *main_irq_cause_addr; 546 void __iomem *main_irq_mask_addr; 547 u32 irq_cause_offset; 548 u32 irq_mask_offset; 549 u32 unmask_all_irqs; 550 551 /* 552 * Needed on some devices that require their clocks to be enabled. 553 * These are optional: if the platform device does not have any 554 * clocks, they won't be used. Also, if the underlying hardware 555 * does not support the common clock framework (CONFIG_HAVE_CLK=n), 556 * all the clock operations become no-ops (see clk.h). 557 */ 558 struct clk *clk; 559 struct clk **port_clks; 560 /* 561 * Some devices have a SATA PHY which can be enabled/disabled 562 * in order to save power. These are optional: if the platform 563 * devices does not have any phy, they won't be used. 564 */ 565 struct phy **port_phys; 566 /* 567 * These consistent DMA memory pools give us guaranteed 568 * alignment for hardware-accessed data structures, 569 * and less memory waste in accomplishing the alignment. 570 */ 571 struct dma_pool *crqb_pool; 572 struct dma_pool *crpb_pool; 573 struct dma_pool *sg_tbl_pool; 574 }; 575 576 struct mv_hw_ops { 577 void (*phy_errata)(struct mv_host_priv *hpriv, void __iomem *mmio, 578 unsigned int port); 579 void (*enable_leds)(struct mv_host_priv *hpriv, void __iomem *mmio); 580 void (*read_preamp)(struct mv_host_priv *hpriv, int idx, 581 void __iomem *mmio); 582 int (*reset_hc)(struct ata_host *host, void __iomem *mmio, 583 unsigned int n_hc); 584 void (*reset_flash)(struct mv_host_priv *hpriv, void __iomem *mmio); 585 void (*reset_bus)(struct ata_host *host, void __iomem *mmio); 586 }; 587 588 static int mv_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val); 589 static int mv_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val); 590 static int mv5_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val); 591 static int mv5_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val); 592 static int mv_port_start(struct ata_port *ap); 593 static void mv_port_stop(struct ata_port *ap); 594 static int mv_qc_defer(struct ata_queued_cmd *qc); 595 static enum ata_completion_errors mv_qc_prep(struct ata_queued_cmd *qc); 596 static enum ata_completion_errors mv_qc_prep_iie(struct ata_queued_cmd *qc); 597 static unsigned int mv_qc_issue(struct ata_queued_cmd *qc); 598 static int mv_hardreset(struct ata_link *link, unsigned int *class, 599 unsigned long deadline); 600 static void mv_eh_freeze(struct ata_port *ap); 601 static void mv_eh_thaw(struct ata_port *ap); 602 static void mv6_dev_config(struct ata_device *dev); 603 604 static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio, 605 unsigned int port); 606 static void mv5_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio); 607 static void mv5_read_preamp(struct mv_host_priv *hpriv, int idx, 608 void __iomem *mmio); 609 static int mv5_reset_hc(struct ata_host *host, void __iomem *mmio, 610 unsigned int n_hc); 611 static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio); 612 static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio); 613 614 static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio, 615 unsigned int port); 616 static void mv6_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio); 617 static void mv6_read_preamp(struct mv_host_priv *hpriv, int idx, 618 void __iomem *mmio); 619 static int mv6_reset_hc(struct ata_host *host, void __iomem *mmio, 620 unsigned int n_hc); 621 static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio); 622 static void mv_soc_enable_leds(struct mv_host_priv *hpriv, 623 void __iomem *mmio); 624 static void mv_soc_read_preamp(struct mv_host_priv *hpriv, int idx, 625 void __iomem *mmio); 626 static int mv_soc_reset_hc(struct ata_host *host, 627 void __iomem *mmio, unsigned int n_hc); 628 static void mv_soc_reset_flash(struct mv_host_priv *hpriv, 629 void __iomem *mmio); 630 static void mv_soc_reset_bus(struct ata_host *host, void __iomem *mmio); 631 static void mv_soc_65n_phy_errata(struct mv_host_priv *hpriv, 632 void __iomem *mmio, unsigned int port); 633 static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio); 634 static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio, 635 unsigned int port_no); 636 static int mv_stop_edma(struct ata_port *ap); 637 static int mv_stop_edma_engine(void __iomem *port_mmio); 638 static void mv_edma_cfg(struct ata_port *ap, int want_ncq, int want_edma); 639 640 static void mv_pmp_select(struct ata_port *ap, int pmp); 641 static int mv_pmp_hardreset(struct ata_link *link, unsigned int *class, 642 unsigned long deadline); 643 static int mv_softreset(struct ata_link *link, unsigned int *class, 644 unsigned long deadline); 645 static void mv_pmp_error_handler(struct ata_port *ap); 646 static void mv_process_crpb_entries(struct ata_port *ap, 647 struct mv_port_priv *pp); 648 649 static void mv_sff_irq_clear(struct ata_port *ap); 650 static int mv_check_atapi_dma(struct ata_queued_cmd *qc); 651 static void mv_bmdma_setup(struct ata_queued_cmd *qc); 652 static void mv_bmdma_start(struct ata_queued_cmd *qc); 653 static void mv_bmdma_stop(struct ata_queued_cmd *qc); 654 static u8 mv_bmdma_status(struct ata_port *ap); 655 static u8 mv_sff_check_status(struct ata_port *ap); 656 657 /* .sg_tablesize is (MV_MAX_SG_CT / 2) in the structures below 658 * because we have to allow room for worst case splitting of 659 * PRDs for 64K boundaries in mv_fill_sg(). 660 */ 661 #ifdef CONFIG_PCI 662 static const struct scsi_host_template mv5_sht = { 663 ATA_BASE_SHT(DRV_NAME), 664 .sg_tablesize = MV_MAX_SG_CT / 2, 665 .dma_boundary = MV_DMA_BOUNDARY, 666 }; 667 #endif 668 static const struct scsi_host_template mv6_sht = { 669 __ATA_BASE_SHT(DRV_NAME), 670 .can_queue = MV_MAX_Q_DEPTH - 1, 671 .sg_tablesize = MV_MAX_SG_CT / 2, 672 .dma_boundary = MV_DMA_BOUNDARY, 673 .sdev_groups = ata_ncq_sdev_groups, 674 .change_queue_depth = ata_scsi_change_queue_depth, 675 .tag_alloc_policy_rr = true, 676 .sdev_configure = ata_scsi_sdev_configure 677 }; 678 679 static struct ata_port_operations mv5_ops = { 680 .inherits = &ata_sff_port_ops, 681 682 .lost_interrupt = ATA_OP_NULL, 683 684 .qc_defer = mv_qc_defer, 685 .qc_prep = mv_qc_prep, 686 .qc_issue = mv_qc_issue, 687 688 .freeze = mv_eh_freeze, 689 .thaw = mv_eh_thaw, 690 .reset.hardreset = mv_hardreset, 691 692 .scr_read = mv5_scr_read, 693 .scr_write = mv5_scr_write, 694 695 .port_start = mv_port_start, 696 .port_stop = mv_port_stop, 697 }; 698 699 static struct ata_port_operations mv6_ops = { 700 .inherits = &ata_bmdma_port_ops, 701 702 .lost_interrupt = ATA_OP_NULL, 703 704 .qc_defer = mv_qc_defer, 705 .qc_prep = mv_qc_prep, 706 .qc_issue = mv_qc_issue, 707 708 .dev_config = mv6_dev_config, 709 710 .freeze = mv_eh_freeze, 711 .thaw = mv_eh_thaw, 712 .reset.hardreset = mv_hardreset, 713 .reset.softreset = mv_softreset, 714 .pmp_reset.hardreset = mv_pmp_hardreset, 715 .pmp_reset.softreset = mv_softreset, 716 .error_handler = mv_pmp_error_handler, 717 718 .scr_read = mv_scr_read, 719 .scr_write = mv_scr_write, 720 721 .sff_check_status = mv_sff_check_status, 722 .sff_irq_clear = mv_sff_irq_clear, 723 .check_atapi_dma = mv_check_atapi_dma, 724 .bmdma_setup = mv_bmdma_setup, 725 .bmdma_start = mv_bmdma_start, 726 .bmdma_stop = mv_bmdma_stop, 727 .bmdma_status = mv_bmdma_status, 728 729 .port_start = mv_port_start, 730 .port_stop = mv_port_stop, 731 }; 732 733 static struct ata_port_operations mv_iie_ops = { 734 .inherits = &mv6_ops, 735 .dev_config = ATA_OP_NULL, 736 .qc_prep = mv_qc_prep_iie, 737 }; 738 739 static const struct ata_port_info mv_port_info[] = { 740 { /* chip_504x */ 741 .flags = MV_GEN_I_FLAGS, 742 .pio_mask = ATA_PIO4, 743 .udma_mask = ATA_UDMA6, 744 .port_ops = &mv5_ops, 745 }, 746 { /* chip_508x */ 747 .flags = MV_GEN_I_FLAGS | MV_FLAG_DUAL_HC, 748 .pio_mask = ATA_PIO4, 749 .udma_mask = ATA_UDMA6, 750 .port_ops = &mv5_ops, 751 }, 752 { /* chip_5080 */ 753 .flags = MV_GEN_I_FLAGS | MV_FLAG_DUAL_HC, 754 .pio_mask = ATA_PIO4, 755 .udma_mask = ATA_UDMA6, 756 .port_ops = &mv5_ops, 757 }, 758 { /* chip_604x */ 759 .flags = MV_GEN_II_FLAGS, 760 .pio_mask = ATA_PIO4, 761 .udma_mask = ATA_UDMA6, 762 .port_ops = &mv6_ops, 763 }, 764 { /* chip_608x */ 765 .flags = MV_GEN_II_FLAGS | MV_FLAG_DUAL_HC, 766 .pio_mask = ATA_PIO4, 767 .udma_mask = ATA_UDMA6, 768 .port_ops = &mv6_ops, 769 }, 770 { /* chip_6042 */ 771 .flags = MV_GEN_IIE_FLAGS, 772 .pio_mask = ATA_PIO4, 773 .udma_mask = ATA_UDMA6, 774 .port_ops = &mv_iie_ops, 775 }, 776 { /* chip_7042 */ 777 .flags = MV_GEN_IIE_FLAGS, 778 .pio_mask = ATA_PIO4, 779 .udma_mask = ATA_UDMA6, 780 .port_ops = &mv_iie_ops, 781 }, 782 { /* chip_soc */ 783 .flags = MV_GEN_IIE_FLAGS, 784 .pio_mask = ATA_PIO4, 785 .udma_mask = ATA_UDMA6, 786 .port_ops = &mv_iie_ops, 787 }, 788 }; 789 790 static const struct mv_hw_ops mv5xxx_ops = { 791 .phy_errata = mv5_phy_errata, 792 .enable_leds = mv5_enable_leds, 793 .read_preamp = mv5_read_preamp, 794 .reset_hc = mv5_reset_hc, 795 .reset_flash = mv5_reset_flash, 796 .reset_bus = mv5_reset_bus, 797 }; 798 799 static const struct mv_hw_ops mv6xxx_ops = { 800 .phy_errata = mv6_phy_errata, 801 .enable_leds = mv6_enable_leds, 802 .read_preamp = mv6_read_preamp, 803 .reset_hc = mv6_reset_hc, 804 .reset_flash = mv6_reset_flash, 805 .reset_bus = mv_reset_pci_bus, 806 }; 807 808 static const struct mv_hw_ops mv_soc_ops = { 809 .phy_errata = mv6_phy_errata, 810 .enable_leds = mv_soc_enable_leds, 811 .read_preamp = mv_soc_read_preamp, 812 .reset_hc = mv_soc_reset_hc, 813 .reset_flash = mv_soc_reset_flash, 814 .reset_bus = mv_soc_reset_bus, 815 }; 816 817 static const struct mv_hw_ops mv_soc_65n_ops = { 818 .phy_errata = mv_soc_65n_phy_errata, 819 .enable_leds = mv_soc_enable_leds, 820 .reset_hc = mv_soc_reset_hc, 821 .reset_flash = mv_soc_reset_flash, 822 .reset_bus = mv_soc_reset_bus, 823 }; 824 825 /* 826 * Functions 827 */ 828 829 static inline void writelfl(unsigned long data, void __iomem *addr) 830 { 831 writel(data, addr); 832 (void) readl(addr); /* flush to avoid PCI posted write */ 833 } 834 835 static inline unsigned int mv_hc_from_port(unsigned int port) 836 { 837 return port >> MV_PORT_HC_SHIFT; 838 } 839 840 static inline unsigned int mv_hardport_from_port(unsigned int port) 841 { 842 return port & MV_PORT_MASK; 843 } 844 845 /* 846 * Consolidate some rather tricky bit shift calculations. 847 * This is hot-path stuff, so not a function. 848 * Simple code, with two return values, so macro rather than inline. 849 * 850 * port is the sole input, in range 0..7. 851 * shift is one output, for use with main_irq_cause / main_irq_mask registers. 852 * hardport is the other output, in range 0..3. 853 * 854 * Note that port and hardport may be the same variable in some cases. 855 */ 856 #define MV_PORT_TO_SHIFT_AND_HARDPORT(port, shift, hardport) \ 857 { \ 858 shift = mv_hc_from_port(port) * HC_SHIFT; \ 859 hardport = mv_hardport_from_port(port); \ 860 shift += hardport * 2; \ 861 } 862 863 static inline void __iomem *mv_hc_base(void __iomem *base, unsigned int hc) 864 { 865 return (base + SATAHC0_REG_BASE + (hc * MV_SATAHC_REG_SZ)); 866 } 867 868 static inline void __iomem *mv_hc_base_from_port(void __iomem *base, 869 unsigned int port) 870 { 871 return mv_hc_base(base, mv_hc_from_port(port)); 872 } 873 874 static inline void __iomem *mv_port_base(void __iomem *base, unsigned int port) 875 { 876 return mv_hc_base_from_port(base, port) + 877 MV_SATAHC_ARBTR_REG_SZ + 878 (mv_hardport_from_port(port) * MV_PORT_REG_SZ); 879 } 880 881 static void __iomem *mv5_phy_base(void __iomem *mmio, unsigned int port) 882 { 883 void __iomem *hc_mmio = mv_hc_base_from_port(mmio, port); 884 unsigned long ofs = (mv_hardport_from_port(port) + 1) * 0x100UL; 885 886 return hc_mmio + ofs; 887 } 888 889 static inline void __iomem *mv_host_base(struct ata_host *host) 890 { 891 struct mv_host_priv *hpriv = host->private_data; 892 return hpriv->base; 893 } 894 895 static inline void __iomem *mv_ap_base(struct ata_port *ap) 896 { 897 return mv_port_base(mv_host_base(ap->host), ap->port_no); 898 } 899 900 static inline int mv_get_hc_count(unsigned long port_flags) 901 { 902 return ((port_flags & MV_FLAG_DUAL_HC) ? 2 : 1); 903 } 904 905 /** 906 * mv_save_cached_regs - (re-)initialize cached port registers 907 * @ap: the port whose registers we are caching 908 * 909 * Initialize the local cache of port registers, 910 * so that reading them over and over again can 911 * be avoided on the hotter paths of this driver. 912 * This saves a few microseconds each time we switch 913 * to/from EDMA mode to perform (eg.) a drive cache flush. 914 */ 915 static void mv_save_cached_regs(struct ata_port *ap) 916 { 917 void __iomem *port_mmio = mv_ap_base(ap); 918 struct mv_port_priv *pp = ap->private_data; 919 920 pp->cached.fiscfg = readl(port_mmio + FISCFG); 921 pp->cached.ltmode = readl(port_mmio + LTMODE); 922 pp->cached.haltcond = readl(port_mmio + EDMA_HALTCOND); 923 pp->cached.unknown_rsvd = readl(port_mmio + EDMA_UNKNOWN_RSVD); 924 } 925 926 /** 927 * mv_write_cached_reg - write to a cached port register 928 * @addr: hardware address of the register 929 * @old: pointer to cached value of the register 930 * @new: new value for the register 931 * 932 * Write a new value to a cached register, 933 * but only if the value is different from before. 934 */ 935 static inline void mv_write_cached_reg(void __iomem *addr, u32 *old, u32 new) 936 { 937 if (new != *old) { 938 unsigned long laddr; 939 *old = new; 940 /* 941 * Workaround for 88SX60x1-B2 FEr SATA#13: 942 * Read-after-write is needed to prevent generating 64-bit 943 * write cycles on the PCI bus for SATA interface registers 944 * at offsets ending in 0x4 or 0xc. 945 * 946 * Looks like a lot of fuss, but it avoids an unnecessary 947 * +1 usec read-after-write delay for unaffected registers. 948 */ 949 laddr = (unsigned long)addr & 0xffff; 950 if (laddr >= 0x300 && laddr <= 0x33c) { 951 laddr &= 0x000f; 952 if (laddr == 0x4 || laddr == 0xc) { 953 writelfl(new, addr); /* read after write */ 954 return; 955 } 956 } 957 writel(new, addr); /* unaffected by the errata */ 958 } 959 } 960 961 static void mv_set_edma_ptrs(void __iomem *port_mmio, 962 struct mv_host_priv *hpriv, 963 struct mv_port_priv *pp) 964 { 965 u32 index; 966 967 /* 968 * initialize request queue 969 */ 970 pp->req_idx &= MV_MAX_Q_DEPTH_MASK; /* paranoia */ 971 index = pp->req_idx << EDMA_REQ_Q_PTR_SHIFT; 972 973 WARN_ON(pp->crqb_dma & 0x3ff); 974 writel((pp->crqb_dma >> 16) >> 16, port_mmio + EDMA_REQ_Q_BASE_HI); 975 writelfl((pp->crqb_dma & EDMA_REQ_Q_BASE_LO_MASK) | index, 976 port_mmio + EDMA_REQ_Q_IN_PTR); 977 writelfl(index, port_mmio + EDMA_REQ_Q_OUT_PTR); 978 979 /* 980 * initialize response queue 981 */ 982 pp->resp_idx &= MV_MAX_Q_DEPTH_MASK; /* paranoia */ 983 index = pp->resp_idx << EDMA_RSP_Q_PTR_SHIFT; 984 985 WARN_ON(pp->crpb_dma & 0xff); 986 writel((pp->crpb_dma >> 16) >> 16, port_mmio + EDMA_RSP_Q_BASE_HI); 987 writelfl(index, port_mmio + EDMA_RSP_Q_IN_PTR); 988 writelfl((pp->crpb_dma & EDMA_RSP_Q_BASE_LO_MASK) | index, 989 port_mmio + EDMA_RSP_Q_OUT_PTR); 990 } 991 992 static void mv_write_main_irq_mask(u32 mask, struct mv_host_priv *hpriv) 993 { 994 /* 995 * When writing to the main_irq_mask in hardware, 996 * we must ensure exclusivity between the interrupt coalescing bits 997 * and the corresponding individual port DONE_IRQ bits. 998 * 999 * Note that this register is really an "IRQ enable" register, 1000 * not an "IRQ mask" register as Marvell's naming might suggest. 1001 */ 1002 if (mask & (ALL_PORTS_COAL_DONE | PORTS_0_3_COAL_DONE)) 1003 mask &= ~DONE_IRQ_0_3; 1004 if (mask & (ALL_PORTS_COAL_DONE | PORTS_4_7_COAL_DONE)) 1005 mask &= ~DONE_IRQ_4_7; 1006 writelfl(mask, hpriv->main_irq_mask_addr); 1007 } 1008 1009 static void mv_set_main_irq_mask(struct ata_host *host, 1010 u32 disable_bits, u32 enable_bits) 1011 { 1012 struct mv_host_priv *hpriv = host->private_data; 1013 u32 old_mask, new_mask; 1014 1015 old_mask = hpriv->main_irq_mask; 1016 new_mask = (old_mask & ~disable_bits) | enable_bits; 1017 if (new_mask != old_mask) { 1018 hpriv->main_irq_mask = new_mask; 1019 mv_write_main_irq_mask(new_mask, hpriv); 1020 } 1021 } 1022 1023 static void mv_enable_port_irqs(struct ata_port *ap, 1024 unsigned int port_bits) 1025 { 1026 unsigned int shift, hardport, port = ap->port_no; 1027 u32 disable_bits, enable_bits; 1028 1029 MV_PORT_TO_SHIFT_AND_HARDPORT(port, shift, hardport); 1030 1031 disable_bits = (DONE_IRQ | ERR_IRQ) << shift; 1032 enable_bits = port_bits << shift; 1033 mv_set_main_irq_mask(ap->host, disable_bits, enable_bits); 1034 } 1035 1036 static void mv_clear_and_enable_port_irqs(struct ata_port *ap, 1037 void __iomem *port_mmio, 1038 unsigned int port_irqs) 1039 { 1040 struct mv_host_priv *hpriv = ap->host->private_data; 1041 int hardport = mv_hardport_from_port(ap->port_no); 1042 void __iomem *hc_mmio = mv_hc_base_from_port( 1043 mv_host_base(ap->host), ap->port_no); 1044 u32 hc_irq_cause; 1045 1046 /* clear EDMA event indicators, if any */ 1047 writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE); 1048 1049 /* clear pending irq events */ 1050 hc_irq_cause = ~((DEV_IRQ | DMA_IRQ) << hardport); 1051 writelfl(hc_irq_cause, hc_mmio + HC_IRQ_CAUSE); 1052 1053 /* clear FIS IRQ Cause */ 1054 if (IS_GEN_IIE(hpriv)) 1055 writelfl(0, port_mmio + FIS_IRQ_CAUSE); 1056 1057 mv_enable_port_irqs(ap, port_irqs); 1058 } 1059 1060 static void mv_set_irq_coalescing(struct ata_host *host, 1061 unsigned int count, unsigned int usecs) 1062 { 1063 struct mv_host_priv *hpriv = host->private_data; 1064 void __iomem *mmio = hpriv->base, *hc_mmio; 1065 u32 coal_enable = 0; 1066 unsigned long flags; 1067 unsigned int clks, is_dual_hc = hpriv->n_ports > MV_PORTS_PER_HC; 1068 const u32 coal_disable = PORTS_0_3_COAL_DONE | PORTS_4_7_COAL_DONE | 1069 ALL_PORTS_COAL_DONE; 1070 1071 /* Disable IRQ coalescing if either threshold is zero */ 1072 if (!usecs || !count) { 1073 clks = count = 0; 1074 } else { 1075 /* Respect maximum limits of the hardware */ 1076 clks = usecs * COAL_CLOCKS_PER_USEC; 1077 if (clks > MAX_COAL_TIME_THRESHOLD) 1078 clks = MAX_COAL_TIME_THRESHOLD; 1079 if (count > MAX_COAL_IO_COUNT) 1080 count = MAX_COAL_IO_COUNT; 1081 } 1082 1083 spin_lock_irqsave(&host->lock, flags); 1084 mv_set_main_irq_mask(host, coal_disable, 0); 1085 1086 if (is_dual_hc && !IS_GEN_I(hpriv)) { 1087 /* 1088 * GEN_II/GEN_IIE with dual host controllers: 1089 * one set of global thresholds for the entire chip. 1090 */ 1091 writel(clks, mmio + IRQ_COAL_TIME_THRESHOLD); 1092 writel(count, mmio + IRQ_COAL_IO_THRESHOLD); 1093 /* clear leftover coal IRQ bit */ 1094 writel(~ALL_PORTS_COAL_IRQ, mmio + IRQ_COAL_CAUSE); 1095 if (count) 1096 coal_enable = ALL_PORTS_COAL_DONE; 1097 clks = count = 0; /* force clearing of regular regs below */ 1098 } 1099 1100 /* 1101 * All chips: independent thresholds for each HC on the chip. 1102 */ 1103 hc_mmio = mv_hc_base_from_port(mmio, 0); 1104 writel(clks, hc_mmio + HC_IRQ_COAL_TIME_THRESHOLD); 1105 writel(count, hc_mmio + HC_IRQ_COAL_IO_THRESHOLD); 1106 writel(~HC_COAL_IRQ, hc_mmio + HC_IRQ_CAUSE); 1107 if (count) 1108 coal_enable |= PORTS_0_3_COAL_DONE; 1109 if (is_dual_hc) { 1110 hc_mmio = mv_hc_base_from_port(mmio, MV_PORTS_PER_HC); 1111 writel(clks, hc_mmio + HC_IRQ_COAL_TIME_THRESHOLD); 1112 writel(count, hc_mmio + HC_IRQ_COAL_IO_THRESHOLD); 1113 writel(~HC_COAL_IRQ, hc_mmio + HC_IRQ_CAUSE); 1114 if (count) 1115 coal_enable |= PORTS_4_7_COAL_DONE; 1116 } 1117 1118 mv_set_main_irq_mask(host, 0, coal_enable); 1119 spin_unlock_irqrestore(&host->lock, flags); 1120 } 1121 1122 /* 1123 * mv_start_edma - Enable eDMA engine 1124 * @pp: port private data 1125 * 1126 * Verify the local cache of the eDMA state is accurate with a 1127 * WARN_ON. 1128 * 1129 * LOCKING: 1130 * Inherited from caller. 1131 */ 1132 static void mv_start_edma(struct ata_port *ap, void __iomem *port_mmio, 1133 struct mv_port_priv *pp, u8 protocol) 1134 { 1135 int want_ncq = (protocol == ATA_PROT_NCQ); 1136 1137 if (pp->pp_flags & MV_PP_FLAG_EDMA_EN) { 1138 int using_ncq = ((pp->pp_flags & MV_PP_FLAG_NCQ_EN) != 0); 1139 if (want_ncq != using_ncq) 1140 mv_stop_edma(ap); 1141 } 1142 if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN)) { 1143 struct mv_host_priv *hpriv = ap->host->private_data; 1144 1145 mv_edma_cfg(ap, want_ncq, 1); 1146 1147 mv_set_edma_ptrs(port_mmio, hpriv, pp); 1148 mv_clear_and_enable_port_irqs(ap, port_mmio, DONE_IRQ|ERR_IRQ); 1149 1150 writelfl(EDMA_EN, port_mmio + EDMA_CMD); 1151 pp->pp_flags |= MV_PP_FLAG_EDMA_EN; 1152 } 1153 } 1154 1155 static void mv_wait_for_edma_empty_idle(struct ata_port *ap) 1156 { 1157 void __iomem *port_mmio = mv_ap_base(ap); 1158 const u32 empty_idle = (EDMA_STATUS_CACHE_EMPTY | EDMA_STATUS_IDLE); 1159 const int per_loop = 5, timeout = (15 * 1000 / per_loop); 1160 int i; 1161 1162 /* 1163 * Wait for the EDMA engine to finish transactions in progress. 1164 * No idea what a good "timeout" value might be, but measurements 1165 * indicate that it often requires hundreds of microseconds 1166 * with two drives in-use. So we use the 15msec value above 1167 * as a rough guess at what even more drives might require. 1168 */ 1169 for (i = 0; i < timeout; ++i) { 1170 u32 edma_stat = readl(port_mmio + EDMA_STATUS); 1171 if ((edma_stat & empty_idle) == empty_idle) 1172 break; 1173 udelay(per_loop); 1174 } 1175 /* ata_port_info(ap, "%s: %u+ usecs\n", __func__, i); */ 1176 } 1177 1178 /** 1179 * mv_stop_edma_engine - Disable eDMA engine 1180 * @port_mmio: io base address 1181 * 1182 * LOCKING: 1183 * Inherited from caller. 1184 */ 1185 static int mv_stop_edma_engine(void __iomem *port_mmio) 1186 { 1187 int i; 1188 1189 /* Disable eDMA. The disable bit auto clears. */ 1190 writelfl(EDMA_DS, port_mmio + EDMA_CMD); 1191 1192 /* Wait for the chip to confirm eDMA is off. */ 1193 for (i = 10000; i > 0; i--) { 1194 u32 reg = readl(port_mmio + EDMA_CMD); 1195 if (!(reg & EDMA_EN)) 1196 return 0; 1197 udelay(10); 1198 } 1199 return -EIO; 1200 } 1201 1202 static int mv_stop_edma(struct ata_port *ap) 1203 { 1204 void __iomem *port_mmio = mv_ap_base(ap); 1205 struct mv_port_priv *pp = ap->private_data; 1206 int err = 0; 1207 1208 if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN)) 1209 return 0; 1210 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN; 1211 mv_wait_for_edma_empty_idle(ap); 1212 if (mv_stop_edma_engine(port_mmio)) { 1213 ata_port_err(ap, "Unable to stop eDMA\n"); 1214 err = -EIO; 1215 } 1216 mv_edma_cfg(ap, 0, 0); 1217 return err; 1218 } 1219 1220 static void mv_dump_mem(struct device *dev, void __iomem *start, unsigned bytes) 1221 { 1222 int b, w, o; 1223 unsigned char linebuf[38]; 1224 1225 for (b = 0; b < bytes; ) { 1226 for (w = 0, o = 0; b < bytes && w < 4; w++) { 1227 o += scnprintf(linebuf + o, sizeof(linebuf) - o, 1228 "%08x ", readl(start + b)); 1229 b += sizeof(u32); 1230 } 1231 dev_dbg(dev, "%s: %p: %s\n", 1232 __func__, start + b, linebuf); 1233 } 1234 } 1235 1236 static void mv_dump_pci_cfg(struct pci_dev *pdev, unsigned bytes) 1237 { 1238 int b, w, o; 1239 u32 dw = 0; 1240 unsigned char linebuf[38]; 1241 1242 for (b = 0; b < bytes; ) { 1243 for (w = 0, o = 0; b < bytes && w < 4; w++) { 1244 (void) pci_read_config_dword(pdev, b, &dw); 1245 o += snprintf(linebuf + o, sizeof(linebuf) - o, 1246 "%08x ", dw); 1247 b += sizeof(u32); 1248 } 1249 dev_dbg(&pdev->dev, "%s: %02x: %s\n", 1250 __func__, b, linebuf); 1251 } 1252 } 1253 1254 static void mv_dump_all_regs(void __iomem *mmio_base, 1255 struct pci_dev *pdev) 1256 { 1257 void __iomem *hc_base; 1258 void __iomem *port_base; 1259 int start_port, num_ports, p, start_hc, num_hcs, hc; 1260 1261 start_hc = start_port = 0; 1262 num_ports = 8; /* should be benign for 4 port devs */ 1263 num_hcs = 2; 1264 dev_dbg(&pdev->dev, 1265 "%s: All registers for port(s) %u-%u:\n", __func__, 1266 start_port, num_ports > 1 ? num_ports - 1 : start_port); 1267 1268 dev_dbg(&pdev->dev, "%s: PCI config space regs:\n", __func__); 1269 mv_dump_pci_cfg(pdev, 0x68); 1270 1271 dev_dbg(&pdev->dev, "%s: PCI regs:\n", __func__); 1272 mv_dump_mem(&pdev->dev, mmio_base+0xc00, 0x3c); 1273 mv_dump_mem(&pdev->dev, mmio_base+0xd00, 0x34); 1274 mv_dump_mem(&pdev->dev, mmio_base+0xf00, 0x4); 1275 mv_dump_mem(&pdev->dev, mmio_base+0x1d00, 0x6c); 1276 for (hc = start_hc; hc < start_hc + num_hcs; hc++) { 1277 hc_base = mv_hc_base(mmio_base, hc); 1278 dev_dbg(&pdev->dev, "%s: HC regs (HC %i):\n", __func__, hc); 1279 mv_dump_mem(&pdev->dev, hc_base, 0x1c); 1280 } 1281 for (p = start_port; p < start_port + num_ports; p++) { 1282 port_base = mv_port_base(mmio_base, p); 1283 dev_dbg(&pdev->dev, "%s: EDMA regs (port %i):\n", __func__, p); 1284 mv_dump_mem(&pdev->dev, port_base, 0x54); 1285 dev_dbg(&pdev->dev, "%s: SATA regs (port %i):\n", __func__, p); 1286 mv_dump_mem(&pdev->dev, port_base+0x300, 0x60); 1287 } 1288 } 1289 1290 static unsigned int mv_scr_offset(unsigned int sc_reg_in) 1291 { 1292 unsigned int ofs; 1293 1294 switch (sc_reg_in) { 1295 case SCR_STATUS: 1296 case SCR_CONTROL: 1297 case SCR_ERROR: 1298 ofs = SATA_STATUS + (sc_reg_in * sizeof(u32)); 1299 break; 1300 case SCR_ACTIVE: 1301 ofs = SATA_ACTIVE; /* active is not with the others */ 1302 break; 1303 default: 1304 ofs = 0xffffffffU; 1305 break; 1306 } 1307 return ofs; 1308 } 1309 1310 static int mv_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val) 1311 { 1312 unsigned int ofs = mv_scr_offset(sc_reg_in); 1313 1314 if (ofs != 0xffffffffU) { 1315 *val = readl(mv_ap_base(link->ap) + ofs); 1316 return 0; 1317 } else 1318 return -EINVAL; 1319 } 1320 1321 static int mv_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val) 1322 { 1323 unsigned int ofs = mv_scr_offset(sc_reg_in); 1324 1325 if (ofs != 0xffffffffU) { 1326 void __iomem *addr = mv_ap_base(link->ap) + ofs; 1327 struct mv_host_priv *hpriv = link->ap->host->private_data; 1328 if (sc_reg_in == SCR_CONTROL) { 1329 /* 1330 * Workaround for 88SX60x1 FEr SATA#26: 1331 * 1332 * COMRESETs have to take care not to accidentally 1333 * put the drive to sleep when writing SCR_CONTROL. 1334 * Setting bits 12..15 prevents this problem. 1335 * 1336 * So if we see an outbound COMMRESET, set those bits. 1337 * Ditto for the followup write that clears the reset. 1338 * 1339 * The proprietary driver does this for 1340 * all chip versions, and so do we. 1341 */ 1342 if ((val & 0xf) == 1 || (readl(addr) & 0xf) == 1) 1343 val |= 0xf000; 1344 1345 if (hpriv->hp_flags & MV_HP_FIX_LP_PHY_CTL) { 1346 void __iomem *lp_phy_addr = 1347 mv_ap_base(link->ap) + LP_PHY_CTL; 1348 /* 1349 * Set PHY speed according to SControl speed. 1350 */ 1351 u32 lp_phy_val = 1352 LP_PHY_CTL_PIN_PU_PLL | 1353 LP_PHY_CTL_PIN_PU_RX | 1354 LP_PHY_CTL_PIN_PU_TX; 1355 1356 if ((val & 0xf0) != 0x10) 1357 lp_phy_val |= 1358 LP_PHY_CTL_GEN_TX_3G | 1359 LP_PHY_CTL_GEN_RX_3G; 1360 1361 writelfl(lp_phy_val, lp_phy_addr); 1362 } 1363 } 1364 writelfl(val, addr); 1365 return 0; 1366 } else 1367 return -EINVAL; 1368 } 1369 1370 static void mv6_dev_config(struct ata_device *adev) 1371 { 1372 /* 1373 * Deal with Gen-II ("mv6") hardware quirks/restrictions: 1374 * 1375 * Gen-II does not support NCQ over a port multiplier 1376 * (no FIS-based switching). 1377 */ 1378 if (adev->flags & ATA_DFLAG_NCQ) { 1379 if (sata_pmp_attached(adev->link->ap)) { 1380 adev->flags &= ~ATA_DFLAG_NCQ; 1381 ata_dev_info(adev, 1382 "NCQ disabled for command-based switching\n"); 1383 } 1384 } 1385 } 1386 1387 static int mv_qc_defer(struct ata_queued_cmd *qc) 1388 { 1389 struct ata_link *link = qc->dev->link; 1390 struct ata_port *ap = link->ap; 1391 struct mv_port_priv *pp = ap->private_data; 1392 1393 /* 1394 * Don't allow new commands if we're in a delayed EH state 1395 * for NCQ and/or FIS-based switching. 1396 */ 1397 if (pp->pp_flags & MV_PP_FLAG_DELAYED_EH) 1398 return ATA_DEFER_PORT; 1399 1400 /* PIO commands need exclusive link: no other commands [DMA or PIO] 1401 * can run concurrently. 1402 * set excl_link when we want to send a PIO command in DMA mode 1403 * or a non-NCQ command in NCQ mode. 1404 * When we receive a command from that link, and there are no 1405 * outstanding commands, mark a flag to clear excl_link and let 1406 * the command go through. 1407 */ 1408 if (unlikely(ap->excl_link)) { 1409 if (link == ap->excl_link) { 1410 if (ap->nr_active_links) 1411 return ATA_DEFER_PORT; 1412 qc->flags |= ATA_QCFLAG_CLEAR_EXCL; 1413 return 0; 1414 } else 1415 return ATA_DEFER_PORT; 1416 } 1417 1418 /* 1419 * If the port is completely idle, then allow the new qc. 1420 */ 1421 if (ap->nr_active_links == 0) 1422 return 0; 1423 1424 /* 1425 * The port is operating in host queuing mode (EDMA) with NCQ 1426 * enabled, allow multiple NCQ commands. EDMA also allows 1427 * queueing multiple DMA commands but libata core currently 1428 * doesn't allow it. 1429 */ 1430 if ((pp->pp_flags & MV_PP_FLAG_EDMA_EN) && 1431 (pp->pp_flags & MV_PP_FLAG_NCQ_EN)) { 1432 if (ata_is_ncq(qc->tf.protocol)) 1433 return 0; 1434 else { 1435 ap->excl_link = link; 1436 return ATA_DEFER_PORT; 1437 } 1438 } 1439 1440 return ATA_DEFER_PORT; 1441 } 1442 1443 static void mv_config_fbs(struct ata_port *ap, int want_ncq, int want_fbs) 1444 { 1445 struct mv_port_priv *pp = ap->private_data; 1446 void __iomem *port_mmio; 1447 1448 u32 fiscfg, *old_fiscfg = &pp->cached.fiscfg; 1449 u32 ltmode, *old_ltmode = &pp->cached.ltmode; 1450 u32 haltcond, *old_haltcond = &pp->cached.haltcond; 1451 1452 ltmode = *old_ltmode & ~LTMODE_BIT8; 1453 haltcond = *old_haltcond | EDMA_ERR_DEV; 1454 1455 if (want_fbs) { 1456 fiscfg = *old_fiscfg | FISCFG_SINGLE_SYNC; 1457 ltmode = *old_ltmode | LTMODE_BIT8; 1458 if (want_ncq) 1459 haltcond &= ~EDMA_ERR_DEV; 1460 else 1461 fiscfg |= FISCFG_WAIT_DEV_ERR; 1462 } else { 1463 fiscfg = *old_fiscfg & ~(FISCFG_SINGLE_SYNC | FISCFG_WAIT_DEV_ERR); 1464 } 1465 1466 port_mmio = mv_ap_base(ap); 1467 mv_write_cached_reg(port_mmio + FISCFG, old_fiscfg, fiscfg); 1468 mv_write_cached_reg(port_mmio + LTMODE, old_ltmode, ltmode); 1469 mv_write_cached_reg(port_mmio + EDMA_HALTCOND, old_haltcond, haltcond); 1470 } 1471 1472 static void mv_60x1_errata_sata25(struct ata_port *ap, int want_ncq) 1473 { 1474 struct mv_host_priv *hpriv = ap->host->private_data; 1475 u32 old, new; 1476 1477 /* workaround for 88SX60x1 FEr SATA#25 (part 1) */ 1478 old = readl(hpriv->base + GPIO_PORT_CTL); 1479 if (want_ncq) 1480 new = old | (1 << 22); 1481 else 1482 new = old & ~(1 << 22); 1483 if (new != old) 1484 writel(new, hpriv->base + GPIO_PORT_CTL); 1485 } 1486 1487 /* 1488 * mv_bmdma_enable - set a magic bit on GEN_IIE to allow bmdma 1489 * @ap: Port being initialized 1490 * 1491 * There are two DMA modes on these chips: basic DMA, and EDMA. 1492 * 1493 * Bit-0 of the "EDMA RESERVED" register enables/disables use 1494 * of basic DMA on the GEN_IIE versions of the chips. 1495 * 1496 * This bit survives EDMA resets, and must be set for basic DMA 1497 * to function, and should be cleared when EDMA is active. 1498 */ 1499 static void mv_bmdma_enable_iie(struct ata_port *ap, int enable_bmdma) 1500 { 1501 struct mv_port_priv *pp = ap->private_data; 1502 u32 new, *old = &pp->cached.unknown_rsvd; 1503 1504 if (enable_bmdma) 1505 new = *old | 1; 1506 else 1507 new = *old & ~1; 1508 mv_write_cached_reg(mv_ap_base(ap) + EDMA_UNKNOWN_RSVD, old, new); 1509 } 1510 1511 /* 1512 * SOC chips have an issue whereby the HDD LEDs don't always blink 1513 * during I/O when NCQ is enabled. Enabling a special "LED blink" mode 1514 * of the SOC takes care of it, generating a steady blink rate when 1515 * any drive on the chip is active. 1516 * 1517 * Unfortunately, the blink mode is a global hardware setting for the SOC, 1518 * so we must use it whenever at least one port on the SOC has NCQ enabled. 1519 * 1520 * We turn "LED blink" off when NCQ is not in use anywhere, because the normal 1521 * LED operation works then, and provides better (more accurate) feedback. 1522 * 1523 * Note that this code assumes that an SOC never has more than one HC onboard. 1524 */ 1525 static void mv_soc_led_blink_enable(struct ata_port *ap) 1526 { 1527 struct ata_host *host = ap->host; 1528 struct mv_host_priv *hpriv = host->private_data; 1529 void __iomem *hc_mmio; 1530 u32 led_ctrl; 1531 1532 if (hpriv->hp_flags & MV_HP_QUIRK_LED_BLINK_EN) 1533 return; 1534 hpriv->hp_flags |= MV_HP_QUIRK_LED_BLINK_EN; 1535 hc_mmio = mv_hc_base_from_port(mv_host_base(host), ap->port_no); 1536 led_ctrl = readl(hc_mmio + SOC_LED_CTRL); 1537 writel(led_ctrl | SOC_LED_CTRL_BLINK, hc_mmio + SOC_LED_CTRL); 1538 } 1539 1540 static void mv_soc_led_blink_disable(struct ata_port *ap) 1541 { 1542 struct ata_host *host = ap->host; 1543 struct mv_host_priv *hpriv = host->private_data; 1544 void __iomem *hc_mmio; 1545 u32 led_ctrl; 1546 unsigned int port; 1547 1548 if (!(hpriv->hp_flags & MV_HP_QUIRK_LED_BLINK_EN)) 1549 return; 1550 1551 /* disable led-blink only if no ports are using NCQ */ 1552 for (port = 0; port < hpriv->n_ports; port++) { 1553 struct ata_port *this_ap = host->ports[port]; 1554 struct mv_port_priv *pp = this_ap->private_data; 1555 1556 if (pp->pp_flags & MV_PP_FLAG_NCQ_EN) 1557 return; 1558 } 1559 1560 hpriv->hp_flags &= ~MV_HP_QUIRK_LED_BLINK_EN; 1561 hc_mmio = mv_hc_base_from_port(mv_host_base(host), ap->port_no); 1562 led_ctrl = readl(hc_mmio + SOC_LED_CTRL); 1563 writel(led_ctrl & ~SOC_LED_CTRL_BLINK, hc_mmio + SOC_LED_CTRL); 1564 } 1565 1566 static void mv_edma_cfg(struct ata_port *ap, int want_ncq, int want_edma) 1567 { 1568 u32 cfg; 1569 struct mv_port_priv *pp = ap->private_data; 1570 struct mv_host_priv *hpriv = ap->host->private_data; 1571 void __iomem *port_mmio = mv_ap_base(ap); 1572 1573 /* set up non-NCQ EDMA configuration */ 1574 cfg = EDMA_CFG_Q_DEPTH; /* always 0x1f for *all* chips */ 1575 pp->pp_flags &= 1576 ~(MV_PP_FLAG_FBS_EN | MV_PP_FLAG_NCQ_EN | MV_PP_FLAG_FAKE_ATA_BUSY); 1577 1578 if (IS_GEN_I(hpriv)) 1579 cfg |= (1 << 8); /* enab config burst size mask */ 1580 1581 else if (IS_GEN_II(hpriv)) { 1582 cfg |= EDMA_CFG_RD_BRST_EXT | EDMA_CFG_WR_BUFF_LEN; 1583 mv_60x1_errata_sata25(ap, want_ncq); 1584 1585 } else if (IS_GEN_IIE(hpriv)) { 1586 int want_fbs = sata_pmp_attached(ap); 1587 /* 1588 * Possible future enhancement: 1589 * 1590 * The chip can use FBS with non-NCQ, if we allow it, 1591 * But first we need to have the error handling in place 1592 * for this mode (datasheet section 7.3.15.4.2.3). 1593 * So disallow non-NCQ FBS for now. 1594 */ 1595 want_fbs &= want_ncq; 1596 1597 mv_config_fbs(ap, want_ncq, want_fbs); 1598 1599 if (want_fbs) { 1600 pp->pp_flags |= MV_PP_FLAG_FBS_EN; 1601 cfg |= EDMA_CFG_EDMA_FBS; /* FIS-based switching */ 1602 } 1603 1604 cfg |= (1 << 23); /* do not mask PM field in rx'd FIS */ 1605 if (want_edma) { 1606 cfg |= (1 << 22); /* enab 4-entry host queue cache */ 1607 if (!IS_SOC(hpriv)) 1608 cfg |= (1 << 18); /* enab early completion */ 1609 } 1610 if (hpriv->hp_flags & MV_HP_CUT_THROUGH) 1611 cfg |= (1 << 17); /* enab cut-thru (dis stor&forwrd) */ 1612 mv_bmdma_enable_iie(ap, !want_edma); 1613 1614 if (IS_SOC(hpriv)) { 1615 if (want_ncq) 1616 mv_soc_led_blink_enable(ap); 1617 else 1618 mv_soc_led_blink_disable(ap); 1619 } 1620 } 1621 1622 if (want_ncq) { 1623 cfg |= EDMA_CFG_NCQ; 1624 pp->pp_flags |= MV_PP_FLAG_NCQ_EN; 1625 } 1626 1627 writelfl(cfg, port_mmio + EDMA_CFG); 1628 } 1629 1630 static void mv_port_free_dma_mem(struct ata_port *ap) 1631 { 1632 struct mv_host_priv *hpriv = ap->host->private_data; 1633 struct mv_port_priv *pp = ap->private_data; 1634 int tag; 1635 1636 if (pp->crqb) { 1637 dma_pool_free(hpriv->crqb_pool, pp->crqb, pp->crqb_dma); 1638 pp->crqb = NULL; 1639 } 1640 if (pp->crpb) { 1641 dma_pool_free(hpriv->crpb_pool, pp->crpb, pp->crpb_dma); 1642 pp->crpb = NULL; 1643 } 1644 /* 1645 * For GEN_I, there's no NCQ, so we have only a single sg_tbl. 1646 * For later hardware, we have one unique sg_tbl per NCQ tag. 1647 */ 1648 for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) { 1649 if (pp->sg_tbl[tag]) { 1650 if (tag == 0 || !IS_GEN_I(hpriv)) 1651 dma_pool_free(hpriv->sg_tbl_pool, 1652 pp->sg_tbl[tag], 1653 pp->sg_tbl_dma[tag]); 1654 pp->sg_tbl[tag] = NULL; 1655 } 1656 } 1657 } 1658 1659 /** 1660 * mv_port_start - Port specific init/start routine. 1661 * @ap: ATA channel to manipulate 1662 * 1663 * Allocate and point to DMA memory, init port private memory, 1664 * zero indices. 1665 * 1666 * LOCKING: 1667 * Inherited from caller. 1668 */ 1669 static int mv_port_start(struct ata_port *ap) 1670 { 1671 struct device *dev = ap->host->dev; 1672 struct mv_host_priv *hpriv = ap->host->private_data; 1673 struct mv_port_priv *pp; 1674 unsigned long flags; 1675 int tag; 1676 1677 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL); 1678 if (!pp) 1679 return -ENOMEM; 1680 ap->private_data = pp; 1681 1682 pp->crqb = dma_pool_zalloc(hpriv->crqb_pool, GFP_KERNEL, &pp->crqb_dma); 1683 if (!pp->crqb) 1684 return -ENOMEM; 1685 1686 pp->crpb = dma_pool_zalloc(hpriv->crpb_pool, GFP_KERNEL, &pp->crpb_dma); 1687 if (!pp->crpb) 1688 goto out_port_free_dma_mem; 1689 1690 /* 6041/6081 Rev. "C0" (and newer) are okay with async notify */ 1691 if (hpriv->hp_flags & MV_HP_ERRATA_60X1C0) 1692 ap->flags |= ATA_FLAG_AN; 1693 /* 1694 * For GEN_I, there's no NCQ, so we only allocate a single sg_tbl. 1695 * For later hardware, we need one unique sg_tbl per NCQ tag. 1696 */ 1697 for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) { 1698 if (tag == 0 || !IS_GEN_I(hpriv)) { 1699 pp->sg_tbl[tag] = dma_pool_alloc(hpriv->sg_tbl_pool, 1700 GFP_KERNEL, &pp->sg_tbl_dma[tag]); 1701 if (!pp->sg_tbl[tag]) 1702 goto out_port_free_dma_mem; 1703 } else { 1704 pp->sg_tbl[tag] = pp->sg_tbl[0]; 1705 pp->sg_tbl_dma[tag] = pp->sg_tbl_dma[0]; 1706 } 1707 } 1708 1709 spin_lock_irqsave(ap->lock, flags); 1710 mv_save_cached_regs(ap); 1711 mv_edma_cfg(ap, 0, 0); 1712 spin_unlock_irqrestore(ap->lock, flags); 1713 1714 return 0; 1715 1716 out_port_free_dma_mem: 1717 mv_port_free_dma_mem(ap); 1718 return -ENOMEM; 1719 } 1720 1721 /** 1722 * mv_port_stop - Port specific cleanup/stop routine. 1723 * @ap: ATA channel to manipulate 1724 * 1725 * Stop DMA, cleanup port memory. 1726 * 1727 * LOCKING: 1728 * This routine uses the host lock to protect the DMA stop. 1729 */ 1730 static void mv_port_stop(struct ata_port *ap) 1731 { 1732 unsigned long flags; 1733 1734 spin_lock_irqsave(ap->lock, flags); 1735 mv_stop_edma(ap); 1736 mv_enable_port_irqs(ap, 0); 1737 spin_unlock_irqrestore(ap->lock, flags); 1738 mv_port_free_dma_mem(ap); 1739 } 1740 1741 /** 1742 * mv_fill_sg - Fill out the Marvell ePRD (scatter gather) entries 1743 * @qc: queued command whose SG list to source from 1744 * 1745 * Populate the SG list and mark the last entry. 1746 * 1747 * LOCKING: 1748 * Inherited from caller. 1749 */ 1750 static void mv_fill_sg(struct ata_queued_cmd *qc) 1751 { 1752 struct mv_port_priv *pp = qc->ap->private_data; 1753 struct scatterlist *sg; 1754 struct mv_sg *mv_sg, *last_sg = NULL; 1755 unsigned int si; 1756 1757 mv_sg = pp->sg_tbl[qc->hw_tag]; 1758 for_each_sg(qc->sg, sg, qc->n_elem, si) { 1759 dma_addr_t addr = sg_dma_address(sg); 1760 u32 sg_len = sg_dma_len(sg); 1761 1762 while (sg_len) { 1763 u32 offset = addr & 0xffff; 1764 u32 len = sg_len; 1765 1766 if (offset + len > 0x10000) 1767 len = 0x10000 - offset; 1768 1769 mv_sg->addr = cpu_to_le32(addr & 0xffffffff); 1770 mv_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16); 1771 mv_sg->flags_size = cpu_to_le32(len & 0xffff); 1772 mv_sg->reserved = 0; 1773 1774 sg_len -= len; 1775 addr += len; 1776 1777 last_sg = mv_sg; 1778 mv_sg++; 1779 } 1780 } 1781 1782 if (likely(last_sg)) 1783 last_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL); 1784 mb(); /* ensure data structure is visible to the chipset */ 1785 } 1786 1787 static void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last) 1788 { 1789 u16 tmp = data | (addr << CRQB_CMD_ADDR_SHIFT) | CRQB_CMD_CS | 1790 (last ? CRQB_CMD_LAST : 0); 1791 *cmdw = cpu_to_le16(tmp); 1792 } 1793 1794 /** 1795 * mv_sff_irq_clear - Clear hardware interrupt after DMA. 1796 * @ap: Port associated with this ATA transaction. 1797 * 1798 * We need this only for ATAPI bmdma transactions, 1799 * as otherwise we experience spurious interrupts 1800 * after libata-sff handles the bmdma interrupts. 1801 */ 1802 static void mv_sff_irq_clear(struct ata_port *ap) 1803 { 1804 mv_clear_and_enable_port_irqs(ap, mv_ap_base(ap), ERR_IRQ); 1805 } 1806 1807 /** 1808 * mv_check_atapi_dma - Filter ATAPI cmds which are unsuitable for DMA. 1809 * @qc: queued command to check for chipset/DMA compatibility. 1810 * 1811 * The bmdma engines cannot handle speculative data sizes 1812 * (bytecount under/over flow). So only allow DMA for 1813 * data transfer commands with known data sizes. 1814 * 1815 * LOCKING: 1816 * Inherited from caller. 1817 */ 1818 static int mv_check_atapi_dma(struct ata_queued_cmd *qc) 1819 { 1820 struct scsi_cmnd *scmd = qc->scsicmd; 1821 1822 if (scmd) { 1823 switch (scmd->cmnd[0]) { 1824 case READ_6: 1825 case READ_10: 1826 case READ_12: 1827 case WRITE_6: 1828 case WRITE_10: 1829 case WRITE_12: 1830 case GPCMD_READ_CD: 1831 case GPCMD_SEND_DVD_STRUCTURE: 1832 case GPCMD_SEND_CUE_SHEET: 1833 return 0; /* DMA is safe */ 1834 } 1835 } 1836 return -EOPNOTSUPP; /* use PIO instead */ 1837 } 1838 1839 /** 1840 * mv_bmdma_setup - Set up BMDMA transaction 1841 * @qc: queued command to prepare DMA for. 1842 * 1843 * LOCKING: 1844 * Inherited from caller. 1845 */ 1846 static void mv_bmdma_setup(struct ata_queued_cmd *qc) 1847 { 1848 struct ata_port *ap = qc->ap; 1849 void __iomem *port_mmio = mv_ap_base(ap); 1850 struct mv_port_priv *pp = ap->private_data; 1851 1852 mv_fill_sg(qc); 1853 1854 /* clear all DMA cmd bits */ 1855 writel(0, port_mmio + BMDMA_CMD); 1856 1857 /* load PRD table addr. */ 1858 writel((pp->sg_tbl_dma[qc->hw_tag] >> 16) >> 16, 1859 port_mmio + BMDMA_PRD_HIGH); 1860 writelfl(pp->sg_tbl_dma[qc->hw_tag], 1861 port_mmio + BMDMA_PRD_LOW); 1862 1863 /* issue r/w command */ 1864 ap->ops->sff_exec_command(ap, &qc->tf); 1865 } 1866 1867 /** 1868 * mv_bmdma_start - Start a BMDMA transaction 1869 * @qc: queued command to start DMA on. 1870 * 1871 * LOCKING: 1872 * Inherited from caller. 1873 */ 1874 static void mv_bmdma_start(struct ata_queued_cmd *qc) 1875 { 1876 struct ata_port *ap = qc->ap; 1877 void __iomem *port_mmio = mv_ap_base(ap); 1878 unsigned int rw = (qc->tf.flags & ATA_TFLAG_WRITE); 1879 u32 cmd = (rw ? 0 : ATA_DMA_WR) | ATA_DMA_START; 1880 1881 /* start host DMA transaction */ 1882 writelfl(cmd, port_mmio + BMDMA_CMD); 1883 } 1884 1885 /** 1886 * mv_bmdma_stop_ap - Stop BMDMA transfer 1887 * @ap: port to stop 1888 * 1889 * Clears the ATA_DMA_START flag in the bmdma control register 1890 * 1891 * LOCKING: 1892 * Inherited from caller. 1893 */ 1894 static void mv_bmdma_stop_ap(struct ata_port *ap) 1895 { 1896 void __iomem *port_mmio = mv_ap_base(ap); 1897 u32 cmd; 1898 1899 /* clear start/stop bit */ 1900 cmd = readl(port_mmio + BMDMA_CMD); 1901 if (cmd & ATA_DMA_START) { 1902 cmd &= ~ATA_DMA_START; 1903 writelfl(cmd, port_mmio + BMDMA_CMD); 1904 1905 /* one-PIO-cycle guaranteed wait, per spec, for HDMA1:0 transition */ 1906 ata_sff_dma_pause(ap); 1907 } 1908 } 1909 1910 static void mv_bmdma_stop(struct ata_queued_cmd *qc) 1911 { 1912 mv_bmdma_stop_ap(qc->ap); 1913 } 1914 1915 /** 1916 * mv_bmdma_status - Read BMDMA status 1917 * @ap: port for which to retrieve DMA status. 1918 * 1919 * Read and return equivalent of the sff BMDMA status register. 1920 * 1921 * LOCKING: 1922 * Inherited from caller. 1923 */ 1924 static u8 mv_bmdma_status(struct ata_port *ap) 1925 { 1926 void __iomem *port_mmio = mv_ap_base(ap); 1927 u32 reg, status; 1928 1929 /* 1930 * Other bits are valid only if ATA_DMA_ACTIVE==0, 1931 * and the ATA_DMA_INTR bit doesn't exist. 1932 */ 1933 reg = readl(port_mmio + BMDMA_STATUS); 1934 if (reg & ATA_DMA_ACTIVE) 1935 status = ATA_DMA_ACTIVE; 1936 else if (reg & ATA_DMA_ERR) 1937 status = (reg & ATA_DMA_ERR) | ATA_DMA_INTR; 1938 else { 1939 /* 1940 * Just because DMA_ACTIVE is 0 (DMA completed), 1941 * this does _not_ mean the device is "done". 1942 * So we should not yet be signalling ATA_DMA_INTR 1943 * in some cases. Eg. DSM/TRIM, and perhaps others. 1944 */ 1945 mv_bmdma_stop_ap(ap); 1946 if (ioread8(ap->ioaddr.altstatus_addr) & ATA_BUSY) 1947 status = 0; 1948 else 1949 status = ATA_DMA_INTR; 1950 } 1951 return status; 1952 } 1953 1954 static void mv_rw_multi_errata_sata24(struct ata_queued_cmd *qc) 1955 { 1956 struct ata_taskfile *tf = &qc->tf; 1957 /* 1958 * Workaround for 88SX60x1 FEr SATA#24. 1959 * 1960 * Chip may corrupt WRITEs if multi_count >= 4kB. 1961 * Note that READs are unaffected. 1962 * 1963 * It's not clear if this errata really means "4K bytes", 1964 * or if it always happens for multi_count > 7 1965 * regardless of device sector_size. 1966 * 1967 * So, for safety, any write with multi_count > 7 1968 * gets converted here into a regular PIO write instead: 1969 */ 1970 if ((tf->flags & ATA_TFLAG_WRITE) && is_multi_taskfile(tf)) { 1971 if (qc->dev->multi_count > 7) { 1972 switch (tf->command) { 1973 case ATA_CMD_WRITE_MULTI: 1974 tf->command = ATA_CMD_PIO_WRITE; 1975 break; 1976 case ATA_CMD_WRITE_MULTI_FUA_EXT: 1977 tf->flags &= ~ATA_TFLAG_FUA; /* ugh */ 1978 fallthrough; 1979 case ATA_CMD_WRITE_MULTI_EXT: 1980 tf->command = ATA_CMD_PIO_WRITE_EXT; 1981 break; 1982 } 1983 } 1984 } 1985 } 1986 1987 /** 1988 * mv_qc_prep - Host specific command preparation. 1989 * @qc: queued command to prepare 1990 * 1991 * This routine simply redirects to the general purpose routine 1992 * if command is not DMA. Else, it handles prep of the CRQB 1993 * (command request block), does some sanity checking, and calls 1994 * the SG load routine. 1995 * 1996 * LOCKING: 1997 * Inherited from caller. 1998 */ 1999 static enum ata_completion_errors mv_qc_prep(struct ata_queued_cmd *qc) 2000 { 2001 struct ata_port *ap = qc->ap; 2002 struct mv_port_priv *pp = ap->private_data; 2003 __le16 *cw; 2004 struct ata_taskfile *tf = &qc->tf; 2005 u16 flags = 0; 2006 unsigned in_index; 2007 2008 switch (tf->protocol) { 2009 case ATA_PROT_DMA: 2010 if (tf->command == ATA_CMD_DSM) 2011 return AC_ERR_OK; 2012 fallthrough; 2013 case ATA_PROT_NCQ: 2014 break; /* continue below */ 2015 case ATA_PROT_PIO: 2016 mv_rw_multi_errata_sata24(qc); 2017 return AC_ERR_OK; 2018 default: 2019 return AC_ERR_OK; 2020 } 2021 2022 /* Fill in command request block 2023 */ 2024 if (!(tf->flags & ATA_TFLAG_WRITE)) 2025 flags |= CRQB_FLAG_READ; 2026 WARN_ON(MV_MAX_Q_DEPTH <= qc->hw_tag); 2027 flags |= qc->hw_tag << CRQB_TAG_SHIFT; 2028 flags |= (qc->dev->link->pmp & 0xf) << CRQB_PMP_SHIFT; 2029 2030 /* get current queue index from software */ 2031 in_index = pp->req_idx; 2032 2033 pp->crqb[in_index].sg_addr = 2034 cpu_to_le32(pp->sg_tbl_dma[qc->hw_tag] & 0xffffffff); 2035 pp->crqb[in_index].sg_addr_hi = 2036 cpu_to_le32((pp->sg_tbl_dma[qc->hw_tag] >> 16) >> 16); 2037 pp->crqb[in_index].ctrl_flags = cpu_to_le16(flags); 2038 2039 cw = &pp->crqb[in_index].ata_cmd[0]; 2040 2041 /* Sadly, the CRQB cannot accommodate all registers--there are 2042 * only 11 bytes...so we must pick and choose required 2043 * registers based on the command. So, we drop feature and 2044 * hob_feature for [RW] DMA commands, but they are needed for 2045 * NCQ. NCQ will drop hob_nsect, which is not needed there 2046 * (nsect is used only for the tag; feat/hob_feat hold true nsect). 2047 */ 2048 switch (tf->command) { 2049 case ATA_CMD_READ: 2050 case ATA_CMD_READ_EXT: 2051 case ATA_CMD_WRITE: 2052 case ATA_CMD_WRITE_EXT: 2053 case ATA_CMD_WRITE_FUA_EXT: 2054 mv_crqb_pack_cmd(cw++, tf->hob_nsect, ATA_REG_NSECT, 0); 2055 break; 2056 case ATA_CMD_FPDMA_READ: 2057 case ATA_CMD_FPDMA_WRITE: 2058 mv_crqb_pack_cmd(cw++, tf->hob_feature, ATA_REG_FEATURE, 0); 2059 mv_crqb_pack_cmd(cw++, tf->feature, ATA_REG_FEATURE, 0); 2060 break; 2061 default: 2062 /* The only other commands EDMA supports in non-queued and 2063 * non-NCQ mode are: [RW] STREAM DMA and W DMA FUA EXT, none 2064 * of which are defined/used by Linux. If we get here, this 2065 * driver needs work. 2066 */ 2067 ata_port_err(ap, "%s: unsupported command: %.2x\n", __func__, 2068 tf->command); 2069 return AC_ERR_INVALID; 2070 } 2071 mv_crqb_pack_cmd(cw++, tf->nsect, ATA_REG_NSECT, 0); 2072 mv_crqb_pack_cmd(cw++, tf->hob_lbal, ATA_REG_LBAL, 0); 2073 mv_crqb_pack_cmd(cw++, tf->lbal, ATA_REG_LBAL, 0); 2074 mv_crqb_pack_cmd(cw++, tf->hob_lbam, ATA_REG_LBAM, 0); 2075 mv_crqb_pack_cmd(cw++, tf->lbam, ATA_REG_LBAM, 0); 2076 mv_crqb_pack_cmd(cw++, tf->hob_lbah, ATA_REG_LBAH, 0); 2077 mv_crqb_pack_cmd(cw++, tf->lbah, ATA_REG_LBAH, 0); 2078 mv_crqb_pack_cmd(cw++, tf->device, ATA_REG_DEVICE, 0); 2079 mv_crqb_pack_cmd(cw++, tf->command, ATA_REG_CMD, 1); /* last */ 2080 2081 if (!(qc->flags & ATA_QCFLAG_DMAMAP)) 2082 return AC_ERR_OK; 2083 mv_fill_sg(qc); 2084 2085 return AC_ERR_OK; 2086 } 2087 2088 /** 2089 * mv_qc_prep_iie - Host specific command preparation. 2090 * @qc: queued command to prepare 2091 * 2092 * This routine simply redirects to the general purpose routine 2093 * if command is not DMA. Else, it handles prep of the CRQB 2094 * (command request block), does some sanity checking, and calls 2095 * the SG load routine. 2096 * 2097 * LOCKING: 2098 * Inherited from caller. 2099 */ 2100 static enum ata_completion_errors mv_qc_prep_iie(struct ata_queued_cmd *qc) 2101 { 2102 struct ata_port *ap = qc->ap; 2103 struct mv_port_priv *pp = ap->private_data; 2104 struct mv_crqb_iie *crqb; 2105 struct ata_taskfile *tf = &qc->tf; 2106 unsigned in_index; 2107 u32 flags = 0; 2108 2109 if ((tf->protocol != ATA_PROT_DMA) && 2110 (tf->protocol != ATA_PROT_NCQ)) 2111 return AC_ERR_OK; 2112 if (tf->command == ATA_CMD_DSM) 2113 return AC_ERR_OK; /* use bmdma for this */ 2114 2115 /* Fill in Gen IIE command request block */ 2116 if (!(tf->flags & ATA_TFLAG_WRITE)) 2117 flags |= CRQB_FLAG_READ; 2118 2119 WARN_ON(MV_MAX_Q_DEPTH <= qc->hw_tag); 2120 flags |= qc->hw_tag << CRQB_TAG_SHIFT; 2121 flags |= qc->hw_tag << CRQB_HOSTQ_SHIFT; 2122 flags |= (qc->dev->link->pmp & 0xf) << CRQB_PMP_SHIFT; 2123 2124 /* get current queue index from software */ 2125 in_index = pp->req_idx; 2126 2127 crqb = (struct mv_crqb_iie *) &pp->crqb[in_index]; 2128 crqb->addr = cpu_to_le32(pp->sg_tbl_dma[qc->hw_tag] & 0xffffffff); 2129 crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma[qc->hw_tag] >> 16) >> 16); 2130 crqb->flags = cpu_to_le32(flags); 2131 2132 crqb->ata_cmd[0] = cpu_to_le32( 2133 (tf->command << 16) | 2134 (tf->feature << 24) 2135 ); 2136 crqb->ata_cmd[1] = cpu_to_le32( 2137 (tf->lbal << 0) | 2138 (tf->lbam << 8) | 2139 (tf->lbah << 16) | 2140 (tf->device << 24) 2141 ); 2142 crqb->ata_cmd[2] = cpu_to_le32( 2143 (tf->hob_lbal << 0) | 2144 (tf->hob_lbam << 8) | 2145 (tf->hob_lbah << 16) | 2146 (tf->hob_feature << 24) 2147 ); 2148 crqb->ata_cmd[3] = cpu_to_le32( 2149 (tf->nsect << 0) | 2150 (tf->hob_nsect << 8) 2151 ); 2152 2153 if (!(qc->flags & ATA_QCFLAG_DMAMAP)) 2154 return AC_ERR_OK; 2155 mv_fill_sg(qc); 2156 2157 return AC_ERR_OK; 2158 } 2159 2160 /** 2161 * mv_sff_check_status - fetch device status, if valid 2162 * @ap: ATA port to fetch status from 2163 * 2164 * When using command issue via mv_qc_issue_fis(), 2165 * the initial ATA_BUSY state does not show up in the 2166 * ATA status (shadow) register. This can confuse libata! 2167 * 2168 * So we have a hook here to fake ATA_BUSY for that situation, 2169 * until the first time a BUSY, DRQ, or ERR bit is seen. 2170 * 2171 * The rest of the time, it simply returns the ATA status register. 2172 */ 2173 static u8 mv_sff_check_status(struct ata_port *ap) 2174 { 2175 u8 stat = ioread8(ap->ioaddr.status_addr); 2176 struct mv_port_priv *pp = ap->private_data; 2177 2178 if (pp->pp_flags & MV_PP_FLAG_FAKE_ATA_BUSY) { 2179 if (stat & (ATA_BUSY | ATA_DRQ | ATA_ERR)) 2180 pp->pp_flags &= ~MV_PP_FLAG_FAKE_ATA_BUSY; 2181 else 2182 stat = ATA_BUSY; 2183 } 2184 return stat; 2185 } 2186 2187 /** 2188 * mv_send_fis - Send a FIS, using the "Vendor-Unique FIS" register 2189 * @ap: ATA port to send a FIS 2190 * @fis: fis to be sent 2191 * @nwords: number of 32-bit words in the fis 2192 */ 2193 static unsigned int mv_send_fis(struct ata_port *ap, u32 *fis, int nwords) 2194 { 2195 void __iomem *port_mmio = mv_ap_base(ap); 2196 u32 ifctl, old_ifctl, ifstat; 2197 int i, timeout = 200, final_word = nwords - 1; 2198 2199 /* Initiate FIS transmission mode */ 2200 old_ifctl = readl(port_mmio + SATA_IFCTL); 2201 ifctl = 0x100 | (old_ifctl & 0xf); 2202 writelfl(ifctl, port_mmio + SATA_IFCTL); 2203 2204 /* Send all words of the FIS except for the final word */ 2205 for (i = 0; i < final_word; ++i) 2206 writel(fis[i], port_mmio + VENDOR_UNIQUE_FIS); 2207 2208 /* Flag end-of-transmission, and then send the final word */ 2209 writelfl(ifctl | 0x200, port_mmio + SATA_IFCTL); 2210 writelfl(fis[final_word], port_mmio + VENDOR_UNIQUE_FIS); 2211 2212 /* 2213 * Wait for FIS transmission to complete. 2214 * This typically takes just a single iteration. 2215 */ 2216 do { 2217 ifstat = readl(port_mmio + SATA_IFSTAT); 2218 } while (!(ifstat & 0x1000) && --timeout); 2219 2220 /* Restore original port configuration */ 2221 writelfl(old_ifctl, port_mmio + SATA_IFCTL); 2222 2223 /* See if it worked */ 2224 if ((ifstat & 0x3000) != 0x1000) { 2225 ata_port_warn(ap, "%s transmission error, ifstat=%08x\n", 2226 __func__, ifstat); 2227 return AC_ERR_OTHER; 2228 } 2229 return 0; 2230 } 2231 2232 /** 2233 * mv_qc_issue_fis - Issue a command directly as a FIS 2234 * @qc: queued command to start 2235 * 2236 * Note that the ATA shadow registers are not updated 2237 * after command issue, so the device will appear "READY" 2238 * if polled, even while it is BUSY processing the command. 2239 * 2240 * So we use a status hook to fake ATA_BUSY until the drive changes state. 2241 * 2242 * Note: we don't get updated shadow regs on *completion* 2243 * of non-data commands. So avoid sending them via this function, 2244 * as they will appear to have completed immediately. 2245 * 2246 * GEN_IIE has special registers that we could get the result tf from, 2247 * but earlier chipsets do not. For now, we ignore those registers. 2248 */ 2249 static unsigned int mv_qc_issue_fis(struct ata_queued_cmd *qc) 2250 { 2251 struct ata_port *ap = qc->ap; 2252 struct mv_port_priv *pp = ap->private_data; 2253 struct ata_link *link = qc->dev->link; 2254 u32 fis[5]; 2255 int err = 0; 2256 2257 ata_tf_to_fis(&qc->tf, link->pmp, 1, (void *)fis); 2258 err = mv_send_fis(ap, fis, ARRAY_SIZE(fis)); 2259 if (err) 2260 return err; 2261 2262 switch (qc->tf.protocol) { 2263 case ATAPI_PROT_PIO: 2264 pp->pp_flags |= MV_PP_FLAG_FAKE_ATA_BUSY; 2265 fallthrough; 2266 case ATAPI_PROT_NODATA: 2267 ap->hsm_task_state = HSM_ST_FIRST; 2268 break; 2269 case ATA_PROT_PIO: 2270 pp->pp_flags |= MV_PP_FLAG_FAKE_ATA_BUSY; 2271 if (qc->tf.flags & ATA_TFLAG_WRITE) 2272 ap->hsm_task_state = HSM_ST_FIRST; 2273 else 2274 ap->hsm_task_state = HSM_ST; 2275 break; 2276 default: 2277 ap->hsm_task_state = HSM_ST_LAST; 2278 break; 2279 } 2280 2281 if (qc->tf.flags & ATA_TFLAG_POLLING) 2282 ata_sff_queue_pio_task(link, 0); 2283 return 0; 2284 } 2285 2286 /** 2287 * mv_qc_issue - Initiate a command to the host 2288 * @qc: queued command to start 2289 * 2290 * This routine simply redirects to the general purpose routine 2291 * if command is not DMA. Else, it sanity checks our local 2292 * caches of the request producer/consumer indices then enables 2293 * DMA and bumps the request producer index. 2294 * 2295 * LOCKING: 2296 * Inherited from caller. 2297 */ 2298 static unsigned int mv_qc_issue(struct ata_queued_cmd *qc) 2299 { 2300 static int limit_warnings = 10; 2301 struct ata_port *ap = qc->ap; 2302 void __iomem *port_mmio = mv_ap_base(ap); 2303 struct mv_port_priv *pp = ap->private_data; 2304 u32 in_index; 2305 unsigned int port_irqs; 2306 2307 pp->pp_flags &= ~MV_PP_FLAG_FAKE_ATA_BUSY; /* paranoia */ 2308 2309 switch (qc->tf.protocol) { 2310 case ATA_PROT_DMA: 2311 if (qc->tf.command == ATA_CMD_DSM) { 2312 if (!ap->ops->bmdma_setup) /* no bmdma on GEN_I */ 2313 return AC_ERR_OTHER; 2314 break; /* use bmdma for this */ 2315 } 2316 fallthrough; 2317 case ATA_PROT_NCQ: 2318 mv_start_edma(ap, port_mmio, pp, qc->tf.protocol); 2319 pp->req_idx = (pp->req_idx + 1) & MV_MAX_Q_DEPTH_MASK; 2320 in_index = pp->req_idx << EDMA_REQ_Q_PTR_SHIFT; 2321 2322 /* Write the request in pointer to kick the EDMA to life */ 2323 writelfl((pp->crqb_dma & EDMA_REQ_Q_BASE_LO_MASK) | in_index, 2324 port_mmio + EDMA_REQ_Q_IN_PTR); 2325 return 0; 2326 2327 case ATA_PROT_PIO: 2328 /* 2329 * Errata SATA#16, SATA#24: warn if multiple DRQs expected. 2330 * 2331 * Someday, we might implement special polling workarounds 2332 * for these, but it all seems rather unnecessary since we 2333 * normally use only DMA for commands which transfer more 2334 * than a single block of data. 2335 * 2336 * Much of the time, this could just work regardless. 2337 * So for now, just log the incident, and allow the attempt. 2338 */ 2339 if (limit_warnings > 0 && (qc->nbytes / qc->sect_size) > 1) { 2340 --limit_warnings; 2341 ata_link_warn(qc->dev->link, DRV_NAME 2342 ": attempting PIO w/multiple DRQ: " 2343 "this may fail due to h/w errata\n"); 2344 } 2345 fallthrough; 2346 case ATA_PROT_NODATA: 2347 case ATAPI_PROT_PIO: 2348 case ATAPI_PROT_NODATA: 2349 if (ap->flags & ATA_FLAG_PIO_POLLING) 2350 qc->tf.flags |= ATA_TFLAG_POLLING; 2351 break; 2352 } 2353 2354 if (qc->tf.flags & ATA_TFLAG_POLLING) 2355 port_irqs = ERR_IRQ; /* mask device interrupt when polling */ 2356 else 2357 port_irqs = ERR_IRQ | DONE_IRQ; /* unmask all interrupts */ 2358 2359 /* 2360 * We're about to send a non-EDMA capable command to the 2361 * port. Turn off EDMA so there won't be problems accessing 2362 * shadow block, etc registers. 2363 */ 2364 mv_stop_edma(ap); 2365 mv_clear_and_enable_port_irqs(ap, mv_ap_base(ap), port_irqs); 2366 mv_pmp_select(ap, qc->dev->link->pmp); 2367 2368 if (qc->tf.command == ATA_CMD_READ_LOG_EXT) { 2369 struct mv_host_priv *hpriv = ap->host->private_data; 2370 /* 2371 * Workaround for 88SX60x1 FEr SATA#25 (part 2). 2372 * 2373 * After any NCQ error, the READ_LOG_EXT command 2374 * from libata-eh *must* use mv_qc_issue_fis(). 2375 * Otherwise it might fail, due to chip errata. 2376 * 2377 * Rather than special-case it, we'll just *always* 2378 * use this method here for READ_LOG_EXT, making for 2379 * easier testing. 2380 */ 2381 if (IS_GEN_II(hpriv)) 2382 return mv_qc_issue_fis(qc); 2383 } 2384 return ata_bmdma_qc_issue(qc); 2385 } 2386 2387 static struct ata_queued_cmd *mv_get_active_qc(struct ata_port *ap) 2388 { 2389 struct mv_port_priv *pp = ap->private_data; 2390 struct ata_queued_cmd *qc; 2391 2392 if (pp->pp_flags & MV_PP_FLAG_NCQ_EN) 2393 return NULL; 2394 qc = ata_qc_from_tag(ap, ap->link.active_tag); 2395 if (qc && !(qc->tf.flags & ATA_TFLAG_POLLING)) 2396 return qc; 2397 return NULL; 2398 } 2399 2400 static void mv_pmp_error_handler(struct ata_port *ap) 2401 __must_hold(&ap->host->eh_mutex) 2402 { 2403 unsigned int pmp, pmp_map; 2404 struct mv_port_priv *pp = ap->private_data; 2405 2406 if (pp->pp_flags & MV_PP_FLAG_DELAYED_EH) { 2407 /* 2408 * Perform NCQ error analysis on failed PMPs 2409 * before we freeze the port entirely. 2410 * 2411 * The failed PMPs are marked earlier by mv_pmp_eh_prep(). 2412 */ 2413 pmp_map = pp->delayed_eh_pmp_map; 2414 pp->pp_flags &= ~MV_PP_FLAG_DELAYED_EH; 2415 for (pmp = 0; pmp_map != 0; pmp++) { 2416 unsigned int this_pmp = (1 << pmp); 2417 if (pmp_map & this_pmp) { 2418 struct ata_link *link = &ap->pmp_link[pmp]; 2419 pmp_map &= ~this_pmp; 2420 ata_eh_analyze_ncq_error(link); 2421 } 2422 } 2423 ata_port_freeze(ap); 2424 } 2425 sata_pmp_error_handler(ap); 2426 } 2427 2428 static unsigned int mv_get_err_pmp_map(struct ata_port *ap) 2429 { 2430 void __iomem *port_mmio = mv_ap_base(ap); 2431 2432 return readl(port_mmio + SATA_TESTCTL) >> 16; 2433 } 2434 2435 static void mv_pmp_eh_prep(struct ata_port *ap, unsigned int pmp_map) 2436 { 2437 unsigned int pmp; 2438 2439 /* 2440 * Initialize EH info for PMPs which saw device errors 2441 */ 2442 for (pmp = 0; pmp_map != 0; pmp++) { 2443 unsigned int this_pmp = (1 << pmp); 2444 if (pmp_map & this_pmp) { 2445 struct ata_link *link = &ap->pmp_link[pmp]; 2446 struct ata_eh_info *ehi = &link->eh_info; 2447 2448 pmp_map &= ~this_pmp; 2449 ata_ehi_clear_desc(ehi); 2450 ata_ehi_push_desc(ehi, "dev err"); 2451 ehi->err_mask |= AC_ERR_DEV; 2452 ehi->action |= ATA_EH_RESET; 2453 ata_link_abort(link); 2454 } 2455 } 2456 } 2457 2458 static int mv_req_q_empty(struct ata_port *ap) 2459 { 2460 void __iomem *port_mmio = mv_ap_base(ap); 2461 u32 in_ptr, out_ptr; 2462 2463 in_ptr = (readl(port_mmio + EDMA_REQ_Q_IN_PTR) 2464 >> EDMA_REQ_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK; 2465 out_ptr = (readl(port_mmio + EDMA_REQ_Q_OUT_PTR) 2466 >> EDMA_REQ_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK; 2467 return (in_ptr == out_ptr); /* 1 == queue_is_empty */ 2468 } 2469 2470 static int mv_handle_fbs_ncq_dev_err(struct ata_port *ap) 2471 { 2472 struct mv_port_priv *pp = ap->private_data; 2473 int failed_links; 2474 unsigned int old_map, new_map; 2475 2476 /* 2477 * Device error during FBS+NCQ operation: 2478 * 2479 * Set a port flag to prevent further I/O being enqueued. 2480 * Leave the EDMA running to drain outstanding commands from this port. 2481 * Perform the post-mortem/EH only when all responses are complete. 2482 * Follow recovery sequence from 6042/7042 datasheet (7.3.15.4.2.2). 2483 */ 2484 if (!(pp->pp_flags & MV_PP_FLAG_DELAYED_EH)) { 2485 pp->pp_flags |= MV_PP_FLAG_DELAYED_EH; 2486 pp->delayed_eh_pmp_map = 0; 2487 } 2488 old_map = pp->delayed_eh_pmp_map; 2489 new_map = old_map | mv_get_err_pmp_map(ap); 2490 2491 if (old_map != new_map) { 2492 pp->delayed_eh_pmp_map = new_map; 2493 mv_pmp_eh_prep(ap, new_map & ~old_map); 2494 } 2495 failed_links = hweight16(new_map); 2496 2497 ata_port_info(ap, 2498 "%s: pmp_map=%04x qc_map=%04llx failed_links=%d nr_active_links=%d\n", 2499 __func__, pp->delayed_eh_pmp_map, 2500 ap->qc_active, failed_links, 2501 ap->nr_active_links); 2502 2503 if (ap->nr_active_links <= failed_links && mv_req_q_empty(ap)) { 2504 mv_process_crpb_entries(ap, pp); 2505 mv_stop_edma(ap); 2506 mv_eh_freeze(ap); 2507 ata_port_info(ap, "%s: done\n", __func__); 2508 return 1; /* handled */ 2509 } 2510 ata_port_info(ap, "%s: waiting\n", __func__); 2511 return 1; /* handled */ 2512 } 2513 2514 static int mv_handle_fbs_non_ncq_dev_err(struct ata_port *ap) 2515 { 2516 /* 2517 * Possible future enhancement: 2518 * 2519 * FBS+non-NCQ operation is not yet implemented. 2520 * See related notes in mv_edma_cfg(). 2521 * 2522 * Device error during FBS+non-NCQ operation: 2523 * 2524 * We need to snapshot the shadow registers for each failed command. 2525 * Follow recovery sequence from 6042/7042 datasheet (7.3.15.4.2.3). 2526 */ 2527 return 0; /* not handled */ 2528 } 2529 2530 static int mv_handle_dev_err(struct ata_port *ap, u32 edma_err_cause) 2531 { 2532 struct mv_port_priv *pp = ap->private_data; 2533 2534 if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN)) 2535 return 0; /* EDMA was not active: not handled */ 2536 if (!(pp->pp_flags & MV_PP_FLAG_FBS_EN)) 2537 return 0; /* FBS was not active: not handled */ 2538 2539 if (!(edma_err_cause & EDMA_ERR_DEV)) 2540 return 0; /* non DEV error: not handled */ 2541 edma_err_cause &= ~EDMA_ERR_IRQ_TRANSIENT; 2542 if (edma_err_cause & ~(EDMA_ERR_DEV | EDMA_ERR_SELF_DIS)) 2543 return 0; /* other problems: not handled */ 2544 2545 if (pp->pp_flags & MV_PP_FLAG_NCQ_EN) { 2546 /* 2547 * EDMA should NOT have self-disabled for this case. 2548 * If it did, then something is wrong elsewhere, 2549 * and we cannot handle it here. 2550 */ 2551 if (edma_err_cause & EDMA_ERR_SELF_DIS) { 2552 ata_port_warn(ap, "%s: err_cause=0x%x pp_flags=0x%x\n", 2553 __func__, edma_err_cause, pp->pp_flags); 2554 return 0; /* not handled */ 2555 } 2556 return mv_handle_fbs_ncq_dev_err(ap); 2557 } else { 2558 /* 2559 * EDMA should have self-disabled for this case. 2560 * If it did not, then something is wrong elsewhere, 2561 * and we cannot handle it here. 2562 */ 2563 if (!(edma_err_cause & EDMA_ERR_SELF_DIS)) { 2564 ata_port_warn(ap, "%s: err_cause=0x%x pp_flags=0x%x\n", 2565 __func__, edma_err_cause, pp->pp_flags); 2566 return 0; /* not handled */ 2567 } 2568 return mv_handle_fbs_non_ncq_dev_err(ap); 2569 } 2570 return 0; /* not handled */ 2571 } 2572 2573 static void mv_unexpected_intr(struct ata_port *ap, int edma_was_enabled) 2574 { 2575 struct ata_eh_info *ehi = &ap->link.eh_info; 2576 char *when = "idle"; 2577 2578 ata_ehi_clear_desc(ehi); 2579 if (edma_was_enabled) { 2580 when = "EDMA enabled"; 2581 } else { 2582 struct ata_queued_cmd *qc = ata_qc_from_tag(ap, ap->link.active_tag); 2583 if (qc && (qc->tf.flags & ATA_TFLAG_POLLING)) 2584 when = "polling"; 2585 } 2586 ata_ehi_push_desc(ehi, "unexpected device interrupt while %s", when); 2587 ehi->err_mask |= AC_ERR_OTHER; 2588 ehi->action |= ATA_EH_RESET; 2589 ata_port_freeze(ap); 2590 } 2591 2592 /** 2593 * mv_err_intr - Handle error interrupts on the port 2594 * @ap: ATA channel to manipulate 2595 * 2596 * Most cases require a full reset of the chip's state machine, 2597 * which also performs a COMRESET. 2598 * Also, if the port disabled DMA, update our cached copy to match. 2599 * 2600 * LOCKING: 2601 * Inherited from caller. 2602 */ 2603 static void mv_err_intr(struct ata_port *ap) 2604 { 2605 void __iomem *port_mmio = mv_ap_base(ap); 2606 u32 edma_err_cause, eh_freeze_mask, serr = 0; 2607 u32 fis_cause = 0; 2608 struct mv_port_priv *pp = ap->private_data; 2609 struct mv_host_priv *hpriv = ap->host->private_data; 2610 unsigned int action = 0, err_mask = 0; 2611 struct ata_eh_info *ehi = &ap->link.eh_info; 2612 struct ata_queued_cmd *qc; 2613 int abort = 0; 2614 2615 /* 2616 * Read and clear the SError and err_cause bits. 2617 * For GenIIe, if EDMA_ERR_TRANS_IRQ_7 is set, we also must read/clear 2618 * the FIS_IRQ_CAUSE register before clearing edma_err_cause. 2619 */ 2620 sata_scr_read(&ap->link, SCR_ERROR, &serr); 2621 sata_scr_write_flush(&ap->link, SCR_ERROR, serr); 2622 2623 edma_err_cause = readl(port_mmio + EDMA_ERR_IRQ_CAUSE); 2624 if (IS_GEN_IIE(hpriv) && (edma_err_cause & EDMA_ERR_TRANS_IRQ_7)) { 2625 fis_cause = readl(port_mmio + FIS_IRQ_CAUSE); 2626 writelfl(~fis_cause, port_mmio + FIS_IRQ_CAUSE); 2627 } 2628 writelfl(~edma_err_cause, port_mmio + EDMA_ERR_IRQ_CAUSE); 2629 2630 if (edma_err_cause & EDMA_ERR_DEV) { 2631 /* 2632 * Device errors during FIS-based switching operation 2633 * require special handling. 2634 */ 2635 if (mv_handle_dev_err(ap, edma_err_cause)) 2636 return; 2637 } 2638 2639 qc = mv_get_active_qc(ap); 2640 ata_ehi_clear_desc(ehi); 2641 ata_ehi_push_desc(ehi, "edma_err_cause=%08x pp_flags=%08x", 2642 edma_err_cause, pp->pp_flags); 2643 2644 if (IS_GEN_IIE(hpriv) && (edma_err_cause & EDMA_ERR_TRANS_IRQ_7)) { 2645 ata_ehi_push_desc(ehi, "fis_cause=%08x", fis_cause); 2646 if (fis_cause & FIS_IRQ_CAUSE_AN) { 2647 u32 ec = edma_err_cause & 2648 ~(EDMA_ERR_TRANS_IRQ_7 | EDMA_ERR_IRQ_TRANSIENT); 2649 sata_async_notification(ap); 2650 if (!ec) 2651 return; /* Just an AN; no need for the nukes */ 2652 ata_ehi_push_desc(ehi, "SDB notify"); 2653 } 2654 } 2655 /* 2656 * All generations share these EDMA error cause bits: 2657 */ 2658 if (edma_err_cause & EDMA_ERR_DEV) { 2659 err_mask |= AC_ERR_DEV; 2660 action |= ATA_EH_RESET; 2661 ata_ehi_push_desc(ehi, "dev error"); 2662 } 2663 if (edma_err_cause & (EDMA_ERR_D_PAR | EDMA_ERR_PRD_PAR | 2664 EDMA_ERR_CRQB_PAR | EDMA_ERR_CRPB_PAR | 2665 EDMA_ERR_INTRL_PAR)) { 2666 err_mask |= AC_ERR_ATA_BUS; 2667 action |= ATA_EH_RESET; 2668 ata_ehi_push_desc(ehi, "parity error"); 2669 } 2670 if (edma_err_cause & (EDMA_ERR_DEV_DCON | EDMA_ERR_DEV_CON)) { 2671 ata_ehi_hotplugged(ehi); 2672 ata_ehi_push_desc(ehi, edma_err_cause & EDMA_ERR_DEV_DCON ? 2673 "dev disconnect" : "dev connect"); 2674 action |= ATA_EH_RESET; 2675 } 2676 2677 /* 2678 * Gen-I has a different SELF_DIS bit, 2679 * different FREEZE bits, and no SERR bit: 2680 */ 2681 if (IS_GEN_I(hpriv)) { 2682 eh_freeze_mask = EDMA_EH_FREEZE_5; 2683 if (edma_err_cause & EDMA_ERR_SELF_DIS_5) { 2684 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN; 2685 ata_ehi_push_desc(ehi, "EDMA self-disable"); 2686 } 2687 } else { 2688 eh_freeze_mask = EDMA_EH_FREEZE; 2689 if (edma_err_cause & EDMA_ERR_SELF_DIS) { 2690 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN; 2691 ata_ehi_push_desc(ehi, "EDMA self-disable"); 2692 } 2693 if (edma_err_cause & EDMA_ERR_SERR) { 2694 ata_ehi_push_desc(ehi, "SError=%08x", serr); 2695 err_mask |= AC_ERR_ATA_BUS; 2696 action |= ATA_EH_RESET; 2697 } 2698 } 2699 2700 if (!err_mask) { 2701 err_mask = AC_ERR_OTHER; 2702 action |= ATA_EH_RESET; 2703 } 2704 2705 ehi->serror |= serr; 2706 ehi->action |= action; 2707 2708 if (qc) 2709 qc->err_mask |= err_mask; 2710 else 2711 ehi->err_mask |= err_mask; 2712 2713 if (err_mask == AC_ERR_DEV) { 2714 /* 2715 * Cannot do ata_port_freeze() here, 2716 * because it would kill PIO access, 2717 * which is needed for further diagnosis. 2718 */ 2719 mv_eh_freeze(ap); 2720 abort = 1; 2721 } else if (edma_err_cause & eh_freeze_mask) { 2722 /* 2723 * Note to self: ata_port_freeze() calls ata_port_abort() 2724 */ 2725 ata_port_freeze(ap); 2726 } else { 2727 abort = 1; 2728 } 2729 2730 if (abort) { 2731 if (qc) 2732 ata_link_abort(qc->dev->link); 2733 else 2734 ata_port_abort(ap); 2735 } 2736 } 2737 2738 static bool mv_process_crpb_response(struct ata_port *ap, 2739 struct mv_crpb *response, unsigned int tag, int ncq_enabled) 2740 { 2741 u8 ata_status; 2742 u16 edma_status = le16_to_cpu(response->flags); 2743 2744 /* 2745 * edma_status from a response queue entry: 2746 * LSB is from EDMA_ERR_IRQ_CAUSE (non-NCQ only). 2747 * MSB is saved ATA status from command completion. 2748 */ 2749 if (!ncq_enabled) { 2750 u8 err_cause = edma_status & 0xff & ~EDMA_ERR_DEV; 2751 if (err_cause) { 2752 /* 2753 * Error will be seen/handled by 2754 * mv_err_intr(). So do nothing at all here. 2755 */ 2756 return false; 2757 } 2758 } 2759 ata_status = edma_status >> CRPB_FLAG_STATUS_SHIFT; 2760 if (!ac_err_mask(ata_status)) 2761 return true; 2762 /* else: leave it for mv_err_intr() */ 2763 return false; 2764 } 2765 2766 static void mv_process_crpb_entries(struct ata_port *ap, struct mv_port_priv *pp) 2767 { 2768 void __iomem *port_mmio = mv_ap_base(ap); 2769 struct mv_host_priv *hpriv = ap->host->private_data; 2770 u32 in_index; 2771 bool work_done = false; 2772 u32 done_mask = 0; 2773 int ncq_enabled = (pp->pp_flags & MV_PP_FLAG_NCQ_EN); 2774 2775 /* Get the hardware queue position index */ 2776 in_index = (readl(port_mmio + EDMA_RSP_Q_IN_PTR) 2777 >> EDMA_RSP_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK; 2778 2779 /* Process new responses from since the last time we looked */ 2780 while (in_index != pp->resp_idx) { 2781 unsigned int tag; 2782 struct mv_crpb *response = &pp->crpb[pp->resp_idx]; 2783 2784 pp->resp_idx = (pp->resp_idx + 1) & MV_MAX_Q_DEPTH_MASK; 2785 2786 if (IS_GEN_I(hpriv)) { 2787 /* 50xx: no NCQ, only one command active at a time */ 2788 tag = ap->link.active_tag; 2789 } else { 2790 /* Gen II/IIE: get command tag from CRPB entry */ 2791 tag = le16_to_cpu(response->id) & 0x1f; 2792 } 2793 if (mv_process_crpb_response(ap, response, tag, ncq_enabled)) 2794 done_mask |= 1 << tag; 2795 work_done = true; 2796 } 2797 2798 if (work_done) { 2799 ata_qc_complete_multiple(ap, ata_qc_get_active(ap) ^ done_mask); 2800 2801 /* Update the software queue position index in hardware */ 2802 writelfl((pp->crpb_dma & EDMA_RSP_Q_BASE_LO_MASK) | 2803 (pp->resp_idx << EDMA_RSP_Q_PTR_SHIFT), 2804 port_mmio + EDMA_RSP_Q_OUT_PTR); 2805 } 2806 } 2807 2808 static void mv_port_intr(struct ata_port *ap, u32 port_cause) 2809 { 2810 struct mv_port_priv *pp; 2811 int edma_was_enabled; 2812 2813 /* 2814 * Grab a snapshot of the EDMA_EN flag setting, 2815 * so that we have a consistent view for this port, 2816 * even if something we call of our routines changes it. 2817 */ 2818 pp = ap->private_data; 2819 edma_was_enabled = (pp->pp_flags & MV_PP_FLAG_EDMA_EN); 2820 /* 2821 * Process completed CRPB response(s) before other events. 2822 */ 2823 if (edma_was_enabled && (port_cause & DONE_IRQ)) { 2824 mv_process_crpb_entries(ap, pp); 2825 if (pp->pp_flags & MV_PP_FLAG_DELAYED_EH) 2826 mv_handle_fbs_ncq_dev_err(ap); 2827 } 2828 /* 2829 * Handle chip-reported errors, or continue on to handle PIO. 2830 */ 2831 if (unlikely(port_cause & ERR_IRQ)) { 2832 mv_err_intr(ap); 2833 } else if (!edma_was_enabled) { 2834 struct ata_queued_cmd *qc = mv_get_active_qc(ap); 2835 if (qc) 2836 ata_bmdma_port_intr(ap, qc); 2837 else 2838 mv_unexpected_intr(ap, edma_was_enabled); 2839 } 2840 } 2841 2842 /** 2843 * mv_host_intr - Handle all interrupts on the given host controller 2844 * @host: host specific structure 2845 * @main_irq_cause: Main interrupt cause register for the chip. 2846 * 2847 * LOCKING: 2848 * Inherited from caller. 2849 */ 2850 static int mv_host_intr(struct ata_host *host, u32 main_irq_cause) 2851 { 2852 struct mv_host_priv *hpriv = host->private_data; 2853 void __iomem *mmio = hpriv->base, *hc_mmio; 2854 unsigned int handled = 0, port; 2855 2856 /* If asserted, clear the "all ports" IRQ coalescing bit */ 2857 if (main_irq_cause & ALL_PORTS_COAL_DONE) 2858 writel(~ALL_PORTS_COAL_IRQ, mmio + IRQ_COAL_CAUSE); 2859 2860 for (port = 0; port < hpriv->n_ports; port++) { 2861 struct ata_port *ap = host->ports[port]; 2862 unsigned int p, shift, hardport, port_cause; 2863 2864 MV_PORT_TO_SHIFT_AND_HARDPORT(port, shift, hardport); 2865 /* 2866 * Each hc within the host has its own hc_irq_cause register, 2867 * where the interrupting ports bits get ack'd. 2868 */ 2869 if (hardport == 0) { /* first port on this hc ? */ 2870 u32 hc_cause = (main_irq_cause >> shift) & HC0_IRQ_PEND; 2871 u32 port_mask, ack_irqs; 2872 /* 2873 * Skip this entire hc if nothing pending for any ports 2874 */ 2875 if (!hc_cause) { 2876 port += MV_PORTS_PER_HC - 1; 2877 continue; 2878 } 2879 /* 2880 * We don't need/want to read the hc_irq_cause register, 2881 * because doing so hurts performance, and 2882 * main_irq_cause already gives us everything we need. 2883 * 2884 * But we do have to *write* to the hc_irq_cause to ack 2885 * the ports that we are handling this time through. 2886 * 2887 * This requires that we create a bitmap for those 2888 * ports which interrupted us, and use that bitmap 2889 * to ack (only) those ports via hc_irq_cause. 2890 */ 2891 ack_irqs = 0; 2892 if (hc_cause & PORTS_0_3_COAL_DONE) 2893 ack_irqs = HC_COAL_IRQ; 2894 for (p = 0; p < MV_PORTS_PER_HC; ++p) { 2895 if ((port + p) >= hpriv->n_ports) 2896 break; 2897 port_mask = (DONE_IRQ | ERR_IRQ) << (p * 2); 2898 if (hc_cause & port_mask) 2899 ack_irqs |= (DMA_IRQ | DEV_IRQ) << p; 2900 } 2901 hc_mmio = mv_hc_base_from_port(mmio, port); 2902 writelfl(~ack_irqs, hc_mmio + HC_IRQ_CAUSE); 2903 handled = 1; 2904 } 2905 /* 2906 * Handle interrupts signalled for this port: 2907 */ 2908 port_cause = (main_irq_cause >> shift) & (DONE_IRQ | ERR_IRQ); 2909 if (port_cause) 2910 mv_port_intr(ap, port_cause); 2911 } 2912 return handled; 2913 } 2914 2915 static int mv_pci_error(struct ata_host *host, void __iomem *mmio) 2916 { 2917 struct mv_host_priv *hpriv = host->private_data; 2918 struct ata_port *ap; 2919 struct ata_queued_cmd *qc; 2920 struct ata_eh_info *ehi; 2921 unsigned int i, err_mask, printed = 0; 2922 u32 err_cause; 2923 2924 err_cause = readl(mmio + hpriv->irq_cause_offset); 2925 2926 dev_err(host->dev, "PCI ERROR; PCI IRQ cause=0x%08x\n", err_cause); 2927 2928 dev_dbg(host->dev, "%s: All regs @ PCI error\n", __func__); 2929 mv_dump_all_regs(mmio, to_pci_dev(host->dev)); 2930 2931 writelfl(0, mmio + hpriv->irq_cause_offset); 2932 2933 for (i = 0; i < host->n_ports; i++) { 2934 ap = host->ports[i]; 2935 if (!ata_link_offline(&ap->link)) { 2936 ehi = &ap->link.eh_info; 2937 ata_ehi_clear_desc(ehi); 2938 if (!printed++) 2939 ata_ehi_push_desc(ehi, 2940 "PCI err cause 0x%08x", err_cause); 2941 err_mask = AC_ERR_HOST_BUS; 2942 ehi->action = ATA_EH_RESET; 2943 qc = ata_qc_from_tag(ap, ap->link.active_tag); 2944 if (qc) 2945 qc->err_mask |= err_mask; 2946 else 2947 ehi->err_mask |= err_mask; 2948 2949 ata_port_freeze(ap); 2950 } 2951 } 2952 return 1; /* handled */ 2953 } 2954 2955 /** 2956 * mv_interrupt - Main interrupt event handler 2957 * @irq: unused 2958 * @dev_instance: private data; in this case the host structure 2959 * 2960 * Read the read only register to determine if any host 2961 * controllers have pending interrupts. If so, call lower level 2962 * routine to handle. Also check for PCI errors which are only 2963 * reported here. 2964 * 2965 * LOCKING: 2966 * This routine holds the host lock while processing pending 2967 * interrupts. 2968 */ 2969 static irqreturn_t mv_interrupt(int irq, void *dev_instance) 2970 { 2971 struct ata_host *host = dev_instance; 2972 struct mv_host_priv *hpriv = host->private_data; 2973 unsigned int handled = 0; 2974 int using_msi = hpriv->hp_flags & MV_HP_FLAG_MSI; 2975 u32 main_irq_cause, pending_irqs; 2976 2977 spin_lock(&host->lock); 2978 2979 /* for MSI: block new interrupts while in here */ 2980 if (using_msi) 2981 mv_write_main_irq_mask(0, hpriv); 2982 2983 main_irq_cause = readl(hpriv->main_irq_cause_addr); 2984 pending_irqs = main_irq_cause & hpriv->main_irq_mask; 2985 /* 2986 * Deal with cases where we either have nothing pending, or have read 2987 * a bogus register value which can indicate HW removal or PCI fault. 2988 */ 2989 if (pending_irqs && main_irq_cause != 0xffffffffU) { 2990 if (unlikely((pending_irqs & PCI_ERR) && !IS_SOC(hpriv))) 2991 handled = mv_pci_error(host, hpriv->base); 2992 else 2993 handled = mv_host_intr(host, pending_irqs); 2994 } 2995 2996 /* for MSI: unmask; interrupt cause bits will retrigger now */ 2997 if (using_msi) 2998 mv_write_main_irq_mask(hpriv->main_irq_mask, hpriv); 2999 3000 spin_unlock(&host->lock); 3001 3002 return IRQ_RETVAL(handled); 3003 } 3004 3005 static unsigned int mv5_scr_offset(unsigned int sc_reg_in) 3006 { 3007 unsigned int ofs; 3008 3009 switch (sc_reg_in) { 3010 case SCR_STATUS: 3011 case SCR_ERROR: 3012 case SCR_CONTROL: 3013 ofs = sc_reg_in * sizeof(u32); 3014 break; 3015 default: 3016 ofs = 0xffffffffU; 3017 break; 3018 } 3019 return ofs; 3020 } 3021 3022 static int mv5_scr_read(struct ata_link *link, unsigned int sc_reg_in, u32 *val) 3023 { 3024 struct mv_host_priv *hpriv = link->ap->host->private_data; 3025 void __iomem *mmio = hpriv->base; 3026 void __iomem *addr = mv5_phy_base(mmio, link->ap->port_no); 3027 unsigned int ofs = mv5_scr_offset(sc_reg_in); 3028 3029 if (ofs != 0xffffffffU) { 3030 *val = readl(addr + ofs); 3031 return 0; 3032 } else 3033 return -EINVAL; 3034 } 3035 3036 static int mv5_scr_write(struct ata_link *link, unsigned int sc_reg_in, u32 val) 3037 { 3038 struct mv_host_priv *hpriv = link->ap->host->private_data; 3039 void __iomem *mmio = hpriv->base; 3040 void __iomem *addr = mv5_phy_base(mmio, link->ap->port_no); 3041 unsigned int ofs = mv5_scr_offset(sc_reg_in); 3042 3043 if (ofs != 0xffffffffU) { 3044 writelfl(val, addr + ofs); 3045 return 0; 3046 } else 3047 return -EINVAL; 3048 } 3049 3050 static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio) 3051 { 3052 struct pci_dev *pdev = to_pci_dev(host->dev); 3053 int early_5080; 3054 3055 early_5080 = (pdev->device == 0x5080) && (pdev->revision == 0); 3056 3057 if (!early_5080) { 3058 u32 tmp = readl(mmio + MV_PCI_EXP_ROM_BAR_CTL); 3059 tmp |= (1 << 0); 3060 writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL); 3061 } 3062 3063 mv_reset_pci_bus(host, mmio); 3064 } 3065 3066 static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio) 3067 { 3068 writel(0x0fcfffff, mmio + FLASH_CTL); 3069 } 3070 3071 static void mv5_read_preamp(struct mv_host_priv *hpriv, int idx, 3072 void __iomem *mmio) 3073 { 3074 void __iomem *phy_mmio = mv5_phy_base(mmio, idx); 3075 u32 tmp; 3076 3077 tmp = readl(phy_mmio + MV5_PHY_MODE); 3078 3079 hpriv->signal[idx].pre = tmp & 0x1800; /* bits 12:11 */ 3080 hpriv->signal[idx].amps = tmp & 0xe0; /* bits 7:5 */ 3081 } 3082 3083 static void mv5_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio) 3084 { 3085 u32 tmp; 3086 3087 writel(0, mmio + GPIO_PORT_CTL); 3088 3089 /* FIXME: handle MV_HP_ERRATA_50XXB2 errata */ 3090 3091 tmp = readl(mmio + MV_PCI_EXP_ROM_BAR_CTL); 3092 tmp |= ~(1 << 0); 3093 writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL); 3094 } 3095 3096 static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio, 3097 unsigned int port) 3098 { 3099 void __iomem *phy_mmio = mv5_phy_base(mmio, port); 3100 const u32 mask = (1<<12) | (1<<11) | (1<<7) | (1<<6) | (1<<5); 3101 u32 tmp; 3102 int fix_apm_sq = (hpriv->hp_flags & MV_HP_ERRATA_50XXB0); 3103 3104 if (fix_apm_sq) { 3105 tmp = readl(phy_mmio + MV5_LTMODE); 3106 tmp |= (1 << 19); 3107 writel(tmp, phy_mmio + MV5_LTMODE); 3108 3109 tmp = readl(phy_mmio + MV5_PHY_CTL); 3110 tmp &= ~0x3; 3111 tmp |= 0x1; 3112 writel(tmp, phy_mmio + MV5_PHY_CTL); 3113 } 3114 3115 tmp = readl(phy_mmio + MV5_PHY_MODE); 3116 tmp &= ~mask; 3117 tmp |= hpriv->signal[port].pre; 3118 tmp |= hpriv->signal[port].amps; 3119 writel(tmp, phy_mmio + MV5_PHY_MODE); 3120 } 3121 3122 3123 #undef ZERO 3124 #define ZERO(reg) writel(0, port_mmio + (reg)) 3125 static void mv5_reset_hc_port(struct mv_host_priv *hpriv, void __iomem *mmio, 3126 unsigned int port) 3127 { 3128 void __iomem *port_mmio = mv_port_base(mmio, port); 3129 3130 mv_reset_channel(hpriv, mmio, port); 3131 3132 ZERO(0x028); /* command */ 3133 writel(0x11f, port_mmio + EDMA_CFG); 3134 ZERO(0x004); /* timer */ 3135 ZERO(0x008); /* irq err cause */ 3136 ZERO(0x00c); /* irq err mask */ 3137 ZERO(0x010); /* rq bah */ 3138 ZERO(0x014); /* rq inp */ 3139 ZERO(0x018); /* rq outp */ 3140 ZERO(0x01c); /* respq bah */ 3141 ZERO(0x024); /* respq outp */ 3142 ZERO(0x020); /* respq inp */ 3143 ZERO(0x02c); /* test control */ 3144 writel(0xbc, port_mmio + EDMA_IORDY_TMOUT); 3145 } 3146 #undef ZERO 3147 3148 #define ZERO(reg) writel(0, hc_mmio + (reg)) 3149 static void mv5_reset_one_hc(struct mv_host_priv *hpriv, void __iomem *mmio, 3150 unsigned int hc) 3151 { 3152 void __iomem *hc_mmio = mv_hc_base(mmio, hc); 3153 u32 tmp; 3154 3155 ZERO(0x00c); 3156 ZERO(0x010); 3157 ZERO(0x014); 3158 ZERO(0x018); 3159 3160 tmp = readl(hc_mmio + 0x20); 3161 tmp &= 0x1c1c1c1c; 3162 tmp |= 0x03030303; 3163 writel(tmp, hc_mmio + 0x20); 3164 } 3165 #undef ZERO 3166 3167 static int mv5_reset_hc(struct ata_host *host, void __iomem *mmio, 3168 unsigned int n_hc) 3169 { 3170 struct mv_host_priv *hpriv = host->private_data; 3171 unsigned int hc, port; 3172 3173 for (hc = 0; hc < n_hc; hc++) { 3174 for (port = 0; port < MV_PORTS_PER_HC; port++) 3175 mv5_reset_hc_port(hpriv, mmio, 3176 (hc * MV_PORTS_PER_HC) + port); 3177 3178 mv5_reset_one_hc(hpriv, mmio, hc); 3179 } 3180 3181 return 0; 3182 } 3183 3184 #undef ZERO 3185 #define ZERO(reg) writel(0, mmio + (reg)) 3186 static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio) 3187 { 3188 struct mv_host_priv *hpriv = host->private_data; 3189 u32 tmp; 3190 3191 tmp = readl(mmio + MV_PCI_MODE); 3192 tmp &= 0xff00ffff; 3193 writel(tmp, mmio + MV_PCI_MODE); 3194 3195 ZERO(MV_PCI_DISC_TIMER); 3196 ZERO(MV_PCI_MSI_TRIGGER); 3197 writel(0x000100ff, mmio + MV_PCI_XBAR_TMOUT); 3198 ZERO(MV_PCI_SERR_MASK); 3199 ZERO(hpriv->irq_cause_offset); 3200 ZERO(hpriv->irq_mask_offset); 3201 ZERO(MV_PCI_ERR_LOW_ADDRESS); 3202 ZERO(MV_PCI_ERR_HIGH_ADDRESS); 3203 ZERO(MV_PCI_ERR_ATTRIBUTE); 3204 ZERO(MV_PCI_ERR_COMMAND); 3205 } 3206 #undef ZERO 3207 3208 static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio) 3209 { 3210 u32 tmp; 3211 3212 mv5_reset_flash(hpriv, mmio); 3213 3214 tmp = readl(mmio + GPIO_PORT_CTL); 3215 tmp &= 0x3; 3216 tmp |= (1 << 5) | (1 << 6); 3217 writel(tmp, mmio + GPIO_PORT_CTL); 3218 } 3219 3220 /* 3221 * mv6_reset_hc - Perform the 6xxx global soft reset 3222 * @mmio: base address of the HBA 3223 * 3224 * This routine only applies to 6xxx parts. 3225 * 3226 * LOCKING: 3227 * Inherited from caller. 3228 */ 3229 static int mv6_reset_hc(struct ata_host *host, void __iomem *mmio, 3230 unsigned int n_hc) 3231 { 3232 void __iomem *reg = mmio + PCI_MAIN_CMD_STS; 3233 int i, rc = 0; 3234 u32 t; 3235 3236 /* Following procedure defined in PCI "main command and status 3237 * register" table. 3238 */ 3239 t = readl(reg); 3240 writel(t | STOP_PCI_MASTER, reg); 3241 3242 for (i = 0; i < 1000; i++) { 3243 udelay(1); 3244 t = readl(reg); 3245 if (PCI_MASTER_EMPTY & t) 3246 break; 3247 } 3248 if (!(PCI_MASTER_EMPTY & t)) { 3249 dev_err(host->dev, "PCI master won't flush\n"); 3250 rc = 1; 3251 goto done; 3252 } 3253 3254 /* set reset */ 3255 i = 5; 3256 do { 3257 writel(t | GLOB_SFT_RST, reg); 3258 t = readl(reg); 3259 udelay(1); 3260 } while (!(GLOB_SFT_RST & t) && (i-- > 0)); 3261 3262 if (!(GLOB_SFT_RST & t)) { 3263 dev_err(host->dev, "can't set global reset\n"); 3264 rc = 1; 3265 goto done; 3266 } 3267 3268 /* clear reset and *reenable the PCI master* (not mentioned in spec) */ 3269 i = 5; 3270 do { 3271 writel(t & ~(GLOB_SFT_RST | STOP_PCI_MASTER), reg); 3272 t = readl(reg); 3273 udelay(1); 3274 } while ((GLOB_SFT_RST & t) && (i-- > 0)); 3275 3276 if (GLOB_SFT_RST & t) { 3277 dev_err(host->dev, "can't clear global reset\n"); 3278 rc = 1; 3279 } 3280 done: 3281 return rc; 3282 } 3283 3284 static void mv6_read_preamp(struct mv_host_priv *hpriv, int idx, 3285 void __iomem *mmio) 3286 { 3287 void __iomem *port_mmio; 3288 u32 tmp; 3289 3290 tmp = readl(mmio + RESET_CFG); 3291 if ((tmp & (1 << 0)) == 0) { 3292 hpriv->signal[idx].amps = 0x7 << 8; 3293 hpriv->signal[idx].pre = 0x1 << 5; 3294 return; 3295 } 3296 3297 port_mmio = mv_port_base(mmio, idx); 3298 tmp = readl(port_mmio + PHY_MODE2); 3299 3300 hpriv->signal[idx].amps = tmp & 0x700; /* bits 10:8 */ 3301 hpriv->signal[idx].pre = tmp & 0xe0; /* bits 7:5 */ 3302 } 3303 3304 static void mv6_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio) 3305 { 3306 writel(0x00000060, mmio + GPIO_PORT_CTL); 3307 } 3308 3309 static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio, 3310 unsigned int port) 3311 { 3312 void __iomem *port_mmio = mv_port_base(mmio, port); 3313 3314 u32 hp_flags = hpriv->hp_flags; 3315 int fix_phy_mode2 = 3316 hp_flags & (MV_HP_ERRATA_60X1B2 | MV_HP_ERRATA_60X1C0); 3317 int fix_phy_mode4 = 3318 hp_flags & (MV_HP_ERRATA_60X1B2 | MV_HP_ERRATA_60X1C0); 3319 u32 m2, m3; 3320 3321 if (fix_phy_mode2) { 3322 m2 = readl(port_mmio + PHY_MODE2); 3323 m2 &= ~(1 << 16); 3324 m2 |= (1 << 31); 3325 writel(m2, port_mmio + PHY_MODE2); 3326 3327 udelay(200); 3328 3329 m2 = readl(port_mmio + PHY_MODE2); 3330 m2 &= ~((1 << 16) | (1 << 31)); 3331 writel(m2, port_mmio + PHY_MODE2); 3332 3333 udelay(200); 3334 } 3335 3336 /* 3337 * Gen-II/IIe PHY_MODE3 errata RM#2: 3338 * Achieves better receiver noise performance than the h/w default: 3339 */ 3340 m3 = readl(port_mmio + PHY_MODE3); 3341 m3 = (m3 & 0x1f) | (0x5555601 << 5); 3342 3343 /* Guideline 88F5182 (GL# SATA-S11) */ 3344 if (IS_SOC(hpriv)) 3345 m3 &= ~0x1c; 3346 3347 if (fix_phy_mode4) { 3348 u32 m4 = readl(port_mmio + PHY_MODE4); 3349 /* 3350 * Enforce reserved-bit restrictions on GenIIe devices only. 3351 * For earlier chipsets, force only the internal config field 3352 * (workaround for errata FEr SATA#10 part 1). 3353 */ 3354 if (IS_GEN_IIE(hpriv)) 3355 m4 = (m4 & ~PHY_MODE4_RSVD_ZEROS) | PHY_MODE4_RSVD_ONES; 3356 else 3357 m4 = (m4 & ~PHY_MODE4_CFG_MASK) | PHY_MODE4_CFG_VALUE; 3358 writel(m4, port_mmio + PHY_MODE4); 3359 } 3360 /* 3361 * Workaround for 60x1-B2 errata SATA#13: 3362 * Any write to PHY_MODE4 (above) may corrupt PHY_MODE3, 3363 * so we must always rewrite PHY_MODE3 after PHY_MODE4. 3364 * Or ensure we use writelfl() when writing PHY_MODE4. 3365 */ 3366 writel(m3, port_mmio + PHY_MODE3); 3367 3368 /* Revert values of pre-emphasis and signal amps to the saved ones */ 3369 m2 = readl(port_mmio + PHY_MODE2); 3370 3371 m2 &= ~MV_M2_PREAMP_MASK; 3372 m2 |= hpriv->signal[port].amps; 3373 m2 |= hpriv->signal[port].pre; 3374 m2 &= ~(1 << 16); 3375 3376 /* according to mvSata 3.6.1, some IIE values are fixed */ 3377 if (IS_GEN_IIE(hpriv)) { 3378 m2 &= ~0xC30FF01F; 3379 m2 |= 0x0000900F; 3380 } 3381 3382 writel(m2, port_mmio + PHY_MODE2); 3383 } 3384 3385 /* TODO: use the generic LED interface to configure the SATA Presence */ 3386 /* & Acitivy LEDs on the board */ 3387 static void mv_soc_enable_leds(struct mv_host_priv *hpriv, 3388 void __iomem *mmio) 3389 { 3390 return; 3391 } 3392 3393 static void mv_soc_read_preamp(struct mv_host_priv *hpriv, int idx, 3394 void __iomem *mmio) 3395 { 3396 void __iomem *port_mmio; 3397 u32 tmp; 3398 3399 port_mmio = mv_port_base(mmio, idx); 3400 tmp = readl(port_mmio + PHY_MODE2); 3401 3402 hpriv->signal[idx].amps = tmp & 0x700; /* bits 10:8 */ 3403 hpriv->signal[idx].pre = tmp & 0xe0; /* bits 7:5 */ 3404 } 3405 3406 #undef ZERO 3407 #define ZERO(reg) writel(0, port_mmio + (reg)) 3408 static void mv_soc_reset_hc_port(struct mv_host_priv *hpriv, 3409 void __iomem *mmio, unsigned int port) 3410 { 3411 void __iomem *port_mmio = mv_port_base(mmio, port); 3412 3413 mv_reset_channel(hpriv, mmio, port); 3414 3415 ZERO(0x028); /* command */ 3416 writel(0x101f, port_mmio + EDMA_CFG); 3417 ZERO(0x004); /* timer */ 3418 ZERO(0x008); /* irq err cause */ 3419 ZERO(0x00c); /* irq err mask */ 3420 ZERO(0x010); /* rq bah */ 3421 ZERO(0x014); /* rq inp */ 3422 ZERO(0x018); /* rq outp */ 3423 ZERO(0x01c); /* respq bah */ 3424 ZERO(0x024); /* respq outp */ 3425 ZERO(0x020); /* respq inp */ 3426 ZERO(0x02c); /* test control */ 3427 writel(0x800, port_mmio + EDMA_IORDY_TMOUT); 3428 } 3429 3430 #undef ZERO 3431 3432 #define ZERO(reg) writel(0, hc_mmio + (reg)) 3433 static void mv_soc_reset_one_hc(struct mv_host_priv *hpriv, 3434 void __iomem *mmio) 3435 { 3436 void __iomem *hc_mmio = mv_hc_base(mmio, 0); 3437 3438 ZERO(0x00c); 3439 ZERO(0x010); 3440 ZERO(0x014); 3441 3442 } 3443 3444 #undef ZERO 3445 3446 static int mv_soc_reset_hc(struct ata_host *host, 3447 void __iomem *mmio, unsigned int n_hc) 3448 { 3449 struct mv_host_priv *hpriv = host->private_data; 3450 unsigned int port; 3451 3452 for (port = 0; port < hpriv->n_ports; port++) 3453 mv_soc_reset_hc_port(hpriv, mmio, port); 3454 3455 mv_soc_reset_one_hc(hpriv, mmio); 3456 3457 return 0; 3458 } 3459 3460 static void mv_soc_reset_flash(struct mv_host_priv *hpriv, 3461 void __iomem *mmio) 3462 { 3463 return; 3464 } 3465 3466 static void mv_soc_reset_bus(struct ata_host *host, void __iomem *mmio) 3467 { 3468 return; 3469 } 3470 3471 static void mv_soc_65n_phy_errata(struct mv_host_priv *hpriv, 3472 void __iomem *mmio, unsigned int port) 3473 { 3474 void __iomem *port_mmio = mv_port_base(mmio, port); 3475 u32 reg; 3476 3477 reg = readl(port_mmio + PHY_MODE3); 3478 reg &= ~(0x3 << 27); /* SELMUPF (bits 28:27) to 1 */ 3479 reg |= (0x1 << 27); 3480 reg &= ~(0x3 << 29); /* SELMUPI (bits 30:29) to 1 */ 3481 reg |= (0x1 << 29); 3482 writel(reg, port_mmio + PHY_MODE3); 3483 3484 reg = readl(port_mmio + PHY_MODE4); 3485 reg &= ~0x1; /* SATU_OD8 (bit 0) to 0, reserved bit 16 must be set */ 3486 reg |= (0x1 << 16); 3487 writel(reg, port_mmio + PHY_MODE4); 3488 3489 reg = readl(port_mmio + PHY_MODE9_GEN2); 3490 reg &= ~0xf; /* TXAMP[3:0] (bits 3:0) to 8 */ 3491 reg |= 0x8; 3492 reg &= ~(0x1 << 14); /* TXAMP[4] (bit 14) to 0 */ 3493 writel(reg, port_mmio + PHY_MODE9_GEN2); 3494 3495 reg = readl(port_mmio + PHY_MODE9_GEN1); 3496 reg &= ~0xf; /* TXAMP[3:0] (bits 3:0) to 8 */ 3497 reg |= 0x8; 3498 reg &= ~(0x1 << 14); /* TXAMP[4] (bit 14) to 0 */ 3499 writel(reg, port_mmio + PHY_MODE9_GEN1); 3500 } 3501 3502 /* 3503 * soc_is_65 - check if the soc is 65 nano device 3504 * 3505 * Detect the type of the SoC, this is done by reading the PHYCFG_OFS 3506 * register, this register should contain non-zero value and it exists only 3507 * in the 65 nano devices, when reading it from older devices we get 0. 3508 */ 3509 static bool soc_is_65n(struct mv_host_priv *hpriv) 3510 { 3511 void __iomem *port0_mmio = mv_port_base(hpriv->base, 0); 3512 3513 if (readl(port0_mmio + PHYCFG_OFS)) 3514 return true; 3515 return false; 3516 } 3517 3518 static void mv_setup_ifcfg(void __iomem *port_mmio, int want_gen2i) 3519 { 3520 u32 ifcfg = readl(port_mmio + SATA_IFCFG); 3521 3522 ifcfg = (ifcfg & 0xf7f) | 0x9b1000; /* from chip spec */ 3523 if (want_gen2i) 3524 ifcfg |= (1 << 7); /* enable gen2i speed */ 3525 writelfl(ifcfg, port_mmio + SATA_IFCFG); 3526 } 3527 3528 static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio, 3529 unsigned int port_no) 3530 { 3531 void __iomem *port_mmio = mv_port_base(mmio, port_no); 3532 3533 /* 3534 * The datasheet warns against setting EDMA_RESET when EDMA is active 3535 * (but doesn't say what the problem might be). So we first try 3536 * to disable the EDMA engine before doing the EDMA_RESET operation. 3537 */ 3538 mv_stop_edma_engine(port_mmio); 3539 writelfl(EDMA_RESET, port_mmio + EDMA_CMD); 3540 3541 if (!IS_GEN_I(hpriv)) { 3542 /* Enable 3.0gb/s link speed: this survives EDMA_RESET */ 3543 mv_setup_ifcfg(port_mmio, 1); 3544 } 3545 /* 3546 * Strobing EDMA_RESET here causes a hard reset of the SATA transport, 3547 * link, and physical layers. It resets all SATA interface registers 3548 * (except for SATA_IFCFG), and issues a COMRESET to the dev. 3549 */ 3550 writelfl(EDMA_RESET, port_mmio + EDMA_CMD); 3551 udelay(25); /* allow reset propagation */ 3552 writelfl(0, port_mmio + EDMA_CMD); 3553 3554 hpriv->ops->phy_errata(hpriv, mmio, port_no); 3555 3556 if (IS_GEN_I(hpriv)) 3557 usleep_range(500, 1000); 3558 } 3559 3560 static void mv_pmp_select(struct ata_port *ap, int pmp) 3561 { 3562 if (sata_pmp_supported(ap)) { 3563 void __iomem *port_mmio = mv_ap_base(ap); 3564 u32 reg = readl(port_mmio + SATA_IFCTL); 3565 int old = reg & 0xf; 3566 3567 if (old != pmp) { 3568 reg = (reg & ~0xf) | pmp; 3569 writelfl(reg, port_mmio + SATA_IFCTL); 3570 } 3571 } 3572 } 3573 3574 static int mv_pmp_hardreset(struct ata_link *link, unsigned int *class, 3575 unsigned long deadline) 3576 { 3577 mv_pmp_select(link->ap, sata_srst_pmp(link)); 3578 return sata_std_hardreset(link, class, deadline); 3579 } 3580 3581 static int mv_softreset(struct ata_link *link, unsigned int *class, 3582 unsigned long deadline) 3583 { 3584 mv_pmp_select(link->ap, sata_srst_pmp(link)); 3585 return ata_sff_softreset(link, class, deadline); 3586 } 3587 3588 static int mv_hardreset(struct ata_link *link, unsigned int *class, 3589 unsigned long deadline) 3590 { 3591 struct ata_port *ap = link->ap; 3592 struct mv_host_priv *hpriv = ap->host->private_data; 3593 struct mv_port_priv *pp = ap->private_data; 3594 void __iomem *mmio = hpriv->base; 3595 int rc, attempts = 0, extra = 0; 3596 u32 sstatus; 3597 bool online; 3598 3599 mv_reset_channel(hpriv, mmio, ap->port_no); 3600 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN; 3601 pp->pp_flags &= 3602 ~(MV_PP_FLAG_FBS_EN | MV_PP_FLAG_NCQ_EN | MV_PP_FLAG_FAKE_ATA_BUSY); 3603 3604 /* Workaround for errata FEr SATA#10 (part 2) */ 3605 do { 3606 const unsigned int *timing = 3607 sata_ehc_deb_timing(&link->eh_context); 3608 3609 rc = sata_link_hardreset(link, timing, deadline + extra, 3610 &online, NULL); 3611 rc = online ? -EAGAIN : rc; 3612 if (rc) 3613 return rc; 3614 sata_scr_read(link, SCR_STATUS, &sstatus); 3615 if (!IS_GEN_I(hpriv) && ++attempts >= 5 && sstatus == 0x121) { 3616 /* Force 1.5gb/s link speed and try again */ 3617 mv_setup_ifcfg(mv_ap_base(ap), 0); 3618 if (time_after(jiffies + HZ, deadline)) 3619 extra = HZ; /* only extend it once, max */ 3620 } 3621 } while (sstatus != 0x0 && sstatus != 0x113 && sstatus != 0x123); 3622 mv_save_cached_regs(ap); 3623 mv_edma_cfg(ap, 0, 0); 3624 3625 return rc; 3626 } 3627 3628 static void mv_eh_freeze(struct ata_port *ap) 3629 { 3630 mv_stop_edma(ap); 3631 mv_enable_port_irqs(ap, 0); 3632 } 3633 3634 static void mv_eh_thaw(struct ata_port *ap) 3635 { 3636 struct mv_host_priv *hpriv = ap->host->private_data; 3637 unsigned int port = ap->port_no; 3638 unsigned int hardport = mv_hardport_from_port(port); 3639 void __iomem *hc_mmio = mv_hc_base_from_port(hpriv->base, port); 3640 void __iomem *port_mmio = mv_ap_base(ap); 3641 u32 hc_irq_cause; 3642 3643 /* clear EDMA errors on this port */ 3644 writel(0, port_mmio + EDMA_ERR_IRQ_CAUSE); 3645 3646 /* clear pending irq events */ 3647 hc_irq_cause = ~((DEV_IRQ | DMA_IRQ) << hardport); 3648 writelfl(hc_irq_cause, hc_mmio + HC_IRQ_CAUSE); 3649 3650 mv_enable_port_irqs(ap, ERR_IRQ); 3651 } 3652 3653 /** 3654 * mv_port_init - Perform some early initialization on a single port. 3655 * @port: libata data structure storing shadow register addresses 3656 * @port_mmio: base address of the port 3657 * 3658 * Initialize shadow register mmio addresses, clear outstanding 3659 * interrupts on the port, and unmask interrupts for the future 3660 * start of the port. 3661 * 3662 * LOCKING: 3663 * Inherited from caller. 3664 */ 3665 static void mv_port_init(struct ata_ioports *port, void __iomem *port_mmio) 3666 { 3667 void __iomem *serr, *shd_base = port_mmio + SHD_BLK; 3668 3669 /* PIO related setup 3670 */ 3671 port->data_addr = shd_base + (sizeof(u32) * ATA_REG_DATA); 3672 port->error_addr = 3673 port->feature_addr = shd_base + (sizeof(u32) * ATA_REG_ERR); 3674 port->nsect_addr = shd_base + (sizeof(u32) * ATA_REG_NSECT); 3675 port->lbal_addr = shd_base + (sizeof(u32) * ATA_REG_LBAL); 3676 port->lbam_addr = shd_base + (sizeof(u32) * ATA_REG_LBAM); 3677 port->lbah_addr = shd_base + (sizeof(u32) * ATA_REG_LBAH); 3678 port->device_addr = shd_base + (sizeof(u32) * ATA_REG_DEVICE); 3679 port->status_addr = 3680 port->command_addr = shd_base + (sizeof(u32) * ATA_REG_STATUS); 3681 /* special case: control/altstatus doesn't have ATA_REG_ address */ 3682 port->altstatus_addr = port->ctl_addr = shd_base + SHD_CTL_AST; 3683 3684 /* Clear any currently outstanding port interrupt conditions */ 3685 serr = port_mmio + mv_scr_offset(SCR_ERROR); 3686 writelfl(readl(serr), serr); 3687 writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE); 3688 3689 /* unmask all non-transient EDMA error interrupts */ 3690 writelfl(~EDMA_ERR_IRQ_TRANSIENT, port_mmio + EDMA_ERR_IRQ_MASK); 3691 } 3692 3693 static unsigned int mv_in_pcix_mode(struct ata_host *host) 3694 { 3695 struct mv_host_priv *hpriv = host->private_data; 3696 void __iomem *mmio = hpriv->base; 3697 u32 reg; 3698 3699 if (IS_SOC(hpriv) || !IS_PCIE(hpriv)) 3700 return 0; /* not PCI-X capable */ 3701 reg = readl(mmio + MV_PCI_MODE); 3702 if ((reg & MV_PCI_MODE_MASK) == 0) 3703 return 0; /* conventional PCI mode */ 3704 return 1; /* chip is in PCI-X mode */ 3705 } 3706 3707 static int mv_pci_cut_through_okay(struct ata_host *host) 3708 { 3709 struct mv_host_priv *hpriv = host->private_data; 3710 void __iomem *mmio = hpriv->base; 3711 u32 reg; 3712 3713 if (!mv_in_pcix_mode(host)) { 3714 reg = readl(mmio + MV_PCI_COMMAND); 3715 if (reg & MV_PCI_COMMAND_MRDTRIG) 3716 return 0; /* not okay */ 3717 } 3718 return 1; /* okay */ 3719 } 3720 3721 static void mv_60x1b2_errata_pci7(struct ata_host *host) 3722 { 3723 struct mv_host_priv *hpriv = host->private_data; 3724 void __iomem *mmio = hpriv->base; 3725 3726 /* workaround for 60x1-B2 errata PCI#7 */ 3727 if (mv_in_pcix_mode(host)) { 3728 u32 reg = readl(mmio + MV_PCI_COMMAND); 3729 writelfl(reg & ~MV_PCI_COMMAND_MWRCOM, mmio + MV_PCI_COMMAND); 3730 } 3731 } 3732 3733 static int mv_chip_id(struct ata_host *host, unsigned int board_idx) 3734 { 3735 struct pci_dev *pdev = to_pci_dev(host->dev); 3736 struct mv_host_priv *hpriv = host->private_data; 3737 u32 hp_flags = hpriv->hp_flags; 3738 3739 switch (board_idx) { 3740 case chip_5080: 3741 hpriv->ops = &mv5xxx_ops; 3742 hp_flags |= MV_HP_GEN_I; 3743 3744 switch (pdev->revision) { 3745 case 0x1: 3746 hp_flags |= MV_HP_ERRATA_50XXB0; 3747 break; 3748 case 0x3: 3749 hp_flags |= MV_HP_ERRATA_50XXB2; 3750 break; 3751 default: 3752 dev_warn(&pdev->dev, 3753 "Applying 50XXB2 workarounds to unknown rev\n"); 3754 hp_flags |= MV_HP_ERRATA_50XXB2; 3755 break; 3756 } 3757 break; 3758 3759 case chip_504x: 3760 case chip_508x: 3761 hpriv->ops = &mv5xxx_ops; 3762 hp_flags |= MV_HP_GEN_I; 3763 3764 switch (pdev->revision) { 3765 case 0x0: 3766 hp_flags |= MV_HP_ERRATA_50XXB0; 3767 break; 3768 case 0x3: 3769 hp_flags |= MV_HP_ERRATA_50XXB2; 3770 break; 3771 default: 3772 dev_warn(&pdev->dev, 3773 "Applying B2 workarounds to unknown rev\n"); 3774 hp_flags |= MV_HP_ERRATA_50XXB2; 3775 break; 3776 } 3777 break; 3778 3779 case chip_604x: 3780 case chip_608x: 3781 hpriv->ops = &mv6xxx_ops; 3782 hp_flags |= MV_HP_GEN_II; 3783 3784 switch (pdev->revision) { 3785 case 0x7: 3786 mv_60x1b2_errata_pci7(host); 3787 hp_flags |= MV_HP_ERRATA_60X1B2; 3788 break; 3789 case 0x9: 3790 hp_flags |= MV_HP_ERRATA_60X1C0; 3791 break; 3792 default: 3793 dev_warn(&pdev->dev, 3794 "Applying B2 workarounds to unknown rev\n"); 3795 hp_flags |= MV_HP_ERRATA_60X1B2; 3796 break; 3797 } 3798 break; 3799 3800 case chip_7042: 3801 hp_flags |= MV_HP_PCIE | MV_HP_CUT_THROUGH; 3802 if (pdev->vendor == PCI_VENDOR_ID_TTI && 3803 (pdev->device == 0x2300 || pdev->device == 0x2310)) 3804 { 3805 /* 3806 * Highpoint RocketRAID PCIe 23xx series cards: 3807 * 3808 * Unconfigured drives are treated as "Legacy" 3809 * by the BIOS, and it overwrites sector 8 with 3810 * a "Lgcy" metadata block prior to Linux boot. 3811 * 3812 * Configured drives (RAID or JBOD) leave sector 8 3813 * alone, but instead overwrite a high numbered 3814 * sector for the RAID metadata. This sector can 3815 * be determined exactly, by truncating the physical 3816 * drive capacity to a nice even GB value. 3817 * 3818 * RAID metadata is at: (dev->n_sectors & ~0xfffff) 3819 * 3820 * Warn the user, lest they think we're just buggy. 3821 */ 3822 dev_warn(&pdev->dev, "Highpoint RocketRAID" 3823 " BIOS CORRUPTS DATA on all attached drives," 3824 " regardless of if/how they are configured." 3825 " BEWARE!\n"); 3826 dev_warn(&pdev->dev, "For data safety, do not" 3827 " use sectors 8-9 on \"Legacy\" drives," 3828 " and avoid the final two gigabytes on" 3829 " all RocketRAID BIOS initialized drives.\n"); 3830 } 3831 fallthrough; 3832 case chip_6042: 3833 hpriv->ops = &mv6xxx_ops; 3834 hp_flags |= MV_HP_GEN_IIE; 3835 if (board_idx == chip_6042 && mv_pci_cut_through_okay(host)) 3836 hp_flags |= MV_HP_CUT_THROUGH; 3837 3838 switch (pdev->revision) { 3839 case 0x2: /* Rev.B0: the first/only public release */ 3840 hp_flags |= MV_HP_ERRATA_60X1C0; 3841 break; 3842 default: 3843 dev_warn(&pdev->dev, 3844 "Applying 60X1C0 workarounds to unknown rev\n"); 3845 hp_flags |= MV_HP_ERRATA_60X1C0; 3846 break; 3847 } 3848 break; 3849 case chip_soc: 3850 if (soc_is_65n(hpriv)) 3851 hpriv->ops = &mv_soc_65n_ops; 3852 else 3853 hpriv->ops = &mv_soc_ops; 3854 hp_flags |= MV_HP_FLAG_SOC | MV_HP_GEN_IIE | 3855 MV_HP_ERRATA_60X1C0; 3856 break; 3857 3858 default: 3859 dev_alert(host->dev, "BUG: invalid board index %u\n", board_idx); 3860 return -EINVAL; 3861 } 3862 3863 hpriv->hp_flags = hp_flags; 3864 if (hp_flags & MV_HP_PCIE) { 3865 hpriv->irq_cause_offset = PCIE_IRQ_CAUSE; 3866 hpriv->irq_mask_offset = PCIE_IRQ_MASK; 3867 hpriv->unmask_all_irqs = PCIE_UNMASK_ALL_IRQS; 3868 } else { 3869 hpriv->irq_cause_offset = PCI_IRQ_CAUSE; 3870 hpriv->irq_mask_offset = PCI_IRQ_MASK; 3871 hpriv->unmask_all_irqs = PCI_UNMASK_ALL_IRQS; 3872 } 3873 3874 return 0; 3875 } 3876 3877 /** 3878 * mv_init_host - Perform some early initialization of the host. 3879 * @host: ATA host to initialize 3880 * 3881 * If possible, do an early global reset of the host. Then do 3882 * our port init and clear/unmask all/relevant host interrupts. 3883 * 3884 * LOCKING: 3885 * Inherited from caller. 3886 */ 3887 static int mv_init_host(struct ata_host *host) 3888 { 3889 int rc = 0, n_hc, port, hc; 3890 struct mv_host_priv *hpriv = host->private_data; 3891 void __iomem *mmio = hpriv->base; 3892 3893 rc = mv_chip_id(host, hpriv->board_idx); 3894 if (rc) 3895 goto done; 3896 3897 if (IS_SOC(hpriv)) { 3898 hpriv->main_irq_cause_addr = mmio + SOC_HC_MAIN_IRQ_CAUSE; 3899 hpriv->main_irq_mask_addr = mmio + SOC_HC_MAIN_IRQ_MASK; 3900 } else { 3901 hpriv->main_irq_cause_addr = mmio + PCI_HC_MAIN_IRQ_CAUSE; 3902 hpriv->main_irq_mask_addr = mmio + PCI_HC_MAIN_IRQ_MASK; 3903 } 3904 3905 /* initialize shadow irq mask with register's value */ 3906 hpriv->main_irq_mask = readl(hpriv->main_irq_mask_addr); 3907 3908 /* global interrupt mask: 0 == mask everything */ 3909 mv_set_main_irq_mask(host, ~0, 0); 3910 3911 n_hc = mv_get_hc_count(host->ports[0]->flags); 3912 3913 for (port = 0; port < host->n_ports; port++) 3914 if (hpriv->ops->read_preamp) 3915 hpriv->ops->read_preamp(hpriv, port, mmio); 3916 3917 rc = hpriv->ops->reset_hc(host, mmio, n_hc); 3918 if (rc) 3919 goto done; 3920 3921 hpriv->ops->reset_flash(hpriv, mmio); 3922 hpriv->ops->reset_bus(host, mmio); 3923 hpriv->ops->enable_leds(hpriv, mmio); 3924 3925 for (port = 0; port < host->n_ports; port++) { 3926 struct ata_port *ap = host->ports[port]; 3927 void __iomem *port_mmio = mv_port_base(mmio, port); 3928 3929 mv_port_init(&ap->ioaddr, port_mmio); 3930 } 3931 3932 for (hc = 0; hc < n_hc; hc++) { 3933 void __iomem *hc_mmio = mv_hc_base(mmio, hc); 3934 3935 dev_dbg(host->dev, "HC%i: HC config=0x%08x HC IRQ cause " 3936 "(before clear)=0x%08x\n", hc, 3937 readl(hc_mmio + HC_CFG), 3938 readl(hc_mmio + HC_IRQ_CAUSE)); 3939 3940 /* Clear any currently outstanding hc interrupt conditions */ 3941 writelfl(0, hc_mmio + HC_IRQ_CAUSE); 3942 } 3943 3944 if (!IS_SOC(hpriv)) { 3945 /* Clear any currently outstanding host interrupt conditions */ 3946 writelfl(0, mmio + hpriv->irq_cause_offset); 3947 3948 /* and unmask interrupt generation for host regs */ 3949 writelfl(hpriv->unmask_all_irqs, mmio + hpriv->irq_mask_offset); 3950 } 3951 3952 /* 3953 * enable only global host interrupts for now. 3954 * The per-port interrupts get done later as ports are set up. 3955 */ 3956 mv_set_main_irq_mask(host, 0, PCI_ERR); 3957 mv_set_irq_coalescing(host, irq_coalescing_io_count, 3958 irq_coalescing_usecs); 3959 done: 3960 return rc; 3961 } 3962 3963 static int mv_create_dma_pools(struct mv_host_priv *hpriv, struct device *dev) 3964 { 3965 hpriv->crqb_pool = dmam_pool_create("crqb_q", dev, MV_CRQB_Q_SZ, 3966 MV_CRQB_Q_SZ, 0); 3967 if (!hpriv->crqb_pool) 3968 return -ENOMEM; 3969 3970 hpriv->crpb_pool = dmam_pool_create("crpb_q", dev, MV_CRPB_Q_SZ, 3971 MV_CRPB_Q_SZ, 0); 3972 if (!hpriv->crpb_pool) 3973 return -ENOMEM; 3974 3975 hpriv->sg_tbl_pool = dmam_pool_create("sg_tbl", dev, MV_SG_TBL_SZ, 3976 MV_SG_TBL_SZ, 0); 3977 if (!hpriv->sg_tbl_pool) 3978 return -ENOMEM; 3979 3980 return 0; 3981 } 3982 3983 static void mv_conf_mbus_windows(struct mv_host_priv *hpriv, 3984 const struct mbus_dram_target_info *dram) 3985 { 3986 int i; 3987 3988 for (i = 0; i < 4; i++) { 3989 writel(0, hpriv->base + WINDOW_CTRL(i)); 3990 writel(0, hpriv->base + WINDOW_BASE(i)); 3991 } 3992 3993 for (i = 0; i < dram->num_cs; i++) { 3994 const struct mbus_dram_window *cs = dram->cs + i; 3995 3996 writel(((cs->size - 1) & 0xffff0000) | 3997 (cs->mbus_attr << 8) | 3998 (dram->mbus_dram_target_id << 4) | 1, 3999 hpriv->base + WINDOW_CTRL(i)); 4000 writel(cs->base, hpriv->base + WINDOW_BASE(i)); 4001 } 4002 } 4003 4004 /** 4005 * mv_platform_probe - handle a positive probe of an soc Marvell 4006 * host 4007 * @pdev: platform device found 4008 * 4009 * LOCKING: 4010 * Inherited from caller. 4011 */ 4012 static int mv_platform_probe(struct platform_device *pdev) 4013 { 4014 const struct mv_sata_platform_data *mv_platform_data; 4015 const struct mbus_dram_target_info *dram; 4016 const struct ata_port_info *ppi[] = 4017 { &mv_port_info[chip_soc], NULL }; 4018 struct ata_host *host; 4019 struct mv_host_priv *hpriv; 4020 struct resource *res; 4021 int n_ports = 0, irq = 0; 4022 int rc; 4023 int port; 4024 4025 ata_print_version_once(&pdev->dev, DRV_VERSION); 4026 4027 /* 4028 * Simple resource validation .. 4029 */ 4030 if (unlikely(pdev->num_resources != 1)) { 4031 dev_err(&pdev->dev, "invalid number of resources\n"); 4032 return -EINVAL; 4033 } 4034 4035 /* 4036 * Get the register base first 4037 */ 4038 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4039 if (res == NULL) 4040 return -EINVAL; 4041 4042 /* allocate host */ 4043 if (pdev->dev.of_node) { 4044 rc = of_property_read_u32(pdev->dev.of_node, "nr-ports", 4045 &n_ports); 4046 if (rc) { 4047 dev_err(&pdev->dev, 4048 "error parsing nr-ports property: %d\n", rc); 4049 return rc; 4050 } 4051 4052 if (n_ports <= 0) { 4053 dev_err(&pdev->dev, "nr-ports must be positive: %d\n", 4054 n_ports); 4055 return -EINVAL; 4056 } 4057 4058 irq = irq_of_parse_and_map(pdev->dev.of_node, 0); 4059 } else { 4060 mv_platform_data = dev_get_platdata(&pdev->dev); 4061 n_ports = mv_platform_data->n_ports; 4062 irq = platform_get_irq(pdev, 0); 4063 } 4064 if (irq < 0) 4065 return irq; 4066 if (!irq) 4067 return -EINVAL; 4068 4069 host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports); 4070 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL); 4071 4072 if (!host || !hpriv) 4073 return -ENOMEM; 4074 hpriv->port_clks = devm_kcalloc(&pdev->dev, 4075 n_ports, sizeof(struct clk *), 4076 GFP_KERNEL); 4077 if (!hpriv->port_clks) 4078 return -ENOMEM; 4079 hpriv->port_phys = devm_kcalloc(&pdev->dev, 4080 n_ports, sizeof(struct phy *), 4081 GFP_KERNEL); 4082 if (!hpriv->port_phys) 4083 return -ENOMEM; 4084 host->private_data = hpriv; 4085 hpriv->board_idx = chip_soc; 4086 4087 host->iomap = NULL; 4088 hpriv->base = devm_ioremap(&pdev->dev, res->start, 4089 resource_size(res)); 4090 if (!hpriv->base) 4091 return -ENOMEM; 4092 4093 hpriv->base -= SATAHC0_REG_BASE; 4094 4095 hpriv->clk = clk_get(&pdev->dev, NULL); 4096 if (IS_ERR(hpriv->clk)) { 4097 dev_notice(&pdev->dev, "cannot get optional clkdev\n"); 4098 } else { 4099 rc = clk_prepare_enable(hpriv->clk); 4100 if (rc) 4101 goto err; 4102 } 4103 4104 for (port = 0; port < n_ports; port++) { 4105 char port_number[16]; 4106 sprintf(port_number, "%d", port); 4107 hpriv->port_clks[port] = clk_get(&pdev->dev, port_number); 4108 if (!IS_ERR(hpriv->port_clks[port])) 4109 clk_prepare_enable(hpriv->port_clks[port]); 4110 4111 sprintf(port_number, "port%d", port); 4112 hpriv->port_phys[port] = devm_phy_optional_get(&pdev->dev, 4113 port_number); 4114 if (IS_ERR(hpriv->port_phys[port])) { 4115 rc = PTR_ERR(hpriv->port_phys[port]); 4116 hpriv->port_phys[port] = NULL; 4117 if (rc != -EPROBE_DEFER) 4118 dev_warn(&pdev->dev, "error getting phy %d", rc); 4119 4120 /* Cleanup only the initialized ports */ 4121 hpriv->n_ports = port; 4122 goto err; 4123 } else 4124 phy_power_on(hpriv->port_phys[port]); 4125 } 4126 4127 /* All the ports have been initialized */ 4128 hpriv->n_ports = n_ports; 4129 4130 /* 4131 * (Re-)program MBUS remapping windows if we are asked to. 4132 */ 4133 dram = mv_mbus_dram_info(); 4134 if (dram) 4135 mv_conf_mbus_windows(hpriv, dram); 4136 4137 rc = mv_create_dma_pools(hpriv, &pdev->dev); 4138 if (rc) 4139 goto err; 4140 4141 /* 4142 * To allow disk hotplug on Armada 370/XP SoCs, the PHY speed must be 4143 * updated in the LP_PHY_CTL register. 4144 */ 4145 if (pdev->dev.of_node && 4146 of_device_is_compatible(pdev->dev.of_node, 4147 "marvell,armada-370-sata")) 4148 hpriv->hp_flags |= MV_HP_FIX_LP_PHY_CTL; 4149 4150 /* initialize adapter */ 4151 rc = mv_init_host(host); 4152 if (rc) 4153 goto err; 4154 4155 dev_info(&pdev->dev, "slots %u ports %d\n", 4156 (unsigned)MV_MAX_Q_DEPTH, host->n_ports); 4157 4158 rc = ata_host_activate(host, irq, mv_interrupt, IRQF_SHARED, &mv6_sht); 4159 if (!rc) 4160 return 0; 4161 4162 err: 4163 if (!IS_ERR(hpriv->clk)) { 4164 clk_disable_unprepare(hpriv->clk); 4165 clk_put(hpriv->clk); 4166 } 4167 for (port = 0; port < hpriv->n_ports; port++) { 4168 if (!IS_ERR(hpriv->port_clks[port])) { 4169 clk_disable_unprepare(hpriv->port_clks[port]); 4170 clk_put(hpriv->port_clks[port]); 4171 } 4172 phy_power_off(hpriv->port_phys[port]); 4173 } 4174 4175 return rc; 4176 } 4177 4178 /* 4179 * 4180 * mv_platform_remove - unplug a platform interface 4181 * @pdev: platform device 4182 * 4183 * A platform bus SATA device has been unplugged. Perform the needed 4184 * cleanup. Also called on module unload for any active devices. 4185 */ 4186 static void mv_platform_remove(struct platform_device *pdev) 4187 { 4188 struct ata_host *host = platform_get_drvdata(pdev); 4189 struct mv_host_priv *hpriv = host->private_data; 4190 int port; 4191 ata_host_detach(host); 4192 4193 if (!IS_ERR(hpriv->clk)) { 4194 clk_disable_unprepare(hpriv->clk); 4195 clk_put(hpriv->clk); 4196 } 4197 for (port = 0; port < host->n_ports; port++) { 4198 if (!IS_ERR(hpriv->port_clks[port])) { 4199 clk_disable_unprepare(hpriv->port_clks[port]); 4200 clk_put(hpriv->port_clks[port]); 4201 } 4202 phy_power_off(hpriv->port_phys[port]); 4203 } 4204 } 4205 4206 #ifdef CONFIG_PM_SLEEP 4207 static int mv_platform_suspend(struct platform_device *pdev, pm_message_t state) 4208 { 4209 struct ata_host *host = platform_get_drvdata(pdev); 4210 4211 if (host) 4212 ata_host_suspend(host, state); 4213 return 0; 4214 } 4215 4216 static int mv_platform_resume(struct platform_device *pdev) 4217 { 4218 struct ata_host *host = platform_get_drvdata(pdev); 4219 const struct mbus_dram_target_info *dram; 4220 int ret; 4221 4222 if (host) { 4223 struct mv_host_priv *hpriv = host->private_data; 4224 4225 /* 4226 * (Re-)program MBUS remapping windows if we are asked to. 4227 */ 4228 dram = mv_mbus_dram_info(); 4229 if (dram) 4230 mv_conf_mbus_windows(hpriv, dram); 4231 4232 /* initialize adapter */ 4233 ret = mv_init_host(host); 4234 if (ret) { 4235 dev_err(&pdev->dev, "Error during HW init\n"); 4236 return ret; 4237 } 4238 ata_host_resume(host); 4239 } 4240 4241 return 0; 4242 } 4243 #else 4244 #define mv_platform_suspend NULL 4245 #define mv_platform_resume NULL 4246 #endif 4247 4248 #ifdef CONFIG_OF 4249 static const struct of_device_id mv_sata_dt_ids[] = { 4250 { .compatible = "marvell,armada-370-sata", }, 4251 { .compatible = "marvell,orion-sata", }, 4252 { /* sentinel */ } 4253 }; 4254 MODULE_DEVICE_TABLE(of, mv_sata_dt_ids); 4255 #endif 4256 4257 static struct platform_driver mv_platform_driver = { 4258 .probe = mv_platform_probe, 4259 .remove = mv_platform_remove, 4260 .suspend = mv_platform_suspend, 4261 .resume = mv_platform_resume, 4262 .driver = { 4263 .name = DRV_NAME, 4264 .of_match_table = of_match_ptr(mv_sata_dt_ids), 4265 }, 4266 }; 4267 4268 4269 #ifdef CONFIG_PCI 4270 static int mv_pci_init_one(struct pci_dev *pdev, 4271 const struct pci_device_id *ent); 4272 #ifdef CONFIG_PM_SLEEP 4273 static int mv_pci_device_resume(struct pci_dev *pdev); 4274 #endif 4275 4276 static const struct pci_device_id mv_pci_tbl[] = { 4277 { PCI_VDEVICE(MARVELL, 0x5040), .driver_data = chip_504x }, 4278 { PCI_VDEVICE(MARVELL, 0x5041), .driver_data = chip_504x }, 4279 { PCI_VDEVICE(MARVELL, 0x5080), .driver_data = chip_5080 }, 4280 { PCI_VDEVICE(MARVELL, 0x5081), .driver_data = chip_508x }, 4281 /* RocketRAID 1720/174x have different identifiers */ 4282 { PCI_VDEVICE(TTI, 0x1720), .driver_data = chip_6042 }, 4283 { PCI_VDEVICE(TTI, 0x1740), .driver_data = chip_6042 }, 4284 { PCI_VDEVICE(TTI, 0x1742), .driver_data = chip_6042 }, 4285 4286 { PCI_VDEVICE(MARVELL, 0x6040), .driver_data = chip_604x }, 4287 { PCI_VDEVICE(MARVELL, 0x6041), .driver_data = chip_604x }, 4288 { PCI_VDEVICE(MARVELL, 0x6042), .driver_data = chip_6042 }, 4289 { PCI_VDEVICE(MARVELL, 0x6080), .driver_data = chip_608x }, 4290 { PCI_VDEVICE(MARVELL, 0x6081), .driver_data = chip_608x }, 4291 4292 { PCI_VDEVICE(ADAPTEC2, 0x0241), .driver_data = chip_604x }, 4293 4294 /* Adaptec 1430SA */ 4295 { PCI_VDEVICE(ADAPTEC2, 0x0243), .driver_data = chip_7042 }, 4296 4297 /* Marvell 7042 support */ 4298 { PCI_VDEVICE(MARVELL, 0x7042), .driver_data = chip_7042 }, 4299 4300 /* Highpoint RocketRAID PCIe series */ 4301 { PCI_VDEVICE(TTI, 0x2300), .driver_data = chip_7042 }, 4302 { PCI_VDEVICE(TTI, 0x2310), .driver_data = chip_7042 }, 4303 4304 { } /* terminate list */ 4305 }; 4306 4307 static struct pci_driver mv_pci_driver = { 4308 .name = DRV_NAME, 4309 .id_table = mv_pci_tbl, 4310 .probe = mv_pci_init_one, 4311 .remove = ata_pci_remove_one, 4312 #ifdef CONFIG_PM_SLEEP 4313 .suspend = ata_pci_device_suspend, 4314 .resume = mv_pci_device_resume, 4315 #endif 4316 4317 }; 4318 MODULE_DEVICE_TABLE(pci, mv_pci_tbl); 4319 4320 /** 4321 * mv_print_info - Dump key info to kernel log for perusal. 4322 * @host: ATA host to print info about 4323 * 4324 * FIXME: complete this. 4325 * 4326 * LOCKING: 4327 * Inherited from caller. 4328 */ 4329 static void mv_print_info(struct ata_host *host) 4330 { 4331 struct pci_dev *pdev = to_pci_dev(host->dev); 4332 struct mv_host_priv *hpriv = host->private_data; 4333 u8 scc; 4334 const char *scc_s, *gen; 4335 4336 /* Use this to determine the HW stepping of the chip so we know 4337 * what errata to workaround 4338 */ 4339 pci_read_config_byte(pdev, PCI_CLASS_DEVICE, &scc); 4340 if (scc == 0) 4341 scc_s = "SCSI"; 4342 else if (scc == 0x01) 4343 scc_s = "RAID"; 4344 else 4345 scc_s = "?"; 4346 4347 if (IS_GEN_I(hpriv)) 4348 gen = "I"; 4349 else if (IS_GEN_II(hpriv)) 4350 gen = "II"; 4351 else if (IS_GEN_IIE(hpriv)) 4352 gen = "IIE"; 4353 else 4354 gen = "?"; 4355 4356 dev_info(&pdev->dev, "Gen-%s %u slots %u ports %s mode IRQ via %s\n", 4357 gen, (unsigned)MV_MAX_Q_DEPTH, host->n_ports, 4358 scc_s, (MV_HP_FLAG_MSI & hpriv->hp_flags) ? "MSI" : "INTx"); 4359 } 4360 4361 /** 4362 * mv_pci_init_one - handle a positive probe of a PCI Marvell host 4363 * @pdev: PCI device found 4364 * @ent: PCI device ID entry for the matched host 4365 * 4366 * LOCKING: 4367 * Inherited from caller. 4368 */ 4369 static int mv_pci_init_one(struct pci_dev *pdev, 4370 const struct pci_device_id *ent) 4371 { 4372 unsigned int board_idx = (unsigned int)ent->driver_data; 4373 const struct ata_port_info *ppi[] = { &mv_port_info[board_idx], NULL }; 4374 struct ata_host *host; 4375 struct mv_host_priv *hpriv; 4376 int n_ports, port, rc; 4377 4378 ata_print_version_once(&pdev->dev, DRV_VERSION); 4379 4380 /* allocate host */ 4381 n_ports = mv_get_hc_count(ppi[0]->flags) * MV_PORTS_PER_HC; 4382 4383 host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports); 4384 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL); 4385 if (!host || !hpriv) 4386 return -ENOMEM; 4387 host->private_data = hpriv; 4388 hpriv->n_ports = n_ports; 4389 hpriv->board_idx = board_idx; 4390 4391 /* acquire resources */ 4392 rc = pcim_enable_device(pdev); 4393 if (rc) 4394 return rc; 4395 4396 rc = pcim_iomap_regions(pdev, 1 << MV_PRIMARY_BAR, DRV_NAME); 4397 if (rc == -EBUSY) 4398 pcim_pin_device(pdev); 4399 if (rc) 4400 return rc; 4401 host->iomap = pcim_iomap_table(pdev); 4402 hpriv->base = host->iomap[MV_PRIMARY_BAR]; 4403 4404 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4405 if (rc) { 4406 dev_err(&pdev->dev, "DMA enable failed\n"); 4407 return rc; 4408 } 4409 4410 rc = mv_create_dma_pools(hpriv, &pdev->dev); 4411 if (rc) 4412 return rc; 4413 4414 for (port = 0; port < host->n_ports; port++) { 4415 struct ata_port *ap = host->ports[port]; 4416 void __iomem *port_mmio = mv_port_base(hpriv->base, port); 4417 unsigned int offset = port_mmio - hpriv->base; 4418 4419 ata_port_pbar_desc(ap, MV_PRIMARY_BAR, -1, "mmio"); 4420 ata_port_pbar_desc(ap, MV_PRIMARY_BAR, offset, "port"); 4421 } 4422 4423 /* initialize adapter */ 4424 rc = mv_init_host(host); 4425 if (rc) 4426 return rc; 4427 4428 /* Enable message-switched interrupts, if requested */ 4429 if (msi && pci_enable_msi(pdev) == 0) 4430 hpriv->hp_flags |= MV_HP_FLAG_MSI; 4431 4432 mv_dump_pci_cfg(pdev, 0x68); 4433 mv_print_info(host); 4434 4435 pci_set_master(pdev); 4436 pci_try_set_mwi(pdev); 4437 return ata_host_activate(host, pdev->irq, mv_interrupt, IRQF_SHARED, 4438 IS_GEN_I(hpriv) ? &mv5_sht : &mv6_sht); 4439 } 4440 4441 #ifdef CONFIG_PM_SLEEP 4442 static int mv_pci_device_resume(struct pci_dev *pdev) 4443 { 4444 struct ata_host *host = pci_get_drvdata(pdev); 4445 int rc; 4446 4447 rc = ata_pci_device_do_resume(pdev); 4448 if (rc) 4449 return rc; 4450 4451 /* initialize adapter */ 4452 rc = mv_init_host(host); 4453 if (rc) 4454 return rc; 4455 4456 ata_host_resume(host); 4457 4458 return 0; 4459 } 4460 #endif 4461 #endif 4462 4463 static int __init mv_init(void) 4464 { 4465 int rc = -ENODEV; 4466 #ifdef CONFIG_PCI 4467 rc = pci_register_driver(&mv_pci_driver); 4468 if (rc < 0) 4469 return rc; 4470 #endif 4471 rc = platform_driver_register(&mv_platform_driver); 4472 4473 #ifdef CONFIG_PCI 4474 if (rc < 0) 4475 pci_unregister_driver(&mv_pci_driver); 4476 #endif 4477 return rc; 4478 } 4479 4480 static void __exit mv_exit(void) 4481 { 4482 #ifdef CONFIG_PCI 4483 pci_unregister_driver(&mv_pci_driver); 4484 #endif 4485 platform_driver_unregister(&mv_platform_driver); 4486 } 4487 4488 MODULE_AUTHOR("Brett Russ"); 4489 MODULE_DESCRIPTION("SCSI low-level driver for Marvell SATA controllers"); 4490 MODULE_LICENSE("GPL v2"); 4491 MODULE_VERSION(DRV_VERSION); 4492 MODULE_ALIAS("platform:" DRV_NAME); 4493 4494 module_init(mv_init); 4495 module_exit(mv_exit); 4496