1 /*- 2 * Copyright (c) 2009-2012 Alexander Motin <mav@FreeBSD.org> 3 * Copyright (c) 2017 Justin Hibbits <jhibbits@FreeBSD.org> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer, 11 * without modification, immediately at the beginning of the file. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/param.h> 29 #include <sys/module.h> 30 #include <sys/systm.h> 31 #include <sys/kernel.h> 32 #include <sys/bus.h> 33 #include <sys/conf.h> 34 #include <sys/endian.h> 35 #include <sys/lock.h> 36 #include <sys/malloc.h> 37 #include <sys/mutex.h> 38 #include <sys/rman.h> 39 40 #include <cam/cam.h> 41 #include <cam/cam_ccb.h> 42 #include <cam/cam_sim.h> 43 #include <cam/cam_xpt_sim.h> 44 #include <cam/cam_debug.h> 45 46 #include <dev/ofw/ofw_bus_subr.h> 47 48 #include <machine/bus.h> 49 #include <machine/resource.h> 50 51 #include "fsl_sata.h" 52 53 struct fsl_sata_channel; 54 struct fsl_sata_slot; 55 enum fsl_sata_err_type; 56 struct fsl_sata_cmd_tab; 57 58 /* local prototypes */ 59 static int fsl_sata_init(device_t dev); 60 static int fsl_sata_deinit(device_t dev); 61 static int fsl_sata_suspend(device_t dev); 62 static int fsl_sata_resume(device_t dev); 63 static void fsl_sata_pm(void *arg); 64 static void fsl_sata_intr(void *arg); 65 static void fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus); 66 static void fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb); 67 static void fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error); 68 static void fsl_sata_execute_transaction(struct fsl_sata_slot *slot); 69 static void fsl_sata_timeout(void *arg); 70 static void fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et); 71 static int fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag); 72 static void fsl_sata_dmainit(device_t dev); 73 static void fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error); 74 static void fsl_sata_dmafini(device_t dev); 75 static void fsl_sata_slotsalloc(device_t dev); 76 static void fsl_sata_slotsfree(device_t dev); 77 static void fsl_sata_reset(struct fsl_sata_channel *ch); 78 static void fsl_sata_start(struct fsl_sata_channel *ch); 79 static void fsl_sata_stop(struct fsl_sata_channel *ch); 80 81 static void fsl_sata_issue_recovery(struct fsl_sata_channel *ch); 82 static void fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb); 83 static void fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb); 84 85 static void fsl_sataaction(struct cam_sim *sim, union ccb *ccb); 86 static void fsl_satapoll(struct cam_sim *sim); 87 88 static MALLOC_DEFINE(M_FSL_SATA, "FSL SATA driver", "FSL SATA driver data buffers"); 89 90 #define recovery_type spriv_field0 91 #define RECOVERY_NONE 0 92 #define RECOVERY_READ_LOG 1 93 #define RECOVERY_REQUEST_SENSE 2 94 #define recovery_slot spriv_field1 95 96 #define FSL_SATA_P_CQR 0x0 97 #define FSL_SATA_P_CAR 0x4 98 #define FSL_SATA_P_CCR 0x10 99 #define FSL_SATA_P_CER 0x18 100 #define FSL_SATA_P_DER 0x20 101 #define FSL_SATA_P_CHBA 0x24 102 #define FSL_SATA_P_HSTS 0x28 103 #define FSL_SATA_P_HSTS_HS_ON 0x80000000 104 #define FSL_SATA_P_HSTS_ME 0x00040000 105 #define FSL_SATA_P_HSTS_DLM 0x00001000 106 #define FSL_SATA_P_HSTS_FOT 0x00000200 107 #define FSL_SATA_P_HSTS_FOR 0x00000100 108 #define FSL_SATA_P_HSTS_FE 0x00000020 109 #define FSL_SATA_P_HSTS_PR 0x00000010 110 #define FSL_SATA_P_HSTS_SNTFU 0x00000004 111 #define FSL_SATA_P_HSTS_DE 0x00000002 112 #define FSL_SATA_P_HCTRL 0x2c 113 #define FSL_SATA_P_HCTRL_HC_ON 0x80000000 114 #define FSL_SATA_P_HCTRL_HC_FORCE_OFF 0x40000000 115 #define FSL_SATA_P_HCTRL_ENT 0x10000000 116 #define FSL_SATA_P_HCTRL_SNOOP 0x00000400 117 #define FSL_SATA_P_HCTRL_PM 0x00000200 118 #define FSL_SATA_P_HCTRL_FATAL 0x00000020 119 #define FSL_SATA_P_HCTRL_PHYRDY 0x00000010 120 #define FSL_SATA_P_HCTRL_SIG 0x00000008 121 #define FSL_SATA_P_HCTRL_SNTFY 0x00000004 122 #define FSL_SATA_P_HCTRL_DE 0x00000002 123 #define FSL_SATA_P_HCTRL_CC 0x00000001 124 #define FSL_SATA_P_HCTRL_INT_MASK 0x0000003f 125 #define FSL_SATA_P_CQPMP 0x30 126 #define FSL_SATA_P_SIG 0x34 127 #define FSL_SATA_P_ICC 0x38 128 #define FSL_SATA_P_ICC_ITC_M 0x1f000000 129 #define FSL_SATA_P_ICC_ITC_S 24 130 #define FSL_SATA_P_ICC_ITTCV_M 0x0007ffff 131 #define FSL_SATA_P_PCC 0x15c 132 #define FSL_SATA_P_PCC_SLUMBER 0x0000000c 133 #define FSL_SATA_P_PCC_PARTIAL 0x0000000a 134 #define FSL_SATA_PCC_LPB_EN 0x0000000e 135 136 #define FSL_SATA_MAX_SLOTS 16 137 /* FSL_SATA register defines */ 138 139 #define FSL_SATA_P_SSTS 0x100 140 #define FSL_SATA_P_SERR 0x104 141 #define FSL_SATA_P_SCTL 0x108 142 #define FSL_SATA_P_SNTF 0x10c 143 144 /* Pessimistic prognosis on number of required S/G entries */ 145 #define FSL_SATA_SG_ENTRIES 63 146 /* Command list. 16 commands. First, 1Kbyte aligned. */ 147 #define FSL_SATA_CL_OFFSET 0 148 #define FSL_SATA_CL_SIZE 16 149 /* Command tables. Up to 32 commands, Each, 4-byte aligned. */ 150 #define FSL_SATA_CT_OFFSET (FSL_SATA_CL_OFFSET + FSL_SATA_CL_SIZE * FSL_SATA_MAX_SLOTS) 151 #define FSL_SATA_CT_SIZE (96 + FSL_SATA_SG_ENTRIES * 16) 152 /* Total main work area. */ 153 #define FSL_SATA_WORK_SIZE (FSL_SATA_CT_OFFSET + FSL_SATA_CT_SIZE * FSL_SATA_MAX_SLOTS) 154 #define FSL_SATA_MAX_XFER (64 * 1024 * 1024) 155 156 /* Some convenience macros for getting the CTP and CLP */ 157 #define FSL_SATA_CTP_BUS(ch, slot) \ 158 ((ch->dma.work_bus + FSL_SATA_CT_OFFSET + (FSL_SATA_CT_SIZE * slot->slot))) 159 #define FSL_SATA_PRD_OFFSET(prd) (96 + (prd) * 16) 160 #define FSL_SATA_CTP(ch, slot) \ 161 ((struct fsl_sata_cmd_tab *)(ch->dma.work + FSL_SATA_CT_OFFSET + \ 162 (FSL_SATA_CT_SIZE * slot->slot))) 163 #define FSL_SATA_CLP(ch, slot) \ 164 ((struct fsl_sata_cmd_list *) (ch->dma.work + FSL_SATA_CL_OFFSET + \ 165 (FSL_SATA_CL_SIZE * slot->slot))) 166 167 struct fsl_sata_dma_prd { 168 uint32_t dba; 169 uint32_t reserved; 170 uint32_t reserved2; 171 uint32_t dwc_flg; /* 0 based */ 172 #define FSL_SATA_PRD_MASK 0x01fffffc /* max 32MB */ 173 #define FSL_SATA_PRD_MAX (FSL_SATA_PRD_MASK + 4) 174 #define FSL_SATA_PRD_SNOOP 0x10000000 175 #define FSL_SATA_PRD_EXT 0x80000000 176 } __packed; 177 178 struct fsl_sata_cmd_tab { 179 uint8_t cfis[32]; 180 uint8_t sfis[32]; 181 uint8_t acmd[16]; 182 uint8_t reserved[16]; 183 struct fsl_sata_dma_prd prd_tab[FSL_SATA_SG_ENTRIES]; 184 #define FSL_SATA_PRD_EXT_INDEX 15 185 #define FSL_SATA_PRD_MAX_DIRECT 16 186 } __packed; 187 188 struct fsl_sata_cmd_list { 189 uint32_t cda; /* word aligned */ 190 uint16_t fis_length; /* length in bytes (aligned to words) */ 191 uint16_t prd_length; /* PRD entries */ 192 uint32_t ttl; 193 uint32_t cmd_flags; 194 #define FSL_SATA_CMD_TAG_MASK 0x001f 195 #define FSL_SATA_CMD_ATAPI 0x0020 196 #define FSL_SATA_CMD_BIST 0x0040 197 #define FSL_SATA_CMD_RESET 0x0080 198 #define FSL_SATA_CMD_QUEUED 0x0100 199 #define FSL_SATA_CMD_SNOOP 0x0200 200 #define FSL_SATA_CMD_VBIST 0x0400 201 #define FSL_SATA_CMD_WRITE 0x0800 202 203 } __packed; 204 205 /* misc defines */ 206 #define ATA_IRQ_RID 0 207 #define ATA_INTR_FLAGS (INTR_MPSAFE|INTR_TYPE_BIO|INTR_ENTROPY) 208 209 struct ata_dmaslot { 210 bus_dmamap_t data_map; /* data DMA map */ 211 int nsegs; /* Number of segs loaded */ 212 }; 213 214 /* structure holding DMA related information */ 215 struct ata_dma { 216 bus_dma_tag_t work_tag; /* workspace DMA tag */ 217 bus_dmamap_t work_map; /* workspace DMA map */ 218 uint8_t *work; /* workspace */ 219 bus_addr_t work_bus; /* bus address of work */ 220 bus_dma_tag_t data_tag; /* data DMA tag */ 221 }; 222 223 enum fsl_sata_slot_states { 224 FSL_SATA_SLOT_EMPTY, 225 FSL_SATA_SLOT_LOADING, 226 FSL_SATA_SLOT_RUNNING, 227 FSL_SATA_SLOT_EXECUTING 228 }; 229 230 struct fsl_sata_slot { 231 struct fsl_sata_channel *ch; /* Channel */ 232 uint8_t slot; /* Number of this slot */ 233 enum fsl_sata_slot_states state; /* Slot state */ 234 union ccb *ccb; /* CCB occupying slot */ 235 struct ata_dmaslot dma; /* DMA data of this slot */ 236 struct callout timeout; /* Execution timeout */ 237 uint32_t ttl; 238 }; 239 240 struct fsl_sata_device { 241 int revision; 242 int mode; 243 u_int bytecount; 244 u_int atapi; 245 u_int tags; 246 u_int caps; 247 }; 248 249 /* structure describing an ATA channel */ 250 struct fsl_sata_channel { 251 device_t dev; /* Device handle */ 252 int r_mid; /* Physical channel RID */ 253 struct resource *r_mem; /* Memory of this channel */ 254 struct resource *r_irq; /* Interrupt of this channel */ 255 void *ih; /* Interrupt handle */ 256 struct ata_dma dma; /* DMA data */ 257 struct cam_sim *sim; 258 struct cam_path *path; 259 uint32_t caps; /* Controller capabilities */ 260 int pm_level; /* power management level */ 261 int devices; /* What is present */ 262 int pm_present; /* PM presence reported */ 263 264 union ccb *hold[FSL_SATA_MAX_SLOTS]; 265 struct fsl_sata_slot slot[FSL_SATA_MAX_SLOTS]; 266 uint32_t oslots; /* Occupied slots */ 267 uint32_t rslots; /* Running slots */ 268 uint32_t aslots; /* Slots with atomic commands */ 269 uint32_t eslots; /* Slots in error */ 270 uint32_t toslots; /* Slots in timeout */ 271 int lastslot; /* Last used slot */ 272 int taggedtarget; /* Last tagged target */ 273 int numrslots; /* Number of running slots */ 274 int numrslotspd[16];/* Number of running slots per dev */ 275 int numtslots; /* Number of tagged slots */ 276 int numtslotspd[16];/* Number of tagged slots per dev */ 277 int numhslots; /* Number of held slots */ 278 int recoverycmd; /* Our READ LOG active */ 279 int fatalerr; /* Fatal error happend */ 280 int resetting; /* Hard-reset in progress. */ 281 int resetpolldiv; /* Hard-reset poll divider. */ 282 union ccb *frozen; /* Frozen command */ 283 struct callout pm_timer; /* Power management events */ 284 struct callout reset_timer; /* Hard-reset timeout */ 285 286 struct fsl_sata_device user[16]; /* User-specified settings */ 287 struct fsl_sata_device curr[16]; /* Current settings */ 288 289 struct mtx_padalign mtx; /* state lock */ 290 STAILQ_HEAD(, ccb_hdr) doneq; /* queue of completed CCBs */ 291 int batch; /* doneq is in use */ 292 }; 293 294 enum fsl_sata_err_type { 295 FSL_SATA_ERR_NONE, /* No error */ 296 FSL_SATA_ERR_INVALID, /* Error detected by us before submitting. */ 297 FSL_SATA_ERR_INNOCENT, /* Innocent victim. */ 298 FSL_SATA_ERR_TFE, /* Task File Error. */ 299 FSL_SATA_ERR_SATA, /* SATA error. */ 300 FSL_SATA_ERR_TIMEOUT, /* Command execution timeout. */ 301 FSL_SATA_ERR_NCQ, /* NCQ command error. CCB should be put on hold 302 * until READ LOG executed to reveal error. */ 303 }; 304 305 /* macros to hide busspace uglyness */ 306 #define ATA_INL(res, offset) \ 307 bus_read_4((res), (offset)) 308 #define ATA_OUTL(res, offset, value) \ 309 bus_write_4((res), (offset), (value)) 310 311 static int 312 fsl_sata_probe(device_t dev) 313 { 314 315 if (!ofw_bus_is_compatible(dev, "fsl,pq-sata-v2") && 316 !ofw_bus_is_compatible(dev, "fsl,pq-sata")) 317 return (ENXIO); 318 319 device_set_desc_copy(dev, "Freescale Integrated SATA Controller"); 320 return (BUS_PROBE_DEFAULT); 321 } 322 323 static int 324 fsl_sata_attach(device_t dev) 325 { 326 struct fsl_sata_channel *ch = device_get_softc(dev); 327 struct cam_devq *devq; 328 int rid, error, i, sata_rev = 0; 329 330 ch->dev = dev; 331 mtx_init(&ch->mtx, "FSL SATA channel lock", NULL, MTX_DEF); 332 ch->pm_level = 0; 333 resource_int_value(device_get_name(dev), 334 device_get_unit(dev), "pm_level", &ch->pm_level); 335 STAILQ_INIT(&ch->doneq); 336 if (ch->pm_level > 3) 337 callout_init_mtx(&ch->pm_timer, &ch->mtx, 0); 338 resource_int_value(device_get_name(dev), 339 device_get_unit(dev), "sata_rev", &sata_rev); 340 for (i = 0; i < 16; i++) { 341 ch->user[i].revision = sata_rev; 342 ch->user[i].mode = 0; 343 ch->user[i].bytecount = 8192; 344 ch->user[i].tags = FSL_SATA_MAX_SLOTS; 345 ch->user[i].caps = 0; 346 ch->curr[i] = ch->user[i]; 347 if (ch->pm_level) { 348 ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ | 349 CTS_SATA_CAPS_D_PMREQ; 350 } 351 ch->user[i].caps |= CTS_SATA_CAPS_H_AN; 352 } 353 ch->r_mid = 0; 354 if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 355 &ch->r_mid, RF_ACTIVE))) 356 return (ENXIO); 357 rman_set_bustag(ch->r_mem, &bs_le_tag); 358 fsl_sata_dmainit(dev); 359 fsl_sata_slotsalloc(dev); 360 fsl_sata_init(dev); 361 rid = ATA_IRQ_RID; 362 if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, 363 &rid, RF_SHAREABLE | RF_ACTIVE))) { 364 device_printf(dev, "Unable to map interrupt\n"); 365 error = ENXIO; 366 goto err0; 367 } 368 if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL, 369 fsl_sata_intr, ch, &ch->ih))) { 370 device_printf(dev, "Unable to setup interrupt\n"); 371 error = ENXIO; 372 goto err1; 373 } 374 mtx_lock(&ch->mtx); 375 /* Create the device queue for our SIM. */ 376 devq = cam_simq_alloc(FSL_SATA_MAX_SLOTS); 377 if (devq == NULL) { 378 device_printf(dev, "Unable to allocate simq\n"); 379 error = ENOMEM; 380 goto err1; 381 } 382 /* Construct SIM entry */ 383 ch->sim = cam_sim_alloc(fsl_sataaction, fsl_satapoll, "fslsata", ch, 384 device_get_unit(dev), (struct mtx *)&ch->mtx, 2, FSL_SATA_MAX_SLOTS, 385 devq); 386 if (ch->sim == NULL) { 387 cam_simq_free(devq); 388 device_printf(dev, "unable to allocate sim\n"); 389 error = ENOMEM; 390 goto err1; 391 } 392 if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) { 393 device_printf(dev, "unable to register xpt bus\n"); 394 error = ENXIO; 395 goto err2; 396 } 397 if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim), 398 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 399 device_printf(dev, "unable to create path\n"); 400 error = ENXIO; 401 goto err3; 402 } 403 if (ch->pm_level > 3) { 404 callout_reset(&ch->pm_timer, 405 (ch->pm_level == 4) ? hz / 1000 : hz / 8, 406 fsl_sata_pm, ch); 407 } 408 mtx_unlock(&ch->mtx); 409 return (0); 410 411 err3: 412 xpt_bus_deregister(cam_sim_path(ch->sim)); 413 err2: 414 cam_sim_free(ch->sim, /*free_devq*/TRUE); 415 err1: 416 mtx_unlock(&ch->mtx); 417 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 418 err0: 419 bus_release_resource(dev, SYS_RES_MEMORY, ch->r_mid, ch->r_mem); 420 mtx_destroy(&ch->mtx); 421 return (error); 422 } 423 424 static int 425 fsl_sata_detach(device_t dev) 426 { 427 struct fsl_sata_channel *ch = device_get_softc(dev); 428 429 mtx_lock(&ch->mtx); 430 xpt_async(AC_LOST_DEVICE, ch->path, NULL); 431 432 xpt_free_path(ch->path); 433 xpt_bus_deregister(cam_sim_path(ch->sim)); 434 cam_sim_free(ch->sim, /*free_devq*/TRUE); 435 mtx_unlock(&ch->mtx); 436 437 if (ch->pm_level > 3) 438 callout_drain(&ch->pm_timer); 439 bus_teardown_intr(dev, ch->r_irq, ch->ih); 440 bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq); 441 442 fsl_sata_deinit(dev); 443 fsl_sata_slotsfree(dev); 444 fsl_sata_dmafini(dev); 445 446 bus_release_resource(dev, SYS_RES_MEMORY, ch->r_mid, ch->r_mem); 447 mtx_destroy(&ch->mtx); 448 return (0); 449 } 450 451 static int 452 fsl_sata_wait_register(struct fsl_sata_channel *ch, bus_size_t off, 453 unsigned int mask, unsigned int val, int t) 454 { 455 int timeout = 0; 456 uint32_t rval; 457 458 while (((rval = ATA_INL(ch->r_mem, off)) & mask) != val) { 459 if (timeout > t) { 460 return (EBUSY); 461 } 462 DELAY(1000); 463 timeout++; 464 } 465 return (0); 466 } 467 468 static int 469 fsl_sata_init(device_t dev) 470 { 471 struct fsl_sata_channel *ch = device_get_softc(dev); 472 uint64_t work; 473 uint32_t r; 474 475 /* Disable port interrupts */ 476 r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL); 477 r &= ~FSL_SATA_P_HCTRL_HC_ON; 478 r |= FSL_SATA_P_HCTRL_HC_FORCE_OFF; 479 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK); 480 fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 481 FSL_SATA_P_HSTS_HS_ON, 0, 1000); 482 /* Setup work areas */ 483 work = ch->dma.work_bus + FSL_SATA_CL_OFFSET; 484 ATA_OUTL(ch->r_mem, FSL_SATA_P_CHBA, work); 485 r &= ~FSL_SATA_P_HCTRL_ENT; 486 r &= ~FSL_SATA_P_HCTRL_PM; 487 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r); 488 r = ATA_INL(ch->r_mem, FSL_SATA_P_PCC); 489 ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, r & ~FSL_SATA_PCC_LPB_EN); 490 ATA_OUTL(ch->r_mem, FSL_SATA_P_ICC, (1 << FSL_SATA_P_ICC_ITC_S)); 491 fsl_sata_start(ch); 492 return (0); 493 } 494 495 static int 496 fsl_sata_deinit(device_t dev) 497 { 498 struct fsl_sata_channel *ch = device_get_softc(dev); 499 uint32_t r; 500 501 /* Disable port interrupts. */ 502 r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL); 503 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_INT_MASK); 504 /* Reset command register. */ 505 fsl_sata_stop(ch); 506 /* Allow everything, including partial and slumber modes. */ 507 ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, 0); 508 DELAY(100); 509 /* Disable PHY. */ 510 ATA_OUTL(ch->r_mem, FSL_SATA_P_SCTL, ATA_SC_DET_DISABLE); 511 r = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL); 512 /* Turn off the controller. */ 513 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, r & ~FSL_SATA_P_HCTRL_HC_ON); 514 return (0); 515 } 516 517 static int 518 fsl_sata_suspend(device_t dev) 519 { 520 struct fsl_sata_channel *ch = device_get_softc(dev); 521 522 mtx_lock(&ch->mtx); 523 xpt_freeze_simq(ch->sim, 1); 524 while (ch->oslots) 525 msleep(ch, &ch->mtx, PRIBIO, "fsl_satasusp", hz/100); 526 fsl_sata_deinit(dev); 527 mtx_unlock(&ch->mtx); 528 return (0); 529 } 530 531 static int 532 fsl_sata_resume(device_t dev) 533 { 534 struct fsl_sata_channel *ch = device_get_softc(dev); 535 536 mtx_lock(&ch->mtx); 537 fsl_sata_init(dev); 538 fsl_sata_reset(ch); 539 xpt_release_simq(ch->sim, TRUE); 540 mtx_unlock(&ch->mtx); 541 return (0); 542 } 543 544 static device_method_t fsl_satach_methods[] = { 545 DEVMETHOD(device_probe, fsl_sata_probe), 546 DEVMETHOD(device_attach, fsl_sata_attach), 547 DEVMETHOD(device_detach, fsl_sata_detach), 548 DEVMETHOD(device_suspend, fsl_sata_suspend), 549 DEVMETHOD(device_resume, fsl_sata_resume), 550 DEVMETHOD_END 551 }; 552 553 static driver_t fsl_satach_driver = { 554 "fslsata", 555 fsl_satach_methods, 556 sizeof(struct fsl_sata_channel) 557 }; 558 559 DRIVER_MODULE(fsl_satach, simplebus, fsl_satach_driver, NULL, NULL); 560 561 struct fsl_sata_dc_cb_args { 562 bus_addr_t maddr; 563 int error; 564 }; 565 566 static void 567 fsl_sata_dmainit(device_t dev) 568 { 569 struct fsl_sata_channel *ch = device_get_softc(dev); 570 struct fsl_sata_dc_cb_args dcba; 571 572 /* Command area. */ 573 if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0, 574 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, 575 NULL, NULL, FSL_SATA_WORK_SIZE, 1, FSL_SATA_WORK_SIZE, 576 0, NULL, NULL, &ch->dma.work_tag)) 577 goto error; 578 if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 579 BUS_DMA_ZERO, &ch->dma.work_map)) 580 goto error; 581 if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work, 582 FSL_SATA_WORK_SIZE, fsl_sata_dmasetupc_cb, &dcba, 0) || dcba.error) { 583 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 584 goto error; 585 } 586 ch->dma.work_bus = dcba.maddr; 587 /* Data area. */ 588 if (bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0, 589 BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, 590 NULL, NULL, FSL_SATA_MAX_XFER, 591 FSL_SATA_SG_ENTRIES - 1, FSL_SATA_PRD_MAX, 592 0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) { 593 goto error; 594 } 595 if (bootverbose) 596 device_printf(dev, "work area: %p\n", ch->dma.work); 597 return; 598 599 error: 600 device_printf(dev, "WARNING - DMA initialization failed\n"); 601 fsl_sata_dmafini(dev); 602 } 603 604 static void 605 fsl_sata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) 606 { 607 struct fsl_sata_dc_cb_args *dcba = (struct fsl_sata_dc_cb_args *)xsc; 608 609 if (!(dcba->error = error)) 610 dcba->maddr = segs[0].ds_addr; 611 } 612 613 static void 614 fsl_sata_dmafini(device_t dev) 615 { 616 struct fsl_sata_channel *ch = device_get_softc(dev); 617 618 if (ch->dma.data_tag) { 619 bus_dma_tag_destroy(ch->dma.data_tag); 620 ch->dma.data_tag = NULL; 621 } 622 if (ch->dma.work_bus) { 623 bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map); 624 bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map); 625 ch->dma.work_bus = 0; 626 ch->dma.work = NULL; 627 } 628 if (ch->dma.work_tag) { 629 bus_dma_tag_destroy(ch->dma.work_tag); 630 ch->dma.work_tag = NULL; 631 } 632 } 633 634 static void 635 fsl_sata_slotsalloc(device_t dev) 636 { 637 struct fsl_sata_channel *ch = device_get_softc(dev); 638 int i; 639 640 /* Alloc and setup command/dma slots */ 641 bzero(ch->slot, sizeof(ch->slot)); 642 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 643 struct fsl_sata_slot *slot = &ch->slot[i]; 644 645 slot->ch = ch; 646 slot->slot = i; 647 slot->state = FSL_SATA_SLOT_EMPTY; 648 slot->ccb = NULL; 649 callout_init_mtx(&slot->timeout, &ch->mtx, 0); 650 651 if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map)) 652 device_printf(ch->dev, "FAILURE - create data_map\n"); 653 } 654 } 655 656 static void 657 fsl_sata_slotsfree(device_t dev) 658 { 659 struct fsl_sata_channel *ch = device_get_softc(dev); 660 int i; 661 662 /* Free all dma slots */ 663 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 664 struct fsl_sata_slot *slot = &ch->slot[i]; 665 666 callout_drain(&slot->timeout); 667 if (slot->dma.data_map) { 668 bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map); 669 slot->dma.data_map = NULL; 670 } 671 } 672 } 673 674 static int 675 fsl_sata_phy_check_events(struct fsl_sata_channel *ch, u_int32_t serr) 676 { 677 678 if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) || 679 ((ch->pm_level != 0) && (serr & ATA_SE_EXCHANGED))) { 680 u_int32_t status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS); 681 union ccb *ccb; 682 683 if (bootverbose) { 684 if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE) 685 device_printf(ch->dev, "CONNECT requested\n"); 686 else 687 device_printf(ch->dev, "DISCONNECT requested\n"); 688 } 689 /* Issue soft reset */ 690 xpt_async(AC_BUS_RESET, ch->path, NULL); 691 if ((ccb = xpt_alloc_ccb_nowait()) == NULL) 692 return (0); 693 if (xpt_create_path(&ccb->ccb_h.path, NULL, 694 cam_sim_path(ch->sim), 695 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 696 xpt_free_ccb(ccb); 697 return (0); 698 } 699 xpt_rescan(ccb); 700 return (1); 701 } 702 return (0); 703 } 704 705 static void 706 fsl_sata_notify_events(struct fsl_sata_channel *ch, u_int32_t status) 707 { 708 struct cam_path *dpath; 709 int i; 710 711 ATA_OUTL(ch->r_mem, FSL_SATA_P_SNTF, status); 712 if (bootverbose) 713 device_printf(ch->dev, "SNTF 0x%04x\n", status); 714 for (i = 0; i < 16; i++) { 715 if ((status & (1 << i)) == 0) 716 continue; 717 if (xpt_create_path(&dpath, NULL, 718 xpt_path_path_id(ch->path), i, 0) == CAM_REQ_CMP) { 719 xpt_async(AC_SCSI_AEN, dpath, NULL); 720 xpt_free_path(dpath); 721 } 722 } 723 } 724 725 static void 726 fsl_sata_done(struct fsl_sata_channel *ch, union ccb *ccb) 727 { 728 729 mtx_assert(&ch->mtx, MA_OWNED); 730 if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) == 0 || 731 ch->batch == 0) { 732 xpt_done(ccb); 733 return; 734 } 735 736 STAILQ_INSERT_TAIL(&ch->doneq, &ccb->ccb_h, sim_links.stqe); 737 } 738 739 static void 740 fsl_sata_intr(void *arg) 741 { 742 struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg; 743 struct ccb_hdr *ccb_h; 744 uint32_t istatus; 745 STAILQ_HEAD(, ccb_hdr) tmp_doneq = STAILQ_HEAD_INITIALIZER(tmp_doneq); 746 747 /* Read interrupt statuses. */ 748 istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & 0x7ffff; 749 if ((istatus & 0x3f) == 0) 750 return; 751 752 mtx_lock(&ch->mtx); 753 ch->batch = 1; 754 fsl_sata_intr_main(ch, istatus); 755 ch->batch = 0; 756 /* 757 * Prevent the possibility of issues caused by processing the queue 758 * while unlocked below by moving the contents to a local queue. 759 */ 760 STAILQ_CONCAT(&tmp_doneq, &ch->doneq); 761 mtx_unlock(&ch->mtx); 762 while ((ccb_h = STAILQ_FIRST(&tmp_doneq)) != NULL) { 763 STAILQ_REMOVE_HEAD(&tmp_doneq, sim_links.stqe); 764 xpt_done_direct((union ccb *)ccb_h); 765 } 766 /* Clear interrupt statuses. */ 767 ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, istatus & 0x3f); 768 769 } 770 771 static void 772 fsl_sata_pm(void *arg) 773 { 774 struct fsl_sata_channel *ch = (struct fsl_sata_channel *)arg; 775 uint32_t work; 776 777 if (ch->numrslots != 0) 778 return; 779 work = ATA_INL(ch->r_mem, FSL_SATA_P_PCC) & ~FSL_SATA_PCC_LPB_EN; 780 if (ch->pm_level == 4) 781 work |= FSL_SATA_P_PCC_PARTIAL; 782 else 783 work |= FSL_SATA_P_PCC_SLUMBER; 784 ATA_OUTL(ch->r_mem, FSL_SATA_P_PCC, work); 785 } 786 787 /* XXX: interrupt todo */ 788 static void 789 fsl_sata_intr_main(struct fsl_sata_channel *ch, uint32_t istatus) 790 { 791 uint32_t cer, der, serr = 0, sntf = 0, ok, err; 792 enum fsl_sata_err_type et; 793 int i; 794 795 /* Complete all successful commands. */ 796 ok = ATA_INL(ch->r_mem, FSL_SATA_P_CCR); 797 /* Mark all commands complete, to complete the interrupt. */ 798 ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, ok); 799 if (ch->aslots == 0 && ok != 0) { 800 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 801 if (((ok >> i) & 1) && ch->slot[i].ccb != NULL) 802 fsl_sata_end_transaction(&ch->slot[i], 803 FSL_SATA_ERR_NONE); 804 } 805 } 806 /* Read command statuses. */ 807 if (istatus & FSL_SATA_P_HSTS_SNTFU) 808 sntf = ATA_INL(ch->r_mem, FSL_SATA_P_SNTF); 809 /* XXX: Process PHY events */ 810 serr = ATA_INL(ch->r_mem, FSL_SATA_P_SERR); 811 ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, serr); 812 if (istatus & (FSL_SATA_P_HSTS_PR)) { 813 if (serr) { 814 fsl_sata_phy_check_events(ch, serr); 815 } 816 } 817 /* Process command errors */ 818 err = (istatus & (FSL_SATA_P_HSTS_FE | FSL_SATA_P_HSTS_DE)); 819 cer = ATA_INL(ch->r_mem, FSL_SATA_P_CER); 820 ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, cer); 821 der = ATA_INL(ch->r_mem, FSL_SATA_P_DER); 822 ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, der); 823 /* On error, complete the rest of commands with error statuses. */ 824 if (err) { 825 if (ch->frozen) { 826 union ccb *fccb = ch->frozen; 827 ch->frozen = NULL; 828 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 829 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 830 xpt_freeze_devq(fccb->ccb_h.path, 1); 831 fccb->ccb_h.status |= CAM_DEV_QFRZN; 832 } 833 fsl_sata_done(ch, fccb); 834 } 835 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 836 if (ch->slot[i].ccb == NULL) 837 continue; 838 if ((cer & (1 << i)) != 0) 839 et = FSL_SATA_ERR_TFE; 840 else if ((der & (1 << ch->slot[i].ccb->ccb_h.target_id)) != 0) 841 et = FSL_SATA_ERR_SATA; 842 else 843 et = FSL_SATA_ERR_INVALID; 844 fsl_sata_end_transaction(&ch->slot[i], et); 845 } 846 } 847 /* Process NOTIFY events */ 848 if (sntf) 849 fsl_sata_notify_events(ch, sntf); 850 } 851 852 /* Must be called with channel locked. */ 853 static int 854 fsl_sata_check_collision(struct fsl_sata_channel *ch, union ccb *ccb) 855 { 856 int t = ccb->ccb_h.target_id; 857 858 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 859 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 860 /* Tagged command while we have no supported tag free. */ 861 if (((~ch->oslots) & (0xffff >> (16 - ch->curr[t].tags))) == 0) 862 return (1); 863 /* Tagged command while untagged are active. */ 864 if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] == 0) 865 return (1); 866 } else { 867 /* Untagged command while tagged are active. */ 868 if (ch->numrslotspd[t] != 0 && ch->numtslotspd[t] != 0) 869 return (1); 870 } 871 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 872 (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) { 873 /* Atomic command while anything active. */ 874 if (ch->numrslots != 0) 875 return (1); 876 } 877 /* We have some atomic command running. */ 878 if (ch->aslots != 0) 879 return (1); 880 return (0); 881 } 882 883 /* Must be called with channel locked. */ 884 static void 885 fsl_sata_begin_transaction(struct fsl_sata_channel *ch, union ccb *ccb) 886 { 887 struct fsl_sata_slot *slot; 888 int tag, tags; 889 890 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 891 ("fsl_sata_begin_transaction func_code=0x%x\n", ccb->ccb_h.func_code)); 892 /* Choose empty slot. */ 893 tags = FSL_SATA_MAX_SLOTS; 894 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 895 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) 896 tags = ch->curr[ccb->ccb_h.target_id].tags; 897 if (ch->lastslot + 1 < tags) 898 tag = ffs(~(ch->oslots >> (ch->lastslot + 1))); 899 else 900 tag = 0; 901 if (tag == 0 || tag + ch->lastslot >= tags) 902 tag = ffs(~ch->oslots) - 1; 903 else 904 tag += ch->lastslot; 905 ch->lastslot = tag; 906 /* Occupy chosen slot. */ 907 slot = &ch->slot[tag]; 908 slot->ccb = ccb; 909 slot->ttl = 0; 910 /* Stop PM timer. */ 911 if (ch->numrslots == 0 && ch->pm_level > 3) 912 callout_stop(&ch->pm_timer); 913 /* Update channel stats. */ 914 ch->oslots |= (1 << tag); 915 ch->numrslots++; 916 ch->numrslotspd[ccb->ccb_h.target_id]++; 917 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 918 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 919 ch->numtslots++; 920 ch->numtslotspd[ccb->ccb_h.target_id]++; 921 ch->taggedtarget = ccb->ccb_h.target_id; 922 } 923 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 924 (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT))) 925 ch->aslots |= (1 << tag); 926 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 927 slot->state = FSL_SATA_SLOT_LOADING; 928 bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb, 929 fsl_sata_dmasetprd, slot, 0); 930 } else { 931 slot->dma.nsegs = 0; 932 fsl_sata_execute_transaction(slot); 933 } 934 935 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 936 ("fsl_sata_begin_transaction exit\n")); 937 } 938 939 /* Locked by busdma engine. */ 940 static void 941 fsl_sata_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 942 { 943 struct fsl_sata_slot *slot = arg; 944 struct fsl_sata_channel *ch = slot->ch; 945 struct fsl_sata_cmd_tab *ctp; 946 struct fsl_sata_dma_prd *prd; 947 int i, j, len, extlen; 948 949 if (error) { 950 device_printf(ch->dev, "DMA load error %d\n", error); 951 fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID); 952 return; 953 } 954 KASSERT(nsegs <= FSL_SATA_SG_ENTRIES - 1, 955 ("too many DMA segment entries\n")); 956 /* Get a piece of the workspace for this request */ 957 ctp = FSL_SATA_CTP(ch, slot); 958 /* Fill S/G table */ 959 prd = &ctp->prd_tab[0]; 960 for (i = 0, j = 0; i < nsegs; i++, j++) { 961 if (j == FSL_SATA_PRD_EXT_INDEX && 962 FSL_SATA_PRD_MAX_DIRECT < nsegs) { 963 prd[j].dba = htole32(FSL_SATA_CTP_BUS(ch, slot) + 964 FSL_SATA_PRD_OFFSET(j+1)); 965 j++; 966 extlen = 0; 967 } 968 len = segs[i].ds_len; 969 len = roundup2(len, sizeof(uint32_t)); 970 prd[j].dba = htole32((uint32_t)segs[i].ds_addr); 971 prd[j].dwc_flg = htole32(FSL_SATA_PRD_SNOOP | len); 972 slot->ttl += len; 973 if (j > FSL_SATA_PRD_MAX_DIRECT) 974 extlen += len; 975 } 976 slot->dma.nsegs = j; 977 if (j > FSL_SATA_PRD_MAX_DIRECT) 978 prd[FSL_SATA_PRD_EXT_INDEX].dwc_flg = 979 htole32(FSL_SATA_PRD_SNOOP | FSL_SATA_PRD_EXT | extlen); 980 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 981 ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ? 982 BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE)); 983 fsl_sata_execute_transaction(slot); 984 } 985 986 /* Must be called with channel locked. */ 987 static void 988 fsl_sata_execute_transaction(struct fsl_sata_slot *slot) 989 { 990 struct fsl_sata_channel *ch = slot->ch; 991 struct fsl_sata_cmd_tab *ctp; 992 struct fsl_sata_cmd_list *clp; 993 union ccb *ccb = slot->ccb; 994 int port = ccb->ccb_h.target_id & 0x0f; 995 int fis_size, i, softreset; 996 uint32_t tmp; 997 uint32_t cmd_flags = FSL_SATA_CMD_WRITE | FSL_SATA_CMD_SNOOP; 998 999 softreset = 0; 1000 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 1001 ("fsl_sata_execute_transaction func_code=0x%x\n", ccb->ccb_h.func_code)); 1002 /* Get a piece of the workspace for this request */ 1003 ctp = FSL_SATA_CTP(ch, slot); 1004 /* Setup the FIS for this request */ 1005 if (!(fis_size = fsl_sata_setup_fis(ch, ctp, ccb, slot->slot))) { 1006 device_printf(ch->dev, "Setting up SATA FIS failed\n"); 1007 fsl_sata_end_transaction(slot, FSL_SATA_ERR_INVALID); 1008 return; 1009 } 1010 /* Setup the command list entry */ 1011 clp = FSL_SATA_CLP(ch, slot); 1012 clp->fis_length = htole16(fis_size); 1013 clp->prd_length = htole16(slot->dma.nsegs); 1014 /* Special handling for Soft Reset command. */ 1015 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1016 (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) { 1017 if (ccb->ataio.cmd.control & ATA_A_RESET) { 1018 softreset = 1; 1019 cmd_flags |= FSL_SATA_CMD_RESET; 1020 } else { 1021 /* Prepare FIS receive area for check. */ 1022 for (i = 0; i < 32; i++) 1023 ctp->sfis[i] = 0xff; 1024 softreset = 2; 1025 } 1026 } 1027 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) 1028 cmd_flags |= FSL_SATA_CMD_QUEUED; 1029 clp->cmd_flags = htole32(cmd_flags | 1030 (ccb->ccb_h.func_code == XPT_SCSI_IO ? FSL_SATA_CMD_ATAPI : 0) | 1031 slot->slot); 1032 clp->ttl = htole32(slot->ttl); 1033 clp->cda = htole32(FSL_SATA_CTP_BUS(ch, slot)); 1034 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 1035 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); 1036 /* Issue command to the controller. */ 1037 slot->state = FSL_SATA_SLOT_RUNNING; 1038 ch->rslots |= (1 << slot->slot); 1039 ATA_OUTL(ch->r_mem, FSL_SATA_P_CQPMP, port); 1040 ATA_OUTL(ch->r_mem, FSL_SATA_P_CQR, (1 << slot->slot)); 1041 /* Device reset commands don't interrupt. Poll them. */ 1042 if (ccb->ccb_h.func_code == XPT_ATA_IO && 1043 (ccb->ataio.cmd.command == ATA_DEVICE_RESET || softreset)) { 1044 int count, timeout = ccb->ccb_h.timeout * 100; 1045 enum fsl_sata_err_type et = FSL_SATA_ERR_NONE; 1046 1047 for (count = 0; count < timeout; count++) { 1048 DELAY(10); 1049 tmp = 0; 1050 if (softreset == 2) { 1051 tmp = ATA_INL(ch->r_mem, FSL_SATA_P_SIG); 1052 if (tmp != 0 && tmp != 0xffffffff) 1053 break; 1054 continue; 1055 } 1056 if ((ATA_INL(ch->r_mem, FSL_SATA_P_CCR) & (1 << slot->slot)) != 0) 1057 break; 1058 } 1059 1060 if (timeout && (count >= timeout)) { 1061 device_printf(ch->dev, "Poll timeout on slot %d port %d (round %d)\n", 1062 slot->slot, port, softreset); 1063 device_printf(ch->dev, "hsts %08x cqr %08x ccr %08x ss %08x " 1064 "rs %08x cer %08x der %08x serr %08x car %08x sig %08x\n", 1065 ATA_INL(ch->r_mem, FSL_SATA_P_HSTS), 1066 ATA_INL(ch->r_mem, FSL_SATA_P_CQR), 1067 ATA_INL(ch->r_mem, FSL_SATA_P_CCR), 1068 ATA_INL(ch->r_mem, FSL_SATA_P_SSTS), ch->rslots, 1069 ATA_INL(ch->r_mem, FSL_SATA_P_CER), 1070 ATA_INL(ch->r_mem, FSL_SATA_P_DER), 1071 ATA_INL(ch->r_mem, FSL_SATA_P_SERR), 1072 ATA_INL(ch->r_mem, FSL_SATA_P_CAR), 1073 ATA_INL(ch->r_mem, FSL_SATA_P_SIG)); 1074 et = FSL_SATA_ERR_TIMEOUT; 1075 } 1076 1077 fsl_sata_end_transaction(slot, et); 1078 return; 1079 } 1080 /* Start command execution timeout */ 1081 callout_reset_sbt(&slot->timeout, SBT_1MS * ccb->ccb_h.timeout / 2, 1082 0, fsl_sata_timeout, slot, 0); 1083 return; 1084 } 1085 1086 /* Must be called with channel locked. */ 1087 static void 1088 fsl_sata_process_timeout(struct fsl_sata_channel *ch) 1089 { 1090 int i; 1091 1092 mtx_assert(&ch->mtx, MA_OWNED); 1093 /* Handle the rest of commands. */ 1094 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1095 /* Do we have a running request on slot? */ 1096 if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING) 1097 continue; 1098 fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_TIMEOUT); 1099 } 1100 } 1101 1102 /* Must be called with channel locked. */ 1103 static void 1104 fsl_sata_rearm_timeout(struct fsl_sata_channel *ch) 1105 { 1106 int i; 1107 1108 mtx_assert(&ch->mtx, MA_OWNED); 1109 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1110 struct fsl_sata_slot *slot = &ch->slot[i]; 1111 1112 /* Do we have a running request on slot? */ 1113 if (slot->state < FSL_SATA_SLOT_RUNNING) 1114 continue; 1115 if ((ch->toslots & (1 << i)) == 0) 1116 continue; 1117 callout_reset_sbt(&slot->timeout, 1118 SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0, 1119 fsl_sata_timeout, slot, 0); 1120 } 1121 } 1122 1123 /* Locked by callout mechanism. */ 1124 static void 1125 fsl_sata_timeout(void *arg) 1126 { 1127 struct fsl_sata_slot *slot = arg; 1128 struct fsl_sata_channel *ch = slot->ch; 1129 device_t dev = ch->dev; 1130 uint32_t sstatus; 1131 1132 /* Check for stale timeout. */ 1133 if (slot->state < FSL_SATA_SLOT_RUNNING) 1134 return; 1135 1136 /* Check if slot was not being executed last time we checked. */ 1137 if (slot->state < FSL_SATA_SLOT_EXECUTING) { 1138 /* Check if slot started executing. */ 1139 sstatus = ATA_INL(ch->r_mem, FSL_SATA_P_CAR); 1140 if ((sstatus & (1 << slot->slot)) != 0) 1141 slot->state = FSL_SATA_SLOT_EXECUTING; 1142 1143 callout_reset_sbt(&slot->timeout, 1144 SBT_1MS * slot->ccb->ccb_h.timeout / 2, 0, 1145 fsl_sata_timeout, slot, 0); 1146 return; 1147 } 1148 1149 device_printf(dev, "Timeout on slot %d port %d\n", 1150 slot->slot, slot->ccb->ccb_h.target_id & 0x0f); 1151 1152 /* Handle frozen command. */ 1153 if (ch->frozen) { 1154 union ccb *fccb = ch->frozen; 1155 ch->frozen = NULL; 1156 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 1157 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 1158 xpt_freeze_devq(fccb->ccb_h.path, 1); 1159 fccb->ccb_h.status |= CAM_DEV_QFRZN; 1160 } 1161 fsl_sata_done(ch, fccb); 1162 } 1163 if (ch->toslots == 0) 1164 xpt_freeze_simq(ch->sim, 1); 1165 ch->toslots |= (1 << slot->slot); 1166 if ((ch->rslots & ~ch->toslots) == 0) 1167 fsl_sata_process_timeout(ch); 1168 else 1169 device_printf(dev, " ... waiting for slots %08x\n", 1170 ch->rslots & ~ch->toslots); 1171 } 1172 1173 /* Must be called with channel locked. */ 1174 static void 1175 fsl_sata_end_transaction(struct fsl_sata_slot *slot, enum fsl_sata_err_type et) 1176 { 1177 struct fsl_sata_channel *ch = slot->ch; 1178 union ccb *ccb = slot->ccb; 1179 struct fsl_sata_cmd_list *clp; 1180 int lastto; 1181 uint32_t sig; 1182 1183 bus_dmamap_sync(ch->dma.work_tag, ch->dma.work_map, 1184 BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); 1185 clp = FSL_SATA_CLP(ch, slot); 1186 /* Read result registers to the result struct */ 1187 if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1188 struct ata_res *res = &ccb->ataio.res; 1189 1190 if ((et == FSL_SATA_ERR_TFE) || 1191 (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) { 1192 struct fsl_sata_cmd_tab *ctp = FSL_SATA_CTP(ch, slot); 1193 uint8_t *fis = ctp->sfis; 1194 1195 res->status = fis[2]; 1196 res->error = fis[3]; 1197 res->lba_low = fis[4]; 1198 res->lba_mid = fis[5]; 1199 res->lba_high = fis[6]; 1200 res->device = fis[7]; 1201 res->lba_low_exp = fis[8]; 1202 res->lba_mid_exp = fis[9]; 1203 res->lba_high_exp = fis[10]; 1204 res->sector_count = fis[12]; 1205 res->sector_count_exp = fis[13]; 1206 1207 if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1208 (ccb->ataio.cmd.control & ATA_A_RESET) == 0) { 1209 sig = ATA_INL(ch->r_mem, FSL_SATA_P_SIG); 1210 res->lba_high = sig >> 24; 1211 res->lba_mid = sig >> 16; 1212 res->lba_low = sig >> 8; 1213 res->sector_count = sig; 1214 } 1215 } else 1216 bzero(res, sizeof(*res)); 1217 if ((ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) == 0 && 1218 (ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1219 ccb->ataio.resid = 1220 ccb->ataio.dxfer_len - le32toh(clp->ttl); 1221 } 1222 } else { 1223 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1224 ccb->csio.resid = 1225 ccb->csio.dxfer_len - le32toh(clp->ttl); 1226 } 1227 } 1228 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) { 1229 bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map, 1230 (ccb->ccb_h.flags & CAM_DIR_IN) ? 1231 BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE); 1232 bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map); 1233 } 1234 if (et != FSL_SATA_ERR_NONE) 1235 ch->eslots |= (1 << slot->slot); 1236 /* In case of error, freeze device for proper recovery. */ 1237 if ((et != FSL_SATA_ERR_NONE) && (!ch->recoverycmd) && 1238 !(ccb->ccb_h.status & CAM_DEV_QFRZN)) { 1239 xpt_freeze_devq(ccb->ccb_h.path, 1); 1240 ccb->ccb_h.status |= CAM_DEV_QFRZN; 1241 } 1242 /* Set proper result status. */ 1243 ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1244 switch (et) { 1245 case FSL_SATA_ERR_NONE: 1246 ccb->ccb_h.status |= CAM_REQ_CMP; 1247 if (ccb->ccb_h.func_code == XPT_SCSI_IO) 1248 ccb->csio.scsi_status = SCSI_STATUS_OK; 1249 break; 1250 case FSL_SATA_ERR_INVALID: 1251 ch->fatalerr = 1; 1252 ccb->ccb_h.status |= CAM_REQ_INVALID; 1253 break; 1254 case FSL_SATA_ERR_INNOCENT: 1255 ccb->ccb_h.status |= CAM_REQUEUE_REQ; 1256 break; 1257 case FSL_SATA_ERR_TFE: 1258 case FSL_SATA_ERR_NCQ: 1259 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 1260 ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR; 1261 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 1262 } else { 1263 ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR; 1264 } 1265 break; 1266 case FSL_SATA_ERR_SATA: 1267 ch->fatalerr = 1; 1268 if (!ch->recoverycmd) { 1269 xpt_freeze_simq(ch->sim, 1); 1270 ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1271 ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1272 } 1273 ccb->ccb_h.status |= CAM_UNCOR_PARITY; 1274 break; 1275 case FSL_SATA_ERR_TIMEOUT: 1276 if (!ch->recoverycmd) { 1277 xpt_freeze_simq(ch->sim, 1); 1278 ccb->ccb_h.status &= ~CAM_STATUS_MASK; 1279 ccb->ccb_h.status |= CAM_RELEASE_SIMQ; 1280 } 1281 ccb->ccb_h.status |= CAM_CMD_TIMEOUT; 1282 break; 1283 default: 1284 ch->fatalerr = 1; 1285 ccb->ccb_h.status |= CAM_REQ_CMP_ERR; 1286 } 1287 /* Free slot. */ 1288 ch->oslots &= ~(1 << slot->slot); 1289 ch->rslots &= ~(1 << slot->slot); 1290 ch->aslots &= ~(1 << slot->slot); 1291 slot->state = FSL_SATA_SLOT_EMPTY; 1292 slot->ccb = NULL; 1293 /* Update channel stats. */ 1294 ch->numrslots--; 1295 ch->numrslotspd[ccb->ccb_h.target_id]--; 1296 ATA_OUTL(ch->r_mem, FSL_SATA_P_CCR, 1 << slot->slot); 1297 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1298 (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) { 1299 ch->numtslots--; 1300 ch->numtslotspd[ccb->ccb_h.target_id]--; 1301 } 1302 /* Cancel timeout state if request completed normally. */ 1303 if (et != FSL_SATA_ERR_TIMEOUT) { 1304 lastto = (ch->toslots == (1 << slot->slot)); 1305 ch->toslots &= ~(1 << slot->slot); 1306 if (lastto) 1307 xpt_release_simq(ch->sim, TRUE); 1308 } 1309 /* If it was first request of reset sequence and there is no error, 1310 * proceed to second request. */ 1311 if ((ccb->ccb_h.func_code == XPT_ATA_IO) && 1312 (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) && 1313 (ccb->ataio.cmd.control & ATA_A_RESET) && 1314 et == FSL_SATA_ERR_NONE) { 1315 ccb->ataio.cmd.control &= ~ATA_A_RESET; 1316 fsl_sata_begin_transaction(ch, ccb); 1317 return; 1318 } 1319 /* If it was our READ LOG command - process it. */ 1320 if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) { 1321 fsl_sata_process_read_log(ch, ccb); 1322 /* If it was our REQUEST SENSE command - process it. */ 1323 } else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) { 1324 fsl_sata_process_request_sense(ch, ccb); 1325 /* If it was NCQ or ATAPI command error, put result on hold. */ 1326 } else if (et == FSL_SATA_ERR_NCQ || 1327 ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR && 1328 (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0)) { 1329 ch->hold[slot->slot] = ccb; 1330 ch->numhslots++; 1331 } else 1332 fsl_sata_done(ch, ccb); 1333 /* If we have no other active commands, ... */ 1334 if (ch->rslots == 0) { 1335 /* if there was fatal error - reset port. */ 1336 if (ch->toslots != 0 || ch->fatalerr) { 1337 fsl_sata_reset(ch); 1338 } else { 1339 /* if we have slots in error, we can reinit port. */ 1340 if (ch->eslots != 0) { 1341 fsl_sata_stop(ch); 1342 fsl_sata_start(ch); 1343 } 1344 /* if there commands on hold, we can do READ LOG. */ 1345 if (!ch->recoverycmd && ch->numhslots) 1346 fsl_sata_issue_recovery(ch); 1347 } 1348 /* If all the rest of commands are in timeout - give them chance. */ 1349 } else if ((ch->rslots & ~ch->toslots) == 0 && 1350 et != FSL_SATA_ERR_TIMEOUT) 1351 fsl_sata_rearm_timeout(ch); 1352 /* Unfreeze frozen command. */ 1353 if (ch->frozen && !fsl_sata_check_collision(ch, ch->frozen)) { 1354 union ccb *fccb = ch->frozen; 1355 ch->frozen = NULL; 1356 fsl_sata_begin_transaction(ch, fccb); 1357 xpt_release_simq(ch->sim, TRUE); 1358 } 1359 /* Start PM timer. */ 1360 if (ch->numrslots == 0 && ch->pm_level > 3 && 1361 (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) { 1362 callout_schedule(&ch->pm_timer, 1363 (ch->pm_level == 4) ? hz / 1000 : hz / 8); 1364 } 1365 } 1366 1367 static void 1368 fsl_sata_issue_recovery(struct fsl_sata_channel *ch) 1369 { 1370 union ccb *ccb; 1371 struct ccb_ataio *ataio; 1372 struct ccb_scsiio *csio; 1373 int i; 1374 1375 /* Find some held command. */ 1376 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1377 if (ch->hold[i]) 1378 break; 1379 } 1380 ccb = xpt_alloc_ccb_nowait(); 1381 if (ccb == NULL) { 1382 device_printf(ch->dev, "Unable to allocate recovery command\n"); 1383 completeall: 1384 /* We can't do anything -- complete held commands. */ 1385 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1386 if (ch->hold[i] == NULL) 1387 continue; 1388 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 1389 ch->hold[i]->ccb_h.status |= CAM_RESRC_UNAVAIL; 1390 fsl_sata_done(ch, ch->hold[i]); 1391 ch->hold[i] = NULL; 1392 ch->numhslots--; 1393 } 1394 fsl_sata_reset(ch); 1395 return; 1396 } 1397 ccb->ccb_h = ch->hold[i]->ccb_h; /* Reuse old header. */ 1398 if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1399 /* READ LOG */ 1400 ccb->ccb_h.recovery_type = RECOVERY_READ_LOG; 1401 ccb->ccb_h.func_code = XPT_ATA_IO; 1402 ccb->ccb_h.flags = CAM_DIR_IN; 1403 ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 1404 ataio = &ccb->ataio; 1405 ataio->data_ptr = malloc(512, M_FSL_SATA, M_NOWAIT); 1406 if (ataio->data_ptr == NULL) { 1407 xpt_free_ccb(ccb); 1408 device_printf(ch->dev, 1409 "Unable to allocate memory for READ LOG command\n"); 1410 goto completeall; 1411 } 1412 ataio->dxfer_len = 512; 1413 bzero(&ataio->cmd, sizeof(ataio->cmd)); 1414 ataio->cmd.flags = CAM_ATAIO_48BIT; 1415 ataio->cmd.command = 0x2F; /* READ LOG EXT */ 1416 ataio->cmd.sector_count = 1; 1417 ataio->cmd.sector_count_exp = 0; 1418 ataio->cmd.lba_low = 0x10; 1419 ataio->cmd.lba_mid = 0; 1420 ataio->cmd.lba_mid_exp = 0; 1421 } else { 1422 /* REQUEST SENSE */ 1423 ccb->ccb_h.recovery_type = RECOVERY_REQUEST_SENSE; 1424 ccb->ccb_h.recovery_slot = i; 1425 ccb->ccb_h.func_code = XPT_SCSI_IO; 1426 ccb->ccb_h.flags = CAM_DIR_IN; 1427 ccb->ccb_h.status = 0; 1428 ccb->ccb_h.timeout = 1000; /* 1s should be enough. */ 1429 csio = &ccb->csio; 1430 csio->data_ptr = (void *)&ch->hold[i]->csio.sense_data; 1431 csio->dxfer_len = ch->hold[i]->csio.sense_len; 1432 csio->cdb_len = 6; 1433 bzero(&csio->cdb_io, sizeof(csio->cdb_io)); 1434 csio->cdb_io.cdb_bytes[0] = 0x03; 1435 csio->cdb_io.cdb_bytes[4] = csio->dxfer_len; 1436 } 1437 /* Freeze SIM while doing recovery. */ 1438 ch->recoverycmd = 1; 1439 xpt_freeze_simq(ch->sim, 1); 1440 fsl_sata_begin_transaction(ch, ccb); 1441 } 1442 1443 static void 1444 fsl_sata_process_read_log(struct fsl_sata_channel *ch, union ccb *ccb) 1445 { 1446 uint8_t *data; 1447 struct ata_res *res; 1448 int i; 1449 1450 ch->recoverycmd = 0; 1451 1452 data = ccb->ataio.data_ptr; 1453 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP && 1454 (data[0] & 0x80) == 0) { 1455 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1456 if (!ch->hold[i]) 1457 continue; 1458 if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO) 1459 continue; 1460 if ((data[0] & 0x1F) == i) { 1461 res = &ch->hold[i]->ataio.res; 1462 res->status = data[2]; 1463 res->error = data[3]; 1464 res->lba_low = data[4]; 1465 res->lba_mid = data[5]; 1466 res->lba_high = data[6]; 1467 res->device = data[7]; 1468 res->lba_low_exp = data[8]; 1469 res->lba_mid_exp = data[9]; 1470 res->lba_high_exp = data[10]; 1471 res->sector_count = data[12]; 1472 res->sector_count_exp = data[13]; 1473 } else { 1474 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 1475 ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ; 1476 } 1477 fsl_sata_done(ch, ch->hold[i]); 1478 ch->hold[i] = NULL; 1479 ch->numhslots--; 1480 } 1481 } else { 1482 if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) 1483 device_printf(ch->dev, "Error while READ LOG EXT\n"); 1484 else if ((data[0] & 0x80) == 0) { 1485 device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n"); 1486 } 1487 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1488 if (!ch->hold[i]) 1489 continue; 1490 if (ch->hold[i]->ccb_h.func_code != XPT_ATA_IO) 1491 continue; 1492 fsl_sata_done(ch, ch->hold[i]); 1493 ch->hold[i] = NULL; 1494 ch->numhslots--; 1495 } 1496 } 1497 free(ccb->ataio.data_ptr, M_FSL_SATA); 1498 xpt_free_ccb(ccb); 1499 xpt_release_simq(ch->sim, TRUE); 1500 } 1501 1502 static void 1503 fsl_sata_process_request_sense(struct fsl_sata_channel *ch, union ccb *ccb) 1504 { 1505 int i; 1506 1507 ch->recoverycmd = 0; 1508 1509 i = ccb->ccb_h.recovery_slot; 1510 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) { 1511 ch->hold[i]->ccb_h.status |= CAM_AUTOSNS_VALID; 1512 } else { 1513 ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK; 1514 ch->hold[i]->ccb_h.status |= CAM_AUTOSENSE_FAIL; 1515 } 1516 fsl_sata_done(ch, ch->hold[i]); 1517 ch->hold[i] = NULL; 1518 ch->numhslots--; 1519 xpt_free_ccb(ccb); 1520 xpt_release_simq(ch->sim, TRUE); 1521 } 1522 1523 static void 1524 fsl_sata_start(struct fsl_sata_channel *ch) 1525 { 1526 u_int32_t cmd; 1527 1528 /* Clear SATA error register */ 1529 ATA_OUTL(ch->r_mem, FSL_SATA_P_SERR, 0xFFFFFFFF); 1530 /* Clear any interrupts pending on this channel */ 1531 ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, 0x3F); 1532 ATA_OUTL(ch->r_mem, FSL_SATA_P_CER, 0xFFFF); 1533 ATA_OUTL(ch->r_mem, FSL_SATA_P_DER, 0xFFFF); 1534 /* Start operations on this channel */ 1535 cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL); 1536 cmd |= FSL_SATA_P_HCTRL_HC_ON | FSL_SATA_P_HCTRL_SNOOP; 1537 cmd &= ~FSL_SATA_P_HCTRL_HC_FORCE_OFF; 1538 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd | 1539 (ch->pm_present ? FSL_SATA_P_HCTRL_PM : 0)); 1540 fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 1541 FSL_SATA_P_HSTS_PR, FSL_SATA_P_HSTS_PR, 500); 1542 ATA_OUTL(ch->r_mem, FSL_SATA_P_HSTS, 1543 ATA_INL(ch->r_mem, FSL_SATA_P_HSTS) & FSL_SATA_P_HSTS_PR); 1544 } 1545 1546 static void 1547 fsl_sata_stop(struct fsl_sata_channel *ch) 1548 { 1549 uint32_t cmd; 1550 int i; 1551 1552 /* Kill all activity on this channel */ 1553 cmd = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL); 1554 cmd &= ~FSL_SATA_P_HCTRL_HC_ON; 1555 1556 for (i = 0; i < 2; i++) { 1557 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, cmd); 1558 if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 1559 FSL_SATA_P_HSTS_HS_ON, 0, 500)) { 1560 if (i != 0) 1561 device_printf(ch->dev, 1562 "stopping FSL SATA engine failed\n"); 1563 cmd |= FSL_SATA_P_HCTRL_HC_FORCE_OFF; 1564 } else 1565 break; 1566 } 1567 ch->eslots = 0; 1568 } 1569 1570 static void 1571 fsl_sata_reset(struct fsl_sata_channel *ch) 1572 { 1573 uint32_t ctrl; 1574 int i; 1575 1576 xpt_freeze_simq(ch->sim, 1); 1577 if (bootverbose) 1578 device_printf(ch->dev, "FSL SATA reset...\n"); 1579 1580 /* Requeue freezed command. */ 1581 if (ch->frozen) { 1582 union ccb *fccb = ch->frozen; 1583 ch->frozen = NULL; 1584 fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ; 1585 if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) { 1586 xpt_freeze_devq(fccb->ccb_h.path, 1); 1587 fccb->ccb_h.status |= CAM_DEV_QFRZN; 1588 } 1589 fsl_sata_done(ch, fccb); 1590 } 1591 /* Kill the engine and requeue all running commands. */ 1592 fsl_sata_stop(ch); 1593 DELAY(1000); /* sleep for 1ms */ 1594 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1595 /* Do we have a running request on slot? */ 1596 if (ch->slot[i].state < FSL_SATA_SLOT_RUNNING) 1597 continue; 1598 /* XXX; Commands in loading state. */ 1599 fsl_sata_end_transaction(&ch->slot[i], FSL_SATA_ERR_INNOCENT); 1600 } 1601 for (i = 0; i < FSL_SATA_MAX_SLOTS; i++) { 1602 if (!ch->hold[i]) 1603 continue; 1604 fsl_sata_done(ch, ch->hold[i]); 1605 ch->hold[i] = NULL; 1606 ch->numhslots--; 1607 } 1608 if (ch->toslots != 0) 1609 xpt_release_simq(ch->sim, TRUE); 1610 ch->eslots = 0; 1611 ch->toslots = 0; 1612 ch->fatalerr = 0; 1613 /* Tell the XPT about the event */ 1614 xpt_async(AC_BUS_RESET, ch->path, NULL); 1615 /* Disable port interrupts */ 1616 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, 1617 ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f); 1618 /* Reset and reconnect PHY, */ 1619 fsl_sata_start(ch); 1620 if (fsl_sata_wait_register(ch, FSL_SATA_P_HSTS, 0x08, 0x08, 500)) { 1621 if (bootverbose) 1622 device_printf(ch->dev, 1623 "FSL SATA reset: device not found\n"); 1624 ch->devices = 0; 1625 /* Enable wanted port interrupts */ 1626 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, 1627 ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) | FSL_SATA_P_HCTRL_PHYRDY); 1628 xpt_release_simq(ch->sim, TRUE); 1629 return; 1630 } 1631 if (bootverbose) 1632 device_printf(ch->dev, "FSL SATA reset: device found\n"); 1633 ch->devices = 1; 1634 /* Enable wanted port interrupts */ 1635 ctrl = ATA_INL(ch->r_mem, FSL_SATA_P_HCTRL) & ~0x3f; 1636 ATA_OUTL(ch->r_mem, FSL_SATA_P_HCTRL, 1637 ctrl | FSL_SATA_P_HCTRL_FATAL | FSL_SATA_P_HCTRL_PHYRDY | 1638 FSL_SATA_P_HCTRL_SIG | FSL_SATA_P_HCTRL_SNTFY | 1639 FSL_SATA_P_HCTRL_DE | FSL_SATA_P_HCTRL_CC); 1640 xpt_release_simq(ch->sim, TRUE); 1641 } 1642 1643 static int 1644 fsl_sata_setup_fis(struct fsl_sata_channel *ch, struct fsl_sata_cmd_tab *ctp, union ccb *ccb, int tag) 1645 { 1646 uint8_t *fis = &ctp->cfis[0]; 1647 1648 bzero(fis, 32); 1649 fis[0] = 0x27; /* host to device */ 1650 fis[1] = (ccb->ccb_h.target_id & 0x0f); 1651 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 1652 fis[1] |= 0x80; 1653 fis[2] = ATA_PACKET_CMD; 1654 if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE && 1655 ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA) 1656 fis[3] = ATA_F_DMA; 1657 else { 1658 fis[5] = ccb->csio.dxfer_len; 1659 fis[6] = ccb->csio.dxfer_len >> 8; 1660 } 1661 fis[7] = ATA_D_LBA; 1662 fis[15] = ATA_A_4BIT; 1663 bcopy((ccb->ccb_h.flags & CAM_CDB_POINTER) ? 1664 ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes, 1665 ctp->acmd, ccb->csio.cdb_len); 1666 bzero(ctp->acmd + ccb->csio.cdb_len, 32 - ccb->csio.cdb_len); 1667 } else if ((ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL) == 0) { 1668 fis[1] |= 0x80; 1669 fis[2] = ccb->ataio.cmd.command; 1670 fis[3] = ccb->ataio.cmd.features; 1671 fis[4] = ccb->ataio.cmd.lba_low; 1672 fis[5] = ccb->ataio.cmd.lba_mid; 1673 fis[6] = ccb->ataio.cmd.lba_high; 1674 fis[7] = ccb->ataio.cmd.device; 1675 fis[8] = ccb->ataio.cmd.lba_low_exp; 1676 fis[9] = ccb->ataio.cmd.lba_mid_exp; 1677 fis[10] = ccb->ataio.cmd.lba_high_exp; 1678 fis[11] = ccb->ataio.cmd.features_exp; 1679 if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) { 1680 fis[12] = tag << 3; 1681 fis[13] = 0; 1682 } else { 1683 fis[12] = ccb->ataio.cmd.sector_count; 1684 fis[13] = ccb->ataio.cmd.sector_count_exp; 1685 } 1686 fis[15] = ATA_A_4BIT; 1687 } else { 1688 fis[15] = ccb->ataio.cmd.control; 1689 } 1690 return (20); 1691 } 1692 1693 static int 1694 fsl_sata_check_ids(struct fsl_sata_channel *ch, union ccb *ccb) 1695 { 1696 1697 if (ccb->ccb_h.target_id > 15) { 1698 ccb->ccb_h.status = CAM_TID_INVALID; 1699 fsl_sata_done(ch, ccb); 1700 return (-1); 1701 } 1702 if (ccb->ccb_h.target_lun != 0) { 1703 ccb->ccb_h.status = CAM_LUN_INVALID; 1704 fsl_sata_done(ch, ccb); 1705 return (-1); 1706 } 1707 return (0); 1708 } 1709 1710 static void 1711 fsl_sataaction(struct cam_sim *sim, union ccb *ccb) 1712 { 1713 struct fsl_sata_channel *ch; 1714 1715 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 1716 ("fsl_sataaction func_code=0x%x\n", ccb->ccb_h.func_code)); 1717 1718 ch = (struct fsl_sata_channel *)cam_sim_softc(sim); 1719 switch (ccb->ccb_h.func_code) { 1720 /* Common cases first */ 1721 case XPT_ATA_IO: /* Execute the requested I/O operation */ 1722 case XPT_SCSI_IO: 1723 if (fsl_sata_check_ids(ch, ccb)) 1724 return; 1725 if (ch->devices == 0 || 1726 (ch->pm_present == 0 && 1727 ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) { 1728 ccb->ccb_h.status = CAM_SEL_TIMEOUT; 1729 break; 1730 } 1731 ccb->ccb_h.recovery_type = RECOVERY_NONE; 1732 /* Check for command collision. */ 1733 if (fsl_sata_check_collision(ch, ccb)) { 1734 /* Freeze command. */ 1735 ch->frozen = ccb; 1736 /* We have only one frozen slot, so freeze simq also. */ 1737 xpt_freeze_simq(ch->sim, 1); 1738 return; 1739 } 1740 fsl_sata_begin_transaction(ch, ccb); 1741 return; 1742 case XPT_ABORT: /* Abort the specified CCB */ 1743 /* XXX Implement */ 1744 ccb->ccb_h.status = CAM_REQ_INVALID; 1745 break; 1746 case XPT_SET_TRAN_SETTINGS: 1747 { 1748 struct ccb_trans_settings *cts = &ccb->cts; 1749 struct fsl_sata_device *d; 1750 1751 if (fsl_sata_check_ids(ch, ccb)) 1752 return; 1753 if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 1754 d = &ch->curr[ccb->ccb_h.target_id]; 1755 else 1756 d = &ch->user[ccb->ccb_h.target_id]; 1757 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION) 1758 d->revision = cts->xport_specific.sata.revision; 1759 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE) 1760 d->mode = cts->xport_specific.sata.mode; 1761 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) 1762 d->bytecount = min(8192, cts->xport_specific.sata.bytecount); 1763 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS) 1764 d->tags = min(FSL_SATA_MAX_SLOTS, cts->xport_specific.sata.tags); 1765 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM) 1766 ch->pm_present = cts->xport_specific.sata.pm_present; 1767 if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI) 1768 d->atapi = cts->xport_specific.sata.atapi; 1769 ccb->ccb_h.status = CAM_REQ_CMP; 1770 break; 1771 } 1772 case XPT_GET_TRAN_SETTINGS: 1773 /* Get default/user set transfer settings for the target */ 1774 { 1775 struct ccb_trans_settings *cts = &ccb->cts; 1776 struct fsl_sata_device *d; 1777 uint32_t status; 1778 1779 if (fsl_sata_check_ids(ch, ccb)) 1780 return; 1781 if (cts->type == CTS_TYPE_CURRENT_SETTINGS) 1782 d = &ch->curr[ccb->ccb_h.target_id]; 1783 else 1784 d = &ch->user[ccb->ccb_h.target_id]; 1785 cts->protocol = PROTO_UNSPECIFIED; 1786 cts->protocol_version = PROTO_VERSION_UNSPECIFIED; 1787 cts->transport = XPORT_SATA; 1788 cts->transport_version = XPORT_VERSION_UNSPECIFIED; 1789 cts->proto_specific.valid = 0; 1790 cts->xport_specific.sata.valid = 0; 1791 if (cts->type == CTS_TYPE_CURRENT_SETTINGS && 1792 (ccb->ccb_h.target_id == 15 || 1793 (ccb->ccb_h.target_id == 0 && !ch->pm_present))) { 1794 status = ATA_INL(ch->r_mem, FSL_SATA_P_SSTS) & ATA_SS_SPD_MASK; 1795 if (status & 0x0f0) { 1796 cts->xport_specific.sata.revision = 1797 (status & 0x0f0) >> 4; 1798 cts->xport_specific.sata.valid |= 1799 CTS_SATA_VALID_REVISION; 1800 } 1801 cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D; 1802 if (ch->pm_level) { 1803 cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ; 1804 } 1805 cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_AN; 1806 cts->xport_specific.sata.caps &= 1807 ch->user[ccb->ccb_h.target_id].caps; 1808 cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 1809 } else { 1810 cts->xport_specific.sata.revision = d->revision; 1811 cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION; 1812 cts->xport_specific.sata.caps = d->caps; 1813 cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS; 1814 } 1815 cts->xport_specific.sata.mode = d->mode; 1816 cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE; 1817 cts->xport_specific.sata.bytecount = d->bytecount; 1818 cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT; 1819 cts->xport_specific.sata.pm_present = ch->pm_present; 1820 cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM; 1821 cts->xport_specific.sata.tags = d->tags; 1822 cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS; 1823 cts->xport_specific.sata.atapi = d->atapi; 1824 cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI; 1825 ccb->ccb_h.status = CAM_REQ_CMP; 1826 break; 1827 } 1828 case XPT_RESET_BUS: /* Reset the specified SCSI bus */ 1829 case XPT_RESET_DEV: /* Bus Device Reset the specified SCSI device */ 1830 fsl_sata_reset(ch); 1831 ccb->ccb_h.status = CAM_REQ_CMP; 1832 break; 1833 case XPT_TERM_IO: /* Terminate the I/O process */ 1834 /* XXX Implement */ 1835 ccb->ccb_h.status = CAM_REQ_INVALID; 1836 break; 1837 case XPT_PATH_INQ: /* Path routing inquiry */ 1838 { 1839 struct ccb_pathinq *cpi = &ccb->cpi; 1840 1841 cpi->version_num = 1; /* XXX??? */ 1842 cpi->hba_inquiry = PI_SDTR_ABLE; 1843 cpi->hba_inquiry |= PI_TAG_ABLE; 1844 #if 0 1845 /* 1846 * XXX: CAM tries to reset port 15 if it sees port multiplier 1847 * support. Disable it for now. 1848 */ 1849 cpi->hba_inquiry |= PI_SATAPM; 1850 #endif 1851 cpi->target_sprt = 0; 1852 cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED; 1853 cpi->hba_eng_cnt = 0; 1854 /* 1855 * XXX: This should be 15, since hardware *does* support a port 1856 * multiplier. See above. 1857 */ 1858 cpi->max_target = 0; 1859 cpi->max_lun = 0; 1860 cpi->initiator_id = 0; 1861 cpi->bus_id = cam_sim_bus(sim); 1862 cpi->base_transfer_speed = 150000; 1863 strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 1864 strncpy(cpi->hba_vid, "FSL SATA", HBA_IDLEN); 1865 strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 1866 cpi->unit_number = cam_sim_unit(sim); 1867 cpi->transport = XPORT_SATA; 1868 cpi->transport_version = XPORT_VERSION_UNSPECIFIED; 1869 cpi->protocol = PROTO_ATA; 1870 cpi->protocol_version = PROTO_VERSION_UNSPECIFIED; 1871 cpi->maxio = (FSL_SATA_SG_ENTRIES - 1) * PAGE_SIZE; 1872 cpi->ccb_h.status = CAM_REQ_CMP; 1873 break; 1874 } 1875 default: 1876 ccb->ccb_h.status = CAM_REQ_INVALID; 1877 break; 1878 } 1879 fsl_sata_done(ch, ccb); 1880 } 1881 1882 static void 1883 fsl_satapoll(struct cam_sim *sim) 1884 { 1885 struct fsl_sata_channel *ch = (struct fsl_sata_channel *)cam_sim_softc(sim); 1886 uint32_t istatus; 1887 1888 /* Read interrupt statuses and process if any. */ 1889 istatus = ATA_INL(ch->r_mem, FSL_SATA_P_HSTS); 1890 if (istatus != 0) 1891 fsl_sata_intr_main(ch, istatus); 1892 } 1893 MODULE_VERSION(fsl_sata, 1); 1894 MODULE_DEPEND(fsl_sata, cam, 1, 1, 1); 1895