1991554f2SKenneth D. Merry /*- 2991554f2SKenneth D. Merry * Copyright (c) 2009 Yahoo! Inc. 3a2c14879SStephen McConnell * Copyright (c) 2011-2015 LSI Corp. 47a2a6a1aSStephen McConnell * Copyright (c) 2013-2016 Avago Technologies 5991554f2SKenneth D. Merry * All rights reserved. 6991554f2SKenneth D. Merry * 7991554f2SKenneth D. Merry * Redistribution and use in source and binary forms, with or without 8991554f2SKenneth D. Merry * modification, are permitted provided that the following conditions 9991554f2SKenneth D. Merry * are met: 10991554f2SKenneth D. Merry * 1. Redistributions of source code must retain the above copyright 11991554f2SKenneth D. Merry * notice, this list of conditions and the following disclaimer. 12991554f2SKenneth D. Merry * 2. Redistributions in binary form must reproduce the above copyright 13991554f2SKenneth D. Merry * notice, this list of conditions and the following disclaimer in the 14991554f2SKenneth D. Merry * documentation and/or other materials provided with the distribution. 15991554f2SKenneth D. Merry * 16991554f2SKenneth D. Merry * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17991554f2SKenneth D. Merry * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18991554f2SKenneth D. Merry * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19991554f2SKenneth D. Merry * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20991554f2SKenneth D. Merry * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21991554f2SKenneth D. Merry * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22991554f2SKenneth D. Merry * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23991554f2SKenneth D. Merry * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24991554f2SKenneth D. Merry * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25991554f2SKenneth D. Merry * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26991554f2SKenneth D. Merry * SUCH DAMAGE. 27991554f2SKenneth D. Merry * 28a2c14879SStephen McConnell * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD 29a2c14879SStephen McConnell * 30991554f2SKenneth D. Merry */ 31991554f2SKenneth D. Merry 32991554f2SKenneth D. Merry #include <sys/cdefs.h> 33991554f2SKenneth D. Merry __FBSDID("$FreeBSD$"); 34991554f2SKenneth D. Merry 35a2c14879SStephen McConnell /* Communications core for Avago Technologies (LSI) MPT3 */ 36991554f2SKenneth D. Merry 37991554f2SKenneth D. Merry /* TODO Move headers to mprvar */ 38991554f2SKenneth D. Merry #include <sys/types.h> 39991554f2SKenneth D. Merry #include <sys/param.h> 40991554f2SKenneth D. Merry #include <sys/systm.h> 41991554f2SKenneth D. Merry #include <sys/kernel.h> 42991554f2SKenneth D. Merry #include <sys/selinfo.h> 43991554f2SKenneth D. Merry #include <sys/lock.h> 44991554f2SKenneth D. Merry #include <sys/mutex.h> 45991554f2SKenneth D. Merry #include <sys/module.h> 46991554f2SKenneth D. Merry #include <sys/bus.h> 47991554f2SKenneth D. Merry #include <sys/conf.h> 48991554f2SKenneth D. Merry #include <sys/bio.h> 49991554f2SKenneth D. Merry #include <sys/malloc.h> 50991554f2SKenneth D. Merry #include <sys/uio.h> 51991554f2SKenneth D. Merry #include <sys/sysctl.h> 52991554f2SKenneth D. Merry #include <sys/queue.h> 53991554f2SKenneth D. Merry #include <sys/kthread.h> 54991554f2SKenneth D. Merry #include <sys/taskqueue.h> 55991554f2SKenneth D. Merry #include <sys/endian.h> 56991554f2SKenneth D. Merry #include <sys/eventhandler.h> 57991554f2SKenneth D. Merry 58991554f2SKenneth D. Merry #include <machine/bus.h> 59991554f2SKenneth D. Merry #include <machine/resource.h> 60991554f2SKenneth D. Merry #include <sys/rman.h> 61991554f2SKenneth D. Merry #include <sys/proc.h> 62991554f2SKenneth D. Merry 63991554f2SKenneth D. Merry #include <dev/pci/pcivar.h> 64991554f2SKenneth D. Merry 65991554f2SKenneth D. Merry #include <cam/cam.h> 6667feec50SStephen McConnell #include <cam/cam_ccb.h> 67991554f2SKenneth D. Merry #include <cam/scsi/scsi_all.h> 68991554f2SKenneth D. Merry 69991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_type.h> 70991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2.h> 71991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_ioc.h> 72991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_sas.h> 7367feec50SStephen McConnell #include <dev/mpr/mpi/mpi2_pci.h> 74991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_cnfg.h> 75991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_init.h> 76991554f2SKenneth D. Merry #include <dev/mpr/mpi/mpi2_tool.h> 77991554f2SKenneth D. Merry #include <dev/mpr/mpr_ioctl.h> 78991554f2SKenneth D. Merry #include <dev/mpr/mprvar.h> 79991554f2SKenneth D. Merry #include <dev/mpr/mpr_table.h> 8067feec50SStephen McConnell #include <dev/mpr/mpr_sas.h> 81991554f2SKenneth D. Merry 82991554f2SKenneth D. Merry static int mpr_diag_reset(struct mpr_softc *sc, int sleep_flag); 83991554f2SKenneth D. Merry static int mpr_init_queues(struct mpr_softc *sc); 84991554f2SKenneth D. Merry static int mpr_message_unit_reset(struct mpr_softc *sc, int sleep_flag); 85991554f2SKenneth D. Merry static int mpr_transition_operational(struct mpr_softc *sc); 86991554f2SKenneth D. Merry static int mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching); 87991554f2SKenneth D. Merry static void mpr_iocfacts_free(struct mpr_softc *sc); 88991554f2SKenneth D. Merry static void mpr_startup(void *arg); 89991554f2SKenneth D. Merry static int mpr_send_iocinit(struct mpr_softc *sc); 90991554f2SKenneth D. Merry static int mpr_alloc_queues(struct mpr_softc *sc); 91991554f2SKenneth D. Merry static int mpr_alloc_replies(struct mpr_softc *sc); 92991554f2SKenneth D. Merry static int mpr_alloc_requests(struct mpr_softc *sc); 9367feec50SStephen McConnell static int mpr_alloc_nvme_prp_pages(struct mpr_softc *sc); 94991554f2SKenneth D. Merry static int mpr_attach_log(struct mpr_softc *sc); 95991554f2SKenneth D. Merry static __inline void mpr_complete_command(struct mpr_softc *sc, 96991554f2SKenneth D. Merry struct mpr_command *cm); 97991554f2SKenneth D. Merry static void mpr_dispatch_event(struct mpr_softc *sc, uintptr_t data, 98991554f2SKenneth D. Merry MPI2_EVENT_NOTIFICATION_REPLY *reply); 997a2a6a1aSStephen McConnell static void mpr_config_complete(struct mpr_softc *sc, struct mpr_command *cm); 100991554f2SKenneth D. Merry static void mpr_periodic(void *); 101991554f2SKenneth D. Merry static int mpr_reregister_events(struct mpr_softc *sc); 1027a2a6a1aSStephen McConnell static void mpr_enqueue_request(struct mpr_softc *sc, struct mpr_command *cm); 1037a2a6a1aSStephen McConnell static int mpr_get_iocfacts(struct mpr_softc *sc, MPI2_IOC_FACTS_REPLY *facts); 104991554f2SKenneth D. Merry static int mpr_wait_db_ack(struct mpr_softc *sc, int timeout, int sleep_flag); 105991554f2SKenneth D. Merry SYSCTL_NODE(_hw, OID_AUTO, mpr, CTLFLAG_RD, 0, "MPR Driver Parameters"); 106991554f2SKenneth D. Merry 107991554f2SKenneth D. Merry MALLOC_DEFINE(M_MPR, "mpr", "mpr driver memory"); 108991554f2SKenneth D. Merry 109991554f2SKenneth D. Merry /* 110991554f2SKenneth D. Merry * Do a "Diagnostic Reset" aka a hard reset. This should get the chip out of 111991554f2SKenneth D. Merry * any state and back to its initialization state machine. 112991554f2SKenneth D. Merry */ 113991554f2SKenneth D. Merry static char mpt2_reset_magic[] = { 0x00, 0x0f, 0x04, 0x0b, 0x02, 0x07, 0x0d }; 114991554f2SKenneth D. Merry 115991554f2SKenneth D. Merry /* 116991554f2SKenneth D. Merry * Added this union to smoothly convert le64toh cm->cm_desc.Words. 11767feec50SStephen McConnell * Compiler only supports uint64_t to be passed as an argument. 118991554f2SKenneth D. Merry * Otherwise it will through this error: 119991554f2SKenneth D. Merry * "aggregate value used where an integer was expected" 120991554f2SKenneth D. Merry */ 121991554f2SKenneth D. Merry typedef union _reply_descriptor { 122991554f2SKenneth D. Merry u64 word; 123991554f2SKenneth D. Merry struct { 124991554f2SKenneth D. Merry u32 low; 125991554f2SKenneth D. Merry u32 high; 126991554f2SKenneth D. Merry } u; 12767feec50SStephen McConnell } reply_descriptor, request_descriptor; 128991554f2SKenneth D. Merry 129991554f2SKenneth D. Merry /* Rate limit chain-fail messages to 1 per minute */ 130991554f2SKenneth D. Merry static struct timeval mpr_chainfail_interval = { 60, 0 }; 131991554f2SKenneth D. Merry 132991554f2SKenneth D. Merry /* 133991554f2SKenneth D. Merry * sleep_flag can be either CAN_SLEEP or NO_SLEEP. 134991554f2SKenneth D. Merry * If this function is called from process context, it can sleep 135991554f2SKenneth D. Merry * and there is no harm to sleep, in case if this fuction is called 136991554f2SKenneth D. Merry * from Interrupt handler, we can not sleep and need NO_SLEEP flag set. 137991554f2SKenneth D. Merry * based on sleep flags driver will call either msleep, pause or DELAY. 138991554f2SKenneth D. Merry * msleep and pause are of same variant, but pause is used when mpr_mtx 139991554f2SKenneth D. Merry * is not hold by driver. 140991554f2SKenneth D. Merry */ 141991554f2SKenneth D. Merry static int 142991554f2SKenneth D. Merry mpr_diag_reset(struct mpr_softc *sc,int sleep_flag) 143991554f2SKenneth D. Merry { 144991554f2SKenneth D. Merry uint32_t reg; 145991554f2SKenneth D. Merry int i, error, tries = 0; 146991554f2SKenneth D. Merry uint8_t first_wait_done = FALSE; 147991554f2SKenneth D. Merry 148991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 149991554f2SKenneth D. Merry 150991554f2SKenneth D. Merry /* Clear any pending interrupts */ 151991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 152991554f2SKenneth D. Merry 153991554f2SKenneth D. Merry /* 154991554f2SKenneth D. Merry * Force NO_SLEEP for threads prohibited to sleep 155991554f2SKenneth D. Merry * e.a Thread from interrupt handler are prohibited to sleep. 156991554f2SKenneth D. Merry */ 157991554f2SKenneth D. Merry #if __FreeBSD_version >= 1000029 158991554f2SKenneth D. Merry if (curthread->td_no_sleeping) 159991554f2SKenneth D. Merry #else //__FreeBSD_version < 1000029 160991554f2SKenneth D. Merry if (curthread->td_pflags & TDP_NOSLEEPING) 161991554f2SKenneth D. Merry #endif //__FreeBSD_version >= 1000029 162991554f2SKenneth D. Merry sleep_flag = NO_SLEEP; 163991554f2SKenneth D. Merry 164991554f2SKenneth D. Merry /* Push the magic sequence */ 165991554f2SKenneth D. Merry error = ETIMEDOUT; 166991554f2SKenneth D. Merry while (tries++ < 20) { 167991554f2SKenneth D. Merry for (i = 0; i < sizeof(mpt2_reset_magic); i++) 168991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET, 169991554f2SKenneth D. Merry mpt2_reset_magic[i]); 170991554f2SKenneth D. Merry 171991554f2SKenneth D. Merry /* wait 100 msec */ 172991554f2SKenneth D. Merry if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP) 173991554f2SKenneth D. Merry msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0, 174991554f2SKenneth D. Merry "mprdiag", hz/10); 175991554f2SKenneth D. Merry else if (sleep_flag == CAN_SLEEP) 176991554f2SKenneth D. Merry pause("mprdiag", hz/10); 177991554f2SKenneth D. Merry else 178991554f2SKenneth D. Merry DELAY(100 * 1000); 179991554f2SKenneth D. Merry 180991554f2SKenneth D. Merry reg = mpr_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET); 181991554f2SKenneth D. Merry if (reg & MPI2_DIAG_DIAG_WRITE_ENABLE) { 182991554f2SKenneth D. Merry error = 0; 183991554f2SKenneth D. Merry break; 184991554f2SKenneth D. Merry } 185991554f2SKenneth D. Merry } 186991554f2SKenneth D. Merry if (error) 187991554f2SKenneth D. Merry return (error); 188991554f2SKenneth D. Merry 189991554f2SKenneth D. Merry /* Send the actual reset. XXX need to refresh the reg? */ 190991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_DIAGNOSTIC_OFFSET, 191991554f2SKenneth D. Merry reg | MPI2_DIAG_RESET_ADAPTER); 192991554f2SKenneth D. Merry 193991554f2SKenneth D. Merry /* Wait up to 300 seconds in 50ms intervals */ 194991554f2SKenneth D. Merry error = ETIMEDOUT; 195991554f2SKenneth D. Merry for (i = 0; i < 6000; i++) { 196991554f2SKenneth D. Merry /* 197991554f2SKenneth D. Merry * Wait 50 msec. If this is the first time through, wait 256 198991554f2SKenneth D. Merry * msec to satisfy Diag Reset timing requirements. 199991554f2SKenneth D. Merry */ 200991554f2SKenneth D. Merry if (first_wait_done) { 201991554f2SKenneth D. Merry if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP) 202991554f2SKenneth D. Merry msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0, 203991554f2SKenneth D. Merry "mprdiag", hz/20); 204991554f2SKenneth D. Merry else if (sleep_flag == CAN_SLEEP) 205991554f2SKenneth D. Merry pause("mprdiag", hz/20); 206991554f2SKenneth D. Merry else 207991554f2SKenneth D. Merry DELAY(50 * 1000); 208991554f2SKenneth D. Merry } else { 209991554f2SKenneth D. Merry DELAY(256 * 1000); 210991554f2SKenneth D. Merry first_wait_done = TRUE; 211991554f2SKenneth D. Merry } 212991554f2SKenneth D. Merry /* 213991554f2SKenneth D. Merry * Check for the RESET_ADAPTER bit to be cleared first, then 214991554f2SKenneth D. Merry * wait for the RESET state to be cleared, which takes a little 215991554f2SKenneth D. Merry * longer. 216991554f2SKenneth D. Merry */ 217991554f2SKenneth D. Merry reg = mpr_regread(sc, MPI2_HOST_DIAGNOSTIC_OFFSET); 218991554f2SKenneth D. Merry if (reg & MPI2_DIAG_RESET_ADAPTER) { 219991554f2SKenneth D. Merry continue; 220991554f2SKenneth D. Merry } 221991554f2SKenneth D. Merry reg = mpr_regread(sc, MPI2_DOORBELL_OFFSET); 222991554f2SKenneth D. Merry if ((reg & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_RESET) { 223991554f2SKenneth D. Merry error = 0; 224991554f2SKenneth D. Merry break; 225991554f2SKenneth D. Merry } 226991554f2SKenneth D. Merry } 227991554f2SKenneth D. Merry if (error) 228991554f2SKenneth D. Merry return (error); 229991554f2SKenneth D. Merry 230991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_WRITE_SEQUENCE_OFFSET, 0x0); 231991554f2SKenneth D. Merry 232991554f2SKenneth D. Merry return (0); 233991554f2SKenneth D. Merry } 234991554f2SKenneth D. Merry 235991554f2SKenneth D. Merry static int 236991554f2SKenneth D. Merry mpr_message_unit_reset(struct mpr_softc *sc, int sleep_flag) 237991554f2SKenneth D. Merry { 238991554f2SKenneth D. Merry 239991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 240991554f2SKenneth D. Merry 241991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_DOORBELL_OFFSET, 242991554f2SKenneth D. Merry MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET << 243991554f2SKenneth D. Merry MPI2_DOORBELL_FUNCTION_SHIFT); 244991554f2SKenneth D. Merry 245991554f2SKenneth D. Merry if (mpr_wait_db_ack(sc, 5, sleep_flag) != 0) { 246991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Doorbell handshake failed : <%s>\n", 247991554f2SKenneth D. Merry __func__); 248991554f2SKenneth D. Merry return (ETIMEDOUT); 249991554f2SKenneth D. Merry } 250991554f2SKenneth D. Merry 251991554f2SKenneth D. Merry return (0); 252991554f2SKenneth D. Merry } 253991554f2SKenneth D. Merry 254991554f2SKenneth D. Merry static int 255991554f2SKenneth D. Merry mpr_transition_ready(struct mpr_softc *sc) 256991554f2SKenneth D. Merry { 257991554f2SKenneth D. Merry uint32_t reg, state; 258991554f2SKenneth D. Merry int error, tries = 0; 259991554f2SKenneth D. Merry int sleep_flags; 260991554f2SKenneth D. Merry 261991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 262991554f2SKenneth D. Merry /* If we are in attach call, do not sleep */ 263991554f2SKenneth D. Merry sleep_flags = (sc->mpr_flags & MPR_FLAGS_ATTACH_DONE) 264991554f2SKenneth D. Merry ? CAN_SLEEP : NO_SLEEP; 265991554f2SKenneth D. Merry 266991554f2SKenneth D. Merry error = 0; 267991554f2SKenneth D. Merry while (tries++ < 1200) { 268991554f2SKenneth D. Merry reg = mpr_regread(sc, MPI2_DOORBELL_OFFSET); 269991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INIT, "Doorbell= 0x%x\n", reg); 270991554f2SKenneth D. Merry 271991554f2SKenneth D. Merry /* 272991554f2SKenneth D. Merry * Ensure the IOC is ready to talk. If it's not, try 273991554f2SKenneth D. Merry * resetting it. 274991554f2SKenneth D. Merry */ 275991554f2SKenneth D. Merry if (reg & MPI2_DOORBELL_USED) { 276991554f2SKenneth D. Merry mpr_diag_reset(sc, sleep_flags); 277991554f2SKenneth D. Merry DELAY(50000); 278991554f2SKenneth D. Merry continue; 279991554f2SKenneth D. Merry } 280991554f2SKenneth D. Merry 281991554f2SKenneth D. Merry /* Is the adapter owned by another peer? */ 282991554f2SKenneth D. Merry if ((reg & MPI2_DOORBELL_WHO_INIT_MASK) == 283991554f2SKenneth D. Merry (MPI2_WHOINIT_PCI_PEER << MPI2_DOORBELL_WHO_INIT_SHIFT)) { 284991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "IOC is under the control " 285991554f2SKenneth D. Merry "of another peer host, aborting initialization.\n"); 286991554f2SKenneth D. Merry return (ENXIO); 287991554f2SKenneth D. Merry } 288991554f2SKenneth D. Merry 289991554f2SKenneth D. Merry state = reg & MPI2_IOC_STATE_MASK; 290991554f2SKenneth D. Merry if (state == MPI2_IOC_STATE_READY) { 291991554f2SKenneth D. Merry /* Ready to go! */ 292991554f2SKenneth D. Merry error = 0; 293991554f2SKenneth D. Merry break; 294991554f2SKenneth D. Merry } else if (state == MPI2_IOC_STATE_FAULT) { 295991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "IOC in fault state 0x%x\n", 296991554f2SKenneth D. Merry state & MPI2_DOORBELL_FAULT_CODE_MASK); 297991554f2SKenneth D. Merry mpr_diag_reset(sc, sleep_flags); 298991554f2SKenneth D. Merry } else if (state == MPI2_IOC_STATE_OPERATIONAL) { 299991554f2SKenneth D. Merry /* Need to take ownership */ 300991554f2SKenneth D. Merry mpr_message_unit_reset(sc, sleep_flags); 301991554f2SKenneth D. Merry } else if (state == MPI2_IOC_STATE_RESET) { 302991554f2SKenneth D. Merry /* Wait a bit, IOC might be in transition */ 303991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 304991554f2SKenneth D. Merry "IOC in unexpected reset state\n"); 305991554f2SKenneth D. Merry } else { 306991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 307991554f2SKenneth D. Merry "IOC in unknown state 0x%x\n", state); 308991554f2SKenneth D. Merry error = EINVAL; 309991554f2SKenneth D. Merry break; 310991554f2SKenneth D. Merry } 311991554f2SKenneth D. Merry 312991554f2SKenneth D. Merry /* Wait 50ms for things to settle down. */ 313991554f2SKenneth D. Merry DELAY(50000); 314991554f2SKenneth D. Merry } 315991554f2SKenneth D. Merry 316991554f2SKenneth D. Merry if (error) 317991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot transition IOC to ready\n"); 318991554f2SKenneth D. Merry return (error); 319991554f2SKenneth D. Merry } 320991554f2SKenneth D. Merry 321991554f2SKenneth D. Merry static int 322991554f2SKenneth D. Merry mpr_transition_operational(struct mpr_softc *sc) 323991554f2SKenneth D. Merry { 324991554f2SKenneth D. Merry uint32_t reg, state; 325991554f2SKenneth D. Merry int error; 326991554f2SKenneth D. Merry 327991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 328991554f2SKenneth D. Merry 329991554f2SKenneth D. Merry error = 0; 330991554f2SKenneth D. Merry reg = mpr_regread(sc, MPI2_DOORBELL_OFFSET); 331991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INIT, "Doorbell= 0x%x\n", reg); 332991554f2SKenneth D. Merry 333991554f2SKenneth D. Merry state = reg & MPI2_IOC_STATE_MASK; 334991554f2SKenneth D. Merry if (state != MPI2_IOC_STATE_READY) { 335991554f2SKenneth D. Merry if ((error = mpr_transition_ready(sc)) != 0) { 336991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 337991554f2SKenneth D. Merry "%s failed to transition ready\n", __func__); 338991554f2SKenneth D. Merry return (error); 339991554f2SKenneth D. Merry } 340991554f2SKenneth D. Merry } 341991554f2SKenneth D. Merry 342991554f2SKenneth D. Merry error = mpr_send_iocinit(sc); 343991554f2SKenneth D. Merry return (error); 344991554f2SKenneth D. Merry } 345991554f2SKenneth D. Merry 346991554f2SKenneth D. Merry /* 347991554f2SKenneth D. Merry * This is called during attach and when re-initializing due to a Diag Reset. 348991554f2SKenneth D. Merry * IOC Facts is used to allocate many of the structures needed by the driver. 349991554f2SKenneth D. Merry * If called from attach, de-allocation is not required because the driver has 350991554f2SKenneth D. Merry * not allocated any structures yet, but if called from a Diag Reset, previously 351991554f2SKenneth D. Merry * allocated structures based on IOC Facts will need to be freed and re- 352991554f2SKenneth D. Merry * allocated bases on the latest IOC Facts. 353991554f2SKenneth D. Merry */ 354991554f2SKenneth D. Merry static int 355991554f2SKenneth D. Merry mpr_iocfacts_allocate(struct mpr_softc *sc, uint8_t attaching) 356991554f2SKenneth D. Merry { 357a2c14879SStephen McConnell int error; 358991554f2SKenneth D. Merry Mpi2IOCFactsReply_t saved_facts; 359991554f2SKenneth D. Merry uint8_t saved_mode, reallocating; 360991554f2SKenneth D. Merry 361991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 362991554f2SKenneth D. Merry 363991554f2SKenneth D. Merry /* Save old IOC Facts and then only reallocate if Facts have changed */ 364991554f2SKenneth D. Merry if (!attaching) { 365991554f2SKenneth D. Merry bcopy(sc->facts, &saved_facts, sizeof(MPI2_IOC_FACTS_REPLY)); 366991554f2SKenneth D. Merry } 367991554f2SKenneth D. Merry 368991554f2SKenneth D. Merry /* 369991554f2SKenneth D. Merry * Get IOC Facts. In all cases throughout this function, panic if doing 370991554f2SKenneth D. Merry * a re-initialization and only return the error if attaching so the OS 371991554f2SKenneth D. Merry * can handle it. 372991554f2SKenneth D. Merry */ 373991554f2SKenneth D. Merry if ((error = mpr_get_iocfacts(sc, sc->facts)) != 0) { 374991554f2SKenneth D. Merry if (attaching) { 375991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "%s failed to get IOC Facts " 376991554f2SKenneth D. Merry "with error %d\n", __func__, error); 377991554f2SKenneth D. Merry return (error); 378991554f2SKenneth D. Merry } else { 379991554f2SKenneth D. Merry panic("%s failed to get IOC Facts with error %d\n", 380991554f2SKenneth D. Merry __func__, error); 381991554f2SKenneth D. Merry } 382991554f2SKenneth D. Merry } 383991554f2SKenneth D. Merry 384991554f2SKenneth D. Merry mpr_print_iocfacts(sc, sc->facts); 385991554f2SKenneth D. Merry 386991554f2SKenneth D. Merry snprintf(sc->fw_version, sizeof(sc->fw_version), 387991554f2SKenneth D. Merry "%02d.%02d.%02d.%02d", 388991554f2SKenneth D. Merry sc->facts->FWVersion.Struct.Major, 389991554f2SKenneth D. Merry sc->facts->FWVersion.Struct.Minor, 390991554f2SKenneth D. Merry sc->facts->FWVersion.Struct.Unit, 391991554f2SKenneth D. Merry sc->facts->FWVersion.Struct.Dev); 392991554f2SKenneth D. Merry 393991554f2SKenneth D. Merry mpr_printf(sc, "Firmware: %s, Driver: %s\n", sc->fw_version, 394991554f2SKenneth D. Merry MPR_DRIVER_VERSION); 395991554f2SKenneth D. Merry mpr_printf(sc, "IOCCapabilities: %b\n", sc->facts->IOCCapabilities, 396991554f2SKenneth D. Merry "\20" "\3ScsiTaskFull" "\4DiagTrace" "\5SnapBuf" "\6ExtBuf" 397991554f2SKenneth D. Merry "\7EEDP" "\10BiDirTarg" "\11Multicast" "\14TransRetry" "\15IR" 39867feec50SStephen McConnell "\16EventReplay" "\17RaidAccel" "\20MSIXIndex" "\21HostDisc" 39967feec50SStephen McConnell "\22FastPath" "\23RDPQArray" "\24AtomicReqDesc" "\25PCIeSRIOV"); 400991554f2SKenneth D. Merry 401991554f2SKenneth D. Merry /* 402991554f2SKenneth D. Merry * If the chip doesn't support event replay then a hard reset will be 403991554f2SKenneth D. Merry * required to trigger a full discovery. Do the reset here then 404991554f2SKenneth D. Merry * retransition to Ready. A hard reset might have already been done, 405991554f2SKenneth D. Merry * but it doesn't hurt to do it again. Only do this if attaching, not 406991554f2SKenneth D. Merry * for a Diag Reset. 407991554f2SKenneth D. Merry */ 408991554f2SKenneth D. Merry if (attaching) { 409991554f2SKenneth D. Merry if ((sc->facts->IOCCapabilities & 410991554f2SKenneth D. Merry MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY) == 0) { 411991554f2SKenneth D. Merry mpr_diag_reset(sc, NO_SLEEP); 412991554f2SKenneth D. Merry if ((error = mpr_transition_ready(sc)) != 0) { 413991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "%s failed to " 414991554f2SKenneth D. Merry "transition to ready with error %d\n", 415991554f2SKenneth D. Merry __func__, error); 416991554f2SKenneth D. Merry return (error); 417991554f2SKenneth D. Merry } 418991554f2SKenneth D. Merry } 419991554f2SKenneth D. Merry } 420991554f2SKenneth D. Merry 421991554f2SKenneth D. Merry /* 422991554f2SKenneth D. Merry * Set flag if IR Firmware is loaded. If the RAID Capability has 423991554f2SKenneth D. Merry * changed from the previous IOC Facts, log a warning, but only if 424991554f2SKenneth D. Merry * checking this after a Diag Reset and not during attach. 425991554f2SKenneth D. Merry */ 426991554f2SKenneth D. Merry saved_mode = sc->ir_firmware; 427991554f2SKenneth D. Merry if (sc->facts->IOCCapabilities & 428991554f2SKenneth D. Merry MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) 429991554f2SKenneth D. Merry sc->ir_firmware = 1; 430991554f2SKenneth D. Merry if (!attaching) { 431991554f2SKenneth D. Merry if (sc->ir_firmware != saved_mode) { 432991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "%s new IR/IT mode in IOC " 433991554f2SKenneth D. Merry "Facts does not match previous mode\n", __func__); 434991554f2SKenneth D. Merry } 435991554f2SKenneth D. Merry } 436991554f2SKenneth D. Merry 437991554f2SKenneth D. Merry /* Only deallocate and reallocate if relevant IOC Facts have changed */ 438991554f2SKenneth D. Merry reallocating = FALSE; 439991554f2SKenneth D. Merry if ((!attaching) && 440991554f2SKenneth D. Merry ((saved_facts.MsgVersion != sc->facts->MsgVersion) || 441991554f2SKenneth D. Merry (saved_facts.HeaderVersion != sc->facts->HeaderVersion) || 442991554f2SKenneth D. Merry (saved_facts.MaxChainDepth != sc->facts->MaxChainDepth) || 443991554f2SKenneth D. Merry (saved_facts.RequestCredit != sc->facts->RequestCredit) || 444991554f2SKenneth D. Merry (saved_facts.ProductID != sc->facts->ProductID) || 445991554f2SKenneth D. Merry (saved_facts.IOCCapabilities != sc->facts->IOCCapabilities) || 446991554f2SKenneth D. Merry (saved_facts.IOCRequestFrameSize != 447991554f2SKenneth D. Merry sc->facts->IOCRequestFrameSize) || 4482bbc5fcbSStephen McConnell (saved_facts.IOCMaxChainSegmentSize != 4492bbc5fcbSStephen McConnell sc->facts->IOCMaxChainSegmentSize) || 450991554f2SKenneth D. Merry (saved_facts.MaxTargets != sc->facts->MaxTargets) || 451991554f2SKenneth D. Merry (saved_facts.MaxSasExpanders != sc->facts->MaxSasExpanders) || 452991554f2SKenneth D. Merry (saved_facts.MaxEnclosures != sc->facts->MaxEnclosures) || 453991554f2SKenneth D. Merry (saved_facts.HighPriorityCredit != sc->facts->HighPriorityCredit) || 454991554f2SKenneth D. Merry (saved_facts.MaxReplyDescriptorPostQueueDepth != 455991554f2SKenneth D. Merry sc->facts->MaxReplyDescriptorPostQueueDepth) || 456991554f2SKenneth D. Merry (saved_facts.ReplyFrameSize != sc->facts->ReplyFrameSize) || 457991554f2SKenneth D. Merry (saved_facts.MaxVolumes != sc->facts->MaxVolumes) || 458991554f2SKenneth D. Merry (saved_facts.MaxPersistentEntries != 459991554f2SKenneth D. Merry sc->facts->MaxPersistentEntries))) { 460991554f2SKenneth D. Merry reallocating = TRUE; 461991554f2SKenneth D. Merry } 462991554f2SKenneth D. Merry 463991554f2SKenneth D. Merry /* 464991554f2SKenneth D. Merry * Some things should be done if attaching or re-allocating after a Diag 465991554f2SKenneth D. Merry * Reset, but are not needed after a Diag Reset if the FW has not 466991554f2SKenneth D. Merry * changed. 467991554f2SKenneth D. Merry */ 468991554f2SKenneth D. Merry if (attaching || reallocating) { 469991554f2SKenneth D. Merry /* 470991554f2SKenneth D. Merry * Check if controller supports FW diag buffers and set flag to 471991554f2SKenneth D. Merry * enable each type. 472991554f2SKenneth D. Merry */ 473991554f2SKenneth D. Merry if (sc->facts->IOCCapabilities & 474991554f2SKenneth D. Merry MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) 475991554f2SKenneth D. Merry sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_TRACE]. 476991554f2SKenneth D. Merry enabled = TRUE; 477991554f2SKenneth D. Merry if (sc->facts->IOCCapabilities & 478991554f2SKenneth D. Merry MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) 479991554f2SKenneth D. Merry sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_SNAPSHOT]. 480991554f2SKenneth D. Merry enabled = TRUE; 481991554f2SKenneth D. Merry if (sc->facts->IOCCapabilities & 482991554f2SKenneth D. Merry MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) 483991554f2SKenneth D. Merry sc->fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_EXTENDED]. 484991554f2SKenneth D. Merry enabled = TRUE; 485991554f2SKenneth D. Merry 486991554f2SKenneth D. Merry /* 48767feec50SStephen McConnell * Set flags for some supported items. 488991554f2SKenneth D. Merry */ 489991554f2SKenneth D. Merry if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) 490991554f2SKenneth D. Merry sc->eedp_enabled = TRUE; 491991554f2SKenneth D. Merry if (sc->facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) 492991554f2SKenneth D. Merry sc->control_TLR = TRUE; 49367feec50SStephen McConnell if (sc->facts->IOCCapabilities & 49467feec50SStephen McConnell MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ) 49567feec50SStephen McConnell sc->atomic_desc_capable = TRUE; 496991554f2SKenneth D. Merry 497991554f2SKenneth D. Merry /* 498991554f2SKenneth D. Merry * Size the queues. Since the reply queues always need one free 499991554f2SKenneth D. Merry * entry, we'll just deduct one reply message here. 500991554f2SKenneth D. Merry */ 501991554f2SKenneth D. Merry sc->num_reqs = MIN(MPR_REQ_FRAMES, sc->facts->RequestCredit); 502991554f2SKenneth D. Merry sc->num_replies = MIN(MPR_REPLY_FRAMES + MPR_EVT_REPLY_FRAMES, 503991554f2SKenneth D. Merry sc->facts->MaxReplyDescriptorPostQueueDepth) - 1; 504991554f2SKenneth D. Merry 505991554f2SKenneth D. Merry /* 506991554f2SKenneth D. Merry * Initialize all Tail Queues 507991554f2SKenneth D. Merry */ 508991554f2SKenneth D. Merry TAILQ_INIT(&sc->req_list); 509991554f2SKenneth D. Merry TAILQ_INIT(&sc->high_priority_req_list); 510991554f2SKenneth D. Merry TAILQ_INIT(&sc->chain_list); 51167feec50SStephen McConnell TAILQ_INIT(&sc->prp_page_list); 512991554f2SKenneth D. Merry TAILQ_INIT(&sc->tm_list); 513991554f2SKenneth D. Merry } 514991554f2SKenneth D. Merry 515991554f2SKenneth D. Merry /* 516991554f2SKenneth D. Merry * If doing a Diag Reset and the FW is significantly different 517991554f2SKenneth D. Merry * (reallocating will be set above in IOC Facts comparison), then all 518991554f2SKenneth D. Merry * buffers based on the IOC Facts will need to be freed before they are 519991554f2SKenneth D. Merry * reallocated. 520991554f2SKenneth D. Merry */ 521991554f2SKenneth D. Merry if (reallocating) { 522991554f2SKenneth D. Merry mpr_iocfacts_free(sc); 523327f2e6cSStephen McConnell mprsas_realloc_targets(sc, saved_facts.MaxTargets + 524327f2e6cSStephen McConnell saved_facts.MaxVolumes); 525991554f2SKenneth D. Merry } 526991554f2SKenneth D. Merry 527991554f2SKenneth D. Merry /* 528991554f2SKenneth D. Merry * Any deallocation has been completed. Now start reallocating 529991554f2SKenneth D. Merry * if needed. Will only need to reallocate if attaching or if the new 530991554f2SKenneth D. Merry * IOC Facts are different from the previous IOC Facts after a Diag 531991554f2SKenneth D. Merry * Reset. Targets have already been allocated above if needed. 532991554f2SKenneth D. Merry */ 533991554f2SKenneth D. Merry if (attaching || reallocating) { 534991554f2SKenneth D. Merry if (((error = mpr_alloc_queues(sc)) != 0) || 535991554f2SKenneth D. Merry ((error = mpr_alloc_replies(sc)) != 0) || 536991554f2SKenneth D. Merry ((error = mpr_alloc_requests(sc)) != 0)) { 537991554f2SKenneth D. Merry if (attaching ) { 538991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "%s failed to alloc " 539991554f2SKenneth D. Merry "queues with error %d\n", __func__, error); 540991554f2SKenneth D. Merry mpr_free(sc); 541991554f2SKenneth D. Merry return (error); 542991554f2SKenneth D. Merry } else { 543991554f2SKenneth D. Merry panic("%s failed to alloc queues with error " 544991554f2SKenneth D. Merry "%d\n", __func__, error); 545991554f2SKenneth D. Merry } 546991554f2SKenneth D. Merry } 547991554f2SKenneth D. Merry } 548991554f2SKenneth D. Merry 549991554f2SKenneth D. Merry /* Always initialize the queues */ 550991554f2SKenneth D. Merry bzero(sc->free_queue, sc->fqdepth * 4); 551991554f2SKenneth D. Merry mpr_init_queues(sc); 552991554f2SKenneth D. Merry 553991554f2SKenneth D. Merry /* 554991554f2SKenneth D. Merry * Always get the chip out of the reset state, but only panic if not 555991554f2SKenneth D. Merry * attaching. If attaching and there is an error, that is handled by 556991554f2SKenneth D. Merry * the OS. 557991554f2SKenneth D. Merry */ 558991554f2SKenneth D. Merry error = mpr_transition_operational(sc); 559991554f2SKenneth D. Merry if (error != 0) { 560991554f2SKenneth D. Merry if (attaching) { 5617a2a6a1aSStephen McConnell mpr_printf(sc, "%s failed to transition to operational " 5627a2a6a1aSStephen McConnell "with error %d\n", __func__, error); 563991554f2SKenneth D. Merry mpr_free(sc); 564991554f2SKenneth D. Merry return (error); 565991554f2SKenneth D. Merry } else { 566991554f2SKenneth D. Merry panic("%s failed to transition to operational with " 567991554f2SKenneth D. Merry "error %d\n", __func__, error); 568991554f2SKenneth D. Merry } 569991554f2SKenneth D. Merry } 570991554f2SKenneth D. Merry 571991554f2SKenneth D. Merry /* 572991554f2SKenneth D. Merry * Finish the queue initialization. 573991554f2SKenneth D. Merry * These are set here instead of in mpr_init_queues() because the 574991554f2SKenneth D. Merry * IOC resets these values during the state transition in 575991554f2SKenneth D. Merry * mpr_transition_operational(). The free index is set to 1 576991554f2SKenneth D. Merry * because the corresponding index in the IOC is set to 0, and the 577991554f2SKenneth D. Merry * IOC treats the queues as full if both are set to the same value. 578991554f2SKenneth D. Merry * Hence the reason that the queue can't hold all of the possible 579991554f2SKenneth D. Merry * replies. 580991554f2SKenneth D. Merry */ 581991554f2SKenneth D. Merry sc->replypostindex = 0; 582991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_REPLY_FREE_HOST_INDEX_OFFSET, sc->replyfreeindex); 583991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET, 0); 584991554f2SKenneth D. Merry 585991554f2SKenneth D. Merry /* 586991554f2SKenneth D. Merry * Attach the subsystems so they can prepare their event masks. 587991554f2SKenneth D. Merry */ 588991554f2SKenneth D. Merry /* XXX Should be dynamic so that IM/IR and user modules can attach */ 589991554f2SKenneth D. Merry if (attaching) { 590991554f2SKenneth D. Merry if (((error = mpr_attach_log(sc)) != 0) || 591991554f2SKenneth D. Merry ((error = mpr_attach_sas(sc)) != 0) || 592991554f2SKenneth D. Merry ((error = mpr_attach_user(sc)) != 0)) { 593991554f2SKenneth D. Merry mpr_printf(sc, "%s failed to attach all subsystems: " 594991554f2SKenneth D. Merry "error %d\n", __func__, error); 595991554f2SKenneth D. Merry mpr_free(sc); 596991554f2SKenneth D. Merry return (error); 597991554f2SKenneth D. Merry } 598991554f2SKenneth D. Merry 599991554f2SKenneth D. Merry if ((error = mpr_pci_setup_interrupts(sc)) != 0) { 600991554f2SKenneth D. Merry mpr_printf(sc, "%s failed to setup interrupts\n", 601991554f2SKenneth D. Merry __func__); 602991554f2SKenneth D. Merry mpr_free(sc); 603991554f2SKenneth D. Merry return (error); 604991554f2SKenneth D. Merry } 605991554f2SKenneth D. Merry } 606991554f2SKenneth D. Merry 607991554f2SKenneth D. Merry return (error); 608991554f2SKenneth D. Merry } 609991554f2SKenneth D. Merry 610991554f2SKenneth D. Merry /* 611991554f2SKenneth D. Merry * This is called if memory is being free (during detach for example) and when 612991554f2SKenneth D. Merry * buffers need to be reallocated due to a Diag Reset. 613991554f2SKenneth D. Merry */ 614991554f2SKenneth D. Merry static void 615991554f2SKenneth D. Merry mpr_iocfacts_free(struct mpr_softc *sc) 616991554f2SKenneth D. Merry { 617991554f2SKenneth D. Merry struct mpr_command *cm; 618991554f2SKenneth D. Merry int i; 619991554f2SKenneth D. Merry 620991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 621991554f2SKenneth D. Merry 622991554f2SKenneth D. Merry if (sc->free_busaddr != 0) 623991554f2SKenneth D. Merry bus_dmamap_unload(sc->queues_dmat, sc->queues_map); 624991554f2SKenneth D. Merry if (sc->free_queue != NULL) 625991554f2SKenneth D. Merry bus_dmamem_free(sc->queues_dmat, sc->free_queue, 626991554f2SKenneth D. Merry sc->queues_map); 627991554f2SKenneth D. Merry if (sc->queues_dmat != NULL) 628991554f2SKenneth D. Merry bus_dma_tag_destroy(sc->queues_dmat); 629991554f2SKenneth D. Merry 630991554f2SKenneth D. Merry if (sc->chain_busaddr != 0) 631991554f2SKenneth D. Merry bus_dmamap_unload(sc->chain_dmat, sc->chain_map); 632991554f2SKenneth D. Merry if (sc->chain_frames != NULL) 633991554f2SKenneth D. Merry bus_dmamem_free(sc->chain_dmat, sc->chain_frames, 634991554f2SKenneth D. Merry sc->chain_map); 635991554f2SKenneth D. Merry if (sc->chain_dmat != NULL) 636991554f2SKenneth D. Merry bus_dma_tag_destroy(sc->chain_dmat); 637991554f2SKenneth D. Merry 638991554f2SKenneth D. Merry if (sc->sense_busaddr != 0) 639991554f2SKenneth D. Merry bus_dmamap_unload(sc->sense_dmat, sc->sense_map); 640991554f2SKenneth D. Merry if (sc->sense_frames != NULL) 641991554f2SKenneth D. Merry bus_dmamem_free(sc->sense_dmat, sc->sense_frames, 642991554f2SKenneth D. Merry sc->sense_map); 643991554f2SKenneth D. Merry if (sc->sense_dmat != NULL) 644991554f2SKenneth D. Merry bus_dma_tag_destroy(sc->sense_dmat); 645991554f2SKenneth D. Merry 64667feec50SStephen McConnell if (sc->prp_page_busaddr != 0) 64767feec50SStephen McConnell bus_dmamap_unload(sc->prp_page_dmat, sc->prp_page_map); 64867feec50SStephen McConnell if (sc->prp_pages != NULL) 64967feec50SStephen McConnell bus_dmamem_free(sc->prp_page_dmat, sc->prp_pages, 65067feec50SStephen McConnell sc->prp_page_map); 65167feec50SStephen McConnell if (sc->prp_page_dmat != NULL) 65267feec50SStephen McConnell bus_dma_tag_destroy(sc->prp_page_dmat); 65367feec50SStephen McConnell 654991554f2SKenneth D. Merry if (sc->reply_busaddr != 0) 655991554f2SKenneth D. Merry bus_dmamap_unload(sc->reply_dmat, sc->reply_map); 656991554f2SKenneth D. Merry if (sc->reply_frames != NULL) 657991554f2SKenneth D. Merry bus_dmamem_free(sc->reply_dmat, sc->reply_frames, 658991554f2SKenneth D. Merry sc->reply_map); 659991554f2SKenneth D. Merry if (sc->reply_dmat != NULL) 660991554f2SKenneth D. Merry bus_dma_tag_destroy(sc->reply_dmat); 661991554f2SKenneth D. Merry 662991554f2SKenneth D. Merry if (sc->req_busaddr != 0) 663991554f2SKenneth D. Merry bus_dmamap_unload(sc->req_dmat, sc->req_map); 664991554f2SKenneth D. Merry if (sc->req_frames != NULL) 665991554f2SKenneth D. Merry bus_dmamem_free(sc->req_dmat, sc->req_frames, sc->req_map); 666991554f2SKenneth D. Merry if (sc->req_dmat != NULL) 667991554f2SKenneth D. Merry bus_dma_tag_destroy(sc->req_dmat); 668991554f2SKenneth D. Merry 669991554f2SKenneth D. Merry if (sc->chains != NULL) 670991554f2SKenneth D. Merry free(sc->chains, M_MPR); 67167feec50SStephen McConnell if (sc->prps != NULL) 67267feec50SStephen McConnell free(sc->prps, M_MPR); 673991554f2SKenneth D. Merry if (sc->commands != NULL) { 674991554f2SKenneth D. Merry for (i = 1; i < sc->num_reqs; i++) { 675991554f2SKenneth D. Merry cm = &sc->commands[i]; 676991554f2SKenneth D. Merry bus_dmamap_destroy(sc->buffer_dmat, cm->cm_dmamap); 677991554f2SKenneth D. Merry } 678991554f2SKenneth D. Merry free(sc->commands, M_MPR); 679991554f2SKenneth D. Merry } 680991554f2SKenneth D. Merry if (sc->buffer_dmat != NULL) 681991554f2SKenneth D. Merry bus_dma_tag_destroy(sc->buffer_dmat); 682991554f2SKenneth D. Merry } 683991554f2SKenneth D. Merry 684991554f2SKenneth D. Merry /* 685991554f2SKenneth D. Merry * The terms diag reset and hard reset are used interchangeably in the MPI 686991554f2SKenneth D. Merry * docs to mean resetting the controller chip. In this code diag reset 687991554f2SKenneth D. Merry * cleans everything up, and the hard reset function just sends the reset 688991554f2SKenneth D. Merry * sequence to the chip. This should probably be refactored so that every 689991554f2SKenneth D. Merry * subsystem gets a reset notification of some sort, and can clean up 690991554f2SKenneth D. Merry * appropriately. 691991554f2SKenneth D. Merry */ 692991554f2SKenneth D. Merry int 693991554f2SKenneth D. Merry mpr_reinit(struct mpr_softc *sc) 694991554f2SKenneth D. Merry { 695991554f2SKenneth D. Merry int error; 696991554f2SKenneth D. Merry struct mprsas_softc *sassc; 697991554f2SKenneth D. Merry 698991554f2SKenneth D. Merry sassc = sc->sassc; 699991554f2SKenneth D. Merry 700991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 701991554f2SKenneth D. Merry 702991554f2SKenneth D. Merry mtx_assert(&sc->mpr_mtx, MA_OWNED); 703991554f2SKenneth D. Merry 704991554f2SKenneth D. Merry if (sc->mpr_flags & MPR_FLAGS_DIAGRESET) { 705991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INIT, "%s reset already in progress\n", 706991554f2SKenneth D. Merry __func__); 707991554f2SKenneth D. Merry return 0; 708991554f2SKenneth D. Merry } 709991554f2SKenneth D. Merry 710991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INFO, "Reinitializing controller,\n"); 711991554f2SKenneth D. Merry /* make sure the completion callbacks can recognize they're getting 712991554f2SKenneth D. Merry * a NULL cm_reply due to a reset. 713991554f2SKenneth D. Merry */ 714991554f2SKenneth D. Merry sc->mpr_flags |= MPR_FLAGS_DIAGRESET; 715991554f2SKenneth D. Merry 716991554f2SKenneth D. Merry /* 717991554f2SKenneth D. Merry * Mask interrupts here. 718991554f2SKenneth D. Merry */ 719991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INIT, "%s mask interrupts\n", __func__); 720991554f2SKenneth D. Merry mpr_mask_intr(sc); 721991554f2SKenneth D. Merry 722991554f2SKenneth D. Merry error = mpr_diag_reset(sc, CAN_SLEEP); 723991554f2SKenneth D. Merry if (error != 0) { 724991554f2SKenneth D. Merry panic("%s hard reset failed with error %d\n", __func__, error); 725991554f2SKenneth D. Merry } 726991554f2SKenneth D. Merry 727991554f2SKenneth D. Merry /* Restore the PCI state, including the MSI-X registers */ 728991554f2SKenneth D. Merry mpr_pci_restore(sc); 729991554f2SKenneth D. Merry 730991554f2SKenneth D. Merry /* Give the I/O subsystem special priority to get itself prepared */ 731991554f2SKenneth D. Merry mprsas_handle_reinit(sc); 732991554f2SKenneth D. Merry 733991554f2SKenneth D. Merry /* 734991554f2SKenneth D. Merry * Get IOC Facts and allocate all structures based on this information. 735991554f2SKenneth D. Merry * The attach function will also call mpr_iocfacts_allocate at startup. 736991554f2SKenneth D. Merry * If relevant values have changed in IOC Facts, this function will free 737991554f2SKenneth D. Merry * all of the memory based on IOC Facts and reallocate that memory. 738991554f2SKenneth D. Merry */ 739991554f2SKenneth D. Merry if ((error = mpr_iocfacts_allocate(sc, FALSE)) != 0) { 740991554f2SKenneth D. Merry panic("%s IOC Facts based allocation failed with error %d\n", 741991554f2SKenneth D. Merry __func__, error); 742991554f2SKenneth D. Merry } 743991554f2SKenneth D. Merry 744991554f2SKenneth D. Merry /* 745991554f2SKenneth D. Merry * Mapping structures will be re-allocated after getting IOC Page8, so 746991554f2SKenneth D. Merry * free these structures here. 747991554f2SKenneth D. Merry */ 748991554f2SKenneth D. Merry mpr_mapping_exit(sc); 749991554f2SKenneth D. Merry 750991554f2SKenneth D. Merry /* 751991554f2SKenneth D. Merry * The static page function currently read is IOC Page8. Others can be 752991554f2SKenneth D. Merry * added in future. It's possible that the values in IOC Page8 have 753991554f2SKenneth D. Merry * changed after a Diag Reset due to user modification, so always read 754991554f2SKenneth D. Merry * these. Interrupts are masked, so unmask them before getting config 755991554f2SKenneth D. Merry * pages. 756991554f2SKenneth D. Merry */ 757991554f2SKenneth D. Merry mpr_unmask_intr(sc); 758991554f2SKenneth D. Merry sc->mpr_flags &= ~MPR_FLAGS_DIAGRESET; 759991554f2SKenneth D. Merry mpr_base_static_config_pages(sc); 760991554f2SKenneth D. Merry 761991554f2SKenneth D. Merry /* 762991554f2SKenneth D. Merry * Some mapping info is based in IOC Page8 data, so re-initialize the 763991554f2SKenneth D. Merry * mapping tables. 764991554f2SKenneth D. Merry */ 765991554f2SKenneth D. Merry mpr_mapping_initialize(sc); 766991554f2SKenneth D. Merry 767991554f2SKenneth D. Merry /* 768991554f2SKenneth D. Merry * Restart will reload the event masks clobbered by the reset, and 769991554f2SKenneth D. Merry * then enable the port. 770991554f2SKenneth D. Merry */ 771991554f2SKenneth D. Merry mpr_reregister_events(sc); 772991554f2SKenneth D. Merry 773991554f2SKenneth D. Merry /* the end of discovery will release the simq, so we're done. */ 774991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INFO, "%s finished sc %p post %u free %u\n", 775991554f2SKenneth D. Merry __func__, sc, sc->replypostindex, sc->replyfreeindex); 776991554f2SKenneth D. Merry mprsas_release_simq_reinit(sassc); 777991554f2SKenneth D. Merry 778991554f2SKenneth D. Merry return 0; 779991554f2SKenneth D. Merry } 780991554f2SKenneth D. Merry 781991554f2SKenneth D. Merry /* Wait for the chip to ACK a word that we've put into its FIFO 782991554f2SKenneth D. Merry * Wait for <timeout> seconds. In single loop wait for busy loop 783991554f2SKenneth D. Merry * for 500 microseconds. 784991554f2SKenneth D. Merry * Total is [ 0.5 * (2000 * <timeout>) ] in miliseconds. 785991554f2SKenneth D. Merry * */ 786991554f2SKenneth D. Merry static int 787991554f2SKenneth D. Merry mpr_wait_db_ack(struct mpr_softc *sc, int timeout, int sleep_flag) 788991554f2SKenneth D. Merry { 789991554f2SKenneth D. Merry u32 cntdn, count; 790991554f2SKenneth D. Merry u32 int_status; 791991554f2SKenneth D. Merry u32 doorbell; 792991554f2SKenneth D. Merry 793991554f2SKenneth D. Merry count = 0; 794991554f2SKenneth D. Merry cntdn = (sleep_flag == CAN_SLEEP) ? 1000*timeout : 2000*timeout; 795991554f2SKenneth D. Merry do { 796991554f2SKenneth D. Merry int_status = mpr_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET); 797991554f2SKenneth D. Merry if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) { 798991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INIT, "%s: successful count(%d), " 799991554f2SKenneth D. Merry "timeout(%d)\n", __func__, count, timeout); 800991554f2SKenneth D. Merry return 0; 801991554f2SKenneth D. Merry } else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) { 802991554f2SKenneth D. Merry doorbell = mpr_regread(sc, MPI2_DOORBELL_OFFSET); 803991554f2SKenneth D. Merry if ((doorbell & MPI2_IOC_STATE_MASK) == 804991554f2SKenneth D. Merry MPI2_IOC_STATE_FAULT) { 805991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 806991554f2SKenneth D. Merry "fault_state(0x%04x)!\n", doorbell); 807991554f2SKenneth D. Merry return (EFAULT); 808991554f2SKenneth D. Merry } 809991554f2SKenneth D. Merry } else if (int_status == 0xFFFFFFFF) 810991554f2SKenneth D. Merry goto out; 811991554f2SKenneth D. Merry 812991554f2SKenneth D. Merry /* 813991554f2SKenneth D. Merry * If it can sleep, sleep for 1 milisecond, else busy loop for 814991554f2SKenneth D. Merry * 0.5 milisecond 815991554f2SKenneth D. Merry */ 816991554f2SKenneth D. Merry if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP) 817a2c14879SStephen McConnell msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0, "mprdba", 818a2c14879SStephen McConnell hz/1000); 819991554f2SKenneth D. Merry else if (sleep_flag == CAN_SLEEP) 820991554f2SKenneth D. Merry pause("mprdba", hz/1000); 821991554f2SKenneth D. Merry else 822991554f2SKenneth D. Merry DELAY(500); 823991554f2SKenneth D. Merry count++; 824991554f2SKenneth D. Merry } while (--cntdn); 825991554f2SKenneth D. Merry 826991554f2SKenneth D. Merry out: 827991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "%s: failed due to timeout count(%d), " 828991554f2SKenneth D. Merry "int_status(%x)!\n", __func__, count, int_status); 829991554f2SKenneth D. Merry return (ETIMEDOUT); 830991554f2SKenneth D. Merry } 831991554f2SKenneth D. Merry 832991554f2SKenneth D. Merry /* Wait for the chip to signal that the next word in its FIFO can be fetched */ 833991554f2SKenneth D. Merry static int 834991554f2SKenneth D. Merry mpr_wait_db_int(struct mpr_softc *sc) 835991554f2SKenneth D. Merry { 836991554f2SKenneth D. Merry int retry; 837991554f2SKenneth D. Merry 838991554f2SKenneth D. Merry for (retry = 0; retry < MPR_DB_MAX_WAIT; retry++) { 839991554f2SKenneth D. Merry if ((mpr_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET) & 840991554f2SKenneth D. Merry MPI2_HIS_IOC2SYS_DB_STATUS) != 0) 841991554f2SKenneth D. Merry return (0); 842991554f2SKenneth D. Merry DELAY(2000); 843991554f2SKenneth D. Merry } 844991554f2SKenneth D. Merry return (ETIMEDOUT); 845991554f2SKenneth D. Merry } 846991554f2SKenneth D. Merry 847991554f2SKenneth D. Merry /* Step through the synchronous command state machine, i.e. "Doorbell mode" */ 848991554f2SKenneth D. Merry static int 849991554f2SKenneth D. Merry mpr_request_sync(struct mpr_softc *sc, void *req, MPI2_DEFAULT_REPLY *reply, 850991554f2SKenneth D. Merry int req_sz, int reply_sz, int timeout) 851991554f2SKenneth D. Merry { 852991554f2SKenneth D. Merry uint32_t *data32; 853991554f2SKenneth D. Merry uint16_t *data16; 854991554f2SKenneth D. Merry int i, count, ioc_sz, residual; 855991554f2SKenneth D. Merry int sleep_flags = CAN_SLEEP; 856991554f2SKenneth D. Merry 857991554f2SKenneth D. Merry #if __FreeBSD_version >= 1000029 858991554f2SKenneth D. Merry if (curthread->td_no_sleeping) 859991554f2SKenneth D. Merry #else //__FreeBSD_version < 1000029 860991554f2SKenneth D. Merry if (curthread->td_pflags & TDP_NOSLEEPING) 861991554f2SKenneth D. Merry #endif //__FreeBSD_version >= 1000029 862991554f2SKenneth D. Merry sleep_flags = NO_SLEEP; 863991554f2SKenneth D. Merry 864991554f2SKenneth D. Merry /* Step 1 */ 865991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 866991554f2SKenneth D. Merry 867991554f2SKenneth D. Merry /* Step 2 */ 868991554f2SKenneth D. Merry if (mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED) 869991554f2SKenneth D. Merry return (EBUSY); 870991554f2SKenneth D. Merry 871991554f2SKenneth D. Merry /* Step 3 872991554f2SKenneth D. Merry * Announce that a message is coming through the doorbell. Messages 873991554f2SKenneth D. Merry * are pushed at 32bit words, so round up if needed. 874991554f2SKenneth D. Merry */ 875991554f2SKenneth D. Merry count = (req_sz + 3) / 4; 876991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_DOORBELL_OFFSET, 877991554f2SKenneth D. Merry (MPI2_FUNCTION_HANDSHAKE << MPI2_DOORBELL_FUNCTION_SHIFT) | 878991554f2SKenneth D. Merry (count << MPI2_DOORBELL_ADD_DWORDS_SHIFT)); 879991554f2SKenneth D. Merry 880991554f2SKenneth D. Merry /* Step 4 */ 881991554f2SKenneth D. Merry if (mpr_wait_db_int(sc) || 882991554f2SKenneth D. Merry (mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED) == 0) { 883991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Doorbell failed to activate\n"); 884991554f2SKenneth D. Merry return (ENXIO); 885991554f2SKenneth D. Merry } 886991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 887991554f2SKenneth D. Merry if (mpr_wait_db_ack(sc, 5, sleep_flags) != 0) { 888991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Doorbell handshake failed\n"); 889991554f2SKenneth D. Merry return (ENXIO); 890991554f2SKenneth D. Merry } 891991554f2SKenneth D. Merry 892991554f2SKenneth D. Merry /* Step 5 */ 893991554f2SKenneth D. Merry /* Clock out the message data synchronously in 32-bit dwords*/ 894991554f2SKenneth D. Merry data32 = (uint32_t *)req; 895991554f2SKenneth D. Merry for (i = 0; i < count; i++) { 896991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_DOORBELL_OFFSET, htole32(data32[i])); 897991554f2SKenneth D. Merry if (mpr_wait_db_ack(sc, 5, sleep_flags) != 0) { 898991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 899991554f2SKenneth D. Merry "Timeout while writing doorbell\n"); 900991554f2SKenneth D. Merry return (ENXIO); 901991554f2SKenneth D. Merry } 902991554f2SKenneth D. Merry } 903991554f2SKenneth D. Merry 904991554f2SKenneth D. Merry /* Step 6 */ 905991554f2SKenneth D. Merry /* Clock in the reply in 16-bit words. The total length of the 906991554f2SKenneth D. Merry * message is always in the 4th byte, so clock out the first 2 words 907991554f2SKenneth D. Merry * manually, then loop the rest. 908991554f2SKenneth D. Merry */ 909991554f2SKenneth D. Merry data16 = (uint16_t *)reply; 910991554f2SKenneth D. Merry if (mpr_wait_db_int(sc) != 0) { 911991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Timeout reading doorbell 0\n"); 912991554f2SKenneth D. Merry return (ENXIO); 913991554f2SKenneth D. Merry } 914991554f2SKenneth D. Merry data16[0] = 915991554f2SKenneth D. Merry mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK; 916991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 917991554f2SKenneth D. Merry if (mpr_wait_db_int(sc) != 0) { 918991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Timeout reading doorbell 1\n"); 919991554f2SKenneth D. Merry return (ENXIO); 920991554f2SKenneth D. Merry } 921991554f2SKenneth D. Merry data16[1] = 922991554f2SKenneth D. Merry mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_DATA_MASK; 923991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 924991554f2SKenneth D. Merry 925991554f2SKenneth D. Merry /* Number of 32bit words in the message */ 926991554f2SKenneth D. Merry ioc_sz = reply->MsgLength; 927991554f2SKenneth D. Merry 928991554f2SKenneth D. Merry /* 929991554f2SKenneth D. Merry * Figure out how many 16bit words to clock in without overrunning. 930991554f2SKenneth D. Merry * The precision loss with dividing reply_sz can safely be 931991554f2SKenneth D. Merry * ignored because the messages can only be multiples of 32bits. 932991554f2SKenneth D. Merry */ 933991554f2SKenneth D. Merry residual = 0; 934991554f2SKenneth D. Merry count = MIN((reply_sz / 4), ioc_sz) * 2; 935991554f2SKenneth D. Merry if (count < ioc_sz * 2) { 936991554f2SKenneth D. Merry residual = ioc_sz * 2 - count; 937991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "Driver error, throwing away %d " 938991554f2SKenneth D. Merry "residual message words\n", residual); 939991554f2SKenneth D. Merry } 940991554f2SKenneth D. Merry 941991554f2SKenneth D. Merry for (i = 2; i < count; i++) { 942991554f2SKenneth D. Merry if (mpr_wait_db_int(sc) != 0) { 943991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 944991554f2SKenneth D. Merry "Timeout reading doorbell %d\n", i); 945991554f2SKenneth D. Merry return (ENXIO); 946991554f2SKenneth D. Merry } 947991554f2SKenneth D. Merry data16[i] = mpr_regread(sc, MPI2_DOORBELL_OFFSET) & 948991554f2SKenneth D. Merry MPI2_DOORBELL_DATA_MASK; 949991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 950991554f2SKenneth D. Merry } 951991554f2SKenneth D. Merry 952991554f2SKenneth D. Merry /* 953991554f2SKenneth D. Merry * Pull out residual words that won't fit into the provided buffer. 954991554f2SKenneth D. Merry * This keeps the chip from hanging due to a driver programming 955991554f2SKenneth D. Merry * error. 956991554f2SKenneth D. Merry */ 957991554f2SKenneth D. Merry while (residual--) { 958991554f2SKenneth D. Merry if (mpr_wait_db_int(sc) != 0) { 959991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Timeout reading doorbell\n"); 960991554f2SKenneth D. Merry return (ENXIO); 961991554f2SKenneth D. Merry } 962991554f2SKenneth D. Merry (void)mpr_regread(sc, MPI2_DOORBELL_OFFSET); 963991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 964991554f2SKenneth D. Merry } 965991554f2SKenneth D. Merry 966991554f2SKenneth D. Merry /* Step 7 */ 967991554f2SKenneth D. Merry if (mpr_wait_db_int(sc) != 0) { 968991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Timeout waiting to exit doorbell\n"); 969991554f2SKenneth D. Merry return (ENXIO); 970991554f2SKenneth D. Merry } 971991554f2SKenneth D. Merry if (mpr_regread(sc, MPI2_DOORBELL_OFFSET) & MPI2_DOORBELL_USED) 972991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Warning, doorbell still active\n"); 973991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET, 0x0); 974991554f2SKenneth D. Merry 975991554f2SKenneth D. Merry return (0); 976991554f2SKenneth D. Merry } 977991554f2SKenneth D. Merry 978991554f2SKenneth D. Merry static void 979991554f2SKenneth D. Merry mpr_enqueue_request(struct mpr_softc *sc, struct mpr_command *cm) 980991554f2SKenneth D. Merry { 98167feec50SStephen McConnell request_descriptor rd; 982991554f2SKenneth D. Merry 983991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 984a2c14879SStephen McConnell mpr_dprint(sc, MPR_TRACE, "SMID %u cm %p ccb %p\n", 985991554f2SKenneth D. Merry cm->cm_desc.Default.SMID, cm, cm->cm_ccb); 986991554f2SKenneth D. Merry 987991554f2SKenneth D. Merry if (sc->mpr_flags & MPR_FLAGS_ATTACH_DONE && !(sc->mpr_flags & 988991554f2SKenneth D. Merry MPR_FLAGS_SHUTDOWN)) 989991554f2SKenneth D. Merry mtx_assert(&sc->mpr_mtx, MA_OWNED); 990991554f2SKenneth D. Merry 991991554f2SKenneth D. Merry if (++sc->io_cmds_active > sc->io_cmds_highwater) 992991554f2SKenneth D. Merry sc->io_cmds_highwater++; 993991554f2SKenneth D. Merry 99467feec50SStephen McConnell if (sc->atomic_desc_capable) { 99567feec50SStephen McConnell rd.u.low = cm->cm_desc.Words.Low; 99667feec50SStephen McConnell mpr_regwrite(sc, MPI26_ATOMIC_REQUEST_DESCRIPTOR_POST_OFFSET, 99767feec50SStephen McConnell rd.u.low); 99867feec50SStephen McConnell } else { 999991554f2SKenneth D. Merry rd.u.low = cm->cm_desc.Words.Low; 1000991554f2SKenneth D. Merry rd.u.high = cm->cm_desc.Words.High; 1001991554f2SKenneth D. Merry rd.word = htole64(rd.word); 1002991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_LOW_OFFSET, 1003991554f2SKenneth D. Merry rd.u.low); 1004991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_REQUEST_DESCRIPTOR_POST_HIGH_OFFSET, 1005991554f2SKenneth D. Merry rd.u.high); 1006991554f2SKenneth D. Merry } 100767feec50SStephen McConnell } 1008991554f2SKenneth D. Merry 1009991554f2SKenneth D. Merry /* 1010991554f2SKenneth D. Merry * Just the FACTS, ma'am. 1011991554f2SKenneth D. Merry */ 1012991554f2SKenneth D. Merry static int 1013991554f2SKenneth D. Merry mpr_get_iocfacts(struct mpr_softc *sc, MPI2_IOC_FACTS_REPLY *facts) 1014991554f2SKenneth D. Merry { 1015991554f2SKenneth D. Merry MPI2_DEFAULT_REPLY *reply; 1016991554f2SKenneth D. Merry MPI2_IOC_FACTS_REQUEST request; 1017991554f2SKenneth D. Merry int error, req_sz, reply_sz; 1018991554f2SKenneth D. Merry 1019991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 1020991554f2SKenneth D. Merry 1021991554f2SKenneth D. Merry req_sz = sizeof(MPI2_IOC_FACTS_REQUEST); 1022991554f2SKenneth D. Merry reply_sz = sizeof(MPI2_IOC_FACTS_REPLY); 1023991554f2SKenneth D. Merry reply = (MPI2_DEFAULT_REPLY *)facts; 1024991554f2SKenneth D. Merry 1025991554f2SKenneth D. Merry bzero(&request, req_sz); 1026991554f2SKenneth D. Merry request.Function = MPI2_FUNCTION_IOC_FACTS; 1027991554f2SKenneth D. Merry error = mpr_request_sync(sc, &request, reply, req_sz, reply_sz, 5); 1028991554f2SKenneth D. Merry 1029991554f2SKenneth D. Merry return (error); 1030991554f2SKenneth D. Merry } 1031991554f2SKenneth D. Merry 1032991554f2SKenneth D. Merry static int 1033991554f2SKenneth D. Merry mpr_send_iocinit(struct mpr_softc *sc) 1034991554f2SKenneth D. Merry { 1035991554f2SKenneth D. Merry MPI2_IOC_INIT_REQUEST init; 1036991554f2SKenneth D. Merry MPI2_DEFAULT_REPLY reply; 1037991554f2SKenneth D. Merry int req_sz, reply_sz, error; 1038991554f2SKenneth D. Merry struct timeval now; 1039991554f2SKenneth D. Merry uint64_t time_in_msec; 1040991554f2SKenneth D. Merry 1041991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 1042991554f2SKenneth D. Merry 1043991554f2SKenneth D. Merry req_sz = sizeof(MPI2_IOC_INIT_REQUEST); 1044991554f2SKenneth D. Merry reply_sz = sizeof(MPI2_IOC_INIT_REPLY); 1045991554f2SKenneth D. Merry bzero(&init, req_sz); 1046991554f2SKenneth D. Merry bzero(&reply, reply_sz); 1047991554f2SKenneth D. Merry 1048991554f2SKenneth D. Merry /* 1049991554f2SKenneth D. Merry * Fill in the init block. Note that most addresses are 1050991554f2SKenneth D. Merry * deliberately in the lower 32bits of memory. This is a micro- 1051991554f2SKenneth D. Merry * optimzation for PCI/PCIX, though it's not clear if it helps PCIe. 1052991554f2SKenneth D. Merry */ 1053991554f2SKenneth D. Merry init.Function = MPI2_FUNCTION_IOC_INIT; 1054991554f2SKenneth D. Merry init.WhoInit = MPI2_WHOINIT_HOST_DRIVER; 1055991554f2SKenneth D. Merry init.MsgVersion = htole16(MPI2_VERSION); 1056991554f2SKenneth D. Merry init.HeaderVersion = htole16(MPI2_HEADER_VERSION); 1057991554f2SKenneth D. Merry init.SystemRequestFrameSize = htole16(sc->facts->IOCRequestFrameSize); 1058991554f2SKenneth D. Merry init.ReplyDescriptorPostQueueDepth = htole16(sc->pqdepth); 1059991554f2SKenneth D. Merry init.ReplyFreeQueueDepth = htole16(sc->fqdepth); 1060991554f2SKenneth D. Merry init.SenseBufferAddressHigh = 0; 1061991554f2SKenneth D. Merry init.SystemReplyAddressHigh = 0; 1062991554f2SKenneth D. Merry init.SystemRequestFrameBaseAddress.High = 0; 1063991554f2SKenneth D. Merry init.SystemRequestFrameBaseAddress.Low = 1064991554f2SKenneth D. Merry htole32((uint32_t)sc->req_busaddr); 1065991554f2SKenneth D. Merry init.ReplyDescriptorPostQueueAddress.High = 0; 1066991554f2SKenneth D. Merry init.ReplyDescriptorPostQueueAddress.Low = 1067991554f2SKenneth D. Merry htole32((uint32_t)sc->post_busaddr); 1068991554f2SKenneth D. Merry init.ReplyFreeQueueAddress.High = 0; 1069991554f2SKenneth D. Merry init.ReplyFreeQueueAddress.Low = htole32((uint32_t)sc->free_busaddr); 1070991554f2SKenneth D. Merry getmicrotime(&now); 1071991554f2SKenneth D. Merry time_in_msec = (now.tv_sec * 1000 + now.tv_usec/1000); 1072991554f2SKenneth D. Merry init.TimeStamp.High = htole32((time_in_msec >> 32) & 0xFFFFFFFF); 1073991554f2SKenneth D. Merry init.TimeStamp.Low = htole32(time_in_msec & 0xFFFFFFFF); 107467feec50SStephen McConnell init.HostPageSize = HOST_PAGE_SIZE_4K; 1075991554f2SKenneth D. Merry 1076991554f2SKenneth D. Merry error = mpr_request_sync(sc, &init, &reply, req_sz, reply_sz, 5); 1077991554f2SKenneth D. Merry if ((reply.IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS) 1078991554f2SKenneth D. Merry error = ENXIO; 1079991554f2SKenneth D. Merry 1080991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INIT, "IOCInit status= 0x%x\n", reply.IOCStatus); 1081991554f2SKenneth D. Merry return (error); 1082991554f2SKenneth D. Merry } 1083991554f2SKenneth D. Merry 1084991554f2SKenneth D. Merry void 1085991554f2SKenneth D. Merry mpr_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 1086991554f2SKenneth D. Merry { 1087991554f2SKenneth D. Merry bus_addr_t *addr; 1088991554f2SKenneth D. Merry 1089991554f2SKenneth D. Merry addr = arg; 1090991554f2SKenneth D. Merry *addr = segs[0].ds_addr; 1091991554f2SKenneth D. Merry } 1092991554f2SKenneth D. Merry 1093991554f2SKenneth D. Merry static int 1094991554f2SKenneth D. Merry mpr_alloc_queues(struct mpr_softc *sc) 1095991554f2SKenneth D. Merry { 1096991554f2SKenneth D. Merry bus_addr_t queues_busaddr; 1097991554f2SKenneth D. Merry uint8_t *queues; 1098991554f2SKenneth D. Merry int qsize, fqsize, pqsize; 1099991554f2SKenneth D. Merry 1100991554f2SKenneth D. Merry /* 1101991554f2SKenneth D. Merry * The reply free queue contains 4 byte entries in multiples of 16 and 1102991554f2SKenneth D. Merry * aligned on a 16 byte boundary. There must always be an unused entry. 1103991554f2SKenneth D. Merry * This queue supplies fresh reply frames for the firmware to use. 1104991554f2SKenneth D. Merry * 1105991554f2SKenneth D. Merry * The reply descriptor post queue contains 8 byte entries in 1106991554f2SKenneth D. Merry * multiples of 16 and aligned on a 16 byte boundary. This queue 1107991554f2SKenneth D. Merry * contains filled-in reply frames sent from the firmware to the host. 1108991554f2SKenneth D. Merry * 1109991554f2SKenneth D. Merry * These two queues are allocated together for simplicity. 1110991554f2SKenneth D. Merry */ 1111d9c9c81cSPedro F. Giffuni sc->fqdepth = roundup2(sc->num_replies + 1, 16); 1112d9c9c81cSPedro F. Giffuni sc->pqdepth = roundup2(sc->num_replies + 1, 16); 1113991554f2SKenneth D. Merry fqsize= sc->fqdepth * 4; 1114991554f2SKenneth D. Merry pqsize = sc->pqdepth * 8; 1115991554f2SKenneth D. Merry qsize = fqsize + pqsize; 1116991554f2SKenneth D. Merry 1117991554f2SKenneth D. Merry if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 1118991554f2SKenneth D. Merry 16, 0, /* algnmnt, boundary */ 1119991554f2SKenneth D. Merry BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ 1120991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* highaddr */ 1121991554f2SKenneth D. Merry NULL, NULL, /* filter, filterarg */ 1122991554f2SKenneth D. Merry qsize, /* maxsize */ 1123991554f2SKenneth D. Merry 1, /* nsegments */ 1124991554f2SKenneth D. Merry qsize, /* maxsegsize */ 1125991554f2SKenneth D. Merry 0, /* flags */ 1126991554f2SKenneth D. Merry NULL, NULL, /* lockfunc, lockarg */ 1127991554f2SKenneth D. Merry &sc->queues_dmat)) { 1128991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate queues DMA tag\n"); 1129991554f2SKenneth D. Merry return (ENOMEM); 1130991554f2SKenneth D. Merry } 1131991554f2SKenneth D. Merry if (bus_dmamem_alloc(sc->queues_dmat, (void **)&queues, BUS_DMA_NOWAIT, 1132991554f2SKenneth D. Merry &sc->queues_map)) { 1133991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate queues memory\n"); 1134991554f2SKenneth D. Merry return (ENOMEM); 1135991554f2SKenneth D. Merry } 1136991554f2SKenneth D. Merry bzero(queues, qsize); 1137991554f2SKenneth D. Merry bus_dmamap_load(sc->queues_dmat, sc->queues_map, queues, qsize, 1138991554f2SKenneth D. Merry mpr_memaddr_cb, &queues_busaddr, 0); 1139991554f2SKenneth D. Merry 1140991554f2SKenneth D. Merry sc->free_queue = (uint32_t *)queues; 1141991554f2SKenneth D. Merry sc->free_busaddr = queues_busaddr; 1142991554f2SKenneth D. Merry sc->post_queue = (MPI2_REPLY_DESCRIPTORS_UNION *)(queues + fqsize); 1143991554f2SKenneth D. Merry sc->post_busaddr = queues_busaddr + fqsize; 1144991554f2SKenneth D. Merry 1145991554f2SKenneth D. Merry return (0); 1146991554f2SKenneth D. Merry } 1147991554f2SKenneth D. Merry 1148991554f2SKenneth D. Merry static int 1149991554f2SKenneth D. Merry mpr_alloc_replies(struct mpr_softc *sc) 1150991554f2SKenneth D. Merry { 1151991554f2SKenneth D. Merry int rsize, num_replies; 1152991554f2SKenneth D. Merry 1153991554f2SKenneth D. Merry /* 1154991554f2SKenneth D. Merry * sc->num_replies should be one less than sc->fqdepth. We need to 1155991554f2SKenneth D. Merry * allocate space for sc->fqdepth replies, but only sc->num_replies 1156991554f2SKenneth D. Merry * replies can be used at once. 1157991554f2SKenneth D. Merry */ 1158991554f2SKenneth D. Merry num_replies = max(sc->fqdepth, sc->num_replies); 1159991554f2SKenneth D. Merry 1160991554f2SKenneth D. Merry rsize = sc->facts->ReplyFrameSize * num_replies * 4; 1161991554f2SKenneth D. Merry if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 1162991554f2SKenneth D. Merry 4, 0, /* algnmnt, boundary */ 1163991554f2SKenneth D. Merry BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ 1164991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* highaddr */ 1165991554f2SKenneth D. Merry NULL, NULL, /* filter, filterarg */ 1166991554f2SKenneth D. Merry rsize, /* maxsize */ 1167991554f2SKenneth D. Merry 1, /* nsegments */ 1168991554f2SKenneth D. Merry rsize, /* maxsegsize */ 1169991554f2SKenneth D. Merry 0, /* flags */ 1170991554f2SKenneth D. Merry NULL, NULL, /* lockfunc, lockarg */ 1171991554f2SKenneth D. Merry &sc->reply_dmat)) { 1172991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate replies DMA tag\n"); 1173991554f2SKenneth D. Merry return (ENOMEM); 1174991554f2SKenneth D. Merry } 1175991554f2SKenneth D. Merry if (bus_dmamem_alloc(sc->reply_dmat, (void **)&sc->reply_frames, 1176991554f2SKenneth D. Merry BUS_DMA_NOWAIT, &sc->reply_map)) { 1177991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate replies memory\n"); 1178991554f2SKenneth D. Merry return (ENOMEM); 1179991554f2SKenneth D. Merry } 1180991554f2SKenneth D. Merry bzero(sc->reply_frames, rsize); 1181991554f2SKenneth D. Merry bus_dmamap_load(sc->reply_dmat, sc->reply_map, sc->reply_frames, rsize, 1182991554f2SKenneth D. Merry mpr_memaddr_cb, &sc->reply_busaddr, 0); 1183991554f2SKenneth D. Merry 1184991554f2SKenneth D. Merry return (0); 1185991554f2SKenneth D. Merry } 1186991554f2SKenneth D. Merry 1187991554f2SKenneth D. Merry static int 1188991554f2SKenneth D. Merry mpr_alloc_requests(struct mpr_softc *sc) 1189991554f2SKenneth D. Merry { 1190991554f2SKenneth D. Merry struct mpr_command *cm; 1191991554f2SKenneth D. Merry struct mpr_chain *chain; 1192991554f2SKenneth D. Merry int i, rsize, nsegs; 1193991554f2SKenneth D. Merry 1194991554f2SKenneth D. Merry rsize = sc->facts->IOCRequestFrameSize * sc->num_reqs * 4; 1195991554f2SKenneth D. Merry if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 1196991554f2SKenneth D. Merry 16, 0, /* algnmnt, boundary */ 1197991554f2SKenneth D. Merry BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ 1198991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* highaddr */ 1199991554f2SKenneth D. Merry NULL, NULL, /* filter, filterarg */ 1200991554f2SKenneth D. Merry rsize, /* maxsize */ 1201991554f2SKenneth D. Merry 1, /* nsegments */ 1202991554f2SKenneth D. Merry rsize, /* maxsegsize */ 1203991554f2SKenneth D. Merry 0, /* flags */ 1204991554f2SKenneth D. Merry NULL, NULL, /* lockfunc, lockarg */ 1205991554f2SKenneth D. Merry &sc->req_dmat)) { 1206991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate request DMA tag\n"); 1207991554f2SKenneth D. Merry return (ENOMEM); 1208991554f2SKenneth D. Merry } 1209991554f2SKenneth D. Merry if (bus_dmamem_alloc(sc->req_dmat, (void **)&sc->req_frames, 1210991554f2SKenneth D. Merry BUS_DMA_NOWAIT, &sc->req_map)) { 1211991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate request memory\n"); 1212991554f2SKenneth D. Merry return (ENOMEM); 1213991554f2SKenneth D. Merry } 1214991554f2SKenneth D. Merry bzero(sc->req_frames, rsize); 1215991554f2SKenneth D. Merry bus_dmamap_load(sc->req_dmat, sc->req_map, sc->req_frames, rsize, 1216991554f2SKenneth D. Merry mpr_memaddr_cb, &sc->req_busaddr, 0); 1217991554f2SKenneth D. Merry 12182bbc5fcbSStephen McConnell /* 12192bbc5fcbSStephen McConnell * Gen3 and beyond uses the IOCMaxChainSegmentSize from IOC Facts to 12202bbc5fcbSStephen McConnell * get the size of a Chain Frame. Previous versions use the size as a 12212bbc5fcbSStephen McConnell * Request Frame for the Chain Frame size. If IOCMaxChainSegmentSize 12222bbc5fcbSStephen McConnell * is 0, use the default value. The IOCMaxChainSegmentSize is the 12232bbc5fcbSStephen McConnell * number of 16-byte elelements that can fit in a Chain Frame, which is 12242bbc5fcbSStephen McConnell * the size of an IEEE Simple SGE. 12252bbc5fcbSStephen McConnell */ 12262bbc5fcbSStephen McConnell if (sc->facts->MsgVersion >= MPI2_VERSION_02_05) { 12272bbc5fcbSStephen McConnell sc->chain_seg_size = 12282bbc5fcbSStephen McConnell htole16(sc->facts->IOCMaxChainSegmentSize); 12292bbc5fcbSStephen McConnell if (sc->chain_seg_size == 0) { 12302bbc5fcbSStephen McConnell sc->chain_frame_size = MPR_DEFAULT_CHAIN_SEG_SIZE * 12312bbc5fcbSStephen McConnell MPR_MAX_CHAIN_ELEMENT_SIZE; 12322bbc5fcbSStephen McConnell } else { 12332bbc5fcbSStephen McConnell sc->chain_frame_size = sc->chain_seg_size * 12342bbc5fcbSStephen McConnell MPR_MAX_CHAIN_ELEMENT_SIZE; 12352bbc5fcbSStephen McConnell } 12362bbc5fcbSStephen McConnell } else { 12372bbc5fcbSStephen McConnell sc->chain_frame_size = sc->facts->IOCRequestFrameSize * 4; 12382bbc5fcbSStephen McConnell } 12392bbc5fcbSStephen McConnell rsize = sc->chain_frame_size * sc->max_chains; 1240991554f2SKenneth D. Merry if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 1241991554f2SKenneth D. Merry 16, 0, /* algnmnt, boundary */ 1242991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* lowaddr */ 1243991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* highaddr */ 1244991554f2SKenneth D. Merry NULL, NULL, /* filter, filterarg */ 1245991554f2SKenneth D. Merry rsize, /* maxsize */ 1246991554f2SKenneth D. Merry 1, /* nsegments */ 1247991554f2SKenneth D. Merry rsize, /* maxsegsize */ 1248991554f2SKenneth D. Merry 0, /* flags */ 1249991554f2SKenneth D. Merry NULL, NULL, /* lockfunc, lockarg */ 1250991554f2SKenneth D. Merry &sc->chain_dmat)) { 1251991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate chain DMA tag\n"); 1252991554f2SKenneth D. Merry return (ENOMEM); 1253991554f2SKenneth D. Merry } 1254991554f2SKenneth D. Merry if (bus_dmamem_alloc(sc->chain_dmat, (void **)&sc->chain_frames, 1255991554f2SKenneth D. Merry BUS_DMA_NOWAIT, &sc->chain_map)) { 1256991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate chain memory\n"); 1257991554f2SKenneth D. Merry return (ENOMEM); 1258991554f2SKenneth D. Merry } 1259991554f2SKenneth D. Merry bzero(sc->chain_frames, rsize); 1260991554f2SKenneth D. Merry bus_dmamap_load(sc->chain_dmat, sc->chain_map, sc->chain_frames, rsize, 1261991554f2SKenneth D. Merry mpr_memaddr_cb, &sc->chain_busaddr, 0); 1262991554f2SKenneth D. Merry 1263991554f2SKenneth D. Merry rsize = MPR_SENSE_LEN * sc->num_reqs; 1264991554f2SKenneth D. Merry if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 1265991554f2SKenneth D. Merry 1, 0, /* algnmnt, boundary */ 1266991554f2SKenneth D. Merry BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ 1267991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* highaddr */ 1268991554f2SKenneth D. Merry NULL, NULL, /* filter, filterarg */ 1269991554f2SKenneth D. Merry rsize, /* maxsize */ 1270991554f2SKenneth D. Merry 1, /* nsegments */ 1271991554f2SKenneth D. Merry rsize, /* maxsegsize */ 1272991554f2SKenneth D. Merry 0, /* flags */ 1273991554f2SKenneth D. Merry NULL, NULL, /* lockfunc, lockarg */ 1274991554f2SKenneth D. Merry &sc->sense_dmat)) { 1275991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate sense DMA tag\n"); 1276991554f2SKenneth D. Merry return (ENOMEM); 1277991554f2SKenneth D. Merry } 1278991554f2SKenneth D. Merry if (bus_dmamem_alloc(sc->sense_dmat, (void **)&sc->sense_frames, 1279991554f2SKenneth D. Merry BUS_DMA_NOWAIT, &sc->sense_map)) { 1280991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate sense memory\n"); 1281991554f2SKenneth D. Merry return (ENOMEM); 1282991554f2SKenneth D. Merry } 1283991554f2SKenneth D. Merry bzero(sc->sense_frames, rsize); 1284991554f2SKenneth D. Merry bus_dmamap_load(sc->sense_dmat, sc->sense_map, sc->sense_frames, rsize, 1285991554f2SKenneth D. Merry mpr_memaddr_cb, &sc->sense_busaddr, 0); 1286991554f2SKenneth D. Merry 1287991554f2SKenneth D. Merry sc->chains = malloc(sizeof(struct mpr_chain) * sc->max_chains, M_MPR, 1288991554f2SKenneth D. Merry M_WAITOK | M_ZERO); 1289991554f2SKenneth D. Merry if (!sc->chains) { 1290991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate memory %s %d\n", 1291991554f2SKenneth D. Merry __func__, __LINE__); 1292991554f2SKenneth D. Merry return (ENOMEM); 1293991554f2SKenneth D. Merry } 1294991554f2SKenneth D. Merry for (i = 0; i < sc->max_chains; i++) { 1295991554f2SKenneth D. Merry chain = &sc->chains[i]; 1296991554f2SKenneth D. Merry chain->chain = (MPI2_SGE_IO_UNION *)(sc->chain_frames + 12972bbc5fcbSStephen McConnell i * sc->chain_frame_size); 1298991554f2SKenneth D. Merry chain->chain_busaddr = sc->chain_busaddr + 12992bbc5fcbSStephen McConnell i * sc->chain_frame_size; 1300991554f2SKenneth D. Merry mpr_free_chain(sc, chain); 1301991554f2SKenneth D. Merry sc->chain_free_lowwater++; 1302991554f2SKenneth D. Merry } 1303991554f2SKenneth D. Merry 130467feec50SStephen McConnell /* 130567feec50SStephen McConnell * Allocate NVMe PRP Pages for NVMe SGL support only if the FW supports 130667feec50SStephen McConnell * these devices. 130767feec50SStephen McConnell */ 130867feec50SStephen McConnell if ((sc->facts->MsgVersion >= MPI2_VERSION_02_06) && 130967feec50SStephen McConnell (sc->facts->ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES)) { 131067feec50SStephen McConnell if (mpr_alloc_nvme_prp_pages(sc) == ENOMEM) 131167feec50SStephen McConnell return (ENOMEM); 131267feec50SStephen McConnell } 131367feec50SStephen McConnell 1314991554f2SKenneth D. Merry /* XXX Need to pick a more precise value */ 1315991554f2SKenneth D. Merry nsegs = (MAXPHYS / PAGE_SIZE) + 1; 1316991554f2SKenneth D. Merry if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 1317991554f2SKenneth D. Merry 1, 0, /* algnmnt, boundary */ 1318991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* lowaddr */ 1319991554f2SKenneth D. Merry BUS_SPACE_MAXADDR, /* highaddr */ 1320991554f2SKenneth D. Merry NULL, NULL, /* filter, filterarg */ 1321991554f2SKenneth D. Merry BUS_SPACE_MAXSIZE_32BIT,/* maxsize */ 1322991554f2SKenneth D. Merry nsegs, /* nsegments */ 1323991554f2SKenneth D. Merry BUS_SPACE_MAXSIZE_32BIT,/* maxsegsize */ 1324991554f2SKenneth D. Merry BUS_DMA_ALLOCNOW, /* flags */ 1325991554f2SKenneth D. Merry busdma_lock_mutex, /* lockfunc */ 1326991554f2SKenneth D. Merry &sc->mpr_mtx, /* lockarg */ 1327991554f2SKenneth D. Merry &sc->buffer_dmat)) { 1328991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate buffer DMA tag\n"); 1329991554f2SKenneth D. Merry return (ENOMEM); 1330991554f2SKenneth D. Merry } 1331991554f2SKenneth D. Merry 1332991554f2SKenneth D. Merry /* 1333991554f2SKenneth D. Merry * SMID 0 cannot be used as a free command per the firmware spec. 1334991554f2SKenneth D. Merry * Just drop that command instead of risking accounting bugs. 1335991554f2SKenneth D. Merry */ 1336991554f2SKenneth D. Merry sc->commands = malloc(sizeof(struct mpr_command) * sc->num_reqs, 1337991554f2SKenneth D. Merry M_MPR, M_WAITOK | M_ZERO); 1338991554f2SKenneth D. Merry if (!sc->commands) { 1339991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate memory %s %d\n", 1340991554f2SKenneth D. Merry __func__, __LINE__); 1341991554f2SKenneth D. Merry return (ENOMEM); 1342991554f2SKenneth D. Merry } 1343991554f2SKenneth D. Merry for (i = 1; i < sc->num_reqs; i++) { 1344991554f2SKenneth D. Merry cm = &sc->commands[i]; 1345991554f2SKenneth D. Merry cm->cm_req = sc->req_frames + 1346991554f2SKenneth D. Merry i * sc->facts->IOCRequestFrameSize * 4; 1347991554f2SKenneth D. Merry cm->cm_req_busaddr = sc->req_busaddr + 1348991554f2SKenneth D. Merry i * sc->facts->IOCRequestFrameSize * 4; 1349991554f2SKenneth D. Merry cm->cm_sense = &sc->sense_frames[i]; 1350991554f2SKenneth D. Merry cm->cm_sense_busaddr = sc->sense_busaddr + i * MPR_SENSE_LEN; 1351991554f2SKenneth D. Merry cm->cm_desc.Default.SMID = i; 1352991554f2SKenneth D. Merry cm->cm_sc = sc; 1353991554f2SKenneth D. Merry TAILQ_INIT(&cm->cm_chain_list); 135467feec50SStephen McConnell TAILQ_INIT(&cm->cm_prp_page_list); 1355991554f2SKenneth D. Merry callout_init_mtx(&cm->cm_callout, &sc->mpr_mtx, 0); 1356991554f2SKenneth D. Merry 1357991554f2SKenneth D. Merry /* XXX Is a failure here a critical problem? */ 135867feec50SStephen McConnell if (bus_dmamap_create(sc->buffer_dmat, 0, &cm->cm_dmamap) 135967feec50SStephen McConnell == 0) { 1360991554f2SKenneth D. Merry if (i <= sc->facts->HighPriorityCredit) 1361991554f2SKenneth D. Merry mpr_free_high_priority_command(sc, cm); 1362991554f2SKenneth D. Merry else 1363991554f2SKenneth D. Merry mpr_free_command(sc, cm); 136467feec50SStephen McConnell } else { 1365991554f2SKenneth D. Merry panic("failed to allocate command %d\n", i); 1366991554f2SKenneth D. Merry sc->num_reqs = i; 1367991554f2SKenneth D. Merry break; 1368991554f2SKenneth D. Merry } 1369991554f2SKenneth D. Merry } 1370991554f2SKenneth D. Merry 1371991554f2SKenneth D. Merry return (0); 1372991554f2SKenneth D. Merry } 1373991554f2SKenneth D. Merry 137467feec50SStephen McConnell /* 137567feec50SStephen McConnell * Allocate contiguous buffers for PCIe NVMe devices for building native PRPs, 137667feec50SStephen McConnell * which are scatter/gather lists for NVMe devices. 137767feec50SStephen McConnell * 137867feec50SStephen McConnell * This buffer must be contiguous due to the nature of how NVMe PRPs are built 137967feec50SStephen McConnell * and translated by FW. 138067feec50SStephen McConnell * 138167feec50SStephen McConnell * returns ENOMEM if memory could not be allocated, otherwise returns 0. 138267feec50SStephen McConnell */ 138367feec50SStephen McConnell static int 138467feec50SStephen McConnell mpr_alloc_nvme_prp_pages(struct mpr_softc *sc) 138567feec50SStephen McConnell { 138667feec50SStephen McConnell int PRPs_per_page, PRPs_required, pages_required; 138767feec50SStephen McConnell int rsize, i; 138867feec50SStephen McConnell struct mpr_prp_page *prp_page; 138967feec50SStephen McConnell 139067feec50SStephen McConnell /* 139167feec50SStephen McConnell * Assuming a MAX_IO_SIZE of 1MB and a PAGE_SIZE of 4k, the max number 139267feec50SStephen McConnell * of PRPs (NVMe's Scatter/Gather Element) needed per I/O is: 139367feec50SStephen McConnell * MAX_IO_SIZE / PAGE_SIZE = 256 139467feec50SStephen McConnell * 139567feec50SStephen McConnell * 1 PRP entry in main frame for PRP list pointer still leaves 255 PRPs 139667feec50SStephen McConnell * required for the remainder of the 1MB I/O. 512 PRPs can fit into one 139767feec50SStephen McConnell * page (4096 / 8 = 512), so only one page is required for each I/O. 139867feec50SStephen McConnell * 139967feec50SStephen McConnell * Each of these buffers will need to be contiguous. For simplicity, 140067feec50SStephen McConnell * only one buffer is allocated here, which has all of the space 140167feec50SStephen McConnell * required for the NVMe Queue Depth. If there are problems allocating 140267feec50SStephen McConnell * this one buffer, this function will need to change to allocate 140367feec50SStephen McConnell * individual, contiguous NVME_QDEPTH buffers. 140467feec50SStephen McConnell * 140567feec50SStephen McConnell * The real calculation will use the real max io size. Above is just an 140667feec50SStephen McConnell * example. 140767feec50SStephen McConnell * 140867feec50SStephen McConnell */ 140967feec50SStephen McConnell PRPs_required = sc->maxio / PAGE_SIZE; 141067feec50SStephen McConnell PRPs_per_page = (PAGE_SIZE / PRP_ENTRY_SIZE) - 1; 141167feec50SStephen McConnell pages_required = (PRPs_required / PRPs_per_page) + 1; 141267feec50SStephen McConnell 141367feec50SStephen McConnell sc->prp_buffer_size = PAGE_SIZE * pages_required; 141467feec50SStephen McConnell rsize = sc->prp_buffer_size * NVME_QDEPTH; 141567feec50SStephen McConnell if (bus_dma_tag_create( sc->mpr_parent_dmat, /* parent */ 141667feec50SStephen McConnell 4, 0, /* algnmnt, boundary */ 141767feec50SStephen McConnell BUS_SPACE_MAXADDR_32BIT,/* lowaddr */ 141867feec50SStephen McConnell BUS_SPACE_MAXADDR, /* highaddr */ 141967feec50SStephen McConnell NULL, NULL, /* filter, filterarg */ 142067feec50SStephen McConnell rsize, /* maxsize */ 142167feec50SStephen McConnell 1, /* nsegments */ 142267feec50SStephen McConnell rsize, /* maxsegsize */ 142367feec50SStephen McConnell 0, /* flags */ 142467feec50SStephen McConnell NULL, NULL, /* lockfunc, lockarg */ 142567feec50SStephen McConnell &sc->prp_page_dmat)) { 142667feec50SStephen McConnell device_printf(sc->mpr_dev, "Cannot allocate NVMe PRP DMA " 142767feec50SStephen McConnell "tag\n"); 142867feec50SStephen McConnell return (ENOMEM); 142967feec50SStephen McConnell } 143067feec50SStephen McConnell if (bus_dmamem_alloc(sc->prp_page_dmat, (void **)&sc->prp_pages, 143167feec50SStephen McConnell BUS_DMA_NOWAIT, &sc->prp_page_map)) { 143267feec50SStephen McConnell device_printf(sc->mpr_dev, "Cannot allocate NVMe PRP memory\n"); 143367feec50SStephen McConnell return (ENOMEM); 143467feec50SStephen McConnell } 143567feec50SStephen McConnell bzero(sc->prp_pages, rsize); 143667feec50SStephen McConnell bus_dmamap_load(sc->prp_page_dmat, sc->prp_page_map, sc->prp_pages, 143767feec50SStephen McConnell rsize, mpr_memaddr_cb, &sc->prp_page_busaddr, 0); 143867feec50SStephen McConnell 143967feec50SStephen McConnell sc->prps = malloc(sizeof(struct mpr_prp_page) * NVME_QDEPTH, M_MPR, 144067feec50SStephen McConnell M_WAITOK | M_ZERO); 144167feec50SStephen McConnell for (i = 0; i < NVME_QDEPTH; i++) { 144267feec50SStephen McConnell prp_page = &sc->prps[i]; 144367feec50SStephen McConnell prp_page->prp_page = (uint64_t *)(sc->prp_pages + 144467feec50SStephen McConnell i * sc->prp_buffer_size); 144567feec50SStephen McConnell prp_page->prp_page_busaddr = (uint64_t)(sc->prp_page_busaddr + 144667feec50SStephen McConnell i * sc->prp_buffer_size); 144767feec50SStephen McConnell mpr_free_prp_page(sc, prp_page); 144867feec50SStephen McConnell sc->prp_pages_free_lowwater++; 144967feec50SStephen McConnell } 145067feec50SStephen McConnell 145167feec50SStephen McConnell return (0); 145267feec50SStephen McConnell } 145367feec50SStephen McConnell 1454991554f2SKenneth D. Merry static int 1455991554f2SKenneth D. Merry mpr_init_queues(struct mpr_softc *sc) 1456991554f2SKenneth D. Merry { 1457991554f2SKenneth D. Merry int i; 1458991554f2SKenneth D. Merry 1459991554f2SKenneth D. Merry memset((uint8_t *)sc->post_queue, 0xff, sc->pqdepth * 8); 1460991554f2SKenneth D. Merry 1461991554f2SKenneth D. Merry /* 1462991554f2SKenneth D. Merry * According to the spec, we need to use one less reply than we 1463991554f2SKenneth D. Merry * have space for on the queue. So sc->num_replies (the number we 1464991554f2SKenneth D. Merry * use) should be less than sc->fqdepth (allocated size). 1465991554f2SKenneth D. Merry */ 1466991554f2SKenneth D. Merry if (sc->num_replies >= sc->fqdepth) 1467991554f2SKenneth D. Merry return (EINVAL); 1468991554f2SKenneth D. Merry 1469991554f2SKenneth D. Merry /* 1470991554f2SKenneth D. Merry * Initialize all of the free queue entries. 1471991554f2SKenneth D. Merry */ 147267feec50SStephen McConnell for (i = 0; i < sc->fqdepth; i++) { 147367feec50SStephen McConnell sc->free_queue[i] = sc->reply_busaddr + 147467feec50SStephen McConnell (i * sc->facts->ReplyFrameSize * 4); 147567feec50SStephen McConnell } 1476991554f2SKenneth D. Merry sc->replyfreeindex = sc->num_replies; 1477991554f2SKenneth D. Merry 1478991554f2SKenneth D. Merry return (0); 1479991554f2SKenneth D. Merry } 1480991554f2SKenneth D. Merry 1481991554f2SKenneth D. Merry /* Get the driver parameter tunables. Lowest priority are the driver defaults. 1482991554f2SKenneth D. Merry * Next are the global settings, if they exist. Highest are the per-unit 1483991554f2SKenneth D. Merry * settings, if they exist. 1484991554f2SKenneth D. Merry */ 1485*252b2b4fSScott Long void 1486991554f2SKenneth D. Merry mpr_get_tunables(struct mpr_softc *sc) 1487991554f2SKenneth D. Merry { 1488991554f2SKenneth D. Merry char tmpstr[80]; 1489991554f2SKenneth D. Merry 1490991554f2SKenneth D. Merry /* XXX default to some debugging for now */ 1491991554f2SKenneth D. Merry sc->mpr_debug = MPR_INFO | MPR_FAULT; 1492991554f2SKenneth D. Merry sc->disable_msix = 0; 1493991554f2SKenneth D. Merry sc->disable_msi = 0; 1494991554f2SKenneth D. Merry sc->max_chains = MPR_CHAIN_FRAMES; 149532b0a21eSStephen McConnell sc->max_io_pages = MPR_MAXIO_PAGES; 1496a2c14879SStephen McConnell sc->enable_ssu = MPR_SSU_ENABLE_SSD_DISABLE_HDD; 1497a2c14879SStephen McConnell sc->spinup_wait_time = DEFAULT_SPINUP_WAIT; 14984ab1cdc5SScott Long sc->use_phynum = 1; 1499991554f2SKenneth D. Merry 1500991554f2SKenneth D. Merry /* 1501991554f2SKenneth D. Merry * Grab the global variables. 1502991554f2SKenneth D. Merry */ 1503991554f2SKenneth D. Merry TUNABLE_INT_FETCH("hw.mpr.debug_level", &sc->mpr_debug); 1504991554f2SKenneth D. Merry TUNABLE_INT_FETCH("hw.mpr.disable_msix", &sc->disable_msix); 1505991554f2SKenneth D. Merry TUNABLE_INT_FETCH("hw.mpr.disable_msi", &sc->disable_msi); 1506991554f2SKenneth D. Merry TUNABLE_INT_FETCH("hw.mpr.max_chains", &sc->max_chains); 150732b0a21eSStephen McConnell TUNABLE_INT_FETCH("hw.mpr.max_io_pages", &sc->max_io_pages); 1508a2c14879SStephen McConnell TUNABLE_INT_FETCH("hw.mpr.enable_ssu", &sc->enable_ssu); 1509a2c14879SStephen McConnell TUNABLE_INT_FETCH("hw.mpr.spinup_wait_time", &sc->spinup_wait_time); 15104ab1cdc5SScott Long TUNABLE_INT_FETCH("hw.mpr.use_phy_num", &sc->use_phynum); 1511991554f2SKenneth D. Merry 1512991554f2SKenneth D. Merry /* Grab the unit-instance variables */ 1513991554f2SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.debug_level", 1514991554f2SKenneth D. Merry device_get_unit(sc->mpr_dev)); 1515991554f2SKenneth D. Merry TUNABLE_INT_FETCH(tmpstr, &sc->mpr_debug); 1516991554f2SKenneth D. Merry 1517991554f2SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.disable_msix", 1518991554f2SKenneth D. Merry device_get_unit(sc->mpr_dev)); 1519991554f2SKenneth D. Merry TUNABLE_INT_FETCH(tmpstr, &sc->disable_msix); 1520991554f2SKenneth D. Merry 1521991554f2SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.disable_msi", 1522991554f2SKenneth D. Merry device_get_unit(sc->mpr_dev)); 1523991554f2SKenneth D. Merry TUNABLE_INT_FETCH(tmpstr, &sc->disable_msi); 1524991554f2SKenneth D. Merry 1525991554f2SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_chains", 1526991554f2SKenneth D. Merry device_get_unit(sc->mpr_dev)); 1527991554f2SKenneth D. Merry TUNABLE_INT_FETCH(tmpstr, &sc->max_chains); 1528991554f2SKenneth D. Merry 152932b0a21eSStephen McConnell snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.max_io_pages", 153032b0a21eSStephen McConnell device_get_unit(sc->mpr_dev)); 153132b0a21eSStephen McConnell TUNABLE_INT_FETCH(tmpstr, &sc->max_io_pages); 153232b0a21eSStephen McConnell 1533991554f2SKenneth D. Merry bzero(sc->exclude_ids, sizeof(sc->exclude_ids)); 1534991554f2SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.exclude_ids", 1535991554f2SKenneth D. Merry device_get_unit(sc->mpr_dev)); 1536991554f2SKenneth D. Merry TUNABLE_STR_FETCH(tmpstr, sc->exclude_ids, sizeof(sc->exclude_ids)); 1537a2c14879SStephen McConnell 1538a2c14879SStephen McConnell snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.enable_ssu", 1539a2c14879SStephen McConnell device_get_unit(sc->mpr_dev)); 1540a2c14879SStephen McConnell TUNABLE_INT_FETCH(tmpstr, &sc->enable_ssu); 1541a2c14879SStephen McConnell 1542a2c14879SStephen McConnell snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.spinup_wait_time", 1543a2c14879SStephen McConnell device_get_unit(sc->mpr_dev)); 1544a2c14879SStephen McConnell TUNABLE_INT_FETCH(tmpstr, &sc->spinup_wait_time); 15454ab1cdc5SScott Long 15464ab1cdc5SScott Long snprintf(tmpstr, sizeof(tmpstr), "dev.mpr.%d.use_phy_num", 15474ab1cdc5SScott Long device_get_unit(sc->mpr_dev)); 15484ab1cdc5SScott Long TUNABLE_INT_FETCH(tmpstr, &sc->use_phynum); 1549991554f2SKenneth D. Merry } 1550991554f2SKenneth D. Merry 1551991554f2SKenneth D. Merry static void 1552991554f2SKenneth D. Merry mpr_setup_sysctl(struct mpr_softc *sc) 1553991554f2SKenneth D. Merry { 1554991554f2SKenneth D. Merry struct sysctl_ctx_list *sysctl_ctx = NULL; 1555991554f2SKenneth D. Merry struct sysctl_oid *sysctl_tree = NULL; 1556991554f2SKenneth D. Merry char tmpstr[80], tmpstr2[80]; 1557991554f2SKenneth D. Merry 1558991554f2SKenneth D. Merry /* 1559991554f2SKenneth D. Merry * Setup the sysctl variable so the user can change the debug level 1560991554f2SKenneth D. Merry * on the fly. 1561991554f2SKenneth D. Merry */ 1562991554f2SKenneth D. Merry snprintf(tmpstr, sizeof(tmpstr), "MPR controller %d", 1563991554f2SKenneth D. Merry device_get_unit(sc->mpr_dev)); 1564991554f2SKenneth D. Merry snprintf(tmpstr2, sizeof(tmpstr2), "%d", device_get_unit(sc->mpr_dev)); 1565991554f2SKenneth D. Merry 1566991554f2SKenneth D. Merry sysctl_ctx = device_get_sysctl_ctx(sc->mpr_dev); 1567991554f2SKenneth D. Merry if (sysctl_ctx != NULL) 1568991554f2SKenneth D. Merry sysctl_tree = device_get_sysctl_tree(sc->mpr_dev); 1569991554f2SKenneth D. Merry 1570991554f2SKenneth D. Merry if (sysctl_tree == NULL) { 1571991554f2SKenneth D. Merry sysctl_ctx_init(&sc->sysctl_ctx); 1572991554f2SKenneth D. Merry sc->sysctl_tree = SYSCTL_ADD_NODE(&sc->sysctl_ctx, 1573991554f2SKenneth D. Merry SYSCTL_STATIC_CHILDREN(_hw_mpr), OID_AUTO, tmpstr2, 1574991554f2SKenneth D. Merry CTLFLAG_RD, 0, tmpstr); 1575991554f2SKenneth D. Merry if (sc->sysctl_tree == NULL) 1576991554f2SKenneth D. Merry return; 1577991554f2SKenneth D. Merry sysctl_ctx = &sc->sysctl_ctx; 1578991554f2SKenneth D. Merry sysctl_tree = sc->sysctl_tree; 1579991554f2SKenneth D. Merry } 1580991554f2SKenneth D. Merry 1581991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1582991554f2SKenneth D. Merry OID_AUTO, "debug_level", CTLFLAG_RW, &sc->mpr_debug, 0, 1583991554f2SKenneth D. Merry "mpr debug level"); 1584991554f2SKenneth D. Merry 1585991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1586991554f2SKenneth D. Merry OID_AUTO, "disable_msix", CTLFLAG_RD, &sc->disable_msix, 0, 1587991554f2SKenneth D. Merry "Disable the use of MSI-X interrupts"); 1588991554f2SKenneth D. Merry 1589991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1590991554f2SKenneth D. Merry OID_AUTO, "disable_msi", CTLFLAG_RD, &sc->disable_msi, 0, 1591991554f2SKenneth D. Merry "Disable the use of MSI interrupts"); 1592991554f2SKenneth D. Merry 1593991554f2SKenneth D. Merry SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1594f0188618SHans Petter Selasky OID_AUTO, "firmware_version", CTLFLAG_RW, sc->fw_version, 1595991554f2SKenneth D. Merry strlen(sc->fw_version), "firmware version"); 1596991554f2SKenneth D. Merry 1597991554f2SKenneth D. Merry SYSCTL_ADD_STRING(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1598991554f2SKenneth D. Merry OID_AUTO, "driver_version", CTLFLAG_RW, MPR_DRIVER_VERSION, 1599991554f2SKenneth D. Merry strlen(MPR_DRIVER_VERSION), "driver version"); 1600991554f2SKenneth D. Merry 1601991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1602991554f2SKenneth D. Merry OID_AUTO, "io_cmds_active", CTLFLAG_RD, 1603991554f2SKenneth D. Merry &sc->io_cmds_active, 0, "number of currently active commands"); 1604991554f2SKenneth D. Merry 1605991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1606991554f2SKenneth D. Merry OID_AUTO, "io_cmds_highwater", CTLFLAG_RD, 1607991554f2SKenneth D. Merry &sc->io_cmds_highwater, 0, "maximum active commands seen"); 1608991554f2SKenneth D. Merry 1609991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1610991554f2SKenneth D. Merry OID_AUTO, "chain_free", CTLFLAG_RD, 1611991554f2SKenneth D. Merry &sc->chain_free, 0, "number of free chain elements"); 1612991554f2SKenneth D. Merry 1613991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1614991554f2SKenneth D. Merry OID_AUTO, "chain_free_lowwater", CTLFLAG_RD, 1615991554f2SKenneth D. Merry &sc->chain_free_lowwater, 0,"lowest number of free chain elements"); 1616991554f2SKenneth D. Merry 1617991554f2SKenneth D. Merry SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1618991554f2SKenneth D. Merry OID_AUTO, "max_chains", CTLFLAG_RD, 1619991554f2SKenneth D. Merry &sc->max_chains, 0,"maximum chain frames that will be allocated"); 1620991554f2SKenneth D. Merry 1621a2c14879SStephen McConnell SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 162232b0a21eSStephen McConnell OID_AUTO, "max_io_pages", CTLFLAG_RD, 162332b0a21eSStephen McConnell &sc->max_io_pages, 0,"maximum pages to allow per I/O (if <1 use " 162432b0a21eSStephen McConnell "IOCFacts)"); 162532b0a21eSStephen McConnell 162632b0a21eSStephen McConnell SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1627a2c14879SStephen McConnell OID_AUTO, "enable_ssu", CTLFLAG_RW, &sc->enable_ssu, 0, 1628a2c14879SStephen McConnell "enable SSU to SATA SSD/HDD at shutdown"); 1629a2c14879SStephen McConnell 1630991554f2SKenneth D. Merry SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1631991554f2SKenneth D. Merry OID_AUTO, "chain_alloc_fail", CTLFLAG_RD, 1632991554f2SKenneth D. Merry &sc->chain_alloc_fail, "chain allocation failures"); 1633a2c14879SStephen McConnell 1634a2c14879SStephen McConnell SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1635a2c14879SStephen McConnell OID_AUTO, "spinup_wait_time", CTLFLAG_RD, 1636a2c14879SStephen McConnell &sc->spinup_wait_time, DEFAULT_SPINUP_WAIT, "seconds to wait for " 1637a2c14879SStephen McConnell "spinup after SATA ID error"); 16384ab1cdc5SScott Long 16394ab1cdc5SScott Long SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 16404ab1cdc5SScott Long OID_AUTO, "use_phy_num", CTLFLAG_RD, &sc->use_phynum, 0, 16414ab1cdc5SScott Long "Use the phy number for enumeration"); 164267feec50SStephen McConnell 164367feec50SStephen McConnell SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 164467feec50SStephen McConnell OID_AUTO, "prp_pages_free", CTLFLAG_RD, 164567feec50SStephen McConnell &sc->prp_pages_free, 0, "number of free PRP pages"); 164667feec50SStephen McConnell 164767feec50SStephen McConnell SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 164867feec50SStephen McConnell OID_AUTO, "prp_pages_free_lowwater", CTLFLAG_RD, 164967feec50SStephen McConnell &sc->prp_pages_free_lowwater, 0,"lowest number of free PRP pages"); 165067feec50SStephen McConnell 165167feec50SStephen McConnell SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 165267feec50SStephen McConnell OID_AUTO, "prp_page_alloc_fail", CTLFLAG_RD, 165367feec50SStephen McConnell &sc->prp_page_alloc_fail, "PRP page allocation failures"); 1654991554f2SKenneth D. Merry } 1655991554f2SKenneth D. Merry 1656991554f2SKenneth D. Merry int 1657991554f2SKenneth D. Merry mpr_attach(struct mpr_softc *sc) 1658991554f2SKenneth D. Merry { 1659991554f2SKenneth D. Merry int error; 1660991554f2SKenneth D. Merry 1661991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 1662991554f2SKenneth D. Merry 1663991554f2SKenneth D. Merry mtx_init(&sc->mpr_mtx, "MPR lock", NULL, MTX_DEF); 1664991554f2SKenneth D. Merry callout_init_mtx(&sc->periodic, &sc->mpr_mtx, 0); 1665327f2e6cSStephen McConnell callout_init_mtx(&sc->device_check_callout, &sc->mpr_mtx, 0); 1666991554f2SKenneth D. Merry TAILQ_INIT(&sc->event_list); 1667991554f2SKenneth D. Merry timevalclear(&sc->lastfail); 1668991554f2SKenneth D. Merry 1669991554f2SKenneth D. Merry if ((error = mpr_transition_ready(sc)) != 0) { 1670991554f2SKenneth D. Merry mpr_printf(sc, "%s failed to transition ready\n", __func__); 1671991554f2SKenneth D. Merry return (error); 1672991554f2SKenneth D. Merry } 1673991554f2SKenneth D. Merry 1674991554f2SKenneth D. Merry sc->facts = malloc(sizeof(MPI2_IOC_FACTS_REPLY), M_MPR, 1675991554f2SKenneth D. Merry M_ZERO|M_NOWAIT); 1676991554f2SKenneth D. Merry if (!sc->facts) { 1677991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate memory %s %d\n", 1678991554f2SKenneth D. Merry __func__, __LINE__); 1679991554f2SKenneth D. Merry return (ENOMEM); 1680991554f2SKenneth D. Merry } 1681991554f2SKenneth D. Merry 1682991554f2SKenneth D. Merry /* 1683991554f2SKenneth D. Merry * Get IOC Facts and allocate all structures based on this information. 1684991554f2SKenneth D. Merry * A Diag Reset will also call mpr_iocfacts_allocate and re-read the IOC 1685991554f2SKenneth D. Merry * Facts. If relevant values have changed in IOC Facts, this function 1686991554f2SKenneth D. Merry * will free all of the memory based on IOC Facts and reallocate that 1687991554f2SKenneth D. Merry * memory. If this fails, any allocated memory should already be freed. 1688991554f2SKenneth D. Merry */ 1689991554f2SKenneth D. Merry if ((error = mpr_iocfacts_allocate(sc, TRUE)) != 0) { 1690991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "%s IOC Facts based allocation " 1691991554f2SKenneth D. Merry "failed with error %d\n", __func__, error); 1692991554f2SKenneth D. Merry return (error); 1693991554f2SKenneth D. Merry } 1694991554f2SKenneth D. Merry 1695991554f2SKenneth D. Merry /* Start the periodic watchdog check on the IOC Doorbell */ 1696991554f2SKenneth D. Merry mpr_periodic(sc); 1697991554f2SKenneth D. Merry 1698991554f2SKenneth D. Merry /* 1699991554f2SKenneth D. Merry * The portenable will kick off discovery events that will drive the 1700991554f2SKenneth D. Merry * rest of the initialization process. The CAM/SAS module will 1701991554f2SKenneth D. Merry * hold up the boot sequence until discovery is complete. 1702991554f2SKenneth D. Merry */ 1703991554f2SKenneth D. Merry sc->mpr_ich.ich_func = mpr_startup; 1704991554f2SKenneth D. Merry sc->mpr_ich.ich_arg = sc; 1705991554f2SKenneth D. Merry if (config_intrhook_establish(&sc->mpr_ich) != 0) { 1706991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "Cannot establish MPR config hook\n"); 1707991554f2SKenneth D. Merry error = EINVAL; 1708991554f2SKenneth D. Merry } 1709991554f2SKenneth D. Merry 1710991554f2SKenneth D. Merry /* 1711991554f2SKenneth D. Merry * Allow IR to shutdown gracefully when shutdown occurs. 1712991554f2SKenneth D. Merry */ 1713991554f2SKenneth D. Merry sc->shutdown_eh = EVENTHANDLER_REGISTER(shutdown_final, 1714991554f2SKenneth D. Merry mprsas_ir_shutdown, sc, SHUTDOWN_PRI_DEFAULT); 1715991554f2SKenneth D. Merry 1716991554f2SKenneth D. Merry if (sc->shutdown_eh == NULL) 1717991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "shutdown event registration " 1718991554f2SKenneth D. Merry "failed\n"); 1719991554f2SKenneth D. Merry 1720991554f2SKenneth D. Merry mpr_setup_sysctl(sc); 1721991554f2SKenneth D. Merry 1722991554f2SKenneth D. Merry sc->mpr_flags |= MPR_FLAGS_ATTACH_DONE; 1723991554f2SKenneth D. Merry 1724991554f2SKenneth D. Merry return (error); 1725991554f2SKenneth D. Merry } 1726991554f2SKenneth D. Merry 1727991554f2SKenneth D. Merry /* Run through any late-start handlers. */ 1728991554f2SKenneth D. Merry static void 1729991554f2SKenneth D. Merry mpr_startup(void *arg) 1730991554f2SKenneth D. Merry { 1731991554f2SKenneth D. Merry struct mpr_softc *sc; 1732991554f2SKenneth D. Merry 1733991554f2SKenneth D. Merry sc = (struct mpr_softc *)arg; 1734991554f2SKenneth D. Merry 1735991554f2SKenneth D. Merry mpr_lock(sc); 1736991554f2SKenneth D. Merry mpr_unmask_intr(sc); 1737991554f2SKenneth D. Merry 1738991554f2SKenneth D. Merry /* initialize device mapping tables */ 1739991554f2SKenneth D. Merry mpr_base_static_config_pages(sc); 1740991554f2SKenneth D. Merry mpr_mapping_initialize(sc); 1741991554f2SKenneth D. Merry mprsas_startup(sc); 1742991554f2SKenneth D. Merry mpr_unlock(sc); 1743991554f2SKenneth D. Merry } 1744991554f2SKenneth D. Merry 1745991554f2SKenneth D. Merry /* Periodic watchdog. Is called with the driver lock already held. */ 1746991554f2SKenneth D. Merry static void 1747991554f2SKenneth D. Merry mpr_periodic(void *arg) 1748991554f2SKenneth D. Merry { 1749991554f2SKenneth D. Merry struct mpr_softc *sc; 1750991554f2SKenneth D. Merry uint32_t db; 1751991554f2SKenneth D. Merry 1752991554f2SKenneth D. Merry sc = (struct mpr_softc *)arg; 1753991554f2SKenneth D. Merry if (sc->mpr_flags & MPR_FLAGS_SHUTDOWN) 1754991554f2SKenneth D. Merry return; 1755991554f2SKenneth D. Merry 1756991554f2SKenneth D. Merry db = mpr_regread(sc, MPI2_DOORBELL_OFFSET); 1757991554f2SKenneth D. Merry if ((db & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) { 1758991554f2SKenneth D. Merry if ((db & MPI2_DOORBELL_FAULT_CODE_MASK) == 1759991554f2SKenneth D. Merry IFAULT_IOP_OVER_TEMP_THRESHOLD_EXCEEDED) { 1760991554f2SKenneth D. Merry panic("TEMPERATURE FAULT: STOPPING."); 1761991554f2SKenneth D. Merry } 1762991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "IOC Fault 0x%08x, Resetting\n", db); 1763991554f2SKenneth D. Merry mpr_reinit(sc); 1764991554f2SKenneth D. Merry } 1765991554f2SKenneth D. Merry 1766991554f2SKenneth D. Merry callout_reset(&sc->periodic, MPR_PERIODIC_DELAY * hz, mpr_periodic, sc); 1767991554f2SKenneth D. Merry } 1768991554f2SKenneth D. Merry 1769991554f2SKenneth D. Merry static void 1770991554f2SKenneth D. Merry mpr_log_evt_handler(struct mpr_softc *sc, uintptr_t data, 1771991554f2SKenneth D. Merry MPI2_EVENT_NOTIFICATION_REPLY *event) 1772991554f2SKenneth D. Merry { 1773991554f2SKenneth D. Merry MPI2_EVENT_DATA_LOG_ENTRY_ADDED *entry; 1774991554f2SKenneth D. Merry 1775991554f2SKenneth D. Merry mpr_print_event(sc, event); 1776991554f2SKenneth D. Merry 1777991554f2SKenneth D. Merry switch (event->Event) { 1778991554f2SKenneth D. Merry case MPI2_EVENT_LOG_DATA: 1779991554f2SKenneth D. Merry mpr_dprint(sc, MPR_EVENT, "MPI2_EVENT_LOG_DATA:\n"); 1780991554f2SKenneth D. Merry if (sc->mpr_debug & MPR_EVENT) 1781991554f2SKenneth D. Merry hexdump(event->EventData, event->EventDataLength, NULL, 1782991554f2SKenneth D. Merry 0); 1783991554f2SKenneth D. Merry break; 1784991554f2SKenneth D. Merry case MPI2_EVENT_LOG_ENTRY_ADDED: 1785991554f2SKenneth D. Merry entry = (MPI2_EVENT_DATA_LOG_ENTRY_ADDED *)event->EventData; 1786991554f2SKenneth D. Merry mpr_dprint(sc, MPR_EVENT, "MPI2_EVENT_LOG_ENTRY_ADDED event " 1787991554f2SKenneth D. Merry "0x%x Sequence %d:\n", entry->LogEntryQualifier, 1788991554f2SKenneth D. Merry entry->LogSequence); 1789991554f2SKenneth D. Merry break; 1790991554f2SKenneth D. Merry default: 1791991554f2SKenneth D. Merry break; 1792991554f2SKenneth D. Merry } 1793991554f2SKenneth D. Merry return; 1794991554f2SKenneth D. Merry } 1795991554f2SKenneth D. Merry 1796991554f2SKenneth D. Merry static int 1797991554f2SKenneth D. Merry mpr_attach_log(struct mpr_softc *sc) 1798991554f2SKenneth D. Merry { 1799991554f2SKenneth D. Merry uint8_t events[16]; 1800991554f2SKenneth D. Merry 1801991554f2SKenneth D. Merry bzero(events, 16); 1802991554f2SKenneth D. Merry setbit(events, MPI2_EVENT_LOG_DATA); 1803991554f2SKenneth D. Merry setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED); 1804991554f2SKenneth D. Merry 1805991554f2SKenneth D. Merry mpr_register_events(sc, events, mpr_log_evt_handler, NULL, 1806991554f2SKenneth D. Merry &sc->mpr_log_eh); 1807991554f2SKenneth D. Merry 1808991554f2SKenneth D. Merry return (0); 1809991554f2SKenneth D. Merry } 1810991554f2SKenneth D. Merry 1811991554f2SKenneth D. Merry static int 1812991554f2SKenneth D. Merry mpr_detach_log(struct mpr_softc *sc) 1813991554f2SKenneth D. Merry { 1814991554f2SKenneth D. Merry 1815991554f2SKenneth D. Merry if (sc->mpr_log_eh != NULL) 1816991554f2SKenneth D. Merry mpr_deregister_events(sc, sc->mpr_log_eh); 1817991554f2SKenneth D. Merry return (0); 1818991554f2SKenneth D. Merry } 1819991554f2SKenneth D. Merry 1820991554f2SKenneth D. Merry /* 1821991554f2SKenneth D. Merry * Free all of the driver resources and detach submodules. Should be called 1822991554f2SKenneth D. Merry * without the lock held. 1823991554f2SKenneth D. Merry */ 1824991554f2SKenneth D. Merry int 1825991554f2SKenneth D. Merry mpr_free(struct mpr_softc *sc) 1826991554f2SKenneth D. Merry { 1827991554f2SKenneth D. Merry int error; 1828991554f2SKenneth D. Merry 1829991554f2SKenneth D. Merry /* Turn off the watchdog */ 1830991554f2SKenneth D. Merry mpr_lock(sc); 1831991554f2SKenneth D. Merry sc->mpr_flags |= MPR_FLAGS_SHUTDOWN; 1832991554f2SKenneth D. Merry mpr_unlock(sc); 1833991554f2SKenneth D. Merry /* Lock must not be held for this */ 1834991554f2SKenneth D. Merry callout_drain(&sc->periodic); 1835327f2e6cSStephen McConnell callout_drain(&sc->device_check_callout); 1836991554f2SKenneth D. Merry 1837991554f2SKenneth D. Merry if (((error = mpr_detach_log(sc)) != 0) || 1838991554f2SKenneth D. Merry ((error = mpr_detach_sas(sc)) != 0)) 1839991554f2SKenneth D. Merry return (error); 1840991554f2SKenneth D. Merry 1841991554f2SKenneth D. Merry mpr_detach_user(sc); 1842991554f2SKenneth D. Merry 1843991554f2SKenneth D. Merry /* Put the IOC back in the READY state. */ 1844991554f2SKenneth D. Merry mpr_lock(sc); 1845991554f2SKenneth D. Merry if ((error = mpr_transition_ready(sc)) != 0) { 1846991554f2SKenneth D. Merry mpr_unlock(sc); 1847991554f2SKenneth D. Merry return (error); 1848991554f2SKenneth D. Merry } 1849991554f2SKenneth D. Merry mpr_unlock(sc); 1850991554f2SKenneth D. Merry 1851991554f2SKenneth D. Merry if (sc->facts != NULL) 1852991554f2SKenneth D. Merry free(sc->facts, M_MPR); 1853991554f2SKenneth D. Merry 1854991554f2SKenneth D. Merry /* 1855991554f2SKenneth D. Merry * Free all buffers that are based on IOC Facts. A Diag Reset may need 1856991554f2SKenneth D. Merry * to free these buffers too. 1857991554f2SKenneth D. Merry */ 1858991554f2SKenneth D. Merry mpr_iocfacts_free(sc); 1859991554f2SKenneth D. Merry 1860991554f2SKenneth D. Merry if (sc->sysctl_tree != NULL) 1861991554f2SKenneth D. Merry sysctl_ctx_free(&sc->sysctl_ctx); 1862991554f2SKenneth D. Merry 1863991554f2SKenneth D. Merry /* Deregister the shutdown function */ 1864991554f2SKenneth D. Merry if (sc->shutdown_eh != NULL) 1865991554f2SKenneth D. Merry EVENTHANDLER_DEREGISTER(shutdown_final, sc->shutdown_eh); 1866991554f2SKenneth D. Merry 1867991554f2SKenneth D. Merry mtx_destroy(&sc->mpr_mtx); 1868991554f2SKenneth D. Merry 1869991554f2SKenneth D. Merry return (0); 1870991554f2SKenneth D. Merry } 1871991554f2SKenneth D. Merry 1872991554f2SKenneth D. Merry static __inline void 1873991554f2SKenneth D. Merry mpr_complete_command(struct mpr_softc *sc, struct mpr_command *cm) 1874991554f2SKenneth D. Merry { 1875991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 1876991554f2SKenneth D. Merry 1877991554f2SKenneth D. Merry if (cm == NULL) { 1878991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "Completing NULL command\n"); 1879991554f2SKenneth D. Merry return; 1880991554f2SKenneth D. Merry } 1881991554f2SKenneth D. Merry 1882991554f2SKenneth D. Merry if (cm->cm_flags & MPR_CM_FLAGS_POLLED) 1883991554f2SKenneth D. Merry cm->cm_flags |= MPR_CM_FLAGS_COMPLETE; 1884991554f2SKenneth D. Merry 1885991554f2SKenneth D. Merry if (cm->cm_complete != NULL) { 1886991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, 1887991554f2SKenneth D. Merry "%s cm %p calling cm_complete %p data %p reply %p\n", 1888991554f2SKenneth D. Merry __func__, cm, cm->cm_complete, cm->cm_complete_data, 1889991554f2SKenneth D. Merry cm->cm_reply); 1890991554f2SKenneth D. Merry cm->cm_complete(sc, cm); 1891991554f2SKenneth D. Merry } 1892991554f2SKenneth D. Merry 1893991554f2SKenneth D. Merry if (cm->cm_flags & MPR_CM_FLAGS_WAKEUP) { 1894991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "waking up %p\n", cm); 1895991554f2SKenneth D. Merry wakeup(cm); 1896991554f2SKenneth D. Merry } 1897991554f2SKenneth D. Merry 1898991554f2SKenneth D. Merry if (sc->io_cmds_active != 0) { 1899991554f2SKenneth D. Merry sc->io_cmds_active--; 1900991554f2SKenneth D. Merry } else { 1901991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "Warning: io_cmds_active is " 1902991554f2SKenneth D. Merry "out of sync - resynching to 0\n"); 1903991554f2SKenneth D. Merry } 1904991554f2SKenneth D. Merry } 1905991554f2SKenneth D. Merry 1906991554f2SKenneth D. Merry static void 1907991554f2SKenneth D. Merry mpr_sas_log_info(struct mpr_softc *sc , u32 log_info) 1908991554f2SKenneth D. Merry { 1909991554f2SKenneth D. Merry union loginfo_type { 1910991554f2SKenneth D. Merry u32 loginfo; 1911991554f2SKenneth D. Merry struct { 1912991554f2SKenneth D. Merry u32 subcode:16; 1913991554f2SKenneth D. Merry u32 code:8; 1914991554f2SKenneth D. Merry u32 originator:4; 1915991554f2SKenneth D. Merry u32 bus_type:4; 1916991554f2SKenneth D. Merry } dw; 1917991554f2SKenneth D. Merry }; 1918991554f2SKenneth D. Merry union loginfo_type sas_loginfo; 1919991554f2SKenneth D. Merry char *originator_str = NULL; 1920991554f2SKenneth D. Merry 1921991554f2SKenneth D. Merry sas_loginfo.loginfo = log_info; 1922991554f2SKenneth D. Merry if (sas_loginfo.dw.bus_type != 3 /*SAS*/) 1923991554f2SKenneth D. Merry return; 1924991554f2SKenneth D. Merry 1925991554f2SKenneth D. Merry /* each nexus loss loginfo */ 1926991554f2SKenneth D. Merry if (log_info == 0x31170000) 1927991554f2SKenneth D. Merry return; 1928991554f2SKenneth D. Merry 1929991554f2SKenneth D. Merry /* eat the loginfos associated with task aborts */ 1930991554f2SKenneth D. Merry if ((log_info == 30050000) || (log_info == 0x31140000) || 1931991554f2SKenneth D. Merry (log_info == 0x31130000)) 1932991554f2SKenneth D. Merry return; 1933991554f2SKenneth D. Merry 1934991554f2SKenneth D. Merry switch (sas_loginfo.dw.originator) { 1935991554f2SKenneth D. Merry case 0: 1936991554f2SKenneth D. Merry originator_str = "IOP"; 1937991554f2SKenneth D. Merry break; 1938991554f2SKenneth D. Merry case 1: 1939991554f2SKenneth D. Merry originator_str = "PL"; 1940991554f2SKenneth D. Merry break; 1941991554f2SKenneth D. Merry case 2: 1942991554f2SKenneth D. Merry originator_str = "IR"; 1943991554f2SKenneth D. Merry break; 1944991554f2SKenneth D. Merry } 1945991554f2SKenneth D. Merry 1946b41c6ff9SStephen McConnell mpr_dprint(sc, MPR_LOG, "log_info(0x%08x): originator(%s), " 19477a2a6a1aSStephen McConnell "code(0x%02x), sub_code(0x%04x)\n", log_info, originator_str, 19487a2a6a1aSStephen McConnell sas_loginfo.dw.code, sas_loginfo.dw.subcode); 1949991554f2SKenneth D. Merry } 1950991554f2SKenneth D. Merry 1951991554f2SKenneth D. Merry static void 1952991554f2SKenneth D. Merry mpr_display_reply_info(struct mpr_softc *sc, uint8_t *reply) 1953991554f2SKenneth D. Merry { 1954991554f2SKenneth D. Merry MPI2DefaultReply_t *mpi_reply; 1955991554f2SKenneth D. Merry u16 sc_status; 1956991554f2SKenneth D. Merry 1957991554f2SKenneth D. Merry mpi_reply = (MPI2DefaultReply_t*)reply; 1958991554f2SKenneth D. Merry sc_status = le16toh(mpi_reply->IOCStatus); 1959991554f2SKenneth D. Merry if (sc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) 1960991554f2SKenneth D. Merry mpr_sas_log_info(sc, le32toh(mpi_reply->IOCLogInfo)); 1961991554f2SKenneth D. Merry } 1962991554f2SKenneth D. Merry 1963991554f2SKenneth D. Merry void 1964991554f2SKenneth D. Merry mpr_intr(void *data) 1965991554f2SKenneth D. Merry { 1966991554f2SKenneth D. Merry struct mpr_softc *sc; 1967991554f2SKenneth D. Merry uint32_t status; 1968991554f2SKenneth D. Merry 1969991554f2SKenneth D. Merry sc = (struct mpr_softc *)data; 1970991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 1971991554f2SKenneth D. Merry 1972991554f2SKenneth D. Merry /* 1973991554f2SKenneth D. Merry * Check interrupt status register to flush the bus. This is 1974991554f2SKenneth D. Merry * needed for both INTx interrupts and driver-driven polling 1975991554f2SKenneth D. Merry */ 1976991554f2SKenneth D. Merry status = mpr_regread(sc, MPI2_HOST_INTERRUPT_STATUS_OFFSET); 1977991554f2SKenneth D. Merry if ((status & MPI2_HIS_REPLY_DESCRIPTOR_INTERRUPT) == 0) 1978991554f2SKenneth D. Merry return; 1979991554f2SKenneth D. Merry 1980991554f2SKenneth D. Merry mpr_lock(sc); 1981991554f2SKenneth D. Merry mpr_intr_locked(data); 1982991554f2SKenneth D. Merry mpr_unlock(sc); 1983991554f2SKenneth D. Merry return; 1984991554f2SKenneth D. Merry } 1985991554f2SKenneth D. Merry 1986991554f2SKenneth D. Merry /* 1987991554f2SKenneth D. Merry * In theory, MSI/MSIX interrupts shouldn't need to read any registers on the 1988991554f2SKenneth D. Merry * chip. Hopefully this theory is correct. 1989991554f2SKenneth D. Merry */ 1990991554f2SKenneth D. Merry void 1991991554f2SKenneth D. Merry mpr_intr_msi(void *data) 1992991554f2SKenneth D. Merry { 1993991554f2SKenneth D. Merry struct mpr_softc *sc; 1994991554f2SKenneth D. Merry 1995991554f2SKenneth D. Merry sc = (struct mpr_softc *)data; 1996991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 1997991554f2SKenneth D. Merry mpr_lock(sc); 1998991554f2SKenneth D. Merry mpr_intr_locked(data); 1999991554f2SKenneth D. Merry mpr_unlock(sc); 2000991554f2SKenneth D. Merry return; 2001991554f2SKenneth D. Merry } 2002991554f2SKenneth D. Merry 2003991554f2SKenneth D. Merry /* 2004991554f2SKenneth D. Merry * The locking is overly broad and simplistic, but easy to deal with for now. 2005991554f2SKenneth D. Merry */ 2006991554f2SKenneth D. Merry void 2007991554f2SKenneth D. Merry mpr_intr_locked(void *data) 2008991554f2SKenneth D. Merry { 2009991554f2SKenneth D. Merry MPI2_REPLY_DESCRIPTORS_UNION *desc; 2010991554f2SKenneth D. Merry struct mpr_softc *sc; 2011991554f2SKenneth D. Merry struct mpr_command *cm = NULL; 2012991554f2SKenneth D. Merry uint8_t flags; 2013991554f2SKenneth D. Merry u_int pq; 2014991554f2SKenneth D. Merry MPI2_DIAG_RELEASE_REPLY *rel_rep; 2015991554f2SKenneth D. Merry mpr_fw_diagnostic_buffer_t *pBuffer; 2016991554f2SKenneth D. Merry 2017991554f2SKenneth D. Merry sc = (struct mpr_softc *)data; 2018991554f2SKenneth D. Merry 2019991554f2SKenneth D. Merry pq = sc->replypostindex; 2020991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, 2021991554f2SKenneth D. Merry "%s sc %p starting with replypostindex %u\n", 2022991554f2SKenneth D. Merry __func__, sc, sc->replypostindex); 2023991554f2SKenneth D. Merry 2024991554f2SKenneth D. Merry for ( ;; ) { 2025991554f2SKenneth D. Merry cm = NULL; 2026991554f2SKenneth D. Merry desc = &sc->post_queue[sc->replypostindex]; 2027991554f2SKenneth D. Merry flags = desc->Default.ReplyFlags & 2028991554f2SKenneth D. Merry MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK; 2029991554f2SKenneth D. Merry if ((flags == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) || 2030991554f2SKenneth D. Merry (le32toh(desc->Words.High) == 0xffffffff)) 2031991554f2SKenneth D. Merry break; 2032991554f2SKenneth D. Merry 2033991554f2SKenneth D. Merry /* increment the replypostindex now, so that event handlers 2034991554f2SKenneth D. Merry * and cm completion handlers which decide to do a diag 2035991554f2SKenneth D. Merry * reset can zero it without it getting incremented again 2036991554f2SKenneth D. Merry * afterwards, and we break out of this loop on the next 2037991554f2SKenneth D. Merry * iteration since the reply post queue has been cleared to 2038991554f2SKenneth D. Merry * 0xFF and all descriptors look unused (which they are). 2039991554f2SKenneth D. Merry */ 2040991554f2SKenneth D. Merry if (++sc->replypostindex >= sc->pqdepth) 2041991554f2SKenneth D. Merry sc->replypostindex = 0; 2042991554f2SKenneth D. Merry 2043991554f2SKenneth D. Merry switch (flags) { 2044991554f2SKenneth D. Merry case MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS: 2045991554f2SKenneth D. Merry case MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS: 204667feec50SStephen McConnell case MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS: 2047991554f2SKenneth D. Merry cm = &sc->commands[le16toh(desc->SCSIIOSuccess.SMID)]; 2048991554f2SKenneth D. Merry cm->cm_reply = NULL; 2049991554f2SKenneth D. Merry break; 2050991554f2SKenneth D. Merry case MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY: 2051991554f2SKenneth D. Merry { 2052991554f2SKenneth D. Merry uint32_t baddr; 2053991554f2SKenneth D. Merry uint8_t *reply; 2054991554f2SKenneth D. Merry 2055991554f2SKenneth D. Merry /* 2056991554f2SKenneth D. Merry * Re-compose the reply address from the address 2057991554f2SKenneth D. Merry * sent back from the chip. The ReplyFrameAddress 2058991554f2SKenneth D. Merry * is the lower 32 bits of the physical address of 2059991554f2SKenneth D. Merry * particular reply frame. Convert that address to 2060991554f2SKenneth D. Merry * host format, and then use that to provide the 2061991554f2SKenneth D. Merry * offset against the virtual address base 2062991554f2SKenneth D. Merry * (sc->reply_frames). 2063991554f2SKenneth D. Merry */ 2064991554f2SKenneth D. Merry baddr = le32toh(desc->AddressReply.ReplyFrameAddress); 2065991554f2SKenneth D. Merry reply = sc->reply_frames + 2066991554f2SKenneth D. Merry (baddr - ((uint32_t)sc->reply_busaddr)); 2067991554f2SKenneth D. Merry /* 2068991554f2SKenneth D. Merry * Make sure the reply we got back is in a valid 2069991554f2SKenneth D. Merry * range. If not, go ahead and panic here, since 2070991554f2SKenneth D. Merry * we'll probably panic as soon as we deference the 2071991554f2SKenneth D. Merry * reply pointer anyway. 2072991554f2SKenneth D. Merry */ 2073991554f2SKenneth D. Merry if ((reply < sc->reply_frames) 2074991554f2SKenneth D. Merry || (reply > (sc->reply_frames + 2075991554f2SKenneth D. Merry (sc->fqdepth * sc->facts->ReplyFrameSize * 4)))) { 2076991554f2SKenneth D. Merry printf("%s: WARNING: reply %p out of range!\n", 2077991554f2SKenneth D. Merry __func__, reply); 2078991554f2SKenneth D. Merry printf("%s: reply_frames %p, fqdepth %d, " 2079991554f2SKenneth D. Merry "frame size %d\n", __func__, 2080991554f2SKenneth D. Merry sc->reply_frames, sc->fqdepth, 2081991554f2SKenneth D. Merry sc->facts->ReplyFrameSize * 4); 2082991554f2SKenneth D. Merry printf("%s: baddr %#x,\n", __func__, baddr); 2083991554f2SKenneth D. Merry /* LSI-TODO. See Linux Code for Graceful exit */ 2084991554f2SKenneth D. Merry panic("Reply address out of range"); 2085991554f2SKenneth D. Merry } 2086991554f2SKenneth D. Merry if (le16toh(desc->AddressReply.SMID) == 0) { 2087991554f2SKenneth D. Merry if (((MPI2_DEFAULT_REPLY *)reply)->Function == 2088991554f2SKenneth D. Merry MPI2_FUNCTION_DIAG_BUFFER_POST) { 2089991554f2SKenneth D. Merry /* 2090991554f2SKenneth D. Merry * If SMID is 0 for Diag Buffer Post, 2091991554f2SKenneth D. Merry * this implies that the reply is due to 2092991554f2SKenneth D. Merry * a release function with a status that 2093991554f2SKenneth D. Merry * the buffer has been released. Set 2094991554f2SKenneth D. Merry * the buffer flags accordingly. 2095991554f2SKenneth D. Merry */ 2096991554f2SKenneth D. Merry rel_rep = 2097991554f2SKenneth D. Merry (MPI2_DIAG_RELEASE_REPLY *)reply; 2098d3f6eabfSStephen McConnell if ((le16toh(rel_rep->IOCStatus) & 2099d3f6eabfSStephen McConnell MPI2_IOCSTATUS_MASK) == 2100991554f2SKenneth D. Merry MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED) 2101991554f2SKenneth D. Merry { 2102991554f2SKenneth D. Merry pBuffer = 2103991554f2SKenneth D. Merry &sc->fw_diag_buffer_list[ 2104991554f2SKenneth D. Merry rel_rep->BufferType]; 2105991554f2SKenneth D. Merry pBuffer->valid_data = TRUE; 2106991554f2SKenneth D. Merry pBuffer->owned_by_firmware = 2107991554f2SKenneth D. Merry FALSE; 2108991554f2SKenneth D. Merry pBuffer->immediate = FALSE; 2109991554f2SKenneth D. Merry } 2110991554f2SKenneth D. Merry } else 2111991554f2SKenneth D. Merry mpr_dispatch_event(sc, baddr, 2112991554f2SKenneth D. Merry (MPI2_EVENT_NOTIFICATION_REPLY *) 2113991554f2SKenneth D. Merry reply); 2114991554f2SKenneth D. Merry } else { 2115991554f2SKenneth D. Merry cm = &sc->commands[ 2116991554f2SKenneth D. Merry le16toh(desc->AddressReply.SMID)]; 2117991554f2SKenneth D. Merry cm->cm_reply = reply; 2118991554f2SKenneth D. Merry cm->cm_reply_data = 2119991554f2SKenneth D. Merry le32toh(desc->AddressReply. 2120991554f2SKenneth D. Merry ReplyFrameAddress); 2121991554f2SKenneth D. Merry } 2122991554f2SKenneth D. Merry break; 2123991554f2SKenneth D. Merry } 2124991554f2SKenneth D. Merry case MPI2_RPY_DESCRIPT_FLAGS_TARGETASSIST_SUCCESS: 2125991554f2SKenneth D. Merry case MPI2_RPY_DESCRIPT_FLAGS_TARGET_COMMAND_BUFFER: 2126991554f2SKenneth D. Merry case MPI2_RPY_DESCRIPT_FLAGS_RAID_ACCELERATOR_SUCCESS: 2127991554f2SKenneth D. Merry default: 2128991554f2SKenneth D. Merry /* Unhandled */ 2129991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "Unhandled reply 0x%x\n", 2130991554f2SKenneth D. Merry desc->Default.ReplyFlags); 2131991554f2SKenneth D. Merry cm = NULL; 2132991554f2SKenneth D. Merry break; 2133991554f2SKenneth D. Merry } 2134991554f2SKenneth D. Merry 2135991554f2SKenneth D. Merry if (cm != NULL) { 2136991554f2SKenneth D. Merry // Print Error reply frame 2137991554f2SKenneth D. Merry if (cm->cm_reply) 2138991554f2SKenneth D. Merry mpr_display_reply_info(sc,cm->cm_reply); 2139991554f2SKenneth D. Merry mpr_complete_command(sc, cm); 2140991554f2SKenneth D. Merry } 2141991554f2SKenneth D. Merry 2142991554f2SKenneth D. Merry desc->Words.Low = 0xffffffff; 2143991554f2SKenneth D. Merry desc->Words.High = 0xffffffff; 2144991554f2SKenneth D. Merry } 2145991554f2SKenneth D. Merry 2146991554f2SKenneth D. Merry if (pq != sc->replypostindex) { 2147991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, 2148991554f2SKenneth D. Merry "%s sc %p writing postindex %d\n", 2149991554f2SKenneth D. Merry __func__, sc, sc->replypostindex); 2150991554f2SKenneth D. Merry mpr_regwrite(sc, MPI2_REPLY_POST_HOST_INDEX_OFFSET, 2151991554f2SKenneth D. Merry sc->replypostindex); 2152991554f2SKenneth D. Merry } 2153991554f2SKenneth D. Merry 2154991554f2SKenneth D. Merry return; 2155991554f2SKenneth D. Merry } 2156991554f2SKenneth D. Merry 2157991554f2SKenneth D. Merry static void 2158991554f2SKenneth D. Merry mpr_dispatch_event(struct mpr_softc *sc, uintptr_t data, 2159991554f2SKenneth D. Merry MPI2_EVENT_NOTIFICATION_REPLY *reply) 2160991554f2SKenneth D. Merry { 2161991554f2SKenneth D. Merry struct mpr_event_handle *eh; 2162991554f2SKenneth D. Merry int event, handled = 0; 2163991554f2SKenneth D. Merry 2164991554f2SKenneth D. Merry event = le16toh(reply->Event); 2165991554f2SKenneth D. Merry TAILQ_FOREACH(eh, &sc->event_list, eh_list) { 2166991554f2SKenneth D. Merry if (isset(eh->mask, event)) { 2167991554f2SKenneth D. Merry eh->callback(sc, data, reply); 2168991554f2SKenneth D. Merry handled++; 2169991554f2SKenneth D. Merry } 2170991554f2SKenneth D. Merry } 2171991554f2SKenneth D. Merry 2172991554f2SKenneth D. Merry if (handled == 0) 2173991554f2SKenneth D. Merry mpr_dprint(sc, MPR_EVENT, "Unhandled event 0x%x\n", 2174991554f2SKenneth D. Merry le16toh(event)); 2175991554f2SKenneth D. Merry 2176991554f2SKenneth D. Merry /* 2177991554f2SKenneth D. Merry * This is the only place that the event/reply should be freed. 2178991554f2SKenneth D. Merry * Anything wanting to hold onto the event data should have 2179991554f2SKenneth D. Merry * already copied it into their own storage. 2180991554f2SKenneth D. Merry */ 2181991554f2SKenneth D. Merry mpr_free_reply(sc, data); 2182991554f2SKenneth D. Merry } 2183991554f2SKenneth D. Merry 2184991554f2SKenneth D. Merry static void 2185991554f2SKenneth D. Merry mpr_reregister_events_complete(struct mpr_softc *sc, struct mpr_command *cm) 2186991554f2SKenneth D. Merry { 2187991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 2188991554f2SKenneth D. Merry 2189991554f2SKenneth D. Merry if (cm->cm_reply) 2190991554f2SKenneth D. Merry mpr_print_event(sc, 2191991554f2SKenneth D. Merry (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply); 2192991554f2SKenneth D. Merry 2193991554f2SKenneth D. Merry mpr_free_command(sc, cm); 2194991554f2SKenneth D. Merry 2195991554f2SKenneth D. Merry /* next, send a port enable */ 2196991554f2SKenneth D. Merry mprsas_startup(sc); 2197991554f2SKenneth D. Merry } 2198991554f2SKenneth D. Merry 2199991554f2SKenneth D. Merry /* 2200991554f2SKenneth D. Merry * For both register_events and update_events, the caller supplies a bitmap 2201991554f2SKenneth D. Merry * of events that it _wants_. These functions then turn that into a bitmask 2202991554f2SKenneth D. Merry * suitable for the controller. 2203991554f2SKenneth D. Merry */ 2204991554f2SKenneth D. Merry int 2205991554f2SKenneth D. Merry mpr_register_events(struct mpr_softc *sc, uint8_t *mask, 2206991554f2SKenneth D. Merry mpr_evt_callback_t *cb, void *data, struct mpr_event_handle **handle) 2207991554f2SKenneth D. Merry { 2208991554f2SKenneth D. Merry struct mpr_event_handle *eh; 2209991554f2SKenneth D. Merry int error = 0; 2210991554f2SKenneth D. Merry 2211991554f2SKenneth D. Merry eh = malloc(sizeof(struct mpr_event_handle), M_MPR, M_WAITOK|M_ZERO); 2212991554f2SKenneth D. Merry if (!eh) { 2213991554f2SKenneth D. Merry device_printf(sc->mpr_dev, "Cannot allocate memory %s %d\n", 2214991554f2SKenneth D. Merry __func__, __LINE__); 2215991554f2SKenneth D. Merry return (ENOMEM); 2216991554f2SKenneth D. Merry } 2217991554f2SKenneth D. Merry eh->callback = cb; 2218991554f2SKenneth D. Merry eh->data = data; 2219991554f2SKenneth D. Merry TAILQ_INSERT_TAIL(&sc->event_list, eh, eh_list); 2220991554f2SKenneth D. Merry if (mask != NULL) 2221991554f2SKenneth D. Merry error = mpr_update_events(sc, eh, mask); 2222991554f2SKenneth D. Merry *handle = eh; 2223991554f2SKenneth D. Merry 2224991554f2SKenneth D. Merry return (error); 2225991554f2SKenneth D. Merry } 2226991554f2SKenneth D. Merry 2227991554f2SKenneth D. Merry int 2228991554f2SKenneth D. Merry mpr_update_events(struct mpr_softc *sc, struct mpr_event_handle *handle, 2229991554f2SKenneth D. Merry uint8_t *mask) 2230991554f2SKenneth D. Merry { 2231991554f2SKenneth D. Merry MPI2_EVENT_NOTIFICATION_REQUEST *evtreq; 2232991554f2SKenneth D. Merry MPI2_EVENT_NOTIFICATION_REPLY *reply; 2233991554f2SKenneth D. Merry struct mpr_command *cm; 2234991554f2SKenneth D. Merry struct mpr_event_handle *eh; 2235991554f2SKenneth D. Merry int error, i; 2236991554f2SKenneth D. Merry 2237991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 2238991554f2SKenneth D. Merry 2239991554f2SKenneth D. Merry if ((mask != NULL) && (handle != NULL)) 2240991554f2SKenneth D. Merry bcopy(mask, &handle->mask[0], 16); 2241991554f2SKenneth D. Merry memset(sc->event_mask, 0xff, 16); 2242991554f2SKenneth D. Merry 2243991554f2SKenneth D. Merry TAILQ_FOREACH(eh, &sc->event_list, eh_list) { 2244991554f2SKenneth D. Merry for (i = 0; i < 16; i++) 2245991554f2SKenneth D. Merry sc->event_mask[i] &= ~eh->mask[i]; 2246991554f2SKenneth D. Merry } 2247991554f2SKenneth D. Merry 2248991554f2SKenneth D. Merry if ((cm = mpr_alloc_command(sc)) == NULL) 2249991554f2SKenneth D. Merry return (EBUSY); 2250991554f2SKenneth D. Merry evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req; 2251991554f2SKenneth D. Merry evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION; 2252991554f2SKenneth D. Merry evtreq->MsgFlags = 0; 2253991554f2SKenneth D. Merry evtreq->SASBroadcastPrimitiveMasks = 0; 2254991554f2SKenneth D. Merry #ifdef MPR_DEBUG_ALL_EVENTS 2255991554f2SKenneth D. Merry { 2256991554f2SKenneth D. Merry u_char fullmask[16]; 2257991554f2SKenneth D. Merry memset(fullmask, 0x00, 16); 2258991554f2SKenneth D. Merry bcopy(fullmask, (uint8_t *)&evtreq->EventMasks, 16); 2259991554f2SKenneth D. Merry } 2260991554f2SKenneth D. Merry #else 2261991554f2SKenneth D. Merry bcopy(sc->event_mask, (uint8_t *)&evtreq->EventMasks, 16); 2262991554f2SKenneth D. Merry #endif 2263991554f2SKenneth D. Merry cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 2264991554f2SKenneth D. Merry cm->cm_data = NULL; 2265991554f2SKenneth D. Merry 2266991554f2SKenneth D. Merry error = mpr_request_polled(sc, cm); 2267991554f2SKenneth D. Merry reply = (MPI2_EVENT_NOTIFICATION_REPLY *)cm->cm_reply; 2268991554f2SKenneth D. Merry if ((reply == NULL) || 2269991554f2SKenneth D. Merry (reply->IOCStatus & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS) 2270991554f2SKenneth D. Merry error = ENXIO; 2271991554f2SKenneth D. Merry 2272991554f2SKenneth D. Merry if (reply) 2273991554f2SKenneth D. Merry mpr_print_event(sc, reply); 2274991554f2SKenneth D. Merry 2275991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s finished error %d\n", __func__, error); 2276991554f2SKenneth D. Merry 2277991554f2SKenneth D. Merry mpr_free_command(sc, cm); 2278991554f2SKenneth D. Merry return (error); 2279991554f2SKenneth D. Merry } 2280991554f2SKenneth D. Merry 2281991554f2SKenneth D. Merry static int 2282991554f2SKenneth D. Merry mpr_reregister_events(struct mpr_softc *sc) 2283991554f2SKenneth D. Merry { 2284991554f2SKenneth D. Merry MPI2_EVENT_NOTIFICATION_REQUEST *evtreq; 2285991554f2SKenneth D. Merry struct mpr_command *cm; 2286991554f2SKenneth D. Merry struct mpr_event_handle *eh; 2287991554f2SKenneth D. Merry int error, i; 2288991554f2SKenneth D. Merry 2289991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 2290991554f2SKenneth D. Merry 2291991554f2SKenneth D. Merry /* first, reregister events */ 2292991554f2SKenneth D. Merry 2293991554f2SKenneth D. Merry memset(sc->event_mask, 0xff, 16); 2294991554f2SKenneth D. Merry 2295991554f2SKenneth D. Merry TAILQ_FOREACH(eh, &sc->event_list, eh_list) { 2296991554f2SKenneth D. Merry for (i = 0; i < 16; i++) 2297991554f2SKenneth D. Merry sc->event_mask[i] &= ~eh->mask[i]; 2298991554f2SKenneth D. Merry } 2299991554f2SKenneth D. Merry 2300991554f2SKenneth D. Merry if ((cm = mpr_alloc_command(sc)) == NULL) 2301991554f2SKenneth D. Merry return (EBUSY); 2302991554f2SKenneth D. Merry evtreq = (MPI2_EVENT_NOTIFICATION_REQUEST *)cm->cm_req; 2303991554f2SKenneth D. Merry evtreq->Function = MPI2_FUNCTION_EVENT_NOTIFICATION; 2304991554f2SKenneth D. Merry evtreq->MsgFlags = 0; 2305991554f2SKenneth D. Merry evtreq->SASBroadcastPrimitiveMasks = 0; 2306991554f2SKenneth D. Merry #ifdef MPR_DEBUG_ALL_EVENTS 2307991554f2SKenneth D. Merry { 2308991554f2SKenneth D. Merry u_char fullmask[16]; 2309991554f2SKenneth D. Merry memset(fullmask, 0x00, 16); 2310991554f2SKenneth D. Merry bcopy(fullmask, (uint8_t *)&evtreq->EventMasks, 16); 2311991554f2SKenneth D. Merry } 2312991554f2SKenneth D. Merry #else 2313991554f2SKenneth D. Merry bcopy(sc->event_mask, (uint8_t *)&evtreq->EventMasks, 16); 2314991554f2SKenneth D. Merry #endif 2315991554f2SKenneth D. Merry cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 2316991554f2SKenneth D. Merry cm->cm_data = NULL; 2317991554f2SKenneth D. Merry cm->cm_complete = mpr_reregister_events_complete; 2318991554f2SKenneth D. Merry 2319991554f2SKenneth D. Merry error = mpr_map_command(sc, cm); 2320991554f2SKenneth D. Merry 2321991554f2SKenneth D. Merry mpr_dprint(sc, MPR_TRACE, "%s finished with error %d\n", __func__, 2322991554f2SKenneth D. Merry error); 2323991554f2SKenneth D. Merry return (error); 2324991554f2SKenneth D. Merry } 2325991554f2SKenneth D. Merry 2326991554f2SKenneth D. Merry int 2327991554f2SKenneth D. Merry mpr_deregister_events(struct mpr_softc *sc, struct mpr_event_handle *handle) 2328991554f2SKenneth D. Merry { 2329991554f2SKenneth D. Merry 2330991554f2SKenneth D. Merry TAILQ_REMOVE(&sc->event_list, handle, eh_list); 2331991554f2SKenneth D. Merry free(handle, M_MPR); 2332991554f2SKenneth D. Merry return (mpr_update_events(sc, NULL, NULL)); 2333991554f2SKenneth D. Merry } 2334991554f2SKenneth D. Merry 233567feec50SStephen McConnell /** 233667feec50SStephen McConnell * mpr_build_nvme_prp - This function is called for NVMe end devices to build a 233767feec50SStephen McConnell * native SGL (NVMe PRP). The native SGL is built starting in the first PRP entry 233867feec50SStephen McConnell * of the NVMe message (PRP1). If the data buffer is small enough to be described 233967feec50SStephen McConnell * entirely using PRP1, then PRP2 is not used. If needed, PRP2 is used to 234067feec50SStephen McConnell * describe a larger data buffer. If the data buffer is too large to describe 234167feec50SStephen McConnell * using the two PRP entriess inside the NVMe message, then PRP1 describes the 234267feec50SStephen McConnell * first data memory segment, and PRP2 contains a pointer to a PRP list located 234367feec50SStephen McConnell * elsewhere in memory to describe the remaining data memory segments. The PRP 234467feec50SStephen McConnell * list will be contiguous. 234567feec50SStephen McConnell 234667feec50SStephen McConnell * The native SGL for NVMe devices is a Physical Region Page (PRP). A PRP 234767feec50SStephen McConnell * consists of a list of PRP entries to describe a number of noncontigous 234867feec50SStephen McConnell * physical memory segments as a single memory buffer, just as a SGL does. Note 234967feec50SStephen McConnell * however, that this function is only used by the IOCTL call, so the memory 235067feec50SStephen McConnell * given will be guaranteed to be contiguous. There is no need to translate 235167feec50SStephen McConnell * non-contiguous SGL into a PRP in this case. All PRPs will describe contiguous 235267feec50SStephen McConnell * space that is one page size each. 235367feec50SStephen McConnell * 235467feec50SStephen McConnell * Each NVMe message contains two PRP entries. The first (PRP1) either contains 235567feec50SStephen McConnell * a PRP list pointer or a PRP element, depending upon the command. PRP2 contains 235667feec50SStephen McConnell * the second PRP element if the memory being described fits within 2 PRP 235767feec50SStephen McConnell * entries, or a PRP list pointer if the PRP spans more than two entries. 235867feec50SStephen McConnell * 235967feec50SStephen McConnell * A PRP list pointer contains the address of a PRP list, structured as a linear 236067feec50SStephen McConnell * array of PRP entries. Each PRP entry in this list describes a segment of 236167feec50SStephen McConnell * physical memory. 236267feec50SStephen McConnell * 236367feec50SStephen McConnell * Each 64-bit PRP entry comprises an address and an offset field. The address 236467feec50SStephen McConnell * always points to the beginning of a PAGE_SIZE physical memory page, and the 236567feec50SStephen McConnell * offset describes where within that page the memory segment begins. Only the 236667feec50SStephen McConnell * first element in a PRP list may contain a non-zero offest, implying that all 236767feec50SStephen McConnell * memory segments following the first begin at the start of a PAGE_SIZE page. 236867feec50SStephen McConnell * 236967feec50SStephen McConnell * Each PRP element normally describes a chunck of PAGE_SIZE physical memory, 237067feec50SStephen McConnell * with exceptions for the first and last elements in the list. If the memory 237167feec50SStephen McConnell * being described by the list begins at a non-zero offset within the first page, 237267feec50SStephen McConnell * then the first PRP element will contain a non-zero offset indicating where the 237367feec50SStephen McConnell * region begins within the page. The last memory segment may end before the end 237467feec50SStephen McConnell * of the PAGE_SIZE segment, depending upon the overall size of the memory being 237567feec50SStephen McConnell * described by the PRP list. 237667feec50SStephen McConnell * 237767feec50SStephen McConnell * Since PRP entries lack any indication of size, the overall data buffer length 237867feec50SStephen McConnell * is used to determine where the end of the data memory buffer is located, and 237967feec50SStephen McConnell * how many PRP entries are required to describe it. 238067feec50SStephen McConnell * 238167feec50SStephen McConnell * Returns nothing. 238267feec50SStephen McConnell */ 238367feec50SStephen McConnell void 238467feec50SStephen McConnell mpr_build_nvme_prp(struct mpr_softc *sc, struct mpr_command *cm, 238567feec50SStephen McConnell Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request, void *data, 238667feec50SStephen McConnell uint32_t data_in_sz, uint32_t data_out_sz) 238767feec50SStephen McConnell { 238867feec50SStephen McConnell int prp_size = PRP_ENTRY_SIZE; 238967feec50SStephen McConnell uint64_t *prp_entry, *prp1_entry, *prp2_entry; 239067feec50SStephen McConnell uint64_t *prp_entry_phys, *prp_page, *prp_page_phys; 239167feec50SStephen McConnell uint32_t offset, entry_len, page_mask_result, page_mask; 239267feec50SStephen McConnell bus_addr_t paddr; 239367feec50SStephen McConnell size_t length; 239467feec50SStephen McConnell struct mpr_prp_page *prp_page_info = NULL; 239567feec50SStephen McConnell 239667feec50SStephen McConnell /* 239767feec50SStephen McConnell * Not all commands require a data transfer. If no data, just return 239867feec50SStephen McConnell * without constructing any PRP. 239967feec50SStephen McConnell */ 240067feec50SStephen McConnell if (!data_in_sz && !data_out_sz) 240167feec50SStephen McConnell return; 240267feec50SStephen McConnell 240367feec50SStephen McConnell /* 240467feec50SStephen McConnell * Set pointers to PRP1 and PRP2, which are in the NVMe command. PRP1 is 240567feec50SStephen McConnell * located at a 24 byte offset from the start of the NVMe command. Then 240667feec50SStephen McConnell * set the current PRP entry pointer to PRP1. 240767feec50SStephen McConnell */ 240867feec50SStephen McConnell prp1_entry = (uint64_t *)(nvme_encap_request->NVMe_Command + 240967feec50SStephen McConnell NVME_CMD_PRP1_OFFSET); 241067feec50SStephen McConnell prp2_entry = (uint64_t *)(nvme_encap_request->NVMe_Command + 241167feec50SStephen McConnell NVME_CMD_PRP2_OFFSET); 241267feec50SStephen McConnell prp_entry = prp1_entry; 241367feec50SStephen McConnell 241467feec50SStephen McConnell /* 241567feec50SStephen McConnell * For the PRP entries, use the specially allocated buffer of 241667feec50SStephen McConnell * contiguous memory. PRP Page allocation failures should not happen 241767feec50SStephen McConnell * because there should be enough PRP page buffers to account for the 241867feec50SStephen McConnell * possible NVMe QDepth. 241967feec50SStephen McConnell */ 242067feec50SStephen McConnell prp_page_info = mpr_alloc_prp_page(sc); 242167feec50SStephen McConnell KASSERT(prp_page_info != NULL, ("%s: There are no PRP Pages left to be " 242267feec50SStephen McConnell "used for building a native NVMe SGL.\n", __func__)); 242367feec50SStephen McConnell prp_page = (uint64_t *)prp_page_info->prp_page; 242467feec50SStephen McConnell prp_page_phys = (uint64_t *)(uintptr_t)prp_page_info->prp_page_busaddr; 242567feec50SStephen McConnell 242667feec50SStephen McConnell /* 242767feec50SStephen McConnell * Insert the allocated PRP page into the command's PRP page list. This 242867feec50SStephen McConnell * will be freed when the command is freed. 242967feec50SStephen McConnell */ 243067feec50SStephen McConnell TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link); 243167feec50SStephen McConnell 243267feec50SStephen McConnell /* 243367feec50SStephen McConnell * Check if we are within 1 entry of a page boundary we don't want our 243467feec50SStephen McConnell * first entry to be a PRP List entry. 243567feec50SStephen McConnell */ 243667feec50SStephen McConnell page_mask = PAGE_SIZE - 1; 243767feec50SStephen McConnell page_mask_result = (uintptr_t)((uint8_t *)prp_page + prp_size) & 243867feec50SStephen McConnell page_mask; 243967feec50SStephen McConnell if (!page_mask_result) 244067feec50SStephen McConnell { 244167feec50SStephen McConnell /* Bump up to next page boundary. */ 244267feec50SStephen McConnell prp_page = (uint64_t *)((uint8_t *)prp_page + prp_size); 244367feec50SStephen McConnell prp_page_phys = (uint64_t *)((uint8_t *)prp_page_phys + 244467feec50SStephen McConnell prp_size); 244567feec50SStephen McConnell } 244667feec50SStephen McConnell 244767feec50SStephen McConnell /* 244867feec50SStephen McConnell * Set PRP physical pointer, which initially points to the current PRP 244967feec50SStephen McConnell * DMA memory page. 245067feec50SStephen McConnell */ 245167feec50SStephen McConnell prp_entry_phys = prp_page_phys; 245267feec50SStephen McConnell 245367feec50SStephen McConnell /* Get physical address and length of the data buffer. */ 245467feec50SStephen McConnell paddr = (bus_addr_t)data; 245567feec50SStephen McConnell if (data_in_sz) 245667feec50SStephen McConnell length = data_in_sz; 245767feec50SStephen McConnell else 245867feec50SStephen McConnell length = data_out_sz; 245967feec50SStephen McConnell 246067feec50SStephen McConnell /* Loop while the length is not zero. */ 246167feec50SStephen McConnell while (length) 246267feec50SStephen McConnell { 246367feec50SStephen McConnell /* 246467feec50SStephen McConnell * Check if we need to put a list pointer here if we are at page 246567feec50SStephen McConnell * boundary - prp_size (8 bytes). 246667feec50SStephen McConnell */ 246767feec50SStephen McConnell page_mask_result = (uintptr_t)((uint8_t *)prp_entry_phys + 246867feec50SStephen McConnell prp_size) & page_mask; 246967feec50SStephen McConnell if (!page_mask_result) 247067feec50SStephen McConnell { 247167feec50SStephen McConnell /* 247267feec50SStephen McConnell * This is the last entry in a PRP List, so we need to 247367feec50SStephen McConnell * put a PRP list pointer here. What this does is: 247467feec50SStephen McConnell * - bump the current memory pointer to the next 247567feec50SStephen McConnell * address, which will be the next full page. 247667feec50SStephen McConnell * - set the PRP Entry to point to that page. This is 247767feec50SStephen McConnell * now the PRP List pointer. 247867feec50SStephen McConnell * - bump the PRP Entry pointer the start of the next 247967feec50SStephen McConnell * page. Since all of this PRP memory is contiguous, 248067feec50SStephen McConnell * no need to get a new page - it's just the next 248167feec50SStephen McConnell * address. 248267feec50SStephen McConnell */ 248367feec50SStephen McConnell prp_entry_phys++; 248467feec50SStephen McConnell *prp_entry = 248567feec50SStephen McConnell htole64((uint64_t)(uintptr_t)prp_entry_phys); 248667feec50SStephen McConnell prp_entry++; 248767feec50SStephen McConnell } 248867feec50SStephen McConnell 248967feec50SStephen McConnell /* Need to handle if entry will be part of a page. */ 249067feec50SStephen McConnell offset = (uint32_t)paddr & page_mask; 249167feec50SStephen McConnell entry_len = PAGE_SIZE - offset; 249267feec50SStephen McConnell 249367feec50SStephen McConnell if (prp_entry == prp1_entry) 249467feec50SStephen McConnell { 249567feec50SStephen McConnell /* 249667feec50SStephen McConnell * Must fill in the first PRP pointer (PRP1) before 249767feec50SStephen McConnell * moving on. 249867feec50SStephen McConnell */ 249967feec50SStephen McConnell *prp1_entry = htole64((uint64_t)paddr); 250067feec50SStephen McConnell 250167feec50SStephen McConnell /* 250267feec50SStephen McConnell * Now point to the second PRP entry within the 250367feec50SStephen McConnell * command (PRP2). 250467feec50SStephen McConnell */ 250567feec50SStephen McConnell prp_entry = prp2_entry; 250667feec50SStephen McConnell } 250767feec50SStephen McConnell else if (prp_entry == prp2_entry) 250867feec50SStephen McConnell { 250967feec50SStephen McConnell /* 251067feec50SStephen McConnell * Should the PRP2 entry be a PRP List pointer or just a 251167feec50SStephen McConnell * regular PRP pointer? If there is more than one more 251267feec50SStephen McConnell * page of data, must use a PRP List pointer. 251367feec50SStephen McConnell */ 251467feec50SStephen McConnell if (length > PAGE_SIZE) 251567feec50SStephen McConnell { 251667feec50SStephen McConnell /* 251767feec50SStephen McConnell * PRP2 will contain a PRP List pointer because 251867feec50SStephen McConnell * more PRP's are needed with this command. The 251967feec50SStephen McConnell * list will start at the beginning of the 252067feec50SStephen McConnell * contiguous buffer. 252167feec50SStephen McConnell */ 252267feec50SStephen McConnell *prp2_entry = 252367feec50SStephen McConnell htole64( 252467feec50SStephen McConnell (uint64_t)(uintptr_t)prp_entry_phys); 252567feec50SStephen McConnell 252667feec50SStephen McConnell /* 252767feec50SStephen McConnell * The next PRP Entry will be the start of the 252867feec50SStephen McConnell * first PRP List. 252967feec50SStephen McConnell */ 253067feec50SStephen McConnell prp_entry = prp_page; 253167feec50SStephen McConnell } 253267feec50SStephen McConnell else 253367feec50SStephen McConnell { 253467feec50SStephen McConnell /* 253567feec50SStephen McConnell * After this, the PRP Entries are complete. 253667feec50SStephen McConnell * This command uses 2 PRP's and no PRP list. 253767feec50SStephen McConnell */ 253867feec50SStephen McConnell *prp2_entry = htole64((uint64_t)paddr); 253967feec50SStephen McConnell } 254067feec50SStephen McConnell } 254167feec50SStephen McConnell else 254267feec50SStephen McConnell { 254367feec50SStephen McConnell /* 254467feec50SStephen McConnell * Put entry in list and bump the addresses. 254567feec50SStephen McConnell * 254667feec50SStephen McConnell * After PRP1 and PRP2 are filled in, this will fill in 254767feec50SStephen McConnell * all remaining PRP entries in a PRP List, one per each 254867feec50SStephen McConnell * time through the loop. 254967feec50SStephen McConnell */ 255067feec50SStephen McConnell *prp_entry = htole64((uint64_t)paddr); 255167feec50SStephen McConnell prp_entry++; 255267feec50SStephen McConnell prp_entry_phys++; 255367feec50SStephen McConnell } 255467feec50SStephen McConnell 255567feec50SStephen McConnell /* 255667feec50SStephen McConnell * Bump the phys address of the command's data buffer by the 255767feec50SStephen McConnell * entry_len. 255867feec50SStephen McConnell */ 255967feec50SStephen McConnell paddr += entry_len; 256067feec50SStephen McConnell 256167feec50SStephen McConnell /* Decrement length accounting for last partial page. */ 256267feec50SStephen McConnell if (entry_len > length) 256367feec50SStephen McConnell length = 0; 256467feec50SStephen McConnell else 256567feec50SStephen McConnell length -= entry_len; 256667feec50SStephen McConnell } 256767feec50SStephen McConnell } 256867feec50SStephen McConnell 256967feec50SStephen McConnell /* 257067feec50SStephen McConnell * mpr_check_pcie_native_sgl - This function is called for PCIe end devices to 257167feec50SStephen McConnell * determine if the driver needs to build a native SGL. If so, that native SGL 257267feec50SStephen McConnell * is built in the contiguous buffers allocated especially for PCIe SGL 257367feec50SStephen McConnell * creation. If the driver will not build a native SGL, return TRUE and a 257467feec50SStephen McConnell * normal IEEE SGL will be built. Currently this routine supports NVMe devices 257567feec50SStephen McConnell * only. 257667feec50SStephen McConnell * 257767feec50SStephen McConnell * Returns FALSE (0) if native SGL was built, TRUE (1) if no SGL was built. 257867feec50SStephen McConnell */ 257967feec50SStephen McConnell static int 258067feec50SStephen McConnell mpr_check_pcie_native_sgl(struct mpr_softc *sc, struct mpr_command *cm, 258167feec50SStephen McConnell bus_dma_segment_t *segs, int segs_left) 258267feec50SStephen McConnell { 258367feec50SStephen McConnell uint32_t i, sge_dwords, length, offset, entry_len; 258467feec50SStephen McConnell uint32_t num_entries, buff_len = 0, sges_in_segment; 258567feec50SStephen McConnell uint32_t page_mask, page_mask_result, *curr_buff; 258667feec50SStephen McConnell uint32_t *ptr_sgl, *ptr_first_sgl, first_page_offset; 258767feec50SStephen McConnell uint32_t first_page_data_size, end_residual; 258867feec50SStephen McConnell uint64_t *msg_phys; 258967feec50SStephen McConnell bus_addr_t paddr; 259067feec50SStephen McConnell int build_native_sgl = 0, first_prp_entry; 259167feec50SStephen McConnell int prp_size = PRP_ENTRY_SIZE; 259267feec50SStephen McConnell Mpi25IeeeSgeChain64_t *main_chain_element = NULL; 259367feec50SStephen McConnell struct mpr_prp_page *prp_page_info = NULL; 259467feec50SStephen McConnell 259567feec50SStephen McConnell mpr_dprint(sc, MPR_TRACE, "%s\n", __func__); 259667feec50SStephen McConnell 259767feec50SStephen McConnell /* 259867feec50SStephen McConnell * Add up the sizes of each segment length to get the total transfer 259967feec50SStephen McConnell * size, which will be checked against the Maximum Data Transfer Size. 260067feec50SStephen McConnell * If the data transfer length exceeds the MDTS for this device, just 260167feec50SStephen McConnell * return 1 so a normal IEEE SGL will be built. F/W will break the I/O 260267feec50SStephen McConnell * up into multiple I/O's. [nvme_mdts = 0 means unlimited] 260367feec50SStephen McConnell */ 260467feec50SStephen McConnell for (i = 0; i < segs_left; i++) 260567feec50SStephen McConnell buff_len += htole32(segs[i].ds_len); 260667feec50SStephen McConnell if ((cm->cm_targ->MDTS > 0) && (buff_len > cm->cm_targ->MDTS)) 260767feec50SStephen McConnell return 1; 260867feec50SStephen McConnell 260967feec50SStephen McConnell /* Create page_mask (to get offset within page) */ 261067feec50SStephen McConnell page_mask = PAGE_SIZE - 1; 261167feec50SStephen McConnell 261267feec50SStephen McConnell /* 261367feec50SStephen McConnell * Check if the number of elements exceeds the max number that can be 261467feec50SStephen McConnell * put in the main message frame (H/W can only translate an SGL that 261567feec50SStephen McConnell * is contained entirely in the main message frame). 261667feec50SStephen McConnell */ 261767feec50SStephen McConnell sges_in_segment = (sc->facts->IOCRequestFrameSize - 261867feec50SStephen McConnell offsetof(Mpi25SCSIIORequest_t, SGL)) / sizeof(MPI25_SGE_IO_UNION); 261967feec50SStephen McConnell if (segs_left > sges_in_segment) 262067feec50SStephen McConnell build_native_sgl = 1; 262167feec50SStephen McConnell else 262267feec50SStephen McConnell { 262367feec50SStephen McConnell /* 262467feec50SStephen McConnell * NVMe uses one PRP for each physical page (or part of physical 262567feec50SStephen McConnell * page). 262667feec50SStephen McConnell * if 4 pages or less then IEEE is OK 262767feec50SStephen McConnell * if > 5 pages then we need to build a native SGL 262867feec50SStephen McConnell * if > 4 and <= 5 pages, then check the physical address of 262967feec50SStephen McConnell * the first SG entry, then if this first size in the page 263067feec50SStephen McConnell * is >= the residual beyond 4 pages then use IEEE, 263167feec50SStephen McConnell * otherwise use native SGL 263267feec50SStephen McConnell */ 263367feec50SStephen McConnell if (buff_len > (PAGE_SIZE * 5)) 263467feec50SStephen McConnell build_native_sgl = 1; 263567feec50SStephen McConnell else if ((buff_len > (PAGE_SIZE * 4)) && 263667feec50SStephen McConnell (buff_len <= (PAGE_SIZE * 5)) ) 263767feec50SStephen McConnell { 263867feec50SStephen McConnell msg_phys = (uint64_t *)segs[0].ds_addr; 263967feec50SStephen McConnell first_page_offset = 264067feec50SStephen McConnell ((uint32_t)(uint64_t)(uintptr_t)msg_phys & 264167feec50SStephen McConnell page_mask); 264267feec50SStephen McConnell first_page_data_size = PAGE_SIZE - first_page_offset; 264367feec50SStephen McConnell end_residual = buff_len % PAGE_SIZE; 264467feec50SStephen McConnell 264567feec50SStephen McConnell /* 264667feec50SStephen McConnell * If offset into first page pushes the end of the data 264767feec50SStephen McConnell * beyond end of the 5th page, we need the extra PRP 264867feec50SStephen McConnell * list. 264967feec50SStephen McConnell */ 265067feec50SStephen McConnell if (first_page_data_size < end_residual) 265167feec50SStephen McConnell build_native_sgl = 1; 265267feec50SStephen McConnell 265367feec50SStephen McConnell /* 265467feec50SStephen McConnell * Check if first SG entry size is < residual beyond 4 265567feec50SStephen McConnell * pages. 265667feec50SStephen McConnell */ 265767feec50SStephen McConnell if (htole32(segs[0].ds_len) < 265867feec50SStephen McConnell (buff_len - (PAGE_SIZE * 4))) 265967feec50SStephen McConnell build_native_sgl = 1; 266067feec50SStephen McConnell } 266167feec50SStephen McConnell } 266267feec50SStephen McConnell 266367feec50SStephen McConnell /* check if native SGL is needed */ 266467feec50SStephen McConnell if (!build_native_sgl) 266567feec50SStephen McConnell return 1; 266667feec50SStephen McConnell 266767feec50SStephen McConnell /* 266867feec50SStephen McConnell * Native SGL is needed. 266967feec50SStephen McConnell * Put a chain element in main message frame that points to the first 267067feec50SStephen McConnell * chain buffer. 267167feec50SStephen McConnell * 267267feec50SStephen McConnell * NOTE: The ChainOffset field must be 0 when using a chain pointer to 267367feec50SStephen McConnell * a native SGL. 267467feec50SStephen McConnell */ 267567feec50SStephen McConnell 267667feec50SStephen McConnell /* Set main message chain element pointer */ 267767feec50SStephen McConnell main_chain_element = (pMpi25IeeeSgeChain64_t)cm->cm_sge; 267867feec50SStephen McConnell 267967feec50SStephen McConnell /* 268067feec50SStephen McConnell * For NVMe the chain element needs to be the 2nd SGL entry in the main 268167feec50SStephen McConnell * message. 268267feec50SStephen McConnell */ 268367feec50SStephen McConnell main_chain_element = (Mpi25IeeeSgeChain64_t *) 268467feec50SStephen McConnell ((uint8_t *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64)); 268567feec50SStephen McConnell 268667feec50SStephen McConnell /* 268767feec50SStephen McConnell * For the PRP entries, use the specially allocated buffer of 268867feec50SStephen McConnell * contiguous memory. PRP Page allocation failures should not happen 268967feec50SStephen McConnell * because there should be enough PRP page buffers to account for the 269067feec50SStephen McConnell * possible NVMe QDepth. 269167feec50SStephen McConnell */ 269267feec50SStephen McConnell prp_page_info = mpr_alloc_prp_page(sc); 269367feec50SStephen McConnell KASSERT(prp_page_info != NULL, ("%s: There are no PRP Pages left to be " 269467feec50SStephen McConnell "used for building a native NVMe SGL.\n", __func__)); 269567feec50SStephen McConnell curr_buff = (uint32_t *)prp_page_info->prp_page; 269667feec50SStephen McConnell msg_phys = (uint64_t *)(uintptr_t)prp_page_info->prp_page_busaddr; 269767feec50SStephen McConnell 269867feec50SStephen McConnell /* 269967feec50SStephen McConnell * Insert the allocated PRP page into the command's PRP page list. This 270067feec50SStephen McConnell * will be freed when the command is freed. 270167feec50SStephen McConnell */ 270267feec50SStephen McConnell TAILQ_INSERT_TAIL(&cm->cm_prp_page_list, prp_page_info, prp_page_link); 270367feec50SStephen McConnell 270467feec50SStephen McConnell /* 270567feec50SStephen McConnell * Check if we are within 1 entry of a page boundary we don't want our 270667feec50SStephen McConnell * first entry to be a PRP List entry. 270767feec50SStephen McConnell */ 270867feec50SStephen McConnell page_mask_result = (uintptr_t)((uint8_t *)curr_buff + prp_size) & 270967feec50SStephen McConnell page_mask; 271067feec50SStephen McConnell if (!page_mask_result) { 271167feec50SStephen McConnell /* Bump up to next page boundary. */ 271267feec50SStephen McConnell curr_buff = (uint32_t *)((uint8_t *)curr_buff + prp_size); 271367feec50SStephen McConnell msg_phys = (uint64_t *)((uint8_t *)msg_phys + prp_size); 271467feec50SStephen McConnell } 271567feec50SStephen McConnell 271667feec50SStephen McConnell /* Fill in the chain element and make it an NVMe segment type. */ 271767feec50SStephen McConnell main_chain_element->Address.High = 271867feec50SStephen McConnell htole32((uint32_t)((uint64_t)(uintptr_t)msg_phys >> 32)); 271967feec50SStephen McConnell main_chain_element->Address.Low = 272067feec50SStephen McConnell htole32((uint32_t)(uintptr_t)msg_phys); 272167feec50SStephen McConnell main_chain_element->NextChainOffset = 0; 272267feec50SStephen McConnell main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT | 272367feec50SStephen McConnell MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR | 272467feec50SStephen McConnell MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP; 272567feec50SStephen McConnell 272667feec50SStephen McConnell /* Set SGL pointer to start of contiguous PCIe buffer. */ 272767feec50SStephen McConnell ptr_sgl = curr_buff; 272867feec50SStephen McConnell sge_dwords = 2; 272967feec50SStephen McConnell num_entries = 0; 273067feec50SStephen McConnell 273167feec50SStephen McConnell /* 273267feec50SStephen McConnell * NVMe has a very convoluted PRP format. One PRP is required for each 273367feec50SStephen McConnell * page or partial page. We need to split up OS SG entries if they are 273467feec50SStephen McConnell * longer than one page or cross a page boundary. We also have to insert 273567feec50SStephen McConnell * a PRP list pointer entry as the last entry in each physical page of 273667feec50SStephen McConnell * the PRP list. 273767feec50SStephen McConnell * 273867feec50SStephen McConnell * NOTE: The first PRP "entry" is actually placed in the first SGL entry 273967feec50SStephen McConnell * in the main message in IEEE 64 format. The 2nd entry in the main 274067feec50SStephen McConnell * message is the chain element, and the rest of the PRP entries are 274167feec50SStephen McConnell * built in the contiguous PCIe buffer. 274267feec50SStephen McConnell */ 274367feec50SStephen McConnell first_prp_entry = 1; 274467feec50SStephen McConnell ptr_first_sgl = (uint32_t *)cm->cm_sge; 274567feec50SStephen McConnell 274667feec50SStephen McConnell for (i = 0; i < segs_left; i++) { 274767feec50SStephen McConnell /* Get physical address and length of this SG entry. */ 274867feec50SStephen McConnell paddr = segs[i].ds_addr; 274967feec50SStephen McConnell length = segs[i].ds_len; 275067feec50SStephen McConnell 275167feec50SStephen McConnell /* 275267feec50SStephen McConnell * Check whether a given SGE buffer lies on a non-PAGED 275367feec50SStephen McConnell * boundary if this is not the first page. If so, this is not 275467feec50SStephen McConnell * expected so have FW build the SGL. 275567feec50SStephen McConnell */ 275667feec50SStephen McConnell if (i) { 275767feec50SStephen McConnell if ((uint32_t)paddr & page_mask) { 275867feec50SStephen McConnell mpr_dprint(sc, MPR_ERROR, "Unaligned SGE while " 275967feec50SStephen McConnell "building NVMe PRPs, low address is 0x%x\n", 276067feec50SStephen McConnell (uint32_t)paddr); 276167feec50SStephen McConnell return 1; 276267feec50SStephen McConnell } 276367feec50SStephen McConnell } 276467feec50SStephen McConnell 276567feec50SStephen McConnell /* Apart from last SGE, if any other SGE boundary is not page 276667feec50SStephen McConnell * aligned then it means that hole exists. Existence of hole 276767feec50SStephen McConnell * leads to data corruption. So fallback to IEEE SGEs. 276867feec50SStephen McConnell */ 276967feec50SStephen McConnell if (i != (segs_left - 1)) { 277067feec50SStephen McConnell if (((uint32_t)paddr + length) & page_mask) { 277167feec50SStephen McConnell mpr_dprint(sc, MPR_ERROR, "Unaligned SGE " 277267feec50SStephen McConnell "boundary while building NVMe PRPs, low " 277367feec50SStephen McConnell "address: 0x%x and length: %u\n", 277467feec50SStephen McConnell (uint32_t)paddr, length); 277567feec50SStephen McConnell return 1; 277667feec50SStephen McConnell } 277767feec50SStephen McConnell } 277867feec50SStephen McConnell 277967feec50SStephen McConnell /* Loop while the length is not zero. */ 278067feec50SStephen McConnell while (length) { 278167feec50SStephen McConnell /* 278267feec50SStephen McConnell * Check if we need to put a list pointer here if we are 278367feec50SStephen McConnell * at page boundary - prp_size. 278467feec50SStephen McConnell */ 278567feec50SStephen McConnell page_mask_result = (uintptr_t)((uint8_t *)ptr_sgl + 278667feec50SStephen McConnell prp_size) & page_mask; 278767feec50SStephen McConnell if (!page_mask_result) { 278867feec50SStephen McConnell /* 278967feec50SStephen McConnell * Need to put a PRP list pointer here. 279067feec50SStephen McConnell */ 279167feec50SStephen McConnell msg_phys = (uint64_t *)((uint8_t *)msg_phys + 279267feec50SStephen McConnell prp_size); 279367feec50SStephen McConnell *ptr_sgl = htole32((uintptr_t)msg_phys); 279467feec50SStephen McConnell *(ptr_sgl+1) = htole32((uint64_t)(uintptr_t) 279567feec50SStephen McConnell msg_phys >> 32); 279667feec50SStephen McConnell ptr_sgl += sge_dwords; 279767feec50SStephen McConnell num_entries++; 279867feec50SStephen McConnell } 279967feec50SStephen McConnell 280067feec50SStephen McConnell /* Need to handle if entry will be part of a page. */ 280167feec50SStephen McConnell offset = (uint32_t)paddr & page_mask; 280267feec50SStephen McConnell entry_len = PAGE_SIZE - offset; 280367feec50SStephen McConnell if (first_prp_entry) { 280467feec50SStephen McConnell /* 280567feec50SStephen McConnell * Put IEEE entry in first SGE in main message. 280667feec50SStephen McConnell * (Simple element, System addr, not end of 280767feec50SStephen McConnell * list.) 280867feec50SStephen McConnell */ 280967feec50SStephen McConnell *ptr_first_sgl = htole32((uint32_t)paddr); 281067feec50SStephen McConnell *(ptr_first_sgl + 1) = 281167feec50SStephen McConnell htole32((uint32_t)((uint64_t)paddr >> 32)); 281267feec50SStephen McConnell *(ptr_first_sgl + 2) = htole32(entry_len); 281367feec50SStephen McConnell *(ptr_first_sgl + 3) = 0; 281467feec50SStephen McConnell 281567feec50SStephen McConnell /* No longer the first PRP entry. */ 281667feec50SStephen McConnell first_prp_entry = 0; 281767feec50SStephen McConnell } else { 281867feec50SStephen McConnell /* Put entry in list. */ 281967feec50SStephen McConnell *ptr_sgl = htole32((uint32_t)paddr); 282067feec50SStephen McConnell *(ptr_sgl + 1) = 282167feec50SStephen McConnell htole32((uint32_t)((uint64_t)paddr >> 32)); 282267feec50SStephen McConnell 282367feec50SStephen McConnell /* Bump ptr_sgl, msg_phys, and num_entries. */ 282467feec50SStephen McConnell ptr_sgl += sge_dwords; 282567feec50SStephen McConnell msg_phys = (uint64_t *)((uint8_t *)msg_phys + 282667feec50SStephen McConnell prp_size); 282767feec50SStephen McConnell num_entries++; 282867feec50SStephen McConnell } 282967feec50SStephen McConnell 283067feec50SStephen McConnell /* Bump the phys address by the entry_len. */ 283167feec50SStephen McConnell paddr += entry_len; 283267feec50SStephen McConnell 283367feec50SStephen McConnell /* Decrement length accounting for last partial page. */ 283467feec50SStephen McConnell if (entry_len > length) 283567feec50SStephen McConnell length = 0; 283667feec50SStephen McConnell else 283767feec50SStephen McConnell length -= entry_len; 283867feec50SStephen McConnell } 283967feec50SStephen McConnell } 284067feec50SStephen McConnell 284167feec50SStephen McConnell /* Set chain element Length. */ 284267feec50SStephen McConnell main_chain_element->Length = htole32(num_entries * prp_size); 284367feec50SStephen McConnell 284467feec50SStephen McConnell /* Return 0, indicating we built a native SGL. */ 284567feec50SStephen McConnell return 0; 284667feec50SStephen McConnell } 284767feec50SStephen McConnell 2848991554f2SKenneth D. Merry /* 2849991554f2SKenneth D. Merry * Add a chain element as the next SGE for the specified command. 2850991554f2SKenneth D. Merry * Reset cm_sge and cm_sgesize to indicate all the available space. Chains are 2851991554f2SKenneth D. Merry * only required for IEEE commands. Therefore there is no code for commands 2852a2c14879SStephen McConnell * that have the MPR_CM_FLAGS_SGE_SIMPLE flag set (and those commands 2853a2c14879SStephen McConnell * shouldn't be requesting chains). 2854991554f2SKenneth D. Merry */ 2855991554f2SKenneth D. Merry static int 2856991554f2SKenneth D. Merry mpr_add_chain(struct mpr_command *cm, int segsleft) 2857991554f2SKenneth D. Merry { 2858991554f2SKenneth D. Merry struct mpr_softc *sc = cm->cm_sc; 2859991554f2SKenneth D. Merry MPI2_REQUEST_HEADER *req; 2860991554f2SKenneth D. Merry MPI25_IEEE_SGE_CHAIN64 *ieee_sgc; 2861991554f2SKenneth D. Merry struct mpr_chain *chain; 28622bbc5fcbSStephen McConnell int sgc_size, current_segs, rem_segs, segs_per_frame; 2863991554f2SKenneth D. Merry uint8_t next_chain_offset = 0; 2864991554f2SKenneth D. Merry 2865991554f2SKenneth D. Merry /* 2866991554f2SKenneth D. Merry * Fail if a command is requesting a chain for SIMPLE SGE's. For SAS3 2867991554f2SKenneth D. Merry * only IEEE commands should be requesting chains. Return some error 2868991554f2SKenneth D. Merry * code other than 0. 2869991554f2SKenneth D. Merry */ 2870991554f2SKenneth D. Merry if (cm->cm_flags & MPR_CM_FLAGS_SGE_SIMPLE) { 2871991554f2SKenneth D. Merry mpr_dprint(sc, MPR_ERROR, "A chain element cannot be added to " 2872991554f2SKenneth D. Merry "an MPI SGL.\n"); 2873991554f2SKenneth D. Merry return(ENOBUFS); 2874991554f2SKenneth D. Merry } 2875991554f2SKenneth D. Merry 2876991554f2SKenneth D. Merry sgc_size = sizeof(MPI25_IEEE_SGE_CHAIN64); 2877991554f2SKenneth D. Merry if (cm->cm_sglsize < sgc_size) 2878991554f2SKenneth D. Merry panic("MPR: Need SGE Error Code\n"); 2879991554f2SKenneth D. Merry 2880991554f2SKenneth D. Merry chain = mpr_alloc_chain(cm->cm_sc); 2881991554f2SKenneth D. Merry if (chain == NULL) 2882991554f2SKenneth D. Merry return (ENOBUFS); 2883991554f2SKenneth D. Merry 2884991554f2SKenneth D. Merry /* 2885991554f2SKenneth D. Merry * Note: a double-linked list is used to make it easier to walk for 2886991554f2SKenneth D. Merry * debugging. 2887991554f2SKenneth D. Merry */ 2888991554f2SKenneth D. Merry TAILQ_INSERT_TAIL(&cm->cm_chain_list, chain, chain_link); 2889991554f2SKenneth D. Merry 2890991554f2SKenneth D. Merry /* 2891991554f2SKenneth D. Merry * Need to know if the number of frames left is more than 1 or not. If 2892991554f2SKenneth D. Merry * more than 1 frame is required, NextChainOffset will need to be set, 2893991554f2SKenneth D. Merry * which will just be the last segment of the frame. 2894991554f2SKenneth D. Merry */ 2895991554f2SKenneth D. Merry rem_segs = 0; 2896991554f2SKenneth D. Merry if (cm->cm_sglsize < (sgc_size * segsleft)) { 2897991554f2SKenneth D. Merry /* 2898991554f2SKenneth D. Merry * rem_segs is the number of segements remaining after the 2899991554f2SKenneth D. Merry * segments that will go into the current frame. Since it is 2900991554f2SKenneth D. Merry * known that at least one more frame is required, account for 2901991554f2SKenneth D. Merry * the chain element. To know if more than one more frame is 2902991554f2SKenneth D. Merry * required, just check if there will be a remainder after using 2903991554f2SKenneth D. Merry * the current frame (with this chain) and the next frame. If 2904991554f2SKenneth D. Merry * so the NextChainOffset must be the last element of the next 2905991554f2SKenneth D. Merry * frame. 2906991554f2SKenneth D. Merry */ 2907991554f2SKenneth D. Merry current_segs = (cm->cm_sglsize / sgc_size) - 1; 2908991554f2SKenneth D. Merry rem_segs = segsleft - current_segs; 29092bbc5fcbSStephen McConnell segs_per_frame = sc->chain_frame_size / sgc_size; 2910991554f2SKenneth D. Merry if (rem_segs > segs_per_frame) { 2911991554f2SKenneth D. Merry next_chain_offset = segs_per_frame - 1; 2912991554f2SKenneth D. Merry } 2913991554f2SKenneth D. Merry } 2914991554f2SKenneth D. Merry ieee_sgc = &((MPI25_SGE_IO_UNION *)cm->cm_sge)->IeeeChain; 29152bbc5fcbSStephen McConnell ieee_sgc->Length = next_chain_offset ? 29162bbc5fcbSStephen McConnell htole32((uint32_t)sc->chain_frame_size) : 2917991554f2SKenneth D. Merry htole32((uint32_t)rem_segs * (uint32_t)sgc_size); 2918991554f2SKenneth D. Merry ieee_sgc->NextChainOffset = next_chain_offset; 2919991554f2SKenneth D. Merry ieee_sgc->Flags = (MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT | 2920991554f2SKenneth D. Merry MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR); 2921991554f2SKenneth D. Merry ieee_sgc->Address.Low = htole32(chain->chain_busaddr); 2922991554f2SKenneth D. Merry ieee_sgc->Address.High = htole32(chain->chain_busaddr >> 32); 2923991554f2SKenneth D. Merry cm->cm_sge = &((MPI25_SGE_IO_UNION *)chain->chain)->IeeeSimple; 2924991554f2SKenneth D. Merry req = (MPI2_REQUEST_HEADER *)cm->cm_req; 29252bbc5fcbSStephen McConnell req->ChainOffset = (sc->chain_frame_size - sgc_size) >> 4; 2926991554f2SKenneth D. Merry 29272bbc5fcbSStephen McConnell cm->cm_sglsize = sc->chain_frame_size; 2928991554f2SKenneth D. Merry return (0); 2929991554f2SKenneth D. Merry } 2930991554f2SKenneth D. Merry 2931991554f2SKenneth D. Merry /* 2932991554f2SKenneth D. Merry * Add one scatter-gather element to the scatter-gather list for a command. 2933a2c14879SStephen McConnell * Maintain cm_sglsize and cm_sge as the remaining size and pointer to the 2934a2c14879SStephen McConnell * next SGE to fill in, respectively. In Gen3, the MPI SGL does not have a 2935a2c14879SStephen McConnell * chain, so don't consider any chain additions. 2936991554f2SKenneth D. Merry */ 2937991554f2SKenneth D. Merry int 2938991554f2SKenneth D. Merry mpr_push_sge(struct mpr_command *cm, MPI2_SGE_SIMPLE64 *sge, size_t len, 2939991554f2SKenneth D. Merry int segsleft) 2940991554f2SKenneth D. Merry { 2941991554f2SKenneth D. Merry uint32_t saved_buf_len, saved_address_low, saved_address_high; 2942991554f2SKenneth D. Merry u32 sge_flags; 2943991554f2SKenneth D. Merry 2944991554f2SKenneth D. Merry /* 2945991554f2SKenneth D. Merry * case 1: >=1 more segment, no room for anything (error) 2946991554f2SKenneth D. Merry * case 2: 1 more segment and enough room for it 2947991554f2SKenneth D. Merry */ 2948991554f2SKenneth D. Merry 2949991554f2SKenneth D. Merry if (cm->cm_sglsize < (segsleft * sizeof(MPI2_SGE_SIMPLE64))) { 2950991554f2SKenneth D. Merry mpr_dprint(cm->cm_sc, MPR_ERROR, 2951991554f2SKenneth D. Merry "%s: warning: Not enough room for MPI SGL in frame.\n", 2952991554f2SKenneth D. Merry __func__); 2953991554f2SKenneth D. Merry return(ENOBUFS); 2954991554f2SKenneth D. Merry } 2955991554f2SKenneth D. Merry 2956991554f2SKenneth D. Merry KASSERT(segsleft == 1, 2957991554f2SKenneth D. Merry ("segsleft cannot be more than 1 for an MPI SGL; segsleft = %d\n", 2958991554f2SKenneth D. Merry segsleft)); 2959991554f2SKenneth D. Merry 2960991554f2SKenneth D. Merry /* 2961991554f2SKenneth D. Merry * There is one more segment left to add for the MPI SGL and there is 2962991554f2SKenneth D. Merry * enough room in the frame to add it. This is the normal case because 2963991554f2SKenneth D. Merry * MPI SGL's don't have chains, otherwise something is wrong. 2964991554f2SKenneth D. Merry * 2965991554f2SKenneth D. Merry * If this is a bi-directional request, need to account for that 2966991554f2SKenneth D. Merry * here. Save the pre-filled sge values. These will be used 2967991554f2SKenneth D. Merry * either for the 2nd SGL or for a single direction SGL. If 2968991554f2SKenneth D. Merry * cm_out_len is non-zero, this is a bi-directional request, so 2969991554f2SKenneth D. Merry * fill in the OUT SGL first, then the IN SGL, otherwise just 2970991554f2SKenneth D. Merry * fill in the IN SGL. Note that at this time, when filling in 2971991554f2SKenneth D. Merry * 2 SGL's for a bi-directional request, they both use the same 2972991554f2SKenneth D. Merry * DMA buffer (same cm command). 2973991554f2SKenneth D. Merry */ 2974991554f2SKenneth D. Merry saved_buf_len = sge->FlagsLength & 0x00FFFFFF; 2975991554f2SKenneth D. Merry saved_address_low = sge->Address.Low; 2976991554f2SKenneth D. Merry saved_address_high = sge->Address.High; 2977991554f2SKenneth D. Merry if (cm->cm_out_len) { 2978991554f2SKenneth D. Merry sge->FlagsLength = cm->cm_out_len | 2979991554f2SKenneth D. Merry ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2980991554f2SKenneth D. Merry MPI2_SGE_FLAGS_END_OF_BUFFER | 2981991554f2SKenneth D. Merry MPI2_SGE_FLAGS_HOST_TO_IOC | 2982991554f2SKenneth D. Merry MPI2_SGE_FLAGS_64_BIT_ADDRESSING) << 2983991554f2SKenneth D. Merry MPI2_SGE_FLAGS_SHIFT); 2984991554f2SKenneth D. Merry cm->cm_sglsize -= len; 2985991554f2SKenneth D. Merry /* Endian Safe code */ 2986991554f2SKenneth D. Merry sge_flags = sge->FlagsLength; 2987991554f2SKenneth D. Merry sge->FlagsLength = htole32(sge_flags); 2988991554f2SKenneth D. Merry sge->Address.High = htole32(sge->Address.High); 2989991554f2SKenneth D. Merry sge->Address.Low = htole32(sge->Address.Low); 2990991554f2SKenneth D. Merry bcopy(sge, cm->cm_sge, len); 2991991554f2SKenneth D. Merry cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len); 2992991554f2SKenneth D. Merry } 2993991554f2SKenneth D. Merry sge->FlagsLength = saved_buf_len | 2994991554f2SKenneth D. Merry ((uint32_t)(MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 2995991554f2SKenneth D. Merry MPI2_SGE_FLAGS_END_OF_BUFFER | 2996991554f2SKenneth D. Merry MPI2_SGE_FLAGS_LAST_ELEMENT | 2997991554f2SKenneth D. Merry MPI2_SGE_FLAGS_END_OF_LIST | 2998991554f2SKenneth D. Merry MPI2_SGE_FLAGS_64_BIT_ADDRESSING) << 2999991554f2SKenneth D. Merry MPI2_SGE_FLAGS_SHIFT); 3000991554f2SKenneth D. Merry if (cm->cm_flags & MPR_CM_FLAGS_DATAIN) { 3001991554f2SKenneth D. Merry sge->FlagsLength |= 3002991554f2SKenneth D. Merry ((uint32_t)(MPI2_SGE_FLAGS_IOC_TO_HOST) << 3003991554f2SKenneth D. Merry MPI2_SGE_FLAGS_SHIFT); 3004991554f2SKenneth D. Merry } else { 3005991554f2SKenneth D. Merry sge->FlagsLength |= 3006991554f2SKenneth D. Merry ((uint32_t)(MPI2_SGE_FLAGS_HOST_TO_IOC) << 3007991554f2SKenneth D. Merry MPI2_SGE_FLAGS_SHIFT); 3008991554f2SKenneth D. Merry } 3009991554f2SKenneth D. Merry sge->Address.Low = saved_address_low; 3010991554f2SKenneth D. Merry sge->Address.High = saved_address_high; 3011991554f2SKenneth D. Merry 3012991554f2SKenneth D. Merry cm->cm_sglsize -= len; 3013991554f2SKenneth D. Merry /* Endian Safe code */ 3014991554f2SKenneth D. Merry sge_flags = sge->FlagsLength; 3015991554f2SKenneth D. Merry sge->FlagsLength = htole32(sge_flags); 3016991554f2SKenneth D. Merry sge->Address.High = htole32(sge->Address.High); 3017991554f2SKenneth D. Merry sge->Address.Low = htole32(sge->Address.Low); 3018991554f2SKenneth D. Merry bcopy(sge, cm->cm_sge, len); 3019991554f2SKenneth D. Merry cm->cm_sge = (MPI2_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + len); 3020991554f2SKenneth D. Merry return (0); 3021991554f2SKenneth D. Merry } 3022991554f2SKenneth D. Merry 3023991554f2SKenneth D. Merry /* 3024991554f2SKenneth D. Merry * Add one IEEE scatter-gather element (chain or simple) to the IEEE scatter- 3025991554f2SKenneth D. Merry * gather list for a command. Maintain cm_sglsize and cm_sge as the 3026991554f2SKenneth D. Merry * remaining size and pointer to the next SGE to fill in, respectively. 3027991554f2SKenneth D. Merry */ 3028991554f2SKenneth D. Merry int 3029991554f2SKenneth D. Merry mpr_push_ieee_sge(struct mpr_command *cm, void *sgep, int segsleft) 3030991554f2SKenneth D. Merry { 3031991554f2SKenneth D. Merry MPI2_IEEE_SGE_SIMPLE64 *sge = sgep; 3032991554f2SKenneth D. Merry int error, ieee_sge_size = sizeof(MPI25_SGE_IO_UNION); 3033991554f2SKenneth D. Merry uint32_t saved_buf_len, saved_address_low, saved_address_high; 3034991554f2SKenneth D. Merry uint32_t sge_length; 3035991554f2SKenneth D. Merry 3036991554f2SKenneth D. Merry /* 3037991554f2SKenneth D. Merry * case 1: No room for chain or segment (error). 3038991554f2SKenneth D. Merry * case 2: Two or more segments left but only room for chain. 3039991554f2SKenneth D. Merry * case 3: Last segment and room for it, so set flags. 3040991554f2SKenneth D. Merry */ 3041991554f2SKenneth D. Merry 3042991554f2SKenneth D. Merry /* 3043991554f2SKenneth D. Merry * There should be room for at least one element, or there is a big 3044991554f2SKenneth D. Merry * problem. 3045991554f2SKenneth D. Merry */ 3046991554f2SKenneth D. Merry if (cm->cm_sglsize < ieee_sge_size) 3047991554f2SKenneth D. Merry panic("MPR: Need SGE Error Code\n"); 3048991554f2SKenneth D. Merry 3049991554f2SKenneth D. Merry if ((segsleft >= 2) && (cm->cm_sglsize < (ieee_sge_size * 2))) { 3050991554f2SKenneth D. Merry if ((error = mpr_add_chain(cm, segsleft)) != 0) 3051991554f2SKenneth D. Merry return (error); 3052991554f2SKenneth D. Merry } 3053991554f2SKenneth D. Merry 3054991554f2SKenneth D. Merry if (segsleft == 1) { 3055991554f2SKenneth D. Merry /* 3056991554f2SKenneth D. Merry * If this is a bi-directional request, need to account for that 3057991554f2SKenneth D. Merry * here. Save the pre-filled sge values. These will be used 3058991554f2SKenneth D. Merry * either for the 2nd SGL or for a single direction SGL. If 3059991554f2SKenneth D. Merry * cm_out_len is non-zero, this is a bi-directional request, so 3060991554f2SKenneth D. Merry * fill in the OUT SGL first, then the IN SGL, otherwise just 3061991554f2SKenneth D. Merry * fill in the IN SGL. Note that at this time, when filling in 3062991554f2SKenneth D. Merry * 2 SGL's for a bi-directional request, they both use the same 3063991554f2SKenneth D. Merry * DMA buffer (same cm command). 3064991554f2SKenneth D. Merry */ 3065991554f2SKenneth D. Merry saved_buf_len = sge->Length; 3066991554f2SKenneth D. Merry saved_address_low = sge->Address.Low; 3067991554f2SKenneth D. Merry saved_address_high = sge->Address.High; 3068991554f2SKenneth D. Merry if (cm->cm_out_len) { 3069991554f2SKenneth D. Merry sge->Length = cm->cm_out_len; 3070991554f2SKenneth D. Merry sge->Flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 3071991554f2SKenneth D. Merry MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR); 3072991554f2SKenneth D. Merry cm->cm_sglsize -= ieee_sge_size; 3073991554f2SKenneth D. Merry /* Endian Safe code */ 3074991554f2SKenneth D. Merry sge_length = sge->Length; 3075991554f2SKenneth D. Merry sge->Length = htole32(sge_length); 3076991554f2SKenneth D. Merry sge->Address.High = htole32(sge->Address.High); 3077991554f2SKenneth D. Merry sge->Address.Low = htole32(sge->Address.Low); 3078991554f2SKenneth D. Merry bcopy(sgep, cm->cm_sge, ieee_sge_size); 3079991554f2SKenneth D. Merry cm->cm_sge = 3080991554f2SKenneth D. Merry (MPI25_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + 3081991554f2SKenneth D. Merry ieee_sge_size); 3082991554f2SKenneth D. Merry } 3083991554f2SKenneth D. Merry sge->Length = saved_buf_len; 3084991554f2SKenneth D. Merry sge->Flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 3085991554f2SKenneth D. Merry MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR | 3086991554f2SKenneth D. Merry MPI25_IEEE_SGE_FLAGS_END_OF_LIST); 3087991554f2SKenneth D. Merry sge->Address.Low = saved_address_low; 3088991554f2SKenneth D. Merry sge->Address.High = saved_address_high; 3089991554f2SKenneth D. Merry } 3090991554f2SKenneth D. Merry 3091991554f2SKenneth D. Merry cm->cm_sglsize -= ieee_sge_size; 3092991554f2SKenneth D. Merry /* Endian Safe code */ 3093991554f2SKenneth D. Merry sge_length = sge->Length; 3094991554f2SKenneth D. Merry sge->Length = htole32(sge_length); 3095991554f2SKenneth D. Merry sge->Address.High = htole32(sge->Address.High); 3096991554f2SKenneth D. Merry sge->Address.Low = htole32(sge->Address.Low); 3097991554f2SKenneth D. Merry bcopy(sgep, cm->cm_sge, ieee_sge_size); 3098991554f2SKenneth D. Merry cm->cm_sge = (MPI25_SGE_IO_UNION *)((uintptr_t)cm->cm_sge + 3099991554f2SKenneth D. Merry ieee_sge_size); 3100991554f2SKenneth D. Merry return (0); 3101991554f2SKenneth D. Merry } 3102991554f2SKenneth D. Merry 3103991554f2SKenneth D. Merry /* 3104991554f2SKenneth D. Merry * Add one dma segment to the scatter-gather list for a command. 3105991554f2SKenneth D. Merry */ 3106991554f2SKenneth D. Merry int 3107991554f2SKenneth D. Merry mpr_add_dmaseg(struct mpr_command *cm, vm_paddr_t pa, size_t len, u_int flags, 3108991554f2SKenneth D. Merry int segsleft) 3109991554f2SKenneth D. Merry { 3110991554f2SKenneth D. Merry MPI2_SGE_SIMPLE64 sge; 3111991554f2SKenneth D. Merry MPI2_IEEE_SGE_SIMPLE64 ieee_sge; 3112991554f2SKenneth D. Merry 3113991554f2SKenneth D. Merry if (!(cm->cm_flags & MPR_CM_FLAGS_SGE_SIMPLE)) { 3114991554f2SKenneth D. Merry ieee_sge.Flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT | 3115991554f2SKenneth D. Merry MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR); 3116991554f2SKenneth D. Merry ieee_sge.Length = len; 3117991554f2SKenneth D. Merry mpr_from_u64(pa, &ieee_sge.Address); 3118991554f2SKenneth D. Merry 3119991554f2SKenneth D. Merry return (mpr_push_ieee_sge(cm, &ieee_sge, segsleft)); 3120991554f2SKenneth D. Merry } else { 3121991554f2SKenneth D. Merry /* 3122991554f2SKenneth D. Merry * This driver always uses 64-bit address elements for 3123991554f2SKenneth D. Merry * simplicity. 3124991554f2SKenneth D. Merry */ 3125991554f2SKenneth D. Merry flags |= MPI2_SGE_FLAGS_SIMPLE_ELEMENT | 3126991554f2SKenneth D. Merry MPI2_SGE_FLAGS_64_BIT_ADDRESSING; 3127991554f2SKenneth D. Merry /* Set Endian safe macro in mpr_push_sge */ 3128991554f2SKenneth D. Merry sge.FlagsLength = len | (flags << MPI2_SGE_FLAGS_SHIFT); 3129991554f2SKenneth D. Merry mpr_from_u64(pa, &sge.Address); 3130991554f2SKenneth D. Merry 3131991554f2SKenneth D. Merry return (mpr_push_sge(cm, &sge, sizeof sge, segsleft)); 3132991554f2SKenneth D. Merry } 3133991554f2SKenneth D. Merry } 3134991554f2SKenneth D. Merry 3135991554f2SKenneth D. Merry static void 3136991554f2SKenneth D. Merry mpr_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error) 3137991554f2SKenneth D. Merry { 3138991554f2SKenneth D. Merry struct mpr_softc *sc; 3139991554f2SKenneth D. Merry struct mpr_command *cm; 3140991554f2SKenneth D. Merry u_int i, dir, sflags; 3141991554f2SKenneth D. Merry 3142991554f2SKenneth D. Merry cm = (struct mpr_command *)arg; 3143991554f2SKenneth D. Merry sc = cm->cm_sc; 3144991554f2SKenneth D. Merry 3145991554f2SKenneth D. Merry /* 3146991554f2SKenneth D. Merry * In this case, just print out a warning and let the chip tell the 3147991554f2SKenneth D. Merry * user they did the wrong thing. 3148991554f2SKenneth D. Merry */ 3149991554f2SKenneth D. Merry if ((cm->cm_max_segs != 0) && (nsegs > cm->cm_max_segs)) { 31507a2a6a1aSStephen McConnell mpr_dprint(sc, MPR_ERROR, "%s: warning: busdma returned %d " 31517a2a6a1aSStephen McConnell "segments, more than the %d allowed\n", __func__, nsegs, 3152991554f2SKenneth D. Merry cm->cm_max_segs); 3153991554f2SKenneth D. Merry } 3154991554f2SKenneth D. Merry 3155991554f2SKenneth D. Merry /* 3156991554f2SKenneth D. Merry * Set up DMA direction flags. Bi-directional requests are also handled 3157991554f2SKenneth D. Merry * here. In that case, both direction flags will be set. 3158991554f2SKenneth D. Merry */ 3159991554f2SKenneth D. Merry sflags = 0; 3160991554f2SKenneth D. Merry if (cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) { 3161991554f2SKenneth D. Merry /* 3162991554f2SKenneth D. Merry * We have to add a special case for SMP passthrough, there 3163991554f2SKenneth D. Merry * is no easy way to generically handle it. The first 3164991554f2SKenneth D. Merry * S/G element is used for the command (therefore the 3165991554f2SKenneth D. Merry * direction bit needs to be set). The second one is used 3166991554f2SKenneth D. Merry * for the reply. We'll leave it to the caller to make 3167991554f2SKenneth D. Merry * sure we only have two buffers. 3168991554f2SKenneth D. Merry */ 3169991554f2SKenneth D. Merry /* 3170991554f2SKenneth D. Merry * Even though the busdma man page says it doesn't make 3171991554f2SKenneth D. Merry * sense to have both direction flags, it does in this case. 3172991554f2SKenneth D. Merry * We have one s/g element being accessed in each direction. 3173991554f2SKenneth D. Merry */ 3174991554f2SKenneth D. Merry dir = BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD; 3175991554f2SKenneth D. Merry 3176991554f2SKenneth D. Merry /* 3177991554f2SKenneth D. Merry * Set the direction flag on the first buffer in the SMP 3178991554f2SKenneth D. Merry * passthrough request. We'll clear it for the second one. 3179991554f2SKenneth D. Merry */ 3180991554f2SKenneth D. Merry sflags |= MPI2_SGE_FLAGS_DIRECTION | 3181991554f2SKenneth D. Merry MPI2_SGE_FLAGS_END_OF_BUFFER; 3182991554f2SKenneth D. Merry } else if (cm->cm_flags & MPR_CM_FLAGS_DATAOUT) { 3183991554f2SKenneth D. Merry sflags |= MPI2_SGE_FLAGS_HOST_TO_IOC; 3184991554f2SKenneth D. Merry dir = BUS_DMASYNC_PREWRITE; 3185991554f2SKenneth D. Merry } else 3186991554f2SKenneth D. Merry dir = BUS_DMASYNC_PREREAD; 3187991554f2SKenneth D. Merry 318867feec50SStephen McConnell /* Check if a native SG list is needed for an NVMe PCIe device. */ 318967feec50SStephen McConnell if (cm->cm_targ && cm->cm_targ->is_nvme && 319067feec50SStephen McConnell mpr_check_pcie_native_sgl(sc, cm, segs, nsegs) == 0) { 319167feec50SStephen McConnell /* A native SG list was built, skip to end. */ 319267feec50SStephen McConnell goto out; 319367feec50SStephen McConnell } 319467feec50SStephen McConnell 3195991554f2SKenneth D. Merry for (i = 0; i < nsegs; i++) { 3196991554f2SKenneth D. Merry if ((cm->cm_flags & MPR_CM_FLAGS_SMP_PASS) && (i != 0)) { 3197991554f2SKenneth D. Merry sflags &= ~MPI2_SGE_FLAGS_DIRECTION; 3198991554f2SKenneth D. Merry } 3199991554f2SKenneth D. Merry error = mpr_add_dmaseg(cm, segs[i].ds_addr, segs[i].ds_len, 3200991554f2SKenneth D. Merry sflags, nsegs - i); 3201991554f2SKenneth D. Merry if (error != 0) { 3202991554f2SKenneth D. Merry /* Resource shortage, roll back! */ 3203991554f2SKenneth D. Merry if (ratecheck(&sc->lastfail, &mpr_chainfail_interval)) 3204991554f2SKenneth D. Merry mpr_dprint(sc, MPR_INFO, "Out of chain frames, " 3205991554f2SKenneth D. Merry "consider increasing hw.mpr.max_chains.\n"); 3206991554f2SKenneth D. Merry cm->cm_flags |= MPR_CM_FLAGS_CHAIN_FAILED; 3207991554f2SKenneth D. Merry mpr_complete_command(sc, cm); 3208991554f2SKenneth D. Merry return; 3209991554f2SKenneth D. Merry } 3210991554f2SKenneth D. Merry } 3211991554f2SKenneth D. Merry 321267feec50SStephen McConnell out: 3213991554f2SKenneth D. Merry bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir); 3214991554f2SKenneth D. Merry mpr_enqueue_request(sc, cm); 3215991554f2SKenneth D. Merry 3216991554f2SKenneth D. Merry return; 3217991554f2SKenneth D. Merry } 3218991554f2SKenneth D. Merry 3219991554f2SKenneth D. Merry static void 3220991554f2SKenneth D. Merry mpr_data_cb2(void *arg, bus_dma_segment_t *segs, int nsegs, bus_size_t mapsize, 3221991554f2SKenneth D. Merry int error) 3222991554f2SKenneth D. Merry { 3223991554f2SKenneth D. Merry mpr_data_cb(arg, segs, nsegs, error); 3224991554f2SKenneth D. Merry } 3225991554f2SKenneth D. Merry 3226991554f2SKenneth D. Merry /* 3227991554f2SKenneth D. Merry * This is the routine to enqueue commands ansynchronously. 3228991554f2SKenneth D. Merry * Note that the only error path here is from bus_dmamap_load(), which can 3229991554f2SKenneth D. Merry * return EINPROGRESS if it is waiting for resources. Other than this, it's 3230991554f2SKenneth D. Merry * assumed that if you have a command in-hand, then you have enough credits 3231991554f2SKenneth D. Merry * to use it. 3232991554f2SKenneth D. Merry */ 3233991554f2SKenneth D. Merry int 3234991554f2SKenneth D. Merry mpr_map_command(struct mpr_softc *sc, struct mpr_command *cm) 3235991554f2SKenneth D. Merry { 3236991554f2SKenneth D. Merry int error = 0; 3237991554f2SKenneth D. Merry 3238991554f2SKenneth D. Merry if (cm->cm_flags & MPR_CM_FLAGS_USE_UIO) { 3239991554f2SKenneth D. Merry error = bus_dmamap_load_uio(sc->buffer_dmat, cm->cm_dmamap, 3240991554f2SKenneth D. Merry &cm->cm_uio, mpr_data_cb2, cm, 0); 3241991554f2SKenneth D. Merry } else if (cm->cm_flags & MPR_CM_FLAGS_USE_CCB) { 3242991554f2SKenneth D. Merry error = bus_dmamap_load_ccb(sc->buffer_dmat, cm->cm_dmamap, 3243991554f2SKenneth D. Merry cm->cm_data, mpr_data_cb, cm, 0); 3244991554f2SKenneth D. Merry } else if ((cm->cm_data != NULL) && (cm->cm_length != 0)) { 3245991554f2SKenneth D. Merry error = bus_dmamap_load(sc->buffer_dmat, cm->cm_dmamap, 3246991554f2SKenneth D. Merry cm->cm_data, cm->cm_length, mpr_data_cb, cm, 0); 3247991554f2SKenneth D. Merry } else { 3248991554f2SKenneth D. Merry /* Add a zero-length element as needed */ 3249991554f2SKenneth D. Merry if (cm->cm_sge != NULL) 3250991554f2SKenneth D. Merry mpr_add_dmaseg(cm, 0, 0, 0, 1); 3251991554f2SKenneth D. Merry mpr_enqueue_request(sc, cm); 3252991554f2SKenneth D. Merry } 3253991554f2SKenneth D. Merry 3254991554f2SKenneth D. Merry return (error); 3255991554f2SKenneth D. Merry } 3256991554f2SKenneth D. Merry 3257991554f2SKenneth D. Merry /* 3258991554f2SKenneth D. Merry * This is the routine to enqueue commands synchronously. An error of 3259991554f2SKenneth D. Merry * EINPROGRESS from mpr_map_command() is ignored since the command will 3260991554f2SKenneth D. Merry * be executed and enqueued automatically. Other errors come from msleep(). 3261991554f2SKenneth D. Merry */ 3262991554f2SKenneth D. Merry int 3263991554f2SKenneth D. Merry mpr_wait_command(struct mpr_softc *sc, struct mpr_command *cm, int timeout, 3264991554f2SKenneth D. Merry int sleep_flag) 3265991554f2SKenneth D. Merry { 3266991554f2SKenneth D. Merry int error, rc; 3267991554f2SKenneth D. Merry struct timeval cur_time, start_time; 3268991554f2SKenneth D. Merry 3269991554f2SKenneth D. Merry if (sc->mpr_flags & MPR_FLAGS_DIAGRESET) 3270991554f2SKenneth D. Merry return EBUSY; 3271991554f2SKenneth D. Merry 3272991554f2SKenneth D. Merry cm->cm_complete = NULL; 3273991554f2SKenneth D. Merry cm->cm_flags |= (MPR_CM_FLAGS_WAKEUP + MPR_CM_FLAGS_POLLED); 3274991554f2SKenneth D. Merry error = mpr_map_command(sc, cm); 3275991554f2SKenneth D. Merry if ((error != 0) && (error != EINPROGRESS)) 3276991554f2SKenneth D. Merry return (error); 3277991554f2SKenneth D. Merry 3278991554f2SKenneth D. Merry // Check for context and wait for 50 mSec at a time until time has 3279991554f2SKenneth D. Merry // expired or the command has finished. If msleep can't be used, need 3280991554f2SKenneth D. Merry // to poll. 3281991554f2SKenneth D. Merry #if __FreeBSD_version >= 1000029 3282991554f2SKenneth D. Merry if (curthread->td_no_sleeping) 3283991554f2SKenneth D. Merry #else //__FreeBSD_version < 1000029 3284991554f2SKenneth D. Merry if (curthread->td_pflags & TDP_NOSLEEPING) 3285991554f2SKenneth D. Merry #endif //__FreeBSD_version >= 1000029 3286991554f2SKenneth D. Merry sleep_flag = NO_SLEEP; 3287417aa6b8SKenneth D. Merry getmicrouptime(&start_time); 3288991554f2SKenneth D. Merry if (mtx_owned(&sc->mpr_mtx) && sleep_flag == CAN_SLEEP) { 3289991554f2SKenneth D. Merry error = msleep(cm, &sc->mpr_mtx, 0, "mprwait", timeout*hz); 3290417aa6b8SKenneth D. Merry if (error == EWOULDBLOCK) { 3291417aa6b8SKenneth D. Merry /* 3292417aa6b8SKenneth D. Merry * Record the actual elapsed time in the case of a 3293417aa6b8SKenneth D. Merry * timeout for the message below. 3294417aa6b8SKenneth D. Merry */ 3295417aa6b8SKenneth D. Merry getmicrouptime(&cur_time); 3296417aa6b8SKenneth D. Merry timevalsub(&cur_time, &start_time); 3297417aa6b8SKenneth D. Merry } 3298991554f2SKenneth D. Merry } else { 3299991554f2SKenneth D. Merry while ((cm->cm_flags & MPR_CM_FLAGS_COMPLETE) == 0) { 3300991554f2SKenneth D. Merry mpr_intr_locked(sc); 3301991554f2SKenneth D. Merry if (sleep_flag == CAN_SLEEP) 3302991554f2SKenneth D. Merry pause("mprwait", hz/20); 3303991554f2SKenneth D. Merry else 3304991554f2SKenneth D. Merry DELAY(50000); 3305991554f2SKenneth D. Merry 3306417aa6b8SKenneth D. Merry getmicrouptime(&cur_time); 3307417aa6b8SKenneth D. Merry timevalsub(&cur_time, &start_time); 3308417aa6b8SKenneth D. Merry if (cur_time.tv_sec > timeout) { 3309991554f2SKenneth D. Merry error = EWOULDBLOCK; 3310991554f2SKenneth D. Merry break; 3311991554f2SKenneth D. Merry } 3312991554f2SKenneth D. Merry } 3313991554f2SKenneth D. Merry } 3314991554f2SKenneth D. Merry 3315991554f2SKenneth D. Merry if (error == EWOULDBLOCK) { 3316417aa6b8SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Calling Reinit from %s, timeout=%d," 3317417aa6b8SKenneth D. Merry " elapsed=%jd\n", __func__, timeout, 3318417aa6b8SKenneth D. Merry (intmax_t)cur_time.tv_sec); 3319991554f2SKenneth D. Merry rc = mpr_reinit(sc); 3320991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Reinit %s\n", (rc == 0) ? "success" : 3321991554f2SKenneth D. Merry "failed"); 3322991554f2SKenneth D. Merry error = ETIMEDOUT; 3323991554f2SKenneth D. Merry } 3324991554f2SKenneth D. Merry return (error); 3325991554f2SKenneth D. Merry } 3326991554f2SKenneth D. Merry 3327991554f2SKenneth D. Merry /* 3328991554f2SKenneth D. Merry * This is the routine to enqueue a command synchonously and poll for 3329991554f2SKenneth D. Merry * completion. Its use should be rare. 3330991554f2SKenneth D. Merry */ 3331991554f2SKenneth D. Merry int 3332991554f2SKenneth D. Merry mpr_request_polled(struct mpr_softc *sc, struct mpr_command *cm) 3333991554f2SKenneth D. Merry { 3334991554f2SKenneth D. Merry int error, timeout = 0, rc; 3335991554f2SKenneth D. Merry struct timeval cur_time, start_time; 3336991554f2SKenneth D. Merry 3337991554f2SKenneth D. Merry error = 0; 3338991554f2SKenneth D. Merry 3339991554f2SKenneth D. Merry cm->cm_flags |= MPR_CM_FLAGS_POLLED; 3340991554f2SKenneth D. Merry cm->cm_complete = NULL; 3341991554f2SKenneth D. Merry mpr_map_command(sc, cm); 3342991554f2SKenneth D. Merry 3343991554f2SKenneth D. Merry getmicrotime(&start_time); 3344991554f2SKenneth D. Merry while ((cm->cm_flags & MPR_CM_FLAGS_COMPLETE) == 0) { 3345991554f2SKenneth D. Merry mpr_intr_locked(sc); 3346991554f2SKenneth D. Merry 3347991554f2SKenneth D. Merry if (mtx_owned(&sc->mpr_mtx)) 3348991554f2SKenneth D. Merry msleep(&sc->msleep_fake_chan, &sc->mpr_mtx, 0, 3349991554f2SKenneth D. Merry "mprpoll", hz/20); 3350991554f2SKenneth D. Merry else 3351991554f2SKenneth D. Merry pause("mprpoll", hz/20); 3352991554f2SKenneth D. Merry 3353991554f2SKenneth D. Merry /* 3354991554f2SKenneth D. Merry * Check for real-time timeout and fail if more than 60 seconds. 3355991554f2SKenneth D. Merry */ 3356991554f2SKenneth D. Merry getmicrotime(&cur_time); 3357991554f2SKenneth D. Merry timeout = cur_time.tv_sec - start_time.tv_sec; 3358991554f2SKenneth D. Merry if (timeout > 60) { 3359991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "polling failed\n"); 3360991554f2SKenneth D. Merry error = ETIMEDOUT; 3361991554f2SKenneth D. Merry break; 3362991554f2SKenneth D. Merry } 3363991554f2SKenneth D. Merry } 3364991554f2SKenneth D. Merry 3365991554f2SKenneth D. Merry if (error) { 3366991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, "Calling Reinit from %s\n", __func__); 3367991554f2SKenneth D. Merry rc = mpr_reinit(sc); 33687a2a6a1aSStephen McConnell mpr_dprint(sc, MPR_FAULT, "Reinit %s\n", (rc == 0) ? "success" : 33697a2a6a1aSStephen McConnell "failed"); 3370991554f2SKenneth D. Merry } 3371991554f2SKenneth D. Merry return (error); 3372991554f2SKenneth D. Merry } 3373991554f2SKenneth D. Merry 3374991554f2SKenneth D. Merry /* 3375991554f2SKenneth D. Merry * The MPT driver had a verbose interface for config pages. In this driver, 3376453130d9SPedro F. Giffuni * reduce it to much simpler terms, similar to the Linux driver. 3377991554f2SKenneth D. Merry */ 3378991554f2SKenneth D. Merry int 3379991554f2SKenneth D. Merry mpr_read_config_page(struct mpr_softc *sc, struct mpr_config_params *params) 3380991554f2SKenneth D. Merry { 3381991554f2SKenneth D. Merry MPI2_CONFIG_REQUEST *req; 3382991554f2SKenneth D. Merry struct mpr_command *cm; 3383991554f2SKenneth D. Merry int error; 3384991554f2SKenneth D. Merry 3385991554f2SKenneth D. Merry if (sc->mpr_flags & MPR_FLAGS_BUSY) { 3386991554f2SKenneth D. Merry return (EBUSY); 3387991554f2SKenneth D. Merry } 3388991554f2SKenneth D. Merry 3389991554f2SKenneth D. Merry cm = mpr_alloc_command(sc); 3390991554f2SKenneth D. Merry if (cm == NULL) { 3391991554f2SKenneth D. Merry return (EBUSY); 3392991554f2SKenneth D. Merry } 3393991554f2SKenneth D. Merry 3394991554f2SKenneth D. Merry req = (MPI2_CONFIG_REQUEST *)cm->cm_req; 3395991554f2SKenneth D. Merry req->Function = MPI2_FUNCTION_CONFIG; 3396991554f2SKenneth D. Merry req->Action = params->action; 3397991554f2SKenneth D. Merry req->SGLFlags = 0; 3398991554f2SKenneth D. Merry req->ChainOffset = 0; 3399991554f2SKenneth D. Merry req->PageAddress = params->page_address; 3400991554f2SKenneth D. Merry if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) { 3401991554f2SKenneth D. Merry MPI2_CONFIG_EXTENDED_PAGE_HEADER *hdr; 3402991554f2SKenneth D. Merry 3403991554f2SKenneth D. Merry hdr = ¶ms->hdr.Ext; 3404991554f2SKenneth D. Merry req->ExtPageType = hdr->ExtPageType; 3405991554f2SKenneth D. Merry req->ExtPageLength = hdr->ExtPageLength; 3406991554f2SKenneth D. Merry req->Header.PageType = MPI2_CONFIG_PAGETYPE_EXTENDED; 3407991554f2SKenneth D. Merry req->Header.PageLength = 0; /* Must be set to zero */ 3408991554f2SKenneth D. Merry req->Header.PageNumber = hdr->PageNumber; 3409991554f2SKenneth D. Merry req->Header.PageVersion = hdr->PageVersion; 3410991554f2SKenneth D. Merry } else { 3411991554f2SKenneth D. Merry MPI2_CONFIG_PAGE_HEADER *hdr; 3412991554f2SKenneth D. Merry 3413991554f2SKenneth D. Merry hdr = ¶ms->hdr.Struct; 3414991554f2SKenneth D. Merry req->Header.PageType = hdr->PageType; 3415991554f2SKenneth D. Merry req->Header.PageNumber = hdr->PageNumber; 3416991554f2SKenneth D. Merry req->Header.PageLength = hdr->PageLength; 3417991554f2SKenneth D. Merry req->Header.PageVersion = hdr->PageVersion; 3418991554f2SKenneth D. Merry } 3419991554f2SKenneth D. Merry 3420991554f2SKenneth D. Merry cm->cm_data = params->buffer; 3421991554f2SKenneth D. Merry cm->cm_length = params->length; 3422a2c14879SStephen McConnell if (cm->cm_data != NULL) { 3423991554f2SKenneth D. Merry cm->cm_sge = &req->PageBufferSGE; 3424991554f2SKenneth D. Merry cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION); 3425991554f2SKenneth D. Merry cm->cm_flags = MPR_CM_FLAGS_SGE_SIMPLE | MPR_CM_FLAGS_DATAIN; 3426a2c14879SStephen McConnell } else 3427a2c14879SStephen McConnell cm->cm_sge = NULL; 3428991554f2SKenneth D. Merry cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE; 3429991554f2SKenneth D. Merry 3430991554f2SKenneth D. Merry cm->cm_complete_data = params; 3431991554f2SKenneth D. Merry if (params->callback != NULL) { 3432991554f2SKenneth D. Merry cm->cm_complete = mpr_config_complete; 3433991554f2SKenneth D. Merry return (mpr_map_command(sc, cm)); 3434991554f2SKenneth D. Merry } else { 3435991554f2SKenneth D. Merry error = mpr_wait_command(sc, cm, 0, CAN_SLEEP); 3436991554f2SKenneth D. Merry if (error) { 3437991554f2SKenneth D. Merry mpr_dprint(sc, MPR_FAULT, 3438991554f2SKenneth D. Merry "Error %d reading config page\n", error); 3439991554f2SKenneth D. Merry mpr_free_command(sc, cm); 3440991554f2SKenneth D. Merry return (error); 3441991554f2SKenneth D. Merry } 3442991554f2SKenneth D. Merry mpr_config_complete(sc, cm); 3443991554f2SKenneth D. Merry } 3444991554f2SKenneth D. Merry 3445991554f2SKenneth D. Merry return (0); 3446991554f2SKenneth D. Merry } 3447991554f2SKenneth D. Merry 3448991554f2SKenneth D. Merry int 3449991554f2SKenneth D. Merry mpr_write_config_page(struct mpr_softc *sc, struct mpr_config_params *params) 3450991554f2SKenneth D. Merry { 3451991554f2SKenneth D. Merry return (EINVAL); 3452991554f2SKenneth D. Merry } 3453991554f2SKenneth D. Merry 3454991554f2SKenneth D. Merry static void 3455991554f2SKenneth D. Merry mpr_config_complete(struct mpr_softc *sc, struct mpr_command *cm) 3456991554f2SKenneth D. Merry { 3457991554f2SKenneth D. Merry MPI2_CONFIG_REPLY *reply; 3458991554f2SKenneth D. Merry struct mpr_config_params *params; 3459991554f2SKenneth D. Merry 3460991554f2SKenneth D. Merry MPR_FUNCTRACE(sc); 3461991554f2SKenneth D. Merry params = cm->cm_complete_data; 3462991554f2SKenneth D. Merry 3463991554f2SKenneth D. Merry if (cm->cm_data != NULL) { 3464991554f2SKenneth D. Merry bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, 3465991554f2SKenneth D. Merry BUS_DMASYNC_POSTREAD); 3466991554f2SKenneth D. Merry bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap); 3467991554f2SKenneth D. Merry } 3468991554f2SKenneth D. Merry 3469991554f2SKenneth D. Merry /* 3470991554f2SKenneth D. Merry * XXX KDM need to do more error recovery? This results in the 3471991554f2SKenneth D. Merry * device in question not getting probed. 3472991554f2SKenneth D. Merry */ 3473991554f2SKenneth D. Merry if ((cm->cm_flags & MPR_CM_FLAGS_ERROR_MASK) != 0) { 3474991554f2SKenneth D. Merry params->status = MPI2_IOCSTATUS_BUSY; 3475991554f2SKenneth D. Merry goto done; 3476991554f2SKenneth D. Merry } 3477991554f2SKenneth D. Merry 3478991554f2SKenneth D. Merry reply = (MPI2_CONFIG_REPLY *)cm->cm_reply; 3479991554f2SKenneth D. Merry if (reply == NULL) { 3480991554f2SKenneth D. Merry params->status = MPI2_IOCSTATUS_BUSY; 3481991554f2SKenneth D. Merry goto done; 3482991554f2SKenneth D. Merry } 3483991554f2SKenneth D. Merry params->status = reply->IOCStatus; 3484a2c14879SStephen McConnell if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) { 3485991554f2SKenneth D. Merry params->hdr.Ext.ExtPageType = reply->ExtPageType; 3486991554f2SKenneth D. Merry params->hdr.Ext.ExtPageLength = reply->ExtPageLength; 3487a2c14879SStephen McConnell params->hdr.Ext.PageType = reply->Header.PageType; 3488a2c14879SStephen McConnell params->hdr.Ext.PageNumber = reply->Header.PageNumber; 3489a2c14879SStephen McConnell params->hdr.Ext.PageVersion = reply->Header.PageVersion; 3490991554f2SKenneth D. Merry } else { 3491991554f2SKenneth D. Merry params->hdr.Struct.PageType = reply->Header.PageType; 3492991554f2SKenneth D. Merry params->hdr.Struct.PageNumber = reply->Header.PageNumber; 3493991554f2SKenneth D. Merry params->hdr.Struct.PageLength = reply->Header.PageLength; 3494991554f2SKenneth D. Merry params->hdr.Struct.PageVersion = reply->Header.PageVersion; 3495991554f2SKenneth D. Merry } 3496991554f2SKenneth D. Merry 3497991554f2SKenneth D. Merry done: 3498991554f2SKenneth D. Merry mpr_free_command(sc, cm); 3499991554f2SKenneth D. Merry if (params->callback != NULL) 3500991554f2SKenneth D. Merry params->callback(sc, params); 3501991554f2SKenneth D. Merry 3502991554f2SKenneth D. Merry return; 3503991554f2SKenneth D. Merry } 3504