1bb0ec6b3SJim Harris /*- 2718cf2ccSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3718cf2ccSPedro F. Giffuni * 4496a2752SJim Harris * Copyright (C) 2012-2014 Intel Corporation 5bb0ec6b3SJim Harris * All rights reserved. 6bb0ec6b3SJim Harris * 7bb0ec6b3SJim Harris * Redistribution and use in source and binary forms, with or without 8bb0ec6b3SJim Harris * modification, are permitted provided that the following conditions 9bb0ec6b3SJim Harris * are met: 10bb0ec6b3SJim Harris * 1. Redistributions of source code must retain the above copyright 11bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer. 12bb0ec6b3SJim Harris * 2. Redistributions in binary form must reproduce the above copyright 13bb0ec6b3SJim Harris * notice, this list of conditions and the following disclaimer in the 14bb0ec6b3SJim Harris * documentation and/or other materials provided with the distribution. 15bb0ec6b3SJim Harris * 16bb0ec6b3SJim Harris * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17bb0ec6b3SJim Harris * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18bb0ec6b3SJim Harris * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19bb0ec6b3SJim Harris * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20bb0ec6b3SJim Harris * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21bb0ec6b3SJim Harris * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22bb0ec6b3SJim Harris * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23bb0ec6b3SJim Harris * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24bb0ec6b3SJim Harris * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25bb0ec6b3SJim Harris * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26bb0ec6b3SJim Harris * SUCH DAMAGE. 27bb0ec6b3SJim Harris */ 28bb0ec6b3SJim Harris 29bb0ec6b3SJim Harris #include <sys/cdefs.h> 30bb0ec6b3SJim Harris __FBSDID("$FreeBSD$"); 31bb0ec6b3SJim Harris 32bb0ec6b3SJim Harris #include <sys/param.h> 33bb0ec6b3SJim Harris #include <sys/bus.h> 34bb0ec6b3SJim Harris #include <sys/conf.h> 35bb0ec6b3SJim Harris #include <sys/module.h> 36bb0ec6b3SJim Harris 37ad697276SJim Harris #include <vm/uma.h> 38ad697276SJim Harris 39bb0ec6b3SJim Harris #include "nvme_private.h" 40bb0ec6b3SJim Harris 41bb0ec6b3SJim Harris struct nvme_consumer { 42038a5ee4SJim Harris uint32_t id; 43038a5ee4SJim Harris nvme_cons_ns_fn_t ns_fn; 44038a5ee4SJim Harris nvme_cons_ctrlr_fn_t ctrlr_fn; 45038a5ee4SJim Harris nvme_cons_async_fn_t async_fn; 46232e2edbSJim Harris nvme_cons_fail_fn_t fail_fn; 47bb0ec6b3SJim Harris }; 48bb0ec6b3SJim Harris 49bb0ec6b3SJim Harris struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS]; 50038a5ee4SJim Harris #define INVALID_CONSUMER_ID 0xFFFF 51bb0ec6b3SJim Harris 52ad697276SJim Harris uma_zone_t nvme_request_zone; 53cb5b7c13SJim Harris int32_t nvme_retry_count; 54c75bdc04SWarner Losh 55ad697276SJim Harris 56bb0ec6b3SJim Harris MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations"); 57bb0ec6b3SJim Harris 58*f182f928SWarner Losh devclass_t nvme_devclass; 59bb0ec6b3SJim Harris 60bb0ec6b3SJim Harris static void 61ad697276SJim Harris nvme_init(void) 62ad697276SJim Harris { 63038a5ee4SJim Harris uint32_t i; 64038a5ee4SJim Harris 65ad697276SJim Harris nvme_request_zone = uma_zcreate("nvme_request", 66ad697276SJim Harris sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0); 67038a5ee4SJim Harris 68038a5ee4SJim Harris for (i = 0; i < NVME_MAX_CONSUMERS; i++) 69038a5ee4SJim Harris nvme_consumer[i].id = INVALID_CONSUMER_ID; 70ad697276SJim Harris } 71ad697276SJim Harris 72ad697276SJim Harris SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL); 73ad697276SJim Harris 74ad697276SJim Harris static void 75ad697276SJim Harris nvme_uninit(void) 76ad697276SJim Harris { 77ad697276SJim Harris uma_zdestroy(nvme_request_zone); 78ad697276SJim Harris } 79ad697276SJim Harris 80ad697276SJim Harris SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL); 81ad697276SJim Harris 82*f182f928SWarner Losh int 83c670f31fSNathan Whitehorn nvme_shutdown(device_t dev) 84bb0ec6b3SJim Harris { 85bb0ec6b3SJim Harris struct nvme_controller *ctrlr; 86bb0ec6b3SJim Harris 87c670f31fSNathan Whitehorn ctrlr = DEVICE2SOFTC(dev); 8856183abcSJim Harris nvme_ctrlr_shutdown(ctrlr); 89bb0ec6b3SJim Harris 90c670f31fSNathan Whitehorn return (0); 91bb0ec6b3SJim Harris } 92bb0ec6b3SJim Harris 93bb0ec6b3SJim Harris void 94bb0ec6b3SJim Harris nvme_dump_command(struct nvme_command *cmd) 95bb0ec6b3SJim Harris { 960d787e9bSWojciech Macek 974b52061eSDavid E. O'Brien printf( 980d787e9bSWojciech Macek "opc:%x f:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n", 999544e6dcSChuck Tuffli cmd->opc, cmd->fuse, cmd->cid, le32toh(cmd->nsid), 100bb0ec6b3SJim Harris cmd->rsvd2, cmd->rsvd3, 1010d787e9bSWojciech Macek (uintmax_t)le64toh(cmd->mptr), (uintmax_t)le64toh(cmd->prp1), (uintmax_t)le64toh(cmd->prp2), 1020d787e9bSWojciech Macek le32toh(cmd->cdw10), le32toh(cmd->cdw11), le32toh(cmd->cdw12), 1030d787e9bSWojciech Macek le32toh(cmd->cdw13), le32toh(cmd->cdw14), le32toh(cmd->cdw15)); 104bb0ec6b3SJim Harris } 105bb0ec6b3SJim Harris 106bb0ec6b3SJim Harris void 107bb0ec6b3SJim Harris nvme_dump_completion(struct nvme_completion *cpl) 108bb0ec6b3SJim Harris { 1090d787e9bSWojciech Macek uint8_t p, sc, sct, m, dnr; 1100d787e9bSWojciech Macek uint16_t status; 1110d787e9bSWojciech Macek 1120d787e9bSWojciech Macek status = le16toh(cpl->status); 1130d787e9bSWojciech Macek 1140d787e9bSWojciech Macek p = NVME_STATUS_GET_P(status); 1150d787e9bSWojciech Macek sc = NVME_STATUS_GET_SC(status); 1160d787e9bSWojciech Macek sct = NVME_STATUS_GET_SCT(status); 1170d787e9bSWojciech Macek m = NVME_STATUS_GET_M(status); 1180d787e9bSWojciech Macek dnr = NVME_STATUS_GET_DNR(status); 1190d787e9bSWojciech Macek 120bb0ec6b3SJim Harris printf("cdw0:%08x sqhd:%04x sqid:%04x " 121bb0ec6b3SJim Harris "cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n", 1220d787e9bSWojciech Macek le32toh(cpl->cdw0), le16toh(cpl->sqhd), le16toh(cpl->sqid), 1230d787e9bSWojciech Macek cpl->cid, p, sc, sct, m, dnr); 124bb0ec6b3SJim Harris } 125bb0ec6b3SJim Harris 126*f182f928SWarner Losh int 127bb0ec6b3SJim Harris nvme_attach(device_t dev) 128bb0ec6b3SJim Harris { 129bb0ec6b3SJim Harris struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 130bb0ec6b3SJim Harris int status; 131bb0ec6b3SJim Harris 132bb0ec6b3SJim Harris status = nvme_ctrlr_construct(ctrlr, dev); 133bb0ec6b3SJim Harris 1347aa27dbaSJim Harris if (status != 0) { 1357aa27dbaSJim Harris nvme_ctrlr_destruct(ctrlr, dev); 136bb0ec6b3SJim Harris return (status); 1377aa27dbaSJim Harris } 138bb0ec6b3SJim Harris 139bb0ec6b3SJim Harris /* 140*f182f928SWarner Losh * Reset controller twice to ensure we do a transition from cc.en==1 141bb0ec6b3SJim Harris * Reset controller twice to ensure we do a transition from cc.en==1 142bb0ec6b3SJim Harris * to cc.en==0. This is because we don't really know what status 143bb0ec6b3SJim Harris * the controller was left in when boot handed off to OS. 144bb0ec6b3SJim Harris */ 145b846efd7SJim Harris status = nvme_ctrlr_hw_reset(ctrlr); 1467aa27dbaSJim Harris if (status != 0) { 1477aa27dbaSJim Harris nvme_ctrlr_destruct(ctrlr, dev); 148bb0ec6b3SJim Harris return (status); 1497aa27dbaSJim Harris } 150bb0ec6b3SJim Harris 151b846efd7SJim Harris status = nvme_ctrlr_hw_reset(ctrlr); 1527aa27dbaSJim Harris if (status != 0) { 1537aa27dbaSJim Harris nvme_ctrlr_destruct(ctrlr, dev); 154bb0ec6b3SJim Harris return (status); 1557aa27dbaSJim Harris } 156bb0ec6b3SJim Harris 157be34f216SJim Harris ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook; 158bb0ec6b3SJim Harris ctrlr->config_hook.ich_arg = ctrlr; 159bb0ec6b3SJim Harris 160bb0ec6b3SJim Harris config_intrhook_establish(&ctrlr->config_hook); 161bb0ec6b3SJim Harris 162bb0ec6b3SJim Harris return (0); 163bb0ec6b3SJim Harris } 164bb0ec6b3SJim Harris 165*f182f928SWarner Losh int 166bb0ec6b3SJim Harris nvme_detach (device_t dev) 167bb0ec6b3SJim Harris { 168bb0ec6b3SJim Harris struct nvme_controller *ctrlr = DEVICE2SOFTC(dev); 169bb0ec6b3SJim Harris 170990e741cSJim Harris nvme_ctrlr_destruct(ctrlr, dev); 171bb0ec6b3SJim Harris return (0); 172bb0ec6b3SJim Harris } 173bb0ec6b3SJim Harris 174bb0ec6b3SJim Harris static void 175496a2752SJim Harris nvme_notify(struct nvme_consumer *cons, 176496a2752SJim Harris struct nvme_controller *ctrlr) 177bb0ec6b3SJim Harris { 178038a5ee4SJim Harris struct nvme_namespace *ns; 179038a5ee4SJim Harris void *ctrlr_cookie; 180496a2752SJim Harris int cmpset, ns_idx; 181bb0ec6b3SJim Harris 182496a2752SJim Harris /* 183496a2752SJim Harris * The consumer may register itself after the nvme devices 184496a2752SJim Harris * have registered with the kernel, but before the 185496a2752SJim Harris * driver has completed initialization. In that case, 186496a2752SJim Harris * return here, and when initialization completes, the 187496a2752SJim Harris * controller will make sure the consumer gets notified. 188496a2752SJim Harris */ 189496a2752SJim Harris if (!ctrlr->is_initialized) 190bb0ec6b3SJim Harris return; 191bb0ec6b3SJim Harris 192496a2752SJim Harris cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1); 193496a2752SJim Harris if (cmpset == 0) 194496a2752SJim Harris return; 195496a2752SJim Harris 196038a5ee4SJim Harris if (cons->ctrlr_fn != NULL) 197038a5ee4SJim Harris ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr); 198038a5ee4SJim Harris else 19951b92c1aSAlexander Motin ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de; 200038a5ee4SJim Harris ctrlr->cons_cookie[cons->id] = ctrlr_cookie; 20151b92c1aSAlexander Motin 20251b92c1aSAlexander Motin /* ctrlr_fn has failed. Nothing to notify here any more. */ 20351b92c1aSAlexander Motin if (ctrlr_cookie == NULL) 20451b92c1aSAlexander Motin return; 20551b92c1aSAlexander Motin 206086d23cfSJim Harris if (ctrlr->is_failed) { 20751b92c1aSAlexander Motin ctrlr->cons_cookie[cons->id] = NULL; 208086d23cfSJim Harris if (cons->fail_fn != NULL) 209086d23cfSJim Harris (*cons->fail_fn)(ctrlr_cookie); 210086d23cfSJim Harris /* 211086d23cfSJim Harris * Do not notify consumers about the namespaces of a 212086d23cfSJim Harris * failed controller. 213086d23cfSJim Harris */ 214496a2752SJim Harris return; 215086d23cfSJim Harris } 216a8a18dd5SWarner Losh for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) { 217038a5ee4SJim Harris ns = &ctrlr->ns[ns_idx]; 218a8a18dd5SWarner Losh if (ns->data.nsze == 0) 219a8a18dd5SWarner Losh continue; 220038a5ee4SJim Harris if (cons->ns_fn != NULL) 221038a5ee4SJim Harris ns->cons_cookie[cons->id] = 222038a5ee4SJim Harris (*cons->ns_fn)(ns, ctrlr_cookie); 223038a5ee4SJim Harris } 224bb0ec6b3SJim Harris } 225bb0ec6b3SJim Harris 226496a2752SJim Harris void 227496a2752SJim Harris nvme_notify_new_controller(struct nvme_controller *ctrlr) 228496a2752SJim Harris { 229496a2752SJim Harris int i; 230496a2752SJim Harris 231496a2752SJim Harris for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 232496a2752SJim Harris if (nvme_consumer[i].id != INVALID_CONSUMER_ID) { 233496a2752SJim Harris nvme_notify(&nvme_consumer[i], ctrlr); 234496a2752SJim Harris } 235496a2752SJim Harris } 236496a2752SJim Harris } 237496a2752SJim Harris 238496a2752SJim Harris static void 239496a2752SJim Harris nvme_notify_new_consumer(struct nvme_consumer *cons) 240496a2752SJim Harris { 241496a2752SJim Harris device_t *devlist; 242496a2752SJim Harris struct nvme_controller *ctrlr; 243496a2752SJim Harris int dev_idx, devcount; 244496a2752SJim Harris 245496a2752SJim Harris if (devclass_get_devices(nvme_devclass, &devlist, &devcount)) 246496a2752SJim Harris return; 247496a2752SJim Harris 248496a2752SJim Harris for (dev_idx = 0; dev_idx < devcount; dev_idx++) { 249496a2752SJim Harris ctrlr = DEVICE2SOFTC(devlist[dev_idx]); 250496a2752SJim Harris nvme_notify(cons, ctrlr); 251496a2752SJim Harris } 252496a2752SJim Harris 253bb0ec6b3SJim Harris free(devlist, M_TEMP); 254bb0ec6b3SJim Harris } 255bb0ec6b3SJim Harris 256038a5ee4SJim Harris void 257038a5ee4SJim Harris nvme_notify_async_consumers(struct nvme_controller *ctrlr, 2580d7e13ecSJim Harris const struct nvme_completion *async_cpl, 2590d7e13ecSJim Harris uint32_t log_page_id, void *log_page_buffer, 2600d7e13ecSJim Harris uint32_t log_page_size) 261038a5ee4SJim Harris { 262038a5ee4SJim Harris struct nvme_consumer *cons; 26351b92c1aSAlexander Motin void *ctrlr_cookie; 264038a5ee4SJim Harris uint32_t i; 265038a5ee4SJim Harris 266038a5ee4SJim Harris for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 267038a5ee4SJim Harris cons = &nvme_consumer[i]; 26851b92c1aSAlexander Motin if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL && 26951b92c1aSAlexander Motin (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) { 27051b92c1aSAlexander Motin (*cons->async_fn)(ctrlr_cookie, async_cpl, 2710d7e13ecSJim Harris log_page_id, log_page_buffer, log_page_size); 272038a5ee4SJim Harris } 273038a5ee4SJim Harris } 27451b92c1aSAlexander Motin } 275038a5ee4SJim Harris 276232e2edbSJim Harris void 277232e2edbSJim Harris nvme_notify_fail_consumers(struct nvme_controller *ctrlr) 278232e2edbSJim Harris { 279232e2edbSJim Harris struct nvme_consumer *cons; 28051b92c1aSAlexander Motin void *ctrlr_cookie; 281232e2edbSJim Harris uint32_t i; 282232e2edbSJim Harris 2830e1fd2ddSJim Harris /* 2840e1fd2ddSJim Harris * This controller failed during initialization (i.e. IDENTIFY 2850e1fd2ddSJim Harris * command failed or timed out). Do not notify any nvme 2860e1fd2ddSJim Harris * consumers of the failure here, since the consumer does not 2870e1fd2ddSJim Harris * even know about the controller yet. 2880e1fd2ddSJim Harris */ 2890e1fd2ddSJim Harris if (!ctrlr->is_initialized) 2900e1fd2ddSJim Harris return; 2910e1fd2ddSJim Harris 292232e2edbSJim Harris for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 293232e2edbSJim Harris cons = &nvme_consumer[i]; 29451b92c1aSAlexander Motin if (cons->id != INVALID_CONSUMER_ID && 29551b92c1aSAlexander Motin (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) { 29651b92c1aSAlexander Motin ctrlr->cons_cookie[i] = NULL; 29751b92c1aSAlexander Motin if (cons->fail_fn != NULL) 29851b92c1aSAlexander Motin cons->fail_fn(ctrlr_cookie); 29951b92c1aSAlexander Motin } 300232e2edbSJim Harris } 301232e2edbSJim Harris } 302232e2edbSJim Harris 303f439e3a4SAlexander Motin void 304f439e3a4SAlexander Motin nvme_notify_ns(struct nvme_controller *ctrlr, int nsid) 305f439e3a4SAlexander Motin { 306f439e3a4SAlexander Motin struct nvme_consumer *cons; 307f439e3a4SAlexander Motin struct nvme_namespace *ns = &ctrlr->ns[nsid - 1]; 30851b92c1aSAlexander Motin void *ctrlr_cookie; 309f439e3a4SAlexander Motin uint32_t i; 310f439e3a4SAlexander Motin 311f439e3a4SAlexander Motin if (!ctrlr->is_initialized) 312f439e3a4SAlexander Motin return; 313f439e3a4SAlexander Motin 314f439e3a4SAlexander Motin for (i = 0; i < NVME_MAX_CONSUMERS; i++) { 315f439e3a4SAlexander Motin cons = &nvme_consumer[i]; 31651b92c1aSAlexander Motin if (cons->id != INVALID_CONSUMER_ID && cons->ns_fn != NULL && 31751b92c1aSAlexander Motin (ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) 31851b92c1aSAlexander Motin ns->cons_cookie[i] = (*cons->ns_fn)(ns, ctrlr_cookie); 319f439e3a4SAlexander Motin } 320f439e3a4SAlexander Motin } 321f439e3a4SAlexander Motin 322bb0ec6b3SJim Harris struct nvme_consumer * 323038a5ee4SJim Harris nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn, 324232e2edbSJim Harris nvme_cons_async_fn_t async_fn, 325232e2edbSJim Harris nvme_cons_fail_fn_t fail_fn) 326bb0ec6b3SJim Harris { 327bb0ec6b3SJim Harris int i; 328bb0ec6b3SJim Harris 329bb0ec6b3SJim Harris /* 330204498d7SWarner Losh * TODO: add locking around consumer registration. 331bb0ec6b3SJim Harris */ 332bb0ec6b3SJim Harris for (i = 0; i < NVME_MAX_CONSUMERS; i++) 333038a5ee4SJim Harris if (nvme_consumer[i].id == INVALID_CONSUMER_ID) { 334038a5ee4SJim Harris nvme_consumer[i].id = i; 335038a5ee4SJim Harris nvme_consumer[i].ns_fn = ns_fn; 336038a5ee4SJim Harris nvme_consumer[i].ctrlr_fn = ctrlr_fn; 337038a5ee4SJim Harris nvme_consumer[i].async_fn = async_fn; 338232e2edbSJim Harris nvme_consumer[i].fail_fn = fail_fn; 339bb0ec6b3SJim Harris 340496a2752SJim Harris nvme_notify_new_consumer(&nvme_consumer[i]); 341bb0ec6b3SJim Harris return (&nvme_consumer[i]); 342bb0ec6b3SJim Harris } 343bb0ec6b3SJim Harris 344bb0ec6b3SJim Harris printf("nvme(4): consumer not registered - no slots available\n"); 345bb0ec6b3SJim Harris return (NULL); 346bb0ec6b3SJim Harris } 347bb0ec6b3SJim Harris 348bb0ec6b3SJim Harris void 349bb0ec6b3SJim Harris nvme_unregister_consumer(struct nvme_consumer *consumer) 350bb0ec6b3SJim Harris { 351bb0ec6b3SJim Harris 352038a5ee4SJim Harris consumer->id = INVALID_CONSUMER_ID; 353bb0ec6b3SJim Harris } 354bb0ec6b3SJim Harris 355955910a9SJim Harris void 356955910a9SJim Harris nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl) 357955910a9SJim Harris { 358955910a9SJim Harris struct nvme_completion_poll_status *status = arg; 359955910a9SJim Harris 360955910a9SJim Harris /* 361955910a9SJim Harris * Copy status into the argument passed by the caller, so that 362955910a9SJim Harris * the caller can check the status to determine if the 363955910a9SJim Harris * the request passed or failed. 364955910a9SJim Harris */ 365955910a9SJim Harris memcpy(&status->cpl, cpl, sizeof(*cpl)); 36629077eb4SWarner Losh atomic_store_rel_int(&status->done, 1); 367955910a9SJim Harris } 368