1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2023-2024 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 */ 7 8 #include <sys/types.h> 9 #include <sys/malloc.h> 10 #include <sys/memdesc.h> 11 #include <sys/refcount.h> 12 13 #include <cam/cam.h> 14 #include <cam/cam_ccb.h> 15 #include <cam/cam_sim.h> 16 #include <cam/cam_xpt_sim.h> 17 #include <cam/cam_debug.h> 18 19 #include <dev/nvmf/host/nvmf_var.h> 20 21 /* 22 * The I/O completion may trigger after the received CQE if the I/O 23 * used a zero-copy mbuf that isn't harvested until after the NIC 24 * driver processes TX completions. Use spriv_field0 to as a refcount. 25 * 26 * Store any I/O error returned in spriv_field1. 27 */ 28 static __inline u_int * 29 ccb_refs(union ccb *ccb) 30 { 31 return ((u_int *)&ccb->ccb_h.spriv_field0); 32 } 33 34 #define spriv_ioerror spriv_field1 35 36 static void 37 nvmf_ccb_done(union ccb *ccb) 38 { 39 if (!refcount_release(ccb_refs(ccb))) 40 return; 41 42 if (nvmf_cqe_aborted(&ccb->nvmeio.cpl)) { 43 struct cam_sim *sim = xpt_path_sim(ccb->ccb_h.path); 44 struct nvmf_softc *sc = cam_sim_softc(sim); 45 46 if (nvmf_fail_disconnect || sc->sim_shutdown) 47 ccb->ccb_h.status = CAM_DEV_NOT_THERE; 48 else 49 ccb->ccb_h.status = CAM_REQUEUE_REQ; 50 xpt_done(ccb); 51 } else if (ccb->nvmeio.cpl.status != 0) { 52 ccb->ccb_h.status = CAM_NVME_STATUS_ERROR; 53 xpt_done(ccb); 54 } else if (ccb->ccb_h.spriv_ioerror != 0) { 55 KASSERT(ccb->ccb_h.spriv_ioerror != EJUSTRETURN, 56 ("%s: zero sized transfer without CQE error", __func__)); 57 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 58 xpt_done(ccb); 59 } else { 60 ccb->ccb_h.status = CAM_REQ_CMP; 61 xpt_done(ccb); 62 } 63 } 64 65 static void 66 nvmf_ccb_io_complete(void *arg, size_t xfered, int error) 67 { 68 union ccb *ccb = arg; 69 70 /* 71 * TODO: Reporting partial completions requires extending 72 * nvmeio to support resid and updating nda to handle partial 73 * reads, either by returning partial success (or an error) to 74 * the caller, or retrying all or part of the request. 75 */ 76 ccb->ccb_h.spriv_ioerror = error; 77 if (error == 0) { 78 if (xfered == 0) { 79 #ifdef INVARIANTS 80 /* 81 * If the request fails with an error in the CQE 82 * there will be no data transferred but also no 83 * I/O error. 84 */ 85 ccb->ccb_h.spriv_ioerror = EJUSTRETURN; 86 #endif 87 } else 88 KASSERT(xfered == ccb->nvmeio.dxfer_len, 89 ("%s: partial CCB completion", __func__)); 90 } 91 92 nvmf_ccb_done(ccb); 93 } 94 95 static void 96 nvmf_ccb_complete(void *arg, const struct nvme_completion *cqe) 97 { 98 union ccb *ccb = arg; 99 100 ccb->nvmeio.cpl = *cqe; 101 nvmf_ccb_done(ccb); 102 } 103 104 static void 105 nvmf_sim_io(struct nvmf_softc *sc, union ccb *ccb) 106 { 107 struct ccb_nvmeio *nvmeio = &ccb->nvmeio; 108 struct memdesc mem; 109 struct nvmf_request *req; 110 struct nvmf_host_qpair *qp; 111 112 mtx_lock(&sc->sim_mtx); 113 if (sc->sim_disconnected) { 114 mtx_unlock(&sc->sim_mtx); 115 if (nvmf_fail_disconnect || sc->sim_shutdown) 116 nvmeio->ccb_h.status = CAM_DEV_NOT_THERE; 117 else 118 nvmeio->ccb_h.status = CAM_REQUEUE_REQ; 119 xpt_done(ccb); 120 return; 121 } 122 if (nvmeio->ccb_h.func_code == XPT_NVME_IO) 123 qp = nvmf_select_io_queue(sc); 124 else 125 qp = sc->admin; 126 req = nvmf_allocate_request(qp, &nvmeio->cmd, nvmf_ccb_complete, 127 ccb, M_NOWAIT); 128 mtx_unlock(&sc->sim_mtx); 129 if (req == NULL) { 130 nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL; 131 xpt_done(ccb); 132 return; 133 } 134 135 if (nvmeio->dxfer_len != 0) { 136 refcount_init(ccb_refs(ccb), 2); 137 mem = memdesc_ccb(ccb); 138 nvmf_capsule_append_data(req->nc, &mem, nvmeio->dxfer_len, 139 (ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT, 140 nvmf_ccb_io_complete, ccb); 141 } else 142 refcount_init(ccb_refs(ccb), 1); 143 144 /* 145 * Clear spriv_ioerror as it can hold an earlier error if this 146 * CCB was aborted and has been retried. 147 */ 148 ccb->ccb_h.spriv_ioerror = 0; 149 KASSERT(ccb->ccb_h.status == CAM_REQ_INPROG, 150 ("%s: incoming CCB is not in-progress", __func__)); 151 ccb->ccb_h.status |= CAM_SIM_QUEUED; 152 nvmf_submit_request(req); 153 } 154 155 static void 156 nvmf_sim_action(struct cam_sim *sim, union ccb *ccb) 157 { 158 struct nvmf_softc *sc = cam_sim_softc(sim); 159 160 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 161 ("nvmf_sim_action: func= %#x\n", 162 ccb->ccb_h.func_code)); 163 164 switch (ccb->ccb_h.func_code) { 165 case XPT_PATH_INQ: /* Path routing inquiry */ 166 { 167 struct ccb_pathinq *cpi = &ccb->cpi; 168 169 cpi->version_num = 1; 170 cpi->hba_inquiry = 0; 171 cpi->target_sprt = 0; 172 cpi->hba_misc = PIM_UNMAPPED | PIM_NOSCAN; 173 cpi->hba_eng_cnt = 0; 174 cpi->max_target = 0; 175 cpi->max_lun = sc->cdata->nn; 176 cpi->async_flags = 0; 177 cpi->hpath_id = 0; 178 cpi->initiator_id = 0; 179 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 180 strlcpy(cpi->hba_vid, "NVMeoF", HBA_IDLEN); 181 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 182 cpi->unit_number = cam_sim_unit(sim); 183 cpi->bus_id = 0; 184 185 /* XXX: Same as iSCSI. */ 186 cpi->base_transfer_speed = 150000; 187 cpi->protocol = PROTO_NVME; 188 cpi->protocol_version = sc->vs; 189 cpi->transport = XPORT_NVMF; 190 cpi->transport_version = sc->vs; 191 cpi->xport_specific.nvmf.nsid = 192 xpt_path_lun_id(ccb->ccb_h.path); 193 cpi->xport_specific.nvmf.trtype = sc->trtype; 194 strlcpy(cpi->xport_specific.nvmf.dev_name, 195 device_get_nameunit(sc->dev), 196 sizeof(cpi->xport_specific.nvmf.dev_name)); 197 cpi->maxio = sc->max_xfer_size; 198 cpi->hba_vendor = 0; 199 cpi->hba_device = 0; 200 cpi->hba_subvendor = 0; 201 cpi->hba_subdevice = 0; 202 cpi->ccb_h.status = CAM_REQ_CMP; 203 break; 204 } 205 case XPT_GET_TRAN_SETTINGS: /* Get transport settings */ 206 { 207 struct ccb_trans_settings *cts = &ccb->cts; 208 struct ccb_trans_settings_nvme *nvme; 209 struct ccb_trans_settings_nvmf *nvmf; 210 211 cts->protocol = PROTO_NVME; 212 cts->protocol_version = sc->vs; 213 cts->transport = XPORT_NVMF; 214 cts->transport_version = sc->vs; 215 216 nvme = &cts->proto_specific.nvme; 217 nvme->valid = CTS_NVME_VALID_SPEC; 218 nvme->spec = sc->vs; 219 220 nvmf = &cts->xport_specific.nvmf; 221 nvmf->valid = CTS_NVMF_VALID_TRTYPE; 222 nvmf->trtype = sc->trtype; 223 cts->ccb_h.status = CAM_REQ_CMP; 224 break; 225 } 226 case XPT_SET_TRAN_SETTINGS: /* Set transport settings */ 227 /* 228 * No transfer settings can be set, but nvme_xpt sends 229 * this anyway. 230 */ 231 ccb->ccb_h.status = CAM_REQ_CMP; 232 break; 233 case XPT_NVME_IO: /* Execute the requested I/O */ 234 case XPT_NVME_ADMIN: /* or Admin operation */ 235 nvmf_sim_io(sc, ccb); 236 return; 237 default: 238 /* XXX */ 239 device_printf(sc->dev, "unhandled sim function %#x\n", 240 ccb->ccb_h.func_code); 241 ccb->ccb_h.status = CAM_REQ_INVALID; 242 break; 243 } 244 xpt_done(ccb); 245 } 246 247 int 248 nvmf_init_sim(struct nvmf_softc *sc) 249 { 250 struct cam_devq *devq; 251 int max_trans; 252 253 max_trans = sc->max_pending_io * 3 / 4; 254 devq = cam_simq_alloc(max_trans); 255 if (devq == NULL) { 256 device_printf(sc->dev, "Failed to allocate CAM simq\n"); 257 return (ENOMEM); 258 } 259 260 mtx_init(&sc->sim_mtx, "nvmf sim", NULL, MTX_DEF); 261 sc->sim = cam_sim_alloc(nvmf_sim_action, NULL, "nvme", sc, 262 device_get_unit(sc->dev), NULL, max_trans, max_trans, devq); 263 if (sc->sim == NULL) { 264 device_printf(sc->dev, "Failed to allocate CAM sim\n"); 265 cam_simq_free(devq); 266 mtx_destroy(&sc->sim_mtx); 267 return (ENXIO); 268 } 269 if (xpt_bus_register(sc->sim, sc->dev, 0) != CAM_SUCCESS) { 270 device_printf(sc->dev, "Failed to create CAM bus\n"); 271 cam_sim_free(sc->sim, TRUE); 272 mtx_destroy(&sc->sim_mtx); 273 return (ENXIO); 274 } 275 if (xpt_create_path(&sc->path, NULL, cam_sim_path(sc->sim), 276 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 277 device_printf(sc->dev, "Failed to create CAM path\n"); 278 xpt_bus_deregister(cam_sim_path(sc->sim)); 279 cam_sim_free(sc->sim, TRUE); 280 mtx_destroy(&sc->sim_mtx); 281 return (ENXIO); 282 } 283 return (0); 284 } 285 286 void 287 nvmf_sim_rescan_ns(struct nvmf_softc *sc, uint32_t id) 288 { 289 union ccb *ccb; 290 291 ccb = xpt_alloc_ccb_nowait(); 292 if (ccb == NULL) { 293 device_printf(sc->dev, 294 "unable to alloc CCB for rescan of namespace %u\n", id); 295 return; 296 } 297 298 /* 299 * As with nvme_sim, map NVMe namespace IDs onto CAM unit 300 * LUNs. 301 */ 302 if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->sim), 0, 303 id) != CAM_REQ_CMP) { 304 device_printf(sc->dev, 305 "Unable to create path for rescan of namespace %u\n", id); 306 xpt_free_ccb(ccb); 307 return; 308 } 309 xpt_rescan(ccb); 310 } 311 312 void 313 nvmf_disconnect_sim(struct nvmf_softc *sc) 314 { 315 mtx_lock(&sc->sim_mtx); 316 sc->sim_disconnected = true; 317 xpt_freeze_simq(sc->sim, 1); 318 mtx_unlock(&sc->sim_mtx); 319 } 320 321 void 322 nvmf_reconnect_sim(struct nvmf_softc *sc) 323 { 324 mtx_lock(&sc->sim_mtx); 325 sc->sim_disconnected = false; 326 mtx_unlock(&sc->sim_mtx); 327 xpt_release_simq(sc->sim, 1); 328 } 329 330 void 331 nvmf_shutdown_sim(struct nvmf_softc *sc) 332 { 333 mtx_lock(&sc->sim_mtx); 334 sc->sim_shutdown = true; 335 mtx_unlock(&sc->sim_mtx); 336 xpt_release_simq(sc->sim, 1); 337 } 338 339 void 340 nvmf_destroy_sim(struct nvmf_softc *sc) 341 { 342 xpt_async(AC_LOST_DEVICE, sc->path, NULL); 343 if (sc->sim_disconnected) 344 xpt_release_simq(sc->sim, 1); 345 xpt_free_path(sc->path); 346 xpt_bus_deregister(cam_sim_path(sc->sim)); 347 cam_sim_free(sc->sim, TRUE); 348 mtx_destroy(&sc->sim_mtx); 349 } 350