1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2023-2024 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 */ 7 8 #include <sys/types.h> 9 #include <sys/malloc.h> 10 #include <sys/memdesc.h> 11 #include <sys/refcount.h> 12 13 #include <cam/cam.h> 14 #include <cam/cam_ccb.h> 15 #include <cam/cam_sim.h> 16 #include <cam/cam_xpt_sim.h> 17 #include <cam/cam_debug.h> 18 19 #include <dev/nvmf/host/nvmf_var.h> 20 21 /* 22 * The I/O completion may trigger after the received CQE if the I/O 23 * used a zero-copy mbuf that isn't harvested until after the NIC 24 * driver processes TX completions. Use spriv_field0 to as a refcount. 25 * 26 * Store any I/O error returned in spriv_field1. 27 */ 28 static __inline u_int * 29 ccb_refs(union ccb *ccb) 30 { 31 return ((u_int *)&ccb->ccb_h.spriv_field0); 32 } 33 34 #define spriv_ioerror spriv_field1 35 36 static void 37 nvmf_ccb_done(union ccb *ccb) 38 { 39 if (!refcount_release(ccb_refs(ccb))) 40 return; 41 42 if (nvmf_cqe_aborted(&ccb->nvmeio.cpl)) { 43 ccb->ccb_h.status = CAM_REQUEUE_REQ; 44 xpt_done(ccb); 45 } else if (ccb->nvmeio.cpl.status != 0) { 46 ccb->ccb_h.status = CAM_NVME_STATUS_ERROR; 47 xpt_done(ccb); 48 } else if (ccb->ccb_h.spriv_ioerror != 0) { 49 KASSERT(ccb->ccb_h.spriv_ioerror != EJUSTRETURN, 50 ("%s: zero sized transfer without CQE error", __func__)); 51 ccb->ccb_h.status = CAM_REQ_CMP_ERR; 52 xpt_done(ccb); 53 } else { 54 ccb->ccb_h.status = CAM_REQ_CMP; 55 xpt_done_direct(ccb); 56 } 57 } 58 59 static void 60 nvmf_ccb_io_complete(void *arg, size_t xfered, int error) 61 { 62 union ccb *ccb = arg; 63 64 /* 65 * TODO: Reporting partial completions requires extending 66 * nvmeio to support resid and updating nda to handle partial 67 * reads, either by returning partial success (or an error) to 68 * the caller, or retrying all or part of the request. 69 */ 70 ccb->ccb_h.spriv_ioerror = error; 71 if (error == 0) { 72 if (xfered == 0) { 73 #ifdef INVARIANTS 74 /* 75 * If the request fails with an error in the CQE 76 * there will be no data transferred but also no 77 * I/O error. 78 */ 79 ccb->ccb_h.spriv_ioerror = EJUSTRETURN; 80 #endif 81 } else 82 KASSERT(xfered == ccb->nvmeio.dxfer_len, 83 ("%s: partial CCB completion", __func__)); 84 } 85 86 nvmf_ccb_done(ccb); 87 } 88 89 static void 90 nvmf_ccb_complete(void *arg, const struct nvme_completion *cqe) 91 { 92 union ccb *ccb = arg; 93 94 ccb->nvmeio.cpl = *cqe; 95 nvmf_ccb_done(ccb); 96 } 97 98 static void 99 nvmf_sim_io(struct nvmf_softc *sc, union ccb *ccb) 100 { 101 struct ccb_nvmeio *nvmeio = &ccb->nvmeio; 102 struct memdesc mem; 103 struct nvmf_request *req; 104 struct nvmf_host_qpair *qp; 105 106 mtx_lock(&sc->sim_mtx); 107 if (sc->sim_disconnected) { 108 mtx_unlock(&sc->sim_mtx); 109 nvmeio->ccb_h.status = CAM_REQUEUE_REQ; 110 xpt_done(ccb); 111 return; 112 } 113 if (nvmeio->ccb_h.func_code == XPT_NVME_IO) 114 qp = nvmf_select_io_queue(sc); 115 else 116 qp = sc->admin; 117 req = nvmf_allocate_request(qp, &nvmeio->cmd, nvmf_ccb_complete, 118 ccb, M_NOWAIT); 119 if (req == NULL) { 120 mtx_unlock(&sc->sim_mtx); 121 nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL; 122 xpt_done(ccb); 123 return; 124 } 125 126 if (nvmeio->dxfer_len != 0) { 127 refcount_init(ccb_refs(ccb), 2); 128 mem = memdesc_ccb(ccb); 129 nvmf_capsule_append_data(req->nc, &mem, nvmeio->dxfer_len, 130 (ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT, 131 nvmf_ccb_io_complete, ccb); 132 } else 133 refcount_init(ccb_refs(ccb), 1); 134 135 /* 136 * Clear spriv_ioerror as it can hold an earlier error if this 137 * CCB was aborted and has been retried. 138 */ 139 ccb->ccb_h.spriv_ioerror = 0; 140 KASSERT(ccb->ccb_h.status == CAM_REQ_INPROG, 141 ("%s: incoming CCB is not in-progress", __func__)); 142 ccb->ccb_h.status |= CAM_SIM_QUEUED; 143 nvmf_submit_request(req); 144 mtx_unlock(&sc->sim_mtx); 145 } 146 147 static void 148 nvmf_sim_action(struct cam_sim *sim, union ccb *ccb) 149 { 150 struct nvmf_softc *sc = cam_sim_softc(sim); 151 152 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, 153 ("nvmf_sim_action: func= %#x\n", 154 ccb->ccb_h.func_code)); 155 156 switch (ccb->ccb_h.func_code) { 157 case XPT_PATH_INQ: /* Path routing inquiry */ 158 { 159 struct ccb_pathinq *cpi = &ccb->cpi; 160 161 cpi->version_num = 1; 162 cpi->hba_inquiry = 0; 163 cpi->target_sprt = 0; 164 cpi->hba_misc = PIM_UNMAPPED | PIM_NOSCAN; 165 cpi->hba_eng_cnt = 0; 166 cpi->max_target = 0; 167 cpi->max_lun = sc->cdata->nn; 168 cpi->async_flags = 0; 169 cpi->hpath_id = 0; 170 cpi->initiator_id = 0; 171 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 172 strlcpy(cpi->hba_vid, "NVMeoF", HBA_IDLEN); 173 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 174 cpi->unit_number = cam_sim_unit(sim); 175 cpi->bus_id = 0; 176 177 /* XXX: Same as iSCSI. */ 178 cpi->base_transfer_speed = 150000; 179 cpi->protocol = PROTO_NVME; 180 cpi->protocol_version = sc->vs; 181 cpi->transport = XPORT_NVMF; 182 cpi->transport_version = sc->vs; 183 cpi->xport_specific.nvmf.nsid = 184 xpt_path_lun_id(ccb->ccb_h.path); 185 cpi->xport_specific.nvmf.trtype = sc->trtype; 186 strlcpy(cpi->xport_specific.nvmf.dev_name, 187 device_get_nameunit(sc->dev), 188 sizeof(cpi->xport_specific.nvmf.dev_name)); 189 cpi->maxio = sc->max_xfer_size; 190 cpi->hba_vendor = 0; 191 cpi->hba_device = 0; 192 cpi->hba_subvendor = 0; 193 cpi->hba_subdevice = 0; 194 cpi->ccb_h.status = CAM_REQ_CMP; 195 break; 196 } 197 case XPT_GET_TRAN_SETTINGS: /* Get transport settings */ 198 { 199 struct ccb_trans_settings *cts = &ccb->cts; 200 struct ccb_trans_settings_nvme *nvme; 201 struct ccb_trans_settings_nvmf *nvmf; 202 203 cts->protocol = PROTO_NVME; 204 cts->protocol_version = sc->vs; 205 cts->transport = XPORT_NVMF; 206 cts->transport_version = sc->vs; 207 208 nvme = &cts->proto_specific.nvme; 209 nvme->valid = CTS_NVME_VALID_SPEC; 210 nvme->spec = sc->vs; 211 212 nvmf = &cts->xport_specific.nvmf; 213 nvmf->valid = CTS_NVMF_VALID_TRTYPE; 214 nvmf->trtype = sc->trtype; 215 cts->ccb_h.status = CAM_REQ_CMP; 216 break; 217 } 218 case XPT_SET_TRAN_SETTINGS: /* Set transport settings */ 219 /* 220 * No transfer settings can be set, but nvme_xpt sends 221 * this anyway. 222 */ 223 ccb->ccb_h.status = CAM_REQ_CMP; 224 break; 225 case XPT_NVME_IO: /* Execute the requested I/O */ 226 case XPT_NVME_ADMIN: /* or Admin operation */ 227 nvmf_sim_io(sc, ccb); 228 return; 229 default: 230 /* XXX */ 231 device_printf(sc->dev, "unhandled sim function %#x\n", 232 ccb->ccb_h.func_code); 233 ccb->ccb_h.status = CAM_REQ_INVALID; 234 break; 235 } 236 xpt_done(ccb); 237 } 238 239 int 240 nvmf_init_sim(struct nvmf_softc *sc) 241 { 242 struct cam_devq *devq; 243 int max_trans; 244 245 max_trans = sc->max_pending_io * 3 / 4; 246 devq = cam_simq_alloc(max_trans); 247 if (devq == NULL) { 248 device_printf(sc->dev, "Failed to allocate CAM simq\n"); 249 return (ENOMEM); 250 } 251 252 mtx_init(&sc->sim_mtx, "nvmf sim", NULL, MTX_DEF); 253 sc->sim = cam_sim_alloc(nvmf_sim_action, NULL, "nvme", sc, 254 device_get_unit(sc->dev), NULL, max_trans, max_trans, devq); 255 if (sc->sim == NULL) { 256 device_printf(sc->dev, "Failed to allocate CAM sim\n"); 257 cam_simq_free(devq); 258 mtx_destroy(&sc->sim_mtx); 259 return (ENXIO); 260 } 261 if (xpt_bus_register(sc->sim, sc->dev, 0) != CAM_SUCCESS) { 262 device_printf(sc->dev, "Failed to create CAM bus\n"); 263 cam_sim_free(sc->sim, TRUE); 264 mtx_destroy(&sc->sim_mtx); 265 return (ENXIO); 266 } 267 if (xpt_create_path(&sc->path, NULL, cam_sim_path(sc->sim), 268 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 269 device_printf(sc->dev, "Failed to create CAM path\n"); 270 xpt_bus_deregister(cam_sim_path(sc->sim)); 271 cam_sim_free(sc->sim, TRUE); 272 mtx_destroy(&sc->sim_mtx); 273 return (ENXIO); 274 } 275 return (0); 276 } 277 278 void 279 nvmf_sim_rescan_ns(struct nvmf_softc *sc, uint32_t id) 280 { 281 union ccb *ccb; 282 283 ccb = xpt_alloc_ccb_nowait(); 284 if (ccb == NULL) { 285 device_printf(sc->dev, 286 "unable to alloc CCB for rescan of namespace %u\n", id); 287 return; 288 } 289 290 /* 291 * As with nvme_sim, map NVMe namespace IDs onto CAM unit 292 * LUNs. 293 */ 294 if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->sim), 0, 295 id) != CAM_REQ_CMP) { 296 device_printf(sc->dev, 297 "Unable to create path for rescan of namespace %u\n", id); 298 xpt_free_ccb(ccb); 299 return; 300 } 301 xpt_rescan(ccb); 302 } 303 304 void 305 nvmf_disconnect_sim(struct nvmf_softc *sc) 306 { 307 mtx_lock(&sc->sim_mtx); 308 sc->sim_disconnected = true; 309 xpt_freeze_simq(sc->sim, 1); 310 mtx_unlock(&sc->sim_mtx); 311 } 312 313 void 314 nvmf_reconnect_sim(struct nvmf_softc *sc) 315 { 316 mtx_lock(&sc->sim_mtx); 317 sc->sim_disconnected = false; 318 mtx_unlock(&sc->sim_mtx); 319 xpt_release_simq(sc->sim, 1); 320 } 321 322 void 323 nvmf_destroy_sim(struct nvmf_softc *sc) 324 { 325 xpt_async(AC_LOST_DEVICE, sc->path, NULL); 326 if (sc->sim_disconnected) 327 xpt_release_simq(sc->sim, 1); 328 xpt_free_path(sc->path); 329 xpt_bus_deregister(cam_sim_path(sc->sim)); 330 cam_sim_free(sc->sim, TRUE); 331 mtx_destroy(&sc->sim_mtx); 332 } 333