1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2023-2024 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 */
7
8 #include <sys/types.h>
9 #include <sys/malloc.h>
10 #include <sys/memdesc.h>
11 #include <sys/refcount.h>
12
13 #include <cam/cam.h>
14 #include <cam/cam_ccb.h>
15 #include <cam/cam_sim.h>
16 #include <cam/cam_xpt_sim.h>
17 #include <cam/cam_debug.h>
18
19 #include <dev/nvmf/host/nvmf_var.h>
20
21 /*
22 * The I/O completion may trigger after the received CQE if the I/O
23 * used a zero-copy mbuf that isn't harvested until after the NIC
24 * driver processes TX completions. Use spriv_field0 to as a refcount.
25 *
26 * Store any I/O error returned in spriv_field1.
27 */
28 static __inline u_int *
ccb_refs(union ccb * ccb)29 ccb_refs(union ccb *ccb)
30 {
31 return ((u_int *)&ccb->ccb_h.spriv_field0);
32 }
33
34 #define spriv_ioerror spriv_field1
35
36 static void
nvmf_ccb_done(union ccb * ccb)37 nvmf_ccb_done(union ccb *ccb)
38 {
39 if (!refcount_release(ccb_refs(ccb)))
40 return;
41
42 if (nvmf_cqe_aborted(&ccb->nvmeio.cpl)) {
43 struct cam_sim *sim = xpt_path_sim(ccb->ccb_h.path);
44 struct nvmf_softc *sc = cam_sim_softc(sim);
45
46 if (nvmf_fail_disconnect || sc->sim_shutdown)
47 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
48 else
49 ccb->ccb_h.status = CAM_REQUEUE_REQ;
50 xpt_done(ccb);
51 } else if (ccb->nvmeio.cpl.status != 0) {
52 ccb->ccb_h.status = CAM_NVME_STATUS_ERROR;
53 xpt_done(ccb);
54 } else if (ccb->ccb_h.spriv_ioerror != 0) {
55 KASSERT(ccb->ccb_h.spriv_ioerror != EJUSTRETURN,
56 ("%s: zero sized transfer without CQE error", __func__));
57 ccb->ccb_h.status = CAM_REQ_CMP_ERR;
58 xpt_done(ccb);
59 } else {
60 ccb->ccb_h.status = CAM_REQ_CMP;
61 xpt_done(ccb);
62 }
63 }
64
65 static void
nvmf_ccb_io_complete(void * arg,size_t xfered,int error)66 nvmf_ccb_io_complete(void *arg, size_t xfered, int error)
67 {
68 union ccb *ccb = arg;
69
70 /*
71 * TODO: Reporting partial completions requires extending
72 * nvmeio to support resid and updating nda to handle partial
73 * reads, either by returning partial success (or an error) to
74 * the caller, or retrying all or part of the request.
75 */
76 ccb->ccb_h.spriv_ioerror = error;
77 if (error == 0) {
78 if (xfered == 0) {
79 #ifdef INVARIANTS
80 /*
81 * If the request fails with an error in the CQE
82 * there will be no data transferred but also no
83 * I/O error.
84 */
85 ccb->ccb_h.spriv_ioerror = EJUSTRETURN;
86 #endif
87 } else
88 KASSERT(xfered == ccb->nvmeio.dxfer_len,
89 ("%s: partial CCB completion", __func__));
90 }
91
92 nvmf_ccb_done(ccb);
93 }
94
95 static void
nvmf_ccb_complete(void * arg,const struct nvme_completion * cqe)96 nvmf_ccb_complete(void *arg, const struct nvme_completion *cqe)
97 {
98 union ccb *ccb = arg;
99
100 ccb->nvmeio.cpl = *cqe;
101 nvmf_ccb_done(ccb);
102 }
103
104 static void
nvmf_sim_io(struct nvmf_softc * sc,union ccb * ccb)105 nvmf_sim_io(struct nvmf_softc *sc, union ccb *ccb)
106 {
107 struct ccb_nvmeio *nvmeio = &ccb->nvmeio;
108 struct memdesc mem;
109 struct nvmf_request *req;
110 struct nvmf_host_qpair *qp;
111
112 mtx_lock(&sc->sim_mtx);
113 if (sc->sim_disconnected) {
114 mtx_unlock(&sc->sim_mtx);
115 if (nvmf_fail_disconnect || sc->sim_shutdown)
116 nvmeio->ccb_h.status = CAM_DEV_NOT_THERE;
117 else
118 nvmeio->ccb_h.status = CAM_REQUEUE_REQ;
119 xpt_done(ccb);
120 return;
121 }
122 if (nvmeio->ccb_h.func_code == XPT_NVME_IO)
123 qp = nvmf_select_io_queue(sc);
124 else
125 qp = sc->admin;
126 req = nvmf_allocate_request(qp, &nvmeio->cmd, nvmf_ccb_complete,
127 ccb, M_NOWAIT);
128 mtx_unlock(&sc->sim_mtx);
129 if (req == NULL) {
130 nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL;
131 xpt_done(ccb);
132 return;
133 }
134
135 if (nvmeio->dxfer_len != 0) {
136 refcount_init(ccb_refs(ccb), 2);
137 mem = memdesc_ccb(ccb);
138 nvmf_capsule_append_data(req->nc, &mem, nvmeio->dxfer_len,
139 (ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT,
140 nvmf_ccb_io_complete, ccb);
141 } else
142 refcount_init(ccb_refs(ccb), 1);
143
144 /*
145 * Clear spriv_ioerror as it can hold an earlier error if this
146 * CCB was aborted and has been retried.
147 */
148 ccb->ccb_h.spriv_ioerror = 0;
149 KASSERT(ccb->ccb_h.status == CAM_REQ_INPROG,
150 ("%s: incoming CCB is not in-progress", __func__));
151 ccb->ccb_h.status |= CAM_SIM_QUEUED;
152 nvmf_submit_request(req);
153 }
154
155 static void
nvmf_sim_action(struct cam_sim * sim,union ccb * ccb)156 nvmf_sim_action(struct cam_sim *sim, union ccb *ccb)
157 {
158 struct nvmf_softc *sc = cam_sim_softc(sim);
159
160 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
161 ("nvmf_sim_action: func= %#x\n",
162 ccb->ccb_h.func_code));
163
164 switch (ccb->ccb_h.func_code) {
165 case XPT_PATH_INQ: /* Path routing inquiry */
166 {
167 struct ccb_pathinq *cpi = &ccb->cpi;
168
169 cpi->version_num = 1;
170 cpi->hba_inquiry = 0;
171 cpi->target_sprt = 0;
172 cpi->hba_misc = PIM_UNMAPPED | PIM_NOSCAN;
173 cpi->hba_eng_cnt = 0;
174 cpi->max_target = 0;
175 cpi->max_lun = sc->cdata->nn;
176 cpi->async_flags = 0;
177 cpi->hpath_id = 0;
178 cpi->initiator_id = 0;
179 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
180 strlcpy(cpi->hba_vid, "NVMeoF", HBA_IDLEN);
181 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
182 cpi->unit_number = cam_sim_unit(sim);
183 cpi->bus_id = 0;
184
185 /* XXX: Same as iSCSI. */
186 cpi->base_transfer_speed = 150000;
187 cpi->protocol = PROTO_NVME;
188 cpi->protocol_version = sc->vs;
189 cpi->transport = XPORT_NVMF;
190 cpi->transport_version = sc->vs;
191 cpi->xport_specific.nvmf.nsid =
192 xpt_path_lun_id(ccb->ccb_h.path);
193 cpi->xport_specific.nvmf.trtype = sc->trtype;
194 strlcpy(cpi->xport_specific.nvmf.dev_name,
195 device_get_nameunit(sc->dev),
196 sizeof(cpi->xport_specific.nvmf.dev_name));
197 cpi->maxio = sc->max_xfer_size;
198 cpi->hba_vendor = 0;
199 cpi->hba_device = 0;
200 cpi->hba_subvendor = 0;
201 cpi->hba_subdevice = 0;
202 cpi->ccb_h.status = CAM_REQ_CMP;
203 break;
204 }
205 case XPT_GET_TRAN_SETTINGS: /* Get transport settings */
206 {
207 struct ccb_trans_settings *cts = &ccb->cts;
208 struct ccb_trans_settings_nvme *nvme;
209 struct ccb_trans_settings_nvmf *nvmf;
210
211 cts->protocol = PROTO_NVME;
212 cts->protocol_version = sc->vs;
213 cts->transport = XPORT_NVMF;
214 cts->transport_version = sc->vs;
215
216 nvme = &cts->proto_specific.nvme;
217 nvme->valid = CTS_NVME_VALID_SPEC;
218 nvme->spec = sc->vs;
219
220 nvmf = &cts->xport_specific.nvmf;
221 nvmf->valid = CTS_NVMF_VALID_TRTYPE;
222 nvmf->trtype = sc->trtype;
223 cts->ccb_h.status = CAM_REQ_CMP;
224 break;
225 }
226 case XPT_SET_TRAN_SETTINGS: /* Set transport settings */
227 /*
228 * No transfer settings can be set, but nvme_xpt sends
229 * this anyway.
230 */
231 ccb->ccb_h.status = CAM_REQ_CMP;
232 break;
233 case XPT_NVME_IO: /* Execute the requested I/O */
234 case XPT_NVME_ADMIN: /* or Admin operation */
235 nvmf_sim_io(sc, ccb);
236 return;
237 default:
238 /* XXX */
239 device_printf(sc->dev, "unhandled sim function %#x\n",
240 ccb->ccb_h.func_code);
241 ccb->ccb_h.status = CAM_REQ_INVALID;
242 break;
243 }
244 xpt_done(ccb);
245 }
246
247 int
nvmf_init_sim(struct nvmf_softc * sc)248 nvmf_init_sim(struct nvmf_softc *sc)
249 {
250 struct cam_devq *devq;
251 int max_trans;
252
253 max_trans = sc->max_pending_io * 3 / 4;
254 devq = cam_simq_alloc(max_trans);
255 if (devq == NULL) {
256 device_printf(sc->dev, "Failed to allocate CAM simq\n");
257 return (ENOMEM);
258 }
259
260 mtx_init(&sc->sim_mtx, "nvmf sim", NULL, MTX_DEF);
261 sc->sim = cam_sim_alloc(nvmf_sim_action, NULL, "nvme", sc,
262 device_get_unit(sc->dev), NULL, max_trans, max_trans, devq);
263 if (sc->sim == NULL) {
264 device_printf(sc->dev, "Failed to allocate CAM sim\n");
265 cam_simq_free(devq);
266 mtx_destroy(&sc->sim_mtx);
267 return (ENXIO);
268 }
269 if (xpt_bus_register(sc->sim, sc->dev, 0) != CAM_SUCCESS) {
270 device_printf(sc->dev, "Failed to create CAM bus\n");
271 cam_sim_free(sc->sim, TRUE);
272 mtx_destroy(&sc->sim_mtx);
273 return (ENXIO);
274 }
275 if (xpt_create_path(&sc->path, NULL, cam_sim_path(sc->sim),
276 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
277 device_printf(sc->dev, "Failed to create CAM path\n");
278 xpt_bus_deregister(cam_sim_path(sc->sim));
279 cam_sim_free(sc->sim, TRUE);
280 mtx_destroy(&sc->sim_mtx);
281 return (ENXIO);
282 }
283 return (0);
284 }
285
286 void
nvmf_sim_rescan_ns(struct nvmf_softc * sc,uint32_t id)287 nvmf_sim_rescan_ns(struct nvmf_softc *sc, uint32_t id)
288 {
289 union ccb *ccb;
290
291 ccb = xpt_alloc_ccb_nowait();
292 if (ccb == NULL) {
293 device_printf(sc->dev,
294 "unable to alloc CCB for rescan of namespace %u\n", id);
295 return;
296 }
297
298 /*
299 * As with nvme_sim, map NVMe namespace IDs onto CAM unit
300 * LUNs.
301 */
302 if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(sc->sim), 0,
303 id) != CAM_REQ_CMP) {
304 device_printf(sc->dev,
305 "Unable to create path for rescan of namespace %u\n", id);
306 xpt_free_ccb(ccb);
307 return;
308 }
309 xpt_rescan(ccb);
310 }
311
312 void
nvmf_disconnect_sim(struct nvmf_softc * sc)313 nvmf_disconnect_sim(struct nvmf_softc *sc)
314 {
315 mtx_lock(&sc->sim_mtx);
316 sc->sim_disconnected = true;
317 xpt_freeze_simq(sc->sim, 1);
318 mtx_unlock(&sc->sim_mtx);
319 }
320
321 void
nvmf_reconnect_sim(struct nvmf_softc * sc)322 nvmf_reconnect_sim(struct nvmf_softc *sc)
323 {
324 mtx_lock(&sc->sim_mtx);
325 sc->sim_disconnected = false;
326 mtx_unlock(&sc->sim_mtx);
327 xpt_release_simq(sc->sim, 1);
328 }
329
330 void
nvmf_shutdown_sim(struct nvmf_softc * sc)331 nvmf_shutdown_sim(struct nvmf_softc *sc)
332 {
333 mtx_lock(&sc->sim_mtx);
334 sc->sim_shutdown = true;
335 mtx_unlock(&sc->sim_mtx);
336 xpt_release_simq(sc->sim, 1);
337 }
338
339 void
nvmf_destroy_sim(struct nvmf_softc * sc)340 nvmf_destroy_sim(struct nvmf_softc *sc)
341 {
342 xpt_async(AC_LOST_DEVICE, sc->path, NULL);
343 if (sc->sim_disconnected)
344 xpt_release_simq(sc->sim, 1);
345 xpt_free_path(sc->path);
346 xpt_bus_deregister(cam_sim_path(sc->sim));
347 cam_sim_free(sc->sim, TRUE);
348 mtx_destroy(&sc->sim_mtx);
349 }
350