1 /*-
2 * Copyright (c) 2016 Netflix, Inc.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer,
9 * without modification, immediately at the beginning of the file.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26 #include <sys/param.h>
27 #include <sys/systm.h>
28 #include <sys/buf.h>
29 #include <sys/bus.h>
30 #include <sys/conf.h>
31 #include <sys/ioccom.h>
32 #include <sys/malloc.h>
33 #include <sys/proc.h>
34 #include <sys/smp.h>
35
36 #include <cam/cam.h>
37 #include <cam/cam_ccb.h>
38 #include <cam/cam_sim.h>
39 #include <cam/cam_xpt_sim.h>
40 #include <cam/cam_debug.h>
41
42 #include <dev/pci/pcivar.h>
43 #include <dev/pci/pcireg.h>
44
45 #include "nvme_private.h"
46
47 #include "nvme_if.h"
48
49 #define ccb_accb_ptr spriv_ptr0
50 #define ccb_ctrlr_ptr spriv_ptr1
51 static void nvme_sim_action(struct cam_sim *sim, union ccb *ccb);
52 static void nvme_sim_poll(struct cam_sim *sim);
53 static int nvme_sim_ns_added(device_t dev, struct nvme_namespace *ns);
54 static int nvme_sim_ns_changed(device_t dev, uint32_t nsid);
55 static int nvme_sim_ns_removed(device_t dev, struct nvme_namespace *ns);
56
57 #define sim2softc(sim) ((struct nvme_sim_softc *)cam_sim_softc(sim))
58 #define sim2ctrlr(sim) (sim2softc(sim)->s_ctrlr)
59
60 struct nvme_sim_softc
61 {
62 struct nvme_controller *s_ctrlr;
63 struct cam_sim *s_sim;
64 struct cam_path *s_path;
65 };
66
67 static void
nvme_sim_nvmeio_done(void * ccb_arg,const struct nvme_completion * cpl)68 nvme_sim_nvmeio_done(void *ccb_arg, const struct nvme_completion *cpl)
69 {
70 union ccb *ccb = (union ccb *)ccb_arg;
71
72 /*
73 * Let the periph know the completion, and let it sort out what
74 * it means. Report an error or success based on SC and SCT.
75 * We do not try to fetch additional data from the error log,
76 * though maybe we should in the future.
77 */
78 memcpy(&ccb->nvmeio.cpl, cpl, sizeof(*cpl));
79 ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
80 if (nvme_completion_is_error(cpl)) {
81 ccb->ccb_h.status = CAM_NVME_STATUS_ERROR;
82 xpt_done(ccb);
83 } else {
84 ccb->ccb_h.status = CAM_REQ_CMP;
85 xpt_done_direct(ccb);
86 }
87 }
88
89 static void
nvme_sim_nvmeio(struct cam_sim * sim,union ccb * ccb)90 nvme_sim_nvmeio(struct cam_sim *sim, union ccb *ccb)
91 {
92 struct ccb_nvmeio *nvmeio = &ccb->nvmeio;
93 struct nvme_request *req;
94 void *payload;
95 uint32_t size;
96 struct nvme_controller *ctrlr;
97
98 ctrlr = sim2ctrlr(sim);
99
100 if (ctrlr->max_identify_cns != 0 &&
101 nvmeio->cmd.opc == NVME_OPC_IDENTIFY &&
102 (le32toh(nvmeio->cmd.cdw10) & 0xff) > ctrlr->max_identify_cns) {
103 nvmeio->ccb_h.status = CAM_REQ_INVALID;
104 xpt_done(ccb);
105 return;
106 }
107
108 payload = nvmeio->data_ptr;
109 size = nvmeio->dxfer_len;
110 /* SG LIST ??? */
111 if ((nvmeio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO)
112 req = nvme_allocate_request_bio((struct bio *)payload,
113 M_NOWAIT, nvme_sim_nvmeio_done, ccb);
114 else if ((nvmeio->ccb_h.flags & CAM_DATA_SG) == CAM_DATA_SG)
115 req = nvme_allocate_request_ccb(ccb, M_NOWAIT,
116 nvme_sim_nvmeio_done, ccb);
117 else if (payload == NULL)
118 req = nvme_allocate_request_null(M_NOWAIT, nvme_sim_nvmeio_done,
119 ccb);
120 else
121 req = nvme_allocate_request_vaddr(payload, size, M_NOWAIT,
122 nvme_sim_nvmeio_done, ccb);
123 if (req == NULL) {
124 nvmeio->ccb_h.status = CAM_RESRC_UNAVAIL;
125 xpt_done(ccb);
126 return;
127 }
128 ccb->ccb_h.status |= CAM_SIM_QUEUED;
129
130 memcpy(&req->cmd, &ccb->nvmeio.cmd, sizeof(ccb->nvmeio.cmd));
131
132 if (ccb->ccb_h.func_code == XPT_NVME_IO)
133 nvme_ctrlr_submit_io_request(ctrlr, req);
134 else
135 nvme_ctrlr_submit_admin_request(ctrlr, req);
136 }
137
138 static uint32_t
nvme_link_kBps(struct nvme_controller * ctrlr)139 nvme_link_kBps(struct nvme_controller *ctrlr)
140 {
141 uint32_t speed, lanes, link[] = { 1, 250000, 500000, 985000, 1970000 };
142 uint32_t status;
143
144 status = pcie_read_config(ctrlr->dev, PCIER_LINK_STA, 2);
145 speed = status & PCIEM_LINK_STA_SPEED;
146 lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4;
147 /*
148 * Failsafe on link speed indicator. If it is insane report the number of
149 * lanes as the speed. Not 100% accurate, but may be diagnostic.
150 */
151 if (speed >= nitems(link))
152 speed = 0;
153 return link[speed] * lanes;
154 }
155
156 static void
nvme_sim_action(struct cam_sim * sim,union ccb * ccb)157 nvme_sim_action(struct cam_sim *sim, union ccb *ccb)
158 {
159 struct nvme_controller *ctrlr;
160
161 CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE,
162 ("nvme_sim_action: func= %#x\n",
163 ccb->ccb_h.func_code));
164
165 ctrlr = sim2ctrlr(sim);
166
167 switch (ccb->ccb_h.func_code) {
168 case XPT_CALC_GEOMETRY: /* Calculate Geometry Totally nuts ? XXX */
169 /*
170 * Only meaningful for old-school SCSI disks since only the SCSI
171 * da driver generates them. Reject all these that slip through.
172 */
173 /*FALLTHROUGH*/
174 case XPT_ABORT: /* Abort the specified CCB */
175 ccb->ccb_h.status = CAM_REQ_INVALID;
176 break;
177 case XPT_SET_TRAN_SETTINGS:
178 /*
179 * NVMe doesn't really have different transfer settings, but
180 * other parts of CAM think failure here is a big deal.
181 */
182 ccb->ccb_h.status = CAM_REQ_CMP;
183 break;
184 case XPT_PATH_INQ: /* Path routing inquiry */
185 {
186 struct ccb_pathinq *cpi = &ccb->cpi;
187 device_t dev = ctrlr->dev;
188
189 /*
190 * For devices that are reported as children of the AHCI
191 * controller, which has no access to the config space for this
192 * controller, report the AHCI controller's data.
193 */
194 if (ctrlr->quirks & QUIRK_AHCI)
195 dev = device_get_parent(dev);
196 cpi->version_num = 1;
197 cpi->hba_inquiry = 0;
198 cpi->target_sprt = 0;
199 cpi->hba_misc = PIM_UNMAPPED | PIM_NOSCAN;
200 cpi->hba_eng_cnt = 0;
201 cpi->max_target = 0;
202 cpi->max_lun = nvme_ctrlr_num_namespaces(ctrlr);
203 cpi->maxio = ctrlr->max_xfer_size;
204 cpi->initiator_id = 0;
205 cpi->bus_id = cam_sim_bus(sim);
206 cpi->base_transfer_speed = nvme_link_kBps(ctrlr);
207 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
208 strlcpy(cpi->hba_vid, "NVMe", HBA_IDLEN);
209 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
210 cpi->unit_number = cam_sim_unit(sim);
211 cpi->transport = XPORT_NVME; /* XXX XPORT_PCIE ? */
212 cpi->transport_version = nvme_mmio_read_4(ctrlr, vs);
213 cpi->protocol = PROTO_NVME;
214 cpi->protocol_version = nvme_mmio_read_4(ctrlr, vs);
215 cpi->xport_specific.nvme.nsid = xpt_path_lun_id(ccb->ccb_h.path);
216 cpi->xport_specific.nvme.domain = pci_get_domain(dev);
217 cpi->xport_specific.nvme.bus = pci_get_bus(dev);
218 cpi->xport_specific.nvme.slot = pci_get_slot(dev);
219 cpi->xport_specific.nvme.function = pci_get_function(dev);
220 cpi->xport_specific.nvme.extra = 0;
221 strlcpy(cpi->xport_specific.nvme.dev_name, device_get_nameunit(dev),
222 sizeof(cpi->xport_specific.nvme.dev_name));
223 cpi->hba_vendor = pci_get_vendor(dev);
224 cpi->hba_device = pci_get_device(dev);
225 cpi->hba_subvendor = pci_get_subvendor(dev);
226 cpi->hba_subdevice = pci_get_subdevice(dev);
227 cpi->ccb_h.status = CAM_REQ_CMP;
228 break;
229 }
230 case XPT_GET_TRAN_SETTINGS: /* Get transport settings */
231 {
232 struct ccb_trans_settings *cts;
233 struct ccb_trans_settings_nvme *nvmep;
234 struct ccb_trans_settings_nvme *nvmex;
235 device_t dev;
236 uint32_t status, caps, flags;
237
238 dev = ctrlr->dev;
239 cts = &ccb->cts;
240 nvmex = &cts->xport_specific.nvme;
241 nvmep = &cts->proto_specific.nvme;
242
243 nvmex->spec = nvme_mmio_read_4(ctrlr, vs);
244 nvmex->valid = CTS_NVME_VALID_SPEC;
245 if ((ctrlr->quirks & QUIRK_AHCI) == 0) {
246 /* AHCI redirect makes it impossible to query */
247 status = pcie_read_config(dev, PCIER_LINK_STA, 2);
248 caps = pcie_read_config(dev, PCIER_LINK_CAP, 2);
249 flags = pcie_read_config(dev, PCIER_FLAGS, 2);
250 if ((flags & PCIEM_FLAGS_TYPE) == PCIEM_TYPE_ENDPOINT) {
251 nvmex->valid |= CTS_NVME_VALID_LINK;
252 nvmex->speed = status & PCIEM_LINK_STA_SPEED;
253 nvmex->lanes = (status & PCIEM_LINK_STA_WIDTH) >> 4;
254 nvmex->max_speed = caps & PCIEM_LINK_CAP_MAX_SPEED;
255 nvmex->max_lanes = (caps & PCIEM_LINK_CAP_MAX_WIDTH) >> 4;
256 }
257 }
258
259 /* XXX these should be something else maybe ? */
260 nvmep->valid = CTS_NVME_VALID_SPEC;
261 nvmep->spec = nvmex->spec;
262
263 cts->transport = XPORT_NVME;
264 cts->transport_version = nvmex->spec;
265 cts->protocol = PROTO_NVME;
266 cts->protocol_version = nvmex->spec;
267 cts->ccb_h.status = CAM_REQ_CMP;
268 break;
269 }
270 case XPT_TERM_IO: /* Terminate the I/O process */
271 /*
272 * every driver handles this, but nothing generates it. Assume
273 * it's OK to just say 'that worked'.
274 */
275 /*FALLTHROUGH*/
276 case XPT_RESET_DEV: /* Bus Device Reset the specified device */
277 case XPT_RESET_BUS: /* Reset the specified bus */
278 /*
279 * NVMe doesn't really support physically resetting the bus. It's part
280 * of the bus scanning dance, so return sucess to tell the process to
281 * proceed.
282 */
283 ccb->ccb_h.status = CAM_REQ_CMP;
284 break;
285 case XPT_NVME_IO: /* Execute the requested I/O operation */
286 if (ctrlr->is_failed) {
287 /*
288 * I/O came in while we were failing the drive, so drop
289 * it. Once falure is complete, we'll be destroyed.
290 */
291 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
292 break;
293 }
294 nvme_sim_nvmeio(sim, ccb);
295 return; /* no done */
296 case XPT_NVME_ADMIN: /* or Admin operation */
297 if (ctrlr->is_failed_admin) {
298 /*
299 * Admin request came in when we can't send admin
300 * commands, so drop it. Once falure is complete, we'll
301 * be destroyed.
302 */
303 ccb->ccb_h.status = CAM_DEV_NOT_THERE;
304 break;
305 }
306 nvme_sim_nvmeio(sim, ccb);
307 return; /* no done */
308 default:
309 ccb->ccb_h.status = CAM_REQ_INVALID;
310 break;
311 }
312 xpt_done(ccb);
313 }
314
315 static void
nvme_sim_poll(struct cam_sim * sim)316 nvme_sim_poll(struct cam_sim *sim)
317 {
318 nvme_ctrlr_poll(sim2ctrlr(sim));
319 }
320
321 static int
nvme_sim_probe(device_t dev)322 nvme_sim_probe(device_t dev)
323 {
324 if (nvme_use_nvd)
325 return (ENXIO);
326
327 if (!NVME_IS_STORAGE_DEVICE(device_get_parent(dev)))
328 return (ENXIO);
329
330 device_set_desc(dev, "nvme cam");
331 return (BUS_PROBE_DEFAULT);
332 }
333
334 static int
nvme_sim_attach(device_t dev)335 nvme_sim_attach(device_t dev)
336 {
337 struct cam_devq *devq;
338 int max_trans;
339 int err = ENXIO;
340 struct nvme_sim_softc *sc = device_get_softc(dev);
341 struct nvme_controller *ctrlr = device_get_ivars(dev);
342
343 max_trans = ctrlr->max_hw_pend_io;
344 devq = cam_simq_alloc(max_trans);
345 if (devq == NULL)
346 return (ENOMEM);
347
348 sc->s_ctrlr = ctrlr;
349
350 sc->s_sim = cam_sim_alloc(nvme_sim_action, nvme_sim_poll,
351 "nvme", sc, device_get_unit(ctrlr->dev),
352 NULL, max_trans, max_trans, devq);
353 if (sc->s_sim == NULL) {
354 device_printf(dev, "Failed to allocate a sim\n");
355 cam_simq_free(devq);
356 return (ENOMEM);
357 }
358 if (xpt_bus_register(sc->s_sim, dev, 0) != CAM_SUCCESS) {
359 device_printf(dev, "Failed to create a bus\n");
360 goto err2;
361 }
362 if (xpt_create_path(&sc->s_path, /*periph*/NULL, cam_sim_path(sc->s_sim),
363 CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
364 device_printf(dev, "Failed to create a path\n");
365 goto err3;
366 }
367
368 for (int i = 0; i < nvme_ctrlr_num_namespaces(ctrlr); i++) {
369 struct nvme_namespace *ns = &ctrlr->ns[i];
370
371 if (ns->data.nsze == 0)
372 continue;
373 nvme_sim_ns_added(dev, ns);
374 }
375
376 return (0);
377 err3:
378 xpt_bus_deregister(cam_sim_path(sc->s_sim));
379 err2:
380 cam_sim_free(sc->s_sim, /*free_devq*/TRUE);
381 return (err);
382 }
383
384
385 static int
nvme_sim_fail_all_ns(device_t dev)386 nvme_sim_fail_all_ns(device_t dev)
387 {
388 struct nvme_sim_softc *sc = device_get_softc(dev);
389 struct nvme_controller *ctrlr = sc->s_ctrlr;
390
391 for (int i = 0; i < nvme_ctrlr_num_namespaces(ctrlr); i++) {
392 struct nvme_namespace *ns = &ctrlr->ns[i];
393
394 if (ns->data.nsze == 0)
395 continue;
396 nvme_sim_ns_removed(dev, ns);
397 }
398 return (0);
399 }
400
401 static int
nvme_sim_detach(device_t dev)402 nvme_sim_detach(device_t dev)
403 {
404 return (nvme_sim_fail_all_ns(dev));
405 }
406
407 static int
nvme_sim_ns_added(device_t dev,struct nvme_namespace * ns)408 nvme_sim_ns_added(device_t dev, struct nvme_namespace *ns)
409 {
410 struct nvme_sim_softc *sc = device_get_softc(dev);
411 union ccb *ccb;
412
413 /*
414 * If we have no namespaces, then we both do not attach the nvme_sim_ns
415 * device. And then get a ns changed AER as well to tell us about it
416 * (which is how we get here). If there's no device attached, then
417 * there's nothing to do. sc->s_sim will be NULL as well (since it
418 * only gets set when we attach).
419 */
420 if (!device_is_attached(dev))
421 return (0);
422
423 /*
424 * We map the NVMe namespace idea onto the CAM unit LUN. For each new
425 * namespace, scan or rescan the path to enumerate it.
426 */
427 ccb = xpt_alloc_ccb();
428 if (xpt_create_path(&ccb->ccb_h.path, /*periph*/NULL,
429 cam_sim_path(sc->s_sim), 0, ns->id) != CAM_REQ_CMP) {
430 printf("unable to create path for rescan\n");
431 return (ENOMEM);
432 }
433 xpt_rescan(ccb);
434
435 return (0);
436 }
437
438 static int
nvme_sim_ns_removed(device_t dev,struct nvme_namespace * ns)439 nvme_sim_ns_removed(device_t dev, struct nvme_namespace *ns)
440 {
441 struct nvme_sim_softc *sc = device_get_softc(dev);
442 struct cam_path *tmppath;
443
444 if (xpt_create_path(&tmppath, /*periph*/NULL,
445 cam_sim_path(sc->s_sim), 0, ns->id) != CAM_REQ_CMP) {
446 printf("unable to create path for ns removal\n");
447 return (ENOMEM);
448 }
449 xpt_async(AC_LOST_DEVICE, tmppath, NULL);
450 xpt_free_path(tmppath);
451
452 return (0);
453 }
454
455 static int
nvme_sim_ns_changed(device_t dev,uint32_t nsid)456 nvme_sim_ns_changed(device_t dev, uint32_t nsid)
457 {
458 struct nvme_sim_softc *sc = device_get_softc(dev);
459 struct nvme_namespace *ns = &sc->s_ctrlr->ns[nsid - 1];
460
461 /*
462 * These wind up being the same. For a configured cam_ed, we generate
463 * AC_GETDEV_CHANGED, but for new one we do AC_FOUND_DEVICE, but the
464 * scan is the same.
465 */
466 return (nvme_sim_ns_added(dev, ns));
467 }
468
469 static int
nvme_sim_controller_failed(device_t dev)470 nvme_sim_controller_failed(device_t dev)
471 {
472 return (nvme_sim_fail_all_ns(dev));
473 }
474
475 static int
nvme_sim_handle_aen(device_t dev,const struct nvme_completion * cpl,uint32_t pg_nr,void * page,uint32_t page_len)476 nvme_sim_handle_aen(device_t dev, const struct nvme_completion *cpl,
477 uint32_t pg_nr, void *page, uint32_t page_len)
478 {
479 /* Do nothing */
480 return (0);
481 }
482
483 static device_method_t nvme_sim_methods[] = {
484 /* Device interface */
485 DEVMETHOD(device_probe, nvme_sim_probe),
486 DEVMETHOD(device_attach, nvme_sim_attach),
487 DEVMETHOD(device_detach, nvme_sim_detach),
488 /* Nvme controller messages */
489 DEVMETHOD(nvme_ns_added, nvme_sim_ns_added),
490 DEVMETHOD(nvme_ns_removed, nvme_sim_ns_removed),
491 DEVMETHOD(nvme_ns_changed, nvme_sim_ns_changed),
492 DEVMETHOD(nvme_controller_failed, nvme_sim_controller_failed),
493 DEVMETHOD(nvme_handle_aen, nvme_sim_handle_aen),
494 DEVMETHOD_END
495 };
496
497 static driver_t nvme_sim_driver = {
498 "nvme_sim",
499 nvme_sim_methods,
500 sizeof(struct nvme_sim_softc),
501 };
502
503 DRIVER_MODULE(nvme_sim, nvme, nvme_sim_driver, NULL, NULL);
504 MODULE_VERSION(nvme_sim, 1);
505