1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
4  * Copyright (c) 2023-2024 Chelsio Communications, Inc.
44 ns_printf(struct nvmf_namespace *ns, const char *fmt, ...)  in ns_printf()  argument
53 	sbuf_printf(&sb, "%sn%u: ", device_get_nameunit(ns->sc->dev),  in ns_printf()
54 	    ns->id);  in ns_printf()
66  * used a zero-copy mbuf that isn't harvested until after the NIC
73 	return ((u_int *)&bio->bio_driver1);  in bio_refs()
79 	struct nvmf_namespace *ns;  in nvmf_ns_biodone()  local
85 	ns = bio->bio_dev->si_drv1;  in nvmf_ns_biodone()
88 	if (bio->bio_error == ECONNABORTED && !nvmf_fail_disconnect) {  in nvmf_ns_biodone()
89 		bio->bio_error = 0;  in nvmf_ns_biodone()
90 		bio->bio_driver2 = 0;  in nvmf_ns_biodone()
91 		mtx_lock(&ns->lock);  in nvmf_ns_biodone()
92 		if (ns->disconnected) {  in nvmf_ns_biodone()
93 			if (nvmf_fail_disconnect || ns->shutdown) {  in nvmf_ns_biodone()
94 				mtx_unlock(&ns->lock);  in nvmf_ns_biodone()
95 				bio->bio_error = ECONNABORTED;  in nvmf_ns_biodone()
96 				bio->bio_flags |= BIO_ERROR;  in nvmf_ns_biodone()
97 				bio->bio_resid = bio->bio_bcount;  in nvmf_ns_biodone()
100 				TAILQ_INSERT_TAIL(&ns->pending_bios, bio,  in nvmf_ns_biodone()
102 				mtx_unlock(&ns->lock);  in nvmf_ns_biodone()
105 			mtx_unlock(&ns->lock);  in nvmf_ns_biodone()
113 		error = (intptr_t)bio->bio_driver2;  in nvmf_ns_biodone()
115 			bio->bio_error = error;  in nvmf_ns_biodone()
116 		if (bio->bio_error != 0)  in nvmf_ns_biodone()
117 			bio->bio_flags |= BIO_ERROR;  in nvmf_ns_biodone()
121 	if (refcount_release(&ns->active_bios))  in nvmf_ns_biodone()
122 		wakeup(ns);  in nvmf_ns_biodone()
130 	KASSERT(xfered <= bio->bio_bcount,  in nvmf_ns_io_complete()
133 	bio->bio_driver2 = (void *)(intptr_t)error;  in nvmf_ns_io_complete()
134 	bio->bio_resid = bio->bio_bcount - xfered;  in nvmf_ns_io_complete()
145 		bio->bio_resid = bio->bio_bcount;  in nvmf_ns_delete_complete()
147 		bio->bio_resid = 0;  in nvmf_ns_delete_complete()
149 	free(bio->bio_driver2, M_NVMF);  in nvmf_ns_delete_complete()
150 	bio->bio_driver2 = (void *)(intptr_t)error;  in nvmf_ns_delete_complete()
161 		bio->bio_error = ECONNABORTED;  in nvmf_ns_bio_complete()
162 	else if (cqe->status != 0)  in nvmf_ns_bio_complete()
163 		bio->bio_error = EIO;  in nvmf_ns_bio_complete()
169 nvmf_ns_submit_bio(struct nvmf_namespace *ns, struct bio *bio)  in nvmf_ns_submit_bio()  argument
180 	switch (bio->bio_cmd) {  in nvmf_ns_submit_bio()
182 		lba = bio->bio_offset / ns->lba_size;  in nvmf_ns_submit_bio()
183 		lba_count = bio->bio_bcount / ns->lba_size;  in nvmf_ns_submit_bio()
184 		nvme_ns_read_cmd(&cmd, ns->id, lba, lba_count);  in nvmf_ns_submit_bio()
187 		lba = bio->bio_offset / ns->lba_size;  in nvmf_ns_submit_bio()
188 		lba_count = bio->bio_bcount / ns->lba_size;  in nvmf_ns_submit_bio()
189 		nvme_ns_write_cmd(&cmd, ns->id, lba, lba_count);  in nvmf_ns_submit_bio()
192 		nvme_ns_flush_cmd(&cmd, ns->id);  in nvmf_ns_submit_bio()
199 		lba = bio->bio_offset / ns->lba_size;  in nvmf_ns_submit_bio()
200 		lba_count = bio->bio_bcount / ns->lba_size;  in nvmf_ns_submit_bio()
201 		dsm_range->starting_lba = htole64(lba);  in nvmf_ns_submit_bio()
202 		dsm_range->length = htole32(lba_count);  in nvmf_ns_submit_bio()
205 		cmd.nsid = htole32(ns->id);  in nvmf_ns_submit_bio()
213 	mtx_lock(&ns->lock);  in nvmf_ns_submit_bio()
214 	if (ns->disconnected) {  in nvmf_ns_submit_bio()
215 		if (nvmf_fail_disconnect || ns->shutdown) {  in nvmf_ns_submit_bio()
218 			TAILQ_INSERT_TAIL(&ns->pending_bios, bio, bio_queue);  in nvmf_ns_submit_bio()
221 		mtx_unlock(&ns->lock);  in nvmf_ns_submit_bio()
226 	req = nvmf_allocate_request(nvmf_select_io_queue(ns->sc), &cmd,  in nvmf_ns_submit_bio()
229 		mtx_unlock(&ns->lock);  in nvmf_ns_submit_bio()
234 	switch (bio->bio_cmd) {  in nvmf_ns_submit_bio()
239 		nvmf_capsule_append_data(req->nc, &mem, bio->bio_bcount,  in nvmf_ns_submit_bio()
240 		    bio->bio_cmd == BIO_WRITE, nvmf_ns_io_complete, bio);  in nvmf_ns_submit_bio()
245 		nvmf_capsule_append_data(req->nc, &mem, sizeof(*dsm_range),  in nvmf_ns_submit_bio()
247 		bio->bio_driver2 = dsm_range;  in nvmf_ns_submit_bio()
251 		KASSERT(bio->bio_resid == 0,  in nvmf_ns_submit_bio()
256 	refcount_acquire(&ns->active_bios);  in nvmf_ns_submit_bio()
258 	mtx_unlock(&ns->lock);  in nvmf_ns_submit_bio()
266 	struct nvmf_namespace *ns = dev->si_drv1;  in nvmf_ns_ioctl()  local
273 		pt->cmd.nsid = htole32(ns->id);  in nvmf_ns_ioctl()
274 		return (nvmf_passthrough_cmd(ns->sc, pt, false));  in nvmf_ns_ioctl()
277 		strlcpy(gnsid->cdev, device_get_nameunit(ns->sc->dev),  in nvmf_ns_ioctl()
278 		    sizeof(gnsid->cdev));  in nvmf_ns_ioctl()
279 		gnsid->nsid = ns->id;  in nvmf_ns_ioctl()
282 		nvme_cdata_get_disk_ident(ns->sc->cdata, (uint8_t *)arg);  in nvmf_ns_ioctl()
285 		*(off_t *)arg = ns->size;  in nvmf_ns_ioctl()
288 		*(u_int *)arg = ns->lba_size;  in nvmf_ns_ioctl()
302 		error = securelevel_gt(td->td_ucred, 0);  in nvmf_ns_open()
309 	struct nvmf_namespace *ns;  in nvmf_ns_strategy()  local
312 	ns = bio->bio_dev->si_drv1;  in nvmf_ns_strategy()
314 	error = nvmf_ns_submit_bio(ns, bio);  in nvmf_ns_strategy()
316 		bio->bio_error = error;  in nvmf_ns_strategy()
317 		bio->bio_flags |= BIO_ERROR;  in nvmf_ns_strategy()
318 		bio->bio_resid = bio->bio_bcount;  in nvmf_ns_strategy()
338 	struct nvmf_namespace *ns;  in nvmf_init_ns()  local
342 	ns = malloc(sizeof(*ns), M_NVMF, M_WAITOK | M_ZERO);  in nvmf_init_ns()
343 	ns->sc = sc;  in nvmf_init_ns()
344 	ns->id = id;  in nvmf_init_ns()
345 	TAILQ_INIT(&ns->pending_bios);  in nvmf_init_ns()
346 	mtx_init(&ns->lock, "nvmf ns", NULL, MTX_DEF);  in nvmf_init_ns()
349 	refcount_init(&ns->active_bios, 1);  in nvmf_init_ns()
351 	if (NVMEV(NVME_NS_DATA_DPS_PIT, data->dps) != 0) {  in nvmf_init_ns()
352 		ns_printf(ns, "End-to-end data protection not supported\n");  in nvmf_init_ns()
356 	lbaf = NVMEV(NVME_NS_DATA_FLBAS_FORMAT, data->flbas);  in nvmf_init_ns()
357 	if (lbaf > data->nlbaf) {  in nvmf_init_ns()
358 		ns_printf(ns, "Invalid LBA format index\n");  in nvmf_init_ns()
362 	if (NVMEV(NVME_NS_DATA_LBAF_MS, data->lbaf[lbaf]) != 0) {  in nvmf_init_ns()
363 		ns_printf(ns, "Namespaces with metadata are not supported\n");  in nvmf_init_ns()
367 	lbads = NVMEV(NVME_NS_DATA_LBAF_LBADS, data->lbaf[lbaf]);  in nvmf_init_ns()
369 		ns_printf(ns, "Invalid LBA format index\n");  in nvmf_init_ns()
373 	ns->lba_size = 1 << lbads;  in nvmf_init_ns()
374 	ns->size = data->nsze * ns->lba_size;  in nvmf_init_ns()
376 	if (nvme_ctrlr_has_dataset_mgmt(sc->cdata))  in nvmf_init_ns()
377 		ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED;  in nvmf_init_ns()
379 	if (NVMEV(NVME_CTRLR_DATA_VWC_PRESENT, sc->cdata->vwc) != 0)  in nvmf_init_ns()
380 		ns->flags |= NVME_NS_FLUSH_SUPPORTED;  in nvmf_init_ns()
392 	mda.mda_si_drv1 = ns;  in nvmf_init_ns()
393 	error = make_dev_s(&mda, &ns->cdev, "%sn%u",  in nvmf_init_ns()
394 	    device_get_nameunit(sc->dev), id);  in nvmf_init_ns()
397 	ns->cdev->si_drv2 = make_dev_alias(ns->cdev, "%sns%u",  in nvmf_init_ns()
398 	    device_get_nameunit(sc->dev), id);  in nvmf_init_ns()
400 	ns->cdev->si_flags |= SI_UNMAPPED;  in nvmf_init_ns()
402 	return (ns);  in nvmf_init_ns()
404 	mtx_destroy(&ns->lock);  in nvmf_init_ns()
405 	free(ns, M_NVMF);  in nvmf_init_ns()
410 nvmf_disconnect_ns(struct nvmf_namespace *ns)  in nvmf_disconnect_ns()  argument
412 	mtx_lock(&ns->lock);  in nvmf_disconnect_ns()
413 	ns->disconnected = true;  in nvmf_disconnect_ns()
414 	mtx_unlock(&ns->lock);  in nvmf_disconnect_ns()
418 nvmf_reconnect_ns(struct nvmf_namespace *ns)  in nvmf_reconnect_ns()  argument
423 	mtx_lock(&ns->lock);  in nvmf_reconnect_ns()
424 	ns->disconnected = false;  in nvmf_reconnect_ns()
426 	TAILQ_CONCAT(&bios, &ns->pending_bios, bio_queue);  in nvmf_reconnect_ns()
427 	mtx_unlock(&ns->lock);  in nvmf_reconnect_ns()
437 nvmf_shutdown_ns(struct nvmf_namespace *ns)  in nvmf_shutdown_ns()  argument
442 	mtx_lock(&ns->lock);  in nvmf_shutdown_ns()
443 	ns->shutdown = true;  in nvmf_shutdown_ns()
445 	TAILQ_CONCAT(&bios, &ns->pending_bios, bio_queue);  in nvmf_shutdown_ns()
446 	mtx_unlock(&ns->lock);  in nvmf_shutdown_ns()
451 		bio->bio_error = ECONNABORTED;  in nvmf_shutdown_ns()
452 		bio->bio_flags |= BIO_ERROR;  in nvmf_shutdown_ns()
453 		bio->bio_resid = bio->bio_bcount;  in nvmf_shutdown_ns()
459 nvmf_destroy_ns(struct nvmf_namespace *ns)  in nvmf_destroy_ns()  argument
464 	if (ns->cdev->si_drv2 != NULL)  in nvmf_destroy_ns()
465 		destroy_dev(ns->cdev->si_drv2);  in nvmf_destroy_ns()
466 	destroy_dev(ns->cdev);  in nvmf_destroy_ns()
473 	mtx_lock(&ns->lock);  in nvmf_destroy_ns()
474 	if (!refcount_release(&ns->active_bios)) {  in nvmf_destroy_ns()
475 		while (ns->active_bios != 0)  in nvmf_destroy_ns()
476 			mtx_sleep(ns, &ns->lock, 0, "nvmfrmns", 0);  in nvmf_destroy_ns()
481 	TAILQ_CONCAT(&bios, &ns->pending_bios, bio_queue);  in nvmf_destroy_ns()
482 	mtx_unlock(&ns->lock);  in nvmf_destroy_ns()
487 		bio->bio_error = ECONNABORTED;  in nvmf_destroy_ns()
488 		bio->bio_flags |= BIO_ERROR;  in nvmf_destroy_ns()
489 		bio->bio_resid = bio->bio_bcount;  in nvmf_destroy_ns()
493 	mtx_destroy(&ns->lock);  in nvmf_destroy_ns()
494 	free(ns, M_NVMF);  in nvmf_destroy_ns()
498 nvmf_update_ns(struct nvmf_namespace *ns,  in nvmf_update_ns()  argument
503 	if (NVMEV(NVME_NS_DATA_DPS_PIT, data->dps) != 0) {  in nvmf_update_ns()
504 		ns_printf(ns, "End-to-end data protection not supported\n");  in nvmf_update_ns()
508 	lbaf = NVMEV(NVME_NS_DATA_FLBAS_FORMAT, data->flbas);  in nvmf_update_ns()
509 	if (lbaf > data->nlbaf) {  in nvmf_update_ns()
510 		ns_printf(ns, "Invalid LBA format index\n");  in nvmf_update_ns()
514 	if (NVMEV(NVME_NS_DATA_LBAF_MS, data->lbaf[lbaf]) != 0) {  in nvmf_update_ns()
515 		ns_printf(ns, "Namespaces with metadata are not supported\n");  in nvmf_update_ns()
519 	lbads = NVMEV(NVME_NS_DATA_LBAF_LBADS, data->lbaf[lbaf]);  in nvmf_update_ns()
521 		ns_printf(ns, "Invalid LBA format index\n");  in nvmf_update_ns()
525 	ns->lba_size = 1 << lbads;  in nvmf_update_ns()
526 	ns->size = data->nsze * ns->lba_size;  in nvmf_update_ns()