Lines Matching +full:write +full:- +full:1 +full:- +full:bps
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
9 * 1. Redistributions of source code must retain the above copyright
139 TAILQ_HEAD(, bio) bps;
166 #define NDA_DEFAULT_SEND_ORDERED 1
185 static int nda_enable_biospeedup = 1;
186 static int nda_nvd_compat = 1;
193 &nda_nvd_compat, 1, "Enable creation of nvd aliases.");
196 * All NVMe media is non-rotational, so all nvme device instances
228 nvme_ns_flush_cmd(&nvmeio->cmd, softc->nsid); in nda_nvme_flush()
242 nvme_ns_trim_cmd(&nvmeio->cmd, softc->nsid, num_ranges); in nda_nvme_trim()
256 nvme_ns_write_cmd(&nvmeio->cmd, softc->nsid, lba, count); in nda_nvme_write()
268 if (bp->bio_flags & BIO_UNMAPPED) { in nda_nvme_rw_bio()
272 payload = bp->bio_data; in nda_nvme_rw_bio()
275 lba = bp->bio_pblkno; in nda_nvme_rw_bio()
276 count = bp->bio_bcount / softc->disk->d_sectorsize; in nda_nvme_rw_bio()
283 bp->bio_bcount, /* dxfer_len */ in nda_nvme_rw_bio()
285 nvme_ns_rw_cmd(&nvmeio->cmd, rwcmd, softc->nsid, lba, count); in nda_nvme_rw_bio()
295 periph = (struct cam_periph *)dp->d_drv1; in ndaopen()
307 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, in ndaopen()
310 softc = (struct nda_softc *)periph->softc; in ndaopen()
311 softc->flags |= NDA_FLAG_OPEN; in ndaopen()
326 periph = (struct cam_periph *)dp->d_drv1; in ndaclose()
327 softc = (struct nda_softc *)periph->softc; in ndaclose()
330 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE | CAM_DEBUG_PERIPH, in ndaclose()
333 if ((softc->flags & NDA_FLAG_DIRTY) != 0 && in ndaclose()
334 (periph->flags & CAM_PERIPH_INVALID) == 0 && in ndaclose()
337 nda_nvme_flush(softc, &ccb->nvmeio); in ndaclose()
339 /*sense_flags*/0, softc->disk->d_devstat); in ndaclose()
342 xpt_print(periph->path, "Synchronize cache failed\n"); in ndaclose()
344 softc->flags &= ~NDA_FLAG_DIRTY; in ndaclose()
349 softc->flags &= ~NDA_FLAG_OPEN; in ndaclose()
351 while (softc->refcount != 0) in ndaclose()
352 cam_periph_sleep(periph, &softc->refcount, PRIBIO, "ndaclose", 1); in ndaclose()
353 KASSERT(softc->outstanding_cmds == 0, in ndaclose()
354 ("nda %d outstanding commands", softc->outstanding_cmds)); in ndaclose()
363 struct nda_softc *softc = (struct nda_softc *)periph->softc; in ndaschedule()
365 if (softc->state != NDA_STATE_NORMAL) in ndaschedule()
368 cam_iosched_schedule(softc->cam_iosched, periph); in ndaschedule()
377 periph = (struct cam_periph *)dp->d_drv1; in ndaioctl()
392 xpt_path_inq(&cpi, periph->path); in ndaioctl()
393 strncpy(gnsid->cdev, cpi.xport_specific.nvme.dev_name, in ndaioctl()
394 sizeof(gnsid->cdev)); in ndaioctl()
395 gnsid->nsid = cpi.xport_specific.nvme.nsid; in ndaioctl()
403 u_int maxmap = dp->d_maxsize; in ndaioctl()
411 xpt_setup_ccb(&ccb->ccb_h, periph->path, CAM_PRIORITY_NORMAL); in ndaioctl()
412 ccb->ccb_state = NDA_CCB_PASS; in ndaioctl()
413 cam_fill_nvmeio(&ccb->nvmeio, in ndaioctl()
416 (pt->is_read ? CAM_DIR_IN : CAM_DIR_OUT) | CAM_DATA_VADDR, in ndaioctl()
417 pt->buf, in ndaioctl()
418 pt->len, in ndaioctl()
420 memcpy(&ccb->nvmeio.cmd, &pt->cmd, sizeof(pt->cmd)); in ndaioctl()
467 periph = (struct cam_periph *)bp->bio_disk->d_drv1; in ndastrategy()
468 softc = (struct nda_softc *)periph->softc; in ndastrategy()
472 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastrategy(%p)\n", bp)); in ndastrategy()
477 if ((periph->flags & CAM_PERIPH_INVALID) != 0) { in ndastrategy()
483 if (bp->bio_cmd == BIO_DELETE) in ndastrategy()
484 softc->deletes++; in ndastrategy()
489 cam_iosched_queue_work(softc->cam_iosched, bp); in ndastrategy()
513 periph = dp->d_drv1; in ndadump()
514 softc = (struct nda_softc *)periph->softc; in ndadump()
515 secsize = softc->disk->d_sectorsize; in ndadump()
519 if ((periph->flags & CAM_PERIPH_INVALID) != 0) in ndadump()
525 xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); in ndadump()
537 xpt_setup_ccb(&nvmeio.ccb_h, periph->path, CAM_PRIORITY_NORMAL); in ndadump()
544 xpt_print(periph->path, "flush cmd failed\n"); in ndadump()
582 periph = (struct cam_periph *)dp->d_drv1; in ndadiskgonecb()
592 softc = (struct nda_softc *)periph->softc; in ndaoninvalidate()
595 * De-register any async callbacks. in ndaoninvalidate()
597 xpt_register_async(0, ndaasync, periph, periph->path); in ndaoninvalidate()
599 softc->invalidations++; in ndaoninvalidate()
608 cam_iosched_flush(softc->cam_iosched, NULL, ENXIO); in ndaoninvalidate()
614 disk_gone(softc->disk); in ndaoninvalidate()
622 softc = (struct nda_softc *)periph->softc; in ndacleanup()
626 cam_iosched_fini(softc->cam_iosched); in ndacleanup()
631 if ((softc->flags & NDA_FLAG_SCTX_INIT) != 0) { in ndacleanup()
633 if (sysctl_ctx_free(&softc->sysctl_stats_ctx) != 0) in ndacleanup()
634 xpt_print(periph->path, in ndacleanup()
637 if (sysctl_ctx_free(&softc->sysctl_ctx) != 0) in ndacleanup()
638 xpt_print(periph->path, in ndacleanup()
642 disk_destroy(softc->disk); in ndacleanup()
664 if (cgd->protocol != PROTO_NVME) in ndaasync()
692 softc = periph->softc; in ndaasync()
693 disk_attr_changed(softc->disk, "GEOM::physpath", in ndaasync()
715 if ((periph->flags & CAM_PERIPH_INVALID) != 0) { in ndasysctlinit()
720 softc = (struct nda_softc *)periph->softc; in ndasysctlinit()
721 snprintf(tmpstr, sizeof(tmpstr), "CAM NDA unit %d", periph->unit_number); in ndasysctlinit()
722 snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number); in ndasysctlinit()
724 sysctl_ctx_init(&softc->sysctl_ctx); in ndasysctlinit()
725 softc->flags |= NDA_FLAG_SCTX_INIT; in ndasysctlinit()
726 softc->sysctl_tree = SYSCTL_ADD_NODE_WITH_LABEL(&softc->sysctl_ctx, in ndasysctlinit()
729 if (softc->sysctl_tree == NULL) { in ndasysctlinit()
735 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), in ndasysctlinit()
737 &softc->unmappedio, 0, "Unmapped I/O leaf"); in ndasysctlinit()
739 SYSCTL_ADD_QUAD(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), in ndasysctlinit()
741 &softc->deletes, "Number of BIO_DELETE requests"); in ndasysctlinit()
743 SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, in ndasysctlinit()
744 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, in ndasysctlinit()
745 "trim_count", CTLFLAG_RD, &softc->trim_count, in ndasysctlinit()
747 SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, in ndasysctlinit()
748 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, in ndasysctlinit()
749 "trim_ranges", CTLFLAG_RD, &softc->trim_ranges, in ndasysctlinit()
751 SYSCTL_ADD_UQUAD(&softc->sysctl_ctx, in ndasysctlinit()
752 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, in ndasysctlinit()
753 "trim_lbas", CTLFLAG_RD, &softc->trim_lbas, in ndasysctlinit()
756 SYSCTL_ADD_INT(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), in ndasysctlinit()
757 OID_AUTO, "rotating", CTLFLAG_RD, &nda_rotating_media, 1, in ndasysctlinit()
760 SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), in ndasysctlinit()
766 softc->sysctl_stats_tree = SYSCTL_ADD_NODE(&softc->sysctl_stats_ctx, in ndasysctlinit()
767 SYSCTL_CHILDREN(softc->sysctl_tree), OID_AUTO, "stats", in ndasysctlinit()
769 if (softc->sysctl_stats_tree == NULL) { in ndasysctlinit()
774 SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, in ndasysctlinit()
775 SYSCTL_CHILDREN(softc->sysctl_stats_tree), in ndasysctlinit()
777 &softc->timeouts, 0, in ndasysctlinit()
779 SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, in ndasysctlinit()
780 SYSCTL_CHILDREN(softc->sysctl_stats_tree), in ndasysctlinit()
782 &softc->errors, 0, in ndasysctlinit()
784 SYSCTL_ADD_INT(&softc->sysctl_stats_ctx, in ndasysctlinit()
785 SYSCTL_CHILDREN(softc->sysctl_stats_tree), in ndasysctlinit()
787 &softc->invalidations, 0, in ndasysctlinit()
792 SYSCTL_ADD_PROC(&softc->sysctl_ctx, SYSCTL_CHILDREN(softc->sysctl_tree), in ndasysctlinit()
795 "Write 1 to invalidate the drive immediately"); in ndasysctlinit()
798 cam_iosched_sysctl_init(softc->cam_iosched, &softc->sysctl_ctx, in ndasysctlinit()
799 softc->sysctl_tree); in ndasysctlinit()
812 if (softc->flags != 0) in ndaflagssysctl()
813 sbuf_printf(&sbuf, "0x%b", (unsigned)softc->flags, NDA_FLAG_STRING); in ndaflagssysctl()
831 periph = (struct cam_periph *)bp->bio_disk->d_drv1; in ndagetattr()
833 ret = xpt_getattr(bp->bio_data, bp->bio_length, bp->bio_attribute, in ndagetattr()
834 periph->path); in ndagetattr()
837 bp->bio_completed = bp->bio_length; in ndagetattr()
868 periph->softc = softc; in ndaregister()
869 softc->quirks = NDA_Q_NONE; in ndaregister()
870 xpt_path_inq(&cpi, periph->path); in ndaregister()
871 TASK_INIT(&softc->sysctl_task, 0, ndasysctlinit, periph); in ndaregister()
876 softc->nsid = (uint32_t)xpt_path_lun_id(periph->path); in ndaregister()
884 "kern.cam.nda.%d.quirks", periph->unit_number); in ndaregister()
885 quirks = softc->quirks; in ndaregister()
887 softc->quirks = quirks; in ndaregister()
888 softc->disk = disk = disk_alloc(); in ndaregister()
889 disk->d_rotation_rate = DISK_RR_NON_ROTATING; in ndaregister()
890 disk->d_open = ndaopen; in ndaregister()
891 disk->d_close = ndaclose; in ndaregister()
892 disk->d_strategy = ndastrategy; in ndaregister()
893 disk->d_ioctl = ndaioctl; in ndaregister()
894 disk->d_getattr = ndagetattr; in ndaregister()
895 if (cam_sim_pollable(periph->sim)) in ndaregister()
896 disk->d_dump = ndadump; in ndaregister()
897 disk->d_gone = ndadiskgonecb; in ndaregister()
898 disk->d_name = "nda"; in ndaregister()
899 disk->d_drv1 = periph; in ndaregister()
900 disk->d_unit = periph->unit_number; in ndaregister()
906 disk->d_maxsize = maxio; in ndaregister()
907 flbas_fmt = NVMEV(NVME_NS_DATA_FLBAS_FORMAT, nsd->flbas); in ndaregister()
908 lbads = NVMEV(NVME_NS_DATA_LBAF_LBADS, nsd->lbaf[flbas_fmt]); in ndaregister()
909 disk->d_sectorsize = 1 << lbads; in ndaregister()
910 disk->d_mediasize = (off_t)(disk->d_sectorsize * nsd->nsze); in ndaregister()
911 disk->d_delmaxsize = disk->d_mediasize; in ndaregister()
912 disk->d_flags = DISKFLAG_DIRECT_COMPLETION; in ndaregister()
914 disk->d_flags |= DISKFLAG_CANDELETE; in ndaregister()
915 vwc_present = NVMEV(NVME_CTRLR_DATA_VWC_PRESENT, cd->vwc); in ndaregister()
917 disk->d_flags |= DISKFLAG_CANFLUSHCACHE; in ndaregister()
919 disk->d_flags |= DISKFLAG_UNMAPPED_BIO; in ndaregister()
920 softc->unmappedio = 1; in ndaregister()
926 cam_strvis_flag(disk->d_descr, cd->mn, NVME_MODEL_NUMBER_LENGTH, in ndaregister()
927 sizeof(disk->d_descr), CAM_STRVIS_FLAG_NONASCII_SPC); in ndaregister()
929 cam_strvis_flag(disk->d_ident, cd->sn, NVME_SERIAL_NUMBER_LENGTH, in ndaregister()
930 sizeof(disk->d_ident), CAM_STRVIS_FLAG_NONASCII_SPC); in ndaregister()
932 disk->d_hba_vendor = cpi.hba_vendor; in ndaregister()
933 disk->d_hba_device = cpi.hba_device; in ndaregister()
934 disk->d_hba_subvendor = cpi.hba_subvendor; in ndaregister()
935 disk->d_hba_subdevice = cpi.hba_subdevice; in ndaregister()
936 snprintf(disk->d_attachment, sizeof(disk->d_attachment), in ndaregister()
938 if (NVMEV(NVME_NS_DATA_NSFEAT_NPVALID, nsd->nsfeat) != 0 && in ndaregister()
939 nsd->npwg != 0) in ndaregister()
940 disk->d_stripesize = ((nsd->npwg + 1) * disk->d_sectorsize); in ndaregister()
942 disk->d_stripesize = nsd->noiob * disk->d_sectorsize; in ndaregister()
943 disk->d_stripeoffset = 0; in ndaregister()
944 disk->d_devstat = devstat_new_entry(periph->periph_name, in ndaregister()
945 periph->unit_number, disk->d_sectorsize, in ndaregister()
950 if (cam_iosched_init(&softc->cam_iosched, periph, disk, in ndaregister()
957 cam_iosched_set_sort_queue(softc->cam_iosched, 0); in ndaregister()
969 (uintmax_t)((uintmax_t)disk->d_mediasize / (1024*1024)), in ndaregister()
970 (uintmax_t)disk->d_mediasize / disk->d_sectorsize, in ndaregister()
971 disk->d_sectorsize); in ndaregister()
973 xpt_announce_quirks(periph, softc->quirks, NDA_Q_BIT_STRING); in ndaregister()
980 taskqueue_enqueue(taskqueue_thread, &softc->sysctl_task); in ndaregister()
987 ndaasync, periph, periph->path); in ndaregister()
989 softc->state = NDA_STATE_NORMAL; in ndaregister()
996 disk_create(softc->disk, DISK_VERSION); in ndaregister()
1005 struct nda_softc *softc = (struct nda_softc *)periph->softc; in ndastart()
1006 struct ccb_nvmeio *nvmeio = &start_ccb->nvmeio; in ndastart()
1008 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart\n")); in ndastart()
1010 switch (softc->state) { in ndastart()
1015 bp = cam_iosched_next_bio(softc->cam_iosched); in ndastart()
1016 CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("ndastart: bio %p\n", bp)); in ndastart()
1022 switch (bp->bio_cmd) { in ndastart()
1024 softc->flags |= NDA_FLAG_DIRTY; in ndastart()
1034 * if a write and pending write errors, then fail this in ndastart()
1040 if (bp->bio_cmd == BIO_READ) { in ndastart()
1041 if (softc->force_read_error) { in ndastart()
1042 softc->force_read_error--; in ndastart()
1043 fail = 1; in ndastart()
1045 if (softc->periodic_read_error > 0) { in ndastart()
1046 if (++softc->periodic_read_count >= in ndastart()
1047 softc->periodic_read_error) { in ndastart()
1048 softc->periodic_read_count = 0; in ndastart()
1049 fail = 1; in ndastart()
1053 if (softc->force_write_error) { in ndastart()
1054 softc->force_write_error--; in ndastart()
1055 fail = 1; in ndastart()
1065 KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 || in ndastart()
1066 round_page(bp->bio_bcount + bp->bio_ma_offset) / in ndastart()
1067 PAGE_SIZE == bp->bio_ma_n, in ndastart()
1069 nda_nvme_rw_bio(softc, &start_ccb->nvmeio, bp, bp->bio_cmd == BIO_READ ? in ndastart()
1098 TAILQ_INIT(&trim->bps); in ndastart()
1100 ents = min(nitems(trim->dsm), nda_max_trim_entries); in ndastart()
1101 ents = max(ents, 1); in ndastart()
1102 dsm_range = trim->dsm; in ndastart()
1105 TAILQ_INSERT_TAIL(&trim->bps, bp1, bio_queue); in ndastart()
1106 dsm_range->length = in ndastart()
1107 htole32(bp1->bio_bcount / softc->disk->d_sectorsize); in ndastart()
1108 dsm_range->starting_lba = in ndastart()
1109 htole64(bp1->bio_offset / softc->disk->d_sectorsize); in ndastart()
1111 totalcount += dsm_range->length; in ndastart()
1115 bp1 = cam_iosched_next_trim(softc->cam_iosched); in ndastart()
1116 /* XXX -- Could collapse adjacent ranges, but we don't for now */ in ndastart()
1117 /* XXX -- Could limit based on total payload size */ in ndastart()
1119 start_ccb->ccb_trim = trim; in ndastart()
1120 nda_nvme_trim(softc, &start_ccb->nvmeio, trim->dsm, in ndastart()
1121 dsm_range - trim->dsm); in ndastart()
1122 start_ccb->ccb_state = NDA_CCB_TRIM; in ndastart()
1123 softc->trim_count++; in ndastart()
1124 softc->trim_ranges += ranges; in ndastart()
1125 softc->trim_lbas += totalcount; in ndastart()
1128 * cam_iosched_submit_trim(softc->cam_iosched); in ndastart()
1143 start_ccb->ccb_state = NDA_CCB_BUFFER_IO; in ndastart()
1144 start_ccb->ccb_bp = bp; in ndastart()
1146 start_ccb->ccb_h.flags |= CAM_UNLOCKED; in ndastart()
1147 softc->outstanding_cmds++; in ndastart()
1148 softc->refcount++; /* For submission only */ in ndastart()
1152 softc->refcount--; /* Submission done */ in ndastart()
1165 struct ccb_nvmeio *nvmeio = &done_ccb->nvmeio; in ndadone()
1169 softc = (struct nda_softc *)periph->softc; in ndadone()
1170 path = done_ccb->ccb_h.path; in ndadone()
1174 state = nvmeio->ccb_state & NDA_CCB_TYPE_MASK; in ndadone()
1182 if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) { in ndadone()
1189 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) in ndadone()
1196 if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) in ndadone()
1203 bp = (struct bio *)done_ccb->ccb_bp; in ndadone()
1204 bp->bio_error = error; in ndadone()
1206 bp->bio_resid = bp->bio_bcount; in ndadone()
1207 bp->bio_flags |= BIO_ERROR; in ndadone()
1209 bp->bio_resid = 0; in ndadone()
1211 softc->outstanding_cmds--; in ndadone()
1219 cam_iosched_bio_complete(softc->cam_iosched, bp, done_ccb); in ndadone()
1229 trim = nvmeio->ccb_trim; in ndadone()
1231 TAILQ_CONCAT(&queue, &trim->bps, bio_queue); in ndadone()
1237 * cam_iosched_trim_done(softc->cam_iosched); in ndadone()
1246 cam_iosched_bio_complete(softc->cam_iosched, bp1, done_ccb); in ndadone()
1248 softc->outstanding_cmds--; in ndadone()
1253 bp2->bio_error = error; in ndadone()
1255 bp2->bio_flags |= BIO_ERROR; in ndadone()
1256 bp2->bio_resid = bp1->bio_bcount; in ndadone()
1258 bp2->bio_resid = 0; in ndadone()
1260 cam_iosched_bio_complete(softc->cam_iosched, bp2, NULL); in ndadone()
1267 /* No-op. We're polling */ in ndadone()
1285 periph = xpt_path_periph(ccb->ccb_h.path); in ndaerror()
1286 softc = (struct nda_softc *)periph->softc; in ndaerror()
1289 switch (ccb->ccb_h.status & CAM_STATUS_MASK) { in ndaerror()
1292 softc->timeouts++; in ndaerror()
1298 softc->errors++; in ndaerror()
1321 softc = (struct nda_softc *)periph->softc; in ndaflush()
1331 (softc->flags & NDA_FLAG_OPEN)) { in ndaflush()
1332 ndadump(softc->disk, NULL, 0, 0); in ndaflush()
1341 if ((softc->flags & NDA_FLAG_OPEN) == 0) { in ndaflush()
1347 nda_nvme_flush(softc, &ccb->nvmeio); in ndaflush()
1350 softc->disk->d_devstat); in ndaflush()
1352 xpt_print(periph->path, "Synchronize cache failed\n"); in ndaflush()