1 // SPDX-License-Identifier: GPL-2.0
4  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
8 #include <linux/blk-integrity.h>
27 	id->nsfeat |= 1 << 1;  in nvmet_bdev_set_limits()
28 	id->nawun = lpp0b;  in nvmet_bdev_set_limits()
29 	id->nawupf = lpp0b;  in nvmet_bdev_set_limits()
30 	id->nacwu = lpp0b;  in nvmet_bdev_set_limits()
37 	id->nsfeat |= 1 << 4;  in nvmet_bdev_set_limits()
39 	id->npwg = to0based(bdev_io_min(bdev) / bdev_logical_block_size(bdev));  in nvmet_bdev_set_limits()
41 	id->npwa = id->npwg;  in nvmet_bdev_set_limits()
43 	id->npdg = to0based(bdev_discard_granularity(bdev) /  in nvmet_bdev_set_limits()
46 	id->npda = id->npdg;  in nvmet_bdev_set_limits()
48 	id->nows = to0based(bdev_io_opt(bdev) / bdev_logical_block_size(bdev));  in nvmet_bdev_set_limits()
52 		id->dlfeat = (1 << 3) | 0x1;  in nvmet_bdev_set_limits()
55 void nvmet_bdev_ns_disable(struct nvmet_ns *ns)  in nvmet_bdev_ns_disable()  argument
57 	if (ns->bdev_file) {  in nvmet_bdev_ns_disable()
58 		fput(ns->bdev_file);  in nvmet_bdev_ns_disable()
59 		ns->bdev = NULL;  in nvmet_bdev_ns_disable()
60 		ns->bdev_file = NULL;  in nvmet_bdev_ns_disable()
64 static void nvmet_bdev_ns_enable_integrity(struct nvmet_ns *ns)  in nvmet_bdev_ns_enable_integrity()  argument
66 	struct blk_integrity *bi = bdev_get_integrity(ns->bdev);  in nvmet_bdev_ns_enable_integrity()
71 	if (bi->csum_type == BLK_INTEGRITY_CSUM_CRC) {  in nvmet_bdev_ns_enable_integrity()
72 		ns->metadata_size = bi->metadata_size;  in nvmet_bdev_ns_enable_integrity()
73 		if (bi->flags & BLK_INTEGRITY_REF_TAG)  in nvmet_bdev_ns_enable_integrity()
74 			ns->pi_type = NVME_NS_DPS_PI_TYPE1;  in nvmet_bdev_ns_enable_integrity()
76 			ns->pi_type = NVME_NS_DPS_PI_TYPE3;  in nvmet_bdev_ns_enable_integrity()
78 		ns->metadata_size = 0;  in nvmet_bdev_ns_enable_integrity()
82 int nvmet_bdev_ns_enable(struct nvmet_ns *ns)  in nvmet_bdev_ns_enable()  argument
91 	if (ns->buffered_io)  in nvmet_bdev_ns_enable()
92 		return -ENOTBLK;  in nvmet_bdev_ns_enable()
94 	ns->bdev_file = bdev_file_open_by_path(ns->device_path,  in nvmet_bdev_ns_enable()
96 	if (IS_ERR(ns->bdev_file)) {  in nvmet_bdev_ns_enable()
97 		ret = PTR_ERR(ns->bdev_file);  in nvmet_bdev_ns_enable()
98 		if (ret != -ENOTBLK) {  in nvmet_bdev_ns_enable()
100 					ns->device_path, ret);  in nvmet_bdev_ns_enable()
102 		ns->bdev_file = NULL;  in nvmet_bdev_ns_enable()
105 	ns->bdev = file_bdev(ns->bdev_file);  in nvmet_bdev_ns_enable()
106 	ns->size = bdev_nr_bytes(ns->bdev);  in nvmet_bdev_ns_enable()
107 	ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev));  in nvmet_bdev_ns_enable()
109 	ns->pi_type = 0;  in nvmet_bdev_ns_enable()
110 	ns->metadata_size = 0;  in nvmet_bdev_ns_enable()
112 		nvmet_bdev_ns_enable_integrity(ns);  in nvmet_bdev_ns_enable()
114 	if (bdev_is_zoned(ns->bdev)) {  in nvmet_bdev_ns_enable()
115 		if (!nvmet_bdev_zns_enable(ns)) {  in nvmet_bdev_ns_enable()
116 			nvmet_bdev_ns_disable(ns);  in nvmet_bdev_ns_enable()
117 			return -EINVAL;  in nvmet_bdev_ns_enable()
119 		ns->csi = NVME_CSI_ZNS;  in nvmet_bdev_ns_enable()
125 void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns)  in nvmet_bdev_ns_revalidate()  argument
127 	ns->size = bdev_nr_bytes(ns->bdev);  in nvmet_bdev_ns_revalidate()
145 		req->error_loc = offsetof(struct nvme_rw_command, length);  in blk_to_nvme_status()
149 		req->error_loc = offsetof(struct nvme_rw_command, slba);  in blk_to_nvme_status()
153 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in blk_to_nvme_status()
157 		req->error_loc = offsetof(struct nvme_rw_command, nsid);  in blk_to_nvme_status()
162 		req->error_loc = offsetof(struct nvme_common_command, opcode);  in blk_to_nvme_status()
165 	switch (req->cmd->common.opcode) {  in blk_to_nvme_status()
168 		req->error_slba = le64_to_cpu(req->cmd->rw.slba);  in blk_to_nvme_status()
171 		req->error_slba =  in blk_to_nvme_status()
172 			le64_to_cpu(req->cmd->write_zeroes.slba);  in blk_to_nvme_status()
175 		req->error_slba = 0;  in blk_to_nvme_status()
182 	struct nvmet_req *req = bio->bi_private;  in nvmet_bio_done()
184 	nvmet_req_complete(req, blk_to_nvme_status(req, bio->bi_status));  in nvmet_bio_done()
197 	bi = bdev_get_integrity(req->ns->bdev);  in nvmet_bdev_alloc_bip()
200 		return -ENODEV;  in nvmet_bdev_alloc_bip()
204 					bio_max_segs(req->metadata_sg_cnt));  in nvmet_bdev_alloc_bip()
211 	bip_set_seed(bip, bio->bi_iter.bi_sector >>  in nvmet_bdev_alloc_bip()
212 		     (bi->interval_exp - SECTOR_SHIFT));  in nvmet_bdev_alloc_bip()
216 		len = min_t(size_t, miter->length, resid);  in nvmet_bdev_alloc_bip()
217 		rc = bio_integrity_add_page(bio, miter->page, len,  in nvmet_bdev_alloc_bip()
218 					    offset_in_page(miter->addr));  in nvmet_bdev_alloc_bip()
222 			return -ENOMEM;  in nvmet_bdev_alloc_bip()
225 		resid -= len;  in nvmet_bdev_alloc_bip()
226 		if (len < miter->length)  in nvmet_bdev_alloc_bip()
227 			miter->consumed -= miter->length - len;  in nvmet_bdev_alloc_bip()
237 	return -EINVAL;  in nvmet_bdev_alloc_bip()
243 	unsigned int sg_cnt = req->sg_cnt;  in nvmet_bdev_execute_rw()
252 	unsigned int total_len = nvmet_rw_data_len(req) + req->metadata_len;  in nvmet_bdev_execute_rw()
257 	if (!req->sg_cnt) {  in nvmet_bdev_execute_rw()
262 	if (req->cmd->rw.opcode == nvme_cmd_write) {  in nvmet_bdev_execute_rw()
264 		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))  in nvmet_bdev_execute_rw()
272 	if (req->cmd->rw.control & cpu_to_le16(NVME_RW_LR))  in nvmet_bdev_execute_rw()
275 	if (is_pci_p2pdma_page(sg_page(req->sg)))  in nvmet_bdev_execute_rw()
278 	sector = nvmet_lba_to_sect(req->ns, req->cmd->rw.slba);  in nvmet_bdev_execute_rw()
281 		bio = &req->b.inline_bio;  in nvmet_bdev_execute_rw()
282 		bio_init(bio, req->ns->bdev, req->inline_bvec,  in nvmet_bdev_execute_rw()
283 			 ARRAY_SIZE(req->inline_bvec), opf);  in nvmet_bdev_execute_rw()
285 		bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt), opf,  in nvmet_bdev_execute_rw()
288 	bio->bi_iter.bi_sector = sector;  in nvmet_bdev_execute_rw()
289 	bio->bi_private = req;  in nvmet_bdev_execute_rw()
290 	bio->bi_end_io = nvmet_bio_done;  in nvmet_bdev_execute_rw()
293 	if (req->metadata_len)  in nvmet_bdev_execute_rw()
294 		sg_miter_start(&prot_miter, req->metadata_sg,  in nvmet_bdev_execute_rw()
295 			       req->metadata_sg_cnt, iter_flags);  in nvmet_bdev_execute_rw()
297 	for_each_sg(req->sg, sg, req->sg_cnt, i) {  in nvmet_bdev_execute_rw()
298 		while (bio_add_page(bio, sg_page(sg), sg->length, sg->offset)  in nvmet_bdev_execute_rw()
299 				!= sg->length) {  in nvmet_bdev_execute_rw()
302 			if (req->metadata_len) {  in nvmet_bdev_execute_rw()
311 			bio = bio_alloc(req->ns->bdev, bio_max_segs(sg_cnt),  in nvmet_bdev_execute_rw()
313 			bio->bi_iter.bi_sector = sector;  in nvmet_bdev_execute_rw()
319 		sector += sg->length >> 9;  in nvmet_bdev_execute_rw()
320 		sg_cnt--;  in nvmet_bdev_execute_rw()
323 	if (req->metadata_len) {  in nvmet_bdev_execute_rw()
337 	struct bio *bio = &req->b.inline_bio;  in nvmet_bdev_execute_flush()
339 	if (!bdev_write_cache(req->ns->bdev)) {  in nvmet_bdev_execute_flush()
347 	bio_init(bio, req->ns->bdev, req->inline_bvec,  in nvmet_bdev_execute_flush()
348 		 ARRAY_SIZE(req->inline_bvec), REQ_OP_WRITE | REQ_PREFLUSH);  in nvmet_bdev_execute_flush()
349 	bio->bi_private = req;  in nvmet_bdev_execute_flush()
350 	bio->bi_end_io = nvmet_bio_done;  in nvmet_bdev_execute_flush()
357 	if (!bdev_write_cache(req->ns->bdev))  in nvmet_bdev_flush()
360 	if (blkdev_issue_flush(req->ns->bdev))  in nvmet_bdev_flush()
368 	struct nvmet_ns *ns = req->ns;  in nvmet_bdev_discard_range()  local
371 	ret = __blkdev_issue_discard(ns->bdev,  in nvmet_bdev_discard_range()
372 			nvmet_lba_to_sect(ns, range->slba),  in nvmet_bdev_discard_range()
373 			le32_to_cpu(range->nlb) << (ns->blksize_shift - 9),  in nvmet_bdev_discard_range()
375 	if (ret && ret != -EOPNOTSUPP) {  in nvmet_bdev_discard_range()
376 		req->error_slba = le64_to_cpu(range->slba);  in nvmet_bdev_discard_range()
389 	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {  in nvmet_bdev_execute_discard()
401 		bio->bi_private = req;  in nvmet_bdev_execute_discard()
402 		bio->bi_end_io = nvmet_bio_done;  in nvmet_bdev_execute_discard()
417 	switch (le32_to_cpu(req->cmd->dsm.attributes)) {  in nvmet_bdev_execute_dsm()
432 	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;  in nvmet_bdev_execute_write_zeroes()
441 	sector = nvmet_lba_to_sect(req->ns, write_zeroes->slba);  in nvmet_bdev_execute_write_zeroes()
442 	nr_sector = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<  in nvmet_bdev_execute_write_zeroes()
443 		(req->ns->blksize_shift - 9));  in nvmet_bdev_execute_write_zeroes()
445 	ret = __blkdev_issue_zeroout(req->ns->bdev, sector, nr_sector,  in nvmet_bdev_execute_write_zeroes()
448 		bio->bi_private = req;  in nvmet_bdev_execute_write_zeroes()
449 		bio->bi_end_io = nvmet_bio_done;  in nvmet_bdev_execute_write_zeroes()
458 	switch (req->cmd->common.opcode) {  in nvmet_bdev_parse_io_cmd()
461 		req->execute = nvmet_bdev_execute_rw;  in nvmet_bdev_parse_io_cmd()
462 		if (req->sq->ctrl->pi_support && nvmet_ns_has_pi(req->ns))  in nvmet_bdev_parse_io_cmd()
463 			req->metadata_len = nvmet_rw_metadata_len(req);  in nvmet_bdev_parse_io_cmd()
466 		req->execute = nvmet_bdev_execute_flush;  in nvmet_bdev_parse_io_cmd()
469 		req->execute = nvmet_bdev_execute_dsm;  in nvmet_bdev_parse_io_cmd()
472 		req->execute = nvmet_bdev_execute_write_zeroes;  in nvmet_bdev_parse_io_cmd()