1 /* SPDX-License-Identifier: GPL-2.0 */
3  * Copyright (c) 2011-2014, Intel Corporation.
13 #include <linux/blk-mq.h>
14 #include <linux/sed-opal.h>
15 #include <linux/fault-inject.h>
18 #include <linux/t10-pi.h>
124 	 * Use non-standard 128 bytes SQEs.
173 	 * MSI (but not MSI-X) interrupts are broken and never fire.
239  * this structure as the first member of their request-private data.
251 	struct nvme_ctrl	*ctrl;  member
273 	if (!req->q->queuedata)  in nvme_req_qid()
276 	return req->mq_hctx->queue_num + 1;  in nvme_req_qid()
300  * @NVME_CTRL_DEAD:		Controller is non-present/unresponsive during
471 static inline enum nvme_ctrl_state nvme_ctrl_state(struct nvme_ctrl *ctrl)  in nvme_ctrl_state()  argument
473 	return READ_ONCE(ctrl->state);  in nvme_ctrl_state()
521  * there is a 1:1 relation to our namespace structures, that is ->list
574 	return IS_ENABLED(CONFIG_NVME_MULTIPATH) && head->disk;  in nvme_ns_head_multipath()
586 	struct nvme_ctrl *ctrl;  member
613 	return head->pi_type && head->ms == head->pi_size;  in nvme_ns_has_pi()
616 static inline unsigned long nvme_get_virt_boundary(struct nvme_ctrl *ctrl,  in nvme_get_virt_boundary()  argument
619 	return NVME_CTRL_PAGE_SIZE - 1;  in nvme_get_virt_boundary()
624 	struct module *module;  member
631 	int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
632 	int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
633 	int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
634 	void (*free_ctrl)(struct nvme_ctrl *ctrl);
635 	void (*submit_async_event)(struct nvme_ctrl *ctrl);
636 	int (*subsystem_reset)(struct nvme_ctrl *ctrl);
637 	void (*delete_ctrl)(struct nvme_ctrl *ctrl);
638 	void (*stop_ctrl)(struct nvme_ctrl *ctrl);
639 	int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
640 	void (*print_device_info)(struct nvme_ctrl *ctrl);
641 	bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl);
642 	unsigned long (*get_virt_boundary)(struct nvme_ctrl *ctrl, bool is_admin);
657 	return nvme_cid_install_genctr(nvme_req(rq)->genctr) | rq->tag;  in nvme_cid()
673 	if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) {  in nvme_find_rq()
674 		dev_err(nvme_req(rq)->ctrl->device,  in nvme_find_rq()
676 			tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr));  in nvme_find_rq()
693 	while (s[len - 1] == ' ')  in nvme_strlen()
694 		len--;  in nvme_strlen()
698 static inline void nvme_print_device_info(struct nvme_ctrl *ctrl)  in nvme_print_device_info()  argument
700 	struct nvme_subsystem *subsys = ctrl->subsys;  in nvme_print_device_info()
702 	if (ctrl->ops->print_device_info) {  in nvme_print_device_info()
703 		ctrl->ops->print_device_info(ctrl);  in nvme_print_device_info()
707 	dev_err(ctrl->device,  in nvme_print_device_info()
708 		"VID:%04x model:%.*s firmware:%.*s\n", subsys->vendor_id,  in nvme_print_device_info()
709 		nvme_strlen(subsys->model, sizeof(subsys->model)),  in nvme_print_device_info()
710 		subsys->model, nvme_strlen(subsys->firmware_rev,  in nvme_print_device_info()
711 					   sizeof(subsys->firmware_rev)),  in nvme_print_device_info()
712 		subsys->firmware_rev);  in nvme_print_device_info()
731 bool nvme_wait_reset(struct nvme_ctrl *ctrl);
732 int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
734 static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)  in nvme_reset_subsystem()  argument
736 	if (!ctrl->subsystem || !ctrl->ops->subsystem_reset)  in nvme_reset_subsystem()
737 		return -ENOTTY;  in nvme_reset_subsystem()
738 	return ctrl->ops->subsystem_reset(ctrl);  in nvme_reset_subsystem()
746 	return sector >> (head->lba_shift - SECTOR_SHIFT);  in nvme_sect_to_lba()
754 	return lba << (head->lba_shift - SECTOR_SHIFT);  in nvme_lba_to_sect()
758  * Convert byte length to nvme's 0-based num dwords
762 	return (len >> 2) - 1;  in nvme_bytes_to_numd()
765 /* Decode a 2-byte "0's based"/"0-based" field */
791  * if blk-mq will need to use IPI magic to complete the request, and if yes do
799 	struct nvme_ctrl *ctrl = rq->ctrl;  in nvme_try_complete_req()  local
801 	if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))  in nvme_try_complete_req()
802 		rq->genctr++;  in nvme_try_complete_req()
804 	rq->status = le16_to_cpu(status) >> 1;  in nvme_try_complete_req()
805 	rq->result = result;  in nvme_try_complete_req()
808 	if (unlikely(blk_should_fake_timeout(req->q)))  in nvme_try_complete_req()
813 static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)  in nvme_get_ctrl()  argument
815 	get_device(ctrl->device);  in nvme_get_ctrl()
818 static inline void nvme_put_ctrl(struct nvme_ctrl *ctrl)  in nvme_put_ctrl()  argument
820 	put_device(ctrl->device);  in nvme_put_ctrl()
832 static inline bool nvme_state_terminal(struct nvme_ctrl *ctrl)  in nvme_state_terminal()  argument
834 	switch (nvme_ctrl_state(ctrl)) {  in nvme_state_terminal()
845 		WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);  in nvme_state_terminal()
859 	rq_list_for_each(&iob->req_list, req) {  in nvme_complete_batch()
868 void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
869 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
870 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
872 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown);
873 int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
874 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
876 int nvme_add_ctrl(struct nvme_ctrl *ctrl);
877 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
878 void nvme_start_ctrl(struct nvme_ctrl *ctrl);
879 void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
880 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended);
881 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
883 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl);
884 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
887 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl);
889 void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
891 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
894 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl);
895 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl);
896 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl);
897 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl);
898 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl);
899 void nvme_sync_queues(struct nvme_ctrl *ctrl);
900 void nvme_sync_io_queues(struct nvme_ctrl *ctrl);
901 void nvme_unfreeze(struct nvme_ctrl *ctrl);
902 void nvme_wait_freeze(struct nvme_ctrl *ctrl);
903 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
904 void nvme_start_freeze(struct nvme_ctrl *ctrl);
911 #define NVME_QID_ANY -1
915 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
917 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
920 static inline bool nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,  in nvme_check_ready()  argument
923 	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);  in nvme_check_ready()
927 	if (ctrl->ops->flags & NVME_F_FABRICS && state == NVME_CTRL_DELETING)  in nvme_check_ready()
929 	return __nvme_check_ready(ctrl, rq, queue_live, state);  in nvme_check_ready()
941 static inline bool nvme_is_unique_nsid(struct nvme_ctrl *ctrl,  in nvme_is_unique_nsid()  argument
944 	return head->shared ||  in nvme_is_unique_nsid()
945 		(ctrl->oacs & NVME_CTRL_OACS_NS_MNGT_SUPP) ||  in nvme_is_unique_nsid()
946 		(ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA) ||  in nvme_is_unique_nsid()
947 		(ctrl->ctratt & NVME_CTRL_CTRATT_NVM_SETS);  in nvme_is_unique_nsid()
977 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
978 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
979 int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
980 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
981 int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
982 void nvme_queue_scan(struct nvme_ctrl *ctrl);
983 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
988 		const struct file_operations *fops, struct module *owner);
1005 int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1019 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
1022 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)  in nvme_ctrl_use_ana()  argument
1024 	return ctrl->ana_log_buf != NULL;  in nvme_ctrl_use_ana()
1032 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
1033 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
1038 int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
1039 void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
1040 void nvme_mpath_update(struct nvme_ctrl *ctrl);
1041 void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
1042 void nvme_mpath_stop(struct nvme_ctrl *ctrl);
1045 void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
1052 	struct nvme_ns *ns = req->q->queuedata;  in nvme_trace_bio_complete()
1054 	if ((req->cmd_flags & REQ_NVME_MPATH) && req->bio)  in nvme_trace_bio_complete()
1055 		trace_block_bio_complete(ns->head->disk->queue, req->bio);  in nvme_trace_bio_complete()
1068 	return disk->fops == &nvme_ns_head_ops;  in nvme_disk_is_ns_head()
1072 	if (test_bit(NVME_NSHEAD_QUEUE_IF_NO_PATH, &head->flags))  in nvme_mpath_queue_if_no_path()
1078 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)  in nvme_ctrl_use_ana()  argument
1085 static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)  in nvme_kick_requeue_lists()  argument
1088 static inline int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,  in nvme_mpath_alloc_disk()  argument
1112 static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)  in nvme_mpath_clear_ctrl_paths()  argument
1121 static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_mpath_init_ctrl()  argument
1124 static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,  in nvme_mpath_init_identify()  argument
1127 	if (ctrl->subsys->cmic & NVME_CTRL_CMIC_ANA)  in nvme_mpath_init_identify()
1128 		dev_warn(ctrl->device,  in nvme_mpath_init_identify()
1129 "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");  in nvme_mpath_init_identify()
1132 static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)  in nvme_mpath_update()  argument
1135 static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)  in nvme_mpath_uninit()  argument
1138 static inline void nvme_mpath_stop(struct nvme_ctrl *ctrl)  in nvme_mpath_stop()  argument
1202 	return disk->private_data;  in nvme_get_ns_from_dev()
1206 int nvme_hwmon_init(struct nvme_ctrl *ctrl);
1207 void nvme_hwmon_exit(struct nvme_ctrl *ctrl);
1209 static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl)  in nvme_hwmon_init()  argument
1214 static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl)  in nvme_hwmon_exit()  argument
1221 	if (rq->cmd_flags & REQ_NVME_MPATH)  in nvme_start_request()
1226 static inline bool nvme_ctrl_sgl_supported(struct nvme_ctrl *ctrl)  in nvme_ctrl_sgl_supported()  argument
1228 	return ctrl->sgls & (NVME_CTRL_SGLS_BYTE_ALIGNED |  in nvme_ctrl_sgl_supported()
1232 static inline bool nvme_ctrl_meta_sgl_supported(struct nvme_ctrl *ctrl)  in nvme_ctrl_meta_sgl_supported()  argument
1234 	if (ctrl->ops->flags & NVME_F_FABRICS)  in nvme_ctrl_meta_sgl_supported()
1236 	return ctrl->sgls & NVME_CTRL_SGLS_MSDS;  in nvme_ctrl_meta_sgl_supported()
1242 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl);
1243 void nvme_auth_stop(struct nvme_ctrl *ctrl);
1244 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid);
1245 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid);
1246 void nvme_auth_free(struct nvme_ctrl *ctrl);
1247 void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl);
1249 static inline int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl)  in nvme_auth_init_ctrl()  argument
1260 static inline void nvme_auth_stop(struct nvme_ctrl *ctrl) {};  in nvme_auth_stop()  argument
1261 static inline int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_negotiate()  argument
1263 	return -EPROTONOSUPPORT;  in nvme_auth_negotiate()
1265 static inline int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid)  in nvme_auth_wait()  argument
1267 	return -EPROTONOSUPPORT;  in nvme_auth_wait()
1269 static inline void nvme_auth_free(struct nvme_ctrl *ctrl) {};  in nvme_auth_free()  argument
1270 static inline void nvme_auth_revoke_tls_key(struct nvme_ctrl *ctrl) {};  in nvme_auth_revoke_tls_key()  argument
1273 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1275 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode);
1277 void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
1280 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
1284 static inline bool nvme_multi_css(struct nvme_ctrl *ctrl)  in nvme_multi_css()  argument
1286 	return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;  in nvme_multi_css()