1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (c) 2015-2016 HGST, a Western Digital Company. 4 */ 5 6 #ifndef _NVMET_H 7 #define _NVMET_H 8 9 #include <linux/dma-mapping.h> 10 #include <linux/types.h> 11 #include <linux/device.h> 12 #include <linux/kref.h> 13 #include <linux/percpu-refcount.h> 14 #include <linux/list.h> 15 #include <linux/mutex.h> 16 #include <linux/uuid.h> 17 #include <linux/nvme.h> 18 #include <linux/configfs.h> 19 #include <linux/rcupdate.h> 20 #include <linux/blkdev.h> 21 #include <linux/radix-tree.h> 22 #include <linux/t10-pi.h> 23 #include <linux/kfifo.h> 24 25 #define NVMET_DEFAULT_VS NVME_VS(2, 1, 0) 26 27 #define NVMET_NS_ENABLED XA_MARK_1 28 #define NVMET_ASYNC_EVENTS 4 29 #define NVMET_ERROR_LOG_SLOTS 128 30 #define NVMET_NO_ERROR_LOC ((u16)-1) 31 #define NVMET_DEFAULT_CTRL_MODEL "Linux" 32 #define NVMET_MN_MAX_SIZE 40 33 #define NVMET_SN_MAX_SIZE 20 34 #define NVMET_FR_MAX_SIZE 8 35 #define NVMET_PR_LOG_QUEUE_SIZE 64 36 37 #define nvmet_for_each_ns(xa, index, entry) \ 38 xa_for_each(xa, index, entry) 39 40 #define nvmet_for_each_enabled_ns(xa, index, entry) \ 41 xa_for_each_marked(xa, index, entry, NVMET_NS_ENABLED) 42 43 /* 44 * Supported optional AENs: 45 */ 46 #define NVMET_AEN_CFG_OPTIONAL \ 47 (NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_ANA_CHANGE) 48 #define NVMET_DISC_AEN_CFG_OPTIONAL \ 49 (NVME_AEN_CFG_DISC_CHANGE) 50 51 /* 52 * Plus mandatory SMART AENs (we'll never send them, but allow enabling them): 53 */ 54 #define NVMET_AEN_CFG_ALL \ 55 (NVME_SMART_CRIT_SPARE | NVME_SMART_CRIT_TEMPERATURE | \ 56 NVME_SMART_CRIT_RELIABILITY | NVME_SMART_CRIT_MEDIA | \ 57 NVME_SMART_CRIT_VOLATILE_MEMORY | NVMET_AEN_CFG_OPTIONAL) 58 59 /* Helper Macros when NVMe error is NVME_SC_CONNECT_INVALID_PARAM 60 * The 16 bit shift is to set IATTR bit to 1, which means offending 61 * offset starts in the data section of connect() 62 */ 63 #define IPO_IATTR_CONNECT_DATA(x) \ 64 (cpu_to_le32((1 << 16) | (offsetof(struct nvmf_connect_data, x)))) 65 #define IPO_IATTR_CONNECT_SQE(x) \ 66 (cpu_to_le32(offsetof(struct nvmf_connect_command, x))) 67 68 struct nvmet_pr_registrant { 69 u64 rkey; 70 uuid_t hostid; 71 enum nvme_pr_type rtype; 72 struct list_head entry; 73 struct rcu_head rcu; 74 }; 75 76 struct nvmet_pr { 77 bool enable; 78 unsigned long notify_mask; 79 atomic_t generation; 80 struct nvmet_pr_registrant __rcu *holder; 81 /* 82 * During the execution of the reservation command, mutual 83 * exclusion is required throughout the process. However, 84 * while waiting asynchronously for the 'per controller 85 * percpu_ref' to complete before the 'preempt and abort' 86 * command finishes, a semaphore is needed to ensure mutual 87 * exclusion instead of a mutex. 88 */ 89 struct semaphore pr_sem; 90 struct list_head registrant_list; 91 }; 92 93 struct nvmet_pr_per_ctrl_ref { 94 struct percpu_ref ref; 95 struct completion free_done; 96 struct completion confirm_done; 97 uuid_t hostid; 98 }; 99 100 struct nvmet_ns { 101 struct percpu_ref ref; 102 struct file *bdev_file; 103 struct block_device *bdev; 104 struct file *file; 105 bool readonly; 106 u32 nsid; 107 u32 blksize_shift; 108 loff_t size; 109 u8 nguid[16]; 110 uuid_t uuid; 111 u32 anagrpid; 112 113 bool buffered_io; 114 bool enabled; 115 #define NVMET_NS_IO_LIVE 0 116 unsigned long flags; 117 struct nvmet_subsys *subsys; 118 const char *device_path; 119 120 struct config_group device_group; 121 struct config_group group; 122 123 struct completion disable_done; 124 mempool_t *bvec_pool; 125 126 struct pci_dev *p2p_dev; 127 int use_p2pmem; 128 int pi_type; 129 int metadata_size; 130 u8 csi; 131 struct nvmet_pr pr; 132 struct xarray pr_per_ctrl_refs; 133 #ifdef CONFIG_NVME_TARGET_DEBUGFS 134 struct dentry *debugfs_dir; 135 #endif 136 }; 137 138 static inline struct nvmet_ns *to_nvmet_ns(struct config_item *item) 139 { 140 return container_of(to_config_group(item), struct nvmet_ns, group); 141 } 142 143 static inline struct device *nvmet_ns_dev(struct nvmet_ns *ns) 144 { 145 return ns->bdev ? disk_to_dev(ns->bdev->bd_disk) : NULL; 146 } 147 148 struct nvmet_cq { 149 struct nvmet_ctrl *ctrl; 150 u16 qid; 151 u16 size; 152 refcount_t ref; 153 }; 154 155 struct nvmet_sq { 156 struct nvmet_ctrl *ctrl; 157 struct percpu_ref ref; 158 struct nvmet_cq *cq; 159 u16 qid; 160 u16 size; 161 u32 sqhd; 162 bool sqhd_disabled; 163 #ifdef CONFIG_NVME_TARGET_AUTH 164 struct mutex auth_lock; 165 bool authenticated; 166 struct delayed_work auth_expired_work; 167 bool auth_expiry_active; 168 u16 dhchap_tid; 169 u8 sc_c; 170 u8 dhchap_status; 171 u8 dhchap_step; 172 u8 *dhchap_c1; 173 u8 *dhchap_c2; 174 u32 dhchap_s1; 175 u32 dhchap_s2; 176 u8 *dhchap_skey; 177 int dhchap_skey_len; 178 #endif 179 #ifdef CONFIG_NVME_TARGET_TCP_TLS 180 struct key *tls_key; 181 #endif 182 struct completion free_done; 183 struct completion confirm_done; 184 }; 185 186 struct nvmet_ana_group { 187 struct config_group group; 188 struct nvmet_port *port; 189 u32 grpid; 190 }; 191 192 static inline struct nvmet_ana_group *to_ana_group(struct config_item *item) 193 { 194 return container_of(to_config_group(item), struct nvmet_ana_group, 195 group); 196 } 197 198 /** 199 * struct nvmet_port - Common structure to keep port 200 * information for the target. 201 * @entry: Entry into referrals or transport list. 202 * @disc_addr: Address information is stored in a format defined 203 * for a discovery log page entry. 204 * @group: ConfigFS group for this element's folder. 205 * @priv: Private data for the transport. 206 */ 207 struct nvmet_port { 208 struct list_head entry; 209 struct nvmf_disc_rsp_page_entry disc_addr; 210 struct config_group group; 211 struct config_group subsys_group; 212 struct list_head subsystems; 213 struct config_group referrals_group; 214 struct list_head referrals; 215 struct list_head global_entry; 216 struct config_group ana_groups_group; 217 struct nvmet_ana_group ana_default_group; 218 struct key *keyring; 219 void *priv; 220 bool enabled; 221 int inline_data_size; 222 int max_queue_size; 223 int mdts; 224 const struct nvmet_fabrics_ops *tr_ops; 225 bool pi_enable; 226 enum nvme_ana_state ana_state[]; 227 }; 228 229 static inline struct nvmet_port *to_nvmet_port(struct config_item *item) 230 { 231 return container_of(to_config_group(item), struct nvmet_port, 232 group); 233 } 234 235 static inline struct nvmet_port *ana_groups_to_port( 236 struct config_item *item) 237 { 238 return container_of(to_config_group(item), struct nvmet_port, 239 ana_groups_group); 240 } 241 242 static inline u8 nvmet_port_disc_addr_treq_secure_channel(struct nvmet_port *port) 243 { 244 return (port->disc_addr.treq & NVME_TREQ_SECURE_CHANNEL_MASK); 245 } 246 247 static inline bool nvmet_port_secure_channel_required(struct nvmet_port *port) 248 { 249 return nvmet_port_disc_addr_treq_secure_channel(port) == NVMF_TREQ_REQUIRED; 250 } 251 252 struct nvmet_pr_log_mgr { 253 struct mutex lock; 254 u64 lost_count; 255 u64 counter; 256 DECLARE_KFIFO(log_queue, struct nvme_pr_log, NVMET_PR_LOG_QUEUE_SIZE); 257 }; 258 259 struct nvmet_ctrl { 260 struct nvmet_subsys *subsys; 261 struct nvmet_sq **sqs; 262 struct nvmet_cq **cqs; 263 264 void *drvdata; 265 266 bool reset_tbkas; 267 268 struct mutex lock; 269 u64 cap; 270 u32 cc; 271 u32 csts; 272 273 uuid_t hostid; 274 u16 cntlid; 275 u16 max_qid; 276 u32 kato; 277 bool tbkas; 278 279 struct nvmet_port *port; 280 281 u32 aen_enabled; 282 unsigned long aen_masked; 283 struct nvmet_req *async_event_cmds[NVMET_ASYNC_EVENTS]; 284 unsigned int nr_async_event_cmds; 285 struct list_head async_events; 286 struct work_struct async_event_work; 287 288 struct list_head subsys_entry; 289 struct kref ref; 290 struct delayed_work ka_work; 291 struct work_struct fatal_err_work; 292 293 const struct nvmet_fabrics_ops *ops; 294 295 __le32 *changed_ns_list; 296 u32 nr_changed_ns; 297 298 char hostnqn[NVMF_NQN_FIELD_LEN]; 299 300 struct device *p2p_client; 301 struct radix_tree_root p2p_ns_map; 302 #ifdef CONFIG_NVME_TARGET_DEBUGFS 303 struct dentry *debugfs_dir; 304 #endif 305 spinlock_t error_lock; 306 u64 err_counter; 307 struct nvme_error_slot slots[NVMET_ERROR_LOG_SLOTS]; 308 bool pi_support; 309 bool concat; 310 #ifdef CONFIG_NVME_TARGET_AUTH 311 struct nvme_dhchap_key *host_key; 312 struct nvme_dhchap_key *ctrl_key; 313 u8 shash_id; 314 struct crypto_kpp *dh_tfm; 315 u8 dh_gid; 316 u8 *dh_key; 317 size_t dh_keysize; 318 #endif 319 #ifdef CONFIG_NVME_TARGET_TCP_TLS 320 struct key *tls_key; 321 #endif 322 struct nvmet_pr_log_mgr pr_log_mgr; 323 }; 324 325 struct nvmet_passthru { 326 struct percpu_ref ref; 327 struct completion disable_done; 328 #define NVMET_PASSTHRU_ENABLED 0 329 unsigned long flags; 330 331 struct nvme_ctrl *ctrl; 332 char *ctrl_path; 333 unsigned int admin_timeout; 334 unsigned int io_timeout; 335 unsigned int clear_ids; 336 }; 337 338 struct nvmet_subsys { 339 enum nvme_subsys_type type; 340 341 struct mutex lock; 342 struct kref ref; 343 344 struct xarray namespaces; 345 unsigned int nr_namespaces; 346 u32 max_nsid; 347 u16 cntlid_min; 348 u16 cntlid_max; 349 350 struct list_head ctrls; 351 352 struct list_head hosts; 353 bool allow_any_host; 354 #ifdef CONFIG_NVME_TARGET_DEBUGFS 355 struct dentry *debugfs_dir; 356 #endif 357 u16 max_qid; 358 359 u64 ver; 360 char serial[NVMET_SN_MAX_SIZE]; 361 bool subsys_discovered; 362 char *subsysnqn; 363 bool pi_support; 364 bool tbkas; 365 366 struct config_group group; 367 368 struct config_group namespaces_group; 369 struct config_group allowed_hosts_group; 370 371 u16 vendor_id; 372 u16 subsys_vendor_id; 373 char *model_number; 374 u32 ieee_oui; 375 char *firmware_rev; 376 377 #ifdef CONFIG_NVME_TARGET_PASSTHRU 378 struct nvmet_passthru *passthru; 379 struct config_group passthru_group; 380 #endif /* CONFIG_NVME_TARGET_PASSTHRU */ 381 382 #ifdef CONFIG_BLK_DEV_ZONED 383 u8 zasl; 384 #endif /* CONFIG_BLK_DEV_ZONED */ 385 }; 386 387 static inline struct nvmet_subsys *to_subsys(struct config_item *item) 388 { 389 return container_of(to_config_group(item), struct nvmet_subsys, group); 390 } 391 392 static inline struct nvmet_subsys *namespaces_to_subsys( 393 struct config_item *item) 394 { 395 return container_of(to_config_group(item), struct nvmet_subsys, 396 namespaces_group); 397 } 398 399 struct nvmet_host { 400 struct config_group group; 401 u8 *dhchap_secret; 402 u8 *dhchap_ctrl_secret; 403 u8 dhchap_key_hash; 404 u8 dhchap_ctrl_key_hash; 405 u8 dhchap_hash_id; 406 u8 dhchap_dhgroup_id; 407 }; 408 409 static inline struct nvmet_host *to_host(struct config_item *item) 410 { 411 return container_of(to_config_group(item), struct nvmet_host, group); 412 } 413 414 static inline char *nvmet_host_name(struct nvmet_host *host) 415 { 416 return config_item_name(&host->group.cg_item); 417 } 418 419 struct nvmet_host_link { 420 struct list_head entry; 421 struct nvmet_host *host; 422 }; 423 424 struct nvmet_subsys_link { 425 struct list_head entry; 426 struct nvmet_subsys *subsys; 427 }; 428 429 struct nvmet_req; 430 struct nvmet_fabrics_ops { 431 struct module *owner; 432 unsigned int type; 433 unsigned int msdbd; 434 unsigned int flags; 435 #define NVMF_KEYED_SGLS (1 << 0) 436 #define NVMF_METADATA_SUPPORTED (1 << 1) 437 void (*queue_response)(struct nvmet_req *req); 438 int (*add_port)(struct nvmet_port *port); 439 void (*remove_port)(struct nvmet_port *port); 440 void (*delete_ctrl)(struct nvmet_ctrl *ctrl); 441 void (*disc_traddr)(struct nvmet_req *req, 442 struct nvmet_port *port, char *traddr); 443 ssize_t (*host_traddr)(struct nvmet_ctrl *ctrl, 444 char *traddr, size_t traddr_len); 445 u16 (*install_queue)(struct nvmet_sq *nvme_sq); 446 void (*discovery_chg)(struct nvmet_port *port); 447 u8 (*get_mdts)(const struct nvmet_ctrl *ctrl); 448 u16 (*get_max_queue_size)(const struct nvmet_ctrl *ctrl); 449 450 /* Operations mandatory for PCI target controllers */ 451 u16 (*create_sq)(struct nvmet_ctrl *ctrl, u16 sqid, u16 cqid, u16 flags, 452 u16 qsize, u64 prp1); 453 u16 (*delete_sq)(struct nvmet_ctrl *ctrl, u16 sqid); 454 u16 (*create_cq)(struct nvmet_ctrl *ctrl, u16 cqid, u16 flags, 455 u16 qsize, u64 prp1, u16 irq_vector); 456 u16 (*delete_cq)(struct nvmet_ctrl *ctrl, u16 cqid); 457 u16 (*set_feature)(const struct nvmet_ctrl *ctrl, u8 feat, 458 void *feat_data); 459 u16 (*get_feature)(const struct nvmet_ctrl *ctrl, u8 feat, 460 void *feat_data); 461 }; 462 463 #define NVMET_MAX_INLINE_BIOVEC 8 464 #define NVMET_MAX_INLINE_DATA_LEN NVMET_MAX_INLINE_BIOVEC * PAGE_SIZE 465 466 struct nvmet_req { 467 struct nvme_command *cmd; 468 struct nvme_completion *cqe; 469 struct nvmet_sq *sq; 470 struct nvmet_cq *cq; 471 struct nvmet_ns *ns; 472 struct scatterlist *sg; 473 struct scatterlist *metadata_sg; 474 struct bio_vec inline_bvec[NVMET_MAX_INLINE_BIOVEC]; 475 union { 476 struct { 477 struct bio inline_bio; 478 } b; 479 struct { 480 bool mpool_alloc; 481 struct kiocb iocb; 482 struct bio_vec *bvec; 483 struct work_struct work; 484 } f; 485 struct { 486 struct bio inline_bio; 487 struct request *rq; 488 struct work_struct work; 489 bool use_workqueue; 490 bool ref_held; 491 } p; 492 #ifdef CONFIG_BLK_DEV_ZONED 493 struct { 494 struct bio inline_bio; 495 struct work_struct zmgmt_work; 496 } z; 497 #endif /* CONFIG_BLK_DEV_ZONED */ 498 struct { 499 struct work_struct abort_work; 500 } r; 501 }; 502 int sg_cnt; 503 int metadata_sg_cnt; 504 /* data length as parsed from the SGL descriptor: */ 505 size_t transfer_len; 506 size_t metadata_len; 507 508 struct nvmet_port *port; 509 510 void (*execute)(struct nvmet_req *req); 511 const struct nvmet_fabrics_ops *ops; 512 513 struct pci_dev *p2p_dev; 514 struct device *p2p_client; 515 u16 error_loc; 516 u64 error_slba; 517 struct nvmet_pr_per_ctrl_ref *pc_ref; 518 }; 519 520 #define NVMET_MAX_MPOOL_BVEC 16 521 extern struct kmem_cache *nvmet_bvec_cache; 522 extern struct workqueue_struct *buffered_io_wq; 523 extern struct workqueue_struct *zbd_wq; 524 extern struct workqueue_struct *nvmet_wq; 525 extern struct workqueue_struct *nvmet_aen_wq; 526 527 static inline void nvmet_set_result(struct nvmet_req *req, u32 result) 528 { 529 req->cqe->result.u32 = cpu_to_le32(result); 530 } 531 532 /* 533 * NVMe command writes actually are DMA reads for us on the target side. 534 */ 535 static inline enum dma_data_direction 536 nvmet_data_dir(struct nvmet_req *req) 537 { 538 return nvme_is_write(req->cmd) ? DMA_FROM_DEVICE : DMA_TO_DEVICE; 539 } 540 541 struct nvmet_async_event { 542 struct list_head entry; 543 u8 event_type; 544 u8 event_info; 545 u8 log_page; 546 }; 547 548 static inline void nvmet_clear_aen_bit(struct nvmet_req *req, u32 bn) 549 { 550 int rae = le32_to_cpu(req->cmd->common.cdw10) & 1 << 15; 551 552 if (!rae) 553 clear_bit(bn, &req->sq->ctrl->aen_masked); 554 } 555 556 static inline bool nvmet_aen_bit_disabled(struct nvmet_ctrl *ctrl, u32 bn) 557 { 558 if (!(READ_ONCE(ctrl->aen_enabled) & (1 << bn))) 559 return true; 560 return test_and_set_bit(bn, &ctrl->aen_masked); 561 } 562 563 void nvmet_get_feat_kato(struct nvmet_req *req); 564 void nvmet_get_feat_async_event(struct nvmet_req *req); 565 u16 nvmet_set_feat_kato(struct nvmet_req *req); 566 u16 nvmet_set_feat_async_event(struct nvmet_req *req, u32 mask); 567 void nvmet_execute_async_event(struct nvmet_req *req); 568 void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl); 569 void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl); 570 571 u16 nvmet_parse_connect_cmd(struct nvmet_req *req); 572 u32 nvmet_connect_cmd_data_len(struct nvmet_req *req); 573 void nvmet_bdev_set_limits(struct block_device *bdev, struct nvme_id_ns *id); 574 void nvmet_bdev_set_nvm_limits(struct block_device *bdev, 575 struct nvme_id_ns_nvm *id); 576 u16 nvmet_bdev_parse_io_cmd(struct nvmet_req *req); 577 u16 nvmet_file_parse_io_cmd(struct nvmet_req *req); 578 u16 nvmet_bdev_zns_parse_io_cmd(struct nvmet_req *req); 579 u32 nvmet_admin_cmd_data_len(struct nvmet_req *req); 580 u16 nvmet_parse_admin_cmd(struct nvmet_req *req); 581 u32 nvmet_discovery_cmd_data_len(struct nvmet_req *req); 582 u16 nvmet_parse_discovery_cmd(struct nvmet_req *req); 583 u16 nvmet_parse_fabrics_admin_cmd(struct nvmet_req *req); 584 u32 nvmet_fabrics_admin_cmd_data_len(struct nvmet_req *req); 585 u16 nvmet_parse_fabrics_io_cmd(struct nvmet_req *req); 586 u32 nvmet_fabrics_io_cmd_data_len(struct nvmet_req *req); 587 588 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_sq *sq, 589 const struct nvmet_fabrics_ops *ops); 590 void nvmet_req_uninit(struct nvmet_req *req); 591 size_t nvmet_req_transfer_len(struct nvmet_req *req); 592 bool nvmet_check_transfer_len(struct nvmet_req *req, size_t len); 593 bool nvmet_check_data_len_lte(struct nvmet_req *req, size_t data_len); 594 void nvmet_req_complete(struct nvmet_req *req, u16 status); 595 int nvmet_req_alloc_sgls(struct nvmet_req *req); 596 void nvmet_req_free_sgls(struct nvmet_req *req); 597 598 void nvmet_execute_set_features(struct nvmet_req *req); 599 void nvmet_execute_get_features(struct nvmet_req *req); 600 void nvmet_execute_keep_alive(struct nvmet_req *req); 601 602 u16 nvmet_check_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create); 603 u16 nvmet_check_io_cqid(struct nvmet_ctrl *ctrl, u16 cqid, bool create); 604 void nvmet_cq_init(struct nvmet_cq *cq); 605 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, u16 qid, 606 u16 size); 607 u16 nvmet_cq_create(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, u16 qid, 608 u16 size); 609 void nvmet_cq_destroy(struct nvmet_cq *cq); 610 bool nvmet_cq_get(struct nvmet_cq *cq); 611 void nvmet_cq_put(struct nvmet_cq *cq); 612 bool nvmet_cq_in_use(struct nvmet_cq *cq); 613 u16 nvmet_check_sqid(struct nvmet_ctrl *ctrl, u16 sqid, bool create); 614 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, u16 qid, 615 u16 size); 616 u16 nvmet_sq_create(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, 617 struct nvmet_cq *cq, u16 qid, u16 size); 618 void nvmet_sq_destroy(struct nvmet_sq *sq); 619 int nvmet_sq_init(struct nvmet_sq *sq, struct nvmet_cq *cq); 620 621 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl); 622 623 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new); 624 625 struct nvmet_alloc_ctrl_args { 626 struct nvmet_port *port; 627 struct nvmet_sq *sq; 628 char *subsysnqn; 629 char *hostnqn; 630 uuid_t *hostid; 631 const struct nvmet_fabrics_ops *ops; 632 struct device *p2p_client; 633 u32 kato; 634 __le32 result; 635 u16 error_loc; 636 u16 status; 637 }; 638 639 struct nvmet_ctrl *nvmet_alloc_ctrl(struct nvmet_alloc_ctrl_args *args); 640 struct nvmet_ctrl *nvmet_ctrl_find_get(const char *subsysnqn, 641 const char *hostnqn, u16 cntlid, 642 struct nvmet_req *req); 643 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl); 644 u16 nvmet_check_ctrl_status(struct nvmet_req *req); 645 ssize_t nvmet_ctrl_host_traddr(struct nvmet_ctrl *ctrl, 646 char *traddr, size_t traddr_len); 647 648 struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn, 649 enum nvme_subsys_type type); 650 void nvmet_subsys_put(struct nvmet_subsys *subsys); 651 void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys); 652 653 u16 nvmet_req_find_ns(struct nvmet_req *req); 654 void nvmet_put_namespace(struct nvmet_ns *ns); 655 int nvmet_ns_enable(struct nvmet_ns *ns); 656 void nvmet_ns_disable(struct nvmet_ns *ns); 657 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid); 658 void nvmet_ns_free(struct nvmet_ns *ns); 659 660 void nvmet_send_ana_event(struct nvmet_subsys *subsys, 661 struct nvmet_port *port); 662 void nvmet_port_send_ana_event(struct nvmet_port *port); 663 664 int nvmet_register_transport(const struct nvmet_fabrics_ops *ops); 665 void nvmet_unregister_transport(const struct nvmet_fabrics_ops *ops); 666 667 void nvmet_port_del_ctrls(struct nvmet_port *port, 668 struct nvmet_subsys *subsys); 669 670 int nvmet_enable_port(struct nvmet_port *port); 671 void nvmet_disable_port(struct nvmet_port *port); 672 673 void nvmet_referral_enable(struct nvmet_port *parent, struct nvmet_port *port); 674 void nvmet_referral_disable(struct nvmet_port *parent, struct nvmet_port *port); 675 676 u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf, 677 size_t len); 678 u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, 679 size_t len); 680 u16 nvmet_zero_sgl(struct nvmet_req *req, off_t off, size_t len); 681 682 u32 nvmet_get_log_page_len(struct nvme_command *cmd); 683 u64 nvmet_get_log_page_offset(struct nvme_command *cmd); 684 685 extern struct list_head *nvmet_ports; 686 void nvmet_port_disc_changed(struct nvmet_port *port, 687 struct nvmet_subsys *subsys); 688 void nvmet_subsys_disc_changed(struct nvmet_subsys *subsys, 689 struct nvmet_host *host); 690 void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, 691 u8 event_info, u8 log_page); 692 693 #define NVMET_MIN_QUEUE_SIZE 16 694 #define NVMET_MAX_QUEUE_SIZE 1024 695 #define NVMET_NR_QUEUES 128 696 #define NVMET_MAX_CMD(ctrl) (NVME_CAP_MQES(ctrl->cap) + 1) 697 #define NVMET_MAX_MDTS 255 698 699 /* 700 * Nice round number that makes a list of nsids fit into a page. 701 * Should become tunable at some point in the future. 702 */ 703 #define NVMET_MAX_NAMESPACES 1024 704 705 /* 706 * 0 is not a valid ANA group ID, so we start numbering at 1. 707 * 708 * ANA Group 1 exists without manual intervention, has namespaces assigned to it 709 * by default, and is available in an optimized state through all ports. 710 */ 711 #define NVMET_MAX_ANAGRPS 128 712 #define NVMET_DEFAULT_ANA_GRPID 1 713 714 #define NVMET_KAS 10 715 #define NVMET_DISC_KATO_MS 120000 716 717 int __init nvmet_init_configfs(void); 718 void __exit nvmet_exit_configfs(void); 719 720 int __init nvmet_init_discovery(void); 721 void nvmet_exit_discovery(void); 722 723 extern struct nvmet_subsys *nvmet_disc_subsys; 724 extern struct rw_semaphore nvmet_config_sem; 725 726 extern u32 nvmet_ana_group_enabled[NVMET_MAX_ANAGRPS + 1]; 727 extern u64 nvmet_ana_chgcnt; 728 extern struct rw_semaphore nvmet_ana_sem; 729 730 bool nvmet_host_allowed(struct nvmet_subsys *subsys, const char *hostnqn); 731 732 int nvmet_bdev_ns_enable(struct nvmet_ns *ns); 733 int nvmet_file_ns_enable(struct nvmet_ns *ns); 734 void nvmet_bdev_ns_disable(struct nvmet_ns *ns); 735 void nvmet_file_ns_disable(struct nvmet_ns *ns); 736 u16 nvmet_bdev_flush(struct nvmet_req *req); 737 u16 nvmet_file_flush(struct nvmet_req *req); 738 void nvmet_ns_changed(struct nvmet_subsys *subsys, u32 nsid); 739 void nvmet_bdev_ns_revalidate(struct nvmet_ns *ns); 740 void nvmet_file_ns_revalidate(struct nvmet_ns *ns); 741 bool nvmet_ns_revalidate(struct nvmet_ns *ns); 742 u16 blk_to_nvme_status(struct nvmet_req *req, blk_status_t blk_sts); 743 744 bool nvmet_bdev_zns_enable(struct nvmet_ns *ns); 745 void nvmet_execute_identify_ctrl_zns(struct nvmet_req *req); 746 void nvmet_execute_identify_ns_zns(struct nvmet_req *req); 747 void nvmet_bdev_execute_zone_mgmt_recv(struct nvmet_req *req); 748 void nvmet_bdev_execute_zone_mgmt_send(struct nvmet_req *req); 749 void nvmet_bdev_execute_zone_append(struct nvmet_req *req); 750 751 static inline u32 nvmet_rw_data_len(struct nvmet_req *req) 752 { 753 return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) << 754 req->ns->blksize_shift; 755 } 756 757 static inline u32 nvmet_rw_metadata_len(struct nvmet_req *req) 758 { 759 if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) 760 return 0; 761 return ((u32)le16_to_cpu(req->cmd->rw.length) + 1) * 762 req->ns->metadata_size; 763 } 764 765 static inline u32 nvmet_dsm_len(struct nvmet_req *req) 766 { 767 return (le32_to_cpu(req->cmd->dsm.nr) + 1) * 768 sizeof(struct nvme_dsm_range); 769 } 770 771 static inline struct nvmet_subsys *nvmet_req_subsys(struct nvmet_req *req) 772 { 773 return req->sq->ctrl->subsys; 774 } 775 776 static inline struct nvmet_ctrl *nvmet_req_ctrl(struct nvmet_req *req) 777 { 778 return req->sq->ctrl; 779 } 780 781 static inline bool nvmet_is_disc_subsys(struct nvmet_subsys *subsys) 782 { 783 return subsys->type != NVME_NQN_NVME; 784 } 785 786 static inline bool nvmet_is_pci_ctrl(struct nvmet_ctrl *ctrl) 787 { 788 return ctrl->port->disc_addr.trtype == NVMF_TRTYPE_PCI; 789 } 790 791 /* Limit MDTS according to port config or transport capability */ 792 static inline u8 nvmet_ctrl_mdts(struct nvmet_req *req) 793 { 794 struct nvmet_ctrl *ctrl = req->sq->ctrl; 795 u8 mdts = req->port->mdts; 796 797 if (!ctrl->ops->get_mdts) 798 return mdts; 799 return min_not_zero(ctrl->ops->get_mdts(ctrl), mdts); 800 } 801 802 #ifdef CONFIG_NVME_TARGET_PASSTHRU 803 void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys); 804 int nvmet_passthru_ctrl_enable(struct nvmet_subsys *subsys); 805 void nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys); 806 u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req); 807 u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req); 808 static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys) 809 { 810 return subsys->passthru && 811 test_bit(NVMET_PASSTHRU_ENABLED, &subsys->passthru->flags); 812 } 813 814 static inline struct nvmet_passthru *nvmet_subsys_passthru( 815 struct nvmet_subsys *subsys) 816 { 817 lockdep_assert_held(&subsys->lock); 818 819 if (!subsys->passthru) { 820 subsys->passthru = kzalloc_obj(*subsys->passthru); 821 if (!subsys->passthru) 822 return NULL; 823 } 824 return subsys->passthru; 825 } 826 827 static inline bool nvmet_get_passthru_ref(struct nvmet_req *req) 828 { 829 if (!percpu_ref_tryget_live(&nvmet_req_subsys(req)->passthru->ref)) 830 return false; 831 832 req->p.ref_held = true; 833 return true; 834 } 835 836 static inline void nvmet_put_passthru_ref(struct nvmet_req *req) 837 { 838 if (!req->p.ref_held) 839 return; 840 841 req->p.ref_held = false; 842 percpu_ref_put(&nvmet_req_subsys(req)->passthru->ref); 843 } 844 #else /* CONFIG_NVME_TARGET_PASSTHRU */ 845 static inline void nvmet_passthru_subsys_free(struct nvmet_subsys *subsys) 846 { 847 } 848 static inline void nvmet_passthru_ctrl_disable(struct nvmet_subsys *subsys) 849 { 850 } 851 static inline u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req) 852 { 853 return 0; 854 } 855 static inline u16 nvmet_parse_passthru_io_cmd(struct nvmet_req *req) 856 { 857 return 0; 858 } 859 static inline bool nvmet_is_passthru_subsys(struct nvmet_subsys *subsys) 860 { 861 return NULL; 862 } 863 static inline struct nvmet_passthru *nvmet_subsys_passthru( 864 struct nvmet_subsys *subsys) 865 { 866 return NULL; 867 } 868 static inline bool nvmet_get_passthru_ref(struct nvmet_req *req) 869 { 870 return false; 871 } 872 static inline void nvmet_put_passthru_ref(struct nvmet_req *req) 873 { 874 } 875 #endif /* CONFIG_NVME_TARGET_PASSTHRU */ 876 877 static inline bool nvmet_is_passthru_req(struct nvmet_req *req) 878 { 879 return nvmet_is_passthru_subsys(nvmet_req_subsys(req)); 880 } 881 882 void nvmet_passthrough_override_cap(struct nvmet_ctrl *ctrl); 883 884 u16 errno_to_nvme_status(struct nvmet_req *req, int errno); 885 u16 nvmet_report_invalid_opcode(struct nvmet_req *req); 886 887 static inline bool nvmet_cc_en(u32 cc) 888 { 889 return (cc & NVME_CC_ENABLE) >> NVME_CC_EN_SHIFT; 890 } 891 892 static inline u8 nvmet_cc_css(u32 cc) 893 { 894 return (cc & NVME_CC_CSS_MASK) >> NVME_CC_CSS_SHIFT; 895 } 896 897 static inline u8 nvmet_cc_mps(u32 cc) 898 { 899 return (cc & NVME_CC_MPS_MASK) >> NVME_CC_MPS_SHIFT; 900 } 901 902 static inline u8 nvmet_cc_ams(u32 cc) 903 { 904 return (cc & NVME_CC_AMS_MASK) >> NVME_CC_AMS_SHIFT; 905 } 906 907 static inline u8 nvmet_cc_shn(u32 cc) 908 { 909 return (cc & NVME_CC_SHN_MASK) >> NVME_CC_SHN_SHIFT; 910 } 911 912 static inline u8 nvmet_cc_iosqes(u32 cc) 913 { 914 return (cc & NVME_CC_IOSQES_MASK) >> NVME_CC_IOSQES_SHIFT; 915 } 916 917 static inline u8 nvmet_cc_iocqes(u32 cc) 918 { 919 return (cc & NVME_CC_IOCQES_MASK) >> NVME_CC_IOCQES_SHIFT; 920 } 921 922 /* Convert a 32-bit number to a 16-bit 0's based number */ 923 static inline __le16 to0based(u32 a) 924 { 925 return cpu_to_le16(clamp(a, 1U, 1U << 16) - 1); 926 } 927 928 static inline bool nvmet_ns_has_pi(struct nvmet_ns *ns) 929 { 930 if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY)) 931 return false; 932 return ns->pi_type && ns->metadata_size == sizeof(struct t10_pi_tuple); 933 } 934 935 static inline __le64 nvmet_sect_to_lba(struct nvmet_ns *ns, sector_t sect) 936 { 937 return cpu_to_le64(sect >> (ns->blksize_shift - SECTOR_SHIFT)); 938 } 939 940 static inline sector_t nvmet_lba_to_sect(struct nvmet_ns *ns, __le64 lba) 941 { 942 return le64_to_cpu(lba) << (ns->blksize_shift - SECTOR_SHIFT); 943 } 944 945 static inline bool nvmet_use_inline_bvec(struct nvmet_req *req) 946 { 947 return req->transfer_len <= NVMET_MAX_INLINE_DATA_LEN && 948 req->sg_cnt <= NVMET_MAX_INLINE_BIOVEC; 949 } 950 951 static inline void nvmet_req_bio_put(struct nvmet_req *req, struct bio *bio) 952 { 953 if (bio != &req->b.inline_bio) 954 bio_put(bio); 955 else 956 bio_uninit(bio); 957 } 958 959 #ifdef CONFIG_NVME_TARGET_TCP_TLS 960 static inline key_serial_t nvmet_queue_tls_keyid(struct nvmet_sq *sq) 961 { 962 return sq->tls_key ? key_serial(sq->tls_key) : 0; 963 } 964 static inline void nvmet_sq_put_tls_key(struct nvmet_sq *sq) 965 { 966 if (sq->tls_key) { 967 key_put(sq->tls_key); 968 sq->tls_key = NULL; 969 } 970 } 971 #else 972 static inline key_serial_t nvmet_queue_tls_keyid(struct nvmet_sq *sq) { return 0; } 973 static inline void nvmet_sq_put_tls_key(struct nvmet_sq *sq) {} 974 #endif 975 #ifdef CONFIG_NVME_TARGET_AUTH 976 u32 nvmet_auth_send_data_len(struct nvmet_req *req); 977 void nvmet_execute_auth_send(struct nvmet_req *req); 978 u32 nvmet_auth_receive_data_len(struct nvmet_req *req); 979 void nvmet_execute_auth_receive(struct nvmet_req *req); 980 int nvmet_auth_set_key(struct nvmet_host *host, const char *secret, 981 bool set_ctrl); 982 int nvmet_auth_set_host_hash(struct nvmet_host *host, const char *hash); 983 u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset); 984 void nvmet_auth_sq_init(struct nvmet_sq *sq); 985 void nvmet_destroy_auth(struct nvmet_ctrl *ctrl); 986 void nvmet_auth_sq_free(struct nvmet_sq *sq); 987 void nvmet_auth_sq_destroy(struct nvmet_sq *sq); 988 int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id); 989 bool nvmet_check_auth_status(struct nvmet_req *req); 990 int nvmet_auth_host_hash(struct nvmet_req *req, u8 *response, 991 unsigned int hash_len); 992 int nvmet_auth_ctrl_hash(struct nvmet_req *req, u8 *response, 993 unsigned int hash_len); 994 static inline bool nvmet_has_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq) 995 { 996 return ctrl->host_key != NULL && !nvmet_queue_tls_keyid(sq); 997 } 998 int nvmet_auth_ctrl_exponential(struct nvmet_req *req, 999 u8 *buf, int buf_size); 1000 int nvmet_auth_ctrl_sesskey(struct nvmet_req *req, 1001 const u8 *pkey, int pkey_size); 1002 void nvmet_auth_insert_psk(struct nvmet_sq *sq); 1003 #else 1004 static inline u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, 1005 struct nvmet_sq *sq, bool reset) 1006 { 1007 return 0; 1008 } 1009 static inline void nvmet_auth_sq_init(struct nvmet_sq *sq) 1010 { 1011 } 1012 static inline void nvmet_destroy_auth(struct nvmet_ctrl *ctrl) {}; 1013 static inline void nvmet_auth_sq_free(struct nvmet_sq *sq) {}; 1014 static inline void nvmet_auth_sq_destroy(struct nvmet_sq *sq) {}; 1015 static inline bool nvmet_check_auth_status(struct nvmet_req *req) 1016 { 1017 return true; 1018 } 1019 static inline bool nvmet_has_auth(struct nvmet_ctrl *ctrl, 1020 struct nvmet_sq *sq) 1021 { 1022 return false; 1023 } 1024 static inline const char *nvmet_dhchap_dhgroup_name(u8 dhgid) { return NULL; } 1025 static inline void nvmet_auth_insert_psk(struct nvmet_sq *sq) {}; 1026 #endif 1027 1028 int nvmet_pr_init_ns(struct nvmet_ns *ns); 1029 u16 nvmet_parse_pr_cmd(struct nvmet_req *req); 1030 u16 nvmet_pr_check_cmd_access(struct nvmet_req *req); 1031 int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl); 1032 void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl); 1033 void nvmet_pr_exit_ns(struct nvmet_ns *ns); 1034 void nvmet_execute_get_log_page_resv(struct nvmet_req *req); 1035 u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask); 1036 u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req); 1037 u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req); 1038 static inline void nvmet_pr_put_ns_pc_ref(struct nvmet_pr_per_ctrl_ref *pc_ref) 1039 { 1040 percpu_ref_put(&pc_ref->ref); 1041 } 1042 1043 /* 1044 * Data for the get_feature() and set_feature() operations of PCI target 1045 * controllers. 1046 */ 1047 struct nvmet_feat_irq_coalesce { 1048 u8 thr; 1049 u8 time; 1050 }; 1051 1052 struct nvmet_feat_irq_config { 1053 u16 iv; 1054 bool cd; 1055 }; 1056 1057 struct nvmet_feat_arbitration { 1058 u8 hpw; 1059 u8 mpw; 1060 u8 lpw; 1061 u8 ab; 1062 }; 1063 1064 #endif /* _NVMET_H */ 1065