xref: /linux/drivers/nvme/host/core.c (revision 31bedc1fb1d93250ae1900ee92ccd56689956d22)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVM Express device driver
4  * Copyright (c) 2011-2014, Intel Corporation.
5  */
6 
7 #include <linux/async.h>
8 #include <linux/blkdev.h>
9 #include <linux/blk-mq.h>
10 #include <linux/blk-integrity.h>
11 #include <linux/compat.h>
12 #include <linux/delay.h>
13 #include <linux/errno.h>
14 #include <linux/hdreg.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/backing-dev.h>
18 #include <linux/slab.h>
19 #include <linux/types.h>
20 #include <linux/pr.h>
21 #include <linux/ptrace.h>
22 #include <linux/nvme_ioctl.h>
23 #include <linux/pm_qos.h>
24 #include <linux/ratelimit.h>
25 #include <linux/unaligned.h>
26 
27 #include "nvme.h"
28 #include "fabrics.h"
29 #include <linux/nvme-auth.h>
30 
31 #define CREATE_TRACE_POINTS
32 #include "trace.h"
33 
34 #define NVME_MINORS		(1U << MINORBITS)
35 
36 struct nvme_ns_info {
37 	struct nvme_ns_ids ids;
38 	u32 nsid;
39 	__le32 anagrpid;
40 	u8 pi_offset;
41 	bool is_shared;
42 	bool is_readonly;
43 	bool is_ready;
44 	bool is_removed;
45 };
46 
47 unsigned int admin_timeout = 60;
48 module_param(admin_timeout, uint, 0644);
49 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
50 EXPORT_SYMBOL_GPL(admin_timeout);
51 
52 unsigned int nvme_io_timeout = 30;
53 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
54 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
55 EXPORT_SYMBOL_GPL(nvme_io_timeout);
56 
57 static unsigned char shutdown_timeout = 5;
58 module_param(shutdown_timeout, byte, 0644);
59 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
60 
61 static u8 nvme_max_retries = 5;
62 module_param_named(max_retries, nvme_max_retries, byte, 0644);
63 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
64 
65 static unsigned long default_ps_max_latency_us = 100000;
66 module_param(default_ps_max_latency_us, ulong, 0644);
67 MODULE_PARM_DESC(default_ps_max_latency_us,
68 		 "max power saving latency for new devices; use PM QOS to change per device");
69 
70 static bool force_apst;
71 module_param(force_apst, bool, 0644);
72 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
73 
74 static unsigned long apst_primary_timeout_ms = 100;
75 module_param(apst_primary_timeout_ms, ulong, 0644);
76 MODULE_PARM_DESC(apst_primary_timeout_ms,
77 	"primary APST timeout in ms");
78 
79 static unsigned long apst_secondary_timeout_ms = 2000;
80 module_param(apst_secondary_timeout_ms, ulong, 0644);
81 MODULE_PARM_DESC(apst_secondary_timeout_ms,
82 	"secondary APST timeout in ms");
83 
84 static unsigned long apst_primary_latency_tol_us = 15000;
85 module_param(apst_primary_latency_tol_us, ulong, 0644);
86 MODULE_PARM_DESC(apst_primary_latency_tol_us,
87 	"primary APST latency tolerance in us");
88 
89 static unsigned long apst_secondary_latency_tol_us = 100000;
90 module_param(apst_secondary_latency_tol_us, ulong, 0644);
91 MODULE_PARM_DESC(apst_secondary_latency_tol_us,
92 	"secondary APST latency tolerance in us");
93 
94 /*
95  * Older kernels didn't enable protection information if it was at an offset.
96  * Newer kernels do, so it breaks reads on the upgrade if such formats were
97  * used in prior kernels since the metadata written did not contain a valid
98  * checksum.
99  */
100 static bool disable_pi_offsets = false;
101 module_param(disable_pi_offsets, bool, 0444);
102 MODULE_PARM_DESC(disable_pi_offsets,
103 	"disable protection information if it has an offset");
104 
105 /*
106  * nvme_wq - hosts nvme related works that are not reset or delete
107  * nvme_reset_wq - hosts nvme reset works
108  * nvme_delete_wq - hosts nvme delete works
109  *
110  * nvme_wq will host works such as scan, aen handling, fw activation,
111  * keep-alive, periodic reconnects etc. nvme_reset_wq
112  * runs reset works which also flush works hosted on nvme_wq for
113  * serialization purposes. nvme_delete_wq host controller deletion
114  * works which flush reset works for serialization.
115  */
116 struct workqueue_struct *nvme_wq;
117 EXPORT_SYMBOL_GPL(nvme_wq);
118 
119 struct workqueue_struct *nvme_reset_wq;
120 EXPORT_SYMBOL_GPL(nvme_reset_wq);
121 
122 struct workqueue_struct *nvme_delete_wq;
123 EXPORT_SYMBOL_GPL(nvme_delete_wq);
124 
125 static LIST_HEAD(nvme_subsystems);
126 DEFINE_MUTEX(nvme_subsystems_lock);
127 
128 static DEFINE_IDA(nvme_instance_ida);
129 static dev_t nvme_ctrl_base_chr_devt;
130 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env);
131 static const struct class nvme_class = {
132 	.name = "nvme",
133 	.dev_uevent = nvme_class_uevent,
134 };
135 
136 static const struct class nvme_subsys_class = {
137 	.name = "nvme-subsystem",
138 };
139 
140 static DEFINE_IDA(nvme_ns_chr_minor_ida);
141 static dev_t nvme_ns_chr_devt;
142 static const struct class nvme_ns_chr_class = {
143 	.name = "nvme-generic",
144 };
145 
146 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
147 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
148 					   unsigned nsid);
149 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
150 				   struct nvme_command *cmd);
151 
152 void nvme_queue_scan(struct nvme_ctrl *ctrl)
153 {
154 	/*
155 	 * Only new queue scan work when admin and IO queues are both alive
156 	 */
157 	if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE && ctrl->tagset)
158 		queue_work(nvme_wq, &ctrl->scan_work);
159 }
160 
161 /*
162  * Use this function to proceed with scheduling reset_work for a controller
163  * that had previously been set to the resetting state. This is intended for
164  * code paths that can't be interrupted by other reset attempts. A hot removal
165  * may prevent this from succeeding.
166  */
167 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
168 {
169 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_RESETTING)
170 		return -EBUSY;
171 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
172 		return -EBUSY;
173 	return 0;
174 }
175 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
176 
177 static void nvme_failfast_work(struct work_struct *work)
178 {
179 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
180 			struct nvme_ctrl, failfast_work);
181 
182 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_CONNECTING)
183 		return;
184 
185 	set_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
186 	dev_info(ctrl->device, "failfast expired\n");
187 	nvme_kick_requeue_lists(ctrl);
188 }
189 
190 static inline void nvme_start_failfast_work(struct nvme_ctrl *ctrl)
191 {
192 	if (!ctrl->opts || ctrl->opts->fast_io_fail_tmo == -1)
193 		return;
194 
195 	schedule_delayed_work(&ctrl->failfast_work,
196 			      ctrl->opts->fast_io_fail_tmo * HZ);
197 }
198 
199 static inline void nvme_stop_failfast_work(struct nvme_ctrl *ctrl)
200 {
201 	if (!ctrl->opts)
202 		return;
203 
204 	cancel_delayed_work_sync(&ctrl->failfast_work);
205 	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
206 }
207 
208 
209 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
210 {
211 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
212 		return -EBUSY;
213 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
214 		return -EBUSY;
215 	return 0;
216 }
217 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
218 
219 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
220 {
221 	int ret;
222 
223 	ret = nvme_reset_ctrl(ctrl);
224 	if (!ret) {
225 		flush_work(&ctrl->reset_work);
226 		if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE)
227 			ret = -ENETRESET;
228 	}
229 
230 	return ret;
231 }
232 
233 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
234 {
235 	dev_info(ctrl->device,
236 		 "Removing ctrl: NQN \"%s\"\n", nvmf_ctrl_subsysnqn(ctrl));
237 
238 	flush_work(&ctrl->reset_work);
239 	nvme_stop_ctrl(ctrl);
240 	nvme_remove_namespaces(ctrl);
241 	ctrl->ops->delete_ctrl(ctrl);
242 	nvme_uninit_ctrl(ctrl);
243 }
244 
245 static void nvme_delete_ctrl_work(struct work_struct *work)
246 {
247 	struct nvme_ctrl *ctrl =
248 		container_of(work, struct nvme_ctrl, delete_work);
249 
250 	nvme_do_delete_ctrl(ctrl);
251 }
252 
253 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
254 {
255 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
256 		return -EBUSY;
257 	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
258 		return -EBUSY;
259 	return 0;
260 }
261 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
262 
263 void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
264 {
265 	/*
266 	 * Keep a reference until nvme_do_delete_ctrl() complete,
267 	 * since ->delete_ctrl can free the controller.
268 	 */
269 	nvme_get_ctrl(ctrl);
270 	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
271 		nvme_do_delete_ctrl(ctrl);
272 	nvme_put_ctrl(ctrl);
273 }
274 
275 static blk_status_t nvme_error_status(u16 status)
276 {
277 	switch (status & NVME_SCT_SC_MASK) {
278 	case NVME_SC_SUCCESS:
279 		return BLK_STS_OK;
280 	case NVME_SC_CAP_EXCEEDED:
281 		return BLK_STS_NOSPC;
282 	case NVME_SC_LBA_RANGE:
283 	case NVME_SC_CMD_INTERRUPTED:
284 	case NVME_SC_NS_NOT_READY:
285 		return BLK_STS_TARGET;
286 	case NVME_SC_BAD_ATTRIBUTES:
287 	case NVME_SC_ONCS_NOT_SUPPORTED:
288 	case NVME_SC_INVALID_OPCODE:
289 	case NVME_SC_INVALID_FIELD:
290 	case NVME_SC_INVALID_NS:
291 		return BLK_STS_NOTSUPP;
292 	case NVME_SC_WRITE_FAULT:
293 	case NVME_SC_READ_ERROR:
294 	case NVME_SC_UNWRITTEN_BLOCK:
295 	case NVME_SC_ACCESS_DENIED:
296 	case NVME_SC_READ_ONLY:
297 	case NVME_SC_COMPARE_FAILED:
298 		return BLK_STS_MEDIUM;
299 	case NVME_SC_GUARD_CHECK:
300 	case NVME_SC_APPTAG_CHECK:
301 	case NVME_SC_REFTAG_CHECK:
302 	case NVME_SC_INVALID_PI:
303 		return BLK_STS_PROTECTION;
304 	case NVME_SC_RESERVATION_CONFLICT:
305 		return BLK_STS_RESV_CONFLICT;
306 	case NVME_SC_HOST_PATH_ERROR:
307 		return BLK_STS_TRANSPORT;
308 	case NVME_SC_ZONE_TOO_MANY_ACTIVE:
309 		return BLK_STS_ZONE_ACTIVE_RESOURCE;
310 	case NVME_SC_ZONE_TOO_MANY_OPEN:
311 		return BLK_STS_ZONE_OPEN_RESOURCE;
312 	default:
313 		return BLK_STS_IOERR;
314 	}
315 }
316 
317 static void nvme_retry_req(struct request *req)
318 {
319 	unsigned long delay = 0;
320 	u16 crd;
321 
322 	/* The mask and shift result must be <= 3 */
323 	crd = (nvme_req(req)->status & NVME_STATUS_CRD) >> 11;
324 	if (crd)
325 		delay = nvme_req(req)->ctrl->crdt[crd - 1] * 100;
326 
327 	nvme_req(req)->retries++;
328 	blk_mq_requeue_request(req, false);
329 	blk_mq_delay_kick_requeue_list(req->q, delay);
330 }
331 
332 static void nvme_log_error(struct request *req)
333 {
334 	struct nvme_ns *ns = req->q->queuedata;
335 	struct nvme_request *nr = nvme_req(req);
336 
337 	if (ns) {
338 		pr_err_ratelimited("%s: %s(0x%x) @ LBA %llu, %u blocks, %s (sct 0x%x / sc 0x%x) %s%s\n",
339 		       ns->disk ? ns->disk->disk_name : "?",
340 		       nvme_get_opcode_str(nr->cmd->common.opcode),
341 		       nr->cmd->common.opcode,
342 		       nvme_sect_to_lba(ns->head, blk_rq_pos(req)),
343 		       blk_rq_bytes(req) >> ns->head->lba_shift,
344 		       nvme_get_error_status_str(nr->status),
345 		       NVME_SCT(nr->status),		/* Status Code Type */
346 		       nr->status & NVME_SC_MASK,	/* Status Code */
347 		       nr->status & NVME_STATUS_MORE ? "MORE " : "",
348 		       nr->status & NVME_STATUS_DNR  ? "DNR "  : "");
349 		return;
350 	}
351 
352 	pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s\n",
353 			   dev_name(nr->ctrl->device),
354 			   nvme_get_admin_opcode_str(nr->cmd->common.opcode),
355 			   nr->cmd->common.opcode,
356 			   nvme_get_error_status_str(nr->status),
357 			   NVME_SCT(nr->status),	/* Status Code Type */
358 			   nr->status & NVME_SC_MASK,	/* Status Code */
359 			   nr->status & NVME_STATUS_MORE ? "MORE " : "",
360 			   nr->status & NVME_STATUS_DNR  ? "DNR "  : "");
361 }
362 
363 static void nvme_log_err_passthru(struct request *req)
364 {
365 	struct nvme_ns *ns = req->q->queuedata;
366 	struct nvme_request *nr = nvme_req(req);
367 
368 	pr_err_ratelimited("%s: %s(0x%x), %s (sct 0x%x / sc 0x%x) %s%s"
369 		"cdw10=0x%x cdw11=0x%x cdw12=0x%x cdw13=0x%x cdw14=0x%x cdw15=0x%x\n",
370 		ns ? ns->disk->disk_name : dev_name(nr->ctrl->device),
371 		ns ? nvme_get_opcode_str(nr->cmd->common.opcode) :
372 		     nvme_get_admin_opcode_str(nr->cmd->common.opcode),
373 		nr->cmd->common.opcode,
374 		nvme_get_error_status_str(nr->status),
375 		NVME_SCT(nr->status),		/* Status Code Type */
376 		nr->status & NVME_SC_MASK,	/* Status Code */
377 		nr->status & NVME_STATUS_MORE ? "MORE " : "",
378 		nr->status & NVME_STATUS_DNR  ? "DNR "  : "",
379 		nr->cmd->common.cdw10,
380 		nr->cmd->common.cdw11,
381 		nr->cmd->common.cdw12,
382 		nr->cmd->common.cdw13,
383 		nr->cmd->common.cdw14,
384 		nr->cmd->common.cdw14);
385 }
386 
387 enum nvme_disposition {
388 	COMPLETE,
389 	RETRY,
390 	FAILOVER,
391 	AUTHENTICATE,
392 };
393 
394 static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
395 {
396 	if (likely(nvme_req(req)->status == 0))
397 		return COMPLETE;
398 
399 	if (blk_noretry_request(req) ||
400 	    (nvme_req(req)->status & NVME_STATUS_DNR) ||
401 	    nvme_req(req)->retries >= nvme_max_retries)
402 		return COMPLETE;
403 
404 	if ((nvme_req(req)->status & NVME_SCT_SC_MASK) == NVME_SC_AUTH_REQUIRED)
405 		return AUTHENTICATE;
406 
407 	if (req->cmd_flags & REQ_NVME_MPATH) {
408 		if (nvme_is_path_error(nvme_req(req)->status) ||
409 		    blk_queue_dying(req->q))
410 			return FAILOVER;
411 	} else {
412 		if (blk_queue_dying(req->q))
413 			return COMPLETE;
414 	}
415 
416 	return RETRY;
417 }
418 
419 static inline void nvme_end_req_zoned(struct request *req)
420 {
421 	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
422 	    req_op(req) == REQ_OP_ZONE_APPEND) {
423 		struct nvme_ns *ns = req->q->queuedata;
424 
425 		req->__sector = nvme_lba_to_sect(ns->head,
426 			le64_to_cpu(nvme_req(req)->result.u64));
427 	}
428 }
429 
430 static inline void __nvme_end_req(struct request *req)
431 {
432 	nvme_end_req_zoned(req);
433 	nvme_trace_bio_complete(req);
434 	if (req->cmd_flags & REQ_NVME_MPATH)
435 		nvme_mpath_end_request(req);
436 }
437 
438 void nvme_end_req(struct request *req)
439 {
440 	blk_status_t status = nvme_error_status(nvme_req(req)->status);
441 
442 	if (unlikely(nvme_req(req)->status && !(req->rq_flags & RQF_QUIET))) {
443 		if (blk_rq_is_passthrough(req))
444 			nvme_log_err_passthru(req);
445 		else
446 			nvme_log_error(req);
447 	}
448 	__nvme_end_req(req);
449 	blk_mq_end_request(req, status);
450 }
451 
452 void nvme_complete_rq(struct request *req)
453 {
454 	struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
455 
456 	trace_nvme_complete_rq(req);
457 	nvme_cleanup_cmd(req);
458 
459 	/*
460 	 * Completions of long-running commands should not be able to
461 	 * defer sending of periodic keep alives, since the controller
462 	 * may have completed processing such commands a long time ago
463 	 * (arbitrarily close to command submission time).
464 	 * req->deadline - req->timeout is the command submission time
465 	 * in jiffies.
466 	 */
467 	if (ctrl->kas &&
468 	    req->deadline - req->timeout >= ctrl->ka_last_check_time)
469 		ctrl->comp_seen = true;
470 
471 	switch (nvme_decide_disposition(req)) {
472 	case COMPLETE:
473 		nvme_end_req(req);
474 		return;
475 	case RETRY:
476 		nvme_retry_req(req);
477 		return;
478 	case FAILOVER:
479 		nvme_failover_req(req);
480 		return;
481 	case AUTHENTICATE:
482 #ifdef CONFIG_NVME_HOST_AUTH
483 		queue_work(nvme_wq, &ctrl->dhchap_auth_work);
484 		nvme_retry_req(req);
485 #else
486 		nvme_end_req(req);
487 #endif
488 		return;
489 	}
490 }
491 EXPORT_SYMBOL_GPL(nvme_complete_rq);
492 
493 void nvme_complete_batch_req(struct request *req)
494 {
495 	trace_nvme_complete_rq(req);
496 	nvme_cleanup_cmd(req);
497 	__nvme_end_req(req);
498 }
499 EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
500 
501 /*
502  * Called to unwind from ->queue_rq on a failed command submission so that the
503  * multipathing code gets called to potentially failover to another path.
504  * The caller needs to unwind all transport specific resource allocations and
505  * must return propagate the return value.
506  */
507 blk_status_t nvme_host_path_error(struct request *req)
508 {
509 	nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
510 	blk_mq_set_request_complete(req);
511 	nvme_complete_rq(req);
512 	return BLK_STS_OK;
513 }
514 EXPORT_SYMBOL_GPL(nvme_host_path_error);
515 
516 bool nvme_cancel_request(struct request *req, void *data)
517 {
518 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
519 				"Cancelling I/O %d", req->tag);
520 
521 	/* don't abort one completed or idle request */
522 	if (blk_mq_rq_state(req) != MQ_RQ_IN_FLIGHT)
523 		return true;
524 
525 	nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
526 	nvme_req(req)->flags |= NVME_REQ_CANCELLED;
527 	blk_mq_complete_request(req);
528 	return true;
529 }
530 EXPORT_SYMBOL_GPL(nvme_cancel_request);
531 
532 void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
533 {
534 	if (ctrl->tagset) {
535 		blk_mq_tagset_busy_iter(ctrl->tagset,
536 				nvme_cancel_request, ctrl);
537 		blk_mq_tagset_wait_completed_request(ctrl->tagset);
538 	}
539 }
540 EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
541 
542 void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
543 {
544 	if (ctrl->admin_tagset) {
545 		blk_mq_tagset_busy_iter(ctrl->admin_tagset,
546 				nvme_cancel_request, ctrl);
547 		blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
548 	}
549 }
550 EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
551 
552 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
553 		enum nvme_ctrl_state new_state)
554 {
555 	enum nvme_ctrl_state old_state;
556 	unsigned long flags;
557 	bool changed = false;
558 
559 	spin_lock_irqsave(&ctrl->lock, flags);
560 
561 	old_state = nvme_ctrl_state(ctrl);
562 	switch (new_state) {
563 	case NVME_CTRL_LIVE:
564 		switch (old_state) {
565 		case NVME_CTRL_NEW:
566 		case NVME_CTRL_RESETTING:
567 		case NVME_CTRL_CONNECTING:
568 			changed = true;
569 			fallthrough;
570 		default:
571 			break;
572 		}
573 		break;
574 	case NVME_CTRL_RESETTING:
575 		switch (old_state) {
576 		case NVME_CTRL_NEW:
577 		case NVME_CTRL_LIVE:
578 			changed = true;
579 			fallthrough;
580 		default:
581 			break;
582 		}
583 		break;
584 	case NVME_CTRL_CONNECTING:
585 		switch (old_state) {
586 		case NVME_CTRL_NEW:
587 		case NVME_CTRL_RESETTING:
588 			changed = true;
589 			fallthrough;
590 		default:
591 			break;
592 		}
593 		break;
594 	case NVME_CTRL_DELETING:
595 		switch (old_state) {
596 		case NVME_CTRL_LIVE:
597 		case NVME_CTRL_RESETTING:
598 		case NVME_CTRL_CONNECTING:
599 			changed = true;
600 			fallthrough;
601 		default:
602 			break;
603 		}
604 		break;
605 	case NVME_CTRL_DELETING_NOIO:
606 		switch (old_state) {
607 		case NVME_CTRL_DELETING:
608 		case NVME_CTRL_DEAD:
609 			changed = true;
610 			fallthrough;
611 		default:
612 			break;
613 		}
614 		break;
615 	case NVME_CTRL_DEAD:
616 		switch (old_state) {
617 		case NVME_CTRL_DELETING:
618 			changed = true;
619 			fallthrough;
620 		default:
621 			break;
622 		}
623 		break;
624 	default:
625 		break;
626 	}
627 
628 	if (changed) {
629 		WRITE_ONCE(ctrl->state, new_state);
630 		wake_up_all(&ctrl->state_wq);
631 	}
632 
633 	spin_unlock_irqrestore(&ctrl->lock, flags);
634 	if (!changed)
635 		return false;
636 
637 	if (new_state == NVME_CTRL_LIVE) {
638 		if (old_state == NVME_CTRL_CONNECTING)
639 			nvme_stop_failfast_work(ctrl);
640 		nvme_kick_requeue_lists(ctrl);
641 	} else if (new_state == NVME_CTRL_CONNECTING &&
642 		old_state == NVME_CTRL_RESETTING) {
643 		nvme_start_failfast_work(ctrl);
644 	}
645 	return changed;
646 }
647 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
648 
649 /*
650  * Waits for the controller state to be resetting, or returns false if it is
651  * not possible to ever transition to that state.
652  */
653 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
654 {
655 	wait_event(ctrl->state_wq,
656 		   nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
657 		   nvme_state_terminal(ctrl));
658 	return nvme_ctrl_state(ctrl) == NVME_CTRL_RESETTING;
659 }
660 EXPORT_SYMBOL_GPL(nvme_wait_reset);
661 
662 static void nvme_free_ns_head(struct kref *ref)
663 {
664 	struct nvme_ns_head *head =
665 		container_of(ref, struct nvme_ns_head, ref);
666 
667 	nvme_mpath_remove_disk(head);
668 	ida_free(&head->subsys->ns_ida, head->instance);
669 	cleanup_srcu_struct(&head->srcu);
670 	nvme_put_subsystem(head->subsys);
671 	kfree(head);
672 }
673 
674 bool nvme_tryget_ns_head(struct nvme_ns_head *head)
675 {
676 	return kref_get_unless_zero(&head->ref);
677 }
678 
679 void nvme_put_ns_head(struct nvme_ns_head *head)
680 {
681 	kref_put(&head->ref, nvme_free_ns_head);
682 }
683 
684 static void nvme_free_ns(struct kref *kref)
685 {
686 	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
687 
688 	put_disk(ns->disk);
689 	nvme_put_ns_head(ns->head);
690 	nvme_put_ctrl(ns->ctrl);
691 	kfree(ns);
692 }
693 
694 bool nvme_get_ns(struct nvme_ns *ns)
695 {
696 	return kref_get_unless_zero(&ns->kref);
697 }
698 
699 void nvme_put_ns(struct nvme_ns *ns)
700 {
701 	kref_put(&ns->kref, nvme_free_ns);
702 }
703 EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
704 
705 static inline void nvme_clear_nvme_request(struct request *req)
706 {
707 	nvme_req(req)->status = 0;
708 	nvme_req(req)->retries = 0;
709 	nvme_req(req)->flags = 0;
710 	req->rq_flags |= RQF_DONTPREP;
711 }
712 
713 /* initialize a passthrough request */
714 void nvme_init_request(struct request *req, struct nvme_command *cmd)
715 {
716 	struct nvme_request *nr = nvme_req(req);
717 	bool logging_enabled;
718 
719 	if (req->q->queuedata) {
720 		struct nvme_ns *ns = req->q->disk->private_data;
721 
722 		logging_enabled = ns->head->passthru_err_log_enabled;
723 		req->timeout = NVME_IO_TIMEOUT;
724 	} else { /* no queuedata implies admin queue */
725 		logging_enabled = nr->ctrl->passthru_err_log_enabled;
726 		req->timeout = NVME_ADMIN_TIMEOUT;
727 	}
728 
729 	if (!logging_enabled)
730 		req->rq_flags |= RQF_QUIET;
731 
732 	/* passthru commands should let the driver set the SGL flags */
733 	cmd->common.flags &= ~NVME_CMD_SGL_ALL;
734 
735 	req->cmd_flags |= REQ_FAILFAST_DRIVER;
736 	if (req->mq_hctx->type == HCTX_TYPE_POLL)
737 		req->cmd_flags |= REQ_POLLED;
738 	nvme_clear_nvme_request(req);
739 	memcpy(nr->cmd, cmd, sizeof(*cmd));
740 }
741 EXPORT_SYMBOL_GPL(nvme_init_request);
742 
743 /*
744  * For something we're not in a state to send to the device the default action
745  * is to busy it and retry it after the controller state is recovered.  However,
746  * if the controller is deleting or if anything is marked for failfast or
747  * nvme multipath it is immediately failed.
748  *
749  * Note: commands used to initialize the controller will be marked for failfast.
750  * Note: nvme cli/ioctl commands are marked for failfast.
751  */
752 blk_status_t nvme_fail_nonready_command(struct nvme_ctrl *ctrl,
753 		struct request *rq)
754 {
755 	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
756 
757 	if (state != NVME_CTRL_DELETING_NOIO &&
758 	    state != NVME_CTRL_DELETING &&
759 	    state != NVME_CTRL_DEAD &&
760 	    !test_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags) &&
761 	    !blk_noretry_request(rq) && !(rq->cmd_flags & REQ_NVME_MPATH))
762 		return BLK_STS_RESOURCE;
763 	return nvme_host_path_error(rq);
764 }
765 EXPORT_SYMBOL_GPL(nvme_fail_nonready_command);
766 
767 bool __nvme_check_ready(struct nvme_ctrl *ctrl, struct request *rq,
768 		bool queue_live, enum nvme_ctrl_state state)
769 {
770 	struct nvme_request *req = nvme_req(rq);
771 
772 	/*
773 	 * currently we have a problem sending passthru commands
774 	 * on the admin_q if the controller is not LIVE because we can't
775 	 * make sure that they are going out after the admin connect,
776 	 * controller enable and/or other commands in the initialization
777 	 * sequence. until the controller will be LIVE, fail with
778 	 * BLK_STS_RESOURCE so that they will be rescheduled.
779 	 */
780 	if (rq->q == ctrl->admin_q && (req->flags & NVME_REQ_USERCMD))
781 		return false;
782 
783 	if (ctrl->ops->flags & NVME_F_FABRICS) {
784 		/*
785 		 * Only allow commands on a live queue, except for the connect
786 		 * command, which is require to set the queue live in the
787 		 * appropinquate states.
788 		 */
789 		switch (state) {
790 		case NVME_CTRL_CONNECTING:
791 			if (blk_rq_is_passthrough(rq) && nvme_is_fabrics(req->cmd) &&
792 			    (req->cmd->fabrics.fctype == nvme_fabrics_type_connect ||
793 			     req->cmd->fabrics.fctype == nvme_fabrics_type_auth_send ||
794 			     req->cmd->fabrics.fctype == nvme_fabrics_type_auth_receive))
795 				return true;
796 			break;
797 		default:
798 			break;
799 		case NVME_CTRL_DEAD:
800 			return false;
801 		}
802 	}
803 
804 	return queue_live;
805 }
806 EXPORT_SYMBOL_GPL(__nvme_check_ready);
807 
808 static inline void nvme_setup_flush(struct nvme_ns *ns,
809 		struct nvme_command *cmnd)
810 {
811 	memset(cmnd, 0, sizeof(*cmnd));
812 	cmnd->common.opcode = nvme_cmd_flush;
813 	cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
814 }
815 
816 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
817 		struct nvme_command *cmnd)
818 {
819 	unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
820 	struct nvme_dsm_range *range;
821 	struct bio *bio;
822 
823 	/*
824 	 * Some devices do not consider the DSM 'Number of Ranges' field when
825 	 * determining how much data to DMA. Always allocate memory for maximum
826 	 * number of segments to prevent device reading beyond end of buffer.
827 	 */
828 	static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
829 
830 	range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
831 	if (!range) {
832 		/*
833 		 * If we fail allocation our range, fallback to the controller
834 		 * discard page. If that's also busy, it's safe to return
835 		 * busy, as we know we can make progress once that's freed.
836 		 */
837 		if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
838 			return BLK_STS_RESOURCE;
839 
840 		range = page_address(ns->ctrl->discard_page);
841 	}
842 
843 	if (queue_max_discard_segments(req->q) == 1) {
844 		u64 slba = nvme_sect_to_lba(ns->head, blk_rq_pos(req));
845 		u32 nlb = blk_rq_sectors(req) >> (ns->head->lba_shift - 9);
846 
847 		range[0].cattr = cpu_to_le32(0);
848 		range[0].nlb = cpu_to_le32(nlb);
849 		range[0].slba = cpu_to_le64(slba);
850 		n = 1;
851 	} else {
852 		__rq_for_each_bio(bio, req) {
853 			u64 slba = nvme_sect_to_lba(ns->head,
854 						    bio->bi_iter.bi_sector);
855 			u32 nlb = bio->bi_iter.bi_size >> ns->head->lba_shift;
856 
857 			if (n < segments) {
858 				range[n].cattr = cpu_to_le32(0);
859 				range[n].nlb = cpu_to_le32(nlb);
860 				range[n].slba = cpu_to_le64(slba);
861 			}
862 			n++;
863 		}
864 	}
865 
866 	if (WARN_ON_ONCE(n != segments)) {
867 		if (virt_to_page(range) == ns->ctrl->discard_page)
868 			clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
869 		else
870 			kfree(range);
871 		return BLK_STS_IOERR;
872 	}
873 
874 	memset(cmnd, 0, sizeof(*cmnd));
875 	cmnd->dsm.opcode = nvme_cmd_dsm;
876 	cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
877 	cmnd->dsm.nr = cpu_to_le32(segments - 1);
878 	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
879 
880 	bvec_set_virt(&req->special_vec, range, alloc_size);
881 	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
882 
883 	return BLK_STS_OK;
884 }
885 
886 static void nvme_set_ref_tag(struct nvme_ns *ns, struct nvme_command *cmnd,
887 			      struct request *req)
888 {
889 	u32 upper, lower;
890 	u64 ref48;
891 
892 	/* both rw and write zeroes share the same reftag format */
893 	switch (ns->head->guard_type) {
894 	case NVME_NVM_NS_16B_GUARD:
895 		cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
896 		break;
897 	case NVME_NVM_NS_64B_GUARD:
898 		ref48 = ext_pi_ref_tag(req);
899 		lower = lower_32_bits(ref48);
900 		upper = upper_32_bits(ref48);
901 
902 		cmnd->rw.reftag = cpu_to_le32(lower);
903 		cmnd->rw.cdw3 = cpu_to_le32(upper);
904 		break;
905 	default:
906 		break;
907 	}
908 }
909 
910 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
911 		struct request *req, struct nvme_command *cmnd)
912 {
913 	memset(cmnd, 0, sizeof(*cmnd));
914 
915 	if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
916 		return nvme_setup_discard(ns, req, cmnd);
917 
918 	cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
919 	cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
920 	cmnd->write_zeroes.slba =
921 		cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req)));
922 	cmnd->write_zeroes.length =
923 		cpu_to_le16((blk_rq_bytes(req) >> ns->head->lba_shift) - 1);
924 
925 	if (!(req->cmd_flags & REQ_NOUNMAP) &&
926 	    (ns->head->features & NVME_NS_DEAC))
927 		cmnd->write_zeroes.control |= cpu_to_le16(NVME_WZ_DEAC);
928 
929 	if (nvme_ns_has_pi(ns->head)) {
930 		cmnd->write_zeroes.control |= cpu_to_le16(NVME_RW_PRINFO_PRACT);
931 
932 		switch (ns->head->pi_type) {
933 		case NVME_NS_DPS_PI_TYPE1:
934 		case NVME_NS_DPS_PI_TYPE2:
935 			nvme_set_ref_tag(ns, cmnd, req);
936 			break;
937 		}
938 	}
939 
940 	return BLK_STS_OK;
941 }
942 
943 /*
944  * NVMe does not support a dedicated command to issue an atomic write. A write
945  * which does adhere to the device atomic limits will silently be executed
946  * non-atomically. The request issuer should ensure that the write is within
947  * the queue atomic writes limits, but just validate this in case it is not.
948  */
949 static bool nvme_valid_atomic_write(struct request *req)
950 {
951 	struct request_queue *q = req->q;
952 	u32 boundary_bytes = queue_atomic_write_boundary_bytes(q);
953 
954 	if (blk_rq_bytes(req) > queue_atomic_write_unit_max_bytes(q))
955 		return false;
956 
957 	if (boundary_bytes) {
958 		u64 mask = boundary_bytes - 1, imask = ~mask;
959 		u64 start = blk_rq_pos(req) << SECTOR_SHIFT;
960 		u64 end = start + blk_rq_bytes(req) - 1;
961 
962 		/* If greater then must be crossing a boundary */
963 		if (blk_rq_bytes(req) > boundary_bytes)
964 			return false;
965 
966 		if ((start & imask) != (end & imask))
967 			return false;
968 	}
969 
970 	return true;
971 }
972 
973 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
974 		struct request *req, struct nvme_command *cmnd,
975 		enum nvme_opcode op)
976 {
977 	u16 control = 0;
978 	u32 dsmgmt = 0;
979 
980 	if (req->cmd_flags & REQ_FUA)
981 		control |= NVME_RW_FUA;
982 	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
983 		control |= NVME_RW_LR;
984 
985 	if (req->cmd_flags & REQ_RAHEAD)
986 		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
987 
988 	if (req->cmd_flags & REQ_ATOMIC && !nvme_valid_atomic_write(req))
989 		return BLK_STS_INVAL;
990 
991 	cmnd->rw.opcode = op;
992 	cmnd->rw.flags = 0;
993 	cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
994 	cmnd->rw.cdw2 = 0;
995 	cmnd->rw.cdw3 = 0;
996 	cmnd->rw.metadata = 0;
997 	cmnd->rw.slba =
998 		cpu_to_le64(nvme_sect_to_lba(ns->head, blk_rq_pos(req)));
999 	cmnd->rw.length =
1000 		cpu_to_le16((blk_rq_bytes(req) >> ns->head->lba_shift) - 1);
1001 	cmnd->rw.reftag = 0;
1002 	cmnd->rw.lbat = 0;
1003 	cmnd->rw.lbatm = 0;
1004 
1005 	if (ns->head->ms) {
1006 		/*
1007 		 * If formated with metadata, the block layer always provides a
1008 		 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
1009 		 * we enable the PRACT bit for protection information or set the
1010 		 * namespace capacity to zero to prevent any I/O.
1011 		 */
1012 		if (!blk_integrity_rq(req)) {
1013 			if (WARN_ON_ONCE(!nvme_ns_has_pi(ns->head)))
1014 				return BLK_STS_NOTSUPP;
1015 			control |= NVME_RW_PRINFO_PRACT;
1016 		}
1017 
1018 		switch (ns->head->pi_type) {
1019 		case NVME_NS_DPS_PI_TYPE3:
1020 			control |= NVME_RW_PRINFO_PRCHK_GUARD;
1021 			break;
1022 		case NVME_NS_DPS_PI_TYPE1:
1023 		case NVME_NS_DPS_PI_TYPE2:
1024 			control |= NVME_RW_PRINFO_PRCHK_GUARD |
1025 					NVME_RW_PRINFO_PRCHK_REF;
1026 			if (op == nvme_cmd_zone_append)
1027 				control |= NVME_RW_APPEND_PIREMAP;
1028 			nvme_set_ref_tag(ns, cmnd, req);
1029 			break;
1030 		}
1031 	}
1032 
1033 	cmnd->rw.control = cpu_to_le16(control);
1034 	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
1035 	return 0;
1036 }
1037 
1038 void nvme_cleanup_cmd(struct request *req)
1039 {
1040 	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
1041 		struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
1042 
1043 		if (req->special_vec.bv_page == ctrl->discard_page)
1044 			clear_bit_unlock(0, &ctrl->discard_page_busy);
1045 		else
1046 			kfree(bvec_virt(&req->special_vec));
1047 		req->rq_flags &= ~RQF_SPECIAL_PAYLOAD;
1048 	}
1049 }
1050 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
1051 
1052 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req)
1053 {
1054 	struct nvme_command *cmd = nvme_req(req)->cmd;
1055 	blk_status_t ret = BLK_STS_OK;
1056 
1057 	if (!(req->rq_flags & RQF_DONTPREP))
1058 		nvme_clear_nvme_request(req);
1059 
1060 	switch (req_op(req)) {
1061 	case REQ_OP_DRV_IN:
1062 	case REQ_OP_DRV_OUT:
1063 		/* these are setup prior to execution in nvme_init_request() */
1064 		break;
1065 	case REQ_OP_FLUSH:
1066 		nvme_setup_flush(ns, cmd);
1067 		break;
1068 	case REQ_OP_ZONE_RESET_ALL:
1069 	case REQ_OP_ZONE_RESET:
1070 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
1071 		break;
1072 	case REQ_OP_ZONE_OPEN:
1073 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
1074 		break;
1075 	case REQ_OP_ZONE_CLOSE:
1076 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
1077 		break;
1078 	case REQ_OP_ZONE_FINISH:
1079 		ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
1080 		break;
1081 	case REQ_OP_WRITE_ZEROES:
1082 		ret = nvme_setup_write_zeroes(ns, req, cmd);
1083 		break;
1084 	case REQ_OP_DISCARD:
1085 		ret = nvme_setup_discard(ns, req, cmd);
1086 		break;
1087 	case REQ_OP_READ:
1088 		ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
1089 		break;
1090 	case REQ_OP_WRITE:
1091 		ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
1092 		break;
1093 	case REQ_OP_ZONE_APPEND:
1094 		ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
1095 		break;
1096 	default:
1097 		WARN_ON_ONCE(1);
1098 		return BLK_STS_IOERR;
1099 	}
1100 
1101 	cmd->common.command_id = nvme_cid(req);
1102 	trace_nvme_setup_cmd(req, cmd);
1103 	return ret;
1104 }
1105 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
1106 
1107 /*
1108  * Return values:
1109  * 0:  success
1110  * >0: nvme controller's cqe status response
1111  * <0: kernel error in lieu of controller response
1112  */
1113 int nvme_execute_rq(struct request *rq, bool at_head)
1114 {
1115 	blk_status_t status;
1116 
1117 	status = blk_execute_rq(rq, at_head);
1118 	if (nvme_req(rq)->flags & NVME_REQ_CANCELLED)
1119 		return -EINTR;
1120 	if (nvme_req(rq)->status)
1121 		return nvme_req(rq)->status;
1122 	return blk_status_to_errno(status);
1123 }
1124 EXPORT_SYMBOL_NS_GPL(nvme_execute_rq, NVME_TARGET_PASSTHRU);
1125 
1126 /*
1127  * Returns 0 on success.  If the result is negative, it's a Linux error code;
1128  * if the result is positive, it's an NVM Express status code
1129  */
1130 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1131 		union nvme_result *result, void *buffer, unsigned bufflen,
1132 		int qid, nvme_submit_flags_t flags)
1133 {
1134 	struct request *req;
1135 	int ret;
1136 	blk_mq_req_flags_t blk_flags = 0;
1137 
1138 	if (flags & NVME_SUBMIT_NOWAIT)
1139 		blk_flags |= BLK_MQ_REQ_NOWAIT;
1140 	if (flags & NVME_SUBMIT_RESERVED)
1141 		blk_flags |= BLK_MQ_REQ_RESERVED;
1142 	if (qid == NVME_QID_ANY)
1143 		req = blk_mq_alloc_request(q, nvme_req_op(cmd), blk_flags);
1144 	else
1145 		req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), blk_flags,
1146 						qid - 1);
1147 
1148 	if (IS_ERR(req))
1149 		return PTR_ERR(req);
1150 	nvme_init_request(req, cmd);
1151 	if (flags & NVME_SUBMIT_RETRY)
1152 		req->cmd_flags &= ~REQ_FAILFAST_DRIVER;
1153 
1154 	if (buffer && bufflen) {
1155 		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
1156 		if (ret)
1157 			goto out;
1158 	}
1159 
1160 	ret = nvme_execute_rq(req, flags & NVME_SUBMIT_AT_HEAD);
1161 	if (result && ret >= 0)
1162 		*result = nvme_req(req)->result;
1163  out:
1164 	blk_mq_free_request(req);
1165 	return ret;
1166 }
1167 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
1168 
1169 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
1170 		void *buffer, unsigned bufflen)
1171 {
1172 	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen,
1173 			NVME_QID_ANY, 0);
1174 }
1175 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
1176 
1177 u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1178 {
1179 	u32 effects = 0;
1180 
1181 	if (ns) {
1182 		effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
1183 		if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1184 			dev_warn_once(ctrl->device,
1185 				"IO command:%02x has unusual effects:%08x\n",
1186 				opcode, effects);
1187 
1188 		/*
1189 		 * NVME_CMD_EFFECTS_CSE_MASK causes a freeze all I/O queues,
1190 		 * which would deadlock when done on an I/O command.  Note that
1191 		 * We already warn about an unusual effect above.
1192 		 */
1193 		effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1194 	} else {
1195 		effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1196 
1197 		/* Ignore execution restrictions if any relaxation bits are set */
1198 		if (effects & NVME_CMD_EFFECTS_CSER_MASK)
1199 			effects &= ~NVME_CMD_EFFECTS_CSE_MASK;
1200 	}
1201 
1202 	return effects;
1203 }
1204 EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
1205 
1206 u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u8 opcode)
1207 {
1208 	u32 effects = nvme_command_effects(ctrl, ns, opcode);
1209 
1210 	/*
1211 	 * For simplicity, IO to all namespaces is quiesced even if the command
1212 	 * effects say only one namespace is affected.
1213 	 */
1214 	if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1215 		mutex_lock(&ctrl->scan_lock);
1216 		mutex_lock(&ctrl->subsys->lock);
1217 		nvme_mpath_start_freeze(ctrl->subsys);
1218 		nvme_mpath_wait_freeze(ctrl->subsys);
1219 		nvme_start_freeze(ctrl);
1220 		nvme_wait_freeze(ctrl);
1221 	}
1222 	return effects;
1223 }
1224 EXPORT_SYMBOL_NS_GPL(nvme_passthru_start, NVME_TARGET_PASSTHRU);
1225 
1226 void nvme_passthru_end(struct nvme_ctrl *ctrl, struct nvme_ns *ns, u32 effects,
1227 		       struct nvme_command *cmd, int status)
1228 {
1229 	if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
1230 		nvme_unfreeze(ctrl);
1231 		nvme_mpath_unfreeze(ctrl->subsys);
1232 		mutex_unlock(&ctrl->subsys->lock);
1233 		mutex_unlock(&ctrl->scan_lock);
1234 	}
1235 	if (effects & NVME_CMD_EFFECTS_CCC) {
1236 		if (!test_and_set_bit(NVME_CTRL_DIRTY_CAPABILITY,
1237 				      &ctrl->flags)) {
1238 			dev_info(ctrl->device,
1239 "controller capabilities changed, reset may be required to take effect.\n");
1240 		}
1241 	}
1242 	if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC)) {
1243 		nvme_queue_scan(ctrl);
1244 		flush_work(&ctrl->scan_work);
1245 	}
1246 	if (ns)
1247 		return;
1248 
1249 	switch (cmd->common.opcode) {
1250 	case nvme_admin_set_features:
1251 		switch (le32_to_cpu(cmd->common.cdw10) & 0xFF) {
1252 		case NVME_FEAT_KATO:
1253 			/*
1254 			 * Keep alive commands interval on the host should be
1255 			 * updated when KATO is modified by Set Features
1256 			 * commands.
1257 			 */
1258 			if (!status)
1259 				nvme_update_keep_alive(ctrl, cmd);
1260 			break;
1261 		default:
1262 			break;
1263 		}
1264 		break;
1265 	default:
1266 		break;
1267 	}
1268 }
1269 EXPORT_SYMBOL_NS_GPL(nvme_passthru_end, NVME_TARGET_PASSTHRU);
1270 
1271 /*
1272  * Recommended frequency for KATO commands per NVMe 1.4 section 7.12.1:
1273  *
1274  *   The host should send Keep Alive commands at half of the Keep Alive Timeout
1275  *   accounting for transport roundtrip times [..].
1276  */
1277 static unsigned long nvme_keep_alive_work_period(struct nvme_ctrl *ctrl)
1278 {
1279 	unsigned long delay = ctrl->kato * HZ / 2;
1280 
1281 	/*
1282 	 * When using Traffic Based Keep Alive, we need to run
1283 	 * nvme_keep_alive_work at twice the normal frequency, as one
1284 	 * command completion can postpone sending a keep alive command
1285 	 * by up to twice the delay between runs.
1286 	 */
1287 	if (ctrl->ctratt & NVME_CTRL_ATTR_TBKAS)
1288 		delay /= 2;
1289 	return delay;
1290 }
1291 
1292 static void nvme_queue_keep_alive_work(struct nvme_ctrl *ctrl)
1293 {
1294 	unsigned long now = jiffies;
1295 	unsigned long delay = nvme_keep_alive_work_period(ctrl);
1296 	unsigned long ka_next_check_tm = ctrl->ka_last_check_time + delay;
1297 
1298 	if (time_after(now, ka_next_check_tm))
1299 		delay = 0;
1300 	else
1301 		delay = ka_next_check_tm - now;
1302 
1303 	queue_delayed_work(nvme_wq, &ctrl->ka_work, delay);
1304 }
1305 
1306 static void nvme_keep_alive_finish(struct request *rq,
1307 		blk_status_t status, struct nvme_ctrl *ctrl)
1308 {
1309 	unsigned long rtt = jiffies - (rq->deadline - rq->timeout);
1310 	unsigned long delay = nvme_keep_alive_work_period(ctrl);
1311 	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
1312 
1313 	/*
1314 	 * Subtract off the keepalive RTT so nvme_keep_alive_work runs
1315 	 * at the desired frequency.
1316 	 */
1317 	if (rtt <= delay) {
1318 		delay -= rtt;
1319 	} else {
1320 		dev_warn(ctrl->device, "long keepalive RTT (%u ms)\n",
1321 			 jiffies_to_msecs(rtt));
1322 		delay = 0;
1323 	}
1324 
1325 	if (status) {
1326 		dev_err(ctrl->device,
1327 			"failed nvme_keep_alive_end_io error=%d\n",
1328 				status);
1329 		return;
1330 	}
1331 
1332 	ctrl->ka_last_check_time = jiffies;
1333 	ctrl->comp_seen = false;
1334 	if (state == NVME_CTRL_LIVE || state == NVME_CTRL_CONNECTING)
1335 		queue_delayed_work(nvme_wq, &ctrl->ka_work, delay);
1336 }
1337 
1338 static void nvme_keep_alive_work(struct work_struct *work)
1339 {
1340 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1341 			struct nvme_ctrl, ka_work);
1342 	bool comp_seen = ctrl->comp_seen;
1343 	struct request *rq;
1344 	blk_status_t status;
1345 
1346 	ctrl->ka_last_check_time = jiffies;
1347 
1348 	if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1349 		dev_dbg(ctrl->device,
1350 			"reschedule traffic based keep-alive timer\n");
1351 		ctrl->comp_seen = false;
1352 		nvme_queue_keep_alive_work(ctrl);
1353 		return;
1354 	}
1355 
1356 	rq = blk_mq_alloc_request(ctrl->admin_q, nvme_req_op(&ctrl->ka_cmd),
1357 				  BLK_MQ_REQ_RESERVED | BLK_MQ_REQ_NOWAIT);
1358 	if (IS_ERR(rq)) {
1359 		/* allocation failure, reset the controller */
1360 		dev_err(ctrl->device, "keep-alive failed: %ld\n", PTR_ERR(rq));
1361 		nvme_reset_ctrl(ctrl);
1362 		return;
1363 	}
1364 	nvme_init_request(rq, &ctrl->ka_cmd);
1365 
1366 	rq->timeout = ctrl->kato * HZ;
1367 	status = blk_execute_rq(rq, false);
1368 	nvme_keep_alive_finish(rq, status, ctrl);
1369 	blk_mq_free_request(rq);
1370 }
1371 
1372 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1373 {
1374 	if (unlikely(ctrl->kato == 0))
1375 		return;
1376 
1377 	nvme_queue_keep_alive_work(ctrl);
1378 }
1379 
1380 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1381 {
1382 	if (unlikely(ctrl->kato == 0))
1383 		return;
1384 
1385 	cancel_delayed_work_sync(&ctrl->ka_work);
1386 }
1387 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1388 
1389 static void nvme_update_keep_alive(struct nvme_ctrl *ctrl,
1390 				   struct nvme_command *cmd)
1391 {
1392 	unsigned int new_kato =
1393 		DIV_ROUND_UP(le32_to_cpu(cmd->common.cdw11), 1000);
1394 
1395 	dev_info(ctrl->device,
1396 		 "keep alive interval updated from %u ms to %u ms\n",
1397 		 ctrl->kato * 1000 / 2, new_kato * 1000 / 2);
1398 
1399 	nvme_stop_keep_alive(ctrl);
1400 	ctrl->kato = new_kato;
1401 	nvme_start_keep_alive(ctrl);
1402 }
1403 
1404 static bool nvme_id_cns_ok(struct nvme_ctrl *ctrl, u8 cns)
1405 {
1406 	/*
1407 	 * The CNS field occupies a full byte starting with NVMe 1.2
1408 	 */
1409 	if (ctrl->vs >= NVME_VS(1, 2, 0))
1410 		return true;
1411 
1412 	/*
1413 	 * NVMe 1.1 expanded the CNS value to two bits, which means values
1414 	 * larger than that could get truncated and treated as an incorrect
1415 	 * value.
1416 	 *
1417 	 * Qemu implemented 1.0 behavior for controllers claiming 1.1
1418 	 * compliance, so they need to be quirked here.
1419 	 */
1420 	if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1421 	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS))
1422 		return cns <= 3;
1423 
1424 	/*
1425 	 * NVMe 1.0 used a single bit for the CNS value.
1426 	 */
1427 	return cns <= 1;
1428 }
1429 
1430 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1431 {
1432 	struct nvme_command c = { };
1433 	int error;
1434 
1435 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1436 	c.identify.opcode = nvme_admin_identify;
1437 	c.identify.cns = NVME_ID_CNS_CTRL;
1438 
1439 	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1440 	if (!*id)
1441 		return -ENOMEM;
1442 
1443 	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1444 			sizeof(struct nvme_id_ctrl));
1445 	if (error) {
1446 		kfree(*id);
1447 		*id = NULL;
1448 	}
1449 	return error;
1450 }
1451 
1452 static int nvme_process_ns_desc(struct nvme_ctrl *ctrl, struct nvme_ns_ids *ids,
1453 		struct nvme_ns_id_desc *cur, bool *csi_seen)
1454 {
1455 	const char *warn_str = "ctrl returned bogus length:";
1456 	void *data = cur;
1457 
1458 	switch (cur->nidt) {
1459 	case NVME_NIDT_EUI64:
1460 		if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1461 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
1462 				 warn_str, cur->nidl);
1463 			return -1;
1464 		}
1465 		if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1466 			return NVME_NIDT_EUI64_LEN;
1467 		memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
1468 		return NVME_NIDT_EUI64_LEN;
1469 	case NVME_NIDT_NGUID:
1470 		if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1471 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
1472 				 warn_str, cur->nidl);
1473 			return -1;
1474 		}
1475 		if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1476 			return NVME_NIDT_NGUID_LEN;
1477 		memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
1478 		return NVME_NIDT_NGUID_LEN;
1479 	case NVME_NIDT_UUID:
1480 		if (cur->nidl != NVME_NIDT_UUID_LEN) {
1481 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
1482 				 warn_str, cur->nidl);
1483 			return -1;
1484 		}
1485 		if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
1486 			return NVME_NIDT_UUID_LEN;
1487 		uuid_copy(&ids->uuid, data + sizeof(*cur));
1488 		return NVME_NIDT_UUID_LEN;
1489 	case NVME_NIDT_CSI:
1490 		if (cur->nidl != NVME_NIDT_CSI_LEN) {
1491 			dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
1492 				 warn_str, cur->nidl);
1493 			return -1;
1494 		}
1495 		memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
1496 		*csi_seen = true;
1497 		return NVME_NIDT_CSI_LEN;
1498 	default:
1499 		/* Skip unknown types */
1500 		return cur->nidl;
1501 	}
1502 }
1503 
1504 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl,
1505 		struct nvme_ns_info *info)
1506 {
1507 	struct nvme_command c = { };
1508 	bool csi_seen = false;
1509 	int status, pos, len;
1510 	void *data;
1511 
1512 	if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
1513 		return 0;
1514 	if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
1515 		return 0;
1516 
1517 	c.identify.opcode = nvme_admin_identify;
1518 	c.identify.nsid = cpu_to_le32(info->nsid);
1519 	c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1520 
1521 	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1522 	if (!data)
1523 		return -ENOMEM;
1524 
1525 	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1526 				      NVME_IDENTIFY_DATA_SIZE);
1527 	if (status) {
1528 		dev_warn(ctrl->device,
1529 			"Identify Descriptors failed (nsid=%u, status=0x%x)\n",
1530 			info->nsid, status);
1531 		goto free_data;
1532 	}
1533 
1534 	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1535 		struct nvme_ns_id_desc *cur = data + pos;
1536 
1537 		if (cur->nidl == 0)
1538 			break;
1539 
1540 		len = nvme_process_ns_desc(ctrl, &info->ids, cur, &csi_seen);
1541 		if (len < 0)
1542 			break;
1543 
1544 		len += sizeof(*cur);
1545 	}
1546 
1547 	if (nvme_multi_css(ctrl) && !csi_seen) {
1548 		dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
1549 			 info->nsid);
1550 		status = -EINVAL;
1551 	}
1552 
1553 free_data:
1554 	kfree(data);
1555 	return status;
1556 }
1557 
1558 int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
1559 			struct nvme_id_ns **id)
1560 {
1561 	struct nvme_command c = { };
1562 	int error;
1563 
1564 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1565 	c.identify.opcode = nvme_admin_identify;
1566 	c.identify.nsid = cpu_to_le32(nsid);
1567 	c.identify.cns = NVME_ID_CNS_NS;
1568 
1569 	*id = kmalloc(sizeof(**id), GFP_KERNEL);
1570 	if (!*id)
1571 		return -ENOMEM;
1572 
1573 	error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1574 	if (error) {
1575 		dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1576 		kfree(*id);
1577 		*id = NULL;
1578 	}
1579 	return error;
1580 }
1581 
1582 static int nvme_ns_info_from_identify(struct nvme_ctrl *ctrl,
1583 		struct nvme_ns_info *info)
1584 {
1585 	struct nvme_ns_ids *ids = &info->ids;
1586 	struct nvme_id_ns *id;
1587 	int ret;
1588 
1589 	ret = nvme_identify_ns(ctrl, info->nsid, &id);
1590 	if (ret)
1591 		return ret;
1592 
1593 	if (id->ncap == 0) {
1594 		/* namespace not allocated or attached */
1595 		info->is_removed = true;
1596 		ret = -ENODEV;
1597 		goto error;
1598 	}
1599 
1600 	info->anagrpid = id->anagrpid;
1601 	info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1602 	info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1603 	info->is_ready = true;
1604 	if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
1605 		dev_info(ctrl->device,
1606 			 "Ignoring bogus Namespace Identifiers\n");
1607 	} else {
1608 		if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1609 		    !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
1610 			memcpy(ids->eui64, id->eui64, sizeof(ids->eui64));
1611 		if (ctrl->vs >= NVME_VS(1, 2, 0) &&
1612 		    !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
1613 			memcpy(ids->nguid, id->nguid, sizeof(ids->nguid));
1614 	}
1615 
1616 error:
1617 	kfree(id);
1618 	return ret;
1619 }
1620 
1621 static int nvme_ns_info_from_id_cs_indep(struct nvme_ctrl *ctrl,
1622 		struct nvme_ns_info *info)
1623 {
1624 	struct nvme_id_ns_cs_indep *id;
1625 	struct nvme_command c = {
1626 		.identify.opcode	= nvme_admin_identify,
1627 		.identify.nsid		= cpu_to_le32(info->nsid),
1628 		.identify.cns		= NVME_ID_CNS_NS_CS_INDEP,
1629 	};
1630 	int ret;
1631 
1632 	id = kmalloc(sizeof(*id), GFP_KERNEL);
1633 	if (!id)
1634 		return -ENOMEM;
1635 
1636 	ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1637 	if (!ret) {
1638 		info->anagrpid = id->anagrpid;
1639 		info->is_shared = id->nmic & NVME_NS_NMIC_SHARED;
1640 		info->is_readonly = id->nsattr & NVME_NS_ATTR_RO;
1641 		info->is_ready = id->nstat & NVME_NSTAT_NRDY;
1642 	}
1643 	kfree(id);
1644 	return ret;
1645 }
1646 
1647 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1648 		unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1649 {
1650 	union nvme_result res = { 0 };
1651 	struct nvme_command c = { };
1652 	int ret;
1653 
1654 	c.features.opcode = op;
1655 	c.features.fid = cpu_to_le32(fid);
1656 	c.features.dword11 = cpu_to_le32(dword11);
1657 
1658 	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1659 			buffer, buflen, NVME_QID_ANY, 0);
1660 	if (ret >= 0 && result)
1661 		*result = le32_to_cpu(res.u32);
1662 	return ret;
1663 }
1664 
1665 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1666 		      unsigned int dword11, void *buffer, size_t buflen,
1667 		      u32 *result)
1668 {
1669 	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1670 			     buflen, result);
1671 }
1672 EXPORT_SYMBOL_GPL(nvme_set_features);
1673 
1674 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1675 		      unsigned int dword11, void *buffer, size_t buflen,
1676 		      u32 *result)
1677 {
1678 	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1679 			     buflen, result);
1680 }
1681 EXPORT_SYMBOL_GPL(nvme_get_features);
1682 
1683 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1684 {
1685 	u32 q_count = (*count - 1) | ((*count - 1) << 16);
1686 	u32 result;
1687 	int status, nr_io_queues;
1688 
1689 	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1690 			&result);
1691 	if (status < 0)
1692 		return status;
1693 
1694 	/*
1695 	 * Degraded controllers might return an error when setting the queue
1696 	 * count.  We still want to be able to bring them online and offer
1697 	 * access to the admin queue, as that might be only way to fix them up.
1698 	 */
1699 	if (status > 0) {
1700 		dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1701 		*count = 0;
1702 	} else {
1703 		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1704 		*count = min(*count, nr_io_queues);
1705 	}
1706 
1707 	return 0;
1708 }
1709 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1710 
1711 #define NVME_AEN_SUPPORTED \
1712 	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1713 	 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1714 
1715 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1716 {
1717 	u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1718 	int status;
1719 
1720 	if (!supported_aens)
1721 		return;
1722 
1723 	status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1724 			NULL, 0, &result);
1725 	if (status)
1726 		dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1727 			 supported_aens);
1728 
1729 	queue_work(nvme_wq, &ctrl->async_event_work);
1730 }
1731 
1732 static int nvme_ns_open(struct nvme_ns *ns)
1733 {
1734 
1735 	/* should never be called due to GENHD_FL_HIDDEN */
1736 	if (WARN_ON_ONCE(nvme_ns_head_multipath(ns->head)))
1737 		goto fail;
1738 	if (!nvme_get_ns(ns))
1739 		goto fail;
1740 	if (!try_module_get(ns->ctrl->ops->module))
1741 		goto fail_put_ns;
1742 
1743 	return 0;
1744 
1745 fail_put_ns:
1746 	nvme_put_ns(ns);
1747 fail:
1748 	return -ENXIO;
1749 }
1750 
1751 static void nvme_ns_release(struct nvme_ns *ns)
1752 {
1753 
1754 	module_put(ns->ctrl->ops->module);
1755 	nvme_put_ns(ns);
1756 }
1757 
1758 static int nvme_open(struct gendisk *disk, blk_mode_t mode)
1759 {
1760 	return nvme_ns_open(disk->private_data);
1761 }
1762 
1763 static void nvme_release(struct gendisk *disk)
1764 {
1765 	nvme_ns_release(disk->private_data);
1766 }
1767 
1768 int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1769 {
1770 	/* some standard values */
1771 	geo->heads = 1 << 6;
1772 	geo->sectors = 1 << 5;
1773 	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1774 	return 0;
1775 }
1776 
1777 static bool nvme_init_integrity(struct nvme_ns_head *head,
1778 		struct queue_limits *lim, struct nvme_ns_info *info)
1779 {
1780 	struct blk_integrity *bi = &lim->integrity;
1781 
1782 	memset(bi, 0, sizeof(*bi));
1783 
1784 	if (!head->ms)
1785 		return true;
1786 
1787 	/*
1788 	 * PI can always be supported as we can ask the controller to simply
1789 	 * insert/strip it, which is not possible for other kinds of metadata.
1790 	 */
1791 	if (!IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) ||
1792 	    !(head->features & NVME_NS_METADATA_SUPPORTED))
1793 		return nvme_ns_has_pi(head);
1794 
1795 	switch (head->pi_type) {
1796 	case NVME_NS_DPS_PI_TYPE3:
1797 		switch (head->guard_type) {
1798 		case NVME_NVM_NS_16B_GUARD:
1799 			bi->csum_type = BLK_INTEGRITY_CSUM_CRC;
1800 			bi->tag_size = sizeof(u16) + sizeof(u32);
1801 			bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1802 			break;
1803 		case NVME_NVM_NS_64B_GUARD:
1804 			bi->csum_type = BLK_INTEGRITY_CSUM_CRC64;
1805 			bi->tag_size = sizeof(u16) + 6;
1806 			bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1807 			break;
1808 		default:
1809 			break;
1810 		}
1811 		break;
1812 	case NVME_NS_DPS_PI_TYPE1:
1813 	case NVME_NS_DPS_PI_TYPE2:
1814 		switch (head->guard_type) {
1815 		case NVME_NVM_NS_16B_GUARD:
1816 			bi->csum_type = BLK_INTEGRITY_CSUM_CRC;
1817 			bi->tag_size = sizeof(u16);
1818 			bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE |
1819 				     BLK_INTEGRITY_REF_TAG;
1820 			break;
1821 		case NVME_NVM_NS_64B_GUARD:
1822 			bi->csum_type = BLK_INTEGRITY_CSUM_CRC64;
1823 			bi->tag_size = sizeof(u16);
1824 			bi->flags |= BLK_INTEGRITY_DEVICE_CAPABLE |
1825 				     BLK_INTEGRITY_REF_TAG;
1826 			break;
1827 		default:
1828 			break;
1829 		}
1830 		break;
1831 	default:
1832 		break;
1833 	}
1834 
1835 	bi->tuple_size = head->ms;
1836 	bi->pi_offset = info->pi_offset;
1837 	return true;
1838 }
1839 
1840 static void nvme_config_discard(struct nvme_ns *ns, struct queue_limits *lim)
1841 {
1842 	struct nvme_ctrl *ctrl = ns->ctrl;
1843 
1844 	if (ctrl->dmrsl && ctrl->dmrsl <= nvme_sect_to_lba(ns->head, UINT_MAX))
1845 		lim->max_hw_discard_sectors =
1846 			nvme_lba_to_sect(ns->head, ctrl->dmrsl);
1847 	else if (ctrl->oncs & NVME_CTRL_ONCS_DSM)
1848 		lim->max_hw_discard_sectors = UINT_MAX;
1849 	else
1850 		lim->max_hw_discard_sectors = 0;
1851 
1852 	lim->discard_granularity = lim->logical_block_size;
1853 
1854 	if (ctrl->dmrl)
1855 		lim->max_discard_segments = ctrl->dmrl;
1856 	else
1857 		lim->max_discard_segments = NVME_DSM_MAX_RANGES;
1858 }
1859 
1860 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1861 {
1862 	return uuid_equal(&a->uuid, &b->uuid) &&
1863 		memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1864 		memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
1865 		a->csi == b->csi;
1866 }
1867 
1868 static int nvme_identify_ns_nvm(struct nvme_ctrl *ctrl, unsigned int nsid,
1869 		struct nvme_id_ns_nvm **nvmp)
1870 {
1871 	struct nvme_command c = {
1872 		.identify.opcode	= nvme_admin_identify,
1873 		.identify.nsid		= cpu_to_le32(nsid),
1874 		.identify.cns		= NVME_ID_CNS_CS_NS,
1875 		.identify.csi		= NVME_CSI_NVM,
1876 	};
1877 	struct nvme_id_ns_nvm *nvm;
1878 	int ret;
1879 
1880 	nvm = kzalloc(sizeof(*nvm), GFP_KERNEL);
1881 	if (!nvm)
1882 		return -ENOMEM;
1883 
1884 	ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, nvm, sizeof(*nvm));
1885 	if (ret)
1886 		kfree(nvm);
1887 	else
1888 		*nvmp = nvm;
1889 	return ret;
1890 }
1891 
1892 static void nvme_configure_pi_elbas(struct nvme_ns_head *head,
1893 		struct nvme_id_ns *id, struct nvme_id_ns_nvm *nvm)
1894 {
1895 	u32 elbaf = le32_to_cpu(nvm->elbaf[nvme_lbaf_index(id->flbas)]);
1896 	u8 guard_type;
1897 
1898 	/* no support for storage tag formats right now */
1899 	if (nvme_elbaf_sts(elbaf))
1900 		return;
1901 
1902 	guard_type = nvme_elbaf_guard_type(elbaf);
1903 	if ((nvm->pic & NVME_ID_NS_NVM_QPIFS) &&
1904 	     guard_type == NVME_NVM_NS_QTYPE_GUARD)
1905 		guard_type = nvme_elbaf_qualified_guard_type(elbaf);
1906 
1907 	head->guard_type = guard_type;
1908 	switch (head->guard_type) {
1909 	case NVME_NVM_NS_64B_GUARD:
1910 		head->pi_size = sizeof(struct crc64_pi_tuple);
1911 		break;
1912 	case NVME_NVM_NS_16B_GUARD:
1913 		head->pi_size = sizeof(struct t10_pi_tuple);
1914 		break;
1915 	default:
1916 		break;
1917 	}
1918 }
1919 
1920 static void nvme_configure_metadata(struct nvme_ctrl *ctrl,
1921 		struct nvme_ns_head *head, struct nvme_id_ns *id,
1922 		struct nvme_id_ns_nvm *nvm, struct nvme_ns_info *info)
1923 {
1924 	head->features &= ~(NVME_NS_METADATA_SUPPORTED | NVME_NS_EXT_LBAS);
1925 	head->pi_type = 0;
1926 	head->pi_size = 0;
1927 	head->ms = le16_to_cpu(id->lbaf[nvme_lbaf_index(id->flbas)].ms);
1928 	if (!head->ms || !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1929 		return;
1930 
1931 	if (nvm && (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)) {
1932 		nvme_configure_pi_elbas(head, id, nvm);
1933 	} else {
1934 		head->pi_size = sizeof(struct t10_pi_tuple);
1935 		head->guard_type = NVME_NVM_NS_16B_GUARD;
1936 	}
1937 
1938 	if (head->pi_size && head->ms >= head->pi_size)
1939 		head->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1940 	if (!(id->dps & NVME_NS_DPS_PI_FIRST)) {
1941 		if (disable_pi_offsets)
1942 			head->pi_type = 0;
1943 		else
1944 			info->pi_offset = head->ms - head->pi_size;
1945 	}
1946 
1947 	if (ctrl->ops->flags & NVME_F_FABRICS) {
1948 		/*
1949 		 * The NVMe over Fabrics specification only supports metadata as
1950 		 * part of the extended data LBA.  We rely on HCA/HBA support to
1951 		 * remap the separate metadata buffer from the block layer.
1952 		 */
1953 		if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
1954 			return;
1955 
1956 		head->features |= NVME_NS_EXT_LBAS;
1957 
1958 		/*
1959 		 * The current fabrics transport drivers support namespace
1960 		 * metadata formats only if nvme_ns_has_pi() returns true.
1961 		 * Suppress support for all other formats so the namespace will
1962 		 * have a 0 capacity and not be usable through the block stack.
1963 		 *
1964 		 * Note, this check will need to be modified if any drivers
1965 		 * gain the ability to use other metadata formats.
1966 		 */
1967 		if (ctrl->max_integrity_segments && nvme_ns_has_pi(head))
1968 			head->features |= NVME_NS_METADATA_SUPPORTED;
1969 	} else {
1970 		/*
1971 		 * For PCIe controllers, we can't easily remap the separate
1972 		 * metadata buffer from the block layer and thus require a
1973 		 * separate metadata buffer for block layer metadata/PI support.
1974 		 * We allow extended LBAs for the passthrough interface, though.
1975 		 */
1976 		if (id->flbas & NVME_NS_FLBAS_META_EXT)
1977 			head->features |= NVME_NS_EXT_LBAS;
1978 		else
1979 			head->features |= NVME_NS_METADATA_SUPPORTED;
1980 	}
1981 }
1982 
1983 
1984 static void nvme_update_atomic_write_disk_info(struct nvme_ns *ns,
1985 			struct nvme_id_ns *id, struct queue_limits *lim,
1986 			u32 bs, u32 atomic_bs)
1987 {
1988 	unsigned int boundary = 0;
1989 
1990 	if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf) {
1991 		if (le16_to_cpu(id->nabspf))
1992 			boundary = (le16_to_cpu(id->nabspf) + 1) * bs;
1993 	}
1994 	lim->atomic_write_hw_max = atomic_bs;
1995 	lim->atomic_write_hw_boundary = boundary;
1996 	lim->atomic_write_hw_unit_min = bs;
1997 	lim->atomic_write_hw_unit_max = rounddown_pow_of_two(atomic_bs);
1998 }
1999 
2000 static u32 nvme_max_drv_segments(struct nvme_ctrl *ctrl)
2001 {
2002 	return ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> SECTOR_SHIFT) + 1;
2003 }
2004 
2005 static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl,
2006 		struct queue_limits *lim)
2007 {
2008 	lim->max_hw_sectors = ctrl->max_hw_sectors;
2009 	lim->max_segments = min_t(u32, USHRT_MAX,
2010 		min_not_zero(nvme_max_drv_segments(ctrl), ctrl->max_segments));
2011 	lim->max_integrity_segments = ctrl->max_integrity_segments;
2012 	lim->virt_boundary_mask = NVME_CTRL_PAGE_SIZE - 1;
2013 	lim->max_segment_size = UINT_MAX;
2014 	lim->dma_alignment = 3;
2015 }
2016 
2017 static bool nvme_update_disk_info(struct nvme_ns *ns, struct nvme_id_ns *id,
2018 		struct queue_limits *lim)
2019 {
2020 	struct nvme_ns_head *head = ns->head;
2021 	u32 bs = 1U << head->lba_shift;
2022 	u32 atomic_bs, phys_bs, io_opt = 0;
2023 	bool valid = true;
2024 
2025 	/*
2026 	 * The block layer can't support LBA sizes larger than the page size
2027 	 * or smaller than a sector size yet, so catch this early and don't
2028 	 * allow block I/O.
2029 	 */
2030 	if (head->lba_shift > PAGE_SHIFT || head->lba_shift < SECTOR_SHIFT) {
2031 		bs = (1 << 9);
2032 		valid = false;
2033 	}
2034 
2035 	atomic_bs = phys_bs = bs;
2036 	if (id->nabo == 0) {
2037 		/*
2038 		 * Bit 1 indicates whether NAWUPF is defined for this namespace
2039 		 * and whether it should be used instead of AWUPF. If NAWUPF ==
2040 		 * 0 then AWUPF must be used instead.
2041 		 */
2042 		if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
2043 			atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
2044 		else
2045 			atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
2046 
2047 		nvme_update_atomic_write_disk_info(ns, id, lim, bs, atomic_bs);
2048 	}
2049 
2050 	if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
2051 		/* NPWG = Namespace Preferred Write Granularity */
2052 		phys_bs = bs * (1 + le16_to_cpu(id->npwg));
2053 		/* NOWS = Namespace Optimal Write Size */
2054 		if (id->nows)
2055 			io_opt = bs * (1 + le16_to_cpu(id->nows));
2056 	}
2057 
2058 	/*
2059 	 * Linux filesystems assume writing a single physical block is
2060 	 * an atomic operation. Hence limit the physical block size to the
2061 	 * value of the Atomic Write Unit Power Fail parameter.
2062 	 */
2063 	lim->logical_block_size = bs;
2064 	lim->physical_block_size = min(phys_bs, atomic_bs);
2065 	lim->io_min = phys_bs;
2066 	lim->io_opt = io_opt;
2067 	if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
2068 		lim->max_write_zeroes_sectors = UINT_MAX;
2069 	else
2070 		lim->max_write_zeroes_sectors = ns->ctrl->max_zeroes_sectors;
2071 	return valid;
2072 }
2073 
2074 static bool nvme_ns_is_readonly(struct nvme_ns *ns, struct nvme_ns_info *info)
2075 {
2076 	return info->is_readonly || test_bit(NVME_NS_FORCE_RO, &ns->flags);
2077 }
2078 
2079 static inline bool nvme_first_scan(struct gendisk *disk)
2080 {
2081 	/* nvme_alloc_ns() scans the disk prior to adding it */
2082 	return !disk_live(disk);
2083 }
2084 
2085 static void nvme_set_chunk_sectors(struct nvme_ns *ns, struct nvme_id_ns *id,
2086 		struct queue_limits *lim)
2087 {
2088 	struct nvme_ctrl *ctrl = ns->ctrl;
2089 	u32 iob;
2090 
2091 	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2092 	    is_power_of_2(ctrl->max_hw_sectors))
2093 		iob = ctrl->max_hw_sectors;
2094 	else
2095 		iob = nvme_lba_to_sect(ns->head, le16_to_cpu(id->noiob));
2096 
2097 	if (!iob)
2098 		return;
2099 
2100 	if (!is_power_of_2(iob)) {
2101 		if (nvme_first_scan(ns->disk))
2102 			pr_warn("%s: ignoring unaligned IO boundary:%u\n",
2103 				ns->disk->disk_name, iob);
2104 		return;
2105 	}
2106 
2107 	if (blk_queue_is_zoned(ns->disk->queue)) {
2108 		if (nvme_first_scan(ns->disk))
2109 			pr_warn("%s: ignoring zoned namespace IO boundary\n",
2110 				ns->disk->disk_name);
2111 		return;
2112 	}
2113 
2114 	lim->chunk_sectors = iob;
2115 }
2116 
2117 static int nvme_update_ns_info_generic(struct nvme_ns *ns,
2118 		struct nvme_ns_info *info)
2119 {
2120 	struct queue_limits lim;
2121 	int ret;
2122 
2123 	blk_mq_freeze_queue(ns->disk->queue);
2124 	lim = queue_limits_start_update(ns->disk->queue);
2125 	nvme_set_ctrl_limits(ns->ctrl, &lim);
2126 	ret = queue_limits_commit_update(ns->disk->queue, &lim);
2127 	set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2128 	blk_mq_unfreeze_queue(ns->disk->queue);
2129 
2130 	/* Hide the block-interface for these devices */
2131 	if (!ret)
2132 		ret = -ENODEV;
2133 	return ret;
2134 }
2135 
2136 static int nvme_update_ns_info_block(struct nvme_ns *ns,
2137 		struct nvme_ns_info *info)
2138 {
2139 	struct queue_limits lim;
2140 	struct nvme_id_ns_nvm *nvm = NULL;
2141 	struct nvme_zone_info zi = {};
2142 	struct nvme_id_ns *id;
2143 	sector_t capacity;
2144 	unsigned lbaf;
2145 	int ret;
2146 
2147 	ret = nvme_identify_ns(ns->ctrl, info->nsid, &id);
2148 	if (ret)
2149 		return ret;
2150 
2151 	if (id->ncap == 0) {
2152 		/* namespace not allocated or attached */
2153 		info->is_removed = true;
2154 		ret = -ENXIO;
2155 		goto out;
2156 	}
2157 	lbaf = nvme_lbaf_index(id->flbas);
2158 
2159 	if (ns->ctrl->ctratt & NVME_CTRL_ATTR_ELBAS) {
2160 		ret = nvme_identify_ns_nvm(ns->ctrl, info->nsid, &nvm);
2161 		if (ret < 0)
2162 			goto out;
2163 	}
2164 
2165 	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
2166 	    ns->head->ids.csi == NVME_CSI_ZNS) {
2167 		ret = nvme_query_zone_info(ns, lbaf, &zi);
2168 		if (ret < 0)
2169 			goto out;
2170 	}
2171 
2172 	blk_mq_freeze_queue(ns->disk->queue);
2173 	ns->head->lba_shift = id->lbaf[lbaf].ds;
2174 	ns->head->nuse = le64_to_cpu(id->nuse);
2175 	capacity = nvme_lba_to_sect(ns->head, le64_to_cpu(id->nsze));
2176 
2177 	lim = queue_limits_start_update(ns->disk->queue);
2178 	nvme_set_ctrl_limits(ns->ctrl, &lim);
2179 	nvme_configure_metadata(ns->ctrl, ns->head, id, nvm, info);
2180 	nvme_set_chunk_sectors(ns, id, &lim);
2181 	if (!nvme_update_disk_info(ns, id, &lim))
2182 		capacity = 0;
2183 	nvme_config_discard(ns, &lim);
2184 	if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
2185 	    ns->head->ids.csi == NVME_CSI_ZNS)
2186 		nvme_update_zone_info(ns, &lim, &zi);
2187 
2188 	if (ns->ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2189 		lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA;
2190 	else
2191 		lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA);
2192 
2193 	/*
2194 	 * Register a metadata profile for PI, or the plain non-integrity NVMe
2195 	 * metadata masquerading as Type 0 if supported, otherwise reject block
2196 	 * I/O to namespaces with metadata except when the namespace supports
2197 	 * PI, as it can strip/insert in that case.
2198 	 */
2199 	if (!nvme_init_integrity(ns->head, &lim, info))
2200 		capacity = 0;
2201 
2202 	ret = queue_limits_commit_update(ns->disk->queue, &lim);
2203 	if (ret) {
2204 		blk_mq_unfreeze_queue(ns->disk->queue);
2205 		goto out;
2206 	}
2207 
2208 	set_capacity_and_notify(ns->disk, capacity);
2209 
2210 	/*
2211 	 * Only set the DEAC bit if the device guarantees that reads from
2212 	 * deallocated data return zeroes.  While the DEAC bit does not
2213 	 * require that, it must be a no-op if reads from deallocated data
2214 	 * do not return zeroes.
2215 	 */
2216 	if ((id->dlfeat & 0x7) == 0x1 && (id->dlfeat & (1 << 3)))
2217 		ns->head->features |= NVME_NS_DEAC;
2218 	set_disk_ro(ns->disk, nvme_ns_is_readonly(ns, info));
2219 	set_bit(NVME_NS_READY, &ns->flags);
2220 	blk_mq_unfreeze_queue(ns->disk->queue);
2221 
2222 	if (blk_queue_is_zoned(ns->queue)) {
2223 		ret = blk_revalidate_disk_zones(ns->disk);
2224 		if (ret && !nvme_first_scan(ns->disk))
2225 			goto out;
2226 	}
2227 
2228 	ret = 0;
2229 out:
2230 	kfree(nvm);
2231 	kfree(id);
2232 	return ret;
2233 }
2234 
2235 static int nvme_update_ns_info(struct nvme_ns *ns, struct nvme_ns_info *info)
2236 {
2237 	bool unsupported = false;
2238 	int ret;
2239 
2240 	switch (info->ids.csi) {
2241 	case NVME_CSI_ZNS:
2242 		if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
2243 			dev_info(ns->ctrl->device,
2244 	"block device for nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
2245 				info->nsid);
2246 			ret = nvme_update_ns_info_generic(ns, info);
2247 			break;
2248 		}
2249 		ret = nvme_update_ns_info_block(ns, info);
2250 		break;
2251 	case NVME_CSI_NVM:
2252 		ret = nvme_update_ns_info_block(ns, info);
2253 		break;
2254 	default:
2255 		dev_info(ns->ctrl->device,
2256 			"block device for nsid %u not supported (csi %u)\n",
2257 			info->nsid, info->ids.csi);
2258 		ret = nvme_update_ns_info_generic(ns, info);
2259 		break;
2260 	}
2261 
2262 	/*
2263 	 * If probing fails due an unsupported feature, hide the block device,
2264 	 * but still allow other access.
2265 	 */
2266 	if (ret == -ENODEV) {
2267 		ns->disk->flags |= GENHD_FL_HIDDEN;
2268 		set_bit(NVME_NS_READY, &ns->flags);
2269 		unsupported = true;
2270 		ret = 0;
2271 	}
2272 
2273 	if (!ret && nvme_ns_head_multipath(ns->head)) {
2274 		struct queue_limits *ns_lim = &ns->disk->queue->limits;
2275 		struct queue_limits lim;
2276 
2277 		blk_mq_freeze_queue(ns->head->disk->queue);
2278 		/*
2279 		 * queue_limits mixes values that are the hardware limitations
2280 		 * for bio splitting with what is the device configuration.
2281 		 *
2282 		 * For NVMe the device configuration can change after e.g. a
2283 		 * Format command, and we really want to pick up the new format
2284 		 * value here.  But we must still stack the queue limits to the
2285 		 * least common denominator for multipathing to split the bios
2286 		 * properly.
2287 		 *
2288 		 * To work around this, we explicitly set the device
2289 		 * configuration to those that we just queried, but only stack
2290 		 * the splitting limits in to make sure we still obey possibly
2291 		 * lower limitations of other controllers.
2292 		 */
2293 		lim = queue_limits_start_update(ns->head->disk->queue);
2294 		lim.logical_block_size = ns_lim->logical_block_size;
2295 		lim.physical_block_size = ns_lim->physical_block_size;
2296 		lim.io_min = ns_lim->io_min;
2297 		lim.io_opt = ns_lim->io_opt;
2298 		queue_limits_stack_bdev(&lim, ns->disk->part0, 0,
2299 					ns->head->disk->disk_name);
2300 		if (unsupported)
2301 			ns->head->disk->flags |= GENHD_FL_HIDDEN;
2302 		else
2303 			nvme_init_integrity(ns->head, &lim, info);
2304 		ret = queue_limits_commit_update(ns->head->disk->queue, &lim);
2305 
2306 		set_capacity_and_notify(ns->head->disk, get_capacity(ns->disk));
2307 		set_disk_ro(ns->head->disk, nvme_ns_is_readonly(ns, info));
2308 		nvme_mpath_revalidate_paths(ns);
2309 
2310 		blk_mq_unfreeze_queue(ns->head->disk->queue);
2311 	}
2312 
2313 	return ret;
2314 }
2315 
2316 int nvme_ns_get_unique_id(struct nvme_ns *ns, u8 id[16],
2317 		enum blk_unique_id type)
2318 {
2319 	struct nvme_ns_ids *ids = &ns->head->ids;
2320 
2321 	if (type != BLK_UID_EUI64)
2322 		return -EINVAL;
2323 
2324 	if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid))) {
2325 		memcpy(id, &ids->nguid, sizeof(ids->nguid));
2326 		return sizeof(ids->nguid);
2327 	}
2328 	if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64))) {
2329 		memcpy(id, &ids->eui64, sizeof(ids->eui64));
2330 		return sizeof(ids->eui64);
2331 	}
2332 
2333 	return -EINVAL;
2334 }
2335 
2336 static int nvme_get_unique_id(struct gendisk *disk, u8 id[16],
2337 		enum blk_unique_id type)
2338 {
2339 	return nvme_ns_get_unique_id(disk->private_data, id, type);
2340 }
2341 
2342 #ifdef CONFIG_BLK_SED_OPAL
2343 static int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2344 		bool send)
2345 {
2346 	struct nvme_ctrl *ctrl = data;
2347 	struct nvme_command cmd = { };
2348 
2349 	if (send)
2350 		cmd.common.opcode = nvme_admin_security_send;
2351 	else
2352 		cmd.common.opcode = nvme_admin_security_recv;
2353 	cmd.common.nsid = 0;
2354 	cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2355 	cmd.common.cdw11 = cpu_to_le32(len);
2356 
2357 	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2358 			NVME_QID_ANY, NVME_SUBMIT_AT_HEAD);
2359 }
2360 
2361 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2362 {
2363 	if (ctrl->oacs & NVME_CTRL_OACS_SEC_SUPP) {
2364 		if (!ctrl->opal_dev)
2365 			ctrl->opal_dev = init_opal_dev(ctrl, &nvme_sec_submit);
2366 		else if (was_suspended)
2367 			opal_unlock_from_suspend(ctrl->opal_dev);
2368 	} else {
2369 		free_opal_dev(ctrl->opal_dev);
2370 		ctrl->opal_dev = NULL;
2371 	}
2372 }
2373 #else
2374 static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended)
2375 {
2376 }
2377 #endif /* CONFIG_BLK_SED_OPAL */
2378 
2379 #ifdef CONFIG_BLK_DEV_ZONED
2380 static int nvme_report_zones(struct gendisk *disk, sector_t sector,
2381 		unsigned int nr_zones, report_zones_cb cb, void *data)
2382 {
2383 	return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb,
2384 			data);
2385 }
2386 #else
2387 #define nvme_report_zones	NULL
2388 #endif /* CONFIG_BLK_DEV_ZONED */
2389 
2390 const struct block_device_operations nvme_bdev_ops = {
2391 	.owner		= THIS_MODULE,
2392 	.ioctl		= nvme_ioctl,
2393 	.compat_ioctl	= blkdev_compat_ptr_ioctl,
2394 	.open		= nvme_open,
2395 	.release	= nvme_release,
2396 	.getgeo		= nvme_getgeo,
2397 	.get_unique_id	= nvme_get_unique_id,
2398 	.report_zones	= nvme_report_zones,
2399 	.pr_ops		= &nvme_pr_ops,
2400 };
2401 
2402 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u32 mask, u32 val,
2403 		u32 timeout, const char *op)
2404 {
2405 	unsigned long timeout_jiffies = jiffies + timeout * HZ;
2406 	u32 csts;
2407 	int ret;
2408 
2409 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2410 		if (csts == ~0)
2411 			return -ENODEV;
2412 		if ((csts & mask) == val)
2413 			break;
2414 
2415 		usleep_range(1000, 2000);
2416 		if (fatal_signal_pending(current))
2417 			return -EINTR;
2418 		if (time_after(jiffies, timeout_jiffies)) {
2419 			dev_err(ctrl->device,
2420 				"Device not ready; aborting %s, CSTS=0x%x\n",
2421 				op, csts);
2422 			return -ENODEV;
2423 		}
2424 	}
2425 
2426 	return ret;
2427 }
2428 
2429 int nvme_disable_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2430 {
2431 	int ret;
2432 
2433 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2434 	if (shutdown)
2435 		ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2436 	else
2437 		ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2438 
2439 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2440 	if (ret)
2441 		return ret;
2442 
2443 	if (shutdown) {
2444 		return nvme_wait_ready(ctrl, NVME_CSTS_SHST_MASK,
2445 				       NVME_CSTS_SHST_CMPLT,
2446 				       ctrl->shutdown_timeout, "shutdown");
2447 	}
2448 	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2449 		msleep(NVME_QUIRK_DELAY_AMOUNT);
2450 	return nvme_wait_ready(ctrl, NVME_CSTS_RDY, 0,
2451 			       (NVME_CAP_TIMEOUT(ctrl->cap) + 1) / 2, "reset");
2452 }
2453 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2454 
2455 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2456 {
2457 	unsigned dev_page_min;
2458 	u32 timeout;
2459 	int ret;
2460 
2461 	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2462 	if (ret) {
2463 		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2464 		return ret;
2465 	}
2466 	dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2467 
2468 	if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
2469 		dev_err(ctrl->device,
2470 			"Minimum device page size %u too large for host (%u)\n",
2471 			1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
2472 		return -ENODEV;
2473 	}
2474 
2475 	if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
2476 		ctrl->ctrl_config = NVME_CC_CSS_CSI;
2477 	else
2478 		ctrl->ctrl_config = NVME_CC_CSS_NVM;
2479 
2480 	/*
2481 	 * Setting CRIME results in CSTS.RDY before the media is ready. This
2482 	 * makes it possible for media related commands to return the error
2483 	 * NVME_SC_ADMIN_COMMAND_MEDIA_NOT_READY. Until the driver is
2484 	 * restructured to handle retries, disable CC.CRIME.
2485 	 */
2486 	ctrl->ctrl_config &= ~NVME_CC_CRIME;
2487 
2488 	ctrl->ctrl_config |= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
2489 	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2490 	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2491 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2492 	if (ret)
2493 		return ret;
2494 
2495 	/* CAP value may change after initial CC write */
2496 	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2497 	if (ret)
2498 		return ret;
2499 
2500 	timeout = NVME_CAP_TIMEOUT(ctrl->cap);
2501 	if (ctrl->cap & NVME_CAP_CRMS_CRWMS) {
2502 		u32 crto, ready_timeout;
2503 
2504 		ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CRTO, &crto);
2505 		if (ret) {
2506 			dev_err(ctrl->device, "Reading CRTO failed (%d)\n",
2507 				ret);
2508 			return ret;
2509 		}
2510 
2511 		/*
2512 		 * CRTO should always be greater or equal to CAP.TO, but some
2513 		 * devices are known to get this wrong. Use the larger of the
2514 		 * two values.
2515 		 */
2516 		ready_timeout = NVME_CRTO_CRWMT(crto);
2517 
2518 		if (ready_timeout < timeout)
2519 			dev_warn_once(ctrl->device, "bad crto:%x cap:%llx\n",
2520 				      crto, ctrl->cap);
2521 		else
2522 			timeout = ready_timeout;
2523 	}
2524 
2525 	ctrl->ctrl_config |= NVME_CC_ENABLE;
2526 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2527 	if (ret)
2528 		return ret;
2529 	return nvme_wait_ready(ctrl, NVME_CSTS_RDY, NVME_CSTS_RDY,
2530 			       (timeout + 1) / 2, "initialisation");
2531 }
2532 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2533 
2534 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2535 {
2536 	__le64 ts;
2537 	int ret;
2538 
2539 	if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2540 		return 0;
2541 
2542 	ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2543 	ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2544 			NULL);
2545 	if (ret)
2546 		dev_warn_once(ctrl->device,
2547 			"could not set timestamp (%d)\n", ret);
2548 	return ret;
2549 }
2550 
2551 static int nvme_configure_host_options(struct nvme_ctrl *ctrl)
2552 {
2553 	struct nvme_feat_host_behavior *host;
2554 	u8 acre = 0, lbafee = 0;
2555 	int ret;
2556 
2557 	/* Don't bother enabling the feature if retry delay is not reported */
2558 	if (ctrl->crdt[0])
2559 		acre = NVME_ENABLE_ACRE;
2560 	if (ctrl->ctratt & NVME_CTRL_ATTR_ELBAS)
2561 		lbafee = NVME_ENABLE_LBAFEE;
2562 
2563 	if (!acre && !lbafee)
2564 		return 0;
2565 
2566 	host = kzalloc(sizeof(*host), GFP_KERNEL);
2567 	if (!host)
2568 		return 0;
2569 
2570 	host->acre = acre;
2571 	host->lbafee = lbafee;
2572 	ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2573 				host, sizeof(*host), NULL);
2574 	kfree(host);
2575 	return ret;
2576 }
2577 
2578 /*
2579  * The function checks whether the given total (exlat + enlat) latency of
2580  * a power state allows the latter to be used as an APST transition target.
2581  * It does so by comparing the latency to the primary and secondary latency
2582  * tolerances defined by module params. If there's a match, the corresponding
2583  * timeout value is returned and the matching tolerance index (1 or 2) is
2584  * reported.
2585  */
2586 static bool nvme_apst_get_transition_time(u64 total_latency,
2587 		u64 *transition_time, unsigned *last_index)
2588 {
2589 	if (total_latency <= apst_primary_latency_tol_us) {
2590 		if (*last_index == 1)
2591 			return false;
2592 		*last_index = 1;
2593 		*transition_time = apst_primary_timeout_ms;
2594 		return true;
2595 	}
2596 	if (apst_secondary_timeout_ms &&
2597 		total_latency <= apst_secondary_latency_tol_us) {
2598 		if (*last_index <= 2)
2599 			return false;
2600 		*last_index = 2;
2601 		*transition_time = apst_secondary_timeout_ms;
2602 		return true;
2603 	}
2604 	return false;
2605 }
2606 
2607 /*
2608  * APST (Autonomous Power State Transition) lets us program a table of power
2609  * state transitions that the controller will perform automatically.
2610  *
2611  * Depending on module params, one of the two supported techniques will be used:
2612  *
2613  * - If the parameters provide explicit timeouts and tolerances, they will be
2614  *   used to build a table with up to 2 non-operational states to transition to.
2615  *   The default parameter values were selected based on the values used by
2616  *   Microsoft's and Intel's NVMe drivers. Yet, since we don't implement dynamic
2617  *   regeneration of the APST table in the event of switching between external
2618  *   and battery power, the timeouts and tolerances reflect a compromise
2619  *   between values used by Microsoft for AC and battery scenarios.
2620  * - If not, we'll configure the table with a simple heuristic: we are willing
2621  *   to spend at most 2% of the time transitioning between power states.
2622  *   Therefore, when running in any given state, we will enter the next
2623  *   lower-power non-operational state after waiting 50 * (enlat + exlat)
2624  *   microseconds, as long as that state's exit latency is under the requested
2625  *   maximum latency.
2626  *
2627  * We will not autonomously enter any non-operational state for which the total
2628  * latency exceeds ps_max_latency_us.
2629  *
2630  * Users can set ps_max_latency_us to zero to turn off APST.
2631  */
2632 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2633 {
2634 	struct nvme_feat_auto_pst *table;
2635 	unsigned apste = 0;
2636 	u64 max_lat_us = 0;
2637 	__le64 target = 0;
2638 	int max_ps = -1;
2639 	int state;
2640 	int ret;
2641 	unsigned last_lt_index = UINT_MAX;
2642 
2643 	/*
2644 	 * If APST isn't supported or if we haven't been initialized yet,
2645 	 * then don't do anything.
2646 	 */
2647 	if (!ctrl->apsta)
2648 		return 0;
2649 
2650 	if (ctrl->npss > 31) {
2651 		dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2652 		return 0;
2653 	}
2654 
2655 	table = kzalloc(sizeof(*table), GFP_KERNEL);
2656 	if (!table)
2657 		return 0;
2658 
2659 	if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2660 		/* Turn off APST. */
2661 		dev_dbg(ctrl->device, "APST disabled\n");
2662 		goto done;
2663 	}
2664 
2665 	/*
2666 	 * Walk through all states from lowest- to highest-power.
2667 	 * According to the spec, lower-numbered states use more power.  NPSS,
2668 	 * despite the name, is the index of the lowest-power state, not the
2669 	 * number of states.
2670 	 */
2671 	for (state = (int)ctrl->npss; state >= 0; state--) {
2672 		u64 total_latency_us, exit_latency_us, transition_ms;
2673 
2674 		if (target)
2675 			table->entries[state] = target;
2676 
2677 		/*
2678 		 * Don't allow transitions to the deepest state if it's quirked
2679 		 * off.
2680 		 */
2681 		if (state == ctrl->npss &&
2682 		    (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2683 			continue;
2684 
2685 		/*
2686 		 * Is this state a useful non-operational state for higher-power
2687 		 * states to autonomously transition to?
2688 		 */
2689 		if (!(ctrl->psd[state].flags & NVME_PS_FLAGS_NON_OP_STATE))
2690 			continue;
2691 
2692 		exit_latency_us = (u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2693 		if (exit_latency_us > ctrl->ps_max_latency_us)
2694 			continue;
2695 
2696 		total_latency_us = exit_latency_us +
2697 			le32_to_cpu(ctrl->psd[state].entry_lat);
2698 
2699 		/*
2700 		 * This state is good. It can be used as the APST idle target
2701 		 * for higher power states.
2702 		 */
2703 		if (apst_primary_timeout_ms && apst_primary_latency_tol_us) {
2704 			if (!nvme_apst_get_transition_time(total_latency_us,
2705 					&transition_ms, &last_lt_index))
2706 				continue;
2707 		} else {
2708 			transition_ms = total_latency_us + 19;
2709 			do_div(transition_ms, 20);
2710 			if (transition_ms > (1 << 24) - 1)
2711 				transition_ms = (1 << 24) - 1;
2712 		}
2713 
2714 		target = cpu_to_le64((state << 3) | (transition_ms << 8));
2715 		if (max_ps == -1)
2716 			max_ps = state;
2717 		if (total_latency_us > max_lat_us)
2718 			max_lat_us = total_latency_us;
2719 	}
2720 
2721 	if (max_ps == -1)
2722 		dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2723 	else
2724 		dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2725 			max_ps, max_lat_us, (int)sizeof(*table), table);
2726 	apste = 1;
2727 
2728 done:
2729 	ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2730 				table, sizeof(*table), NULL);
2731 	if (ret)
2732 		dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2733 	kfree(table);
2734 	return ret;
2735 }
2736 
2737 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2738 {
2739 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2740 	u64 latency;
2741 
2742 	switch (val) {
2743 	case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2744 	case PM_QOS_LATENCY_ANY:
2745 		latency = U64_MAX;
2746 		break;
2747 
2748 	default:
2749 		latency = val;
2750 	}
2751 
2752 	if (ctrl->ps_max_latency_us != latency) {
2753 		ctrl->ps_max_latency_us = latency;
2754 		if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
2755 			nvme_configure_apst(ctrl);
2756 	}
2757 }
2758 
2759 struct nvme_core_quirk_entry {
2760 	/*
2761 	 * NVMe model and firmware strings are padded with spaces.  For
2762 	 * simplicity, strings in the quirk table are padded with NULLs
2763 	 * instead.
2764 	 */
2765 	u16 vid;
2766 	const char *mn;
2767 	const char *fr;
2768 	unsigned long quirks;
2769 };
2770 
2771 static const struct nvme_core_quirk_entry core_quirks[] = {
2772 	{
2773 		/*
2774 		 * This Toshiba device seems to die using any APST states.  See:
2775 		 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2776 		 */
2777 		.vid = 0x1179,
2778 		.mn = "THNSF5256GPUK TOSHIBA",
2779 		.quirks = NVME_QUIRK_NO_APST,
2780 	},
2781 	{
2782 		/*
2783 		 * This LiteON CL1-3D*-Q11 firmware version has a race
2784 		 * condition associated with actions related to suspend to idle
2785 		 * LiteON has resolved the problem in future firmware
2786 		 */
2787 		.vid = 0x14a4,
2788 		.fr = "22301111",
2789 		.quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2790 	},
2791 	{
2792 		/*
2793 		 * This Kioxia CD6-V Series / HPE PE8030 device times out and
2794 		 * aborts I/O during any load, but more easily reproducible
2795 		 * with discards (fstrim).
2796 		 *
2797 		 * The device is left in a state where it is also not possible
2798 		 * to use "nvme set-feature" to disable APST, but booting with
2799 		 * nvme_core.default_ps_max_latency=0 works.
2800 		 */
2801 		.vid = 0x1e0f,
2802 		.mn = "KCD6XVUL6T40",
2803 		.quirks = NVME_QUIRK_NO_APST,
2804 	},
2805 	{
2806 		/*
2807 		 * The external Samsung X5 SSD fails initialization without a
2808 		 * delay before checking if it is ready and has a whole set of
2809 		 * other problems.  To make this even more interesting, it
2810 		 * shares the PCI ID with internal Samsung 970 Evo Plus that
2811 		 * does not need or want these quirks.
2812 		 */
2813 		.vid = 0x144d,
2814 		.mn = "Samsung Portable SSD X5",
2815 		.quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY |
2816 			  NVME_QUIRK_NO_DEEPEST_PS |
2817 			  NVME_QUIRK_IGNORE_DEV_SUBNQN,
2818 	}
2819 };
2820 
2821 /* match is null-terminated but idstr is space-padded. */
2822 static bool string_matches(const char *idstr, const char *match, size_t len)
2823 {
2824 	size_t matchlen;
2825 
2826 	if (!match)
2827 		return true;
2828 
2829 	matchlen = strlen(match);
2830 	WARN_ON_ONCE(matchlen > len);
2831 
2832 	if (memcmp(idstr, match, matchlen))
2833 		return false;
2834 
2835 	for (; matchlen < len; matchlen++)
2836 		if (idstr[matchlen] != ' ')
2837 			return false;
2838 
2839 	return true;
2840 }
2841 
2842 static bool quirk_matches(const struct nvme_id_ctrl *id,
2843 			  const struct nvme_core_quirk_entry *q)
2844 {
2845 	return q->vid == le16_to_cpu(id->vid) &&
2846 		string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2847 		string_matches(id->fr, q->fr, sizeof(id->fr));
2848 }
2849 
2850 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2851 		struct nvme_id_ctrl *id)
2852 {
2853 	size_t nqnlen;
2854 	int off;
2855 
2856 	if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2857 		nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2858 		if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2859 			strscpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2860 			return;
2861 		}
2862 
2863 		if (ctrl->vs >= NVME_VS(1, 2, 1))
2864 			dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2865 	}
2866 
2867 	/*
2868 	 * Generate a "fake" NQN similar to the one in Section 4.5 of the NVMe
2869 	 * Base Specification 2.0.  It is slightly different from the format
2870 	 * specified there due to historic reasons, and we can't change it now.
2871 	 */
2872 	off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2873 			"nqn.2014.08.org.nvmexpress:%04x%04x",
2874 			le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2875 	memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2876 	off += sizeof(id->sn);
2877 	memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2878 	off += sizeof(id->mn);
2879 	memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2880 }
2881 
2882 static void nvme_release_subsystem(struct device *dev)
2883 {
2884 	struct nvme_subsystem *subsys =
2885 		container_of(dev, struct nvme_subsystem, dev);
2886 
2887 	if (subsys->instance >= 0)
2888 		ida_free(&nvme_instance_ida, subsys->instance);
2889 	kfree(subsys);
2890 }
2891 
2892 static void nvme_destroy_subsystem(struct kref *ref)
2893 {
2894 	struct nvme_subsystem *subsys =
2895 			container_of(ref, struct nvme_subsystem, ref);
2896 
2897 	mutex_lock(&nvme_subsystems_lock);
2898 	list_del(&subsys->entry);
2899 	mutex_unlock(&nvme_subsystems_lock);
2900 
2901 	ida_destroy(&subsys->ns_ida);
2902 	device_del(&subsys->dev);
2903 	put_device(&subsys->dev);
2904 }
2905 
2906 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2907 {
2908 	kref_put(&subsys->ref, nvme_destroy_subsystem);
2909 }
2910 
2911 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2912 {
2913 	struct nvme_subsystem *subsys;
2914 
2915 	lockdep_assert_held(&nvme_subsystems_lock);
2916 
2917 	/*
2918 	 * Fail matches for discovery subsystems. This results
2919 	 * in each discovery controller bound to a unique subsystem.
2920 	 * This avoids issues with validating controller values
2921 	 * that can only be true when there is a single unique subsystem.
2922 	 * There may be multiple and completely independent entities
2923 	 * that provide discovery controllers.
2924 	 */
2925 	if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2926 		return NULL;
2927 
2928 	list_for_each_entry(subsys, &nvme_subsystems, entry) {
2929 		if (strcmp(subsys->subnqn, subsysnqn))
2930 			continue;
2931 		if (!kref_get_unless_zero(&subsys->ref))
2932 			continue;
2933 		return subsys;
2934 	}
2935 
2936 	return NULL;
2937 }
2938 
2939 static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2940 {
2941 	return ctrl->opts && ctrl->opts->discovery_nqn;
2942 }
2943 
2944 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2945 		struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2946 {
2947 	struct nvme_ctrl *tmp;
2948 
2949 	lockdep_assert_held(&nvme_subsystems_lock);
2950 
2951 	list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2952 		if (nvme_state_terminal(tmp))
2953 			continue;
2954 
2955 		if (tmp->cntlid == ctrl->cntlid) {
2956 			dev_err(ctrl->device,
2957 				"Duplicate cntlid %u with %s, subsys %s, rejecting\n",
2958 				ctrl->cntlid, dev_name(tmp->device),
2959 				subsys->subnqn);
2960 			return false;
2961 		}
2962 
2963 		if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) ||
2964 		    nvme_discovery_ctrl(ctrl))
2965 			continue;
2966 
2967 		dev_err(ctrl->device,
2968 			"Subsystem does not support multiple controllers\n");
2969 		return false;
2970 	}
2971 
2972 	return true;
2973 }
2974 
2975 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2976 {
2977 	struct nvme_subsystem *subsys, *found;
2978 	int ret;
2979 
2980 	subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2981 	if (!subsys)
2982 		return -ENOMEM;
2983 
2984 	subsys->instance = -1;
2985 	mutex_init(&subsys->lock);
2986 	kref_init(&subsys->ref);
2987 	INIT_LIST_HEAD(&subsys->ctrls);
2988 	INIT_LIST_HEAD(&subsys->nsheads);
2989 	nvme_init_subnqn(subsys, ctrl, id);
2990 	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2991 	memcpy(subsys->model, id->mn, sizeof(subsys->model));
2992 	subsys->vendor_id = le16_to_cpu(id->vid);
2993 	subsys->cmic = id->cmic;
2994 
2995 	/* Versions prior to 1.4 don't necessarily report a valid type */
2996 	if (id->cntrltype == NVME_CTRL_DISC ||
2997 	    !strcmp(subsys->subnqn, NVME_DISC_SUBSYS_NAME))
2998 		subsys->subtype = NVME_NQN_DISC;
2999 	else
3000 		subsys->subtype = NVME_NQN_NVME;
3001 
3002 	if (nvme_discovery_ctrl(ctrl) && subsys->subtype != NVME_NQN_DISC) {
3003 		dev_err(ctrl->device,
3004 			"Subsystem %s is not a discovery controller",
3005 			subsys->subnqn);
3006 		kfree(subsys);
3007 		return -EINVAL;
3008 	}
3009 	subsys->awupf = le16_to_cpu(id->awupf);
3010 	nvme_mpath_default_iopolicy(subsys);
3011 
3012 	subsys->dev.class = &nvme_subsys_class;
3013 	subsys->dev.release = nvme_release_subsystem;
3014 	subsys->dev.groups = nvme_subsys_attrs_groups;
3015 	dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
3016 	device_initialize(&subsys->dev);
3017 
3018 	mutex_lock(&nvme_subsystems_lock);
3019 	found = __nvme_find_get_subsystem(subsys->subnqn);
3020 	if (found) {
3021 		put_device(&subsys->dev);
3022 		subsys = found;
3023 
3024 		if (!nvme_validate_cntlid(subsys, ctrl, id)) {
3025 			ret = -EINVAL;
3026 			goto out_put_subsystem;
3027 		}
3028 	} else {
3029 		ret = device_add(&subsys->dev);
3030 		if (ret) {
3031 			dev_err(ctrl->device,
3032 				"failed to register subsystem device.\n");
3033 			put_device(&subsys->dev);
3034 			goto out_unlock;
3035 		}
3036 		ida_init(&subsys->ns_ida);
3037 		list_add_tail(&subsys->entry, &nvme_subsystems);
3038 	}
3039 
3040 	ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
3041 				dev_name(ctrl->device));
3042 	if (ret) {
3043 		dev_err(ctrl->device,
3044 			"failed to create sysfs link from subsystem.\n");
3045 		goto out_put_subsystem;
3046 	}
3047 
3048 	if (!found)
3049 		subsys->instance = ctrl->instance;
3050 	ctrl->subsys = subsys;
3051 	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
3052 	mutex_unlock(&nvme_subsystems_lock);
3053 	return 0;
3054 
3055 out_put_subsystem:
3056 	nvme_put_subsystem(subsys);
3057 out_unlock:
3058 	mutex_unlock(&nvme_subsystems_lock);
3059 	return ret;
3060 }
3061 
3062 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
3063 		void *log, size_t size, u64 offset)
3064 {
3065 	struct nvme_command c = { };
3066 	u32 dwlen = nvme_bytes_to_numd(size);
3067 
3068 	c.get_log_page.opcode = nvme_admin_get_log_page;
3069 	c.get_log_page.nsid = cpu_to_le32(nsid);
3070 	c.get_log_page.lid = log_page;
3071 	c.get_log_page.lsp = lsp;
3072 	c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
3073 	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
3074 	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
3075 	c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
3076 	c.get_log_page.csi = csi;
3077 
3078 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
3079 }
3080 
3081 static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
3082 				struct nvme_effects_log **log)
3083 {
3084 	struct nvme_effects_log	*cel = xa_load(&ctrl->cels, csi);
3085 	int ret;
3086 
3087 	if (cel)
3088 		goto out;
3089 
3090 	cel = kzalloc(sizeof(*cel), GFP_KERNEL);
3091 	if (!cel)
3092 		return -ENOMEM;
3093 
3094 	ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
3095 			cel, sizeof(*cel), 0);
3096 	if (ret) {
3097 		kfree(cel);
3098 		return ret;
3099 	}
3100 
3101 	xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
3102 out:
3103 	*log = cel;
3104 	return 0;
3105 }
3106 
3107 static inline u32 nvme_mps_to_sectors(struct nvme_ctrl *ctrl, u32 units)
3108 {
3109 	u32 page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12, val;
3110 
3111 	if (check_shl_overflow(1U, units + page_shift - 9, &val))
3112 		return UINT_MAX;
3113 	return val;
3114 }
3115 
3116 static int nvme_init_non_mdts_limits(struct nvme_ctrl *ctrl)
3117 {
3118 	struct nvme_command c = { };
3119 	struct nvme_id_ctrl_nvm *id;
3120 	int ret;
3121 
3122 	/*
3123 	 * Even though NVMe spec explicitly states that MDTS is not applicable
3124 	 * to the write-zeroes, we are cautious and limit the size to the
3125 	 * controllers max_hw_sectors value, which is based on the MDTS field
3126 	 * and possibly other limiting factors.
3127 	 */
3128 	if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
3129 	    !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
3130 		ctrl->max_zeroes_sectors = ctrl->max_hw_sectors;
3131 	else
3132 		ctrl->max_zeroes_sectors = 0;
3133 
3134 	if (ctrl->subsys->subtype != NVME_NQN_NVME ||
3135 	    !nvme_id_cns_ok(ctrl, NVME_ID_CNS_CS_CTRL) ||
3136 	    test_bit(NVME_CTRL_SKIP_ID_CNS_CS, &ctrl->flags))
3137 		return 0;
3138 
3139 	id = kzalloc(sizeof(*id), GFP_KERNEL);
3140 	if (!id)
3141 		return -ENOMEM;
3142 
3143 	c.identify.opcode = nvme_admin_identify;
3144 	c.identify.cns = NVME_ID_CNS_CS_CTRL;
3145 	c.identify.csi = NVME_CSI_NVM;
3146 
3147 	ret = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
3148 	if (ret)
3149 		goto free_data;
3150 
3151 	ctrl->dmrl = id->dmrl;
3152 	ctrl->dmrsl = le32_to_cpu(id->dmrsl);
3153 	if (id->wzsl)
3154 		ctrl->max_zeroes_sectors = nvme_mps_to_sectors(ctrl, id->wzsl);
3155 
3156 free_data:
3157 	if (ret > 0)
3158 		set_bit(NVME_CTRL_SKIP_ID_CNS_CS, &ctrl->flags);
3159 	kfree(id);
3160 	return ret;
3161 }
3162 
3163 static void nvme_init_known_nvm_effects(struct nvme_ctrl *ctrl)
3164 {
3165 	struct nvme_effects_log	*log = ctrl->effects;
3166 
3167 	log->acs[nvme_admin_format_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3168 						NVME_CMD_EFFECTS_NCC |
3169 						NVME_CMD_EFFECTS_CSE_MASK);
3170 	log->acs[nvme_admin_sanitize_nvm] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC |
3171 						NVME_CMD_EFFECTS_CSE_MASK);
3172 
3173 	/*
3174 	 * The spec says the result of a security receive command depends on
3175 	 * the previous security send command. As such, many vendors log this
3176 	 * command as one to submitted only when no other commands to the same
3177 	 * namespace are outstanding. The intention is to tell the host to
3178 	 * prevent mixing security send and receive.
3179 	 *
3180 	 * This driver can only enforce such exclusive access against IO
3181 	 * queues, though. We are not readily able to enforce such a rule for
3182 	 * two commands to the admin queue, which is the only queue that
3183 	 * matters for this command.
3184 	 *
3185 	 * Rather than blindly freezing the IO queues for this effect that
3186 	 * doesn't even apply to IO, mask it off.
3187 	 */
3188 	log->acs[nvme_admin_security_recv] &= cpu_to_le32(~NVME_CMD_EFFECTS_CSE_MASK);
3189 
3190 	log->iocs[nvme_cmd_write] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3191 	log->iocs[nvme_cmd_write_zeroes] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3192 	log->iocs[nvme_cmd_write_uncor] |= cpu_to_le32(NVME_CMD_EFFECTS_LBCC);
3193 }
3194 
3195 static int nvme_init_effects(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
3196 {
3197 	int ret = 0;
3198 
3199 	if (ctrl->effects)
3200 		return 0;
3201 
3202 	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
3203 		ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
3204 		if (ret < 0)
3205 			return ret;
3206 	}
3207 
3208 	if (!ctrl->effects) {
3209 		ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
3210 		if (!ctrl->effects)
3211 			return -ENOMEM;
3212 		xa_store(&ctrl->cels, NVME_CSI_NVM, ctrl->effects, GFP_KERNEL);
3213 	}
3214 
3215 	nvme_init_known_nvm_effects(ctrl);
3216 	return 0;
3217 }
3218 
3219 static int nvme_check_ctrl_fabric_info(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
3220 {
3221 	/*
3222 	 * In fabrics we need to verify the cntlid matches the
3223 	 * admin connect
3224 	 */
3225 	if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
3226 		dev_err(ctrl->device,
3227 			"Mismatching cntlid: Connect %u vs Identify %u, rejecting\n",
3228 			ctrl->cntlid, le16_to_cpu(id->cntlid));
3229 		return -EINVAL;
3230 	}
3231 
3232 	if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
3233 		dev_err(ctrl->device,
3234 			"keep-alive support is mandatory for fabrics\n");
3235 		return -EINVAL;
3236 	}
3237 
3238 	if (!nvme_discovery_ctrl(ctrl) && ctrl->ioccsz < 4) {
3239 		dev_err(ctrl->device,
3240 			"I/O queue command capsule supported size %d < 4\n",
3241 			ctrl->ioccsz);
3242 		return -EINVAL;
3243 	}
3244 
3245 	if (!nvme_discovery_ctrl(ctrl) && ctrl->iorcsz < 1) {
3246 		dev_err(ctrl->device,
3247 			"I/O queue response capsule supported size %d < 1\n",
3248 			ctrl->iorcsz);
3249 		return -EINVAL;
3250 	}
3251 
3252 	if (!ctrl->maxcmd) {
3253 		dev_err(ctrl->device, "Maximum outstanding commands is 0\n");
3254 		return -EINVAL;
3255 	}
3256 
3257 	return 0;
3258 }
3259 
3260 static int nvme_init_identify(struct nvme_ctrl *ctrl)
3261 {
3262 	struct queue_limits lim;
3263 	struct nvme_id_ctrl *id;
3264 	u32 max_hw_sectors;
3265 	bool prev_apst_enabled;
3266 	int ret;
3267 
3268 	ret = nvme_identify_ctrl(ctrl, &id);
3269 	if (ret) {
3270 		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
3271 		return -EIO;
3272 	}
3273 
3274 	if (!(ctrl->ops->flags & NVME_F_FABRICS))
3275 		ctrl->cntlid = le16_to_cpu(id->cntlid);
3276 
3277 	if (!ctrl->identified) {
3278 		unsigned int i;
3279 
3280 		/*
3281 		 * Check for quirks.  Quirk can depend on firmware version,
3282 		 * so, in principle, the set of quirks present can change
3283 		 * across a reset.  As a possible future enhancement, we
3284 		 * could re-scan for quirks every time we reinitialize
3285 		 * the device, but we'd have to make sure that the driver
3286 		 * behaves intelligently if the quirks change.
3287 		 */
3288 		for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
3289 			if (quirk_matches(id, &core_quirks[i]))
3290 				ctrl->quirks |= core_quirks[i].quirks;
3291 		}
3292 
3293 		ret = nvme_init_subsystem(ctrl, id);
3294 		if (ret)
3295 			goto out_free;
3296 
3297 		ret = nvme_init_effects(ctrl, id);
3298 		if (ret)
3299 			goto out_free;
3300 	}
3301 	memcpy(ctrl->subsys->firmware_rev, id->fr,
3302 	       sizeof(ctrl->subsys->firmware_rev));
3303 
3304 	if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
3305 		dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
3306 		ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
3307 	}
3308 
3309 	ctrl->crdt[0] = le16_to_cpu(id->crdt1);
3310 	ctrl->crdt[1] = le16_to_cpu(id->crdt2);
3311 	ctrl->crdt[2] = le16_to_cpu(id->crdt3);
3312 
3313 	ctrl->oacs = le16_to_cpu(id->oacs);
3314 	ctrl->oncs = le16_to_cpu(id->oncs);
3315 	ctrl->mtfa = le16_to_cpu(id->mtfa);
3316 	ctrl->oaes = le32_to_cpu(id->oaes);
3317 	ctrl->wctemp = le16_to_cpu(id->wctemp);
3318 	ctrl->cctemp = le16_to_cpu(id->cctemp);
3319 
3320 	atomic_set(&ctrl->abort_limit, id->acl + 1);
3321 	ctrl->vwc = id->vwc;
3322 	if (id->mdts)
3323 		max_hw_sectors = nvme_mps_to_sectors(ctrl, id->mdts);
3324 	else
3325 		max_hw_sectors = UINT_MAX;
3326 	ctrl->max_hw_sectors =
3327 		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
3328 
3329 	lim = queue_limits_start_update(ctrl->admin_q);
3330 	nvme_set_ctrl_limits(ctrl, &lim);
3331 	ret = queue_limits_commit_update(ctrl->admin_q, &lim);
3332 	if (ret)
3333 		goto out_free;
3334 
3335 	ctrl->sgls = le32_to_cpu(id->sgls);
3336 	ctrl->kas = le16_to_cpu(id->kas);
3337 	ctrl->max_namespaces = le32_to_cpu(id->mnan);
3338 	ctrl->ctratt = le32_to_cpu(id->ctratt);
3339 
3340 	ctrl->cntrltype = id->cntrltype;
3341 	ctrl->dctype = id->dctype;
3342 
3343 	if (id->rtd3e) {
3344 		/* us -> s */
3345 		u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
3346 
3347 		ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
3348 						 shutdown_timeout, 60);
3349 
3350 		if (ctrl->shutdown_timeout != shutdown_timeout)
3351 			dev_info(ctrl->device,
3352 				 "D3 entry latency set to %u seconds\n",
3353 				 ctrl->shutdown_timeout);
3354 	} else
3355 		ctrl->shutdown_timeout = shutdown_timeout;
3356 
3357 	ctrl->npss = id->npss;
3358 	ctrl->apsta = id->apsta;
3359 	prev_apst_enabled = ctrl->apst_enabled;
3360 	if (ctrl->quirks & NVME_QUIRK_NO_APST) {
3361 		if (force_apst && id->apsta) {
3362 			dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
3363 			ctrl->apst_enabled = true;
3364 		} else {
3365 			ctrl->apst_enabled = false;
3366 		}
3367 	} else {
3368 		ctrl->apst_enabled = id->apsta;
3369 	}
3370 	memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
3371 
3372 	if (ctrl->ops->flags & NVME_F_FABRICS) {
3373 		ctrl->icdoff = le16_to_cpu(id->icdoff);
3374 		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
3375 		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
3376 		ctrl->maxcmd = le16_to_cpu(id->maxcmd);
3377 
3378 		ret = nvme_check_ctrl_fabric_info(ctrl, id);
3379 		if (ret)
3380 			goto out_free;
3381 	} else {
3382 		ctrl->hmpre = le32_to_cpu(id->hmpre);
3383 		ctrl->hmmin = le32_to_cpu(id->hmmin);
3384 		ctrl->hmminds = le32_to_cpu(id->hmminds);
3385 		ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
3386 	}
3387 
3388 	ret = nvme_mpath_init_identify(ctrl, id);
3389 	if (ret < 0)
3390 		goto out_free;
3391 
3392 	if (ctrl->apst_enabled && !prev_apst_enabled)
3393 		dev_pm_qos_expose_latency_tolerance(ctrl->device);
3394 	else if (!ctrl->apst_enabled && prev_apst_enabled)
3395 		dev_pm_qos_hide_latency_tolerance(ctrl->device);
3396 
3397 out_free:
3398 	kfree(id);
3399 	return ret;
3400 }
3401 
3402 /*
3403  * Initialize the cached copies of the Identify data and various controller
3404  * register in our nvme_ctrl structure.  This should be called as soon as
3405  * the admin queue is fully up and running.
3406  */
3407 int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended)
3408 {
3409 	int ret;
3410 
3411 	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
3412 	if (ret) {
3413 		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
3414 		return ret;
3415 	}
3416 
3417 	ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
3418 
3419 	if (ctrl->vs >= NVME_VS(1, 1, 0))
3420 		ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
3421 
3422 	ret = nvme_init_identify(ctrl);
3423 	if (ret)
3424 		return ret;
3425 
3426 	ret = nvme_configure_apst(ctrl);
3427 	if (ret < 0)
3428 		return ret;
3429 
3430 	ret = nvme_configure_timestamp(ctrl);
3431 	if (ret < 0)
3432 		return ret;
3433 
3434 	ret = nvme_configure_host_options(ctrl);
3435 	if (ret < 0)
3436 		return ret;
3437 
3438 	nvme_configure_opal(ctrl, was_suspended);
3439 
3440 	if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
3441 		/*
3442 		 * Do not return errors unless we are in a controller reset,
3443 		 * the controller works perfectly fine without hwmon.
3444 		 */
3445 		ret = nvme_hwmon_init(ctrl);
3446 		if (ret == -EINTR)
3447 			return ret;
3448 	}
3449 
3450 	clear_bit(NVME_CTRL_DIRTY_CAPABILITY, &ctrl->flags);
3451 	ctrl->identified = true;
3452 
3453 	nvme_start_keep_alive(ctrl);
3454 
3455 	return 0;
3456 }
3457 EXPORT_SYMBOL_GPL(nvme_init_ctrl_finish);
3458 
3459 static int nvme_dev_open(struct inode *inode, struct file *file)
3460 {
3461 	struct nvme_ctrl *ctrl =
3462 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3463 
3464 	switch (nvme_ctrl_state(ctrl)) {
3465 	case NVME_CTRL_LIVE:
3466 		break;
3467 	default:
3468 		return -EWOULDBLOCK;
3469 	}
3470 
3471 	nvme_get_ctrl(ctrl);
3472 	if (!try_module_get(ctrl->ops->module)) {
3473 		nvme_put_ctrl(ctrl);
3474 		return -EINVAL;
3475 	}
3476 
3477 	file->private_data = ctrl;
3478 	return 0;
3479 }
3480 
3481 static int nvme_dev_release(struct inode *inode, struct file *file)
3482 {
3483 	struct nvme_ctrl *ctrl =
3484 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
3485 
3486 	module_put(ctrl->ops->module);
3487 	nvme_put_ctrl(ctrl);
3488 	return 0;
3489 }
3490 
3491 static const struct file_operations nvme_dev_fops = {
3492 	.owner		= THIS_MODULE,
3493 	.open		= nvme_dev_open,
3494 	.release	= nvme_dev_release,
3495 	.unlocked_ioctl	= nvme_dev_ioctl,
3496 	.compat_ioctl	= compat_ptr_ioctl,
3497 	.uring_cmd	= nvme_dev_uring_cmd,
3498 };
3499 
3500 static struct nvme_ns_head *nvme_find_ns_head(struct nvme_ctrl *ctrl,
3501 		unsigned nsid)
3502 {
3503 	struct nvme_ns_head *h;
3504 
3505 	lockdep_assert_held(&ctrl->subsys->lock);
3506 
3507 	list_for_each_entry(h, &ctrl->subsys->nsheads, entry) {
3508 		/*
3509 		 * Private namespaces can share NSIDs under some conditions.
3510 		 * In that case we can't use the same ns_head for namespaces
3511 		 * with the same NSID.
3512 		 */
3513 		if (h->ns_id != nsid || !nvme_is_unique_nsid(ctrl, h))
3514 			continue;
3515 		if (!list_empty(&h->list) && nvme_tryget_ns_head(h))
3516 			return h;
3517 	}
3518 
3519 	return NULL;
3520 }
3521 
3522 static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
3523 		struct nvme_ns_ids *ids)
3524 {
3525 	bool has_uuid = !uuid_is_null(&ids->uuid);
3526 	bool has_nguid = memchr_inv(ids->nguid, 0, sizeof(ids->nguid));
3527 	bool has_eui64 = memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
3528 	struct nvme_ns_head *h;
3529 
3530 	lockdep_assert_held(&subsys->lock);
3531 
3532 	list_for_each_entry(h, &subsys->nsheads, entry) {
3533 		if (has_uuid && uuid_equal(&ids->uuid, &h->ids.uuid))
3534 			return -EINVAL;
3535 		if (has_nguid &&
3536 		    memcmp(&ids->nguid, &h->ids.nguid, sizeof(ids->nguid)) == 0)
3537 			return -EINVAL;
3538 		if (has_eui64 &&
3539 		    memcmp(&ids->eui64, &h->ids.eui64, sizeof(ids->eui64)) == 0)
3540 			return -EINVAL;
3541 	}
3542 
3543 	return 0;
3544 }
3545 
3546 static void nvme_cdev_rel(struct device *dev)
3547 {
3548 	ida_free(&nvme_ns_chr_minor_ida, MINOR(dev->devt));
3549 }
3550 
3551 void nvme_cdev_del(struct cdev *cdev, struct device *cdev_device)
3552 {
3553 	cdev_device_del(cdev, cdev_device);
3554 	put_device(cdev_device);
3555 }
3556 
3557 int nvme_cdev_add(struct cdev *cdev, struct device *cdev_device,
3558 		const struct file_operations *fops, struct module *owner)
3559 {
3560 	int minor, ret;
3561 
3562 	minor = ida_alloc(&nvme_ns_chr_minor_ida, GFP_KERNEL);
3563 	if (minor < 0)
3564 		return minor;
3565 	cdev_device->devt = MKDEV(MAJOR(nvme_ns_chr_devt), minor);
3566 	cdev_device->class = &nvme_ns_chr_class;
3567 	cdev_device->release = nvme_cdev_rel;
3568 	device_initialize(cdev_device);
3569 	cdev_init(cdev, fops);
3570 	cdev->owner = owner;
3571 	ret = cdev_device_add(cdev, cdev_device);
3572 	if (ret)
3573 		put_device(cdev_device);
3574 
3575 	return ret;
3576 }
3577 
3578 static int nvme_ns_chr_open(struct inode *inode, struct file *file)
3579 {
3580 	return nvme_ns_open(container_of(inode->i_cdev, struct nvme_ns, cdev));
3581 }
3582 
3583 static int nvme_ns_chr_release(struct inode *inode, struct file *file)
3584 {
3585 	nvme_ns_release(container_of(inode->i_cdev, struct nvme_ns, cdev));
3586 	return 0;
3587 }
3588 
3589 static const struct file_operations nvme_ns_chr_fops = {
3590 	.owner		= THIS_MODULE,
3591 	.open		= nvme_ns_chr_open,
3592 	.release	= nvme_ns_chr_release,
3593 	.unlocked_ioctl	= nvme_ns_chr_ioctl,
3594 	.compat_ioctl	= compat_ptr_ioctl,
3595 	.uring_cmd	= nvme_ns_chr_uring_cmd,
3596 	.uring_cmd_iopoll = nvme_ns_chr_uring_cmd_iopoll,
3597 };
3598 
3599 static int nvme_add_ns_cdev(struct nvme_ns *ns)
3600 {
3601 	int ret;
3602 
3603 	ns->cdev_device.parent = ns->ctrl->device;
3604 	ret = dev_set_name(&ns->cdev_device, "ng%dn%d",
3605 			   ns->ctrl->instance, ns->head->instance);
3606 	if (ret)
3607 		return ret;
3608 
3609 	return nvme_cdev_add(&ns->cdev, &ns->cdev_device, &nvme_ns_chr_fops,
3610 			     ns->ctrl->ops->module);
3611 }
3612 
3613 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3614 		struct nvme_ns_info *info)
3615 {
3616 	struct nvme_ns_head *head;
3617 	size_t size = sizeof(*head);
3618 	int ret = -ENOMEM;
3619 
3620 #ifdef CONFIG_NVME_MULTIPATH
3621 	size += num_possible_nodes() * sizeof(struct nvme_ns *);
3622 #endif
3623 
3624 	head = kzalloc(size, GFP_KERNEL);
3625 	if (!head)
3626 		goto out;
3627 	ret = ida_alloc_min(&ctrl->subsys->ns_ida, 1, GFP_KERNEL);
3628 	if (ret < 0)
3629 		goto out_free_head;
3630 	head->instance = ret;
3631 	INIT_LIST_HEAD(&head->list);
3632 	ret = init_srcu_struct(&head->srcu);
3633 	if (ret)
3634 		goto out_ida_remove;
3635 	head->subsys = ctrl->subsys;
3636 	head->ns_id = info->nsid;
3637 	head->ids = info->ids;
3638 	head->shared = info->is_shared;
3639 	ratelimit_state_init(&head->rs_nuse, 5 * HZ, 1);
3640 	ratelimit_set_flags(&head->rs_nuse, RATELIMIT_MSG_ON_RELEASE);
3641 	kref_init(&head->ref);
3642 
3643 	if (head->ids.csi) {
3644 		ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
3645 		if (ret)
3646 			goto out_cleanup_srcu;
3647 	} else
3648 		head->effects = ctrl->effects;
3649 
3650 	ret = nvme_mpath_alloc_disk(ctrl, head);
3651 	if (ret)
3652 		goto out_cleanup_srcu;
3653 
3654 	list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3655 
3656 	kref_get(&ctrl->subsys->ref);
3657 
3658 	return head;
3659 out_cleanup_srcu:
3660 	cleanup_srcu_struct(&head->srcu);
3661 out_ida_remove:
3662 	ida_free(&ctrl->subsys->ns_ida, head->instance);
3663 out_free_head:
3664 	kfree(head);
3665 out:
3666 	if (ret > 0)
3667 		ret = blk_status_to_errno(nvme_error_status(ret));
3668 	return ERR_PTR(ret);
3669 }
3670 
3671 static int nvme_global_check_duplicate_ids(struct nvme_subsystem *this,
3672 		struct nvme_ns_ids *ids)
3673 {
3674 	struct nvme_subsystem *s;
3675 	int ret = 0;
3676 
3677 	/*
3678 	 * Note that this check is racy as we try to avoid holding the global
3679 	 * lock over the whole ns_head creation.  But it is only intended as
3680 	 * a sanity check anyway.
3681 	 */
3682 	mutex_lock(&nvme_subsystems_lock);
3683 	list_for_each_entry(s, &nvme_subsystems, entry) {
3684 		if (s == this)
3685 			continue;
3686 		mutex_lock(&s->lock);
3687 		ret = nvme_subsys_check_duplicate_ids(s, ids);
3688 		mutex_unlock(&s->lock);
3689 		if (ret)
3690 			break;
3691 	}
3692 	mutex_unlock(&nvme_subsystems_lock);
3693 
3694 	return ret;
3695 }
3696 
3697 static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
3698 {
3699 	struct nvme_ctrl *ctrl = ns->ctrl;
3700 	struct nvme_ns_head *head = NULL;
3701 	int ret;
3702 
3703 	ret = nvme_global_check_duplicate_ids(ctrl->subsys, &info->ids);
3704 	if (ret) {
3705 		/*
3706 		 * We've found two different namespaces on two different
3707 		 * subsystems that report the same ID.  This is pretty nasty
3708 		 * for anything that actually requires unique device
3709 		 * identification.  In the kernel we need this for multipathing,
3710 		 * and in user space the /dev/disk/by-id/ links rely on it.
3711 		 *
3712 		 * If the device also claims to be multi-path capable back off
3713 		 * here now and refuse the probe the second device as this is a
3714 		 * recipe for data corruption.  If not this is probably a
3715 		 * cheap consumer device if on the PCIe bus, so let the user
3716 		 * proceed and use the shiny toy, but warn that with changing
3717 		 * probing order (which due to our async probing could just be
3718 		 * device taking longer to startup) the other device could show
3719 		 * up at any time.
3720 		 */
3721 		nvme_print_device_info(ctrl);
3722 		if ((ns->ctrl->ops->flags & NVME_F_FABRICS) || /* !PCIe */
3723 		    ((ns->ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) &&
3724 		     info->is_shared)) {
3725 			dev_err(ctrl->device,
3726 				"ignoring nsid %d because of duplicate IDs\n",
3727 				info->nsid);
3728 			return ret;
3729 		}
3730 
3731 		dev_err(ctrl->device,
3732 			"clearing duplicate IDs for nsid %d\n", info->nsid);
3733 		dev_err(ctrl->device,
3734 			"use of /dev/disk/by-id/ may cause data corruption\n");
3735 		memset(&info->ids.nguid, 0, sizeof(info->ids.nguid));
3736 		memset(&info->ids.uuid, 0, sizeof(info->ids.uuid));
3737 		memset(&info->ids.eui64, 0, sizeof(info->ids.eui64));
3738 		ctrl->quirks |= NVME_QUIRK_BOGUS_NID;
3739 	}
3740 
3741 	mutex_lock(&ctrl->subsys->lock);
3742 	head = nvme_find_ns_head(ctrl, info->nsid);
3743 	if (!head) {
3744 		ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &info->ids);
3745 		if (ret) {
3746 			dev_err(ctrl->device,
3747 				"duplicate IDs in subsystem for nsid %d\n",
3748 				info->nsid);
3749 			goto out_unlock;
3750 		}
3751 		head = nvme_alloc_ns_head(ctrl, info);
3752 		if (IS_ERR(head)) {
3753 			ret = PTR_ERR(head);
3754 			goto out_unlock;
3755 		}
3756 	} else {
3757 		ret = -EINVAL;
3758 		if (!info->is_shared || !head->shared) {
3759 			dev_err(ctrl->device,
3760 				"Duplicate unshared namespace %d\n",
3761 				info->nsid);
3762 			goto out_put_ns_head;
3763 		}
3764 		if (!nvme_ns_ids_equal(&head->ids, &info->ids)) {
3765 			dev_err(ctrl->device,
3766 				"IDs don't match for shared namespace %d\n",
3767 					info->nsid);
3768 			goto out_put_ns_head;
3769 		}
3770 
3771 		if (!multipath) {
3772 			dev_warn(ctrl->device,
3773 				"Found shared namespace %d, but multipathing not supported.\n",
3774 				info->nsid);
3775 			dev_warn_once(ctrl->device,
3776 				"Support for shared namespaces without CONFIG_NVME_MULTIPATH is deprecated and will be removed in Linux 6.0.\n");
3777 		}
3778 	}
3779 
3780 	list_add_tail_rcu(&ns->siblings, &head->list);
3781 	ns->head = head;
3782 	mutex_unlock(&ctrl->subsys->lock);
3783 	return 0;
3784 
3785 out_put_ns_head:
3786 	nvme_put_ns_head(head);
3787 out_unlock:
3788 	mutex_unlock(&ctrl->subsys->lock);
3789 	return ret;
3790 }
3791 
3792 struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3793 {
3794 	struct nvme_ns *ns, *ret = NULL;
3795 	int srcu_idx;
3796 
3797 	srcu_idx = srcu_read_lock(&ctrl->srcu);
3798 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
3799 		if (ns->head->ns_id == nsid) {
3800 			if (!nvme_get_ns(ns))
3801 				continue;
3802 			ret = ns;
3803 			break;
3804 		}
3805 		if (ns->head->ns_id > nsid)
3806 			break;
3807 	}
3808 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
3809 	return ret;
3810 }
3811 EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3812 
3813 /*
3814  * Add the namespace to the controller list while keeping the list ordered.
3815  */
3816 static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
3817 {
3818 	struct nvme_ns *tmp;
3819 
3820 	list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
3821 		if (tmp->head->ns_id < ns->head->ns_id) {
3822 			list_add_rcu(&ns->list, &tmp->list);
3823 			return;
3824 		}
3825 	}
3826 	list_add(&ns->list, &ns->ctrl->namespaces);
3827 }
3828 
3829 static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
3830 {
3831 	struct queue_limits lim = { };
3832 	struct nvme_ns *ns;
3833 	struct gendisk *disk;
3834 	int node = ctrl->numa_node;
3835 
3836 	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3837 	if (!ns)
3838 		return;
3839 
3840 	if (ctrl->opts && ctrl->opts->data_digest)
3841 		lim.features |= BLK_FEAT_STABLE_WRITES;
3842 	if (ctrl->ops->supports_pci_p2pdma &&
3843 	    ctrl->ops->supports_pci_p2pdma(ctrl))
3844 		lim.features |= BLK_FEAT_PCI_P2PDMA;
3845 
3846 	disk = blk_mq_alloc_disk(ctrl->tagset, &lim, ns);
3847 	if (IS_ERR(disk))
3848 		goto out_free_ns;
3849 	disk->fops = &nvme_bdev_ops;
3850 	disk->private_data = ns;
3851 
3852 	ns->disk = disk;
3853 	ns->queue = disk->queue;
3854 	ns->ctrl = ctrl;
3855 	kref_init(&ns->kref);
3856 
3857 	if (nvme_init_ns_head(ns, info))
3858 		goto out_cleanup_disk;
3859 
3860 	/*
3861 	 * If multipathing is enabled, the device name for all disks and not
3862 	 * just those that represent shared namespaces needs to be based on the
3863 	 * subsystem instance.  Using the controller instance for private
3864 	 * namespaces could lead to naming collisions between shared and private
3865 	 * namespaces if they don't use a common numbering scheme.
3866 	 *
3867 	 * If multipathing is not enabled, disk names must use the controller
3868 	 * instance as shared namespaces will show up as multiple block
3869 	 * devices.
3870 	 */
3871 	if (nvme_ns_head_multipath(ns->head)) {
3872 		sprintf(disk->disk_name, "nvme%dc%dn%d", ctrl->subsys->instance,
3873 			ctrl->instance, ns->head->instance);
3874 		disk->flags |= GENHD_FL_HIDDEN;
3875 	} else if (multipath) {
3876 		sprintf(disk->disk_name, "nvme%dn%d", ctrl->subsys->instance,
3877 			ns->head->instance);
3878 	} else {
3879 		sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance,
3880 			ns->head->instance);
3881 	}
3882 
3883 	if (nvme_update_ns_info(ns, info))
3884 		goto out_unlink_ns;
3885 
3886 	mutex_lock(&ctrl->namespaces_lock);
3887 	/*
3888 	 * Ensure that no namespaces are added to the ctrl list after the queues
3889 	 * are frozen, thereby avoiding a deadlock between scan and reset.
3890 	 */
3891 	if (test_bit(NVME_CTRL_FROZEN, &ctrl->flags)) {
3892 		mutex_unlock(&ctrl->namespaces_lock);
3893 		goto out_unlink_ns;
3894 	}
3895 	nvme_ns_add_to_ctrl_list(ns);
3896 	mutex_unlock(&ctrl->namespaces_lock);
3897 	synchronize_srcu(&ctrl->srcu);
3898 	nvme_get_ctrl(ctrl);
3899 
3900 	if (device_add_disk(ctrl->device, ns->disk, nvme_ns_attr_groups))
3901 		goto out_cleanup_ns_from_list;
3902 
3903 	if (!nvme_ns_head_multipath(ns->head))
3904 		nvme_add_ns_cdev(ns);
3905 
3906 	nvme_mpath_add_disk(ns, info->anagrpid);
3907 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3908 
3909 	/*
3910 	 * Set ns->disk->device->driver_data to ns so we can access
3911 	 * ns->head->passthru_err_log_enabled in
3912 	 * nvme_io_passthru_err_log_enabled_[store | show]().
3913 	 */
3914 	dev_set_drvdata(disk_to_dev(ns->disk), ns);
3915 
3916 	return;
3917 
3918  out_cleanup_ns_from_list:
3919 	nvme_put_ctrl(ctrl);
3920 	mutex_lock(&ctrl->namespaces_lock);
3921 	list_del_rcu(&ns->list);
3922 	mutex_unlock(&ctrl->namespaces_lock);
3923 	synchronize_srcu(&ctrl->srcu);
3924  out_unlink_ns:
3925 	mutex_lock(&ctrl->subsys->lock);
3926 	list_del_rcu(&ns->siblings);
3927 	if (list_empty(&ns->head->list))
3928 		list_del_init(&ns->head->entry);
3929 	mutex_unlock(&ctrl->subsys->lock);
3930 	nvme_put_ns_head(ns->head);
3931  out_cleanup_disk:
3932 	put_disk(disk);
3933  out_free_ns:
3934 	kfree(ns);
3935 }
3936 
3937 static void nvme_ns_remove(struct nvme_ns *ns)
3938 {
3939 	bool last_path = false;
3940 
3941 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3942 		return;
3943 
3944 	clear_bit(NVME_NS_READY, &ns->flags);
3945 	set_capacity(ns->disk, 0);
3946 	nvme_fault_inject_fini(&ns->fault_inject);
3947 
3948 	/*
3949 	 * Ensure that !NVME_NS_READY is seen by other threads to prevent
3950 	 * this ns going back into current_path.
3951 	 */
3952 	synchronize_srcu(&ns->head->srcu);
3953 
3954 	/* wait for concurrent submissions */
3955 	if (nvme_mpath_clear_current_path(ns))
3956 		synchronize_srcu(&ns->head->srcu);
3957 
3958 	mutex_lock(&ns->ctrl->subsys->lock);
3959 	list_del_rcu(&ns->siblings);
3960 	if (list_empty(&ns->head->list)) {
3961 		list_del_init(&ns->head->entry);
3962 		last_path = true;
3963 	}
3964 	mutex_unlock(&ns->ctrl->subsys->lock);
3965 
3966 	/* guarantee not available in head->list */
3967 	synchronize_srcu(&ns->head->srcu);
3968 
3969 	if (!nvme_ns_head_multipath(ns->head))
3970 		nvme_cdev_del(&ns->cdev, &ns->cdev_device);
3971 	del_gendisk(ns->disk);
3972 
3973 	mutex_lock(&ns->ctrl->namespaces_lock);
3974 	list_del_rcu(&ns->list);
3975 	mutex_unlock(&ns->ctrl->namespaces_lock);
3976 	synchronize_srcu(&ns->ctrl->srcu);
3977 
3978 	if (last_path)
3979 		nvme_mpath_shutdown_disk(ns->head);
3980 	nvme_put_ns(ns);
3981 }
3982 
3983 static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
3984 {
3985 	struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
3986 
3987 	if (ns) {
3988 		nvme_ns_remove(ns);
3989 		nvme_put_ns(ns);
3990 	}
3991 }
3992 
3993 static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info)
3994 {
3995 	int ret = NVME_SC_INVALID_NS | NVME_STATUS_DNR;
3996 
3997 	if (!nvme_ns_ids_equal(&ns->head->ids, &info->ids)) {
3998 		dev_err(ns->ctrl->device,
3999 			"identifiers changed for nsid %d\n", ns->head->ns_id);
4000 		goto out;
4001 	}
4002 
4003 	ret = nvme_update_ns_info(ns, info);
4004 out:
4005 	/*
4006 	 * Only remove the namespace if we got a fatal error back from the
4007 	 * device, otherwise ignore the error and just move on.
4008 	 *
4009 	 * TODO: we should probably schedule a delayed retry here.
4010 	 */
4011 	if (ret > 0 && (ret & NVME_STATUS_DNR))
4012 		nvme_ns_remove(ns);
4013 }
4014 
4015 static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid)
4016 {
4017 	struct nvme_ns_info info = { .nsid = nsid };
4018 	struct nvme_ns *ns;
4019 	int ret;
4020 
4021 	if (nvme_identify_ns_descs(ctrl, &info))
4022 		return;
4023 
4024 	if (info.ids.csi != NVME_CSI_NVM && !nvme_multi_css(ctrl)) {
4025 		dev_warn(ctrl->device,
4026 			"command set not reported for nsid: %d\n", nsid);
4027 		return;
4028 	}
4029 
4030 	/*
4031 	 * If available try to use the Command Set Idependent Identify Namespace
4032 	 * data structure to find all the generic information that is needed to
4033 	 * set up a namespace.  If not fall back to the legacy version.
4034 	 */
4035 	if ((ctrl->cap & NVME_CAP_CRMS_CRIMS) ||
4036 	    (info.ids.csi != NVME_CSI_NVM && info.ids.csi != NVME_CSI_ZNS))
4037 		ret = nvme_ns_info_from_id_cs_indep(ctrl, &info);
4038 	else
4039 		ret = nvme_ns_info_from_identify(ctrl, &info);
4040 
4041 	if (info.is_removed)
4042 		nvme_ns_remove_by_nsid(ctrl, nsid);
4043 
4044 	/*
4045 	 * Ignore the namespace if it is not ready. We will get an AEN once it
4046 	 * becomes ready and restart the scan.
4047 	 */
4048 	if (ret || !info.is_ready)
4049 		return;
4050 
4051 	ns = nvme_find_get_ns(ctrl, nsid);
4052 	if (ns) {
4053 		nvme_validate_ns(ns, &info);
4054 		nvme_put_ns(ns);
4055 	} else {
4056 		nvme_alloc_ns(ctrl, &info);
4057 	}
4058 }
4059 
4060 /**
4061  * struct async_scan_info - keeps track of controller & NSIDs to scan
4062  * @ctrl:	Controller on which namespaces are being scanned
4063  * @next_nsid:	Index of next NSID to scan in ns_list
4064  * @ns_list:	Pointer to list of NSIDs to scan
4065  *
4066  * Note: There is a single async_scan_info structure shared by all instances
4067  * of nvme_scan_ns_async() scanning a given controller, so the atomic
4068  * operations on next_nsid are critical to ensure each instance scans a unique
4069  * NSID.
4070  */
4071 struct async_scan_info {
4072 	struct nvme_ctrl *ctrl;
4073 	atomic_t next_nsid;
4074 	__le32 *ns_list;
4075 };
4076 
4077 static void nvme_scan_ns_async(void *data, async_cookie_t cookie)
4078 {
4079 	struct async_scan_info *scan_info = data;
4080 	int idx;
4081 	u32 nsid;
4082 
4083 	idx = (u32)atomic_fetch_inc(&scan_info->next_nsid);
4084 	nsid = le32_to_cpu(scan_info->ns_list[idx]);
4085 
4086 	nvme_scan_ns(scan_info->ctrl, nsid);
4087 }
4088 
4089 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
4090 					unsigned nsid)
4091 {
4092 	struct nvme_ns *ns, *next;
4093 	LIST_HEAD(rm_list);
4094 
4095 	mutex_lock(&ctrl->namespaces_lock);
4096 	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
4097 		if (ns->head->ns_id > nsid) {
4098 			list_del_rcu(&ns->list);
4099 			synchronize_srcu(&ctrl->srcu);
4100 			list_add_tail_rcu(&ns->list, &rm_list);
4101 		}
4102 	}
4103 	mutex_unlock(&ctrl->namespaces_lock);
4104 
4105 	list_for_each_entry_safe(ns, next, &rm_list, list)
4106 		nvme_ns_remove(ns);
4107 }
4108 
4109 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
4110 {
4111 	const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
4112 	__le32 *ns_list;
4113 	u32 prev = 0;
4114 	int ret = 0, i;
4115 	ASYNC_DOMAIN(domain);
4116 	struct async_scan_info scan_info;
4117 
4118 	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
4119 	if (!ns_list)
4120 		return -ENOMEM;
4121 
4122 	scan_info.ctrl = ctrl;
4123 	scan_info.ns_list = ns_list;
4124 	for (;;) {
4125 		struct nvme_command cmd = {
4126 			.identify.opcode	= nvme_admin_identify,
4127 			.identify.cns		= NVME_ID_CNS_NS_ACTIVE_LIST,
4128 			.identify.nsid		= cpu_to_le32(prev),
4129 		};
4130 
4131 		ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
4132 					    NVME_IDENTIFY_DATA_SIZE);
4133 		if (ret) {
4134 			dev_warn(ctrl->device,
4135 				"Identify NS List failed (status=0x%x)\n", ret);
4136 			goto free;
4137 		}
4138 
4139 		atomic_set(&scan_info.next_nsid, 0);
4140 		for (i = 0; i < nr_entries; i++) {
4141 			u32 nsid = le32_to_cpu(ns_list[i]);
4142 
4143 			if (!nsid)	/* end of the list? */
4144 				goto out;
4145 			async_schedule_domain(nvme_scan_ns_async, &scan_info,
4146 						&domain);
4147 			while (++prev < nsid)
4148 				nvme_ns_remove_by_nsid(ctrl, prev);
4149 		}
4150 		async_synchronize_full_domain(&domain);
4151 	}
4152  out:
4153 	nvme_remove_invalid_namespaces(ctrl, prev);
4154  free:
4155 	async_synchronize_full_domain(&domain);
4156 	kfree(ns_list);
4157 	return ret;
4158 }
4159 
4160 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
4161 {
4162 	struct nvme_id_ctrl *id;
4163 	u32 nn, i;
4164 
4165 	if (nvme_identify_ctrl(ctrl, &id))
4166 		return;
4167 	nn = le32_to_cpu(id->nn);
4168 	kfree(id);
4169 
4170 	for (i = 1; i <= nn; i++)
4171 		nvme_scan_ns(ctrl, i);
4172 
4173 	nvme_remove_invalid_namespaces(ctrl, nn);
4174 }
4175 
4176 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
4177 {
4178 	size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
4179 	__le32 *log;
4180 	int error;
4181 
4182 	log = kzalloc(log_size, GFP_KERNEL);
4183 	if (!log)
4184 		return;
4185 
4186 	/*
4187 	 * We need to read the log to clear the AEN, but we don't want to rely
4188 	 * on it for the changed namespace information as userspace could have
4189 	 * raced with us in reading the log page, which could cause us to miss
4190 	 * updates.
4191 	 */
4192 	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
4193 			NVME_CSI_NVM, log, log_size, 0);
4194 	if (error)
4195 		dev_warn(ctrl->device,
4196 			"reading changed ns log failed: %d\n", error);
4197 
4198 	kfree(log);
4199 }
4200 
4201 static void nvme_scan_work(struct work_struct *work)
4202 {
4203 	struct nvme_ctrl *ctrl =
4204 		container_of(work, struct nvme_ctrl, scan_work);
4205 	int ret;
4206 
4207 	/* No tagset on a live ctrl means IO queues could not created */
4208 	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE || !ctrl->tagset)
4209 		return;
4210 
4211 	/*
4212 	 * Identify controller limits can change at controller reset due to
4213 	 * new firmware download, even though it is not common we cannot ignore
4214 	 * such scenario. Controller's non-mdts limits are reported in the unit
4215 	 * of logical blocks that is dependent on the format of attached
4216 	 * namespace. Hence re-read the limits at the time of ns allocation.
4217 	 */
4218 	ret = nvme_init_non_mdts_limits(ctrl);
4219 	if (ret < 0) {
4220 		dev_warn(ctrl->device,
4221 			"reading non-mdts-limits failed: %d\n", ret);
4222 		return;
4223 	}
4224 
4225 	if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
4226 		dev_info(ctrl->device, "rescanning namespaces.\n");
4227 		nvme_clear_changed_ns_log(ctrl);
4228 	}
4229 
4230 	mutex_lock(&ctrl->scan_lock);
4231 	if (!nvme_id_cns_ok(ctrl, NVME_ID_CNS_NS_ACTIVE_LIST)) {
4232 		nvme_scan_ns_sequential(ctrl);
4233 	} else {
4234 		/*
4235 		 * Fall back to sequential scan if DNR is set to handle broken
4236 		 * devices which should support Identify NS List (as per the VS
4237 		 * they report) but don't actually support it.
4238 		 */
4239 		ret = nvme_scan_ns_list(ctrl);
4240 		if (ret > 0 && ret & NVME_STATUS_DNR)
4241 			nvme_scan_ns_sequential(ctrl);
4242 	}
4243 	mutex_unlock(&ctrl->scan_lock);
4244 }
4245 
4246 /*
4247  * This function iterates the namespace list unlocked to allow recovery from
4248  * controller failure. It is up to the caller to ensure the namespace list is
4249  * not modified by scan work while this function is executing.
4250  */
4251 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
4252 {
4253 	struct nvme_ns *ns, *next;
4254 	LIST_HEAD(ns_list);
4255 
4256 	/*
4257 	 * make sure to requeue I/O to all namespaces as these
4258 	 * might result from the scan itself and must complete
4259 	 * for the scan_work to make progress
4260 	 */
4261 	nvme_mpath_clear_ctrl_paths(ctrl);
4262 
4263 	/*
4264 	 * Unquiesce io queues so any pending IO won't hang, especially
4265 	 * those submitted from scan work
4266 	 */
4267 	nvme_unquiesce_io_queues(ctrl);
4268 
4269 	/* prevent racing with ns scanning */
4270 	flush_work(&ctrl->scan_work);
4271 
4272 	/*
4273 	 * The dead states indicates the controller was not gracefully
4274 	 * disconnected. In that case, we won't be able to flush any data while
4275 	 * removing the namespaces' disks; fail all the queues now to avoid
4276 	 * potentially having to clean up the failed sync later.
4277 	 */
4278 	if (nvme_ctrl_state(ctrl) == NVME_CTRL_DEAD)
4279 		nvme_mark_namespaces_dead(ctrl);
4280 
4281 	/* this is a no-op when called from the controller reset handler */
4282 	nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
4283 
4284 	mutex_lock(&ctrl->namespaces_lock);
4285 	list_splice_init_rcu(&ctrl->namespaces, &ns_list, synchronize_rcu);
4286 	mutex_unlock(&ctrl->namespaces_lock);
4287 	synchronize_srcu(&ctrl->srcu);
4288 
4289 	list_for_each_entry_safe(ns, next, &ns_list, list)
4290 		nvme_ns_remove(ns);
4291 }
4292 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
4293 
4294 static int nvme_class_uevent(const struct device *dev, struct kobj_uevent_env *env)
4295 {
4296 	const struct nvme_ctrl *ctrl =
4297 		container_of(dev, struct nvme_ctrl, ctrl_device);
4298 	struct nvmf_ctrl_options *opts = ctrl->opts;
4299 	int ret;
4300 
4301 	ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
4302 	if (ret)
4303 		return ret;
4304 
4305 	if (opts) {
4306 		ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
4307 		if (ret)
4308 			return ret;
4309 
4310 		ret = add_uevent_var(env, "NVME_TRSVCID=%s",
4311 				opts->trsvcid ?: "none");
4312 		if (ret)
4313 			return ret;
4314 
4315 		ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
4316 				opts->host_traddr ?: "none");
4317 		if (ret)
4318 			return ret;
4319 
4320 		ret = add_uevent_var(env, "NVME_HOST_IFACE=%s",
4321 				opts->host_iface ?: "none");
4322 	}
4323 	return ret;
4324 }
4325 
4326 static void nvme_change_uevent(struct nvme_ctrl *ctrl, char *envdata)
4327 {
4328 	char *envp[2] = { envdata, NULL };
4329 
4330 	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4331 }
4332 
4333 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
4334 {
4335 	char *envp[2] = { NULL, NULL };
4336 	u32 aen_result = ctrl->aen_result;
4337 
4338 	ctrl->aen_result = 0;
4339 	if (!aen_result)
4340 		return;
4341 
4342 	envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
4343 	if (!envp[0])
4344 		return;
4345 	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
4346 	kfree(envp[0]);
4347 }
4348 
4349 static void nvme_async_event_work(struct work_struct *work)
4350 {
4351 	struct nvme_ctrl *ctrl =
4352 		container_of(work, struct nvme_ctrl, async_event_work);
4353 
4354 	nvme_aen_uevent(ctrl);
4355 
4356 	/*
4357 	 * The transport drivers must guarantee AER submission here is safe by
4358 	 * flushing ctrl async_event_work after changing the controller state
4359 	 * from LIVE and before freeing the admin queue.
4360 	*/
4361 	if (nvme_ctrl_state(ctrl) == NVME_CTRL_LIVE)
4362 		ctrl->ops->submit_async_event(ctrl);
4363 }
4364 
4365 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
4366 {
4367 
4368 	u32 csts;
4369 
4370 	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
4371 		return false;
4372 
4373 	if (csts == ~0)
4374 		return false;
4375 
4376 	return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
4377 }
4378 
4379 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
4380 {
4381 	struct nvme_fw_slot_info_log *log;
4382 	u8 next_fw_slot, cur_fw_slot;
4383 
4384 	log = kmalloc(sizeof(*log), GFP_KERNEL);
4385 	if (!log)
4386 		return;
4387 
4388 	if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
4389 			 log, sizeof(*log), 0)) {
4390 		dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
4391 		goto out_free_log;
4392 	}
4393 
4394 	cur_fw_slot = log->afi & 0x7;
4395 	next_fw_slot = (log->afi & 0x70) >> 4;
4396 	if (!cur_fw_slot || (next_fw_slot && (cur_fw_slot != next_fw_slot))) {
4397 		dev_info(ctrl->device,
4398 			 "Firmware is activated after next Controller Level Reset\n");
4399 		goto out_free_log;
4400 	}
4401 
4402 	memcpy(ctrl->subsys->firmware_rev, &log->frs[cur_fw_slot - 1],
4403 		sizeof(ctrl->subsys->firmware_rev));
4404 
4405 out_free_log:
4406 	kfree(log);
4407 }
4408 
4409 static void nvme_fw_act_work(struct work_struct *work)
4410 {
4411 	struct nvme_ctrl *ctrl = container_of(work,
4412 				struct nvme_ctrl, fw_act_work);
4413 	unsigned long fw_act_timeout;
4414 
4415 	nvme_auth_stop(ctrl);
4416 
4417 	if (ctrl->mtfa)
4418 		fw_act_timeout = jiffies +
4419 				msecs_to_jiffies(ctrl->mtfa * 100);
4420 	else
4421 		fw_act_timeout = jiffies +
4422 				msecs_to_jiffies(admin_timeout * 1000);
4423 
4424 	nvme_quiesce_io_queues(ctrl);
4425 	while (nvme_ctrl_pp_status(ctrl)) {
4426 		if (time_after(jiffies, fw_act_timeout)) {
4427 			dev_warn(ctrl->device,
4428 				"Fw activation timeout, reset controller\n");
4429 			nvme_try_sched_reset(ctrl);
4430 			return;
4431 		}
4432 		msleep(100);
4433 	}
4434 
4435 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
4436 		return;
4437 
4438 	nvme_unquiesce_io_queues(ctrl);
4439 	/* read FW slot information to clear the AER */
4440 	nvme_get_fw_slot_info(ctrl);
4441 
4442 	queue_work(nvme_wq, &ctrl->async_event_work);
4443 }
4444 
4445 static u32 nvme_aer_type(u32 result)
4446 {
4447 	return result & 0x7;
4448 }
4449 
4450 static u32 nvme_aer_subtype(u32 result)
4451 {
4452 	return (result & 0xff00) >> 8;
4453 }
4454 
4455 static bool nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
4456 {
4457 	u32 aer_notice_type = nvme_aer_subtype(result);
4458 	bool requeue = true;
4459 
4460 	switch (aer_notice_type) {
4461 	case NVME_AER_NOTICE_NS_CHANGED:
4462 		set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
4463 		nvme_queue_scan(ctrl);
4464 		break;
4465 	case NVME_AER_NOTICE_FW_ACT_STARTING:
4466 		/*
4467 		 * We are (ab)using the RESETTING state to prevent subsequent
4468 		 * recovery actions from interfering with the controller's
4469 		 * firmware activation.
4470 		 */
4471 		if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) {
4472 			requeue = false;
4473 			queue_work(nvme_wq, &ctrl->fw_act_work);
4474 		}
4475 		break;
4476 #ifdef CONFIG_NVME_MULTIPATH
4477 	case NVME_AER_NOTICE_ANA:
4478 		if (!ctrl->ana_log_buf)
4479 			break;
4480 		queue_work(nvme_wq, &ctrl->ana_work);
4481 		break;
4482 #endif
4483 	case NVME_AER_NOTICE_DISC_CHANGED:
4484 		ctrl->aen_result = result;
4485 		break;
4486 	default:
4487 		dev_warn(ctrl->device, "async event result %08x\n", result);
4488 	}
4489 	return requeue;
4490 }
4491 
4492 static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
4493 {
4494 	dev_warn(ctrl->device,
4495 		"resetting controller due to persistent internal error\n");
4496 	nvme_reset_ctrl(ctrl);
4497 }
4498 
4499 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
4500 		volatile union nvme_result *res)
4501 {
4502 	u32 result = le32_to_cpu(res->u32);
4503 	u32 aer_type = nvme_aer_type(result);
4504 	u32 aer_subtype = nvme_aer_subtype(result);
4505 	bool requeue = true;
4506 
4507 	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
4508 		return;
4509 
4510 	trace_nvme_async_event(ctrl, result);
4511 	switch (aer_type) {
4512 	case NVME_AER_NOTICE:
4513 		requeue = nvme_handle_aen_notice(ctrl, result);
4514 		break;
4515 	case NVME_AER_ERROR:
4516 		/*
4517 		 * For a persistent internal error, don't run async_event_work
4518 		 * to submit a new AER. The controller reset will do it.
4519 		 */
4520 		if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
4521 			nvme_handle_aer_persistent_error(ctrl);
4522 			return;
4523 		}
4524 		fallthrough;
4525 	case NVME_AER_SMART:
4526 	case NVME_AER_CSS:
4527 	case NVME_AER_VS:
4528 		ctrl->aen_result = result;
4529 		break;
4530 	default:
4531 		break;
4532 	}
4533 
4534 	if (requeue)
4535 		queue_work(nvme_wq, &ctrl->async_event_work);
4536 }
4537 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
4538 
4539 int nvme_alloc_admin_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4540 		const struct blk_mq_ops *ops, unsigned int cmd_size)
4541 {
4542 	struct queue_limits lim = {};
4543 	int ret;
4544 
4545 	memset(set, 0, sizeof(*set));
4546 	set->ops = ops;
4547 	set->queue_depth = NVME_AQ_MQ_TAG_DEPTH;
4548 	if (ctrl->ops->flags & NVME_F_FABRICS)
4549 		/* Reserved for fabric connect and keep alive */
4550 		set->reserved_tags = 2;
4551 	set->numa_node = ctrl->numa_node;
4552 	set->flags = BLK_MQ_F_NO_SCHED;
4553 	if (ctrl->ops->flags & NVME_F_BLOCKING)
4554 		set->flags |= BLK_MQ_F_BLOCKING;
4555 	set->cmd_size = cmd_size;
4556 	set->driver_data = ctrl;
4557 	set->nr_hw_queues = 1;
4558 	set->timeout = NVME_ADMIN_TIMEOUT;
4559 	ret = blk_mq_alloc_tag_set(set);
4560 	if (ret)
4561 		return ret;
4562 
4563 	ctrl->admin_q = blk_mq_alloc_queue(set, &lim, NULL);
4564 	if (IS_ERR(ctrl->admin_q)) {
4565 		ret = PTR_ERR(ctrl->admin_q);
4566 		goto out_free_tagset;
4567 	}
4568 
4569 	if (ctrl->ops->flags & NVME_F_FABRICS) {
4570 		ctrl->fabrics_q = blk_mq_alloc_queue(set, NULL, NULL);
4571 		if (IS_ERR(ctrl->fabrics_q)) {
4572 			ret = PTR_ERR(ctrl->fabrics_q);
4573 			goto out_cleanup_admin_q;
4574 		}
4575 	}
4576 
4577 	ctrl->admin_tagset = set;
4578 	return 0;
4579 
4580 out_cleanup_admin_q:
4581 	blk_mq_destroy_queue(ctrl->admin_q);
4582 	blk_put_queue(ctrl->admin_q);
4583 out_free_tagset:
4584 	blk_mq_free_tag_set(set);
4585 	ctrl->admin_q = NULL;
4586 	ctrl->fabrics_q = NULL;
4587 	return ret;
4588 }
4589 EXPORT_SYMBOL_GPL(nvme_alloc_admin_tag_set);
4590 
4591 void nvme_remove_admin_tag_set(struct nvme_ctrl *ctrl)
4592 {
4593 	blk_mq_destroy_queue(ctrl->admin_q);
4594 	blk_put_queue(ctrl->admin_q);
4595 	if (ctrl->ops->flags & NVME_F_FABRICS) {
4596 		blk_mq_destroy_queue(ctrl->fabrics_q);
4597 		blk_put_queue(ctrl->fabrics_q);
4598 	}
4599 	blk_mq_free_tag_set(ctrl->admin_tagset);
4600 }
4601 EXPORT_SYMBOL_GPL(nvme_remove_admin_tag_set);
4602 
4603 int nvme_alloc_io_tag_set(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set,
4604 		const struct blk_mq_ops *ops, unsigned int nr_maps,
4605 		unsigned int cmd_size)
4606 {
4607 	int ret;
4608 
4609 	memset(set, 0, sizeof(*set));
4610 	set->ops = ops;
4611 	set->queue_depth = min_t(unsigned, ctrl->sqsize, BLK_MQ_MAX_DEPTH - 1);
4612 	/*
4613 	 * Some Apple controllers requires tags to be unique across admin and
4614 	 * the (only) I/O queue, so reserve the first 32 tags of the I/O queue.
4615 	 */
4616 	if (ctrl->quirks & NVME_QUIRK_SHARED_TAGS)
4617 		set->reserved_tags = NVME_AQ_DEPTH;
4618 	else if (ctrl->ops->flags & NVME_F_FABRICS)
4619 		/* Reserved for fabric connect */
4620 		set->reserved_tags = 1;
4621 	set->numa_node = ctrl->numa_node;
4622 	set->flags = BLK_MQ_F_SHOULD_MERGE;
4623 	if (ctrl->ops->flags & NVME_F_BLOCKING)
4624 		set->flags |= BLK_MQ_F_BLOCKING;
4625 	set->cmd_size = cmd_size;
4626 	set->driver_data = ctrl;
4627 	set->nr_hw_queues = ctrl->queue_count - 1;
4628 	set->timeout = NVME_IO_TIMEOUT;
4629 	set->nr_maps = nr_maps;
4630 	ret = blk_mq_alloc_tag_set(set);
4631 	if (ret)
4632 		return ret;
4633 
4634 	if (ctrl->ops->flags & NVME_F_FABRICS) {
4635 		struct queue_limits lim = {
4636 			.features	= BLK_FEAT_SKIP_TAGSET_QUIESCE,
4637 		};
4638 
4639 		ctrl->connect_q = blk_mq_alloc_queue(set, &lim, NULL);
4640         	if (IS_ERR(ctrl->connect_q)) {
4641 			ret = PTR_ERR(ctrl->connect_q);
4642 			goto out_free_tag_set;
4643 		}
4644 	}
4645 
4646 	ctrl->tagset = set;
4647 	return 0;
4648 
4649 out_free_tag_set:
4650 	blk_mq_free_tag_set(set);
4651 	ctrl->connect_q = NULL;
4652 	return ret;
4653 }
4654 EXPORT_SYMBOL_GPL(nvme_alloc_io_tag_set);
4655 
4656 void nvme_remove_io_tag_set(struct nvme_ctrl *ctrl)
4657 {
4658 	if (ctrl->ops->flags & NVME_F_FABRICS) {
4659 		blk_mq_destroy_queue(ctrl->connect_q);
4660 		blk_put_queue(ctrl->connect_q);
4661 	}
4662 	blk_mq_free_tag_set(ctrl->tagset);
4663 }
4664 EXPORT_SYMBOL_GPL(nvme_remove_io_tag_set);
4665 
4666 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
4667 {
4668 	nvme_mpath_stop(ctrl);
4669 	nvme_auth_stop(ctrl);
4670 	nvme_stop_failfast_work(ctrl);
4671 	flush_work(&ctrl->async_event_work);
4672 	cancel_work_sync(&ctrl->fw_act_work);
4673 	if (ctrl->ops->stop_ctrl)
4674 		ctrl->ops->stop_ctrl(ctrl);
4675 }
4676 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
4677 
4678 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
4679 {
4680 	nvme_enable_aen(ctrl);
4681 
4682 	/*
4683 	 * persistent discovery controllers need to send indication to userspace
4684 	 * to re-read the discovery log page to learn about possible changes
4685 	 * that were missed. We identify persistent discovery controllers by
4686 	 * checking that they started once before, hence are reconnecting back.
4687 	 */
4688 	if (test_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags) &&
4689 	    nvme_discovery_ctrl(ctrl))
4690 		nvme_change_uevent(ctrl, "NVME_EVENT=rediscover");
4691 
4692 	if (ctrl->queue_count > 1) {
4693 		nvme_queue_scan(ctrl);
4694 		nvme_unquiesce_io_queues(ctrl);
4695 		nvme_mpath_update(ctrl);
4696 	}
4697 
4698 	nvme_change_uevent(ctrl, "NVME_EVENT=connected");
4699 	set_bit(NVME_CTRL_STARTED_ONCE, &ctrl->flags);
4700 }
4701 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
4702 
4703 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
4704 {
4705 	nvme_stop_keep_alive(ctrl);
4706 	nvme_hwmon_exit(ctrl);
4707 	nvme_fault_inject_fini(&ctrl->fault_inject);
4708 	dev_pm_qos_hide_latency_tolerance(ctrl->device);
4709 	cdev_device_del(&ctrl->cdev, ctrl->device);
4710 	nvme_put_ctrl(ctrl);
4711 }
4712 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4713 
4714 static void nvme_free_cels(struct nvme_ctrl *ctrl)
4715 {
4716 	struct nvme_effects_log	*cel;
4717 	unsigned long i;
4718 
4719 	xa_for_each(&ctrl->cels, i, cel) {
4720 		xa_erase(&ctrl->cels, i);
4721 		kfree(cel);
4722 	}
4723 
4724 	xa_destroy(&ctrl->cels);
4725 }
4726 
4727 static void nvme_free_ctrl(struct device *dev)
4728 {
4729 	struct nvme_ctrl *ctrl =
4730 		container_of(dev, struct nvme_ctrl, ctrl_device);
4731 	struct nvme_subsystem *subsys = ctrl->subsys;
4732 
4733 	if (!subsys || ctrl->instance != subsys->instance)
4734 		ida_free(&nvme_instance_ida, ctrl->instance);
4735 	nvme_free_cels(ctrl);
4736 	nvme_mpath_uninit(ctrl);
4737 	cleanup_srcu_struct(&ctrl->srcu);
4738 	nvme_auth_stop(ctrl);
4739 	nvme_auth_free(ctrl);
4740 	__free_page(ctrl->discard_page);
4741 	free_opal_dev(ctrl->opal_dev);
4742 
4743 	if (subsys) {
4744 		mutex_lock(&nvme_subsystems_lock);
4745 		list_del(&ctrl->subsys_entry);
4746 		sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4747 		mutex_unlock(&nvme_subsystems_lock);
4748 	}
4749 
4750 	ctrl->ops->free_ctrl(ctrl);
4751 
4752 	if (subsys)
4753 		nvme_put_subsystem(subsys);
4754 }
4755 
4756 /*
4757  * Initialize a NVMe controller structures.  This needs to be called during
4758  * earliest initialization so that we have the initialized structured around
4759  * during probing.
4760  *
4761  * On success, the caller must use the nvme_put_ctrl() to release this when
4762  * needed, which also invokes the ops->free_ctrl() callback.
4763  */
4764 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4765 		const struct nvme_ctrl_ops *ops, unsigned long quirks)
4766 {
4767 	int ret;
4768 
4769 	WRITE_ONCE(ctrl->state, NVME_CTRL_NEW);
4770 	ctrl->passthru_err_log_enabled = false;
4771 	clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
4772 	spin_lock_init(&ctrl->lock);
4773 	mutex_init(&ctrl->namespaces_lock);
4774 
4775 	ret = init_srcu_struct(&ctrl->srcu);
4776 	if (ret)
4777 		return ret;
4778 
4779 	mutex_init(&ctrl->scan_lock);
4780 	INIT_LIST_HEAD(&ctrl->namespaces);
4781 	xa_init(&ctrl->cels);
4782 	ctrl->dev = dev;
4783 	ctrl->ops = ops;
4784 	ctrl->quirks = quirks;
4785 	ctrl->numa_node = NUMA_NO_NODE;
4786 	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4787 	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4788 	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4789 	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4790 	init_waitqueue_head(&ctrl->state_wq);
4791 
4792 	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4793 	INIT_DELAYED_WORK(&ctrl->failfast_work, nvme_failfast_work);
4794 	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4795 	ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4796 	ctrl->ka_last_check_time = jiffies;
4797 
4798 	BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4799 			PAGE_SIZE);
4800 	ctrl->discard_page = alloc_page(GFP_KERNEL);
4801 	if (!ctrl->discard_page) {
4802 		ret = -ENOMEM;
4803 		goto out;
4804 	}
4805 
4806 	ret = ida_alloc(&nvme_instance_ida, GFP_KERNEL);
4807 	if (ret < 0)
4808 		goto out;
4809 	ctrl->instance = ret;
4810 
4811 	ret = nvme_auth_init_ctrl(ctrl);
4812 	if (ret)
4813 		goto out_release_instance;
4814 
4815 	nvme_mpath_init_ctrl(ctrl);
4816 
4817 	device_initialize(&ctrl->ctrl_device);
4818 	ctrl->device = &ctrl->ctrl_device;
4819 	ctrl->device->devt = MKDEV(MAJOR(nvme_ctrl_base_chr_devt),
4820 			ctrl->instance);
4821 	ctrl->device->class = &nvme_class;
4822 	ctrl->device->parent = ctrl->dev;
4823 	if (ops->dev_attr_groups)
4824 		ctrl->device->groups = ops->dev_attr_groups;
4825 	else
4826 		ctrl->device->groups = nvme_dev_attr_groups;
4827 	ctrl->device->release = nvme_free_ctrl;
4828 	dev_set_drvdata(ctrl->device, ctrl);
4829 
4830 	return ret;
4831 
4832 out_release_instance:
4833 	ida_free(&nvme_instance_ida, ctrl->instance);
4834 out:
4835 	if (ctrl->discard_page)
4836 		__free_page(ctrl->discard_page);
4837 	cleanup_srcu_struct(&ctrl->srcu);
4838 	return ret;
4839 }
4840 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4841 
4842 /*
4843  * On success, returns with an elevated controller reference and caller must
4844  * use nvme_uninit_ctrl() to properly free resources associated with the ctrl.
4845  */
4846 int nvme_add_ctrl(struct nvme_ctrl *ctrl)
4847 {
4848 	int ret;
4849 
4850 	ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4851 	if (ret)
4852 		return ret;
4853 
4854 	cdev_init(&ctrl->cdev, &nvme_dev_fops);
4855 	ctrl->cdev.owner = ctrl->ops->module;
4856 	ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4857 	if (ret)
4858 		return ret;
4859 
4860 	/*
4861 	 * Initialize latency tolerance controls.  The sysfs files won't
4862 	 * be visible to userspace unless the device actually supports APST.
4863 	 */
4864 	ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4865 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4866 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4867 
4868 	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4869 	nvme_get_ctrl(ctrl);
4870 
4871 	return 0;
4872 }
4873 EXPORT_SYMBOL_GPL(nvme_add_ctrl);
4874 
4875 /* let I/O to all namespaces fail in preparation for surprise removal */
4876 void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl)
4877 {
4878 	struct nvme_ns *ns;
4879 	int srcu_idx;
4880 
4881 	srcu_idx = srcu_read_lock(&ctrl->srcu);
4882 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4883 		blk_mark_disk_dead(ns->disk);
4884 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
4885 }
4886 EXPORT_SYMBOL_GPL(nvme_mark_namespaces_dead);
4887 
4888 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4889 {
4890 	struct nvme_ns *ns;
4891 	int srcu_idx;
4892 
4893 	srcu_idx = srcu_read_lock(&ctrl->srcu);
4894 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4895 		blk_mq_unfreeze_queue(ns->queue);
4896 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
4897 	clear_bit(NVME_CTRL_FROZEN, &ctrl->flags);
4898 }
4899 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4900 
4901 int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4902 {
4903 	struct nvme_ns *ns;
4904 	int srcu_idx;
4905 
4906 	srcu_idx = srcu_read_lock(&ctrl->srcu);
4907 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
4908 		timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4909 		if (timeout <= 0)
4910 			break;
4911 	}
4912 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
4913 	return timeout;
4914 }
4915 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4916 
4917 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4918 {
4919 	struct nvme_ns *ns;
4920 	int srcu_idx;
4921 
4922 	srcu_idx = srcu_read_lock(&ctrl->srcu);
4923 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4924 		blk_mq_freeze_queue_wait(ns->queue);
4925 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
4926 }
4927 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4928 
4929 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4930 {
4931 	struct nvme_ns *ns;
4932 	int srcu_idx;
4933 
4934 	set_bit(NVME_CTRL_FROZEN, &ctrl->flags);
4935 	srcu_idx = srcu_read_lock(&ctrl->srcu);
4936 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4937 		blk_freeze_queue_start(ns->queue);
4938 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
4939 }
4940 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4941 
4942 void nvme_quiesce_io_queues(struct nvme_ctrl *ctrl)
4943 {
4944 	if (!ctrl->tagset)
4945 		return;
4946 	if (!test_and_set_bit(NVME_CTRL_STOPPED, &ctrl->flags))
4947 		blk_mq_quiesce_tagset(ctrl->tagset);
4948 	else
4949 		blk_mq_wait_quiesce_done(ctrl->tagset);
4950 }
4951 EXPORT_SYMBOL_GPL(nvme_quiesce_io_queues);
4952 
4953 void nvme_unquiesce_io_queues(struct nvme_ctrl *ctrl)
4954 {
4955 	if (!ctrl->tagset)
4956 		return;
4957 	if (test_and_clear_bit(NVME_CTRL_STOPPED, &ctrl->flags))
4958 		blk_mq_unquiesce_tagset(ctrl->tagset);
4959 }
4960 EXPORT_SYMBOL_GPL(nvme_unquiesce_io_queues);
4961 
4962 void nvme_quiesce_admin_queue(struct nvme_ctrl *ctrl)
4963 {
4964 	if (!test_and_set_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4965 		blk_mq_quiesce_queue(ctrl->admin_q);
4966 	else
4967 		blk_mq_wait_quiesce_done(ctrl->admin_q->tag_set);
4968 }
4969 EXPORT_SYMBOL_GPL(nvme_quiesce_admin_queue);
4970 
4971 void nvme_unquiesce_admin_queue(struct nvme_ctrl *ctrl)
4972 {
4973 	if (test_and_clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->flags))
4974 		blk_mq_unquiesce_queue(ctrl->admin_q);
4975 }
4976 EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);
4977 
4978 void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
4979 {
4980 	struct nvme_ns *ns;
4981 	int srcu_idx;
4982 
4983 	srcu_idx = srcu_read_lock(&ctrl->srcu);
4984 	list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
4985 		blk_sync_queue(ns->queue);
4986 	srcu_read_unlock(&ctrl->srcu, srcu_idx);
4987 }
4988 EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
4989 
4990 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4991 {
4992 	nvme_sync_io_queues(ctrl);
4993 	if (ctrl->admin_q)
4994 		blk_sync_queue(ctrl->admin_q);
4995 }
4996 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4997 
4998 struct nvme_ctrl *nvme_ctrl_from_file(struct file *file)
4999 {
5000 	if (file->f_op != &nvme_dev_fops)
5001 		return NULL;
5002 	return file->private_data;
5003 }
5004 EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
5005 
5006 /*
5007  * Check we didn't inadvertently grow the command structure sizes:
5008  */
5009 static inline void _nvme_check_size(void)
5010 {
5011 	BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
5012 	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
5013 	BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
5014 	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
5015 	BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
5016 	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
5017 	BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
5018 	BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
5019 	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
5020 	BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
5021 	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
5022 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
5023 	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
5024 	BUILD_BUG_ON(sizeof(struct nvme_id_ns_cs_indep) !=
5025 			NVME_IDENTIFY_DATA_SIZE);
5026 	BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
5027 	BUILD_BUG_ON(sizeof(struct nvme_id_ns_nvm) != NVME_IDENTIFY_DATA_SIZE);
5028 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
5029 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_nvm) != NVME_IDENTIFY_DATA_SIZE);
5030 	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
5031 	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
5032 	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
5033 	BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
5034 	BUILD_BUG_ON(sizeof(struct nvme_feat_host_behavior) != 512);
5035 }
5036 
5037 
5038 static int __init nvme_core_init(void)
5039 {
5040 	int result = -ENOMEM;
5041 
5042 	_nvme_check_size();
5043 
5044 	nvme_wq = alloc_workqueue("nvme-wq",
5045 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5046 	if (!nvme_wq)
5047 		goto out;
5048 
5049 	nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
5050 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5051 	if (!nvme_reset_wq)
5052 		goto destroy_wq;
5053 
5054 	nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
5055 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
5056 	if (!nvme_delete_wq)
5057 		goto destroy_reset_wq;
5058 
5059 	result = alloc_chrdev_region(&nvme_ctrl_base_chr_devt, 0,
5060 			NVME_MINORS, "nvme");
5061 	if (result < 0)
5062 		goto destroy_delete_wq;
5063 
5064 	result = class_register(&nvme_class);
5065 	if (result)
5066 		goto unregister_chrdev;
5067 
5068 	result = class_register(&nvme_subsys_class);
5069 	if (result)
5070 		goto destroy_class;
5071 
5072 	result = alloc_chrdev_region(&nvme_ns_chr_devt, 0, NVME_MINORS,
5073 				     "nvme-generic");
5074 	if (result < 0)
5075 		goto destroy_subsys_class;
5076 
5077 	result = class_register(&nvme_ns_chr_class);
5078 	if (result)
5079 		goto unregister_generic_ns;
5080 
5081 	result = nvme_init_auth();
5082 	if (result)
5083 		goto destroy_ns_chr;
5084 	return 0;
5085 
5086 destroy_ns_chr:
5087 	class_unregister(&nvme_ns_chr_class);
5088 unregister_generic_ns:
5089 	unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5090 destroy_subsys_class:
5091 	class_unregister(&nvme_subsys_class);
5092 destroy_class:
5093 	class_unregister(&nvme_class);
5094 unregister_chrdev:
5095 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5096 destroy_delete_wq:
5097 	destroy_workqueue(nvme_delete_wq);
5098 destroy_reset_wq:
5099 	destroy_workqueue(nvme_reset_wq);
5100 destroy_wq:
5101 	destroy_workqueue(nvme_wq);
5102 out:
5103 	return result;
5104 }
5105 
5106 static void __exit nvme_core_exit(void)
5107 {
5108 	nvme_exit_auth();
5109 	class_unregister(&nvme_ns_chr_class);
5110 	class_unregister(&nvme_subsys_class);
5111 	class_unregister(&nvme_class);
5112 	unregister_chrdev_region(nvme_ns_chr_devt, NVME_MINORS);
5113 	unregister_chrdev_region(nvme_ctrl_base_chr_devt, NVME_MINORS);
5114 	destroy_workqueue(nvme_delete_wq);
5115 	destroy_workqueue(nvme_reset_wq);
5116 	destroy_workqueue(nvme_wq);
5117 	ida_destroy(&nvme_ns_chr_minor_ida);
5118 	ida_destroy(&nvme_instance_ida);
5119 }
5120 
5121 MODULE_LICENSE("GPL");
5122 MODULE_VERSION("1.0");
5123 MODULE_DESCRIPTION("NVMe host core framework");
5124 module_init(nvme_core_init);
5125 module_exit(nvme_core_exit);
5126