xref: /linux/drivers/nvme/host/core.c (revision 2c97b5ae83dca56718774e7b4bf9640f05d11867)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVM Express device driver
4  * Copyright (c) 2011-2014, Intel Corporation.
5  */
6 
7 #include <linux/blkdev.h>
8 #include <linux/blk-mq.h>
9 #include <linux/delay.h>
10 #include <linux/errno.h>
11 #include <linux/hdreg.h>
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/backing-dev.h>
15 #include <linux/list_sort.h>
16 #include <linux/slab.h>
17 #include <linux/types.h>
18 #include <linux/pr.h>
19 #include <linux/ptrace.h>
20 #include <linux/nvme_ioctl.h>
21 #include <linux/t10-pi.h>
22 #include <linux/pm_qos.h>
23 #include <asm/unaligned.h>
24 
25 #include "nvme.h"
26 #include "fabrics.h"
27 
28 #define CREATE_TRACE_POINTS
29 #include "trace.h"
30 
31 #define NVME_MINORS		(1U << MINORBITS)
32 
33 unsigned int admin_timeout = 60;
34 module_param(admin_timeout, uint, 0644);
35 MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
36 EXPORT_SYMBOL_GPL(admin_timeout);
37 
38 unsigned int nvme_io_timeout = 30;
39 module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
40 MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
41 EXPORT_SYMBOL_GPL(nvme_io_timeout);
42 
43 static unsigned char shutdown_timeout = 5;
44 module_param(shutdown_timeout, byte, 0644);
45 MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");
46 
47 static u8 nvme_max_retries = 5;
48 module_param_named(max_retries, nvme_max_retries, byte, 0644);
49 MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
50 
51 static unsigned long default_ps_max_latency_us = 100000;
52 module_param(default_ps_max_latency_us, ulong, 0644);
53 MODULE_PARM_DESC(default_ps_max_latency_us,
54 		 "max power saving latency for new devices; use PM QOS to change per device");
55 
56 static bool force_apst;
57 module_param(force_apst, bool, 0644);
58 MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");
59 
60 static bool streams;
61 module_param(streams, bool, 0644);
62 MODULE_PARM_DESC(streams, "turn on support for Streams write directives");
63 
64 /*
65  * nvme_wq - hosts nvme related works that are not reset or delete
66  * nvme_reset_wq - hosts nvme reset works
67  * nvme_delete_wq - hosts nvme delete works
68  *
69  * nvme_wq will host works such are scan, aen handling, fw activation,
70  * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq
71  * runs reset works which also flush works hosted on nvme_wq for
72  * serialization purposes. nvme_delete_wq host controller deletion
73  * works which flush reset works for serialization.
74  */
75 struct workqueue_struct *nvme_wq;
76 EXPORT_SYMBOL_GPL(nvme_wq);
77 
78 struct workqueue_struct *nvme_reset_wq;
79 EXPORT_SYMBOL_GPL(nvme_reset_wq);
80 
81 struct workqueue_struct *nvme_delete_wq;
82 EXPORT_SYMBOL_GPL(nvme_delete_wq);
83 
84 static LIST_HEAD(nvme_subsystems);
85 static DEFINE_MUTEX(nvme_subsystems_lock);
86 
87 static DEFINE_IDA(nvme_instance_ida);
88 static dev_t nvme_chr_devt;
89 static struct class *nvme_class;
90 static struct class *nvme_subsys_class;
91 
92 static int nvme_revalidate_disk(struct gendisk *disk);
93 static void nvme_put_subsystem(struct nvme_subsystem *subsys);
94 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
95 					   unsigned nsid);
96 
97 static void nvme_set_queue_dying(struct nvme_ns *ns)
98 {
99 	/*
100 	 * Revalidating a dead namespace sets capacity to 0. This will end
101 	 * buffered writers dirtying pages that can't be synced.
102 	 */
103 	if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
104 		return;
105 	blk_set_queue_dying(ns->queue);
106 	/* Forcibly unquiesce queues to avoid blocking dispatch */
107 	blk_mq_unquiesce_queue(ns->queue);
108 	/*
109 	 * Revalidate after unblocking dispatchers that may be holding bd_butex
110 	 */
111 	revalidate_disk(ns->disk);
112 }
113 
114 static void nvme_queue_scan(struct nvme_ctrl *ctrl)
115 {
116 	/*
117 	 * Only new queue scan work when admin and IO queues are both alive
118 	 */
119 	if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
120 		queue_work(nvme_wq, &ctrl->scan_work);
121 }
122 
123 /*
124  * Use this function to proceed with scheduling reset_work for a controller
125  * that had previously been set to the resetting state. This is intended for
126  * code paths that can't be interrupted by other reset attempts. A hot removal
127  * may prevent this from succeeding.
128  */
129 int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
130 {
131 	if (ctrl->state != NVME_CTRL_RESETTING)
132 		return -EBUSY;
133 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
134 		return -EBUSY;
135 	return 0;
136 }
137 EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
138 
139 int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
140 {
141 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
142 		return -EBUSY;
143 	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
144 		return -EBUSY;
145 	return 0;
146 }
147 EXPORT_SYMBOL_GPL(nvme_reset_ctrl);
148 
149 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
150 {
151 	int ret;
152 
153 	ret = nvme_reset_ctrl(ctrl);
154 	if (!ret) {
155 		flush_work(&ctrl->reset_work);
156 		if (ctrl->state != NVME_CTRL_LIVE)
157 			ret = -ENETRESET;
158 	}
159 
160 	return ret;
161 }
162 EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
163 
164 static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
165 {
166 	dev_info(ctrl->device,
167 		 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
168 
169 	flush_work(&ctrl->reset_work);
170 	nvme_stop_ctrl(ctrl);
171 	nvme_remove_namespaces(ctrl);
172 	ctrl->ops->delete_ctrl(ctrl);
173 	nvme_uninit_ctrl(ctrl);
174 	nvme_put_ctrl(ctrl);
175 }
176 
177 static void nvme_delete_ctrl_work(struct work_struct *work)
178 {
179 	struct nvme_ctrl *ctrl =
180 		container_of(work, struct nvme_ctrl, delete_work);
181 
182 	nvme_do_delete_ctrl(ctrl);
183 }
184 
185 int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
186 {
187 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
188 		return -EBUSY;
189 	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
190 		return -EBUSY;
191 	return 0;
192 }
193 EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
194 
195 static int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
196 {
197 	int ret = 0;
198 
199 	/*
200 	 * Keep a reference until nvme_do_delete_ctrl() complete,
201 	 * since ->delete_ctrl can free the controller.
202 	 */
203 	nvme_get_ctrl(ctrl);
204 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
205 		ret = -EBUSY;
206 	if (!ret)
207 		nvme_do_delete_ctrl(ctrl);
208 	nvme_put_ctrl(ctrl);
209 	return ret;
210 }
211 
212 static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
213 {
214 	return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
215 }
216 
217 static blk_status_t nvme_error_status(u16 status)
218 {
219 	switch (status & 0x7ff) {
220 	case NVME_SC_SUCCESS:
221 		return BLK_STS_OK;
222 	case NVME_SC_CAP_EXCEEDED:
223 		return BLK_STS_NOSPC;
224 	case NVME_SC_LBA_RANGE:
225 		return BLK_STS_TARGET;
226 	case NVME_SC_BAD_ATTRIBUTES:
227 	case NVME_SC_ONCS_NOT_SUPPORTED:
228 	case NVME_SC_INVALID_OPCODE:
229 	case NVME_SC_INVALID_FIELD:
230 	case NVME_SC_INVALID_NS:
231 		return BLK_STS_NOTSUPP;
232 	case NVME_SC_WRITE_FAULT:
233 	case NVME_SC_READ_ERROR:
234 	case NVME_SC_UNWRITTEN_BLOCK:
235 	case NVME_SC_ACCESS_DENIED:
236 	case NVME_SC_READ_ONLY:
237 	case NVME_SC_COMPARE_FAILED:
238 		return BLK_STS_MEDIUM;
239 	case NVME_SC_GUARD_CHECK:
240 	case NVME_SC_APPTAG_CHECK:
241 	case NVME_SC_REFTAG_CHECK:
242 	case NVME_SC_INVALID_PI:
243 		return BLK_STS_PROTECTION;
244 	case NVME_SC_RESERVATION_CONFLICT:
245 		return BLK_STS_NEXUS;
246 	case NVME_SC_HOST_PATH_ERROR:
247 		return BLK_STS_TRANSPORT;
248 	default:
249 		return BLK_STS_IOERR;
250 	}
251 }
252 
253 static inline bool nvme_req_needs_retry(struct request *req)
254 {
255 	if (blk_noretry_request(req))
256 		return false;
257 	if (nvme_req(req)->status & NVME_SC_DNR)
258 		return false;
259 	if (nvme_req(req)->retries >= nvme_max_retries)
260 		return false;
261 	return true;
262 }
263 
264 static void nvme_retry_req(struct request *req)
265 {
266 	struct nvme_ns *ns = req->q->queuedata;
267 	unsigned long delay = 0;
268 	u16 crd;
269 
270 	/* The mask and shift result must be <= 3 */
271 	crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
272 	if (ns && crd)
273 		delay = ns->ctrl->crdt[crd - 1] * 100;
274 
275 	nvme_req(req)->retries++;
276 	blk_mq_requeue_request(req, false);
277 	blk_mq_delay_kick_requeue_list(req->q, delay);
278 }
279 
280 void nvme_complete_rq(struct request *req)
281 {
282 	blk_status_t status = nvme_error_status(nvme_req(req)->status);
283 
284 	trace_nvme_complete_rq(req);
285 
286 	nvme_cleanup_cmd(req);
287 
288 	if (nvme_req(req)->ctrl->kas)
289 		nvme_req(req)->ctrl->comp_seen = true;
290 
291 	if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
292 		if ((req->cmd_flags & REQ_NVME_MPATH) &&
293 		    blk_path_error(status)) {
294 			nvme_failover_req(req);
295 			return;
296 		}
297 
298 		if (!blk_queue_dying(req->q)) {
299 			nvme_retry_req(req);
300 			return;
301 		}
302 	}
303 
304 	nvme_trace_bio_complete(req, status);
305 	blk_mq_end_request(req, status);
306 }
307 EXPORT_SYMBOL_GPL(nvme_complete_rq);
308 
309 bool nvme_cancel_request(struct request *req, void *data, bool reserved)
310 {
311 	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
312 				"Cancelling I/O %d", req->tag);
313 
314 	/* don't abort one completed request */
315 	if (blk_mq_request_completed(req))
316 		return true;
317 
318 	nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
319 	blk_mq_complete_request(req);
320 	return true;
321 }
322 EXPORT_SYMBOL_GPL(nvme_cancel_request);
323 
324 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
325 		enum nvme_ctrl_state new_state)
326 {
327 	enum nvme_ctrl_state old_state;
328 	unsigned long flags;
329 	bool changed = false;
330 
331 	spin_lock_irqsave(&ctrl->lock, flags);
332 
333 	old_state = ctrl->state;
334 	switch (new_state) {
335 	case NVME_CTRL_LIVE:
336 		switch (old_state) {
337 		case NVME_CTRL_NEW:
338 		case NVME_CTRL_RESETTING:
339 		case NVME_CTRL_CONNECTING:
340 			changed = true;
341 			/* FALLTHRU */
342 		default:
343 			break;
344 		}
345 		break;
346 	case NVME_CTRL_RESETTING:
347 		switch (old_state) {
348 		case NVME_CTRL_NEW:
349 		case NVME_CTRL_LIVE:
350 			changed = true;
351 			/* FALLTHRU */
352 		default:
353 			break;
354 		}
355 		break;
356 	case NVME_CTRL_CONNECTING:
357 		switch (old_state) {
358 		case NVME_CTRL_NEW:
359 		case NVME_CTRL_RESETTING:
360 			changed = true;
361 			/* FALLTHRU */
362 		default:
363 			break;
364 		}
365 		break;
366 	case NVME_CTRL_DELETING:
367 		switch (old_state) {
368 		case NVME_CTRL_LIVE:
369 		case NVME_CTRL_RESETTING:
370 		case NVME_CTRL_CONNECTING:
371 			changed = true;
372 			/* FALLTHRU */
373 		default:
374 			break;
375 		}
376 		break;
377 	case NVME_CTRL_DEAD:
378 		switch (old_state) {
379 		case NVME_CTRL_DELETING:
380 			changed = true;
381 			/* FALLTHRU */
382 		default:
383 			break;
384 		}
385 		break;
386 	default:
387 		break;
388 	}
389 
390 	if (changed) {
391 		ctrl->state = new_state;
392 		wake_up_all(&ctrl->state_wq);
393 	}
394 
395 	spin_unlock_irqrestore(&ctrl->lock, flags);
396 	if (changed && ctrl->state == NVME_CTRL_LIVE)
397 		nvme_kick_requeue_lists(ctrl);
398 	return changed;
399 }
400 EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
401 
402 /*
403  * Returns true for sink states that can't ever transition back to live.
404  */
405 static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
406 {
407 	switch (ctrl->state) {
408 	case NVME_CTRL_NEW:
409 	case NVME_CTRL_LIVE:
410 	case NVME_CTRL_RESETTING:
411 	case NVME_CTRL_CONNECTING:
412 		return false;
413 	case NVME_CTRL_DELETING:
414 	case NVME_CTRL_DEAD:
415 		return true;
416 	default:
417 		WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
418 		return true;
419 	}
420 }
421 
422 /*
423  * Waits for the controller state to be resetting, or returns false if it is
424  * not possible to ever transition to that state.
425  */
426 bool nvme_wait_reset(struct nvme_ctrl *ctrl)
427 {
428 	wait_event(ctrl->state_wq,
429 		   nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) ||
430 		   nvme_state_terminal(ctrl));
431 	return ctrl->state == NVME_CTRL_RESETTING;
432 }
433 EXPORT_SYMBOL_GPL(nvme_wait_reset);
434 
435 static void nvme_free_ns_head(struct kref *ref)
436 {
437 	struct nvme_ns_head *head =
438 		container_of(ref, struct nvme_ns_head, ref);
439 
440 	nvme_mpath_remove_disk(head);
441 	ida_simple_remove(&head->subsys->ns_ida, head->instance);
442 	list_del_init(&head->entry);
443 	cleanup_srcu_struct(&head->srcu);
444 	nvme_put_subsystem(head->subsys);
445 	kfree(head);
446 }
447 
448 static void nvme_put_ns_head(struct nvme_ns_head *head)
449 {
450 	kref_put(&head->ref, nvme_free_ns_head);
451 }
452 
453 static void nvme_free_ns(struct kref *kref)
454 {
455 	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
456 
457 	if (ns->ndev)
458 		nvme_nvm_unregister(ns);
459 
460 	put_disk(ns->disk);
461 	nvme_put_ns_head(ns->head);
462 	nvme_put_ctrl(ns->ctrl);
463 	kfree(ns);
464 }
465 
466 static void nvme_put_ns(struct nvme_ns *ns)
467 {
468 	kref_put(&ns->kref, nvme_free_ns);
469 }
470 
471 static inline void nvme_clear_nvme_request(struct request *req)
472 {
473 	if (!(req->rq_flags & RQF_DONTPREP)) {
474 		nvme_req(req)->retries = 0;
475 		nvme_req(req)->flags = 0;
476 		req->rq_flags |= RQF_DONTPREP;
477 	}
478 }
479 
480 struct request *nvme_alloc_request(struct request_queue *q,
481 		struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
482 {
483 	unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
484 	struct request *req;
485 
486 	if (qid == NVME_QID_ANY) {
487 		req = blk_mq_alloc_request(q, op, flags);
488 	} else {
489 		req = blk_mq_alloc_request_hctx(q, op, flags,
490 				qid ? qid - 1 : 0);
491 	}
492 	if (IS_ERR(req))
493 		return req;
494 
495 	req->cmd_flags |= REQ_FAILFAST_DRIVER;
496 	nvme_clear_nvme_request(req);
497 	nvme_req(req)->cmd = cmd;
498 
499 	return req;
500 }
501 EXPORT_SYMBOL_GPL(nvme_alloc_request);
502 
503 static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
504 {
505 	struct nvme_command c;
506 
507 	memset(&c, 0, sizeof(c));
508 
509 	c.directive.opcode = nvme_admin_directive_send;
510 	c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
511 	c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
512 	c.directive.dtype = NVME_DIR_IDENTIFY;
513 	c.directive.tdtype = NVME_DIR_STREAMS;
514 	c.directive.endir = enable ? NVME_DIR_ENDIR : 0;
515 
516 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
517 }
518 
519 static int nvme_disable_streams(struct nvme_ctrl *ctrl)
520 {
521 	return nvme_toggle_streams(ctrl, false);
522 }
523 
524 static int nvme_enable_streams(struct nvme_ctrl *ctrl)
525 {
526 	return nvme_toggle_streams(ctrl, true);
527 }
528 
529 static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
530 				  struct streams_directive_params *s, u32 nsid)
531 {
532 	struct nvme_command c;
533 
534 	memset(&c, 0, sizeof(c));
535 	memset(s, 0, sizeof(*s));
536 
537 	c.directive.opcode = nvme_admin_directive_recv;
538 	c.directive.nsid = cpu_to_le32(nsid);
539 	c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
540 	c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
541 	c.directive.dtype = NVME_DIR_STREAMS;
542 
543 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
544 }
545 
546 static int nvme_configure_directives(struct nvme_ctrl *ctrl)
547 {
548 	struct streams_directive_params s;
549 	int ret;
550 
551 	if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
552 		return 0;
553 	if (!streams)
554 		return 0;
555 
556 	ret = nvme_enable_streams(ctrl);
557 	if (ret)
558 		return ret;
559 
560 	ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
561 	if (ret)
562 		return ret;
563 
564 	ctrl->nssa = le16_to_cpu(s.nssa);
565 	if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
566 		dev_info(ctrl->device, "too few streams (%u) available\n",
567 					ctrl->nssa);
568 		nvme_disable_streams(ctrl);
569 		return 0;
570 	}
571 
572 	ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
573 	dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
574 	return 0;
575 }
576 
577 /*
578  * Check if 'req' has a write hint associated with it. If it does, assign
579  * a valid namespace stream to the write.
580  */
581 static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
582 				     struct request *req, u16 *control,
583 				     u32 *dsmgmt)
584 {
585 	enum rw_hint streamid = req->write_hint;
586 
587 	if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
588 		streamid = 0;
589 	else {
590 		streamid--;
591 		if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
592 			return;
593 
594 		*control |= NVME_RW_DTYPE_STREAMS;
595 		*dsmgmt |= streamid << 16;
596 	}
597 
598 	if (streamid < ARRAY_SIZE(req->q->write_hints))
599 		req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
600 }
601 
602 static inline void nvme_setup_flush(struct nvme_ns *ns,
603 		struct nvme_command *cmnd)
604 {
605 	cmnd->common.opcode = nvme_cmd_flush;
606 	cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
607 }
608 
609 static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
610 		struct nvme_command *cmnd)
611 {
612 	unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
613 	struct nvme_dsm_range *range;
614 	struct bio *bio;
615 
616 	/*
617 	 * Some devices do not consider the DSM 'Number of Ranges' field when
618 	 * determining how much data to DMA. Always allocate memory for maximum
619 	 * number of segments to prevent device reading beyond end of buffer.
620 	 */
621 	static const size_t alloc_size = sizeof(*range) * NVME_DSM_MAX_RANGES;
622 
623 	range = kzalloc(alloc_size, GFP_ATOMIC | __GFP_NOWARN);
624 	if (!range) {
625 		/*
626 		 * If we fail allocation our range, fallback to the controller
627 		 * discard page. If that's also busy, it's safe to return
628 		 * busy, as we know we can make progress once that's freed.
629 		 */
630 		if (test_and_set_bit_lock(0, &ns->ctrl->discard_page_busy))
631 			return BLK_STS_RESOURCE;
632 
633 		range = page_address(ns->ctrl->discard_page);
634 	}
635 
636 	__rq_for_each_bio(bio, req) {
637 		u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
638 		u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
639 
640 		if (n < segments) {
641 			range[n].cattr = cpu_to_le32(0);
642 			range[n].nlb = cpu_to_le32(nlb);
643 			range[n].slba = cpu_to_le64(slba);
644 		}
645 		n++;
646 	}
647 
648 	if (WARN_ON_ONCE(n != segments)) {
649 		if (virt_to_page(range) == ns->ctrl->discard_page)
650 			clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
651 		else
652 			kfree(range);
653 		return BLK_STS_IOERR;
654 	}
655 
656 	cmnd->dsm.opcode = nvme_cmd_dsm;
657 	cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
658 	cmnd->dsm.nr = cpu_to_le32(segments - 1);
659 	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
660 
661 	req->special_vec.bv_page = virt_to_page(range);
662 	req->special_vec.bv_offset = offset_in_page(range);
663 	req->special_vec.bv_len = alloc_size;
664 	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
665 
666 	return BLK_STS_OK;
667 }
668 
669 static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
670 		struct request *req, struct nvme_command *cmnd)
671 {
672 	if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
673 		return nvme_setup_discard(ns, req, cmnd);
674 
675 	cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
676 	cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
677 	cmnd->write_zeroes.slba =
678 		cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
679 	cmnd->write_zeroes.length =
680 		cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
681 	cmnd->write_zeroes.control = 0;
682 	return BLK_STS_OK;
683 }
684 
685 static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
686 		struct request *req, struct nvme_command *cmnd)
687 {
688 	struct nvme_ctrl *ctrl = ns->ctrl;
689 	u16 control = 0;
690 	u32 dsmgmt = 0;
691 
692 	if (req->cmd_flags & REQ_FUA)
693 		control |= NVME_RW_FUA;
694 	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
695 		control |= NVME_RW_LR;
696 
697 	if (req->cmd_flags & REQ_RAHEAD)
698 		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;
699 
700 	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
701 	cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
702 	cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
703 	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
704 
705 	if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
706 		nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);
707 
708 	if (ns->ms) {
709 		/*
710 		 * If formated with metadata, the block layer always provides a
711 		 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
712 		 * we enable the PRACT bit for protection information or set the
713 		 * namespace capacity to zero to prevent any I/O.
714 		 */
715 		if (!blk_integrity_rq(req)) {
716 			if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
717 				return BLK_STS_NOTSUPP;
718 			control |= NVME_RW_PRINFO_PRACT;
719 		}
720 
721 		switch (ns->pi_type) {
722 		case NVME_NS_DPS_PI_TYPE3:
723 			control |= NVME_RW_PRINFO_PRCHK_GUARD;
724 			break;
725 		case NVME_NS_DPS_PI_TYPE1:
726 		case NVME_NS_DPS_PI_TYPE2:
727 			control |= NVME_RW_PRINFO_PRCHK_GUARD |
728 					NVME_RW_PRINFO_PRCHK_REF;
729 			cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
730 			break;
731 		}
732 	}
733 
734 	cmnd->rw.control = cpu_to_le16(control);
735 	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
736 	return 0;
737 }
738 
739 void nvme_cleanup_cmd(struct request *req)
740 {
741 	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
742 		struct nvme_ns *ns = req->rq_disk->private_data;
743 		struct page *page = req->special_vec.bv_page;
744 
745 		if (page == ns->ctrl->discard_page)
746 			clear_bit_unlock(0, &ns->ctrl->discard_page_busy);
747 		else
748 			kfree(page_address(page) + req->special_vec.bv_offset);
749 	}
750 }
751 EXPORT_SYMBOL_GPL(nvme_cleanup_cmd);
752 
753 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
754 		struct nvme_command *cmd)
755 {
756 	blk_status_t ret = BLK_STS_OK;
757 
758 	nvme_clear_nvme_request(req);
759 
760 	memset(cmd, 0, sizeof(*cmd));
761 	switch (req_op(req)) {
762 	case REQ_OP_DRV_IN:
763 	case REQ_OP_DRV_OUT:
764 		memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
765 		break;
766 	case REQ_OP_FLUSH:
767 		nvme_setup_flush(ns, cmd);
768 		break;
769 	case REQ_OP_WRITE_ZEROES:
770 		ret = nvme_setup_write_zeroes(ns, req, cmd);
771 		break;
772 	case REQ_OP_DISCARD:
773 		ret = nvme_setup_discard(ns, req, cmd);
774 		break;
775 	case REQ_OP_READ:
776 	case REQ_OP_WRITE:
777 		ret = nvme_setup_rw(ns, req, cmd);
778 		break;
779 	default:
780 		WARN_ON_ONCE(1);
781 		return BLK_STS_IOERR;
782 	}
783 
784 	cmd->common.command_id = req->tag;
785 	trace_nvme_setup_cmd(req, cmd);
786 	return ret;
787 }
788 EXPORT_SYMBOL_GPL(nvme_setup_cmd);
789 
790 static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
791 {
792 	struct completion *waiting = rq->end_io_data;
793 
794 	rq->end_io_data = NULL;
795 	complete(waiting);
796 }
797 
798 static void nvme_execute_rq_polled(struct request_queue *q,
799 		struct gendisk *bd_disk, struct request *rq, int at_head)
800 {
801 	DECLARE_COMPLETION_ONSTACK(wait);
802 
803 	WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
804 
805 	rq->cmd_flags |= REQ_HIPRI;
806 	rq->end_io_data = &wait;
807 	blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
808 
809 	while (!completion_done(&wait)) {
810 		blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
811 		cond_resched();
812 	}
813 }
814 
815 /*
816  * Returns 0 on success.  If the result is negative, it's a Linux error code;
817  * if the result is positive, it's an NVM Express status code
818  */
819 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
820 		union nvme_result *result, void *buffer, unsigned bufflen,
821 		unsigned timeout, int qid, int at_head,
822 		blk_mq_req_flags_t flags, bool poll)
823 {
824 	struct request *req;
825 	int ret;
826 
827 	req = nvme_alloc_request(q, cmd, flags, qid);
828 	if (IS_ERR(req))
829 		return PTR_ERR(req);
830 
831 	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
832 
833 	if (buffer && bufflen) {
834 		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
835 		if (ret)
836 			goto out;
837 	}
838 
839 	if (poll)
840 		nvme_execute_rq_polled(req->q, NULL, req, at_head);
841 	else
842 		blk_execute_rq(req->q, NULL, req, at_head);
843 	if (result)
844 		*result = nvme_req(req)->result;
845 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
846 		ret = -EINTR;
847 	else
848 		ret = nvme_req(req)->status;
849  out:
850 	blk_mq_free_request(req);
851 	return ret;
852 }
853 EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
854 
855 int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
856 		void *buffer, unsigned bufflen)
857 {
858 	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
859 			NVME_QID_ANY, 0, 0, false);
860 }
861 EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
862 
863 static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
864 		unsigned len, u32 seed, bool write)
865 {
866 	struct bio_integrity_payload *bip;
867 	int ret = -ENOMEM;
868 	void *buf;
869 
870 	buf = kmalloc(len, GFP_KERNEL);
871 	if (!buf)
872 		goto out;
873 
874 	ret = -EFAULT;
875 	if (write && copy_from_user(buf, ubuf, len))
876 		goto out_free_meta;
877 
878 	bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
879 	if (IS_ERR(bip)) {
880 		ret = PTR_ERR(bip);
881 		goto out_free_meta;
882 	}
883 
884 	bip->bip_iter.bi_size = len;
885 	bip->bip_iter.bi_sector = seed;
886 	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
887 			offset_in_page(buf));
888 	if (ret == len)
889 		return buf;
890 	ret = -ENOMEM;
891 out_free_meta:
892 	kfree(buf);
893 out:
894 	return ERR_PTR(ret);
895 }
896 
897 static int nvme_submit_user_cmd(struct request_queue *q,
898 		struct nvme_command *cmd, void __user *ubuffer,
899 		unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
900 		u32 meta_seed, u64 *result, unsigned timeout)
901 {
902 	bool write = nvme_is_write(cmd);
903 	struct nvme_ns *ns = q->queuedata;
904 	struct gendisk *disk = ns ? ns->disk : NULL;
905 	struct request *req;
906 	struct bio *bio = NULL;
907 	void *meta = NULL;
908 	int ret;
909 
910 	req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
911 	if (IS_ERR(req))
912 		return PTR_ERR(req);
913 
914 	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
915 	nvme_req(req)->flags |= NVME_REQ_USERCMD;
916 
917 	if (ubuffer && bufflen) {
918 		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
919 				GFP_KERNEL);
920 		if (ret)
921 			goto out;
922 		bio = req->bio;
923 		bio->bi_disk = disk;
924 		if (disk && meta_buffer && meta_len) {
925 			meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
926 					meta_seed, write);
927 			if (IS_ERR(meta)) {
928 				ret = PTR_ERR(meta);
929 				goto out_unmap;
930 			}
931 			req->cmd_flags |= REQ_INTEGRITY;
932 		}
933 	}
934 
935 	blk_execute_rq(req->q, disk, req, 0);
936 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
937 		ret = -EINTR;
938 	else
939 		ret = nvme_req(req)->status;
940 	if (result)
941 		*result = le64_to_cpu(nvme_req(req)->result.u64);
942 	if (meta && !ret && !write) {
943 		if (copy_to_user(meta_buffer, meta, meta_len))
944 			ret = -EFAULT;
945 	}
946 	kfree(meta);
947  out_unmap:
948 	if (bio)
949 		blk_rq_unmap_user(bio);
950  out:
951 	blk_mq_free_request(req);
952 	return ret;
953 }
954 
955 static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
956 {
957 	struct nvme_ctrl *ctrl = rq->end_io_data;
958 	unsigned long flags;
959 	bool startka = false;
960 
961 	blk_mq_free_request(rq);
962 
963 	if (status) {
964 		dev_err(ctrl->device,
965 			"failed nvme_keep_alive_end_io error=%d\n",
966 				status);
967 		return;
968 	}
969 
970 	ctrl->comp_seen = false;
971 	spin_lock_irqsave(&ctrl->lock, flags);
972 	if (ctrl->state == NVME_CTRL_LIVE ||
973 	    ctrl->state == NVME_CTRL_CONNECTING)
974 		startka = true;
975 	spin_unlock_irqrestore(&ctrl->lock, flags);
976 	if (startka)
977 		schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
978 }
979 
980 static int nvme_keep_alive(struct nvme_ctrl *ctrl)
981 {
982 	struct request *rq;
983 
984 	rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
985 			NVME_QID_ANY);
986 	if (IS_ERR(rq))
987 		return PTR_ERR(rq);
988 
989 	rq->timeout = ctrl->kato * HZ;
990 	rq->end_io_data = ctrl;
991 
992 	blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);
993 
994 	return 0;
995 }
996 
997 static void nvme_keep_alive_work(struct work_struct *work)
998 {
999 	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
1000 			struct nvme_ctrl, ka_work);
1001 	bool comp_seen = ctrl->comp_seen;
1002 
1003 	if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
1004 		dev_dbg(ctrl->device,
1005 			"reschedule traffic based keep-alive timer\n");
1006 		ctrl->comp_seen = false;
1007 		schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
1008 		return;
1009 	}
1010 
1011 	if (nvme_keep_alive(ctrl)) {
1012 		/* allocation failure, reset the controller */
1013 		dev_err(ctrl->device, "keep-alive failed\n");
1014 		nvme_reset_ctrl(ctrl);
1015 		return;
1016 	}
1017 }
1018 
1019 static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
1020 {
1021 	if (unlikely(ctrl->kato == 0))
1022 		return;
1023 
1024 	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
1025 }
1026 
1027 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
1028 {
1029 	if (unlikely(ctrl->kato == 0))
1030 		return;
1031 
1032 	cancel_delayed_work_sync(&ctrl->ka_work);
1033 }
1034 EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);
1035 
1036 static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
1037 {
1038 	struct nvme_command c = { };
1039 	int error;
1040 
1041 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1042 	c.identify.opcode = nvme_admin_identify;
1043 	c.identify.cns = NVME_ID_CNS_CTRL;
1044 
1045 	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
1046 	if (!*id)
1047 		return -ENOMEM;
1048 
1049 	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
1050 			sizeof(struct nvme_id_ctrl));
1051 	if (error)
1052 		kfree(*id);
1053 	return error;
1054 }
1055 
1056 static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
1057 		struct nvme_ns_ids *ids)
1058 {
1059 	struct nvme_command c = { };
1060 	int status;
1061 	void *data;
1062 	int pos;
1063 	int len;
1064 
1065 	c.identify.opcode = nvme_admin_identify;
1066 	c.identify.nsid = cpu_to_le32(nsid);
1067 	c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;
1068 
1069 	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
1070 	if (!data)
1071 		return -ENOMEM;
1072 
1073 	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
1074 				      NVME_IDENTIFY_DATA_SIZE);
1075 	if (status)
1076 		goto free_data;
1077 
1078 	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
1079 		struct nvme_ns_id_desc *cur = data + pos;
1080 
1081 		if (cur->nidl == 0)
1082 			break;
1083 
1084 		switch (cur->nidt) {
1085 		case NVME_NIDT_EUI64:
1086 			if (cur->nidl != NVME_NIDT_EUI64_LEN) {
1087 				dev_warn(ctrl->device,
1088 					 "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
1089 					 cur->nidl);
1090 				goto free_data;
1091 			}
1092 			len = NVME_NIDT_EUI64_LEN;
1093 			memcpy(ids->eui64, data + pos + sizeof(*cur), len);
1094 			break;
1095 		case NVME_NIDT_NGUID:
1096 			if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1097 				dev_warn(ctrl->device,
1098 					 "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
1099 					 cur->nidl);
1100 				goto free_data;
1101 			}
1102 			len = NVME_NIDT_NGUID_LEN;
1103 			memcpy(ids->nguid, data + pos + sizeof(*cur), len);
1104 			break;
1105 		case NVME_NIDT_UUID:
1106 			if (cur->nidl != NVME_NIDT_UUID_LEN) {
1107 				dev_warn(ctrl->device,
1108 					 "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
1109 					 cur->nidl);
1110 				goto free_data;
1111 			}
1112 			len = NVME_NIDT_UUID_LEN;
1113 			uuid_copy(&ids->uuid, data + pos + sizeof(*cur));
1114 			break;
1115 		default:
1116 			/* Skip unknown types */
1117 			len = cur->nidl;
1118 			break;
1119 		}
1120 
1121 		len += sizeof(*cur);
1122 	}
1123 free_data:
1124 	kfree(data);
1125 	return status;
1126 }
1127 
1128 static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
1129 {
1130 	struct nvme_command c = { };
1131 
1132 	c.identify.opcode = nvme_admin_identify;
1133 	c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
1134 	c.identify.nsid = cpu_to_le32(nsid);
1135 	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
1136 				    NVME_IDENTIFY_DATA_SIZE);
1137 }
1138 
1139 static int nvme_identify_ns(struct nvme_ctrl *ctrl,
1140 		unsigned nsid, struct nvme_id_ns **id)
1141 {
1142 	struct nvme_command c = { };
1143 	int error;
1144 
1145 	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
1146 	c.identify.opcode = nvme_admin_identify;
1147 	c.identify.nsid = cpu_to_le32(nsid);
1148 	c.identify.cns = NVME_ID_CNS_NS;
1149 
1150 	*id = kmalloc(sizeof(**id), GFP_KERNEL);
1151 	if (!*id)
1152 		return -ENOMEM;
1153 
1154 	error = nvme_submit_sync_cmd(ctrl->admin_q, &c, *id, sizeof(**id));
1155 	if (error) {
1156 		dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
1157 		kfree(*id);
1158 	}
1159 
1160 	return error;
1161 }
1162 
1163 static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
1164 		unsigned int dword11, void *buffer, size_t buflen, u32 *result)
1165 {
1166 	struct nvme_command c;
1167 	union nvme_result res;
1168 	int ret;
1169 
1170 	memset(&c, 0, sizeof(c));
1171 	c.features.opcode = op;
1172 	c.features.fid = cpu_to_le32(fid);
1173 	c.features.dword11 = cpu_to_le32(dword11);
1174 
1175 	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1176 			buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1177 	if (ret >= 0 && result)
1178 		*result = le32_to_cpu(res.u32);
1179 	return ret;
1180 }
1181 
1182 int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
1183 		      unsigned int dword11, void *buffer, size_t buflen,
1184 		      u32 *result)
1185 {
1186 	return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
1187 			     buflen, result);
1188 }
1189 EXPORT_SYMBOL_GPL(nvme_set_features);
1190 
1191 int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
1192 		      unsigned int dword11, void *buffer, size_t buflen,
1193 		      u32 *result)
1194 {
1195 	return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
1196 			     buflen, result);
1197 }
1198 EXPORT_SYMBOL_GPL(nvme_get_features);
1199 
1200 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
1201 {
1202 	u32 q_count = (*count - 1) | ((*count - 1) << 16);
1203 	u32 result;
1204 	int status, nr_io_queues;
1205 
1206 	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
1207 			&result);
1208 	if (status < 0)
1209 		return status;
1210 
1211 	/*
1212 	 * Degraded controllers might return an error when setting the queue
1213 	 * count.  We still want to be able to bring them online and offer
1214 	 * access to the admin queue, as that might be only way to fix them up.
1215 	 */
1216 	if (status > 0) {
1217 		dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1218 		*count = 0;
1219 	} else {
1220 		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
1221 		*count = min(*count, nr_io_queues);
1222 	}
1223 
1224 	return 0;
1225 }
1226 EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1227 
1228 #define NVME_AEN_SUPPORTED \
1229 	(NVME_AEN_CFG_NS_ATTR | NVME_AEN_CFG_FW_ACT | \
1230 	 NVME_AEN_CFG_ANA_CHANGE | NVME_AEN_CFG_DISC_CHANGE)
1231 
1232 static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1233 {
1234 	u32 result, supported_aens = ctrl->oaes & NVME_AEN_SUPPORTED;
1235 	int status;
1236 
1237 	if (!supported_aens)
1238 		return;
1239 
1240 	status = nvme_set_features(ctrl, NVME_FEAT_ASYNC_EVENT, supported_aens,
1241 			NULL, 0, &result);
1242 	if (status)
1243 		dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1244 			 supported_aens);
1245 
1246 	queue_work(nvme_wq, &ctrl->async_event_work);
1247 }
1248 
1249 static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
1250 {
1251 	struct nvme_user_io io;
1252 	struct nvme_command c;
1253 	unsigned length, meta_len;
1254 	void __user *metadata;
1255 
1256 	if (copy_from_user(&io, uio, sizeof(io)))
1257 		return -EFAULT;
1258 	if (io.flags)
1259 		return -EINVAL;
1260 
1261 	switch (io.opcode) {
1262 	case nvme_cmd_write:
1263 	case nvme_cmd_read:
1264 	case nvme_cmd_compare:
1265 		break;
1266 	default:
1267 		return -EINVAL;
1268 	}
1269 
1270 	length = (io.nblocks + 1) << ns->lba_shift;
1271 	meta_len = (io.nblocks + 1) * ns->ms;
1272 	metadata = (void __user *)(uintptr_t)io.metadata;
1273 
1274 	if (ns->ext) {
1275 		length += meta_len;
1276 		meta_len = 0;
1277 	} else if (meta_len) {
1278 		if ((io.metadata & 3) || !io.metadata)
1279 			return -EINVAL;
1280 	}
1281 
1282 	memset(&c, 0, sizeof(c));
1283 	c.rw.opcode = io.opcode;
1284 	c.rw.flags = io.flags;
1285 	c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1286 	c.rw.slba = cpu_to_le64(io.slba);
1287 	c.rw.length = cpu_to_le16(io.nblocks);
1288 	c.rw.control = cpu_to_le16(io.control);
1289 	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
1290 	c.rw.reftag = cpu_to_le32(io.reftag);
1291 	c.rw.apptag = cpu_to_le16(io.apptag);
1292 	c.rw.appmask = cpu_to_le16(io.appmask);
1293 
1294 	return nvme_submit_user_cmd(ns->queue, &c,
1295 			(void __user *)(uintptr_t)io.addr, length,
1296 			metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1297 }
1298 
1299 static u32 nvme_known_admin_effects(u8 opcode)
1300 {
1301 	switch (opcode) {
1302 	case nvme_admin_format_nvm:
1303 		return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
1304 					NVME_CMD_EFFECTS_CSE_MASK;
1305 	case nvme_admin_sanitize_nvm:
1306 		return NVME_CMD_EFFECTS_CSE_MASK;
1307 	default:
1308 		break;
1309 	}
1310 	return 0;
1311 }
1312 
1313 static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1314 								u8 opcode)
1315 {
1316 	u32 effects = 0;
1317 
1318 	if (ns) {
1319 		if (ctrl->effects)
1320 			effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1321 		if (effects & ~(NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC))
1322 			dev_warn(ctrl->device,
1323 				 "IO command:%02x has unhandled effects:%08x\n",
1324 				 opcode, effects);
1325 		return 0;
1326 	}
1327 
1328 	if (ctrl->effects)
1329 		effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1330 	effects |= nvme_known_admin_effects(opcode);
1331 
1332 	/*
1333 	 * For simplicity, IO to all namespaces is quiesced even if the command
1334 	 * effects say only one namespace is affected.
1335 	 */
1336 	if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1337 		mutex_lock(&ctrl->scan_lock);
1338 		mutex_lock(&ctrl->subsys->lock);
1339 		nvme_mpath_start_freeze(ctrl->subsys);
1340 		nvme_mpath_wait_freeze(ctrl->subsys);
1341 		nvme_start_freeze(ctrl);
1342 		nvme_wait_freeze(ctrl);
1343 	}
1344 	return effects;
1345 }
1346 
1347 static void nvme_update_formats(struct nvme_ctrl *ctrl)
1348 {
1349 	struct nvme_ns *ns;
1350 
1351 	down_read(&ctrl->namespaces_rwsem);
1352 	list_for_each_entry(ns, &ctrl->namespaces, list)
1353 		if (ns->disk && nvme_revalidate_disk(ns->disk))
1354 			nvme_set_queue_dying(ns);
1355 	up_read(&ctrl->namespaces_rwsem);
1356 }
1357 
1358 static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1359 {
1360 	/*
1361 	 * Revalidate LBA changes prior to unfreezing. This is necessary to
1362 	 * prevent memory corruption if a logical block size was changed by
1363 	 * this command.
1364 	 */
1365 	if (effects & NVME_CMD_EFFECTS_LBCC)
1366 		nvme_update_formats(ctrl);
1367 	if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
1368 		nvme_unfreeze(ctrl);
1369 		nvme_mpath_unfreeze(ctrl->subsys);
1370 		mutex_unlock(&ctrl->subsys->lock);
1371 		nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1372 		mutex_unlock(&ctrl->scan_lock);
1373 	}
1374 	if (effects & NVME_CMD_EFFECTS_CCC)
1375 		nvme_init_identify(ctrl);
1376 	if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))
1377 		nvme_queue_scan(ctrl);
1378 }
1379 
1380 static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1381 			struct nvme_passthru_cmd __user *ucmd)
1382 {
1383 	struct nvme_passthru_cmd cmd;
1384 	struct nvme_command c;
1385 	unsigned timeout = 0;
1386 	u32 effects;
1387 	u64 result;
1388 	int status;
1389 
1390 	if (!capable(CAP_SYS_ADMIN))
1391 		return -EACCES;
1392 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1393 		return -EFAULT;
1394 	if (cmd.flags)
1395 		return -EINVAL;
1396 
1397 	memset(&c, 0, sizeof(c));
1398 	c.common.opcode = cmd.opcode;
1399 	c.common.flags = cmd.flags;
1400 	c.common.nsid = cpu_to_le32(cmd.nsid);
1401 	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1402 	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1403 	c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1404 	c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1405 	c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1406 	c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1407 	c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1408 	c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1409 
1410 	if (cmd.timeout_ms)
1411 		timeout = msecs_to_jiffies(cmd.timeout_ms);
1412 
1413 	effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1414 	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1415 			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1416 			(void __user *)(uintptr_t)cmd.metadata,
1417 			cmd.metadata_len, 0, &result, timeout);
1418 	nvme_passthru_end(ctrl, effects);
1419 
1420 	if (status >= 0) {
1421 		if (put_user(result, &ucmd->result))
1422 			return -EFAULT;
1423 	}
1424 
1425 	return status;
1426 }
1427 
1428 static int nvme_user_cmd64(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1429 			struct nvme_passthru_cmd64 __user *ucmd)
1430 {
1431 	struct nvme_passthru_cmd64 cmd;
1432 	struct nvme_command c;
1433 	unsigned timeout = 0;
1434 	u32 effects;
1435 	int status;
1436 
1437 	if (!capable(CAP_SYS_ADMIN))
1438 		return -EACCES;
1439 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1440 		return -EFAULT;
1441 	if (cmd.flags)
1442 		return -EINVAL;
1443 
1444 	memset(&c, 0, sizeof(c));
1445 	c.common.opcode = cmd.opcode;
1446 	c.common.flags = cmd.flags;
1447 	c.common.nsid = cpu_to_le32(cmd.nsid);
1448 	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1449 	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1450 	c.common.cdw10 = cpu_to_le32(cmd.cdw10);
1451 	c.common.cdw11 = cpu_to_le32(cmd.cdw11);
1452 	c.common.cdw12 = cpu_to_le32(cmd.cdw12);
1453 	c.common.cdw13 = cpu_to_le32(cmd.cdw13);
1454 	c.common.cdw14 = cpu_to_le32(cmd.cdw14);
1455 	c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1456 
1457 	if (cmd.timeout_ms)
1458 		timeout = msecs_to_jiffies(cmd.timeout_ms);
1459 
1460 	effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1461 	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1462 			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1463 			(void __user *)(uintptr_t)cmd.metadata, cmd.metadata_len,
1464 			0, &cmd.result, timeout);
1465 	nvme_passthru_end(ctrl, effects);
1466 
1467 	if (status >= 0) {
1468 		if (put_user(cmd.result, &ucmd->result))
1469 			return -EFAULT;
1470 	}
1471 
1472 	return status;
1473 }
1474 
1475 /*
1476  * Issue ioctl requests on the first available path.  Note that unlike normal
1477  * block layer requests we will not retry failed request on another controller.
1478  */
1479 static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
1480 		struct nvme_ns_head **head, int *srcu_idx)
1481 {
1482 #ifdef CONFIG_NVME_MULTIPATH
1483 	if (disk->fops == &nvme_ns_head_ops) {
1484 		struct nvme_ns *ns;
1485 
1486 		*head = disk->private_data;
1487 		*srcu_idx = srcu_read_lock(&(*head)->srcu);
1488 		ns = nvme_find_path(*head);
1489 		if (!ns)
1490 			srcu_read_unlock(&(*head)->srcu, *srcu_idx);
1491 		return ns;
1492 	}
1493 #endif
1494 	*head = NULL;
1495 	*srcu_idx = -1;
1496 	return disk->private_data;
1497 }
1498 
1499 static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1500 {
1501 	if (head)
1502 		srcu_read_unlock(&head->srcu, idx);
1503 }
1504 
1505 static bool is_ctrl_ioctl(unsigned int cmd)
1506 {
1507 	if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)
1508 		return true;
1509 	if (is_sed_ioctl(cmd))
1510 		return true;
1511 	return false;
1512 }
1513 
1514 static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
1515 				  void __user *argp,
1516 				  struct nvme_ns_head *head,
1517 				  int srcu_idx)
1518 {
1519 	struct nvme_ctrl *ctrl = ns->ctrl;
1520 	int ret;
1521 
1522 	nvme_get_ctrl(ns->ctrl);
1523 	nvme_put_ns_from_disk(head, srcu_idx);
1524 
1525 	switch (cmd) {
1526 	case NVME_IOCTL_ADMIN_CMD:
1527 		ret = nvme_user_cmd(ctrl, NULL, argp);
1528 		break;
1529 	case NVME_IOCTL_ADMIN64_CMD:
1530 		ret = nvme_user_cmd64(ctrl, NULL, argp);
1531 		break;
1532 	default:
1533 		ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1534 		break;
1535 	}
1536 	nvme_put_ctrl(ctrl);
1537 	return ret;
1538 }
1539 
1540 static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
1541 		unsigned int cmd, unsigned long arg)
1542 {
1543 	struct nvme_ns_head *head = NULL;
1544 	void __user *argp = (void __user *)arg;
1545 	struct nvme_ns *ns;
1546 	int srcu_idx, ret;
1547 
1548 	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1549 	if (unlikely(!ns))
1550 		return -EWOULDBLOCK;
1551 
1552 	/*
1553 	 * Handle ioctls that apply to the controller instead of the namespace
1554 	 * seperately and drop the ns SRCU reference early.  This avoids a
1555 	 * deadlock when deleting namespaces using the passthrough interface.
1556 	 */
1557 	if (is_ctrl_ioctl(cmd))
1558 		return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1559 
1560 	switch (cmd) {
1561 	case NVME_IOCTL_ID:
1562 		force_successful_syscall_return();
1563 		ret = ns->head->ns_id;
1564 		break;
1565 	case NVME_IOCTL_IO_CMD:
1566 		ret = nvme_user_cmd(ns->ctrl, ns, argp);
1567 		break;
1568 	case NVME_IOCTL_SUBMIT_IO:
1569 		ret = nvme_submit_io(ns, argp);
1570 		break;
1571 	case NVME_IOCTL_IO64_CMD:
1572 		ret = nvme_user_cmd64(ns->ctrl, ns, argp);
1573 		break;
1574 	default:
1575 		if (ns->ndev)
1576 			ret = nvme_nvm_ioctl(ns, cmd, arg);
1577 		else
1578 			ret = -ENOTTY;
1579 	}
1580 
1581 	nvme_put_ns_from_disk(head, srcu_idx);
1582 	return ret;
1583 }
1584 
1585 static int nvme_open(struct block_device *bdev, fmode_t mode)
1586 {
1587 	struct nvme_ns *ns = bdev->bd_disk->private_data;
1588 
1589 #ifdef CONFIG_NVME_MULTIPATH
1590 	/* should never be called due to GENHD_FL_HIDDEN */
1591 	if (WARN_ON_ONCE(ns->head->disk))
1592 		goto fail;
1593 #endif
1594 	if (!kref_get_unless_zero(&ns->kref))
1595 		goto fail;
1596 	if (!try_module_get(ns->ctrl->ops->module))
1597 		goto fail_put_ns;
1598 
1599 	return 0;
1600 
1601 fail_put_ns:
1602 	nvme_put_ns(ns);
1603 fail:
1604 	return -ENXIO;
1605 }
1606 
1607 static void nvme_release(struct gendisk *disk, fmode_t mode)
1608 {
1609 	struct nvme_ns *ns = disk->private_data;
1610 
1611 	module_put(ns->ctrl->ops->module);
1612 	nvme_put_ns(ns);
1613 }
1614 
1615 static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
1616 {
1617 	/* some standard values */
1618 	geo->heads = 1 << 6;
1619 	geo->sectors = 1 << 5;
1620 	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
1621 	return 0;
1622 }
1623 
1624 #ifdef CONFIG_BLK_DEV_INTEGRITY
1625 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1626 {
1627 	struct blk_integrity integrity;
1628 
1629 	memset(&integrity, 0, sizeof(integrity));
1630 	switch (pi_type) {
1631 	case NVME_NS_DPS_PI_TYPE3:
1632 		integrity.profile = &t10_pi_type3_crc;
1633 		integrity.tag_size = sizeof(u16) + sizeof(u32);
1634 		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1635 		break;
1636 	case NVME_NS_DPS_PI_TYPE1:
1637 	case NVME_NS_DPS_PI_TYPE2:
1638 		integrity.profile = &t10_pi_type1_crc;
1639 		integrity.tag_size = sizeof(u16);
1640 		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1641 		break;
1642 	default:
1643 		integrity.profile = NULL;
1644 		break;
1645 	}
1646 	integrity.tuple_size = ms;
1647 	blk_integrity_register(disk, &integrity);
1648 	blk_queue_max_integrity_segments(disk->queue, 1);
1649 }
1650 #else
1651 static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1652 {
1653 }
1654 #endif /* CONFIG_BLK_DEV_INTEGRITY */
1655 
1656 static void nvme_set_chunk_size(struct nvme_ns *ns)
1657 {
1658 	u32 chunk_size = nvme_lba_to_sect(ns, ns->noiob);
1659 	blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
1660 }
1661 
1662 static void nvme_config_discard(struct gendisk *disk, struct nvme_ns *ns)
1663 {
1664 	struct nvme_ctrl *ctrl = ns->ctrl;
1665 	struct request_queue *queue = disk->queue;
1666 	u32 size = queue_logical_block_size(queue);
1667 
1668 	if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
1669 		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
1670 		return;
1671 	}
1672 
1673 	if (ctrl->nr_streams && ns->sws && ns->sgs)
1674 		size *= ns->sws * ns->sgs;
1675 
1676 	BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
1677 			NVME_DSM_MAX_RANGES);
1678 
1679 	queue->limits.discard_alignment = 0;
1680 	queue->limits.discard_granularity = size;
1681 
1682 	/* If discard is already enabled, don't reset queue limits */
1683 	if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
1684 		return;
1685 
1686 	blk_queue_max_discard_sectors(queue, UINT_MAX);
1687 	blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1688 
1689 	if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1690 		blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1691 }
1692 
1693 static void nvme_config_write_zeroes(struct gendisk *disk, struct nvme_ns *ns)
1694 {
1695 	u64 max_blocks;
1696 
1697 	if (!(ns->ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) ||
1698 	    (ns->ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
1699 		return;
1700 	/*
1701 	 * Even though NVMe spec explicitly states that MDTS is not
1702 	 * applicable to the write-zeroes:- "The restriction does not apply to
1703 	 * commands that do not transfer data between the host and the
1704 	 * controller (e.g., Write Uncorrectable ro Write Zeroes command).".
1705 	 * In order to be more cautious use controller's max_hw_sectors value
1706 	 * to configure the maximum sectors for the write-zeroes which is
1707 	 * configured based on the controller's MDTS field in the
1708 	 * nvme_init_identify() if available.
1709 	 */
1710 	if (ns->ctrl->max_hw_sectors == UINT_MAX)
1711 		max_blocks = (u64)USHRT_MAX + 1;
1712 	else
1713 		max_blocks = ns->ctrl->max_hw_sectors + 1;
1714 
1715 	blk_queue_max_write_zeroes_sectors(disk->queue,
1716 					   nvme_lba_to_sect(ns, max_blocks));
1717 }
1718 
1719 static int nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1720 		struct nvme_id_ns *id, struct nvme_ns_ids *ids)
1721 {
1722 	int ret = 0;
1723 
1724 	memset(ids, 0, sizeof(*ids));
1725 
1726 	if (ctrl->vs >= NVME_VS(1, 1, 0))
1727 		memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
1728 	if (ctrl->vs >= NVME_VS(1, 2, 0))
1729 		memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
1730 	if (ctrl->vs >= NVME_VS(1, 3, 0)) {
1731 		 /* Don't treat error as fatal we potentially
1732 		  * already have a NGUID or EUI-64
1733 		  */
1734 		ret = nvme_identify_ns_descs(ctrl, nsid, ids);
1735 		if (ret)
1736 			dev_warn(ctrl->device,
1737 				 "Identify Descriptors failed (%d)\n", ret);
1738 	}
1739 	return ret;
1740 }
1741 
1742 static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
1743 {
1744 	return !uuid_is_null(&ids->uuid) ||
1745 		memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
1746 		memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
1747 }
1748 
1749 static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
1750 {
1751 	return uuid_equal(&a->uuid, &b->uuid) &&
1752 		memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1753 		memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0;
1754 }
1755 
1756 static void nvme_update_disk_info(struct gendisk *disk,
1757 		struct nvme_ns *ns, struct nvme_id_ns *id)
1758 {
1759 	sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
1760 	unsigned short bs = 1 << ns->lba_shift;
1761 	u32 atomic_bs, phys_bs, io_opt;
1762 
1763 	if (ns->lba_shift > PAGE_SHIFT) {
1764 		/* unsupported block size, set capacity to 0 later */
1765 		bs = (1 << 9);
1766 	}
1767 	blk_mq_freeze_queue(disk->queue);
1768 	blk_integrity_unregister(disk);
1769 
1770 	if (id->nabo == 0) {
1771 		/*
1772 		 * Bit 1 indicates whether NAWUPF is defined for this namespace
1773 		 * and whether it should be used instead of AWUPF. If NAWUPF ==
1774 		 * 0 then AWUPF must be used instead.
1775 		 */
1776 		if (id->nsfeat & (1 << 1) && id->nawupf)
1777 			atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
1778 		else
1779 			atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
1780 	} else {
1781 		atomic_bs = bs;
1782 	}
1783 	phys_bs = bs;
1784 	io_opt = bs;
1785 	if (id->nsfeat & (1 << 4)) {
1786 		/* NPWG = Namespace Preferred Write Granularity */
1787 		phys_bs *= 1 + le16_to_cpu(id->npwg);
1788 		/* NOWS = Namespace Optimal Write Size */
1789 		io_opt *= 1 + le16_to_cpu(id->nows);
1790 	}
1791 
1792 	blk_queue_logical_block_size(disk->queue, bs);
1793 	/*
1794 	 * Linux filesystems assume writing a single physical block is
1795 	 * an atomic operation. Hence limit the physical block size to the
1796 	 * value of the Atomic Write Unit Power Fail parameter.
1797 	 */
1798 	blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
1799 	blk_queue_io_min(disk->queue, phys_bs);
1800 	blk_queue_io_opt(disk->queue, io_opt);
1801 
1802 	if (ns->ms && !ns->ext &&
1803 	    (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1804 		nvme_init_integrity(disk, ns->ms, ns->pi_type);
1805 	if ((ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) ||
1806 	    ns->lba_shift > PAGE_SHIFT)
1807 		capacity = 0;
1808 
1809 	set_capacity(disk, capacity);
1810 
1811 	nvme_config_discard(disk, ns);
1812 	nvme_config_write_zeroes(disk, ns);
1813 
1814 	if (id->nsattr & (1 << 0))
1815 		set_disk_ro(disk, true);
1816 	else
1817 		set_disk_ro(disk, false);
1818 
1819 	blk_mq_unfreeze_queue(disk->queue);
1820 }
1821 
1822 static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
1823 {
1824 	struct nvme_ns *ns = disk->private_data;
1825 
1826 	/*
1827 	 * If identify namespace failed, use default 512 byte block size so
1828 	 * block layer can use before failing read/write for 0 capacity.
1829 	 */
1830 	ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
1831 	if (ns->lba_shift == 0)
1832 		ns->lba_shift = 9;
1833 	ns->noiob = le16_to_cpu(id->noiob);
1834 	ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1835 	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
1836 	/* the PI implementation requires metadata equal t10 pi tuple size */
1837 	if (ns->ms == sizeof(struct t10_pi_tuple))
1838 		ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1839 	else
1840 		ns->pi_type = 0;
1841 
1842 	if (ns->noiob)
1843 		nvme_set_chunk_size(ns);
1844 	nvme_update_disk_info(disk, ns, id);
1845 #ifdef CONFIG_NVME_MULTIPATH
1846 	if (ns->head->disk) {
1847 		nvme_update_disk_info(ns->head->disk, ns, id);
1848 		blk_queue_stack_limits(ns->head->disk->queue, ns->queue);
1849 		revalidate_disk(ns->head->disk);
1850 	}
1851 #endif
1852 }
1853 
1854 static int nvme_revalidate_disk(struct gendisk *disk)
1855 {
1856 	struct nvme_ns *ns = disk->private_data;
1857 	struct nvme_ctrl *ctrl = ns->ctrl;
1858 	struct nvme_id_ns *id;
1859 	struct nvme_ns_ids ids;
1860 	int ret = 0;
1861 
1862 	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
1863 		set_capacity(disk, 0);
1864 		return -ENODEV;
1865 	}
1866 
1867 	ret = nvme_identify_ns(ctrl, ns->head->ns_id, &id);
1868 	if (ret)
1869 		goto out;
1870 
1871 	if (id->ncap == 0) {
1872 		ret = -ENODEV;
1873 		goto free_id;
1874 	}
1875 
1876 	__nvme_revalidate_disk(disk, id);
1877 	ret = nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
1878 	if (ret)
1879 		goto free_id;
1880 
1881 	if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
1882 		dev_err(ctrl->device,
1883 			"identifiers changed for nsid %d\n", ns->head->ns_id);
1884 		ret = -ENODEV;
1885 	}
1886 
1887 free_id:
1888 	kfree(id);
1889 out:
1890 	/*
1891 	 * Only fail the function if we got a fatal error back from the
1892 	 * device, otherwise ignore the error and just move on.
1893 	 */
1894 	if (ret == -ENOMEM || (ret > 0 && !(ret & NVME_SC_DNR)))
1895 		ret = 0;
1896 	else if (ret > 0)
1897 		ret = blk_status_to_errno(nvme_error_status(ret));
1898 	return ret;
1899 }
1900 
1901 static char nvme_pr_type(enum pr_type type)
1902 {
1903 	switch (type) {
1904 	case PR_WRITE_EXCLUSIVE:
1905 		return 1;
1906 	case PR_EXCLUSIVE_ACCESS:
1907 		return 2;
1908 	case PR_WRITE_EXCLUSIVE_REG_ONLY:
1909 		return 3;
1910 	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
1911 		return 4;
1912 	case PR_WRITE_EXCLUSIVE_ALL_REGS:
1913 		return 5;
1914 	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
1915 		return 6;
1916 	default:
1917 		return 0;
1918 	}
1919 };
1920 
1921 static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
1922 				u64 key, u64 sa_key, u8 op)
1923 {
1924 	struct nvme_ns_head *head = NULL;
1925 	struct nvme_ns *ns;
1926 	struct nvme_command c;
1927 	int srcu_idx, ret;
1928 	u8 data[16] = { 0, };
1929 
1930 	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
1931 	if (unlikely(!ns))
1932 		return -EWOULDBLOCK;
1933 
1934 	put_unaligned_le64(key, &data[0]);
1935 	put_unaligned_le64(sa_key, &data[8]);
1936 
1937 	memset(&c, 0, sizeof(c));
1938 	c.common.opcode = op;
1939 	c.common.nsid = cpu_to_le32(ns->head->ns_id);
1940 	c.common.cdw10 = cpu_to_le32(cdw10);
1941 
1942 	ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1943 	nvme_put_ns_from_disk(head, srcu_idx);
1944 	return ret;
1945 }
1946 
1947 static int nvme_pr_register(struct block_device *bdev, u64 old,
1948 		u64 new, unsigned flags)
1949 {
1950 	u32 cdw10;
1951 
1952 	if (flags & ~PR_FL_IGNORE_KEY)
1953 		return -EOPNOTSUPP;
1954 
1955 	cdw10 = old ? 2 : 0;
1956 	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
1957 	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
1958 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
1959 }
1960 
1961 static int nvme_pr_reserve(struct block_device *bdev, u64 key,
1962 		enum pr_type type, unsigned flags)
1963 {
1964 	u32 cdw10;
1965 
1966 	if (flags & ~PR_FL_IGNORE_KEY)
1967 		return -EOPNOTSUPP;
1968 
1969 	cdw10 = nvme_pr_type(type) << 8;
1970 	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
1971 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
1972 }
1973 
1974 static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
1975 		enum pr_type type, bool abort)
1976 {
1977 	u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
1978 	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
1979 }
1980 
1981 static int nvme_pr_clear(struct block_device *bdev, u64 key)
1982 {
1983 	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1984 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
1985 }
1986 
1987 static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
1988 {
1989 	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
1990 	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1991 }
1992 
1993 static const struct pr_ops nvme_pr_ops = {
1994 	.pr_register	= nvme_pr_register,
1995 	.pr_reserve	= nvme_pr_reserve,
1996 	.pr_release	= nvme_pr_release,
1997 	.pr_preempt	= nvme_pr_preempt,
1998 	.pr_clear	= nvme_pr_clear,
1999 };
2000 
2001 #ifdef CONFIG_BLK_SED_OPAL
2002 int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
2003 		bool send)
2004 {
2005 	struct nvme_ctrl *ctrl = data;
2006 	struct nvme_command cmd;
2007 
2008 	memset(&cmd, 0, sizeof(cmd));
2009 	if (send)
2010 		cmd.common.opcode = nvme_admin_security_send;
2011 	else
2012 		cmd.common.opcode = nvme_admin_security_recv;
2013 	cmd.common.nsid = 0;
2014 	cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
2015 	cmd.common.cdw11 = cpu_to_le32(len);
2016 
2017 	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
2018 				      ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
2019 }
2020 EXPORT_SYMBOL_GPL(nvme_sec_submit);
2021 #endif /* CONFIG_BLK_SED_OPAL */
2022 
2023 static const struct block_device_operations nvme_fops = {
2024 	.owner		= THIS_MODULE,
2025 	.ioctl		= nvme_ioctl,
2026 	.compat_ioctl	= nvme_ioctl,
2027 	.open		= nvme_open,
2028 	.release	= nvme_release,
2029 	.getgeo		= nvme_getgeo,
2030 	.revalidate_disk= nvme_revalidate_disk,
2031 	.pr_ops		= &nvme_pr_ops,
2032 };
2033 
2034 #ifdef CONFIG_NVME_MULTIPATH
2035 static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
2036 {
2037 	struct nvme_ns_head *head = bdev->bd_disk->private_data;
2038 
2039 	if (!kref_get_unless_zero(&head->ref))
2040 		return -ENXIO;
2041 	return 0;
2042 }
2043 
2044 static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
2045 {
2046 	nvme_put_ns_head(disk->private_data);
2047 }
2048 
2049 const struct block_device_operations nvme_ns_head_ops = {
2050 	.owner		= THIS_MODULE,
2051 	.open		= nvme_ns_head_open,
2052 	.release	= nvme_ns_head_release,
2053 	.ioctl		= nvme_ioctl,
2054 	.compat_ioctl	= nvme_ioctl,
2055 	.getgeo		= nvme_getgeo,
2056 	.pr_ops		= &nvme_pr_ops,
2057 };
2058 #endif /* CONFIG_NVME_MULTIPATH */
2059 
2060 static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
2061 {
2062 	unsigned long timeout =
2063 		((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
2064 	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
2065 	int ret;
2066 
2067 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2068 		if (csts == ~0)
2069 			return -ENODEV;
2070 		if ((csts & NVME_CSTS_RDY) == bit)
2071 			break;
2072 
2073 		msleep(100);
2074 		if (fatal_signal_pending(current))
2075 			return -EINTR;
2076 		if (time_after(jiffies, timeout)) {
2077 			dev_err(ctrl->device,
2078 				"Device not ready; aborting %s\n", enabled ?
2079 						"initialisation" : "reset");
2080 			return -ENODEV;
2081 		}
2082 	}
2083 
2084 	return ret;
2085 }
2086 
2087 /*
2088  * If the device has been passed off to us in an enabled state, just clear
2089  * the enabled bit.  The spec says we should set the 'shutdown notification
2090  * bits', but doing so may cause the device to complete commands to the
2091  * admin queue ... and we don't know what memory that might be pointing at!
2092  */
2093 int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
2094 {
2095 	int ret;
2096 
2097 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2098 	ctrl->ctrl_config &= ~NVME_CC_ENABLE;
2099 
2100 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2101 	if (ret)
2102 		return ret;
2103 
2104 	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
2105 		msleep(NVME_QUIRK_DELAY_AMOUNT);
2106 
2107 	return nvme_wait_ready(ctrl, ctrl->cap, false);
2108 }
2109 EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
2110 
2111 int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
2112 {
2113 	/*
2114 	 * Default to a 4K page size, with the intention to update this
2115 	 * path in the future to accomodate architectures with differing
2116 	 * kernel and IO page sizes.
2117 	 */
2118 	unsigned dev_page_min, page_shift = 12;
2119 	int ret;
2120 
2121 	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
2122 	if (ret) {
2123 		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2124 		return ret;
2125 	}
2126 	dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2127 
2128 	if (page_shift < dev_page_min) {
2129 		dev_err(ctrl->device,
2130 			"Minimum device page size %u too large for host (%u)\n",
2131 			1 << dev_page_min, 1 << page_shift);
2132 		return -ENODEV;
2133 	}
2134 
2135 	ctrl->page_size = 1 << page_shift;
2136 
2137 	ctrl->ctrl_config = NVME_CC_CSS_NVM;
2138 	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
2139 	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
2140 	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
2141 	ctrl->ctrl_config |= NVME_CC_ENABLE;
2142 
2143 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2144 	if (ret)
2145 		return ret;
2146 	return nvme_wait_ready(ctrl, ctrl->cap, true);
2147 }
2148 EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
2149 
2150 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
2151 {
2152 	unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
2153 	u32 csts;
2154 	int ret;
2155 
2156 	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
2157 	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;
2158 
2159 	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
2160 	if (ret)
2161 		return ret;
2162 
2163 	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
2164 		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
2165 			break;
2166 
2167 		msleep(100);
2168 		if (fatal_signal_pending(current))
2169 			return -EINTR;
2170 		if (time_after(jiffies, timeout)) {
2171 			dev_err(ctrl->device,
2172 				"Device shutdown incomplete; abort shutdown\n");
2173 			return -ENODEV;
2174 		}
2175 	}
2176 
2177 	return ret;
2178 }
2179 EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
2180 
2181 static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
2182 		struct request_queue *q)
2183 {
2184 	bool vwc = false;
2185 
2186 	if (ctrl->max_hw_sectors) {
2187 		u32 max_segments =
2188 			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
2189 
2190 		max_segments = min_not_zero(max_segments, ctrl->max_segments);
2191 		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
2192 		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
2193 	}
2194 	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
2195 	    is_power_of_2(ctrl->max_hw_sectors))
2196 		blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
2197 	blk_queue_virt_boundary(q, ctrl->page_size - 1);
2198 	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
2199 		vwc = true;
2200 	blk_queue_write_cache(q, vwc, vwc);
2201 }
2202 
2203 static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
2204 {
2205 	__le64 ts;
2206 	int ret;
2207 
2208 	if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
2209 		return 0;
2210 
2211 	ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
2212 	ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
2213 			NULL);
2214 	if (ret)
2215 		dev_warn_once(ctrl->device,
2216 			"could not set timestamp (%d)\n", ret);
2217 	return ret;
2218 }
2219 
2220 static int nvme_configure_acre(struct nvme_ctrl *ctrl)
2221 {
2222 	struct nvme_feat_host_behavior *host;
2223 	int ret;
2224 
2225 	/* Don't bother enabling the feature if retry delay is not reported */
2226 	if (!ctrl->crdt[0])
2227 		return 0;
2228 
2229 	host = kzalloc(sizeof(*host), GFP_KERNEL);
2230 	if (!host)
2231 		return 0;
2232 
2233 	host->acre = NVME_ENABLE_ACRE;
2234 	ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
2235 				host, sizeof(*host), NULL);
2236 	kfree(host);
2237 	return ret;
2238 }
2239 
2240 static int nvme_configure_apst(struct nvme_ctrl *ctrl)
2241 {
2242 	/*
2243 	 * APST (Autonomous Power State Transition) lets us program a
2244 	 * table of power state transitions that the controller will
2245 	 * perform automatically.  We configure it with a simple
2246 	 * heuristic: we are willing to spend at most 2% of the time
2247 	 * transitioning between power states.  Therefore, when running
2248 	 * in any given state, we will enter the next lower-power
2249 	 * non-operational state after waiting 50 * (enlat + exlat)
2250 	 * microseconds, as long as that state's exit latency is under
2251 	 * the requested maximum latency.
2252 	 *
2253 	 * We will not autonomously enter any non-operational state for
2254 	 * which the total latency exceeds ps_max_latency_us.  Users
2255 	 * can set ps_max_latency_us to zero to turn off APST.
2256 	 */
2257 
2258 	unsigned apste;
2259 	struct nvme_feat_auto_pst *table;
2260 	u64 max_lat_us = 0;
2261 	int max_ps = -1;
2262 	int ret;
2263 
2264 	/*
2265 	 * If APST isn't supported or if we haven't been initialized yet,
2266 	 * then don't do anything.
2267 	 */
2268 	if (!ctrl->apsta)
2269 		return 0;
2270 
2271 	if (ctrl->npss > 31) {
2272 		dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
2273 		return 0;
2274 	}
2275 
2276 	table = kzalloc(sizeof(*table), GFP_KERNEL);
2277 	if (!table)
2278 		return 0;
2279 
2280 	if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
2281 		/* Turn off APST. */
2282 		apste = 0;
2283 		dev_dbg(ctrl->device, "APST disabled\n");
2284 	} else {
2285 		__le64 target = cpu_to_le64(0);
2286 		int state;
2287 
2288 		/*
2289 		 * Walk through all states from lowest- to highest-power.
2290 		 * According to the spec, lower-numbered states use more
2291 		 * power.  NPSS, despite the name, is the index of the
2292 		 * lowest-power state, not the number of states.
2293 		 */
2294 		for (state = (int)ctrl->npss; state >= 0; state--) {
2295 			u64 total_latency_us, exit_latency_us, transition_ms;
2296 
2297 			if (target)
2298 				table->entries[state] = target;
2299 
2300 			/*
2301 			 * Don't allow transitions to the deepest state
2302 			 * if it's quirked off.
2303 			 */
2304 			if (state == ctrl->npss &&
2305 			    (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
2306 				continue;
2307 
2308 			/*
2309 			 * Is this state a useful non-operational state for
2310 			 * higher-power states to autonomously transition to?
2311 			 */
2312 			if (!(ctrl->psd[state].flags &
2313 			      NVME_PS_FLAGS_NON_OP_STATE))
2314 				continue;
2315 
2316 			exit_latency_us =
2317 				(u64)le32_to_cpu(ctrl->psd[state].exit_lat);
2318 			if (exit_latency_us > ctrl->ps_max_latency_us)
2319 				continue;
2320 
2321 			total_latency_us =
2322 				exit_latency_us +
2323 				le32_to_cpu(ctrl->psd[state].entry_lat);
2324 
2325 			/*
2326 			 * This state is good.  Use it as the APST idle
2327 			 * target for higher power states.
2328 			 */
2329 			transition_ms = total_latency_us + 19;
2330 			do_div(transition_ms, 20);
2331 			if (transition_ms > (1 << 24) - 1)
2332 				transition_ms = (1 << 24) - 1;
2333 
2334 			target = cpu_to_le64((state << 3) |
2335 					     (transition_ms << 8));
2336 
2337 			if (max_ps == -1)
2338 				max_ps = state;
2339 
2340 			if (total_latency_us > max_lat_us)
2341 				max_lat_us = total_latency_us;
2342 		}
2343 
2344 		apste = 1;
2345 
2346 		if (max_ps == -1) {
2347 			dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
2348 		} else {
2349 			dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
2350 				max_ps, max_lat_us, (int)sizeof(*table), table);
2351 		}
2352 	}
2353 
2354 	ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
2355 				table, sizeof(*table), NULL);
2356 	if (ret)
2357 		dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);
2358 
2359 	kfree(table);
2360 	return ret;
2361 }
2362 
2363 static void nvme_set_latency_tolerance(struct device *dev, s32 val)
2364 {
2365 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2366 	u64 latency;
2367 
2368 	switch (val) {
2369 	case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
2370 	case PM_QOS_LATENCY_ANY:
2371 		latency = U64_MAX;
2372 		break;
2373 
2374 	default:
2375 		latency = val;
2376 	}
2377 
2378 	if (ctrl->ps_max_latency_us != latency) {
2379 		ctrl->ps_max_latency_us = latency;
2380 		nvme_configure_apst(ctrl);
2381 	}
2382 }
2383 
2384 struct nvme_core_quirk_entry {
2385 	/*
2386 	 * NVMe model and firmware strings are padded with spaces.  For
2387 	 * simplicity, strings in the quirk table are padded with NULLs
2388 	 * instead.
2389 	 */
2390 	u16 vid;
2391 	const char *mn;
2392 	const char *fr;
2393 	unsigned long quirks;
2394 };
2395 
2396 static const struct nvme_core_quirk_entry core_quirks[] = {
2397 	{
2398 		/*
2399 		 * This Toshiba device seems to die using any APST states.  See:
2400 		 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
2401 		 */
2402 		.vid = 0x1179,
2403 		.mn = "THNSF5256GPUK TOSHIBA",
2404 		.quirks = NVME_QUIRK_NO_APST,
2405 	},
2406 	{
2407 		/*
2408 		 * This LiteON CL1-3D*-Q11 firmware version has a race
2409 		 * condition associated with actions related to suspend to idle
2410 		 * LiteON has resolved the problem in future firmware
2411 		 */
2412 		.vid = 0x14a4,
2413 		.fr = "22301111",
2414 		.quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2415 	},
2416 	{
2417 		/*
2418 		 * This Kingston E8FK11.T firmware version has no interrupt
2419 		 * after resume with actions related to suspend to idle
2420 		 * https://bugzilla.kernel.org/show_bug.cgi?id=204887
2421 		 */
2422 		.vid = 0x2646,
2423 		.fr = "E8FK11.T",
2424 		.quirks = NVME_QUIRK_SIMPLE_SUSPEND,
2425 	}
2426 };
2427 
2428 /* match is null-terminated but idstr is space-padded. */
2429 static bool string_matches(const char *idstr, const char *match, size_t len)
2430 {
2431 	size_t matchlen;
2432 
2433 	if (!match)
2434 		return true;
2435 
2436 	matchlen = strlen(match);
2437 	WARN_ON_ONCE(matchlen > len);
2438 
2439 	if (memcmp(idstr, match, matchlen))
2440 		return false;
2441 
2442 	for (; matchlen < len; matchlen++)
2443 		if (idstr[matchlen] != ' ')
2444 			return false;
2445 
2446 	return true;
2447 }
2448 
2449 static bool quirk_matches(const struct nvme_id_ctrl *id,
2450 			  const struct nvme_core_quirk_entry *q)
2451 {
2452 	return q->vid == le16_to_cpu(id->vid) &&
2453 		string_matches(id->mn, q->mn, sizeof(id->mn)) &&
2454 		string_matches(id->fr, q->fr, sizeof(id->fr));
2455 }
2456 
2457 static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
2458 		struct nvme_id_ctrl *id)
2459 {
2460 	size_t nqnlen;
2461 	int off;
2462 
2463 	if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
2464 		nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2465 		if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2466 			strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2467 			return;
2468 		}
2469 
2470 		if (ctrl->vs >= NVME_VS(1, 2, 1))
2471 			dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
2472 	}
2473 
2474 	/* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2475 	off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2476 			"nqn.2014.08.org.nvmexpress:%04x%04x",
2477 			le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
2478 	memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2479 	off += sizeof(id->sn);
2480 	memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2481 	off += sizeof(id->mn);
2482 	memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2483 }
2484 
2485 static void nvme_release_subsystem(struct device *dev)
2486 {
2487 	struct nvme_subsystem *subsys =
2488 		container_of(dev, struct nvme_subsystem, dev);
2489 
2490 	if (subsys->instance >= 0)
2491 		ida_simple_remove(&nvme_instance_ida, subsys->instance);
2492 	kfree(subsys);
2493 }
2494 
2495 static void nvme_destroy_subsystem(struct kref *ref)
2496 {
2497 	struct nvme_subsystem *subsys =
2498 			container_of(ref, struct nvme_subsystem, ref);
2499 
2500 	mutex_lock(&nvme_subsystems_lock);
2501 	list_del(&subsys->entry);
2502 	mutex_unlock(&nvme_subsystems_lock);
2503 
2504 	ida_destroy(&subsys->ns_ida);
2505 	device_del(&subsys->dev);
2506 	put_device(&subsys->dev);
2507 }
2508 
2509 static void nvme_put_subsystem(struct nvme_subsystem *subsys)
2510 {
2511 	kref_put(&subsys->ref, nvme_destroy_subsystem);
2512 }
2513 
2514 static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
2515 {
2516 	struct nvme_subsystem *subsys;
2517 
2518 	lockdep_assert_held(&nvme_subsystems_lock);
2519 
2520 	/*
2521 	 * Fail matches for discovery subsystems. This results
2522 	 * in each discovery controller bound to a unique subsystem.
2523 	 * This avoids issues with validating controller values
2524 	 * that can only be true when there is a single unique subsystem.
2525 	 * There may be multiple and completely independent entities
2526 	 * that provide discovery controllers.
2527 	 */
2528 	if (!strcmp(subsysnqn, NVME_DISC_SUBSYS_NAME))
2529 		return NULL;
2530 
2531 	list_for_each_entry(subsys, &nvme_subsystems, entry) {
2532 		if (strcmp(subsys->subnqn, subsysnqn))
2533 			continue;
2534 		if (!kref_get_unless_zero(&subsys->ref))
2535 			continue;
2536 		return subsys;
2537 	}
2538 
2539 	return NULL;
2540 }
2541 
2542 #define SUBSYS_ATTR_RO(_name, _mode, _show)			\
2543 	struct device_attribute subsys_attr_##_name = \
2544 		__ATTR(_name, _mode, _show, NULL)
2545 
2546 static ssize_t nvme_subsys_show_nqn(struct device *dev,
2547 				    struct device_attribute *attr,
2548 				    char *buf)
2549 {
2550 	struct nvme_subsystem *subsys =
2551 		container_of(dev, struct nvme_subsystem, dev);
2552 
2553 	return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
2554 }
2555 static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);
2556 
2557 #define nvme_subsys_show_str_function(field)				\
2558 static ssize_t subsys_##field##_show(struct device *dev,		\
2559 			    struct device_attribute *attr, char *buf)	\
2560 {									\
2561 	struct nvme_subsystem *subsys =					\
2562 		container_of(dev, struct nvme_subsystem, dev);		\
2563 	return sprintf(buf, "%.*s\n",					\
2564 		       (int)sizeof(subsys->field), subsys->field);	\
2565 }									\
2566 static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2567 
2568 nvme_subsys_show_str_function(model);
2569 nvme_subsys_show_str_function(serial);
2570 nvme_subsys_show_str_function(firmware_rev);
2571 
2572 static struct attribute *nvme_subsys_attrs[] = {
2573 	&subsys_attr_model.attr,
2574 	&subsys_attr_serial.attr,
2575 	&subsys_attr_firmware_rev.attr,
2576 	&subsys_attr_subsysnqn.attr,
2577 #ifdef CONFIG_NVME_MULTIPATH
2578 	&subsys_attr_iopolicy.attr,
2579 #endif
2580 	NULL,
2581 };
2582 
2583 static struct attribute_group nvme_subsys_attrs_group = {
2584 	.attrs = nvme_subsys_attrs,
2585 };
2586 
2587 static const struct attribute_group *nvme_subsys_attrs_groups[] = {
2588 	&nvme_subsys_attrs_group,
2589 	NULL,
2590 };
2591 
2592 static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
2593 		struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2594 {
2595 	struct nvme_ctrl *tmp;
2596 
2597 	lockdep_assert_held(&nvme_subsystems_lock);
2598 
2599 	list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
2600 		if (tmp->state == NVME_CTRL_DELETING ||
2601 		    tmp->state == NVME_CTRL_DEAD)
2602 			continue;
2603 
2604 		if (tmp->cntlid == ctrl->cntlid) {
2605 			dev_err(ctrl->device,
2606 				"Duplicate cntlid %u with %s, rejecting\n",
2607 				ctrl->cntlid, dev_name(tmp->device));
2608 			return false;
2609 		}
2610 
2611 		if ((id->cmic & (1 << 1)) ||
2612 		    (ctrl->opts && ctrl->opts->discovery_nqn))
2613 			continue;
2614 
2615 		dev_err(ctrl->device,
2616 			"Subsystem does not support multiple controllers\n");
2617 		return false;
2618 	}
2619 
2620 	return true;
2621 }
2622 
2623 static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
2624 {
2625 	struct nvme_subsystem *subsys, *found;
2626 	int ret;
2627 
2628 	subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2629 	if (!subsys)
2630 		return -ENOMEM;
2631 
2632 	subsys->instance = -1;
2633 	mutex_init(&subsys->lock);
2634 	kref_init(&subsys->ref);
2635 	INIT_LIST_HEAD(&subsys->ctrls);
2636 	INIT_LIST_HEAD(&subsys->nsheads);
2637 	nvme_init_subnqn(subsys, ctrl, id);
2638 	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2639 	memcpy(subsys->model, id->mn, sizeof(subsys->model));
2640 	memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2641 	subsys->vendor_id = le16_to_cpu(id->vid);
2642 	subsys->cmic = id->cmic;
2643 	subsys->awupf = le16_to_cpu(id->awupf);
2644 #ifdef CONFIG_NVME_MULTIPATH
2645 	subsys->iopolicy = NVME_IOPOLICY_NUMA;
2646 #endif
2647 
2648 	subsys->dev.class = nvme_subsys_class;
2649 	subsys->dev.release = nvme_release_subsystem;
2650 	subsys->dev.groups = nvme_subsys_attrs_groups;
2651 	dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2652 	device_initialize(&subsys->dev);
2653 
2654 	mutex_lock(&nvme_subsystems_lock);
2655 	found = __nvme_find_get_subsystem(subsys->subnqn);
2656 	if (found) {
2657 		put_device(&subsys->dev);
2658 		subsys = found;
2659 
2660 		if (!nvme_validate_cntlid(subsys, ctrl, id)) {
2661 			ret = -EINVAL;
2662 			goto out_put_subsystem;
2663 		}
2664 	} else {
2665 		ret = device_add(&subsys->dev);
2666 		if (ret) {
2667 			dev_err(ctrl->device,
2668 				"failed to register subsystem device.\n");
2669 			put_device(&subsys->dev);
2670 			goto out_unlock;
2671 		}
2672 		ida_init(&subsys->ns_ida);
2673 		list_add_tail(&subsys->entry, &nvme_subsystems);
2674 	}
2675 
2676 	ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2677 				dev_name(ctrl->device));
2678 	if (ret) {
2679 		dev_err(ctrl->device,
2680 			"failed to create sysfs link from subsystem.\n");
2681 		goto out_put_subsystem;
2682 	}
2683 
2684 	if (!found)
2685 		subsys->instance = ctrl->instance;
2686 	ctrl->subsys = subsys;
2687 	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2688 	mutex_unlock(&nvme_subsystems_lock);
2689 	return 0;
2690 
2691 out_put_subsystem:
2692 	nvme_put_subsystem(subsys);
2693 out_unlock:
2694 	mutex_unlock(&nvme_subsystems_lock);
2695 	return ret;
2696 }
2697 
2698 int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
2699 		void *log, size_t size, u64 offset)
2700 {
2701 	struct nvme_command c = { };
2702 	unsigned long dwlen = size / 4 - 1;
2703 
2704 	c.get_log_page.opcode = nvme_admin_get_log_page;
2705 	c.get_log_page.nsid = cpu_to_le32(nsid);
2706 	c.get_log_page.lid = log_page;
2707 	c.get_log_page.lsp = lsp;
2708 	c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
2709 	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2710 	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2711 	c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
2712 
2713 	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2714 }
2715 
2716 static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
2717 {
2718 	int ret;
2719 
2720 	if (!ctrl->effects)
2721 		ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
2722 
2723 	if (!ctrl->effects)
2724 		return 0;
2725 
2726 	ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0,
2727 			ctrl->effects, sizeof(*ctrl->effects), 0);
2728 	if (ret) {
2729 		kfree(ctrl->effects);
2730 		ctrl->effects = NULL;
2731 	}
2732 	return ret;
2733 }
2734 
2735 /*
2736  * Initialize the cached copies of the Identify data and various controller
2737  * register in our nvme_ctrl structure.  This should be called as soon as
2738  * the admin queue is fully up and running.
2739  */
2740 int nvme_init_identify(struct nvme_ctrl *ctrl)
2741 {
2742 	struct nvme_id_ctrl *id;
2743 	int ret, page_shift;
2744 	u32 max_hw_sectors;
2745 	bool prev_apst_enabled;
2746 
2747 	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
2748 	if (ret) {
2749 		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2750 		return ret;
2751 	}
2752 	page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
2753 	ctrl->sqsize = min_t(int, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
2754 
2755 	if (ctrl->vs >= NVME_VS(1, 1, 0))
2756 		ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
2757 
2758 	ret = nvme_identify_ctrl(ctrl, &id);
2759 	if (ret) {
2760 		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2761 		return -EIO;
2762 	}
2763 
2764 	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2765 		ret = nvme_get_effects_log(ctrl);
2766 		if (ret < 0)
2767 			goto out_free;
2768 	}
2769 
2770 	if (!(ctrl->ops->flags & NVME_F_FABRICS))
2771 		ctrl->cntlid = le16_to_cpu(id->cntlid);
2772 
2773 	if (!ctrl->identified) {
2774 		int i;
2775 
2776 		ret = nvme_init_subsystem(ctrl, id);
2777 		if (ret)
2778 			goto out_free;
2779 
2780 		/*
2781 		 * Check for quirks.  Quirk can depend on firmware version,
2782 		 * so, in principle, the set of quirks present can change
2783 		 * across a reset.  As a possible future enhancement, we
2784 		 * could re-scan for quirks every time we reinitialize
2785 		 * the device, but we'd have to make sure that the driver
2786 		 * behaves intelligently if the quirks change.
2787 		 */
2788 		for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
2789 			if (quirk_matches(id, &core_quirks[i]))
2790 				ctrl->quirks |= core_quirks[i].quirks;
2791 		}
2792 	}
2793 
2794 	if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
2795 		dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
2796 		ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
2797 	}
2798 
2799 	ctrl->crdt[0] = le16_to_cpu(id->crdt1);
2800 	ctrl->crdt[1] = le16_to_cpu(id->crdt2);
2801 	ctrl->crdt[2] = le16_to_cpu(id->crdt3);
2802 
2803 	ctrl->oacs = le16_to_cpu(id->oacs);
2804 	ctrl->oncs = le16_to_cpu(id->oncs);
2805 	ctrl->mtfa = le16_to_cpu(id->mtfa);
2806 	ctrl->oaes = le32_to_cpu(id->oaes);
2807 	ctrl->wctemp = le16_to_cpu(id->wctemp);
2808 	ctrl->cctemp = le16_to_cpu(id->cctemp);
2809 
2810 	atomic_set(&ctrl->abort_limit, id->acl + 1);
2811 	ctrl->vwc = id->vwc;
2812 	if (id->mdts)
2813 		max_hw_sectors = 1 << (id->mdts + page_shift - 9);
2814 	else
2815 		max_hw_sectors = UINT_MAX;
2816 	ctrl->max_hw_sectors =
2817 		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
2818 
2819 	nvme_set_queue_limits(ctrl, ctrl->admin_q);
2820 	ctrl->sgls = le32_to_cpu(id->sgls);
2821 	ctrl->kas = le16_to_cpu(id->kas);
2822 	ctrl->max_namespaces = le32_to_cpu(id->mnan);
2823 	ctrl->ctratt = le32_to_cpu(id->ctratt);
2824 
2825 	if (id->rtd3e) {
2826 		/* us -> s */
2827 		u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
2828 
2829 		ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
2830 						 shutdown_timeout, 60);
2831 
2832 		if (ctrl->shutdown_timeout != shutdown_timeout)
2833 			dev_info(ctrl->device,
2834 				 "Shutdown timeout set to %u seconds\n",
2835 				 ctrl->shutdown_timeout);
2836 	} else
2837 		ctrl->shutdown_timeout = shutdown_timeout;
2838 
2839 	ctrl->npss = id->npss;
2840 	ctrl->apsta = id->apsta;
2841 	prev_apst_enabled = ctrl->apst_enabled;
2842 	if (ctrl->quirks & NVME_QUIRK_NO_APST) {
2843 		if (force_apst && id->apsta) {
2844 			dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
2845 			ctrl->apst_enabled = true;
2846 		} else {
2847 			ctrl->apst_enabled = false;
2848 		}
2849 	} else {
2850 		ctrl->apst_enabled = id->apsta;
2851 	}
2852 	memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));
2853 
2854 	if (ctrl->ops->flags & NVME_F_FABRICS) {
2855 		ctrl->icdoff = le16_to_cpu(id->icdoff);
2856 		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
2857 		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
2858 		ctrl->maxcmd = le16_to_cpu(id->maxcmd);
2859 
2860 		/*
2861 		 * In fabrics we need to verify the cntlid matches the
2862 		 * admin connect
2863 		 */
2864 		if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
2865 			ret = -EINVAL;
2866 			goto out_free;
2867 		}
2868 
2869 		if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
2870 			dev_err(ctrl->device,
2871 				"keep-alive support is mandatory for fabrics\n");
2872 			ret = -EINVAL;
2873 			goto out_free;
2874 		}
2875 	} else {
2876 		ctrl->hmpre = le32_to_cpu(id->hmpre);
2877 		ctrl->hmmin = le32_to_cpu(id->hmmin);
2878 		ctrl->hmminds = le32_to_cpu(id->hmminds);
2879 		ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
2880 	}
2881 
2882 	ret = nvme_mpath_init(ctrl, id);
2883 	kfree(id);
2884 
2885 	if (ret < 0)
2886 		return ret;
2887 
2888 	if (ctrl->apst_enabled && !prev_apst_enabled)
2889 		dev_pm_qos_expose_latency_tolerance(ctrl->device);
2890 	else if (!ctrl->apst_enabled && prev_apst_enabled)
2891 		dev_pm_qos_hide_latency_tolerance(ctrl->device);
2892 
2893 	ret = nvme_configure_apst(ctrl);
2894 	if (ret < 0)
2895 		return ret;
2896 
2897 	ret = nvme_configure_timestamp(ctrl);
2898 	if (ret < 0)
2899 		return ret;
2900 
2901 	ret = nvme_configure_directives(ctrl);
2902 	if (ret < 0)
2903 		return ret;
2904 
2905 	ret = nvme_configure_acre(ctrl);
2906 	if (ret < 0)
2907 		return ret;
2908 
2909 	if (!ctrl->identified)
2910 		nvme_hwmon_init(ctrl);
2911 
2912 	ctrl->identified = true;
2913 
2914 	return 0;
2915 
2916 out_free:
2917 	kfree(id);
2918 	return ret;
2919 }
2920 EXPORT_SYMBOL_GPL(nvme_init_identify);
2921 
2922 static int nvme_dev_open(struct inode *inode, struct file *file)
2923 {
2924 	struct nvme_ctrl *ctrl =
2925 		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
2926 
2927 	switch (ctrl->state) {
2928 	case NVME_CTRL_LIVE:
2929 		break;
2930 	default:
2931 		return -EWOULDBLOCK;
2932 	}
2933 
2934 	file->private_data = ctrl;
2935 	return 0;
2936 }
2937 
2938 static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
2939 {
2940 	struct nvme_ns *ns;
2941 	int ret;
2942 
2943 	down_read(&ctrl->namespaces_rwsem);
2944 	if (list_empty(&ctrl->namespaces)) {
2945 		ret = -ENOTTY;
2946 		goto out_unlock;
2947 	}
2948 
2949 	ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
2950 	if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
2951 		dev_warn(ctrl->device,
2952 			"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
2953 		ret = -EINVAL;
2954 		goto out_unlock;
2955 	}
2956 
2957 	dev_warn(ctrl->device,
2958 		"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
2959 	kref_get(&ns->kref);
2960 	up_read(&ctrl->namespaces_rwsem);
2961 
2962 	ret = nvme_user_cmd(ctrl, ns, argp);
2963 	nvme_put_ns(ns);
2964 	return ret;
2965 
2966 out_unlock:
2967 	up_read(&ctrl->namespaces_rwsem);
2968 	return ret;
2969 }
2970 
2971 static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
2972 		unsigned long arg)
2973 {
2974 	struct nvme_ctrl *ctrl = file->private_data;
2975 	void __user *argp = (void __user *)arg;
2976 
2977 	switch (cmd) {
2978 	case NVME_IOCTL_ADMIN_CMD:
2979 		return nvme_user_cmd(ctrl, NULL, argp);
2980 	case NVME_IOCTL_ADMIN64_CMD:
2981 		return nvme_user_cmd64(ctrl, NULL, argp);
2982 	case NVME_IOCTL_IO_CMD:
2983 		return nvme_dev_user_cmd(ctrl, argp);
2984 	case NVME_IOCTL_RESET:
2985 		dev_warn(ctrl->device, "resetting controller\n");
2986 		return nvme_reset_ctrl_sync(ctrl);
2987 	case NVME_IOCTL_SUBSYS_RESET:
2988 		return nvme_reset_subsystem(ctrl);
2989 	case NVME_IOCTL_RESCAN:
2990 		nvme_queue_scan(ctrl);
2991 		return 0;
2992 	default:
2993 		return -ENOTTY;
2994 	}
2995 }
2996 
2997 static const struct file_operations nvme_dev_fops = {
2998 	.owner		= THIS_MODULE,
2999 	.open		= nvme_dev_open,
3000 	.unlocked_ioctl	= nvme_dev_ioctl,
3001 	.compat_ioctl	= compat_ptr_ioctl,
3002 };
3003 
3004 static ssize_t nvme_sysfs_reset(struct device *dev,
3005 				struct device_attribute *attr, const char *buf,
3006 				size_t count)
3007 {
3008 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3009 	int ret;
3010 
3011 	ret = nvme_reset_ctrl_sync(ctrl);
3012 	if (ret < 0)
3013 		return ret;
3014 	return count;
3015 }
3016 static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
3017 
3018 static ssize_t nvme_sysfs_rescan(struct device *dev,
3019 				struct device_attribute *attr, const char *buf,
3020 				size_t count)
3021 {
3022 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3023 
3024 	nvme_queue_scan(ctrl);
3025 	return count;
3026 }
3027 static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);
3028 
3029 static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
3030 {
3031 	struct gendisk *disk = dev_to_disk(dev);
3032 
3033 	if (disk->fops == &nvme_fops)
3034 		return nvme_get_ns_from_dev(dev)->head;
3035 	else
3036 		return disk->private_data;
3037 }
3038 
3039 static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
3040 		char *buf)
3041 {
3042 	struct nvme_ns_head *head = dev_to_ns_head(dev);
3043 	struct nvme_ns_ids *ids = &head->ids;
3044 	struct nvme_subsystem *subsys = head->subsys;
3045 	int serial_len = sizeof(subsys->serial);
3046 	int model_len = sizeof(subsys->model);
3047 
3048 	if (!uuid_is_null(&ids->uuid))
3049 		return sprintf(buf, "uuid.%pU\n", &ids->uuid);
3050 
3051 	if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3052 		return sprintf(buf, "eui.%16phN\n", ids->nguid);
3053 
3054 	if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3055 		return sprintf(buf, "eui.%8phN\n", ids->eui64);
3056 
3057 	while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
3058 				  subsys->serial[serial_len - 1] == '\0'))
3059 		serial_len--;
3060 	while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
3061 				 subsys->model[model_len - 1] == '\0'))
3062 		model_len--;
3063 
3064 	return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
3065 		serial_len, subsys->serial, model_len, subsys->model,
3066 		head->ns_id);
3067 }
3068 static DEVICE_ATTR_RO(wwid);
3069 
3070 static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
3071 		char *buf)
3072 {
3073 	return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
3074 }
3075 static DEVICE_ATTR_RO(nguid);
3076 
3077 static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
3078 		char *buf)
3079 {
3080 	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3081 
3082 	/* For backward compatibility expose the NGUID to userspace if
3083 	 * we have no UUID set
3084 	 */
3085 	if (uuid_is_null(&ids->uuid)) {
3086 		printk_ratelimited(KERN_WARNING
3087 				   "No UUID available providing old NGUID\n");
3088 		return sprintf(buf, "%pU\n", ids->nguid);
3089 	}
3090 	return sprintf(buf, "%pU\n", &ids->uuid);
3091 }
3092 static DEVICE_ATTR_RO(uuid);
3093 
3094 static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
3095 		char *buf)
3096 {
3097 	return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
3098 }
3099 static DEVICE_ATTR_RO(eui);
3100 
3101 static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
3102 		char *buf)
3103 {
3104 	return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
3105 }
3106 static DEVICE_ATTR_RO(nsid);
3107 
3108 static struct attribute *nvme_ns_id_attrs[] = {
3109 	&dev_attr_wwid.attr,
3110 	&dev_attr_uuid.attr,
3111 	&dev_attr_nguid.attr,
3112 	&dev_attr_eui.attr,
3113 	&dev_attr_nsid.attr,
3114 #ifdef CONFIG_NVME_MULTIPATH
3115 	&dev_attr_ana_grpid.attr,
3116 	&dev_attr_ana_state.attr,
3117 #endif
3118 	NULL,
3119 };
3120 
3121 static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
3122 		struct attribute *a, int n)
3123 {
3124 	struct device *dev = container_of(kobj, struct device, kobj);
3125 	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
3126 
3127 	if (a == &dev_attr_uuid.attr) {
3128 		if (uuid_is_null(&ids->uuid) &&
3129 		    !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3130 			return 0;
3131 	}
3132 	if (a == &dev_attr_nguid.attr) {
3133 		if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
3134 			return 0;
3135 	}
3136 	if (a == &dev_attr_eui.attr) {
3137 		if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
3138 			return 0;
3139 	}
3140 #ifdef CONFIG_NVME_MULTIPATH
3141 	if (a == &dev_attr_ana_grpid.attr || a == &dev_attr_ana_state.attr) {
3142 		if (dev_to_disk(dev)->fops != &nvme_fops) /* per-path attr */
3143 			return 0;
3144 		if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
3145 			return 0;
3146 	}
3147 #endif
3148 	return a->mode;
3149 }
3150 
3151 static const struct attribute_group nvme_ns_id_attr_group = {
3152 	.attrs		= nvme_ns_id_attrs,
3153 	.is_visible	= nvme_ns_id_attrs_are_visible,
3154 };
3155 
3156 const struct attribute_group *nvme_ns_id_attr_groups[] = {
3157 	&nvme_ns_id_attr_group,
3158 #ifdef CONFIG_NVM
3159 	&nvme_nvm_attr_group,
3160 #endif
3161 	NULL,
3162 };
3163 
3164 #define nvme_show_str_function(field)						\
3165 static ssize_t  field##_show(struct device *dev,				\
3166 			    struct device_attribute *attr, char *buf)		\
3167 {										\
3168         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3169         return sprintf(buf, "%.*s\n",						\
3170 		(int)sizeof(ctrl->subsys->field), ctrl->subsys->field);		\
3171 }										\
3172 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3173 
3174 nvme_show_str_function(model);
3175 nvme_show_str_function(serial);
3176 nvme_show_str_function(firmware_rev);
3177 
3178 #define nvme_show_int_function(field)						\
3179 static ssize_t  field##_show(struct device *dev,				\
3180 			    struct device_attribute *attr, char *buf)		\
3181 {										\
3182         struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
3183         return sprintf(buf, "%d\n", ctrl->field);	\
3184 }										\
3185 static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
3186 
3187 nvme_show_int_function(cntlid);
3188 nvme_show_int_function(numa_node);
3189 nvme_show_int_function(queue_count);
3190 nvme_show_int_function(sqsize);
3191 
3192 static ssize_t nvme_sysfs_delete(struct device *dev,
3193 				struct device_attribute *attr, const char *buf,
3194 				size_t count)
3195 {
3196 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3197 
3198 	if (device_remove_file_self(dev, attr))
3199 		nvme_delete_ctrl_sync(ctrl);
3200 	return count;
3201 }
3202 static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);
3203 
3204 static ssize_t nvme_sysfs_show_transport(struct device *dev,
3205 					 struct device_attribute *attr,
3206 					 char *buf)
3207 {
3208 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3209 
3210 	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
3211 }
3212 static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);
3213 
3214 static ssize_t nvme_sysfs_show_state(struct device *dev,
3215 				     struct device_attribute *attr,
3216 				     char *buf)
3217 {
3218 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3219 	static const char *const state_name[] = {
3220 		[NVME_CTRL_NEW]		= "new",
3221 		[NVME_CTRL_LIVE]	= "live",
3222 		[NVME_CTRL_RESETTING]	= "resetting",
3223 		[NVME_CTRL_CONNECTING]	= "connecting",
3224 		[NVME_CTRL_DELETING]	= "deleting",
3225 		[NVME_CTRL_DEAD]	= "dead",
3226 	};
3227 
3228 	if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
3229 	    state_name[ctrl->state])
3230 		return sprintf(buf, "%s\n", state_name[ctrl->state]);
3231 
3232 	return sprintf(buf, "unknown state\n");
3233 }
3234 
3235 static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
3236 
3237 static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
3238 					 struct device_attribute *attr,
3239 					 char *buf)
3240 {
3241 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3242 
3243 	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
3244 }
3245 static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
3246 
3247 static ssize_t nvme_sysfs_show_address(struct device *dev,
3248 					 struct device_attribute *attr,
3249 					 char *buf)
3250 {
3251 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3252 
3253 	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
3254 }
3255 static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
3256 
3257 static struct attribute *nvme_dev_attrs[] = {
3258 	&dev_attr_reset_controller.attr,
3259 	&dev_attr_rescan_controller.attr,
3260 	&dev_attr_model.attr,
3261 	&dev_attr_serial.attr,
3262 	&dev_attr_firmware_rev.attr,
3263 	&dev_attr_cntlid.attr,
3264 	&dev_attr_delete_controller.attr,
3265 	&dev_attr_transport.attr,
3266 	&dev_attr_subsysnqn.attr,
3267 	&dev_attr_address.attr,
3268 	&dev_attr_state.attr,
3269 	&dev_attr_numa_node.attr,
3270 	&dev_attr_queue_count.attr,
3271 	&dev_attr_sqsize.attr,
3272 	NULL
3273 };
3274 
3275 static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
3276 		struct attribute *a, int n)
3277 {
3278 	struct device *dev = container_of(kobj, struct device, kobj);
3279 	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
3280 
3281 	if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
3282 		return 0;
3283 	if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
3284 		return 0;
3285 
3286 	return a->mode;
3287 }
3288 
3289 static struct attribute_group nvme_dev_attrs_group = {
3290 	.attrs		= nvme_dev_attrs,
3291 	.is_visible	= nvme_dev_attrs_are_visible,
3292 };
3293 
3294 static const struct attribute_group *nvme_dev_attr_groups[] = {
3295 	&nvme_dev_attrs_group,
3296 	NULL,
3297 };
3298 
3299 static struct nvme_ns_head *__nvme_find_ns_head(struct nvme_subsystem *subsys,
3300 		unsigned nsid)
3301 {
3302 	struct nvme_ns_head *h;
3303 
3304 	lockdep_assert_held(&subsys->lock);
3305 
3306 	list_for_each_entry(h, &subsys->nsheads, entry) {
3307 		if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
3308 			return h;
3309 	}
3310 
3311 	return NULL;
3312 }
3313 
3314 static int __nvme_check_ids(struct nvme_subsystem *subsys,
3315 		struct nvme_ns_head *new)
3316 {
3317 	struct nvme_ns_head *h;
3318 
3319 	lockdep_assert_held(&subsys->lock);
3320 
3321 	list_for_each_entry(h, &subsys->nsheads, entry) {
3322 		if (nvme_ns_ids_valid(&new->ids) &&
3323 		    !list_empty(&h->list) &&
3324 		    nvme_ns_ids_equal(&new->ids, &h->ids))
3325 			return -EINVAL;
3326 	}
3327 
3328 	return 0;
3329 }
3330 
3331 static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
3332 		unsigned nsid, struct nvme_id_ns *id)
3333 {
3334 	struct nvme_ns_head *head;
3335 	size_t size = sizeof(*head);
3336 	int ret = -ENOMEM;
3337 
3338 #ifdef CONFIG_NVME_MULTIPATH
3339 	size += num_possible_nodes() * sizeof(struct nvme_ns *);
3340 #endif
3341 
3342 	head = kzalloc(size, GFP_KERNEL);
3343 	if (!head)
3344 		goto out;
3345 	ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
3346 	if (ret < 0)
3347 		goto out_free_head;
3348 	head->instance = ret;
3349 	INIT_LIST_HEAD(&head->list);
3350 	ret = init_srcu_struct(&head->srcu);
3351 	if (ret)
3352 		goto out_ida_remove;
3353 	head->subsys = ctrl->subsys;
3354 	head->ns_id = nsid;
3355 	kref_init(&head->ref);
3356 
3357 	ret = nvme_report_ns_ids(ctrl, nsid, id, &head->ids);
3358 	if (ret)
3359 		goto out_cleanup_srcu;
3360 
3361 	ret = __nvme_check_ids(ctrl->subsys, head);
3362 	if (ret) {
3363 		dev_err(ctrl->device,
3364 			"duplicate IDs for nsid %d\n", nsid);
3365 		goto out_cleanup_srcu;
3366 	}
3367 
3368 	ret = nvme_mpath_alloc_disk(ctrl, head);
3369 	if (ret)
3370 		goto out_cleanup_srcu;
3371 
3372 	list_add_tail(&head->entry, &ctrl->subsys->nsheads);
3373 
3374 	kref_get(&ctrl->subsys->ref);
3375 
3376 	return head;
3377 out_cleanup_srcu:
3378 	cleanup_srcu_struct(&head->srcu);
3379 out_ida_remove:
3380 	ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
3381 out_free_head:
3382 	kfree(head);
3383 out:
3384 	if (ret > 0)
3385 		ret = blk_status_to_errno(nvme_error_status(ret));
3386 	return ERR_PTR(ret);
3387 }
3388 
3389 static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3390 		struct nvme_id_ns *id)
3391 {
3392 	struct nvme_ctrl *ctrl = ns->ctrl;
3393 	bool is_shared = id->nmic & (1 << 0);
3394 	struct nvme_ns_head *head = NULL;
3395 	int ret = 0;
3396 
3397 	mutex_lock(&ctrl->subsys->lock);
3398 	if (is_shared)
3399 		head = __nvme_find_ns_head(ctrl->subsys, nsid);
3400 	if (!head) {
3401 		head = nvme_alloc_ns_head(ctrl, nsid, id);
3402 		if (IS_ERR(head)) {
3403 			ret = PTR_ERR(head);
3404 			goto out_unlock;
3405 		}
3406 	} else {
3407 		struct nvme_ns_ids ids;
3408 
3409 		ret = nvme_report_ns_ids(ctrl, nsid, id, &ids);
3410 		if (ret)
3411 			goto out_unlock;
3412 
3413 		if (!nvme_ns_ids_equal(&head->ids, &ids)) {
3414 			dev_err(ctrl->device,
3415 				"IDs don't match for shared namespace %d\n",
3416 					nsid);
3417 			ret = -EINVAL;
3418 			goto out_unlock;
3419 		}
3420 	}
3421 
3422 	list_add_tail(&ns->siblings, &head->list);
3423 	ns->head = head;
3424 
3425 out_unlock:
3426 	mutex_unlock(&ctrl->subsys->lock);
3427 	if (ret > 0)
3428 		ret = blk_status_to_errno(nvme_error_status(ret));
3429 	return ret;
3430 }
3431 
3432 static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
3433 {
3434 	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3435 	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3436 
3437 	return nsa->head->ns_id - nsb->head->ns_id;
3438 }
3439 
3440 static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3441 {
3442 	struct nvme_ns *ns, *ret = NULL;
3443 
3444 	down_read(&ctrl->namespaces_rwsem);
3445 	list_for_each_entry(ns, &ctrl->namespaces, list) {
3446 		if (ns->head->ns_id == nsid) {
3447 			if (!kref_get_unless_zero(&ns->kref))
3448 				continue;
3449 			ret = ns;
3450 			break;
3451 		}
3452 		if (ns->head->ns_id > nsid)
3453 			break;
3454 	}
3455 	up_read(&ctrl->namespaces_rwsem);
3456 	return ret;
3457 }
3458 
3459 static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
3460 {
3461 	struct streams_directive_params s;
3462 	int ret;
3463 
3464 	if (!ctrl->nr_streams)
3465 		return 0;
3466 
3467 	ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
3468 	if (ret)
3469 		return ret;
3470 
3471 	ns->sws = le32_to_cpu(s.sws);
3472 	ns->sgs = le16_to_cpu(s.sgs);
3473 
3474 	if (ns->sws) {
3475 		unsigned int bs = 1 << ns->lba_shift;
3476 
3477 		blk_queue_io_min(ns->queue, bs * ns->sws);
3478 		if (ns->sgs)
3479 			blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
3480 	}
3481 
3482 	return 0;
3483 }
3484 
3485 static int nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3486 {
3487 	struct nvme_ns *ns;
3488 	struct gendisk *disk;
3489 	struct nvme_id_ns *id;
3490 	char disk_name[DISK_NAME_LEN];
3491 	int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
3492 
3493 	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3494 	if (!ns)
3495 		return -ENOMEM;
3496 
3497 	ns->queue = blk_mq_init_queue(ctrl->tagset);
3498 	if (IS_ERR(ns->queue)) {
3499 		ret = PTR_ERR(ns->queue);
3500 		goto out_free_ns;
3501 	}
3502 
3503 	if (ctrl->opts && ctrl->opts->data_digest)
3504 		ns->queue->backing_dev_info->capabilities
3505 			|= BDI_CAP_STABLE_WRITES;
3506 
3507 	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
3508 	if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
3509 		blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
3510 
3511 	ns->queue->queuedata = ns;
3512 	ns->ctrl = ctrl;
3513 
3514 	kref_init(&ns->kref);
3515 	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
3516 
3517 	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
3518 	nvme_set_queue_limits(ctrl, ns->queue);
3519 
3520 	ret = nvme_identify_ns(ctrl, nsid, &id);
3521 	if (ret)
3522 		goto out_free_queue;
3523 
3524 	if (id->ncap == 0) {
3525 		ret = -EINVAL;
3526 		goto out_free_id;
3527 	}
3528 
3529 	ret = nvme_init_ns_head(ns, nsid, id);
3530 	if (ret)
3531 		goto out_free_id;
3532 	nvme_setup_streams_ns(ctrl, ns);
3533 	nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3534 
3535 	disk = alloc_disk_node(0, node);
3536 	if (!disk) {
3537 		ret = -ENOMEM;
3538 		goto out_unlink_ns;
3539 	}
3540 
3541 	disk->fops = &nvme_fops;
3542 	disk->private_data = ns;
3543 	disk->queue = ns->queue;
3544 	disk->flags = flags;
3545 	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3546 	ns->disk = disk;
3547 
3548 	__nvme_revalidate_disk(disk, id);
3549 
3550 	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3551 		ret = nvme_nvm_register(ns, disk_name, node);
3552 		if (ret) {
3553 			dev_warn(ctrl->device, "LightNVM init failure\n");
3554 			goto out_put_disk;
3555 		}
3556 	}
3557 
3558 	down_write(&ctrl->namespaces_rwsem);
3559 	list_add_tail(&ns->list, &ctrl->namespaces);
3560 	up_write(&ctrl->namespaces_rwsem);
3561 
3562 	nvme_get_ctrl(ctrl);
3563 
3564 	device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3565 
3566 	nvme_mpath_add_disk(ns, id);
3567 	nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3568 	kfree(id);
3569 
3570 	return 0;
3571  out_put_disk:
3572 	put_disk(ns->disk);
3573  out_unlink_ns:
3574 	mutex_lock(&ctrl->subsys->lock);
3575 	list_del_rcu(&ns->siblings);
3576 	mutex_unlock(&ctrl->subsys->lock);
3577 	nvme_put_ns_head(ns->head);
3578  out_free_id:
3579 	kfree(id);
3580  out_free_queue:
3581 	blk_cleanup_queue(ns->queue);
3582  out_free_ns:
3583 	kfree(ns);
3584 	if (ret > 0)
3585 		ret = blk_status_to_errno(nvme_error_status(ret));
3586 	return ret;
3587 }
3588 
3589 static void nvme_ns_remove(struct nvme_ns *ns)
3590 {
3591 	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3592 		return;
3593 
3594 	nvme_fault_inject_fini(&ns->fault_inject);
3595 
3596 	mutex_lock(&ns->ctrl->subsys->lock);
3597 	list_del_rcu(&ns->siblings);
3598 	mutex_unlock(&ns->ctrl->subsys->lock);
3599 	synchronize_rcu(); /* guarantee not available in head->list */
3600 	nvme_mpath_clear_current_path(ns);
3601 	synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3602 
3603 	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
3604 		del_gendisk(ns->disk);
3605 		blk_cleanup_queue(ns->queue);
3606 		if (blk_get_integrity(ns->disk))
3607 			blk_integrity_unregister(ns->disk);
3608 	}
3609 
3610 	down_write(&ns->ctrl->namespaces_rwsem);
3611 	list_del_init(&ns->list);
3612 	up_write(&ns->ctrl->namespaces_rwsem);
3613 
3614 	nvme_mpath_check_last_path(ns);
3615 	nvme_put_ns(ns);
3616 }
3617 
3618 static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
3619 {
3620 	struct nvme_ns *ns;
3621 
3622 	ns = nvme_find_get_ns(ctrl, nsid);
3623 	if (ns) {
3624 		if (ns->disk && revalidate_disk(ns->disk))
3625 			nvme_ns_remove(ns);
3626 		nvme_put_ns(ns);
3627 	} else
3628 		nvme_alloc_ns(ctrl, nsid);
3629 }
3630 
3631 static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
3632 					unsigned nsid)
3633 {
3634 	struct nvme_ns *ns, *next;
3635 	LIST_HEAD(rm_list);
3636 
3637 	down_write(&ctrl->namespaces_rwsem);
3638 	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
3639 		if (ns->head->ns_id > nsid || test_bit(NVME_NS_DEAD, &ns->flags))
3640 			list_move_tail(&ns->list, &rm_list);
3641 	}
3642 	up_write(&ctrl->namespaces_rwsem);
3643 
3644 	list_for_each_entry_safe(ns, next, &rm_list, list)
3645 		nvme_ns_remove(ns);
3646 
3647 }
3648 
3649 static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
3650 {
3651 	struct nvme_ns *ns;
3652 	__le32 *ns_list;
3653 	unsigned i, j, nsid, prev = 0;
3654 	unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024);
3655 	int ret = 0;
3656 
3657 	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
3658 	if (!ns_list)
3659 		return -ENOMEM;
3660 
3661 	for (i = 0; i < num_lists; i++) {
3662 		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
3663 		if (ret)
3664 			goto free;
3665 
3666 		for (j = 0; j < min(nn, 1024U); j++) {
3667 			nsid = le32_to_cpu(ns_list[j]);
3668 			if (!nsid)
3669 				goto out;
3670 
3671 			nvme_validate_ns(ctrl, nsid);
3672 
3673 			while (++prev < nsid) {
3674 				ns = nvme_find_get_ns(ctrl, prev);
3675 				if (ns) {
3676 					nvme_ns_remove(ns);
3677 					nvme_put_ns(ns);
3678 				}
3679 			}
3680 		}
3681 		nn -= j;
3682 	}
3683  out:
3684 	nvme_remove_invalid_namespaces(ctrl, prev);
3685  free:
3686 	kfree(ns_list);
3687 	return ret;
3688 }
3689 
3690 static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
3691 {
3692 	unsigned i;
3693 
3694 	for (i = 1; i <= nn; i++)
3695 		nvme_validate_ns(ctrl, i);
3696 
3697 	nvme_remove_invalid_namespaces(ctrl, nn);
3698 }
3699 
3700 static void nvme_clear_changed_ns_log(struct nvme_ctrl *ctrl)
3701 {
3702 	size_t log_size = NVME_MAX_CHANGED_NAMESPACES * sizeof(__le32);
3703 	__le32 *log;
3704 	int error;
3705 
3706 	log = kzalloc(log_size, GFP_KERNEL);
3707 	if (!log)
3708 		return;
3709 
3710 	/*
3711 	 * We need to read the log to clear the AEN, but we don't want to rely
3712 	 * on it for the changed namespace information as userspace could have
3713 	 * raced with us in reading the log page, which could cause us to miss
3714 	 * updates.
3715 	 */
3716 	error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, log,
3717 			log_size, 0);
3718 	if (error)
3719 		dev_warn(ctrl->device,
3720 			"reading changed ns log failed: %d\n", error);
3721 
3722 	kfree(log);
3723 }
3724 
3725 static void nvme_scan_work(struct work_struct *work)
3726 {
3727 	struct nvme_ctrl *ctrl =
3728 		container_of(work, struct nvme_ctrl, scan_work);
3729 	struct nvme_id_ctrl *id;
3730 	unsigned nn;
3731 
3732 	/* No tagset on a live ctrl means IO queues could not created */
3733 	if (ctrl->state != NVME_CTRL_LIVE || !ctrl->tagset)
3734 		return;
3735 
3736 	if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
3737 		dev_info(ctrl->device, "rescanning namespaces.\n");
3738 		nvme_clear_changed_ns_log(ctrl);
3739 	}
3740 
3741 	if (nvme_identify_ctrl(ctrl, &id))
3742 		return;
3743 
3744 	mutex_lock(&ctrl->scan_lock);
3745 	nn = le32_to_cpu(id->nn);
3746 	if (ctrl->vs >= NVME_VS(1, 1, 0) &&
3747 	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
3748 		if (!nvme_scan_ns_list(ctrl, nn))
3749 			goto out_free_id;
3750 	}
3751 	nvme_scan_ns_sequential(ctrl, nn);
3752 out_free_id:
3753 	mutex_unlock(&ctrl->scan_lock);
3754 	kfree(id);
3755 	down_write(&ctrl->namespaces_rwsem);
3756 	list_sort(NULL, &ctrl->namespaces, ns_cmp);
3757 	up_write(&ctrl->namespaces_rwsem);
3758 }
3759 
3760 /*
3761  * This function iterates the namespace list unlocked to allow recovery from
3762  * controller failure. It is up to the caller to ensure the namespace list is
3763  * not modified by scan work while this function is executing.
3764  */
3765 void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
3766 {
3767 	struct nvme_ns *ns, *next;
3768 	LIST_HEAD(ns_list);
3769 
3770 	/*
3771 	 * make sure to requeue I/O to all namespaces as these
3772 	 * might result from the scan itself and must complete
3773 	 * for the scan_work to make progress
3774 	 */
3775 	nvme_mpath_clear_ctrl_paths(ctrl);
3776 
3777 	/* prevent racing with ns scanning */
3778 	flush_work(&ctrl->scan_work);
3779 
3780 	/*
3781 	 * The dead states indicates the controller was not gracefully
3782 	 * disconnected. In that case, we won't be able to flush any data while
3783 	 * removing the namespaces' disks; fail all the queues now to avoid
3784 	 * potentially having to clean up the failed sync later.
3785 	 */
3786 	if (ctrl->state == NVME_CTRL_DEAD)
3787 		nvme_kill_queues(ctrl);
3788 
3789 	down_write(&ctrl->namespaces_rwsem);
3790 	list_splice_init(&ctrl->namespaces, &ns_list);
3791 	up_write(&ctrl->namespaces_rwsem);
3792 
3793 	list_for_each_entry_safe(ns, next, &ns_list, list)
3794 		nvme_ns_remove(ns);
3795 }
3796 EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
3797 
3798 static int nvme_class_uevent(struct device *dev, struct kobj_uevent_env *env)
3799 {
3800 	struct nvme_ctrl *ctrl =
3801 		container_of(dev, struct nvme_ctrl, ctrl_device);
3802 	struct nvmf_ctrl_options *opts = ctrl->opts;
3803 	int ret;
3804 
3805 	ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
3806 	if (ret)
3807 		return ret;
3808 
3809 	if (opts) {
3810 		ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
3811 		if (ret)
3812 			return ret;
3813 
3814 		ret = add_uevent_var(env, "NVME_TRSVCID=%s",
3815 				opts->trsvcid ?: "none");
3816 		if (ret)
3817 			return ret;
3818 
3819 		ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
3820 				opts->host_traddr ?: "none");
3821 	}
3822 	return ret;
3823 }
3824 
3825 static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
3826 {
3827 	char *envp[2] = { NULL, NULL };
3828 	u32 aen_result = ctrl->aen_result;
3829 
3830 	ctrl->aen_result = 0;
3831 	if (!aen_result)
3832 		return;
3833 
3834 	envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
3835 	if (!envp[0])
3836 		return;
3837 	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
3838 	kfree(envp[0]);
3839 }
3840 
3841 static void nvme_async_event_work(struct work_struct *work)
3842 {
3843 	struct nvme_ctrl *ctrl =
3844 		container_of(work, struct nvme_ctrl, async_event_work);
3845 
3846 	nvme_aen_uevent(ctrl);
3847 	ctrl->ops->submit_async_event(ctrl);
3848 }
3849 
3850 static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
3851 {
3852 
3853 	u32 csts;
3854 
3855 	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
3856 		return false;
3857 
3858 	if (csts == ~0)
3859 		return false;
3860 
3861 	return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
3862 }
3863 
3864 static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
3865 {
3866 	struct nvme_fw_slot_info_log *log;
3867 
3868 	log = kmalloc(sizeof(*log), GFP_KERNEL);
3869 	if (!log)
3870 		return;
3871 
3872 	if (nvme_get_log(ctrl, NVME_NSID_ALL, 0, NVME_LOG_FW_SLOT, log,
3873 			sizeof(*log), 0))
3874 		dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
3875 	kfree(log);
3876 }
3877 
3878 static void nvme_fw_act_work(struct work_struct *work)
3879 {
3880 	struct nvme_ctrl *ctrl = container_of(work,
3881 				struct nvme_ctrl, fw_act_work);
3882 	unsigned long fw_act_timeout;
3883 
3884 	if (ctrl->mtfa)
3885 		fw_act_timeout = jiffies +
3886 				msecs_to_jiffies(ctrl->mtfa * 100);
3887 	else
3888 		fw_act_timeout = jiffies +
3889 				msecs_to_jiffies(admin_timeout * 1000);
3890 
3891 	nvme_stop_queues(ctrl);
3892 	while (nvme_ctrl_pp_status(ctrl)) {
3893 		if (time_after(jiffies, fw_act_timeout)) {
3894 			dev_warn(ctrl->device,
3895 				"Fw activation timeout, reset controller\n");
3896 			nvme_try_sched_reset(ctrl);
3897 			return;
3898 		}
3899 		msleep(100);
3900 	}
3901 
3902 	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
3903 		return;
3904 
3905 	nvme_start_queues(ctrl);
3906 	/* read FW slot information to clear the AER */
3907 	nvme_get_fw_slot_info(ctrl);
3908 }
3909 
3910 static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
3911 {
3912 	u32 aer_notice_type = (result & 0xff00) >> 8;
3913 
3914 	trace_nvme_async_event(ctrl, aer_notice_type);
3915 
3916 	switch (aer_notice_type) {
3917 	case NVME_AER_NOTICE_NS_CHANGED:
3918 		set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
3919 		nvme_queue_scan(ctrl);
3920 		break;
3921 	case NVME_AER_NOTICE_FW_ACT_STARTING:
3922 		/*
3923 		 * We are (ab)using the RESETTING state to prevent subsequent
3924 		 * recovery actions from interfering with the controller's
3925 		 * firmware activation.
3926 		 */
3927 		if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
3928 			queue_work(nvme_wq, &ctrl->fw_act_work);
3929 		break;
3930 #ifdef CONFIG_NVME_MULTIPATH
3931 	case NVME_AER_NOTICE_ANA:
3932 		if (!ctrl->ana_log_buf)
3933 			break;
3934 		queue_work(nvme_wq, &ctrl->ana_work);
3935 		break;
3936 #endif
3937 	case NVME_AER_NOTICE_DISC_CHANGED:
3938 		ctrl->aen_result = result;
3939 		break;
3940 	default:
3941 		dev_warn(ctrl->device, "async event result %08x\n", result);
3942 	}
3943 }
3944 
3945 void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
3946 		volatile union nvme_result *res)
3947 {
3948 	u32 result = le32_to_cpu(res->u32);
3949 	u32 aer_type = result & 0x07;
3950 
3951 	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
3952 		return;
3953 
3954 	switch (aer_type) {
3955 	case NVME_AER_NOTICE:
3956 		nvme_handle_aen_notice(ctrl, result);
3957 		break;
3958 	case NVME_AER_ERROR:
3959 	case NVME_AER_SMART:
3960 	case NVME_AER_CSS:
3961 	case NVME_AER_VS:
3962 		trace_nvme_async_event(ctrl, aer_type);
3963 		ctrl->aen_result = result;
3964 		break;
3965 	default:
3966 		break;
3967 	}
3968 	queue_work(nvme_wq, &ctrl->async_event_work);
3969 }
3970 EXPORT_SYMBOL_GPL(nvme_complete_async_event);
3971 
3972 void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
3973 {
3974 	nvme_mpath_stop(ctrl);
3975 	nvme_stop_keep_alive(ctrl);
3976 	flush_work(&ctrl->async_event_work);
3977 	cancel_work_sync(&ctrl->fw_act_work);
3978 }
3979 EXPORT_SYMBOL_GPL(nvme_stop_ctrl);
3980 
3981 void nvme_start_ctrl(struct nvme_ctrl *ctrl)
3982 {
3983 	if (ctrl->kato)
3984 		nvme_start_keep_alive(ctrl);
3985 
3986 	nvme_enable_aen(ctrl);
3987 
3988 	if (ctrl->queue_count > 1) {
3989 		nvme_queue_scan(ctrl);
3990 		nvme_start_queues(ctrl);
3991 	}
3992 }
3993 EXPORT_SYMBOL_GPL(nvme_start_ctrl);
3994 
3995 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
3996 {
3997 	nvme_fault_inject_fini(&ctrl->fault_inject);
3998 	dev_pm_qos_hide_latency_tolerance(ctrl->device);
3999 	cdev_device_del(&ctrl->cdev, ctrl->device);
4000 }
4001 EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
4002 
4003 static void nvme_free_ctrl(struct device *dev)
4004 {
4005 	struct nvme_ctrl *ctrl =
4006 		container_of(dev, struct nvme_ctrl, ctrl_device);
4007 	struct nvme_subsystem *subsys = ctrl->subsys;
4008 
4009 	if (subsys && ctrl->instance != subsys->instance)
4010 		ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4011 
4012 	kfree(ctrl->effects);
4013 	nvme_mpath_uninit(ctrl);
4014 	__free_page(ctrl->discard_page);
4015 
4016 	if (subsys) {
4017 		mutex_lock(&nvme_subsystems_lock);
4018 		list_del(&ctrl->subsys_entry);
4019 		sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
4020 		mutex_unlock(&nvme_subsystems_lock);
4021 	}
4022 
4023 	ctrl->ops->free_ctrl(ctrl);
4024 
4025 	if (subsys)
4026 		nvme_put_subsystem(subsys);
4027 }
4028 
4029 /*
4030  * Initialize a NVMe controller structures.  This needs to be called during
4031  * earliest initialization so that we have the initialized structured around
4032  * during probing.
4033  */
4034 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
4035 		const struct nvme_ctrl_ops *ops, unsigned long quirks)
4036 {
4037 	int ret;
4038 
4039 	ctrl->state = NVME_CTRL_NEW;
4040 	spin_lock_init(&ctrl->lock);
4041 	mutex_init(&ctrl->scan_lock);
4042 	INIT_LIST_HEAD(&ctrl->namespaces);
4043 	init_rwsem(&ctrl->namespaces_rwsem);
4044 	ctrl->dev = dev;
4045 	ctrl->ops = ops;
4046 	ctrl->quirks = quirks;
4047 	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
4048 	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
4049 	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
4050 	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
4051 	init_waitqueue_head(&ctrl->state_wq);
4052 
4053 	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
4054 	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
4055 	ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
4056 
4057 	BUILD_BUG_ON(NVME_DSM_MAX_RANGES * sizeof(struct nvme_dsm_range) >
4058 			PAGE_SIZE);
4059 	ctrl->discard_page = alloc_page(GFP_KERNEL);
4060 	if (!ctrl->discard_page) {
4061 		ret = -ENOMEM;
4062 		goto out;
4063 	}
4064 
4065 	ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
4066 	if (ret < 0)
4067 		goto out;
4068 	ctrl->instance = ret;
4069 
4070 	device_initialize(&ctrl->ctrl_device);
4071 	ctrl->device = &ctrl->ctrl_device;
4072 	ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
4073 	ctrl->device->class = nvme_class;
4074 	ctrl->device->parent = ctrl->dev;
4075 	ctrl->device->groups = nvme_dev_attr_groups;
4076 	ctrl->device->release = nvme_free_ctrl;
4077 	dev_set_drvdata(ctrl->device, ctrl);
4078 	ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
4079 	if (ret)
4080 		goto out_release_instance;
4081 
4082 	cdev_init(&ctrl->cdev, &nvme_dev_fops);
4083 	ctrl->cdev.owner = ops->module;
4084 	ret = cdev_device_add(&ctrl->cdev, ctrl->device);
4085 	if (ret)
4086 		goto out_free_name;
4087 
4088 	/*
4089 	 * Initialize latency tolerance controls.  The sysfs files won't
4090 	 * be visible to userspace unless the device actually supports APST.
4091 	 */
4092 	ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
4093 	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
4094 		min(default_ps_max_latency_us, (unsigned long)S32_MAX));
4095 
4096 	nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
4097 
4098 	return 0;
4099 out_free_name:
4100 	kfree_const(ctrl->device->kobj.name);
4101 out_release_instance:
4102 	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
4103 out:
4104 	if (ctrl->discard_page)
4105 		__free_page(ctrl->discard_page);
4106 	return ret;
4107 }
4108 EXPORT_SYMBOL_GPL(nvme_init_ctrl);
4109 
4110 /**
4111  * nvme_kill_queues(): Ends all namespace queues
4112  * @ctrl: the dead controller that needs to end
4113  *
4114  * Call this function when the driver determines it is unable to get the
4115  * controller in a state capable of servicing IO.
4116  */
4117 void nvme_kill_queues(struct nvme_ctrl *ctrl)
4118 {
4119 	struct nvme_ns *ns;
4120 
4121 	down_read(&ctrl->namespaces_rwsem);
4122 
4123 	/* Forcibly unquiesce queues to avoid blocking dispatch */
4124 	if (ctrl->admin_q && !blk_queue_dying(ctrl->admin_q))
4125 		blk_mq_unquiesce_queue(ctrl->admin_q);
4126 
4127 	list_for_each_entry(ns, &ctrl->namespaces, list)
4128 		nvme_set_queue_dying(ns);
4129 
4130 	up_read(&ctrl->namespaces_rwsem);
4131 }
4132 EXPORT_SYMBOL_GPL(nvme_kill_queues);
4133 
4134 void nvme_unfreeze(struct nvme_ctrl *ctrl)
4135 {
4136 	struct nvme_ns *ns;
4137 
4138 	down_read(&ctrl->namespaces_rwsem);
4139 	list_for_each_entry(ns, &ctrl->namespaces, list)
4140 		blk_mq_unfreeze_queue(ns->queue);
4141 	up_read(&ctrl->namespaces_rwsem);
4142 }
4143 EXPORT_SYMBOL_GPL(nvme_unfreeze);
4144 
4145 void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
4146 {
4147 	struct nvme_ns *ns;
4148 
4149 	down_read(&ctrl->namespaces_rwsem);
4150 	list_for_each_entry(ns, &ctrl->namespaces, list) {
4151 		timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
4152 		if (timeout <= 0)
4153 			break;
4154 	}
4155 	up_read(&ctrl->namespaces_rwsem);
4156 }
4157 EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
4158 
4159 void nvme_wait_freeze(struct nvme_ctrl *ctrl)
4160 {
4161 	struct nvme_ns *ns;
4162 
4163 	down_read(&ctrl->namespaces_rwsem);
4164 	list_for_each_entry(ns, &ctrl->namespaces, list)
4165 		blk_mq_freeze_queue_wait(ns->queue);
4166 	up_read(&ctrl->namespaces_rwsem);
4167 }
4168 EXPORT_SYMBOL_GPL(nvme_wait_freeze);
4169 
4170 void nvme_start_freeze(struct nvme_ctrl *ctrl)
4171 {
4172 	struct nvme_ns *ns;
4173 
4174 	down_read(&ctrl->namespaces_rwsem);
4175 	list_for_each_entry(ns, &ctrl->namespaces, list)
4176 		blk_freeze_queue_start(ns->queue);
4177 	up_read(&ctrl->namespaces_rwsem);
4178 }
4179 EXPORT_SYMBOL_GPL(nvme_start_freeze);
4180 
4181 void nvme_stop_queues(struct nvme_ctrl *ctrl)
4182 {
4183 	struct nvme_ns *ns;
4184 
4185 	down_read(&ctrl->namespaces_rwsem);
4186 	list_for_each_entry(ns, &ctrl->namespaces, list)
4187 		blk_mq_quiesce_queue(ns->queue);
4188 	up_read(&ctrl->namespaces_rwsem);
4189 }
4190 EXPORT_SYMBOL_GPL(nvme_stop_queues);
4191 
4192 void nvme_start_queues(struct nvme_ctrl *ctrl)
4193 {
4194 	struct nvme_ns *ns;
4195 
4196 	down_read(&ctrl->namespaces_rwsem);
4197 	list_for_each_entry(ns, &ctrl->namespaces, list)
4198 		blk_mq_unquiesce_queue(ns->queue);
4199 	up_read(&ctrl->namespaces_rwsem);
4200 }
4201 EXPORT_SYMBOL_GPL(nvme_start_queues);
4202 
4203 
4204 void nvme_sync_queues(struct nvme_ctrl *ctrl)
4205 {
4206 	struct nvme_ns *ns;
4207 
4208 	down_read(&ctrl->namespaces_rwsem);
4209 	list_for_each_entry(ns, &ctrl->namespaces, list)
4210 		blk_sync_queue(ns->queue);
4211 	up_read(&ctrl->namespaces_rwsem);
4212 
4213 	if (ctrl->admin_q)
4214 		blk_sync_queue(ctrl->admin_q);
4215 }
4216 EXPORT_SYMBOL_GPL(nvme_sync_queues);
4217 
4218 /*
4219  * Check we didn't inadvertently grow the command structure sizes:
4220  */
4221 static inline void _nvme_check_size(void)
4222 {
4223 	BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
4224 	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
4225 	BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
4226 	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
4227 	BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
4228 	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
4229 	BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
4230 	BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
4231 	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
4232 	BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
4233 	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
4234 	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
4235 	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
4236 	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
4237 	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
4238 	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
4239 	BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
4240 }
4241 
4242 
4243 static int __init nvme_core_init(void)
4244 {
4245 	int result = -ENOMEM;
4246 
4247 	_nvme_check_size();
4248 
4249 	nvme_wq = alloc_workqueue("nvme-wq",
4250 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4251 	if (!nvme_wq)
4252 		goto out;
4253 
4254 	nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
4255 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4256 	if (!nvme_reset_wq)
4257 		goto destroy_wq;
4258 
4259 	nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
4260 			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
4261 	if (!nvme_delete_wq)
4262 		goto destroy_reset_wq;
4263 
4264 	result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
4265 	if (result < 0)
4266 		goto destroy_delete_wq;
4267 
4268 	nvme_class = class_create(THIS_MODULE, "nvme");
4269 	if (IS_ERR(nvme_class)) {
4270 		result = PTR_ERR(nvme_class);
4271 		goto unregister_chrdev;
4272 	}
4273 	nvme_class->dev_uevent = nvme_class_uevent;
4274 
4275 	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
4276 	if (IS_ERR(nvme_subsys_class)) {
4277 		result = PTR_ERR(nvme_subsys_class);
4278 		goto destroy_class;
4279 	}
4280 	return 0;
4281 
4282 destroy_class:
4283 	class_destroy(nvme_class);
4284 unregister_chrdev:
4285 	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4286 destroy_delete_wq:
4287 	destroy_workqueue(nvme_delete_wq);
4288 destroy_reset_wq:
4289 	destroy_workqueue(nvme_reset_wq);
4290 destroy_wq:
4291 	destroy_workqueue(nvme_wq);
4292 out:
4293 	return result;
4294 }
4295 
4296 static void __exit nvme_core_exit(void)
4297 {
4298 	class_destroy(nvme_subsys_class);
4299 	class_destroy(nvme_class);
4300 	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
4301 	destroy_workqueue(nvme_delete_wq);
4302 	destroy_workqueue(nvme_reset_wq);
4303 	destroy_workqueue(nvme_wq);
4304 }
4305 
4306 MODULE_LICENSE("GPL");
4307 MODULE_VERSION("1.0");
4308 module_init(nvme_core_init);
4309 module_exit(nvme_core_exit);
4310