xref: /freebsd/usr.sbin/bhyve/pci_nvme.c (revision 190cef3d52236565eb22e18b33e9e865ec634aa3)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2017 Shunsuke Mie
5  * Copyright (c) 2018 Leon Dang
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 /*
30  * bhyve PCIe-NVMe device emulation.
31  *
32  * options:
33  *  -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z
34  *
35  *  accepted devpath:
36  *    /dev/blockdev
37  *    /path/to/image
38  *    ram=size_in_MiB
39  *
40  *  maxq    = max number of queues
41  *  qsz     = max elements in each queue
42  *  ioslots = max number of concurrent io requests
43  *  sectsz  = sector size (defaults to blockif sector size)
44  *  ser     = serial number (20-chars max)
45  *
46  */
47 
48 /* TODO:
49     - create async event for smart and log
50     - intr coalesce
51  */
52 
53 #include <sys/cdefs.h>
54 __FBSDID("$FreeBSD$");
55 
56 #include <sys/types.h>
57 
58 #include <assert.h>
59 #include <pthread.h>
60 #include <semaphore.h>
61 #include <stdbool.h>
62 #include <stddef.h>
63 #include <stdint.h>
64 #include <stdio.h>
65 #include <stdlib.h>
66 #include <string.h>
67 
68 #include <machine/atomic.h>
69 #include <machine/vmm.h>
70 #include <vmmapi.h>
71 
72 #include <dev/nvme/nvme.h>
73 
74 #include "bhyverun.h"
75 #include "block_if.h"
76 #include "pci_emul.h"
77 
78 
79 static int nvme_debug = 0;
80 #define	DPRINTF(params) if (nvme_debug) printf params
81 #define	WPRINTF(params) printf params
82 
83 /* defaults; can be overridden */
84 #define	NVME_MSIX_BAR		4
85 
86 #define	NVME_IOSLOTS		8
87 
88 #define	NVME_QUEUES		16
89 #define	NVME_MAX_QENTRIES	2048
90 
91 #define	NVME_PRP2_ITEMS		(PAGE_SIZE/sizeof(uint64_t))
92 #define	NVME_MAX_BLOCKIOVS	512
93 
94 /* helpers */
95 
96 #define	NVME_DOORBELL_OFFSET	offsetof(struct nvme_registers, doorbell)
97 
98 enum nvme_controller_register_offsets {
99 	NVME_CR_CAP_LOW = 0x00,
100 	NVME_CR_CAP_HI  = 0x04,
101 	NVME_CR_VS      = 0x08,
102 	NVME_CR_INTMS   = 0x0c,
103 	NVME_CR_INTMC   = 0x10,
104 	NVME_CR_CC      = 0x14,
105 	NVME_CR_CSTS    = 0x1c,
106 	NVME_CR_NSSR    = 0x20,
107 	NVME_CR_AQA     = 0x24,
108 	NVME_CR_ASQ_LOW = 0x28,
109 	NVME_CR_ASQ_HI  = 0x2c,
110 	NVME_CR_ACQ_LOW = 0x30,
111 	NVME_CR_ACQ_HI  = 0x34,
112 };
113 
114 enum nvme_cmd_cdw11 {
115 	NVME_CMD_CDW11_PC  = 0x0001,
116 	NVME_CMD_CDW11_IEN = 0x0002,
117 	NVME_CMD_CDW11_IV  = 0xFFFF0000,
118 };
119 
120 #define	NVME_CQ_INTEN	0x01
121 #define	NVME_CQ_INTCOAL	0x02
122 
123 struct nvme_completion_queue {
124 	struct nvme_completion *qbase;
125 	uint32_t	size;
126 	uint16_t	tail; /* nvme progress */
127 	uint16_t	head; /* guest progress */
128 	uint16_t	intr_vec;
129 	uint32_t	intr_en;
130 	pthread_mutex_t	mtx;
131 };
132 
133 struct nvme_submission_queue {
134 	struct nvme_command *qbase;
135 	uint32_t	size;
136 	uint16_t	head; /* nvme progress */
137 	uint16_t	tail; /* guest progress */
138 	uint16_t	cqid; /* completion queue id */
139 	int		busy; /* queue is being processed */
140 	int		qpriority;
141 };
142 
143 enum nvme_storage_type {
144 	NVME_STOR_BLOCKIF = 0,
145 	NVME_STOR_RAM = 1,
146 };
147 
148 struct pci_nvme_blockstore {
149 	enum nvme_storage_type type;
150 	void		*ctx;
151 	uint64_t	size;
152 	uint32_t	sectsz;
153 	uint32_t	sectsz_bits;
154 };
155 
156 struct pci_nvme_ioreq {
157 	struct pci_nvme_softc *sc;
158 	struct pci_nvme_ioreq *next;
159 	struct nvme_submission_queue *nvme_sq;
160 	uint16_t	sqid;
161 
162 	/* command information */
163 	uint16_t	opc;
164 	uint16_t	cid;
165 	uint32_t	nsid;
166 
167 	uint64_t	prev_gpaddr;
168 	size_t		prev_size;
169 
170 	/*
171 	 * lock if all iovs consumed (big IO);
172 	 * complete transaction before continuing
173 	 */
174 	pthread_mutex_t	mtx;
175 	pthread_cond_t	cv;
176 
177 	struct blockif_req io_req;
178 
179 	/* pad to fit up to 512 page descriptors from guest IO request */
180 	struct iovec	iovpadding[NVME_MAX_BLOCKIOVS-BLOCKIF_IOV_MAX];
181 };
182 
183 struct pci_nvme_softc {
184 	struct pci_devinst *nsc_pi;
185 
186 	pthread_mutex_t	mtx;
187 
188 	struct nvme_registers regs;
189 
190 	struct nvme_namespace_data  nsdata;
191 	struct nvme_controller_data ctrldata;
192 
193 	struct pci_nvme_blockstore nvstore;
194 
195 	uint16_t	max_qentries; /* max entries per queue */
196 	uint32_t	max_queues;
197 	uint32_t	num_cqueues;
198 	uint32_t	num_squeues;
199 
200 	struct pci_nvme_ioreq *ioreqs;
201 	struct pci_nvme_ioreq *ioreqs_free; /* free list of ioreqs */
202 	uint32_t	pending_ios;
203 	uint32_t	ioslots;
204 	sem_t		iosemlock;
205 
206 	/* status and guest memory mapped queues */
207 	struct nvme_completion_queue *compl_queues;
208 	struct nvme_submission_queue *submit_queues;
209 
210 	/* controller features */
211 	uint32_t	intr_coales_aggr_time;   /* 0x08: uS to delay intr */
212 	uint32_t	intr_coales_aggr_thresh; /* 0x08: compl-Q entries */
213 	uint32_t	async_ev_config;         /* 0x0B: async event config */
214 };
215 
216 
217 static void pci_nvme_io_partial(struct blockif_req *br, int err);
218 
219 /* Controller Configuration utils */
220 #define	NVME_CC_GET_EN(cc) \
221 	((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK)
222 #define	NVME_CC_GET_CSS(cc) \
223 	((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK)
224 #define	NVME_CC_GET_SHN(cc) \
225 	((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK)
226 #define	NVME_CC_GET_IOSQES(cc) \
227 	((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK)
228 #define	NVME_CC_GET_IOCQES(cc) \
229 	((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK)
230 
231 #define	NVME_CC_WRITE_MASK \
232 	((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \
233 	 (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \
234 	 (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT))
235 
236 #define	NVME_CC_NEN_WRITE_MASK \
237 	((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \
238 	 (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \
239 	 (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT))
240 
241 /* Controller Status utils */
242 #define	NVME_CSTS_GET_RDY(sts) \
243 	((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK)
244 
245 #define	NVME_CSTS_RDY	(1 << NVME_CSTS_REG_RDY_SHIFT)
246 
247 /* Completion Queue status word utils */
248 #define	NVME_STATUS_P	(1 << NVME_STATUS_P_SHIFT)
249 #define	NVME_STATUS_MASK \
250 	((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\
251 	 (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT))
252 
253 static __inline void
254 cpywithpad(char *dst, int dst_size, const char *src, char pad)
255 {
256 	int len = strnlen(src, dst_size);
257 	memcpy(dst, src, len);
258 	memset(dst + len, pad, dst_size - len);
259 }
260 
261 static __inline void
262 pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code)
263 {
264 
265 	*status &= ~NVME_STATUS_MASK;
266 	*status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT |
267 		(code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT;
268 }
269 
270 static __inline void
271 pci_nvme_status_genc(uint16_t *status, uint16_t code)
272 {
273 
274 	pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
275 }
276 
277 static __inline void
278 pci_nvme_toggle_phase(uint16_t *status, int prev)
279 {
280 
281 	if (prev)
282 		*status &= ~NVME_STATUS_P;
283 	else
284 		*status |= NVME_STATUS_P;
285 }
286 
287 static void
288 pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
289 {
290 	struct nvme_controller_data *cd = &sc->ctrldata;
291 
292 	cd->vid = 0xFB5D;
293 	cd->ssvid = 0x0000;
294 
295 	cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');
296 	cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' ');
297 
298 	/* Num of submission commands that we can handle at a time (2^rab) */
299 	cd->rab   = 4;
300 
301 	/* FreeBSD OUI */
302 	cd->ieee[0] = 0x58;
303 	cd->ieee[1] = 0x9c;
304 	cd->ieee[2] = 0xfc;
305 
306 	cd->mic = 0;
307 
308 	cd->mdts = 9;	/* max data transfer size (2^mdts * CAP.MPSMIN) */
309 
310 	cd->ver = 0x00010300;
311 
312 	cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT;
313 	cd->acl = 2;
314 	cd->aerl = 4;
315 
316 	cd->lpa = 0;	/* TODO: support some simple things like SMART */
317 	cd->elpe = 0;	/* max error log page entries */
318 	cd->npss = 1;	/* number of power states support */
319 
320 	/* Warning Composite Temperature Threshold */
321 	cd->wctemp = 0x0157;
322 
323 	cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) |
324 	    (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT);
325 	cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) |
326 	    (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT);
327 	cd->nn = 1;	/* number of namespaces */
328 
329 	cd->fna = 0x03;
330 
331 	cd->power_state[0].mp = 10;
332 }
333 
334 static void
335 pci_nvme_init_nsdata(struct pci_nvme_softc *sc)
336 {
337 	struct nvme_namespace_data *nd;
338 
339 	nd = &sc->nsdata;
340 
341 	nd->nsze = sc->nvstore.size / sc->nvstore.sectsz;
342 	nd->ncap = nd->nsze;
343 	nd->nuse = nd->nsze;
344 
345 	/* Get LBA and backstore information from backing store */
346 	nd->nlbaf = 1;
347 	/* LBA data-sz = 2^lbads */
348 	nd->lbaf[0] = sc->nvstore.sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT;
349 
350 	nd->flbas = 0;
351 }
352 
353 static void
354 pci_nvme_reset_locked(struct pci_nvme_softc *sc)
355 {
356 	DPRINTF(("%s\r\n", __func__));
357 
358 	sc->regs.cap_lo = (sc->max_qentries & NVME_CAP_LO_REG_MQES_MASK) |
359 	    (1 << NVME_CAP_LO_REG_CQR_SHIFT) |
360 	    (60 << NVME_CAP_LO_REG_TO_SHIFT);
361 
362 	sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;
363 
364 	sc->regs.vs = 0x00010300;	/* NVMe v1.3 */
365 
366 	sc->regs.cc = 0;
367 	sc->regs.csts = 0;
368 
369 	sc->num_cqueues = sc->num_squeues = sc->max_queues;
370 	if (sc->submit_queues != NULL) {
371 		for (int i = 0; i <= sc->max_queues; i++) {
372 			/*
373 			 * The Admin Submission Queue is at index 0.
374 			 * It must not be changed at reset otherwise the
375 			 * emulation will be out of sync with the guest.
376 			 */
377 			if (i != 0) {
378 				sc->submit_queues[i].qbase = NULL;
379 				sc->submit_queues[i].size = 0;
380 				sc->submit_queues[i].cqid = 0;
381 
382 				sc->compl_queues[i].qbase = NULL;
383 				sc->compl_queues[i].size = 0;
384 			}
385 			sc->submit_queues[i].tail = 0;
386 			sc->submit_queues[i].head = 0;
387 			sc->submit_queues[i].busy = 0;
388 
389 			sc->compl_queues[i].tail = 0;
390 			sc->compl_queues[i].head = 0;
391 		}
392 	} else
393 		sc->submit_queues = calloc(sc->max_queues + 1,
394 		                        sizeof(struct nvme_submission_queue));
395 
396 	if (sc->compl_queues == NULL) {
397 		sc->compl_queues = calloc(sc->max_queues + 1,
398 		                        sizeof(struct nvme_completion_queue));
399 
400 		for (int i = 0; i <= sc->num_cqueues; i++)
401 			pthread_mutex_init(&sc->compl_queues[i].mtx, NULL);
402 	}
403 }
404 
405 static void
406 pci_nvme_reset(struct pci_nvme_softc *sc)
407 {
408 	pthread_mutex_lock(&sc->mtx);
409 	pci_nvme_reset_locked(sc);
410 	pthread_mutex_unlock(&sc->mtx);
411 }
412 
413 static void
414 pci_nvme_init_controller(struct vmctx *ctx, struct pci_nvme_softc *sc)
415 {
416 	uint16_t acqs, asqs;
417 
418 	DPRINTF(("%s\r\n", __func__));
419 
420 	asqs = (sc->regs.aqa & NVME_AQA_REG_ASQS_MASK) + 1;
421 	sc->submit_queues[0].size = asqs;
422 	sc->submit_queues[0].qbase = vm_map_gpa(ctx, sc->regs.asq,
423 	            sizeof(struct nvme_command) * asqs);
424 
425 	DPRINTF(("%s mapping Admin-SQ guest 0x%lx, host: %p\r\n",
426 	        __func__, sc->regs.asq, sc->submit_queues[0].qbase));
427 
428 	acqs = ((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
429 	    NVME_AQA_REG_ACQS_MASK) + 1;
430 	sc->compl_queues[0].size = acqs;
431 	sc->compl_queues[0].qbase = vm_map_gpa(ctx, sc->regs.acq,
432 	         sizeof(struct nvme_completion) * acqs);
433 	DPRINTF(("%s mapping Admin-CQ guest 0x%lx, host: %p\r\n",
434 	        __func__, sc->regs.acq, sc->compl_queues[0].qbase));
435 }
436 
437 static int
438 nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
439 	struct nvme_completion* compl)
440 {
441 	uint16_t qid = command->cdw10 & 0xffff;
442 
443 	DPRINTF(("%s DELETE_IO_SQ %u\r\n", __func__, qid));
444 	if (qid == 0 || qid > sc->num_cqueues) {
445 		WPRINTF(("%s NOT PERMITTED queue id %u / num_squeues %u\r\n",
446 		        __func__, qid, sc->num_squeues));
447 		pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
448 		    NVME_SC_INVALID_QUEUE_IDENTIFIER);
449 		return (1);
450 	}
451 
452 	sc->submit_queues[qid].qbase = NULL;
453 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
454 	return (1);
455 }
456 
457 static int
458 nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
459 	struct nvme_completion* compl)
460 {
461 	if (command->cdw11 & NVME_CMD_CDW11_PC) {
462 		uint16_t qid = command->cdw10 & 0xffff;
463 		struct nvme_submission_queue *nsq;
464 
465 		if (qid > sc->num_squeues) {
466 			WPRINTF(("%s queue index %u > num_squeues %u\r\n",
467 			        __func__, qid, sc->num_squeues));
468 			pci_nvme_status_tc(&compl->status,
469 			    NVME_SCT_COMMAND_SPECIFIC,
470 			    NVME_SC_INVALID_QUEUE_IDENTIFIER);
471 			return (1);
472 		}
473 
474 		nsq = &sc->submit_queues[qid];
475 		nsq->size = ((command->cdw10 >> 16) & 0xffff) + 1;
476 
477 		nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
478 		              sizeof(struct nvme_command) * (size_t)nsq->size);
479 		nsq->cqid = (command->cdw11 >> 16) & 0xffff;
480 		nsq->qpriority = (command->cdw11 >> 1) & 0x03;
481 
482 		DPRINTF(("%s sq %u size %u gaddr %p cqid %u\r\n", __func__,
483 		        qid, nsq->size, nsq->qbase, nsq->cqid));
484 
485 		pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
486 
487 		DPRINTF(("%s completed creating IOSQ qid %u\r\n",
488 		         __func__, qid));
489 	} else {
490 		/*
491 		 * Guest sent non-cont submission queue request.
492 		 * This setting is unsupported by this emulation.
493 		 */
494 		WPRINTF(("%s unsupported non-contig (list-based) "
495 		         "create i/o submission queue\r\n", __func__));
496 
497 		pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
498 	}
499 	return (1);
500 }
501 
502 static int
503 nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
504 	struct nvme_completion* compl)
505 {
506 	uint16_t qid = command->cdw10 & 0xffff;
507 
508 	DPRINTF(("%s DELETE_IO_CQ %u\r\n", __func__, qid));
509 	if (qid == 0 || qid > sc->num_cqueues) {
510 		WPRINTF(("%s queue index %u / num_cqueues %u\r\n",
511 		        __func__, qid, sc->num_cqueues));
512 		pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
513 		    NVME_SC_INVALID_QUEUE_IDENTIFIER);
514 		return (1);
515 	}
516 
517 	sc->compl_queues[qid].qbase = NULL;
518 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
519 	return (1);
520 }
521 
522 static int
523 nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
524 	struct nvme_completion* compl)
525 {
526 	if (command->cdw11 & NVME_CMD_CDW11_PC) {
527 		uint16_t qid = command->cdw10 & 0xffff;
528 		struct nvme_completion_queue *ncq;
529 
530 		if (qid > sc->num_cqueues) {
531 			WPRINTF(("%s queue index %u > num_cqueues %u\r\n",
532 			        __func__, qid, sc->num_cqueues));
533 			pci_nvme_status_tc(&compl->status,
534 			    NVME_SCT_COMMAND_SPECIFIC,
535 			    NVME_SC_INVALID_QUEUE_IDENTIFIER);
536 			return (1);
537 		}
538 
539 		ncq = &sc->compl_queues[qid];
540 		ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1;
541 		ncq->intr_vec = (command->cdw11 >> 16) & 0xffff;
542 		ncq->size = ((command->cdw10 >> 16) & 0xffff) + 1;
543 
544 		ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
545 		             command->prp1,
546 		             sizeof(struct nvme_command) * (size_t)ncq->size);
547 
548 		pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
549 	} else {
550 		/*
551 		 * Non-contig completion queue unsupported.
552 		 */
553 		WPRINTF(("%s unsupported non-contig (list-based) "
554 		         "create i/o completion queue\r\n",
555 		         __func__));
556 
557 		/* 0x12 = Invalid Use of Controller Memory Buffer */
558 		pci_nvme_status_genc(&compl->status, 0x12);
559 	}
560 
561 	return (1);
562 }
563 
564 static int
565 nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
566 	struct nvme_completion* compl)
567 {
568 	uint32_t logsize = (1 + ((command->cdw10 >> 16) & 0xFFF)) * 2;
569 	uint8_t logpage = command->cdw10 & 0xFF;
570 	void *data;
571 
572 	DPRINTF(("%s log page %u len %u\r\n", __func__, logpage, logsize));
573 
574 	if (logpage >= 1 && logpage <= 3)
575 		data = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
576 		                  PAGE_SIZE);
577 
578 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
579 
580 	switch (logpage) {
581 	case 0x01: /* Error information */
582 		memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
583 		break;
584 	case 0x02: /* SMART/Health information */
585 		/* TODO: present some smart info */
586 		memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
587 		break;
588 	case 0x03: /* Firmware slot information */
589 		memset(data, 0, logsize > PAGE_SIZE ? PAGE_SIZE : logsize);
590 		break;
591 	default:
592 		WPRINTF(("%s get log page %x command not supported\r\n",
593 		        __func__, logpage));
594 
595 		pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
596 		    NVME_SC_INVALID_LOG_PAGE);
597 	}
598 
599 	return (1);
600 }
601 
602 static int
603 nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
604 	struct nvme_completion* compl)
605 {
606 	void *dest;
607 
608 	DPRINTF(("%s identify 0x%x nsid 0x%x\r\n", __func__,
609 	        command->cdw10 & 0xFF, command->nsid));
610 
611 	switch (command->cdw10 & 0xFF) {
612 	case 0x00: /* return Identify Namespace data structure */
613 		dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
614 		                  sizeof(sc->nsdata));
615 		memcpy(dest, &sc->nsdata, sizeof(sc->nsdata));
616 		break;
617 	case 0x01: /* return Identify Controller data structure */
618 		dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
619 		                  sizeof(sc->ctrldata));
620 		memcpy(dest, &sc->ctrldata, sizeof(sc->ctrldata));
621 		break;
622 	case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */
623 		dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
624 		                  sizeof(uint32_t) * 1024);
625 		((uint32_t *)dest)[0] = 1;
626 		((uint32_t *)dest)[1] = 0;
627 		break;
628 	case 0x11:
629 		pci_nvme_status_genc(&compl->status,
630 		    NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
631 		return (1);
632 	case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */
633 	case 0x10:
634 	case 0x12:
635 	case 0x13:
636 	case 0x14:
637 	case 0x15:
638 	default:
639 		DPRINTF(("%s unsupported identify command requested 0x%x\r\n",
640 		         __func__, command->cdw10 & 0xFF));
641 		pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
642 		return (1);
643 	}
644 
645 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
646 	return (1);
647 }
648 
649 static int
650 nvme_opc_set_features(struct pci_nvme_softc* sc, struct nvme_command* command,
651 	struct nvme_completion* compl)
652 {
653 	int feature = command->cdw10 & 0xFF;
654 	uint32_t iv;
655 
656 	DPRINTF(("%s feature 0x%x\r\n", __func__, feature));
657 	compl->cdw0 = 0;
658 
659 	switch (feature) {
660 	case NVME_FEAT_ARBITRATION:
661 		DPRINTF(("  arbitration 0x%x\r\n", command->cdw11));
662 		break;
663 	case NVME_FEAT_POWER_MANAGEMENT:
664 		DPRINTF(("  power management 0x%x\r\n", command->cdw11));
665 		break;
666 	case NVME_FEAT_LBA_RANGE_TYPE:
667 		DPRINTF(("  lba range 0x%x\r\n", command->cdw11));
668 		break;
669 	case NVME_FEAT_TEMPERATURE_THRESHOLD:
670 		DPRINTF(("  temperature threshold 0x%x\r\n", command->cdw11));
671 		break;
672 	case NVME_FEAT_ERROR_RECOVERY:
673 		DPRINTF(("  error recovery 0x%x\r\n", command->cdw11));
674 		break;
675 	case NVME_FEAT_VOLATILE_WRITE_CACHE:
676 		DPRINTF(("  volatile write cache 0x%x\r\n", command->cdw11));
677 		break;
678 	case NVME_FEAT_NUMBER_OF_QUEUES:
679 		sc->num_squeues = command->cdw11 & 0xFFFF;
680 		sc->num_cqueues = (command->cdw11 >> 16) & 0xFFFF;
681 		DPRINTF(("  number of queues (submit %u, completion %u)\r\n",
682 		        sc->num_squeues, sc->num_cqueues));
683 
684 		if (sc->num_squeues == 0 || sc->num_squeues > sc->max_queues)
685 			sc->num_squeues = sc->max_queues;
686 		if (sc->num_cqueues == 0 || sc->num_cqueues > sc->max_queues)
687 			sc->num_cqueues = sc->max_queues;
688 
689 		compl->cdw0 = (sc->num_squeues & 0xFFFF) |
690 		              ((sc->num_cqueues & 0xFFFF) << 16);
691 
692 		break;
693 	case NVME_FEAT_INTERRUPT_COALESCING:
694 		DPRINTF(("  interrupt coalescing 0x%x\r\n", command->cdw11));
695 
696 		/* in uS */
697 		sc->intr_coales_aggr_time = ((command->cdw11 >> 8) & 0xFF)*100;
698 
699 		sc->intr_coales_aggr_thresh = command->cdw11 & 0xFF;
700 		break;
701 	case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
702 		iv = command->cdw11 & 0xFFFF;
703 
704 		DPRINTF(("  interrupt vector configuration 0x%x\r\n",
705 		        command->cdw11));
706 
707 		for (uint32_t i = 0; i <= sc->num_cqueues; i++) {
708 			if (sc->compl_queues[i].intr_vec == iv) {
709 				if (command->cdw11 & (1 << 16))
710 					sc->compl_queues[i].intr_en |=
711 					                      NVME_CQ_INTCOAL;
712 				else
713 					sc->compl_queues[i].intr_en &=
714 					                     ~NVME_CQ_INTCOAL;
715 			}
716 		}
717 		break;
718 	case NVME_FEAT_WRITE_ATOMICITY:
719 		DPRINTF(("  write atomicity 0x%x\r\n", command->cdw11));
720 		break;
721 	case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
722 		DPRINTF(("  async event configuration 0x%x\r\n",
723 		        command->cdw11));
724 		sc->async_ev_config = command->cdw11;
725 		break;
726 	case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
727 		DPRINTF(("  software progress marker 0x%x\r\n",
728 		        command->cdw11));
729 		break;
730 	case 0x0C:
731 		DPRINTF(("  autonomous power state transition 0x%x\r\n",
732 		        command->cdw11));
733 		break;
734 	default:
735 		WPRINTF(("%s invalid feature\r\n", __func__));
736 		pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
737 		return (1);
738 	}
739 
740 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
741 	return (1);
742 }
743 
744 static int
745 nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command,
746 	struct nvme_completion* compl)
747 {
748 	int feature = command->cdw10 & 0xFF;
749 
750 	DPRINTF(("%s feature 0x%x\r\n", __func__, feature));
751 
752 	compl->cdw0 = 0;
753 
754 	switch (feature) {
755 	case NVME_FEAT_ARBITRATION:
756 		DPRINTF(("  arbitration\r\n"));
757 		break;
758 	case NVME_FEAT_POWER_MANAGEMENT:
759 		DPRINTF(("  power management\r\n"));
760 		break;
761 	case NVME_FEAT_LBA_RANGE_TYPE:
762 		DPRINTF(("  lba range\r\n"));
763 		break;
764 	case NVME_FEAT_TEMPERATURE_THRESHOLD:
765 		DPRINTF(("  temperature threshold\r\n"));
766 		switch ((command->cdw11 >> 20) & 0x3) {
767 		case 0:
768 			/* Over temp threshold */
769 			compl->cdw0 = 0xFFFF;
770 			break;
771 		case 1:
772 			/* Under temp threshold */
773 			compl->cdw0 = 0;
774 			break;
775 		default:
776 			WPRINTF(("  invalid threshold type select\r\n"));
777 			pci_nvme_status_genc(&compl->status,
778 			    NVME_SC_INVALID_FIELD);
779 			return (1);
780 		}
781 		break;
782 	case NVME_FEAT_ERROR_RECOVERY:
783 		DPRINTF(("  error recovery\r\n"));
784 		break;
785 	case NVME_FEAT_VOLATILE_WRITE_CACHE:
786 		DPRINTF(("  volatile write cache\r\n"));
787 		break;
788 	case NVME_FEAT_NUMBER_OF_QUEUES:
789 		compl->cdw0 = 0;
790 		if (sc->num_squeues == 0)
791 			compl->cdw0 |= sc->max_queues & 0xFFFF;
792 		else
793 			compl->cdw0 |= sc->num_squeues & 0xFFFF;
794 
795 		if (sc->num_cqueues == 0)
796 			compl->cdw0 |= (sc->max_queues & 0xFFFF) << 16;
797 		else
798 			compl->cdw0 |= (sc->num_cqueues & 0xFFFF) << 16;
799 
800 		DPRINTF(("  number of queues (submit %u, completion %u)\r\n",
801 		        compl->cdw0 & 0xFFFF,
802 		        (compl->cdw0 >> 16) & 0xFFFF));
803 
804 		break;
805 	case NVME_FEAT_INTERRUPT_COALESCING:
806 		DPRINTF(("  interrupt coalescing\r\n"));
807 		break;
808 	case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
809 		DPRINTF(("  interrupt vector configuration\r\n"));
810 		break;
811 	case NVME_FEAT_WRITE_ATOMICITY:
812 		DPRINTF(("  write atomicity\r\n"));
813 		break;
814 	case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
815 		DPRINTF(("  async event configuration\r\n"));
816 		sc->async_ev_config = command->cdw11;
817 		break;
818 	case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
819 		DPRINTF(("  software progress marker\r\n"));
820 		break;
821 	case 0x0C:
822 		DPRINTF(("  autonomous power state transition\r\n"));
823 		break;
824 	default:
825 		WPRINTF(("%s invalid feature 0x%x\r\n", __func__, feature));
826 		pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
827 		return (1);
828 	}
829 
830 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
831 	return (1);
832 }
833 
834 static int
835 nvme_opc_abort(struct pci_nvme_softc* sc, struct nvme_command* command,
836 	struct nvme_completion* compl)
837 {
838 	DPRINTF(("%s submission queue %u, command ID 0x%x\r\n", __func__,
839 	        command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF));
840 
841 	/* TODO: search for the command ID and abort it */
842 
843 	compl->cdw0 = 1;
844 	pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
845 	return (1);
846 }
847 
848 static int
849 nvme_opc_async_event_req(struct pci_nvme_softc* sc,
850 	struct nvme_command* command, struct nvme_completion* compl)
851 {
852 	DPRINTF(("%s async event request 0x%x\r\n", __func__, command->cdw11));
853 
854 	/*
855 	 * TODO: raise events when they happen based on the Set Features cmd.
856 	 * These events happen async, so only set completion successful if
857 	 * there is an event reflective of the request to get event.
858 	 */
859 	pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
860 	    NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
861 	return (0);
862 }
863 
864 static void
865 pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
866 {
867 	struct nvme_completion compl;
868 	struct nvme_command *cmd;
869 	struct nvme_submission_queue *sq;
870 	struct nvme_completion_queue *cq;
871 	int do_intr = 0;
872 	uint16_t sqhead;
873 
874 	DPRINTF(("%s index %u\r\n", __func__, (uint32_t)value));
875 
876 	sq = &sc->submit_queues[0];
877 
878 	sqhead = atomic_load_acq_short(&sq->head);
879 
880 	if (atomic_testandset_int(&sq->busy, 1)) {
881 		DPRINTF(("%s SQ busy, head %u, tail %u\r\n",
882 		        __func__, sqhead, sq->tail));
883 		return;
884 	}
885 
886 	DPRINTF(("sqhead %u, tail %u\r\n", sqhead, sq->tail));
887 
888 	while (sqhead != atomic_load_acq_short(&sq->tail)) {
889 		cmd = &(sq->qbase)[sqhead];
890 		compl.status = 0;
891 
892 		switch (cmd->opc) {
893 		case NVME_OPC_DELETE_IO_SQ:
894 			DPRINTF(("%s command DELETE_IO_SQ\r\n", __func__));
895 			do_intr |= nvme_opc_delete_io_sq(sc, cmd, &compl);
896 			break;
897 		case NVME_OPC_CREATE_IO_SQ:
898 			DPRINTF(("%s command CREATE_IO_SQ\r\n", __func__));
899 			do_intr |= nvme_opc_create_io_sq(sc, cmd, &compl);
900 			break;
901 		case NVME_OPC_DELETE_IO_CQ:
902 			DPRINTF(("%s command DELETE_IO_CQ\r\n", __func__));
903 			do_intr |= nvme_opc_delete_io_cq(sc, cmd, &compl);
904 			break;
905 		case NVME_OPC_CREATE_IO_CQ:
906 			DPRINTF(("%s command CREATE_IO_CQ\r\n", __func__));
907 			do_intr |= nvme_opc_create_io_cq(sc, cmd, &compl);
908 			break;
909 		case NVME_OPC_GET_LOG_PAGE:
910 			DPRINTF(("%s command GET_LOG_PAGE\r\n", __func__));
911 			do_intr |= nvme_opc_get_log_page(sc, cmd, &compl);
912 			break;
913 		case NVME_OPC_IDENTIFY:
914 			DPRINTF(("%s command IDENTIFY\r\n", __func__));
915 			do_intr |= nvme_opc_identify(sc, cmd, &compl);
916 			break;
917 		case NVME_OPC_ABORT:
918 			DPRINTF(("%s command ABORT\r\n", __func__));
919 			do_intr |= nvme_opc_abort(sc, cmd, &compl);
920 			break;
921 		case NVME_OPC_SET_FEATURES:
922 			DPRINTF(("%s command SET_FEATURES\r\n", __func__));
923 			do_intr |= nvme_opc_set_features(sc, cmd, &compl);
924 			break;
925 		case NVME_OPC_GET_FEATURES:
926 			DPRINTF(("%s command GET_FEATURES\r\n", __func__));
927 			do_intr |= nvme_opc_get_features(sc, cmd, &compl);
928 			break;
929 		case NVME_OPC_ASYNC_EVENT_REQUEST:
930 			DPRINTF(("%s command ASYNC_EVENT_REQ\r\n", __func__));
931 			/* XXX dont care, unhandled for now
932 			do_intr |= nvme_opc_async_event_req(sc, cmd, &compl);
933 			*/
934 			break;
935 		default:
936 			WPRINTF(("0x%x command is not implemented\r\n",
937 			    cmd->opc));
938 		}
939 
940 		/* for now skip async event generation */
941 		if (cmd->opc != NVME_OPC_ASYNC_EVENT_REQUEST) {
942 			struct nvme_completion *cp;
943 			int phase;
944 
945 			cq = &sc->compl_queues[0];
946 
947 			cp = &(cq->qbase)[cq->tail];
948 			cp->sqid = 0;
949 			cp->sqhd = sqhead;
950 			cp->cid = cmd->cid;
951 
952 			phase = NVME_STATUS_GET_P(cp->status);
953 			cp->status = compl.status;
954 			pci_nvme_toggle_phase(&cp->status, phase);
955 
956 			cq->tail = (cq->tail + 1) % cq->size;
957 		}
958 		sqhead = (sqhead + 1) % sq->size;
959 	}
960 
961 	DPRINTF(("setting sqhead %u\r\n", sqhead));
962 	atomic_store_short(&sq->head, sqhead);
963 	atomic_store_int(&sq->busy, 0);
964 
965 	if (do_intr)
966 		pci_generate_msix(sc->nsc_pi, 0);
967 
968 }
969 
970 static int
971 pci_nvme_append_iov_req(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req,
972 	uint64_t gpaddr, size_t size, int do_write, uint64_t lba)
973 {
974 	int iovidx;
975 
976 	if (req != NULL) {
977 		/* concatenate contig block-iovs to minimize number of iovs */
978 		if ((req->prev_gpaddr + req->prev_size) == gpaddr) {
979 			iovidx = req->io_req.br_iovcnt - 1;
980 
981 			req->io_req.br_iov[iovidx].iov_base =
982 			    paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
983 			                     req->prev_gpaddr, size);
984 
985 			req->prev_size += size;
986 			req->io_req.br_resid += size;
987 
988 			req->io_req.br_iov[iovidx].iov_len = req->prev_size;
989 		} else {
990 			pthread_mutex_lock(&req->mtx);
991 
992 			iovidx = req->io_req.br_iovcnt;
993 			if (iovidx == NVME_MAX_BLOCKIOVS) {
994 				int err = 0;
995 
996 				DPRINTF(("large I/O, doing partial req\r\n"));
997 
998 				iovidx = 0;
999 				req->io_req.br_iovcnt = 0;
1000 
1001 				req->io_req.br_callback = pci_nvme_io_partial;
1002 
1003 				if (!do_write)
1004 					err = blockif_read(sc->nvstore.ctx,
1005 					                   &req->io_req);
1006 				else
1007 					err = blockif_write(sc->nvstore.ctx,
1008 					                    &req->io_req);
1009 
1010 				/* wait until req completes before cont */
1011 				if (err == 0)
1012 					pthread_cond_wait(&req->cv, &req->mtx);
1013 			}
1014 			if (iovidx == 0) {
1015 				req->io_req.br_offset = lba;
1016 				req->io_req.br_resid = 0;
1017 				req->io_req.br_param = req;
1018 			}
1019 
1020 			req->io_req.br_iov[iovidx].iov_base =
1021 			    paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
1022 			                     gpaddr, size);
1023 
1024 			req->io_req.br_iov[iovidx].iov_len = size;
1025 
1026 			req->prev_gpaddr = gpaddr;
1027 			req->prev_size = size;
1028 			req->io_req.br_resid += size;
1029 
1030 			req->io_req.br_iovcnt++;
1031 
1032 			pthread_mutex_unlock(&req->mtx);
1033 		}
1034 	} else {
1035 		/* RAM buffer: read/write directly */
1036 		void *p = sc->nvstore.ctx;
1037 		void *gptr;
1038 
1039 		if ((lba + size) > sc->nvstore.size) {
1040 			WPRINTF(("%s write would overflow RAM\r\n", __func__));
1041 			return (-1);
1042 		}
1043 
1044 		p = (void *)((uintptr_t)p + (uintptr_t)lba);
1045 		gptr = paddr_guest2host(sc->nsc_pi->pi_vmctx, gpaddr, size);
1046 		if (do_write)
1047 			memcpy(p, gptr, size);
1048 		else
1049 			memcpy(gptr, p, size);
1050 	}
1051 	return (0);
1052 }
1053 
1054 static void
1055 pci_nvme_set_completion(struct pci_nvme_softc *sc,
1056 	struct nvme_submission_queue *sq, int sqid, uint16_t cid,
1057 	uint32_t cdw0, uint16_t status, int ignore_busy)
1058 {
1059 	struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
1060 	struct nvme_completion *compl;
1061 	int do_intr = 0;
1062 	int phase;
1063 
1064 	DPRINTF(("%s sqid %d cqid %u cid %u status: 0x%x 0x%x\r\n",
1065 		 __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
1066 		 NVME_STATUS_GET_SC(status)));
1067 
1068 	pthread_mutex_lock(&cq->mtx);
1069 
1070 	assert(cq->qbase != NULL);
1071 
1072 	compl = &cq->qbase[cq->tail];
1073 
1074 	compl->sqhd = atomic_load_acq_short(&sq->head);
1075 	compl->sqid = sqid;
1076 	compl->cid = cid;
1077 
1078 	// toggle phase
1079 	phase = NVME_STATUS_GET_P(compl->status);
1080 	compl->status = status;
1081 	pci_nvme_toggle_phase(&compl->status, phase);
1082 
1083 	cq->tail = (cq->tail + 1) % cq->size;
1084 
1085 	if (cq->intr_en & NVME_CQ_INTEN)
1086 		do_intr = 1;
1087 
1088 	pthread_mutex_unlock(&cq->mtx);
1089 
1090 	if (ignore_busy || !atomic_load_acq_int(&sq->busy))
1091 		if (do_intr)
1092 			pci_generate_msix(sc->nsc_pi, cq->intr_vec);
1093 }
1094 
1095 static void
1096 pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req)
1097 {
1098 	req->sc = NULL;
1099 	req->nvme_sq = NULL;
1100 	req->sqid = 0;
1101 
1102 	pthread_mutex_lock(&sc->mtx);
1103 
1104 	req->next = sc->ioreqs_free;
1105 	sc->ioreqs_free = req;
1106 	sc->pending_ios--;
1107 
1108 	/* when no more IO pending, can set to ready if device reset/enabled */
1109 	if (sc->pending_ios == 0 &&
1110 	    NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts)))
1111 		sc->regs.csts |= NVME_CSTS_RDY;
1112 
1113 	pthread_mutex_unlock(&sc->mtx);
1114 
1115 	sem_post(&sc->iosemlock);
1116 }
1117 
1118 static struct pci_nvme_ioreq *
1119 pci_nvme_get_ioreq(struct pci_nvme_softc *sc)
1120 {
1121 	struct pci_nvme_ioreq *req = NULL;;
1122 
1123 	sem_wait(&sc->iosemlock);
1124 	pthread_mutex_lock(&sc->mtx);
1125 
1126 	req = sc->ioreqs_free;
1127 	assert(req != NULL);
1128 
1129 	sc->ioreqs_free = req->next;
1130 
1131 	req->next = NULL;
1132 	req->sc = sc;
1133 
1134 	sc->pending_ios++;
1135 
1136 	pthread_mutex_unlock(&sc->mtx);
1137 
1138 	req->io_req.br_iovcnt = 0;
1139 	req->io_req.br_offset = 0;
1140 	req->io_req.br_resid = 0;
1141 	req->io_req.br_param = req;
1142 	req->prev_gpaddr = 0;
1143 	req->prev_size = 0;
1144 
1145 	return req;
1146 }
1147 
1148 static void
1149 pci_nvme_io_done(struct blockif_req *br, int err)
1150 {
1151 	struct pci_nvme_ioreq *req = br->br_param;
1152 	struct nvme_submission_queue *sq = req->nvme_sq;
1153 	uint16_t code, status;
1154 
1155 	DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err)));
1156 
1157 	/* TODO return correct error */
1158 	code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
1159 	pci_nvme_status_genc(&status, code);
1160 
1161 	pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, 0, status, 0);
1162 	pci_nvme_release_ioreq(req->sc, req);
1163 }
1164 
1165 static void
1166 pci_nvme_io_partial(struct blockif_req *br, int err)
1167 {
1168 	struct pci_nvme_ioreq *req = br->br_param;
1169 
1170 	DPRINTF(("%s error %d %s\r\n", __func__, err, strerror(err)));
1171 
1172 	pthread_cond_signal(&req->cv);
1173 }
1174 
1175 
1176 static void
1177 pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
1178 {
1179 	struct nvme_submission_queue *sq;
1180 	uint16_t status;
1181 	uint16_t sqhead;
1182 	int err;
1183 
1184 	/* handle all submissions up to sq->tail index */
1185 	sq = &sc->submit_queues[idx];
1186 
1187 	if (atomic_testandset_int(&sq->busy, 1)) {
1188 		DPRINTF(("%s sqid %u busy\r\n", __func__, idx));
1189 		return;
1190 	}
1191 
1192 	sqhead = atomic_load_acq_short(&sq->head);
1193 
1194 	DPRINTF(("nvme_handle_io qid %u head %u tail %u cmdlist %p\r\n",
1195 	         idx, sqhead, sq->tail, sq->qbase));
1196 
1197 	while (sqhead != atomic_load_acq_short(&sq->tail)) {
1198 		struct nvme_command *cmd;
1199 		struct pci_nvme_ioreq *req = NULL;
1200 		uint64_t lba;
1201 		uint64_t nblocks, bytes, size, cpsz;
1202 
1203 		/* TODO: support scatter gather list handling */
1204 
1205 		cmd = &sq->qbase[sqhead];
1206 		sqhead = (sqhead + 1) % sq->size;
1207 
1208 		lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
1209 
1210 		if (cmd->opc == NVME_OPC_FLUSH) {
1211 			pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1212 			pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1213 			                        status, 1);
1214 
1215 			continue;
1216 		} else if (cmd->opc == 0x08) {
1217 			/* TODO: write zeroes */
1218 			WPRINTF(("%s write zeroes lba 0x%lx blocks %u\r\n",
1219 			        __func__, lba, cmd->cdw12 & 0xFFFF));
1220 			pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
1221 			pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1222 			                        status, 1);
1223 
1224 			continue;
1225 		}
1226 
1227 		nblocks = (cmd->cdw12 & 0xFFFF) + 1;
1228 
1229 		bytes = nblocks * sc->nvstore.sectsz;
1230 
1231 		if (sc->nvstore.type == NVME_STOR_BLOCKIF) {
1232 			req = pci_nvme_get_ioreq(sc);
1233 			req->nvme_sq = sq;
1234 			req->sqid = idx;
1235 		}
1236 
1237 		/*
1238 		 * If data starts mid-page and flows into the next page, then
1239 		 * increase page count
1240 		 */
1241 
1242 		DPRINTF(("[h%u:t%u:n%u] %s starting LBA 0x%lx blocks %lu "
1243 		         "(%lu-bytes)\r\n",
1244 		         sqhead==0 ? sq->size-1 : sqhead-1, sq->tail, sq->size,
1245 		         cmd->opc == NVME_OPC_WRITE ?
1246 			     "WRITE" : "READ",
1247 		         lba, nblocks, bytes));
1248 
1249 		cmd->prp1 &= ~(0x03UL);
1250 		cmd->prp2 &= ~(0x03UL);
1251 
1252 		DPRINTF((" prp1 0x%lx prp2 0x%lx\r\n", cmd->prp1, cmd->prp2));
1253 
1254 		size = bytes;
1255 		lba *= sc->nvstore.sectsz;
1256 
1257 		cpsz = PAGE_SIZE - (cmd->prp1 % PAGE_SIZE);
1258 
1259 		if (cpsz > bytes)
1260 			cpsz = bytes;
1261 
1262 		if (req != NULL) {
1263 			req->io_req.br_offset = ((uint64_t)cmd->cdw11 << 32) |
1264 			                        cmd->cdw10;
1265 			req->opc = cmd->opc;
1266 			req->cid = cmd->cid;
1267 			req->nsid = cmd->nsid;
1268 		}
1269 
1270 		err = pci_nvme_append_iov_req(sc, req, cmd->prp1, cpsz,
1271 		    cmd->opc == NVME_OPC_WRITE, lba);
1272 		lba += cpsz;
1273 		size -= cpsz;
1274 
1275 		if (size == 0)
1276 			goto iodone;
1277 
1278 		if (size <= PAGE_SIZE) {
1279 			/* prp2 is second (and final) page in transfer */
1280 
1281 			err = pci_nvme_append_iov_req(sc, req, cmd->prp2,
1282 			    size,
1283 			    cmd->opc == NVME_OPC_WRITE,
1284 			    lba);
1285 		} else {
1286 			uint64_t *prp_list;
1287 			int i;
1288 
1289 			/* prp2 is pointer to a physical region page list */
1290 			prp_list = paddr_guest2host(sc->nsc_pi->pi_vmctx,
1291 			                            cmd->prp2, PAGE_SIZE);
1292 
1293 			i = 0;
1294 			while (size != 0) {
1295 				cpsz = MIN(size, PAGE_SIZE);
1296 
1297 				/*
1298 				 * Move to linked physical region page list
1299 				 * in last item.
1300 				 */
1301 				if (i == (NVME_PRP2_ITEMS-1) &&
1302 				    size > PAGE_SIZE) {
1303 					assert((prp_list[i] & (PAGE_SIZE-1)) == 0);
1304 					prp_list = paddr_guest2host(
1305 					              sc->nsc_pi->pi_vmctx,
1306 					              prp_list[i], PAGE_SIZE);
1307 					i = 0;
1308 				}
1309 				if (prp_list[i] == 0) {
1310 					WPRINTF(("PRP2[%d] = 0 !!!\r\n", i));
1311 					err = 1;
1312 					break;
1313 				}
1314 
1315 				err = pci_nvme_append_iov_req(sc, req,
1316 				    prp_list[i], cpsz,
1317 				    cmd->opc == NVME_OPC_WRITE, lba);
1318 				if (err)
1319 					break;
1320 
1321 				lba += cpsz;
1322 				size -= cpsz;
1323 				i++;
1324 			}
1325 		}
1326 
1327 iodone:
1328 		if (sc->nvstore.type == NVME_STOR_RAM) {
1329 			uint16_t code, status;
1330 
1331 			code = err ? NVME_SC_LBA_OUT_OF_RANGE :
1332 			    NVME_SC_SUCCESS;
1333 			pci_nvme_status_genc(&status, code);
1334 
1335 			pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1336 			                        status, 1);
1337 
1338 			continue;
1339 		}
1340 
1341 
1342 		if (err)
1343 			goto do_error;
1344 
1345 		req->io_req.br_callback = pci_nvme_io_done;
1346 
1347 		err = 0;
1348 		switch (cmd->opc) {
1349 		case NVME_OPC_READ:
1350 			err = blockif_read(sc->nvstore.ctx, &req->io_req);
1351 			break;
1352 		case NVME_OPC_WRITE:
1353 			err = blockif_write(sc->nvstore.ctx, &req->io_req);
1354 			break;
1355 		default:
1356 			WPRINTF(("%s unhandled io command 0x%x\r\n",
1357 				 __func__, cmd->opc));
1358 			err = 1;
1359 		}
1360 
1361 do_error:
1362 		if (err) {
1363 			uint16_t status;
1364 
1365 			pci_nvme_status_genc(&status,
1366 			    NVME_SC_DATA_TRANSFER_ERROR);
1367 
1368 			pci_nvme_set_completion(sc, sq, idx, cmd->cid, 0,
1369 			                        status, 1);
1370 			pci_nvme_release_ioreq(sc, req);
1371 		}
1372 	}
1373 
1374 	atomic_store_short(&sq->head, sqhead);
1375 	atomic_store_int(&sq->busy, 0);
1376 }
1377 
1378 static void
1379 pci_nvme_handle_doorbell(struct vmctx *ctx, struct pci_nvme_softc* sc,
1380 	uint64_t idx, int is_sq, uint64_t value)
1381 {
1382 	DPRINTF(("nvme doorbell %lu, %s, val 0x%lx\r\n",
1383 	        idx, is_sq ? "SQ" : "CQ", value & 0xFFFF));
1384 
1385 	if (is_sq) {
1386 		atomic_store_short(&sc->submit_queues[idx].tail,
1387 		                   (uint16_t)value);
1388 
1389 		if (idx == 0) {
1390 			pci_nvme_handle_admin_cmd(sc, value);
1391 		} else {
1392 			/* submission queue; handle new entries in SQ */
1393 			if (idx > sc->num_squeues) {
1394 				WPRINTF(("%s SQ index %lu overflow from "
1395 				         "guest (max %u)\r\n",
1396 				         __func__, idx, sc->num_squeues));
1397 				return;
1398 			}
1399 			pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
1400 		}
1401 	} else {
1402 		if (idx > sc->num_cqueues) {
1403 			WPRINTF(("%s queue index %lu overflow from "
1404 			         "guest (max %u)\r\n",
1405 			         __func__, idx, sc->num_cqueues));
1406 			return;
1407 		}
1408 
1409 		sc->compl_queues[idx].head = (uint16_t)value;
1410 	}
1411 }
1412 
1413 static void
1414 pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
1415 {
1416 	const char *s = iswrite ? "WRITE" : "READ";
1417 
1418 	switch (offset) {
1419 	case NVME_CR_CAP_LOW:
1420 		DPRINTF(("%s %s NVME_CR_CAP_LOW\r\n", func, s));
1421 		break;
1422 	case NVME_CR_CAP_HI:
1423 		DPRINTF(("%s %s NVME_CR_CAP_HI\r\n", func, s));
1424 		break;
1425 	case NVME_CR_VS:
1426 		DPRINTF(("%s %s NVME_CR_VS\r\n", func, s));
1427 		break;
1428 	case NVME_CR_INTMS:
1429 		DPRINTF(("%s %s NVME_CR_INTMS\r\n", func, s));
1430 		break;
1431 	case NVME_CR_INTMC:
1432 		DPRINTF(("%s %s NVME_CR_INTMC\r\n", func, s));
1433 		break;
1434 	case NVME_CR_CC:
1435 		DPRINTF(("%s %s NVME_CR_CC\r\n", func, s));
1436 		break;
1437 	case NVME_CR_CSTS:
1438 		DPRINTF(("%s %s NVME_CR_CSTS\r\n", func, s));
1439 		break;
1440 	case NVME_CR_NSSR:
1441 		DPRINTF(("%s %s NVME_CR_NSSR\r\n", func, s));
1442 		break;
1443 	case NVME_CR_AQA:
1444 		DPRINTF(("%s %s NVME_CR_AQA\r\n", func, s));
1445 		break;
1446 	case NVME_CR_ASQ_LOW:
1447 		DPRINTF(("%s %s NVME_CR_ASQ_LOW\r\n", func, s));
1448 		break;
1449 	case NVME_CR_ASQ_HI:
1450 		DPRINTF(("%s %s NVME_CR_ASQ_HI\r\n", func, s));
1451 		break;
1452 	case NVME_CR_ACQ_LOW:
1453 		DPRINTF(("%s %s NVME_CR_ACQ_LOW\r\n", func, s));
1454 		break;
1455 	case NVME_CR_ACQ_HI:
1456 		DPRINTF(("%s %s NVME_CR_ACQ_HI\r\n", func, s));
1457 		break;
1458 	default:
1459 		DPRINTF(("unknown nvme bar-0 offset 0x%lx\r\n", offset));
1460 	}
1461 
1462 }
1463 
1464 static void
1465 pci_nvme_write_bar_0(struct vmctx *ctx, struct pci_nvme_softc* sc,
1466 	uint64_t offset, int size, uint64_t value)
1467 {
1468 	uint32_t ccreg;
1469 
1470 	if (offset >= NVME_DOORBELL_OFFSET) {
1471 		uint64_t belloffset = offset - NVME_DOORBELL_OFFSET;
1472 		uint64_t idx = belloffset / 8; /* door bell size = 2*int */
1473 		int is_sq = (belloffset % 8) < 4;
1474 
1475 		if (belloffset > ((sc->max_queues+1) * 8 - 4)) {
1476 			WPRINTF(("guest attempted an overflow write offset "
1477 			         "0x%lx, val 0x%lx in %s",
1478 			         offset, value, __func__));
1479 			return;
1480 		}
1481 
1482 		pci_nvme_handle_doorbell(ctx, sc, idx, is_sq, value);
1483 		return;
1484 	}
1485 
1486 	DPRINTF(("nvme-write offset 0x%lx, size %d, value 0x%lx\r\n",
1487 	        offset, size, value));
1488 
1489 	if (size != 4) {
1490 		WPRINTF(("guest wrote invalid size %d (offset 0x%lx, "
1491 		         "val 0x%lx) to bar0 in %s",
1492 		         size, offset, value, __func__));
1493 		/* TODO: shutdown device */
1494 		return;
1495 	}
1496 
1497 	pci_nvme_bar0_reg_dumps(__func__, offset, 1);
1498 
1499 	pthread_mutex_lock(&sc->mtx);
1500 
1501 	switch (offset) {
1502 	case NVME_CR_CAP_LOW:
1503 	case NVME_CR_CAP_HI:
1504 		/* readonly */
1505 		break;
1506 	case NVME_CR_VS:
1507 		/* readonly */
1508 		break;
1509 	case NVME_CR_INTMS:
1510 		/* MSI-X, so ignore */
1511 		break;
1512 	case NVME_CR_INTMC:
1513 		/* MSI-X, so ignore */
1514 		break;
1515 	case NVME_CR_CC:
1516 		ccreg = (uint32_t)value;
1517 
1518 		DPRINTF(("%s NVME_CR_CC en %x css %x shn %x iosqes %u "
1519 		         "iocqes %u\r\n",
1520 		        __func__,
1521 			 NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg),
1522 			 NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg),
1523 			 NVME_CC_GET_IOCQES(ccreg)));
1524 
1525 		if (NVME_CC_GET_SHN(ccreg)) {
1526 			/* perform shutdown - flush out data to backend */
1527 			sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK <<
1528 			    NVME_CSTS_REG_SHST_SHIFT);
1529 			sc->regs.csts |= NVME_SHST_COMPLETE <<
1530 			    NVME_CSTS_REG_SHST_SHIFT;
1531 		}
1532 		if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) {
1533 			if (NVME_CC_GET_EN(ccreg) == 0)
1534 				/* transition 1-> causes controller reset */
1535 				pci_nvme_reset_locked(sc);
1536 			else
1537 				pci_nvme_init_controller(ctx, sc);
1538 		}
1539 
1540 		/* Insert the iocqes, iosqes and en bits from the write */
1541 		sc->regs.cc &= ~NVME_CC_WRITE_MASK;
1542 		sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK;
1543 		if (NVME_CC_GET_EN(ccreg) == 0) {
1544 			/* Insert the ams, mps and css bit fields */
1545 			sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK;
1546 			sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK;
1547 			sc->regs.csts &= ~NVME_CSTS_RDY;
1548 		} else if (sc->pending_ios == 0) {
1549 			sc->regs.csts |= NVME_CSTS_RDY;
1550 		}
1551 		break;
1552 	case NVME_CR_CSTS:
1553 		break;
1554 	case NVME_CR_NSSR:
1555 		/* ignore writes; don't support subsystem reset */
1556 		break;
1557 	case NVME_CR_AQA:
1558 		sc->regs.aqa = (uint32_t)value;
1559 		break;
1560 	case NVME_CR_ASQ_LOW:
1561 		sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) |
1562 		               (0xFFFFF000 & value);
1563 		break;
1564 	case NVME_CR_ASQ_HI:
1565 		sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) |
1566 		               (value << 32);
1567 		break;
1568 	case NVME_CR_ACQ_LOW:
1569 		sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) |
1570 		               (0xFFFFF000 & value);
1571 		break;
1572 	case NVME_CR_ACQ_HI:
1573 		sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) |
1574 		               (value << 32);
1575 		break;
1576 	default:
1577 		DPRINTF(("%s unknown offset 0x%lx, value 0x%lx size %d\r\n",
1578 		         __func__, offset, value, size));
1579 	}
1580 	pthread_mutex_unlock(&sc->mtx);
1581 }
1582 
1583 static void
1584 pci_nvme_write(struct vmctx *ctx, int vcpu, struct pci_devinst *pi,
1585                 int baridx, uint64_t offset, int size, uint64_t value)
1586 {
1587 	struct pci_nvme_softc* sc = pi->pi_arg;
1588 
1589 	if (baridx == pci_msix_table_bar(pi) ||
1590 	    baridx == pci_msix_pba_bar(pi)) {
1591 		DPRINTF(("nvme-write baridx %d, msix: off 0x%lx, size %d, "
1592 		         " value 0x%lx\r\n", baridx, offset, size, value));
1593 
1594 		pci_emul_msix_twrite(pi, offset, size, value);
1595 		return;
1596 	}
1597 
1598 	switch (baridx) {
1599 	case 0:
1600 		pci_nvme_write_bar_0(ctx, sc, offset, size, value);
1601 		break;
1602 
1603 	default:
1604 		DPRINTF(("%s unknown baridx %d, val 0x%lx\r\n",
1605 		         __func__, baridx, value));
1606 	}
1607 }
1608 
1609 static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc,
1610 	uint64_t offset, int size)
1611 {
1612 	uint64_t value;
1613 
1614 	pci_nvme_bar0_reg_dumps(__func__, offset, 0);
1615 
1616 	if (offset < NVME_DOORBELL_OFFSET) {
1617 		void *p = &(sc->regs);
1618 		pthread_mutex_lock(&sc->mtx);
1619 		memcpy(&value, (void *)((uintptr_t)p + offset), size);
1620 		pthread_mutex_unlock(&sc->mtx);
1621 	} else {
1622 		value = 0;
1623                 WPRINTF(("pci_nvme: read invalid offset %ld\r\n", offset));
1624 	}
1625 
1626 	switch (size) {
1627 	case 1:
1628 		value &= 0xFF;
1629 		break;
1630 	case 2:
1631 		value &= 0xFFFF;
1632 		break;
1633 	case 4:
1634 		value &= 0xFFFFFFFF;
1635 		break;
1636 	}
1637 
1638 	DPRINTF(("   nvme-read offset 0x%lx, size %d -> value 0x%x\r\n",
1639 	         offset, size, (uint32_t)value));
1640 
1641 	return (value);
1642 }
1643 
1644 
1645 
1646 static uint64_t
1647 pci_nvme_read(struct vmctx *ctx, int vcpu, struct pci_devinst *pi, int baridx,
1648     uint64_t offset, int size)
1649 {
1650 	struct pci_nvme_softc* sc = pi->pi_arg;
1651 
1652 	if (baridx == pci_msix_table_bar(pi) ||
1653 	    baridx == pci_msix_pba_bar(pi)) {
1654 		DPRINTF(("nvme-read bar: %d, msix: regoff 0x%lx, size %d\r\n",
1655 		        baridx, offset, size));
1656 
1657 		return pci_emul_msix_tread(pi, offset, size);
1658 	}
1659 
1660 	switch (baridx) {
1661 	case 0:
1662        		return pci_nvme_read_bar_0(sc, offset, size);
1663 
1664 	default:
1665 		DPRINTF(("unknown bar %d, 0x%lx\r\n", baridx, offset));
1666 	}
1667 
1668 	return (0);
1669 }
1670 
1671 
1672 static int
1673 pci_nvme_parse_opts(struct pci_nvme_softc *sc, char *opts)
1674 {
1675 	char bident[sizeof("XX:X:X")];
1676 	char	*uopt, *xopts, *config;
1677 	uint32_t sectsz;
1678 	int optidx;
1679 
1680 	sc->max_queues = NVME_QUEUES;
1681 	sc->max_qentries = NVME_MAX_QENTRIES;
1682 	sc->ioslots = NVME_IOSLOTS;
1683 	sc->num_squeues = sc->max_queues;
1684 	sc->num_cqueues = sc->max_queues;
1685 	sectsz = 0;
1686 
1687 	uopt = strdup(opts);
1688 	optidx = 0;
1689 	snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
1690 	         "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
1691 	for (xopts = strtok(uopt, ",");
1692 	     xopts != NULL;
1693 	     xopts = strtok(NULL, ",")) {
1694 
1695 		if ((config = strchr(xopts, '=')) != NULL)
1696 			*config++ = '\0';
1697 
1698 		if (!strcmp("maxq", xopts)) {
1699 			sc->max_queues = atoi(config);
1700 		} else if (!strcmp("qsz", xopts)) {
1701 			sc->max_qentries = atoi(config);
1702 		} else if (!strcmp("ioslots", xopts)) {
1703 			sc->ioslots = atoi(config);
1704 		} else if (!strcmp("sectsz", xopts)) {
1705 			sectsz = atoi(config);
1706 		} else if (!strcmp("ser", xopts)) {
1707 			/*
1708 			 * This field indicates the Product Serial Number in
1709 			 * 7-bit ASCII, unused bytes should be space characters.
1710 			 * Ref: NVMe v1.3c.
1711 			 */
1712 			cpywithpad((char *)sc->ctrldata.sn,
1713 			           sizeof(sc->ctrldata.sn), config, ' ');
1714 		} else if (!strcmp("ram", xopts)) {
1715 			uint64_t sz = strtoull(&xopts[4], NULL, 10);
1716 
1717 			sc->nvstore.type = NVME_STOR_RAM;
1718 			sc->nvstore.size = sz * 1024 * 1024;
1719 			sc->nvstore.ctx = calloc(1, sc->nvstore.size);
1720 			sc->nvstore.sectsz = 4096;
1721 			sc->nvstore.sectsz_bits = 12;
1722 			if (sc->nvstore.ctx == NULL) {
1723 				perror("Unable to allocate RAM");
1724 				free(uopt);
1725 				return (-1);
1726 			}
1727 		} else if (optidx == 0) {
1728 			snprintf(bident, sizeof(bident), "%d:%d",
1729 			         sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
1730 			sc->nvstore.ctx = blockif_open(xopts, bident);
1731 			if (sc->nvstore.ctx == NULL) {
1732 				perror("Could not open backing file");
1733 				free(uopt);
1734 				return (-1);
1735 			}
1736 			sc->nvstore.type = NVME_STOR_BLOCKIF;
1737 			sc->nvstore.size = blockif_size(sc->nvstore.ctx);
1738 		} else {
1739 			fprintf(stderr, "Invalid option %s\n", xopts);
1740 			free(uopt);
1741 			return (-1);
1742 		}
1743 
1744 		optidx++;
1745 	}
1746 	free(uopt);
1747 
1748 	if (sc->nvstore.ctx == NULL || sc->nvstore.size == 0) {
1749 		fprintf(stderr, "backing store not specified\n");
1750 		return (-1);
1751 	}
1752 	if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
1753 		sc->nvstore.sectsz = sectsz;
1754 	else if (sc->nvstore.type != NVME_STOR_RAM)
1755 		sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);
1756 	for (sc->nvstore.sectsz_bits = 9;
1757 	     (1 << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
1758 	     sc->nvstore.sectsz_bits++);
1759 
1760 	if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES)
1761 		sc->max_queues = NVME_QUEUES;
1762 
1763 	if (sc->max_qentries <= 0) {
1764 		fprintf(stderr, "Invalid qsz option\n");
1765 		return (-1);
1766 	}
1767 	if (sc->ioslots <= 0) {
1768 		fprintf(stderr, "Invalid ioslots option\n");
1769 		return (-1);
1770 	}
1771 
1772 	return (0);
1773 }
1774 
1775 static int
1776 pci_nvme_init(struct vmctx *ctx, struct pci_devinst *pi, char *opts)
1777 {
1778 	struct pci_nvme_softc *sc;
1779 	uint32_t pci_membar_sz;
1780 	int	error;
1781 
1782 	error = 0;
1783 
1784 	sc = calloc(1, sizeof(struct pci_nvme_softc));
1785 	pi->pi_arg = sc;
1786 	sc->nsc_pi = pi;
1787 
1788 	error = pci_nvme_parse_opts(sc, opts);
1789 	if (error < 0)
1790 		goto done;
1791 	else
1792 		error = 0;
1793 
1794 	sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq));
1795 	for (int i = 0; i < sc->ioslots; i++) {
1796 		if (i < (sc->ioslots-1))
1797 			sc->ioreqs[i].next = &sc->ioreqs[i+1];
1798 		pthread_mutex_init(&sc->ioreqs[i].mtx, NULL);
1799 		pthread_cond_init(&sc->ioreqs[i].cv, NULL);
1800 	}
1801 	sc->ioreqs_free = sc->ioreqs;
1802 	sc->intr_coales_aggr_thresh = 1;
1803 
1804 	pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A);
1805 	pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D);
1806 	pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
1807 	pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM);
1808 	pci_set_cfgdata8(pi, PCIR_PROGIF,
1809 	                 PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0);
1810 
1811 	/* allocate size of nvme registers + doorbell space for all queues */
1812 	pci_membar_sz = sizeof(struct nvme_registers) +
1813 	                2*sizeof(uint32_t)*(sc->max_queues);
1814 
1815 	DPRINTF(("nvme membar size: %u\r\n", pci_membar_sz));
1816 
1817 	error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz);
1818 	if (error) {
1819 		WPRINTF(("%s pci alloc mem bar failed\r\n", __func__));
1820 		goto done;
1821 	}
1822 
1823 	error = pci_emul_add_msixcap(pi, sc->max_queues, NVME_MSIX_BAR);
1824 	if (error) {
1825 		WPRINTF(("%s pci add msixcap failed\r\n", __func__));
1826 		goto done;
1827 	}
1828 
1829 	pthread_mutex_init(&sc->mtx, NULL);
1830 	sem_init(&sc->iosemlock, 0, sc->ioslots);
1831 
1832 	pci_nvme_reset(sc);
1833 	pci_nvme_init_ctrldata(sc);
1834 	pci_nvme_init_nsdata(sc);
1835 
1836 	pci_lintr_request(pi);
1837 
1838 done:
1839 	return (error);
1840 }
1841 
1842 
1843 struct pci_devemu pci_de_nvme = {
1844 	.pe_emu =	"nvme",
1845 	.pe_init =	pci_nvme_init,
1846 	.pe_barwrite =	pci_nvme_write,
1847 	.pe_barread =	pci_nvme_read
1848 };
1849 PCI_EMUL_SET(pci_de_nvme);
1850