1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2024 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 */
7
8 #include <sys/sysctl.h>
9 #include <errno.h>
10 #include <fcntl.h>
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <string.h>
14 #include <unistd.h>
15 #include <uuid.h>
16
17 #include "libnvmf.h"
18 #include "internal.h"
19
20 static void
nvmf_init_sqe(void * sqe,uint8_t opcode)21 nvmf_init_sqe(void *sqe, uint8_t opcode)
22 {
23 struct nvme_command *cmd = sqe;
24
25 memset(cmd, 0, sizeof(*cmd));
26 cmd->opc = opcode;
27 }
28
29 static void
nvmf_init_fabrics_sqe(void * sqe,uint8_t fctype)30 nvmf_init_fabrics_sqe(void *sqe, uint8_t fctype)
31 {
32 struct nvmf_capsule_cmd *cmd = sqe;
33
34 nvmf_init_sqe(sqe, NVME_OPC_FABRICS_COMMANDS);
35 cmd->fctype = fctype;
36 }
37
38 struct nvmf_qpair *
nvmf_connect(struct nvmf_association * na,const struct nvmf_qpair_params * params,uint16_t qid,u_int queue_size,const uint8_t hostid[16],uint16_t cntlid,const char * subnqn,const char * hostnqn,uint32_t kato)39 nvmf_connect(struct nvmf_association *na,
40 const struct nvmf_qpair_params *params, uint16_t qid, u_int queue_size,
41 const uint8_t hostid[16], uint16_t cntlid, const char *subnqn,
42 const char *hostnqn, uint32_t kato)
43 {
44 struct nvmf_fabric_connect_cmd cmd;
45 struct nvmf_fabric_connect_data data;
46 const struct nvmf_fabric_connect_rsp *rsp;
47 struct nvmf_qpair *qp;
48 struct nvmf_capsule *cc, *rc;
49 int error;
50 uint16_t sqhd, status;
51
52 qp = NULL;
53 cc = NULL;
54 rc = NULL;
55 na_clear_error(na);
56 if (na->na_controller) {
57 na_error(na, "Cannot connect on a controller");
58 goto error;
59 }
60
61 if (params->admin != (qid == 0)) {
62 na_error(na, "Admin queue must use Queue ID 0");
63 goto error;
64 }
65
66 if (qid == 0) {
67 if (queue_size < NVME_MIN_ADMIN_ENTRIES ||
68 queue_size > NVME_MAX_ADMIN_ENTRIES) {
69 na_error(na, "Invalid queue size %u", queue_size);
70 goto error;
71 }
72 } else {
73 if (queue_size < NVME_MIN_IO_ENTRIES ||
74 queue_size > NVME_MAX_IO_ENTRIES) {
75 na_error(na, "Invalid queue size %u", queue_size);
76 goto error;
77 }
78
79 /* KATO is only for Admin queues. */
80 if (kato != 0) {
81 na_error(na, "Cannot set KATO on I/O queues");
82 goto error;
83 }
84 }
85
86 qp = nvmf_allocate_qpair(na, params);
87 if (qp == NULL)
88 goto error;
89
90 nvmf_init_fabrics_sqe(&cmd, NVMF_FABRIC_COMMAND_CONNECT);
91 cmd.recfmt = 0;
92 cmd.qid = htole16(qid);
93
94 /* N.B. sqsize is 0's based. */
95 cmd.sqsize = htole16(queue_size - 1);
96 if (!na->na_params.sq_flow_control)
97 cmd.cattr |= NVMF_CONNECT_ATTR_DISABLE_SQ_FC;
98 cmd.kato = htole32(kato);
99
100 cc = nvmf_allocate_command(qp, &cmd);
101 if (cc == NULL) {
102 na_error(na, "Failed to allocate command capsule: %s",
103 strerror(errno));
104 goto error;
105 }
106
107 memset(&data, 0, sizeof(data));
108 memcpy(data.hostid, hostid, sizeof(data.hostid));
109 data.cntlid = htole16(cntlid);
110 strlcpy(data.subnqn, subnqn, sizeof(data.subnqn));
111 strlcpy(data.hostnqn, hostnqn, sizeof(data.hostnqn));
112
113 error = nvmf_capsule_append_data(cc, &data, sizeof(data), true);
114 if (error != 0) {
115 na_error(na, "Failed to append data to CONNECT capsule: %s",
116 strerror(error));
117 goto error;
118 }
119
120 error = nvmf_transmit_capsule(cc);
121 if (error != 0) {
122 na_error(na, "Failed to transmit CONNECT capsule: %s",
123 strerror(errno));
124 goto error;
125 }
126
127 error = nvmf_receive_capsule(qp, &rc);
128 if (error != 0) {
129 na_error(na, "Failed to receive CONNECT response: %s",
130 strerror(error));
131 goto error;
132 }
133
134 rsp = (const struct nvmf_fabric_connect_rsp *)&rc->nc_cqe;
135 status = le16toh(rc->nc_cqe.status);
136 if (status != 0) {
137 if (NVME_STATUS_GET_SC(status) == NVMF_FABRIC_SC_INVALID_PARAM)
138 na_error(na,
139 "CONNECT invalid parameter IATTR: %#x IPO: %#x",
140 rsp->status_code_specific.invalid.iattr,
141 rsp->status_code_specific.invalid.ipo);
142 else
143 na_error(na, "CONNECT failed, status %#x", status);
144 goto error;
145 }
146
147 if (rc->nc_cqe.cid != cmd.cid) {
148 na_error(na, "Mismatched CID in CONNECT response");
149 goto error;
150 }
151
152 if (!rc->nc_sqhd_valid) {
153 na_error(na, "CONNECT response without valid SQHD");
154 goto error;
155 }
156
157 sqhd = le16toh(rsp->sqhd);
158 if (sqhd == 0xffff) {
159 if (na->na_params.sq_flow_control) {
160 na_error(na, "Controller disabled SQ flow control");
161 goto error;
162 }
163 qp->nq_flow_control = false;
164 } else {
165 qp->nq_flow_control = true;
166 qp->nq_sqhd = sqhd;
167 qp->nq_sqtail = sqhd;
168 }
169
170 if (rsp->status_code_specific.success.authreq) {
171 na_error(na, "CONNECT response requests authentication\n");
172 goto error;
173 }
174
175 qp->nq_qsize = queue_size;
176 qp->nq_cntlid = le16toh(rsp->status_code_specific.success.cntlid);
177 qp->nq_kato = kato;
178 /* XXX: Save qid in qp? */
179 return (qp);
180
181 error:
182 if (rc != NULL)
183 nvmf_free_capsule(rc);
184 if (cc != NULL)
185 nvmf_free_capsule(cc);
186 if (qp != NULL)
187 nvmf_free_qpair(qp);
188 return (NULL);
189 }
190
191 uint16_t
nvmf_cntlid(struct nvmf_qpair * qp)192 nvmf_cntlid(struct nvmf_qpair *qp)
193 {
194 return (qp->nq_cntlid);
195 }
196
197 int
nvmf_host_transmit_command(struct nvmf_capsule * nc)198 nvmf_host_transmit_command(struct nvmf_capsule *nc)
199 {
200 struct nvmf_qpair *qp = nc->nc_qpair;
201 uint16_t new_sqtail;
202 int error;
203
204 /* Fail if the queue is full. */
205 new_sqtail = (qp->nq_sqtail + 1) % qp->nq_qsize;
206 if (new_sqtail == qp->nq_sqhd)
207 return (EBUSY);
208
209 nc->nc_sqe.cid = htole16(qp->nq_cid);
210
211 /* 4.2 Skip CID of 0xFFFF. */
212 qp->nq_cid++;
213 if (qp->nq_cid == 0xFFFF)
214 qp->nq_cid = 0;
215
216 error = nvmf_transmit_capsule(nc);
217 if (error != 0)
218 return (error);
219
220 qp->nq_sqtail = new_sqtail;
221 return (0);
222 }
223
224 /* Receive a single capsule and update SQ FC accounting. */
225 static int
nvmf_host_receive_capsule(struct nvmf_qpair * qp,struct nvmf_capsule ** ncp)226 nvmf_host_receive_capsule(struct nvmf_qpair *qp, struct nvmf_capsule **ncp)
227 {
228 struct nvmf_capsule *nc;
229 int error;
230
231 /* If the SQ is empty, there is no response to wait for. */
232 if (qp->nq_sqhd == qp->nq_sqtail)
233 return (EWOULDBLOCK);
234
235 error = nvmf_receive_capsule(qp, &nc);
236 if (error != 0)
237 return (error);
238
239 if (qp->nq_flow_control) {
240 if (nc->nc_sqhd_valid)
241 qp->nq_sqhd = le16toh(nc->nc_cqe.sqhd);
242 } else {
243 /*
244 * If SQ FC is disabled, just advance the head for
245 * each response capsule received so that we track the
246 * number of outstanding commands.
247 */
248 qp->nq_sqhd = (qp->nq_sqhd + 1) % qp->nq_qsize;
249 }
250 *ncp = nc;
251 return (0);
252 }
253
254 int
nvmf_host_receive_response(struct nvmf_qpair * qp,struct nvmf_capsule ** ncp)255 nvmf_host_receive_response(struct nvmf_qpair *qp, struct nvmf_capsule **ncp)
256 {
257 struct nvmf_capsule *nc;
258
259 /* Return the oldest previously received response. */
260 if (!TAILQ_EMPTY(&qp->nq_rx_capsules)) {
261 nc = TAILQ_FIRST(&qp->nq_rx_capsules);
262 TAILQ_REMOVE(&qp->nq_rx_capsules, nc, nc_link);
263 *ncp = nc;
264 return (0);
265 }
266
267 return (nvmf_host_receive_capsule(qp, ncp));
268 }
269
270 int
nvmf_host_wait_for_response(struct nvmf_capsule * cc,struct nvmf_capsule ** rcp)271 nvmf_host_wait_for_response(struct nvmf_capsule *cc,
272 struct nvmf_capsule **rcp)
273 {
274 struct nvmf_qpair *qp = cc->nc_qpair;
275 struct nvmf_capsule *rc;
276 int error;
277
278 /* Check if a response was already received. */
279 TAILQ_FOREACH(rc, &qp->nq_rx_capsules, nc_link) {
280 if (rc->nc_cqe.cid == cc->nc_sqe.cid) {
281 TAILQ_REMOVE(&qp->nq_rx_capsules, rc, nc_link);
282 *rcp = rc;
283 return (0);
284 }
285 }
286
287 /* Wait for a response. */
288 for (;;) {
289 error = nvmf_host_receive_capsule(qp, &rc);
290 if (error != 0)
291 return (error);
292
293 if (rc->nc_cqe.cid != cc->nc_sqe.cid) {
294 TAILQ_INSERT_TAIL(&qp->nq_rx_capsules, rc, nc_link);
295 continue;
296 }
297
298 *rcp = rc;
299 return (0);
300 }
301 }
302
303 struct nvmf_capsule *
nvmf_keepalive(struct nvmf_qpair * qp)304 nvmf_keepalive(struct nvmf_qpair *qp)
305 {
306 struct nvme_command cmd;
307
308 if (!qp->nq_admin) {
309 errno = EINVAL;
310 return (NULL);
311 }
312
313 nvmf_init_sqe(&cmd, NVME_OPC_KEEP_ALIVE);
314
315 return (nvmf_allocate_command(qp, &cmd));
316 }
317
318 static struct nvmf_capsule *
nvmf_get_property(struct nvmf_qpair * qp,uint32_t offset,uint8_t size)319 nvmf_get_property(struct nvmf_qpair *qp, uint32_t offset, uint8_t size)
320 {
321 struct nvmf_fabric_prop_get_cmd cmd;
322
323 nvmf_init_fabrics_sqe(&cmd, NVMF_FABRIC_COMMAND_PROPERTY_GET);
324 switch (size) {
325 case 4:
326 cmd.attrib.size = NVMF_PROP_SIZE_4;
327 break;
328 case 8:
329 cmd.attrib.size = NVMF_PROP_SIZE_8;
330 break;
331 default:
332 errno = EINVAL;
333 return (NULL);
334 }
335 cmd.ofst = htole32(offset);
336
337 return (nvmf_allocate_command(qp, &cmd));
338 }
339
340 int
nvmf_read_property(struct nvmf_qpair * qp,uint32_t offset,uint8_t size,uint64_t * value)341 nvmf_read_property(struct nvmf_qpair *qp, uint32_t offset, uint8_t size,
342 uint64_t *value)
343 {
344 struct nvmf_capsule *cc, *rc;
345 const struct nvmf_fabric_prop_get_rsp *rsp;
346 uint16_t status;
347 int error;
348
349 if (!qp->nq_admin)
350 return (EINVAL);
351
352 cc = nvmf_get_property(qp, offset, size);
353 if (cc == NULL)
354 return (errno);
355
356 error = nvmf_host_transmit_command(cc);
357 if (error != 0) {
358 nvmf_free_capsule(cc);
359 return (error);
360 }
361
362 error = nvmf_host_wait_for_response(cc, &rc);
363 nvmf_free_capsule(cc);
364 if (error != 0)
365 return (error);
366
367 rsp = (const struct nvmf_fabric_prop_get_rsp *)&rc->nc_cqe;
368 status = le16toh(rc->nc_cqe.status);
369 if (status != 0) {
370 printf("NVMF: PROPERTY_GET failed, status %#x\n", status);
371 nvmf_free_capsule(rc);
372 return (EIO);
373 }
374
375 if (size == 8)
376 *value = le64toh(rsp->value.u64);
377 else
378 *value = le32toh(rsp->value.u32.low);
379 nvmf_free_capsule(rc);
380 return (0);
381 }
382
383 static struct nvmf_capsule *
nvmf_set_property(struct nvmf_qpair * qp,uint32_t offset,uint8_t size,uint64_t value)384 nvmf_set_property(struct nvmf_qpair *qp, uint32_t offset, uint8_t size,
385 uint64_t value)
386 {
387 struct nvmf_fabric_prop_set_cmd cmd;
388
389 nvmf_init_fabrics_sqe(&cmd, NVMF_FABRIC_COMMAND_PROPERTY_SET);
390 switch (size) {
391 case 4:
392 cmd.attrib.size = NVMF_PROP_SIZE_4;
393 cmd.value.u32.low = htole32(value);
394 break;
395 case 8:
396 cmd.attrib.size = NVMF_PROP_SIZE_8;
397 cmd.value.u64 = htole64(value);
398 break;
399 default:
400 errno = EINVAL;
401 return (NULL);
402 }
403 cmd.ofst = htole32(offset);
404
405 return (nvmf_allocate_command(qp, &cmd));
406 }
407
408 int
nvmf_write_property(struct nvmf_qpair * qp,uint32_t offset,uint8_t size,uint64_t value)409 nvmf_write_property(struct nvmf_qpair *qp, uint32_t offset, uint8_t size,
410 uint64_t value)
411 {
412 struct nvmf_capsule *cc, *rc;
413 uint16_t status;
414 int error;
415
416 if (!qp->nq_admin)
417 return (EINVAL);
418
419 cc = nvmf_set_property(qp, offset, size, value);
420 if (cc == NULL)
421 return (errno);
422
423 error = nvmf_host_transmit_command(cc);
424 if (error != 0) {
425 nvmf_free_capsule(cc);
426 return (error);
427 }
428
429 error = nvmf_host_wait_for_response(cc, &rc);
430 nvmf_free_capsule(cc);
431 if (error != 0)
432 return (error);
433
434 status = le16toh(rc->nc_cqe.status);
435 if (status != 0) {
436 printf("NVMF: PROPERTY_SET failed, status %#x\n", status);
437 nvmf_free_capsule(rc);
438 return (EIO);
439 }
440
441 nvmf_free_capsule(rc);
442 return (0);
443 }
444
445 int
nvmf_hostid_from_hostuuid(uint8_t hostid[16])446 nvmf_hostid_from_hostuuid(uint8_t hostid[16])
447 {
448 char hostuuid_str[64];
449 uuid_t hostuuid;
450 size_t len;
451 uint32_t status;
452
453 len = sizeof(hostuuid_str);
454 if (sysctlbyname("kern.hostuuid", hostuuid_str, &len, NULL, 0) != 0)
455 return (errno);
456
457 uuid_from_string(hostuuid_str, &hostuuid, &status);
458 switch (status) {
459 case uuid_s_ok:
460 break;
461 case uuid_s_no_memory:
462 return (ENOMEM);
463 default:
464 return (EINVAL);
465 }
466
467 uuid_enc_le(hostid, &hostuuid);
468 return (0);
469 }
470
471 int
nvmf_nqn_from_hostuuid(char nqn[NVMF_NQN_MAX_LEN])472 nvmf_nqn_from_hostuuid(char nqn[NVMF_NQN_MAX_LEN])
473 {
474 char hostuuid_str[64];
475 size_t len;
476
477 len = sizeof(hostuuid_str);
478 if (sysctlbyname("kern.hostuuid", hostuuid_str, &len, NULL, 0) != 0)
479 return (errno);
480
481 strlcpy(nqn, NVMF_NQN_UUID_PRE, NVMF_NQN_MAX_LEN);
482 strlcat(nqn, hostuuid_str, NVMF_NQN_MAX_LEN);
483 return (0);
484 }
485
486 int
nvmf_host_identify_controller(struct nvmf_qpair * qp,struct nvme_controller_data * cdata)487 nvmf_host_identify_controller(struct nvmf_qpair *qp,
488 struct nvme_controller_data *cdata)
489 {
490 struct nvme_command cmd;
491 struct nvmf_capsule *cc, *rc;
492 int error;
493 uint16_t status;
494
495 if (!qp->nq_admin)
496 return (EINVAL);
497
498 nvmf_init_sqe(&cmd, NVME_OPC_IDENTIFY);
499
500 /* 5.15.1 Use CNS of 0x01 for controller data. */
501 cmd.cdw10 = htole32(1);
502
503 cc = nvmf_allocate_command(qp, &cmd);
504 if (cc == NULL)
505 return (errno);
506
507 error = nvmf_capsule_append_data(cc, cdata, sizeof(*cdata), false);
508 if (error != 0) {
509 nvmf_free_capsule(cc);
510 return (error);
511 }
512
513 error = nvmf_host_transmit_command(cc);
514 if (error != 0) {
515 nvmf_free_capsule(cc);
516 return (error);
517 }
518
519 error = nvmf_host_wait_for_response(cc, &rc);
520 nvmf_free_capsule(cc);
521 if (error != 0)
522 return (error);
523
524 status = le16toh(rc->nc_cqe.status);
525 if (status != 0) {
526 printf("NVMF: IDENTIFY failed, status %#x\n", status);
527 nvmf_free_capsule(rc);
528 return (EIO);
529 }
530
531 nvmf_free_capsule(rc);
532 return (0);
533 }
534
535 int
nvmf_host_identify_namespace(struct nvmf_qpair * qp,uint32_t nsid,struct nvme_namespace_data * nsdata)536 nvmf_host_identify_namespace(struct nvmf_qpair *qp, uint32_t nsid,
537 struct nvme_namespace_data *nsdata)
538 {
539 struct nvme_command cmd;
540 struct nvmf_capsule *cc, *rc;
541 int error;
542 uint16_t status;
543
544 if (!qp->nq_admin)
545 return (EINVAL);
546
547 nvmf_init_sqe(&cmd, NVME_OPC_IDENTIFY);
548
549 /* 5.15.1 Use CNS of 0x00 for namespace data. */
550 cmd.cdw10 = htole32(0);
551 cmd.nsid = htole32(nsid);
552
553 cc = nvmf_allocate_command(qp, &cmd);
554 if (cc == NULL)
555 return (errno);
556
557 error = nvmf_capsule_append_data(cc, nsdata, sizeof(*nsdata), false);
558 if (error != 0) {
559 nvmf_free_capsule(cc);
560 return (error);
561 }
562
563 error = nvmf_host_transmit_command(cc);
564 if (error != 0) {
565 nvmf_free_capsule(cc);
566 return (error);
567 }
568
569 error = nvmf_host_wait_for_response(cc, &rc);
570 nvmf_free_capsule(cc);
571 if (error != 0)
572 return (error);
573
574 status = le16toh(rc->nc_cqe.status);
575 if (status != 0) {
576 printf("NVMF: IDENTIFY failed, status %#x\n", status);
577 nvmf_free_capsule(rc);
578 return (EIO);
579 }
580
581 nvmf_free_capsule(rc);
582 return (0);
583 }
584
585 static int
nvmf_get_discovery_log_page(struct nvmf_qpair * qp,uint64_t offset,void * buf,size_t len)586 nvmf_get_discovery_log_page(struct nvmf_qpair *qp, uint64_t offset, void *buf,
587 size_t len)
588 {
589 struct nvme_command cmd;
590 struct nvmf_capsule *cc, *rc;
591 size_t numd;
592 int error;
593 uint16_t status;
594
595 if (len % 4 != 0 || len == 0 || offset % 4 != 0)
596 return (EINVAL);
597
598 numd = (len / 4) - 1;
599 nvmf_init_sqe(&cmd, NVME_OPC_GET_LOG_PAGE);
600 cmd.cdw10 = htole32(numd << 16 | NVME_LOG_DISCOVERY);
601 cmd.cdw11 = htole32(numd >> 16);
602 cmd.cdw12 = htole32(offset);
603 cmd.cdw13 = htole32(offset >> 32);
604
605 cc = nvmf_allocate_command(qp, &cmd);
606 if (cc == NULL)
607 return (errno);
608
609 error = nvmf_capsule_append_data(cc, buf, len, false);
610 if (error != 0) {
611 nvmf_free_capsule(cc);
612 return (error);
613 }
614
615 error = nvmf_host_transmit_command(cc);
616 if (error != 0) {
617 nvmf_free_capsule(cc);
618 return (error);
619 }
620
621 error = nvmf_host_wait_for_response(cc, &rc);
622 nvmf_free_capsule(cc);
623 if (error != 0)
624 return (error);
625
626 status = le16toh(rc->nc_cqe.status);
627 if (NVMEV(NVME_STATUS_SC, status) ==
628 NVMF_FABRIC_SC_LOG_RESTART_DISCOVERY) {
629 nvmf_free_capsule(rc);
630 return (EAGAIN);
631 }
632 if (status != 0) {
633 printf("NVMF: GET_LOG_PAGE failed, status %#x\n", status);
634 nvmf_free_capsule(rc);
635 return (EIO);
636 }
637
638 nvmf_free_capsule(rc);
639 return (0);
640 }
641
642 int
nvmf_host_fetch_discovery_log_page(struct nvmf_qpair * qp,struct nvme_discovery_log ** logp)643 nvmf_host_fetch_discovery_log_page(struct nvmf_qpair *qp,
644 struct nvme_discovery_log **logp)
645 {
646 struct nvme_discovery_log hdr, *log;
647 size_t payload_len;
648 int error;
649
650 if (!qp->nq_admin)
651 return (EINVAL);
652
653 log = NULL;
654 for (;;) {
655 error = nvmf_get_discovery_log_page(qp, 0, &hdr, sizeof(hdr));
656 if (error != 0) {
657 free(log);
658 return (error);
659 }
660 nvme_discovery_log_swapbytes(&hdr);
661
662 if (hdr.recfmt != 0) {
663 printf("NVMF: Unsupported discovery log format: %d\n",
664 hdr.recfmt);
665 free(log);
666 return (EINVAL);
667 }
668
669 if (hdr.numrec > 1024) {
670 printf("NVMF: Too many discovery log entries: %ju\n",
671 (uintmax_t)hdr.numrec);
672 free(log);
673 return (EFBIG);
674 }
675
676 payload_len = sizeof(log->entries[0]) * hdr.numrec;
677 log = reallocf(log, sizeof(*log) + payload_len);
678 if (log == NULL)
679 return (ENOMEM);
680 *log = hdr;
681 if (hdr.numrec == 0)
682 break;
683
684 error = nvmf_get_discovery_log_page(qp, sizeof(hdr),
685 log->entries, payload_len);
686 if (error == EAGAIN)
687 continue;
688 if (error != 0) {
689 free(log);
690 return (error);
691 }
692
693 /* Re-read the header and check the generation count. */
694 error = nvmf_get_discovery_log_page(qp, 0, &hdr, sizeof(hdr));
695 if (error != 0) {
696 free(log);
697 return (error);
698 }
699 nvme_discovery_log_swapbytes(&hdr);
700
701 if (log->genctr != hdr.genctr)
702 continue;
703
704 for (u_int i = 0; i < log->numrec; i++)
705 nvme_discovery_log_entry_swapbytes(&log->entries[i]);
706 break;
707 }
708 *logp = log;
709 return (0);
710 }
711
712 int
nvmf_init_dle_from_admin_qp(struct nvmf_qpair * qp,const struct nvme_controller_data * cdata,struct nvme_discovery_log_entry * dle)713 nvmf_init_dle_from_admin_qp(struct nvmf_qpair *qp,
714 const struct nvme_controller_data *cdata,
715 struct nvme_discovery_log_entry *dle)
716 {
717 int error;
718 uint16_t cntlid;
719
720 memset(dle, 0, sizeof(*dle));
721 error = nvmf_populate_dle(qp, dle);
722 if (error != 0)
723 return (error);
724 if ((cdata->fcatt & 1) == 0)
725 cntlid = NVMF_CNTLID_DYNAMIC;
726 else
727 cntlid = cdata->ctrlr_id;
728 dle->cntlid = htole16(cntlid);
729 memcpy(dle->subnqn, cdata->subnqn, sizeof(dle->subnqn));
730 return (0);
731 }
732
733 int
nvmf_host_request_queues(struct nvmf_qpair * qp,u_int requested,u_int * actual)734 nvmf_host_request_queues(struct nvmf_qpair *qp, u_int requested, u_int *actual)
735 {
736 struct nvme_command cmd;
737 struct nvmf_capsule *cc, *rc;
738 int error;
739 uint16_t status;
740
741 if (!qp->nq_admin || requested < 1 || requested > 65535)
742 return (EINVAL);
743
744 /* The number of queues is 0's based. */
745 requested--;
746
747 nvmf_init_sqe(&cmd, NVME_OPC_SET_FEATURES);
748 cmd.cdw10 = htole32(NVME_FEAT_NUMBER_OF_QUEUES);
749
750 /* Same number of completion and submission queues. */
751 cmd.cdw11 = htole32((requested << 16) | requested);
752
753 cc = nvmf_allocate_command(qp, &cmd);
754 if (cc == NULL)
755 return (errno);
756
757 error = nvmf_host_transmit_command(cc);
758 if (error != 0) {
759 nvmf_free_capsule(cc);
760 return (error);
761 }
762
763 error = nvmf_host_wait_for_response(cc, &rc);
764 nvmf_free_capsule(cc);
765 if (error != 0)
766 return (error);
767
768 status = le16toh(rc->nc_cqe.status);
769 if (status != 0) {
770 printf("NVMF: SET_FEATURES failed, status %#x\n", status);
771 nvmf_free_capsule(rc);
772 return (EIO);
773 }
774
775 *actual = (le32toh(rc->nc_cqe.cdw0) & 0xffff) + 1;
776 nvmf_free_capsule(rc);
777 return (0);
778 }
779
780 static bool
is_queue_pair_idle(struct nvmf_qpair * qp)781 is_queue_pair_idle(struct nvmf_qpair *qp)
782 {
783 if (qp->nq_sqhd != qp->nq_sqtail)
784 return (false);
785 if (!TAILQ_EMPTY(&qp->nq_rx_capsules))
786 return (false);
787 return (true);
788 }
789
790 static int
prepare_queues_for_handoff(struct nvmf_ioc_nv * nv,const struct nvme_discovery_log_entry * dle,const char * hostnqn,struct nvmf_qpair * admin_qp,u_int num_queues,struct nvmf_qpair ** io_queues,const struct nvme_controller_data * cdata)791 prepare_queues_for_handoff(struct nvmf_ioc_nv *nv,
792 const struct nvme_discovery_log_entry *dle, const char *hostnqn,
793 struct nvmf_qpair *admin_qp, u_int num_queues,
794 struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata)
795 {
796 const struct nvmf_association *na = admin_qp->nq_association;
797 nvlist_t *nvl, *nvl_qp, *nvl_rparams;
798 u_int i;
799 int error;
800
801 if (num_queues == 0)
802 return (EINVAL);
803
804 /* Ensure trtype matches. */
805 if (dle->trtype != na->na_trtype)
806 return (EINVAL);
807
808 /* All queue pairs must be idle. */
809 if (!is_queue_pair_idle(admin_qp))
810 return (EBUSY);
811 for (i = 0; i < num_queues; i++) {
812 if (!is_queue_pair_idle(io_queues[i]))
813 return (EBUSY);
814 }
815
816 /* Fill out reconnect parameters. */
817 nvl_rparams = nvlist_create(0);
818 nvlist_add_binary(nvl_rparams, "dle", dle, sizeof(*dle));
819 nvlist_add_string(nvl_rparams, "hostnqn", hostnqn);
820 nvlist_add_number(nvl_rparams, "num_io_queues", num_queues);
821 nvlist_add_number(nvl_rparams, "kato", admin_qp->nq_kato);
822 nvlist_add_number(nvl_rparams, "io_qsize", io_queues[0]->nq_qsize);
823 nvlist_add_bool(nvl_rparams, "sq_flow_control",
824 na->na_params.sq_flow_control);
825 switch (na->na_trtype) {
826 case NVMF_TRTYPE_TCP:
827 nvlist_add_bool(nvl_rparams, "header_digests",
828 na->na_params.tcp.header_digests);
829 nvlist_add_bool(nvl_rparams, "data_digests",
830 na->na_params.tcp.data_digests);
831 break;
832 default:
833 __unreachable();
834 }
835 error = nvlist_error(nvl_rparams);
836 if (error != 0) {
837 nvlist_destroy(nvl_rparams);
838 return (error);
839 }
840
841 nvl = nvlist_create(0);
842 nvlist_add_number(nvl, "trtype", na->na_trtype);
843 nvlist_add_number(nvl, "kato", admin_qp->nq_kato);
844 nvlist_move_nvlist(nvl, "rparams", nvl_rparams);
845
846 /* First, the admin queue. */
847 error = nvmf_kernel_handoff_params(admin_qp, &nvl_qp);
848 if (error) {
849 nvlist_destroy(nvl);
850 return (error);
851 }
852 nvlist_move_nvlist(nvl, "admin", nvl_qp);
853
854 /* Next, the I/O queues. */
855 for (i = 0; i < num_queues; i++) {
856 error = nvmf_kernel_handoff_params(io_queues[i], &nvl_qp);
857 if (error) {
858 nvlist_destroy(nvl);
859 return (error);
860 }
861 nvlist_append_nvlist_array(nvl, "io", nvl_qp);
862 }
863
864 nvlist_add_binary(nvl, "cdata", cdata, sizeof(*cdata));
865
866 error = nvmf_pack_ioc_nvlist(nv, nvl);
867 nvlist_destroy(nvl);
868 return (error);
869 }
870
871 int
nvmf_handoff_host(const struct nvme_discovery_log_entry * dle,const char * hostnqn,struct nvmf_qpair * admin_qp,u_int num_queues,struct nvmf_qpair ** io_queues,const struct nvme_controller_data * cdata)872 nvmf_handoff_host(const struct nvme_discovery_log_entry *dle,
873 const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues,
874 struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata)
875 {
876 struct nvmf_ioc_nv nv;
877 u_int i;
878 int error, fd;
879
880 fd = open("/dev/nvmf", O_RDWR);
881 if (fd == -1) {
882 error = errno;
883 goto out;
884 }
885
886 error = prepare_queues_for_handoff(&nv, dle, hostnqn, admin_qp,
887 num_queues, io_queues, cdata);
888 if (error != 0)
889 goto out;
890
891 if (ioctl(fd, NVMF_HANDOFF_HOST, &nv) == -1)
892 error = errno;
893 free(nv.data);
894
895 out:
896 if (fd >= 0)
897 close(fd);
898 for (i = 0; i < num_queues; i++)
899 (void)nvmf_free_qpair(io_queues[i]);
900 (void)nvmf_free_qpair(admin_qp);
901 return (error);
902 }
903
904 int
nvmf_disconnect_host(const char * host)905 nvmf_disconnect_host(const char *host)
906 {
907 int error, fd;
908
909 error = 0;
910 fd = open("/dev/nvmf", O_RDWR);
911 if (fd == -1) {
912 error = errno;
913 goto out;
914 }
915
916 if (ioctl(fd, NVMF_DISCONNECT_HOST, &host) == -1)
917 error = errno;
918
919 out:
920 if (fd >= 0)
921 close(fd);
922 return (error);
923 }
924
925 int
nvmf_disconnect_all(void)926 nvmf_disconnect_all(void)
927 {
928 int error, fd;
929
930 error = 0;
931 fd = open("/dev/nvmf", O_RDWR);
932 if (fd == -1) {
933 error = errno;
934 goto out;
935 }
936
937 if (ioctl(fd, NVMF_DISCONNECT_ALL) == -1)
938 error = errno;
939
940 out:
941 if (fd >= 0)
942 close(fd);
943 return (error);
944 }
945
946 static int
nvmf_read_ioc_nv(int fd,u_long com,nvlist_t ** nvlp)947 nvmf_read_ioc_nv(int fd, u_long com, nvlist_t **nvlp)
948 {
949 struct nvmf_ioc_nv nv;
950 nvlist_t *nvl;
951 int error;
952
953 memset(&nv, 0, sizeof(nv));
954 if (ioctl(fd, com, &nv) == -1)
955 return (errno);
956
957 nv.data = malloc(nv.len);
958 nv.size = nv.len;
959 if (ioctl(fd, com, &nv) == -1) {
960 error = errno;
961 free(nv.data);
962 return (error);
963 }
964
965 nvl = nvlist_unpack(nv.data, nv.len, 0);
966 free(nv.data);
967 if (nvl == NULL)
968 return (EINVAL);
969
970 *nvlp = nvl;
971 return (0);
972 }
973
974 int
nvmf_reconnect_params(int fd,nvlist_t ** nvlp)975 nvmf_reconnect_params(int fd, nvlist_t **nvlp)
976 {
977 return (nvmf_read_ioc_nv(fd, NVMF_RECONNECT_PARAMS, nvlp));
978 }
979
980 int
nvmf_reconnect_host(int fd,const struct nvme_discovery_log_entry * dle,const char * hostnqn,struct nvmf_qpair * admin_qp,u_int num_queues,struct nvmf_qpair ** io_queues,const struct nvme_controller_data * cdata)981 nvmf_reconnect_host(int fd, const struct nvme_discovery_log_entry *dle,
982 const char *hostnqn, struct nvmf_qpair *admin_qp, u_int num_queues,
983 struct nvmf_qpair **io_queues, const struct nvme_controller_data *cdata)
984 {
985 struct nvmf_ioc_nv nv;
986 u_int i;
987 int error;
988
989 error = prepare_queues_for_handoff(&nv, dle, hostnqn, admin_qp,
990 num_queues, io_queues, cdata);
991 if (error != 0)
992 goto out;
993
994 if (ioctl(fd, NVMF_RECONNECT_HOST, &nv) == -1)
995 error = errno;
996 free(nv.data);
997
998 out:
999 for (i = 0; i < num_queues; i++)
1000 (void)nvmf_free_qpair(io_queues[i]);
1001 (void)nvmf_free_qpair(admin_qp);
1002 return (error);
1003 }
1004