1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2023-2024 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 */
7
8 #include <sys/dnv.h>
9 #include <sys/nv.h>
10 #include <sys/socket.h>
11 #include <err.h>
12 #include <libnvmf.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #include <sysexits.h>
16 #include <unistd.h>
17
18 #include "nvmecontrol.h"
19 #include "fabrics.h"
20
21 /*
22 * See comment about other possible settings in connect.c.
23 */
24
25 static struct options {
26 const char *dev;
27 const char *transport;
28 const char *hostnqn;
29 uint32_t kato;
30 uint32_t reconnect_delay;
31 uint32_t controller_loss_timeout;
32 uint16_t num_io_queues;
33 uint16_t queue_size;
34 bool data_digests;
35 bool flow_control;
36 bool header_digests;
37 } opt = {
38 .dev = NULL,
39 .transport = "tcp",
40 .hostnqn = NULL,
41 .kato = NVMF_KATO_DEFAULT / 1000,
42 .reconnect_delay = NVMF_DEFAULT_RECONNECT_DELAY,
43 .controller_loss_timeout = NVMF_DEFAULT_CONTROLLER_LOSS,
44 .num_io_queues = 1,
45 .queue_size = 0,
46 .data_digests = false,
47 .flow_control = false,
48 .header_digests = false,
49 };
50
51 static void
tcp_association_params(struct nvmf_association_params * params,bool header_digests,bool data_digests)52 tcp_association_params(struct nvmf_association_params *params,
53 bool header_digests, bool data_digests)
54 {
55 params->tcp.pda = 0;
56 params->tcp.header_digests = header_digests;
57 params->tcp.data_digests = data_digests;
58 /* XXX */
59 params->tcp.maxr2t = 1;
60 }
61
62 static int
reconnect_nvm_controller(int fd,const struct nvmf_association_params * aparams,enum nvmf_trtype trtype,int adrfam,const char * address,const char * port,uint16_t cntlid,const char * subnqn,const char * hostnqn,uint32_t kato,uint32_t reconnect_delay,uint32_t controller_loss_timeout,u_int num_io_queues,u_int queue_size,const struct nvme_discovery_log_entry * dle)63 reconnect_nvm_controller(int fd, const struct nvmf_association_params *aparams,
64 enum nvmf_trtype trtype, int adrfam, const char *address, const char *port,
65 uint16_t cntlid, const char *subnqn, const char *hostnqn, uint32_t kato,
66 uint32_t reconnect_delay, uint32_t controller_loss_timeout,
67 u_int num_io_queues, u_int queue_size,
68 const struct nvme_discovery_log_entry *dle)
69 {
70 struct nvme_controller_data cdata;
71 struct nvme_discovery_log_entry dle_thunk;
72 struct nvmf_qpair *admin, **io;
73 int error;
74
75 io = calloc(num_io_queues, sizeof(*io));
76 error = connect_nvm_queues(aparams, trtype, adrfam, address, port,
77 cntlid, subnqn, hostnqn, kato, &admin, io, num_io_queues,
78 queue_size, &cdata);
79 if (error != 0) {
80 free(io);
81 return (error);
82 }
83
84 if (dle == NULL) {
85 error = nvmf_init_dle_from_admin_qp(admin, &cdata, &dle_thunk);
86 if (error != 0) {
87 warnc(error, "Failed to generate handoff parameters");
88 disconnect_nvm_queues(admin, io, num_io_queues);
89 free(io);
90 return (EX_IOERR);
91 }
92 dle = &dle_thunk;
93 }
94
95 error = nvmf_reconnect_host(fd, dle, hostnqn, admin, num_io_queues, io,
96 &cdata, reconnect_delay, controller_loss_timeout);
97 if (error != 0) {
98 warnc(error, "Failed to handoff queues to kernel");
99 free(io);
100 return (EX_IOERR);
101 }
102 free(io);
103 return (0);
104 }
105
106 static int
reconnect_by_address(int fd,const nvlist_t * rparams,const char * addr)107 reconnect_by_address(int fd, const nvlist_t *rparams, const char *addr)
108 {
109 const struct nvme_discovery_log_entry *dle;
110 struct nvmf_association_params aparams;
111 enum nvmf_trtype trtype;
112 const char *address, *hostnqn, *port;
113 char *subnqn, *tofree;
114 int error;
115
116 memset(&aparams, 0, sizeof(aparams));
117 aparams.sq_flow_control = opt.flow_control;
118 if (strcasecmp(opt.transport, "tcp") == 0) {
119 trtype = NVMF_TRTYPE_TCP;
120 tcp_association_params(&aparams, opt.header_digests,
121 opt.data_digests);
122 } else {
123 warnx("Unsupported or invalid transport");
124 return (EX_USAGE);
125 }
126
127 nvmf_parse_address(addr, &address, &port, &tofree);
128 if (port == NULL) {
129 free(tofree);
130 warnx("Explicit port required");
131 return (EX_USAGE);
132 }
133
134 dle = nvlist_get_binary(rparams, "dle", NULL);
135
136 hostnqn = opt.hostnqn;
137 if (hostnqn == NULL)
138 hostnqn = nvmf_default_hostnqn();
139
140 /* Ensure subnqn is a terminated C string. */
141 subnqn = strndup(dle->subnqn, sizeof(dle->subnqn));
142
143 error = reconnect_nvm_controller(fd, &aparams, trtype, AF_UNSPEC,
144 address, port, le16toh(dle->cntlid), subnqn, hostnqn,
145 opt.kato * 1000, opt.reconnect_delay, opt.controller_loss_timeout,
146 opt.num_io_queues, opt.queue_size, NULL);
147 free(subnqn);
148 free(tofree);
149 return (error);
150 }
151
152 static int
reconnect_by_params(int fd,const nvlist_t * rparams)153 reconnect_by_params(int fd, const nvlist_t *rparams)
154 {
155 struct nvmf_association_params aparams;
156 const struct nvme_discovery_log_entry *dle;
157 char *address, *port, *subnqn;
158 int adrfam, error;
159
160 dle = nvlist_get_binary(rparams, "dle", NULL);
161
162 memset(&aparams, 0, sizeof(aparams));
163 aparams.sq_flow_control = nvlist_get_bool(rparams, "sq_flow_control");
164 switch (dle->trtype) {
165 case NVMF_TRTYPE_TCP:
166 switch (dle->adrfam) {
167 case NVMF_ADRFAM_IPV4:
168 adrfam = AF_INET;
169 break;
170 case NVMF_ADRFAM_IPV6:
171 adrfam = AF_INET6;
172 break;
173 default:
174 warnx("Unsupported address family");
175 return (EX_UNAVAILABLE);
176 }
177 switch (dle->tsas.tcp.sectype) {
178 case NVME_TCP_SECURITY_NONE:
179 break;
180 default:
181 warnx("Unsupported TCP security type");
182 return (EX_UNAVAILABLE);
183 }
184 break;
185
186 tcp_association_params(&aparams,
187 nvlist_get_bool(rparams, "header_digests"),
188 nvlist_get_bool(rparams, "data_digests"));
189 break;
190 default:
191 warnx("Unsupported transport %s",
192 nvmf_transport_type(dle->trtype));
193 return (EX_UNAVAILABLE);
194 }
195
196 /* Ensure address, port, and subnqn is a terminated C string. */
197 address = strndup(dle->traddr, sizeof(dle->traddr));
198 port = strndup(dle->trsvcid, sizeof(dle->trsvcid));
199 subnqn = strndup(dle->subnqn, sizeof(dle->subnqn));
200
201 error = reconnect_nvm_controller(fd, &aparams, dle->trtype, adrfam,
202 address, port, le16toh(dle->cntlid), dle->subnqn,
203 nvlist_get_string(rparams, "hostnqn"),
204 dnvlist_get_number(rparams, "kato", 0),
205 dnvlist_get_number(rparams, "reconnect_delay", 0),
206 dnvlist_get_number(rparams, "controller_loss_timeout", 0),
207 nvlist_get_number(rparams, "num_io_queues"),
208 nvlist_get_number(rparams, "io_qsize"), dle);
209 free(subnqn);
210 free(port);
211 free(address);
212 return (error);
213 }
214
215 static int
fetch_and_validate_rparams(int fd,nvlist_t ** rparamsp)216 fetch_and_validate_rparams(int fd, nvlist_t **rparamsp)
217 {
218 const struct nvme_discovery_log_entry *dle;
219 nvlist_t *rparams;
220 size_t len;
221 int error;
222
223 error = nvmf_reconnect_params(fd, &rparams);
224 if (error != 0) {
225 warnc(error, "Failed to fetch reconnect parameters");
226 return (EX_IOERR);
227 }
228
229 if (!nvlist_exists_binary(rparams, "dle") ||
230 !nvlist_exists_string(rparams, "hostnqn") ||
231 !nvlist_exists_number(rparams, "num_io_queues") ||
232 !nvlist_exists_number(rparams, "io_qsize") ||
233 !nvlist_exists_bool(rparams, "sq_flow_control")) {
234 nvlist_destroy(rparams);
235 warnx("Missing required reconnect parameters");
236 return (EX_IOERR);
237 }
238
239 dle = nvlist_get_binary(rparams, "dle", &len);
240 if (len != sizeof(*dle)) {
241 nvlist_destroy(rparams);
242 warnx("Discovery Log entry reconnect parameter is wrong size");
243 return (EX_IOERR);
244 }
245
246 switch (dle->trtype) {
247 case NVMF_TRTYPE_TCP:
248 if (!nvlist_exists_bool(rparams, "header_digests") ||
249 !nvlist_exists_bool(rparams, "data_digests")) {
250 nvlist_destroy(rparams);
251 warnx("Missing required reconnect parameters");
252 return (EX_IOERR);
253 }
254 break;
255 default:
256 nvlist_destroy(rparams);
257 warnx("Unsupported transport %s",
258 nvmf_transport_type(dle->trtype));
259 return (EX_UNAVAILABLE);
260 }
261
262 *rparamsp = rparams;
263 return (0);
264 }
265
266 static void
reconnect_fn(const struct cmd * f,int argc,char * argv[])267 reconnect_fn(const struct cmd *f, int argc, char *argv[])
268 {
269 nvlist_t *rparams;
270 int error, fd;
271
272 if (arg_parse(argc, argv, f))
273 return;
274
275 open_dev(opt.dev, &fd, 1, 1);
276 error = fetch_and_validate_rparams(fd, &rparams);
277 if (error != 0)
278 exit(error);
279
280 /* Check for optional address. */
281 if (optind < argc)
282 error = reconnect_by_address(fd, rparams, argv[optind]);
283 else
284 error = reconnect_by_params(fd, rparams);
285 if (error != 0)
286 exit(error);
287
288 nvlist_destroy(rparams);
289 close(fd);
290 }
291
292 static const struct opts reconnect_opts[] = {
293 #define OPT(l, s, t, opt, addr, desc) { l, s, t, &opt.addr, desc }
294 OPT("transport", 't', arg_string, opt, transport,
295 "Transport type"),
296 OPT("nr-io-queues", 'i', arg_uint16, opt, num_io_queues,
297 "Number of I/O queues"),
298 OPT("queue-size", 'Q', arg_uint16, opt, queue_size,
299 "Number of entries in each I/O queue"),
300 OPT("keep-alive-tmo", 'k', arg_uint32, opt, kato,
301 "Keep Alive timeout (in seconds)"),
302 OPT("reconnect-delay", 'r', arg_uint32, opt, reconnect_delay,
303 "Delay between reconnect attempts after connection loss "
304 "(in seconds)"),
305 OPT("ctrl-loss-tmo", 'l', arg_uint32, opt, controller_loss_timeout,
306 "Controller loss timeout after connection loss (in seconds)"),
307 OPT("hostnqn", 'q', arg_string, opt, hostnqn,
308 "Host NQN"),
309 OPT("flow_control", 'F', arg_none, opt, flow_control,
310 "Request SQ flow control"),
311 OPT("hdr_digests", 'g', arg_none, opt, header_digests,
312 "Enable TCP PDU header digests"),
313 OPT("data_digests", 'G', arg_none, opt, data_digests,
314 "Enable TCP PDU data digests"),
315 { NULL, 0, arg_none, NULL, NULL }
316 };
317 #undef OPT
318
319 static const struct args reconnect_args[] = {
320 { arg_string, &opt.dev, "controller-id" },
321 { arg_none, NULL, NULL },
322 };
323
324 static struct cmd reconnect_cmd = {
325 .name = "reconnect",
326 .fn = reconnect_fn,
327 .descr = "Reconnect to a fabrics controller",
328 .ctx_size = sizeof(opt),
329 .opts = reconnect_opts,
330 .args = reconnect_args,
331 };
332
333 CMD_COMMAND(reconnect_cmd);
334