1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 */
7
8 #include <sys/param.h>
9 #include <sys/linker.h>
10 #include <sys/module.h>
11 #include <sys/time.h>
12 #include <assert.h>
13 #include <ctype.h>
14 #include <errno.h>
15 #include <libiscsiutil.h>
16 #include <libnvmf.h>
17 #include <libutil.h>
18 #include <limits.h>
19 #include <stdbool.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <cam/ctl/ctl.h>
25 #include <cam/ctl/ctl_io.h>
26 #include <cam/ctl/ctl_ioctl.h>
27
28 #include <memory>
29
30 #include "ctld.hh"
31 #include "nvmf.hh"
32
33 #define DEFAULT_MAXH2CDATA (256 * 1024)
34
35 struct nvmf_io_portal final : public nvmf_portal {
nvmf_io_portalnvmf_io_portal36 nvmf_io_portal(struct portal_group *pg, const char *listen,
37 portal_protocol protocol, freebsd::addrinfo_up ai) :
38 nvmf_portal(pg, listen, protocol, std::move(ai)) {}
39
40 void handle_connection(freebsd::fd_up fd, const char *host,
41 const struct sockaddr *client_sa) override;
42 };
43
44 struct nvmf_transport_group final : public portal_group {
nvmf_transport_groupnvmf_transport_group45 nvmf_transport_group(struct conf *conf, std::string_view name) :
46 portal_group(conf, name) {}
47
keywordnvmf_transport_group48 const char *keyword() const override
49 { return "transport-group"; }
50
51 void allocate_tag() override;
52 bool add_portal(const char *value, portal_protocol protocol)
53 override;
54 void add_default_portals() override;
55 bool set_filter(const char *str) override;
56
57 virtual port_up create_port(struct target *target, auth_group_sp ag)
58 override;
59 virtual port_up create_port(struct target *target, uint32_t ctl_port)
60 override;
61
62 private:
63 static uint16_t last_port_id;
64 };
65
66 struct nvmf_port final : public portal_group_port {
nvmf_portnvmf_port67 nvmf_port(struct target *target, struct portal_group *pg,
68 auth_group_sp ag) :
69 portal_group_port(target, pg, ag) {}
nvmf_portnvmf_port70 nvmf_port(struct target *target, struct portal_group *pg,
71 uint32_t ctl_port) :
72 portal_group_port(target, pg, ctl_port) {}
73
74 bool kernel_create_port() override;
75 bool kernel_remove_port() override;
76
77 private:
78 static bool modules_loaded;
79 static void load_kernel_modules();
80 };
81
82 struct nvmf_controller final : public target {
nvmf_controllernvmf_controller83 nvmf_controller(struct conf *conf, std::string_view name) :
84 target(conf, "controller", name) {}
85
86 bool add_host_nqn(std::string_view name) override;
87 bool add_host_address(const char *addr) override;
88 bool add_namespace(u_int id, const char *lun_name) override;
89 bool add_portal_group(const char *pg_name, const char *ag_name)
90 override;
91 struct lun *start_namespace(u_int id) override;
92
93 protected:
94 struct portal_group *default_portal_group() override;
95 };
96
97 uint16_t nvmf_transport_group::last_port_id = 0;
98 bool nvmf_port::modules_loaded = false;
99
100 static bool need_tcp_transport = false;
101
102 static bool
parse_bool(const nvlist_t * nvl,const char * key,bool def)103 parse_bool(const nvlist_t *nvl, const char *key, bool def)
104 {
105 const char *value;
106
107 if (!nvlist_exists_string(nvl, key))
108 return def;
109
110 value = nvlist_get_string(nvl, key);
111 if (strcasecmp(value, "true") == 0 ||
112 strcasecmp(value, "1") == 0)
113 return true;
114 if (strcasecmp(value, "false") == 0 ||
115 strcasecmp(value, "0") == 0)
116 return false;
117
118 log_warnx("Invalid value \"%s\" for boolean option %s", value, key);
119 return def;
120 }
121
122 static uint64_t
parse_number(const nvlist_t * nvl,const char * key,uint64_t def,uint64_t minv,uint64_t maxv)123 parse_number(const nvlist_t *nvl, const char *key, uint64_t def, uint64_t minv,
124 uint64_t maxv)
125 {
126 const char *value;
127 int64_t val;
128
129 if (!nvlist_exists_string(nvl, key))
130 return def;
131
132 value = nvlist_get_string(nvl, key);
133 if (expand_number(value, &val) == 0 && val >= 0 &&
134 (uint64_t)val >= minv && (uint64_t)val <= maxv)
135 return (uint64_t)val;
136
137 log_warnx("Invalid value \"%s\" for numeric option %s", value, key);
138 return def;
139 }
140
141 bool
prepare()142 nvmf_portal::prepare()
143 {
144 memset(&p_aparams, 0, sizeof(p_aparams));
145
146 /* Options shared between discovery and I/O associations. */
147 freebsd::nvlist_up nvl = portal_group()->options();
148 p_aparams.tcp.header_digests = parse_bool(nvl.get(), "HDGST", false);
149 p_aparams.tcp.data_digests = parse_bool(nvl.get(), "DDGST", false);
150 uint64_t value = parse_number(nvl.get(), "MAXH2CDATA",
151 DEFAULT_MAXH2CDATA, 4096, UINT32_MAX);
152 if (value % 4 != 0) {
153 log_warnx("Invalid value \"%ju\" for option MAXH2CDATA",
154 (uintmax_t)value);
155 value = DEFAULT_MAXH2CDATA;
156 }
157 p_aparams.tcp.maxh2cdata = value;
158
159 switch (protocol()) {
160 case portal_protocol::NVME_TCP:
161 p_aparams.sq_flow_control = parse_bool(nvl.get(), "SQFC",
162 false);
163 p_aparams.dynamic_controller_model = true;
164 p_aparams.max_admin_qsize = parse_number(nvl.get(),
165 "max_admin_qsize", NVME_MAX_ADMIN_ENTRIES,
166 NVME_MIN_ADMIN_ENTRIES, NVME_MAX_ADMIN_ENTRIES);
167 p_aparams.max_io_qsize = parse_number(nvl.get(), "max_io_qsize",
168 NVME_MAX_IO_ENTRIES, NVME_MIN_IO_ENTRIES,
169 NVME_MAX_IO_ENTRIES);
170 p_aparams.tcp.pda = 0;
171 break;
172 case portal_protocol::NVME_DISCOVERY_TCP:
173 p_aparams.sq_flow_control = false;
174 p_aparams.dynamic_controller_model = true;
175 p_aparams.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES;
176 p_aparams.tcp.pda = 0;
177 break;
178 default:
179 __assert_unreachable();
180 }
181
182 p_association.reset(nvmf_allocate_association(NVMF_TRTYPE_TCP, true,
183 &p_aparams));
184 if (!p_association) {
185 log_warn("Failed to create NVMe controller association");
186 return false;
187 }
188
189 return true;
190 }
191
192 portal_group_up
nvmf_make_transport_group(struct conf * conf,std::string_view name)193 nvmf_make_transport_group(struct conf *conf, std::string_view name)
194 {
195 return std::make_unique<nvmf_transport_group>(conf, name);
196 }
197
198 target_up
nvmf_make_controller(struct conf * conf,std::string_view name)199 nvmf_make_controller(struct conf *conf, std::string_view name)
200 {
201 return std::make_unique<nvmf_controller>(conf, name);
202 }
203
204 void
allocate_tag()205 nvmf_transport_group::allocate_tag()
206 {
207 set_tag(++last_port_id);
208 }
209
210 bool
add_portal(const char * value,portal_protocol protocol)211 nvmf_transport_group::add_portal(const char *value, portal_protocol protocol)
212 {
213 freebsd::addrinfo_up ai;
214
215 switch (protocol) {
216 case portal_protocol::NVME_TCP:
217 ai = parse_addr_port(value, "4420");
218 break;
219 case portal_protocol::NVME_DISCOVERY_TCP:
220 ai = parse_addr_port(value, "8009");
221 break;
222 default:
223 log_warnx("unsupported transport protocol for %s", value);
224 return false;
225 }
226
227 if (!ai) {
228 log_warnx("invalid listen address %s", value);
229 return false;
230 }
231
232 /*
233 * XXX: getaddrinfo(3) may return multiple addresses; we should turn
234 * those into multiple portals.
235 */
236
237 portal_up portal;
238 if (protocol == portal_protocol::NVME_DISCOVERY_TCP) {
239 portal = std::make_unique<nvmf_discovery_portal>(this, value,
240 protocol, std::move(ai));
241 } else {
242 portal = std::make_unique<nvmf_io_portal>(this, value,
243 protocol, std::move(ai));
244 need_tcp_transport = true;
245 }
246
247 pg_portals.emplace_back(std::move(portal));
248 return true;
249 }
250
251 void
add_default_portals()252 nvmf_transport_group::add_default_portals()
253 {
254 add_portal("0.0.0.0", portal_protocol::NVME_DISCOVERY_TCP);
255 add_portal("[::]", portal_protocol::NVME_DISCOVERY_TCP);
256 add_portal("0.0.0.0", portal_protocol::NVME_TCP);
257 add_portal("[::]", portal_protocol::NVME_TCP);
258 }
259
260 bool
set_filter(const char * str)261 nvmf_transport_group::set_filter(const char *str)
262 {
263 enum discovery_filter filter;
264
265 if (strcmp(str, "none") == 0) {
266 filter = discovery_filter::NONE;
267 } else if (strcmp(str, "address") == 0) {
268 filter = discovery_filter::PORTAL;
269 } else if (strcmp(str, "address-name") == 0) {
270 filter = discovery_filter::PORTAL_NAME;
271 } else {
272 log_warnx("invalid discovery-filter \"%s\" for transport-group "
273 "\"%s\"; valid values are \"none\", \"address\", "
274 "and \"address-name\"",
275 str, name());
276 return false;
277 }
278
279 if (pg_discovery_filter != discovery_filter::UNKNOWN &&
280 pg_discovery_filter != filter) {
281 log_warnx("cannot set discovery-filter to \"%s\" for "
282 "transport-group \"%s\"; already has a different "
283 "value", str, name());
284 return false;
285 }
286
287 pg_discovery_filter = filter;
288 return true;
289 }
290
291 port_up
create_port(struct target * target,auth_group_sp ag)292 nvmf_transport_group::create_port(struct target *target, auth_group_sp ag)
293 {
294 return std::make_unique<nvmf_port>(target, this, ag);
295 }
296
297 port_up
create_port(struct target * target,uint32_t ctl_port)298 nvmf_transport_group::create_port(struct target *target, uint32_t ctl_port)
299 {
300 return std::make_unique<nvmf_port>(target, this, ctl_port);
301 }
302
303 void
load_kernel_modules()304 nvmf_port::load_kernel_modules()
305 {
306 int saved_errno;
307
308 if (modules_loaded)
309 return;
310
311 saved_errno = errno;
312 if (modfind("nvmft") == -1 && kldload("nvmft") == -1)
313 log_warn("couldn't load nvmft");
314
315 if (need_tcp_transport) {
316 if (modfind("nvmf/tcp") == -1 && kldload("nvmf_tcp") == -1)
317 log_warn("couldn't load nvmf_tcp");
318 }
319
320 errno = saved_errno;
321 modules_loaded = true;
322 }
323
324 bool
kernel_create_port()325 nvmf_port::kernel_create_port()
326 {
327 struct portal_group *pg = p_portal_group;
328 struct target *targ = p_target;
329
330 load_kernel_modules();
331
332 freebsd::nvlist_up nvl = pg->options();
333 nvlist_add_string(nvl.get(), "subnqn", targ->name());
334 nvlist_add_string(nvl.get(), "ctld_transport_group_name",
335 pg->name());
336 nvlist_add_stringf(nvl.get(), "portid", "%u", pg->tag());
337 if (!nvlist_exists_string(nvl.get(), "max_io_qsize"))
338 nvlist_add_stringf(nvl.get(), "max_io_qsize", "%u",
339 NVME_MAX_IO_ENTRIES);
340
341 return ctl_create_port("nvmf", nvl.get(), &p_ctl_port);
342 }
343
344 bool
kernel_remove_port()345 nvmf_port::kernel_remove_port()
346 {
347 freebsd::nvlist_up nvl(nvlist_create(0));
348 nvlist_add_string(nvl.get(), "subnqn", p_target->name());
349
350 return ctl_remove_port("nvmf", nvl.get());
351 }
352
353 bool
add_host_nqn(std::string_view name)354 nvmf_controller::add_host_nqn(std::string_view name)
355 {
356 if (!use_private_auth("host-nqn"))
357 return false;
358 return t_auth_group->add_host_nqn(name);
359 }
360
361 bool
add_host_address(const char * addr)362 nvmf_controller::add_host_address(const char *addr)
363 {
364 if (!use_private_auth("host-address"))
365 return false;
366 return t_auth_group->add_host_address(addr);
367 }
368
369 bool
add_namespace(u_int id,const char * lun_name)370 nvmf_controller::add_namespace(u_int id, const char *lun_name)
371 {
372 if (id == 0) {
373 log_warnx("namespace ID cannot be 0 for %s", label());
374 return false;
375 }
376
377 std::string lun_label = "namespace ID " + std::to_string(id - 1);
378 return target::add_lun(id, lun_label.c_str(), lun_name);
379 }
380
381 bool
add_portal_group(const char * pg_name,const char * ag_name)382 nvmf_controller::add_portal_group(const char *pg_name, const char *ag_name)
383 {
384 struct portal_group *pg;
385 auth_group_sp ag;
386
387 pg = t_conf->find_transport_group(pg_name);
388 if (pg == NULL) {
389 log_warnx("unknown transport-group \"%s\" for %s", pg_name,
390 label());
391 return false;
392 }
393
394 if (ag_name != NULL) {
395 ag = t_conf->find_auth_group(ag_name);
396 if (ag == NULL) {
397 log_warnx("unknown auth-group \"%s\" for %s", ag_name,
398 label());
399 return false;
400 }
401 }
402
403 if (!t_conf->add_port(this, pg, std::move(ag))) {
404 log_warnx("can't link transport-group \"%s\" to %s", pg_name,
405 label());
406 return false;
407 }
408 return true;
409 }
410
411 struct lun *
start_namespace(u_int id)412 nvmf_controller::start_namespace(u_int id)
413 {
414 if (id == 0) {
415 log_warnx("namespace ID cannot be 0 for %s", label());
416 return nullptr;
417 }
418
419 std::string lun_label = "namespace ID " + std::to_string(id - 1);
420 std::string lun_name = freebsd::stringf("%s,nsid,%u", name(), id);
421 return target::start_lun(id, lun_label.c_str(), lun_name.c_str());
422 }
423
424 struct portal_group *
default_portal_group()425 nvmf_controller::default_portal_group()
426 {
427 return t_conf->find_transport_group("default");
428 }
429
430 void
handle_connection(freebsd::fd_up fd,const char * host __unused,const struct sockaddr * client_sa __unused)431 nvmf_io_portal::handle_connection(freebsd::fd_up fd, const char *host __unused,
432 const struct sockaddr *client_sa __unused)
433 {
434 struct nvmf_qpair_params qparams;
435 memset(&qparams, 0, sizeof(qparams));
436 qparams.tcp.fd = fd;
437
438 struct nvmf_capsule *nc = NULL;
439 struct nvmf_fabric_connect_data data;
440 nvmf_qpair_up qp(nvmf_accept(association(), &qparams, &nc, &data));
441 if (!qp) {
442 log_warnx("Failed to create NVMe I/O qpair: %s",
443 nvmf_association_error(association()));
444 return;
445 }
446 nvmf_capsule_up nc_guard(nc);
447 const struct nvmf_fabric_connect_cmd *cmd =
448 (const struct nvmf_fabric_connect_cmd *)nvmf_capsule_sqe(nc);
449
450 struct ctl_nvmf req;
451 memset(&req, 0, sizeof(req));
452 req.type = CTL_NVMF_HANDOFF;
453 int error = nvmf_handoff_controller_qpair(qp.get(), cmd, &data,
454 &req.data.handoff);
455 if (error != 0) {
456 log_warnc(error,
457 "Failed to prepare NVMe I/O qpair for handoff");
458 return;
459 }
460
461 if (ioctl(ctl_fd, CTL_NVMF, &req) != 0)
462 log_warn("ioctl(CTL_NVMF/CTL_NVMF_HANDOFF)");
463 if (req.status == CTL_NVMF_ERROR)
464 log_warnx("Failed to handoff NVMF connection: %s",
465 req.error_str);
466 else if (req.status != CTL_NVMF_OK)
467 log_warnx("Failed to handoff NVMF connection with status %d",
468 req.status);
469 }
470