1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2025 Chelsio Communications, Inc.
5 * Written by: John Baldwin <jhb@FreeBSD.org>
6 */
7
8 #include <sys/param.h>
9 #include <sys/linker.h>
10 #include <sys/module.h>
11 #include <sys/time.h>
12 #include <assert.h>
13 #include <ctype.h>
14 #include <errno.h>
15 #include <libiscsiutil.h>
16 #include <libnvmf.h>
17 #include <libutil.h>
18 #include <limits.h>
19 #include <stdbool.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <unistd.h>
24 #include <cam/ctl/ctl.h>
25 #include <cam/ctl/ctl_io.h>
26 #include <cam/ctl/ctl_ioctl.h>
27
28 #include <memory>
29
30 #include "ctld.hh"
31 #include "nvmf.hh"
32
33 #define DEFAULT_MAXH2CDATA (256 * 1024)
34
35 struct nvmf_io_portal final : public nvmf_portal {
nvmf_io_portalnvmf_io_portal36 nvmf_io_portal(struct portal_group *pg, const char *listen,
37 portal_protocol protocol, freebsd::addrinfo_up ai,
38 const struct nvmf_association_params &aparams,
39 nvmf_association_up na) :
40 nvmf_portal(pg, listen, protocol, std::move(ai), aparams,
41 std::move(na)) {}
42
43 void handle_connection(freebsd::fd_up fd, const char *host,
44 const struct sockaddr *client_sa) override;
45 };
46
47 struct nvmf_transport_group final : public portal_group {
nvmf_transport_groupnvmf_transport_group48 nvmf_transport_group(struct conf *conf, std::string_view name) :
49 portal_group(conf, name) {}
50
keywordnvmf_transport_group51 const char *keyword() const override
52 { return "transport-group"; }
53
54 void allocate_tag() override;
55 bool add_portal(const char *value, portal_protocol protocol)
56 override;
57 void add_default_portals() override;
58 bool set_filter(const char *str) override;
59
60 virtual port_up create_port(struct target *target, auth_group_sp ag)
61 override;
62 virtual port_up create_port(struct target *target, uint32_t ctl_port)
63 override;
64
65 private:
66 struct nvmf_association_params init_aparams(portal_protocol protocol);
67
68 static uint16_t last_port_id;
69 };
70
71 struct nvmf_port final : public portal_group_port {
nvmf_portnvmf_port72 nvmf_port(struct target *target, struct portal_group *pg,
73 auth_group_sp ag) :
74 portal_group_port(target, pg, ag) {}
nvmf_portnvmf_port75 nvmf_port(struct target *target, struct portal_group *pg,
76 uint32_t ctl_port) :
77 portal_group_port(target, pg, ctl_port) {}
78
79 bool kernel_create_port() override;
80 bool kernel_remove_port() override;
81
82 private:
83 static bool modules_loaded;
84 static void load_kernel_modules();
85 };
86
87 struct nvmf_controller final : public target {
nvmf_controllernvmf_controller88 nvmf_controller(struct conf *conf, std::string_view name) :
89 target(conf, "controller", name) {}
90
91 bool add_host_nqn(std::string_view name) override;
92 bool add_host_address(const char *addr) override;
93 bool add_namespace(u_int id, const char *lun_name) override;
94 bool add_portal_group(const char *pg_name, const char *ag_name)
95 override;
96 struct lun *start_namespace(u_int id) override;
97
98 protected:
99 struct portal_group *default_portal_group() override;
100 };
101
102 uint16_t nvmf_transport_group::last_port_id = 0;
103 bool nvmf_port::modules_loaded = false;
104
105 static bool need_tcp_transport = false;
106
107 static bool
parse_bool(const nvlist_t * nvl,const char * key,bool def)108 parse_bool(const nvlist_t *nvl, const char *key, bool def)
109 {
110 const char *value;
111
112 if (!nvlist_exists_string(nvl, key))
113 return def;
114
115 value = nvlist_get_string(nvl, key);
116 if (strcasecmp(value, "true") == 0 ||
117 strcasecmp(value, "1") == 0)
118 return true;
119 if (strcasecmp(value, "false") == 0 ||
120 strcasecmp(value, "0") == 0)
121 return false;
122
123 log_warnx("Invalid value \"%s\" for boolean option %s", value, key);
124 return def;
125 }
126
127 static uint64_t
parse_number(const nvlist_t * nvl,const char * key,uint64_t def,uint64_t minv,uint64_t maxv)128 parse_number(const nvlist_t *nvl, const char *key, uint64_t def, uint64_t minv,
129 uint64_t maxv)
130 {
131 const char *value;
132 int64_t val;
133
134 if (!nvlist_exists_string(nvl, key))
135 return def;
136
137 value = nvlist_get_string(nvl, key);
138 if (expand_number(value, &val) == 0 && val >= 0 &&
139 (uint64_t)val >= minv && (uint64_t)val <= maxv)
140 return (uint64_t)val;
141
142 log_warnx("Invalid value \"%s\" for numeric option %s", value, key);
143 return def;
144 }
145
146 struct nvmf_association_params
init_aparams(portal_protocol protocol)147 nvmf_transport_group::init_aparams(portal_protocol protocol)
148 {
149 struct nvmf_association_params params;
150 memset(¶ms, 0, sizeof(params));
151
152 /* Options shared between discovery and I/O associations. */
153 const nvlist_t *nvl = pg_options.get();
154 params.tcp.header_digests = parse_bool(nvl, "HDGST", false);
155 params.tcp.data_digests = parse_bool(nvl, "DDGST", false);
156 uint64_t value = parse_number(nvl, "MAXH2CDATA", DEFAULT_MAXH2CDATA,
157 4096, UINT32_MAX);
158 if (value % 4 != 0) {
159 log_warnx("Invalid value \"%ju\" for option MAXH2CDATA",
160 (uintmax_t)value);
161 value = DEFAULT_MAXH2CDATA;
162 }
163 params.tcp.maxh2cdata = value;
164
165 switch (protocol) {
166 case portal_protocol::NVME_TCP:
167 params.sq_flow_control = parse_bool(nvl, "SQFC", false);
168 params.dynamic_controller_model = true;
169 params.max_admin_qsize = parse_number(nvl, "max_admin_qsize",
170 NVME_MAX_ADMIN_ENTRIES, NVME_MIN_ADMIN_ENTRIES,
171 NVME_MAX_ADMIN_ENTRIES);
172 params.max_io_qsize = parse_number(nvl, "max_io_qsize",
173 NVME_MAX_IO_ENTRIES, NVME_MIN_IO_ENTRIES,
174 NVME_MAX_IO_ENTRIES);
175 params.tcp.pda = 0;
176 break;
177 case portal_protocol::NVME_DISCOVERY_TCP:
178 params.sq_flow_control = false;
179 params.dynamic_controller_model = true;
180 params.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES;
181 params.tcp.pda = 0;
182 break;
183 default:
184 __assert_unreachable();
185 }
186
187 return params;
188 }
189
190 portal_group_up
nvmf_make_transport_group(struct conf * conf,std::string_view name)191 nvmf_make_transport_group(struct conf *conf, std::string_view name)
192 {
193 return std::make_unique<nvmf_transport_group>(conf, name);
194 }
195
196 target_up
nvmf_make_controller(struct conf * conf,std::string_view name)197 nvmf_make_controller(struct conf *conf, std::string_view name)
198 {
199 return std::make_unique<nvmf_controller>(conf, name);
200 }
201
202 void
allocate_tag()203 nvmf_transport_group::allocate_tag()
204 {
205 set_tag(++last_port_id);
206 }
207
208 bool
add_portal(const char * value,portal_protocol protocol)209 nvmf_transport_group::add_portal(const char *value, portal_protocol protocol)
210 {
211 freebsd::addrinfo_up ai;
212 enum nvmf_trtype trtype;
213
214 switch (protocol) {
215 case portal_protocol::NVME_TCP:
216 trtype = NVMF_TRTYPE_TCP;
217 ai = parse_addr_port(value, "4420");
218 break;
219 case portal_protocol::NVME_DISCOVERY_TCP:
220 trtype = NVMF_TRTYPE_TCP;
221 ai = parse_addr_port(value, "8009");
222 break;
223 default:
224 log_warnx("unsupported transport protocol for %s", value);
225 return false;
226 }
227
228 if (!ai) {
229 log_warnx("invalid listen address %s", value);
230 return false;
231 }
232
233 struct nvmf_association_params aparams = init_aparams(protocol);
234 nvmf_association_up association(nvmf_allocate_association(trtype, true,
235 &aparams));
236 if (!association) {
237 log_warn("Failed to create NVMe controller association");
238 return false;
239 }
240
241 /*
242 * XXX: getaddrinfo(3) may return multiple addresses; we should turn
243 * those into multiple portals.
244 */
245
246 portal_up portal;
247 if (protocol == portal_protocol::NVME_DISCOVERY_TCP) {
248 portal = std::make_unique<nvmf_discovery_portal>(this, value,
249 protocol, std::move(ai), aparams, std::move(association));
250 } else {
251 portal = std::make_unique<nvmf_io_portal>(this, value,
252 protocol, std::move(ai), aparams, std::move(association));
253 need_tcp_transport = true;
254 }
255
256 pg_portals.emplace_back(std::move(portal));
257 return true;
258 }
259
260 void
add_default_portals()261 nvmf_transport_group::add_default_portals()
262 {
263 add_portal("0.0.0.0", portal_protocol::NVME_DISCOVERY_TCP);
264 add_portal("[::]", portal_protocol::NVME_DISCOVERY_TCP);
265 add_portal("0.0.0.0", portal_protocol::NVME_TCP);
266 add_portal("[::]", portal_protocol::NVME_TCP);
267 }
268
269 bool
set_filter(const char * str)270 nvmf_transport_group::set_filter(const char *str)
271 {
272 enum discovery_filter filter;
273
274 if (strcmp(str, "none") == 0) {
275 filter = discovery_filter::NONE;
276 } else if (strcmp(str, "address") == 0) {
277 filter = discovery_filter::PORTAL;
278 } else if (strcmp(str, "address-name") == 0) {
279 filter = discovery_filter::PORTAL_NAME;
280 } else {
281 log_warnx("invalid discovery-filter \"%s\" for transport-group "
282 "\"%s\"; valid values are \"none\", \"address\", "
283 "and \"address-name\"",
284 str, name());
285 return false;
286 }
287
288 if (pg_discovery_filter != discovery_filter::UNKNOWN &&
289 pg_discovery_filter != filter) {
290 log_warnx("cannot set discovery-filter to \"%s\" for "
291 "transport-group \"%s\"; already has a different "
292 "value", str, name());
293 return false;
294 }
295
296 pg_discovery_filter = filter;
297 return true;
298 }
299
300 port_up
create_port(struct target * target,auth_group_sp ag)301 nvmf_transport_group::create_port(struct target *target, auth_group_sp ag)
302 {
303 return std::make_unique<nvmf_port>(target, this, ag);
304 }
305
306 port_up
create_port(struct target * target,uint32_t ctl_port)307 nvmf_transport_group::create_port(struct target *target, uint32_t ctl_port)
308 {
309 return std::make_unique<nvmf_port>(target, this, ctl_port);
310 }
311
312 void
load_kernel_modules()313 nvmf_port::load_kernel_modules()
314 {
315 int saved_errno;
316
317 if (modules_loaded)
318 return;
319
320 saved_errno = errno;
321 if (modfind("nvmft") == -1 && kldload("nvmft") == -1)
322 log_warn("couldn't load nvmft");
323
324 if (need_tcp_transport) {
325 if (modfind("nvmf/tcp") == -1 && kldload("nvmf_tcp") == -1)
326 log_warn("couldn't load nvmf_tcp");
327 }
328
329 errno = saved_errno;
330 modules_loaded = true;
331 }
332
333 bool
kernel_create_port()334 nvmf_port::kernel_create_port()
335 {
336 struct portal_group *pg = p_portal_group;
337 struct target *targ = p_target;
338
339 load_kernel_modules();
340
341 freebsd::nvlist_up nvl = pg->options();
342 nvlist_add_string(nvl.get(), "subnqn", targ->name());
343 nvlist_add_string(nvl.get(), "ctld_transport_group_name",
344 pg->name());
345 nvlist_add_stringf(nvl.get(), "portid", "%u", pg->tag());
346 if (!nvlist_exists_string(nvl.get(), "max_io_qsize"))
347 nvlist_add_stringf(nvl.get(), "max_io_qsize", "%u",
348 NVME_MAX_IO_ENTRIES);
349
350 return ctl_create_port("nvmf", nvl.get(), &p_ctl_port);
351 }
352
353 bool
kernel_remove_port()354 nvmf_port::kernel_remove_port()
355 {
356 freebsd::nvlist_up nvl(nvlist_create(0));
357 nvlist_add_string(nvl.get(), "subnqn", p_target->name());
358
359 return ctl_remove_port("nvmf", nvl.get());
360 }
361
362 bool
add_host_nqn(std::string_view name)363 nvmf_controller::add_host_nqn(std::string_view name)
364 {
365 if (!use_private_auth("host-nqn"))
366 return false;
367 return t_auth_group->add_host_nqn(name);
368 }
369
370 bool
add_host_address(const char * addr)371 nvmf_controller::add_host_address(const char *addr)
372 {
373 if (!use_private_auth("host-address"))
374 return false;
375 return t_auth_group->add_host_address(addr);
376 }
377
378 bool
add_namespace(u_int id,const char * lun_name)379 nvmf_controller::add_namespace(u_int id, const char *lun_name)
380 {
381 if (id == 0) {
382 log_warnx("namespace ID cannot be 0 for %s", label());
383 return false;
384 }
385
386 std::string lun_label = "namespace ID " + std::to_string(id - 1);
387 return target::add_lun(id, lun_label.c_str(), lun_name);
388 }
389
390 bool
add_portal_group(const char * pg_name,const char * ag_name)391 nvmf_controller::add_portal_group(const char *pg_name, const char *ag_name)
392 {
393 struct portal_group *pg;
394 auth_group_sp ag;
395
396 pg = t_conf->find_transport_group(pg_name);
397 if (pg == NULL) {
398 log_warnx("unknown transport-group \"%s\" for %s", pg_name,
399 label());
400 return false;
401 }
402
403 if (ag_name != NULL) {
404 ag = t_conf->find_auth_group(ag_name);
405 if (ag == NULL) {
406 log_warnx("unknown auth-group \"%s\" for %s", ag_name,
407 label());
408 return false;
409 }
410 }
411
412 if (!t_conf->add_port(this, pg, std::move(ag))) {
413 log_warnx("can't link transport-group \"%s\" to %s", pg_name,
414 label());
415 return false;
416 }
417 return true;
418 }
419
420 struct lun *
start_namespace(u_int id)421 nvmf_controller::start_namespace(u_int id)
422 {
423 if (id == 0) {
424 log_warnx("namespace ID cannot be 0 for %s", label());
425 return nullptr;
426 }
427
428 std::string lun_label = "namespace ID " + std::to_string(id - 1);
429 std::string lun_name = freebsd::stringf("%s,nsid,%u", name(), id);
430 return target::start_lun(id, lun_label.c_str(), lun_name.c_str());
431 }
432
433 struct portal_group *
default_portal_group()434 nvmf_controller::default_portal_group()
435 {
436 return t_conf->find_transport_group("default");
437 }
438
439 void
handle_connection(freebsd::fd_up fd,const char * host __unused,const struct sockaddr * client_sa __unused)440 nvmf_io_portal::handle_connection(freebsd::fd_up fd, const char *host __unused,
441 const struct sockaddr *client_sa __unused)
442 {
443 struct nvmf_qpair_params qparams;
444 memset(&qparams, 0, sizeof(qparams));
445 qparams.tcp.fd = fd;
446
447 struct nvmf_capsule *nc = NULL;
448 struct nvmf_fabric_connect_data data;
449 nvmf_qpair_up qp(nvmf_accept(association(), &qparams, &nc, &data));
450 if (!qp) {
451 log_warnx("Failed to create NVMe I/O qpair: %s",
452 nvmf_association_error(association()));
453 return;
454 }
455 nvmf_capsule_up nc_guard(nc);
456 const struct nvmf_fabric_connect_cmd *cmd =
457 (const struct nvmf_fabric_connect_cmd *)nvmf_capsule_sqe(nc);
458
459 struct ctl_nvmf req;
460 memset(&req, 0, sizeof(req));
461 req.type = CTL_NVMF_HANDOFF;
462 int error = nvmf_handoff_controller_qpair(qp.get(), cmd, &data,
463 &req.data.handoff);
464 if (error != 0) {
465 log_warnc(error,
466 "Failed to prepare NVMe I/O qpair for handoff");
467 return;
468 }
469
470 if (ioctl(ctl_fd, CTL_NVMF, &req) != 0)
471 log_warn("ioctl(CTL_NVMF/CTL_NVMF_HANDOFF)");
472 if (req.status == CTL_NVMF_ERROR)
473 log_warnx("Failed to handoff NVMF connection: %s",
474 req.error_str);
475 else if (req.status != CTL_NVMF_OK)
476 log_warnx("Failed to handoff NVMF connection with status %d",
477 req.status);
478 }
479