1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2025 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 #include <sys/linker.h> 10 #include <sys/module.h> 11 #include <sys/time.h> 12 #include <assert.h> 13 #include <ctype.h> 14 #include <errno.h> 15 #include <libiscsiutil.h> 16 #include <libnvmf.h> 17 #include <libutil.h> 18 #include <limits.h> 19 #include <stdbool.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <cam/ctl/ctl.h> 25 #include <cam/ctl/ctl_io.h> 26 #include <cam/ctl/ctl_ioctl.h> 27 28 #include <memory> 29 30 #include "ctld.hh" 31 #include "nvmf.hh" 32 33 #define DEFAULT_MAXH2CDATA (256 * 1024) 34 35 struct nvmf_io_portal final : public nvmf_portal { 36 nvmf_io_portal(struct portal_group *pg, const char *listen, 37 portal_protocol protocol, freebsd::addrinfo_up ai) : 38 nvmf_portal(pg, listen, protocol, std::move(ai)) {} 39 40 void handle_connection(freebsd::fd_up fd, const char *host, 41 const struct sockaddr *client_sa) override; 42 }; 43 44 struct nvmf_transport_group final : public portal_group { 45 nvmf_transport_group(struct conf *conf, std::string_view name) : 46 portal_group(conf, name) {} 47 48 const char *keyword() const override 49 { return "transport-group"; } 50 51 void allocate_tag() override; 52 bool add_portal(const char *value, portal_protocol protocol) 53 override; 54 void add_default_portals() override; 55 bool set_filter(const char *str) override; 56 57 virtual port_up create_port(struct target *target, auth_group_sp ag) 58 override; 59 virtual port_up create_port(struct target *target, uint32_t ctl_port) 60 override; 61 62 private: 63 static uint16_t last_port_id; 64 }; 65 66 struct nvmf_port final : public portal_group_port { 67 nvmf_port(struct target *target, struct portal_group *pg, 68 auth_group_sp ag) : 69 portal_group_port(target, pg, ag) {} 70 nvmf_port(struct target *target, struct portal_group *pg, 71 uint32_t ctl_port) : 72 portal_group_port(target, pg, ctl_port) {} 73 74 bool kernel_create_port() override; 75 bool kernel_remove_port() override; 76 77 private: 78 static bool modules_loaded; 79 static void load_kernel_modules(); 80 }; 81 82 struct nvmf_controller final : public target { 83 nvmf_controller(struct conf *conf, std::string_view name) : 84 target(conf, "controller", name) {} 85 86 bool add_host_nqn(std::string_view name) override; 87 bool add_host_address(const char *addr) override; 88 bool add_namespace(u_int id, const char *lun_name) override; 89 bool add_portal_group(const char *pg_name, const char *ag_name) 90 override; 91 struct lun *start_namespace(u_int id) override; 92 93 protected: 94 struct portal_group *default_portal_group() override; 95 }; 96 97 uint16_t nvmf_transport_group::last_port_id = 0; 98 bool nvmf_port::modules_loaded = false; 99 100 static bool need_tcp_transport = false; 101 102 static bool 103 parse_bool(const nvlist_t *nvl, const char *key, bool def) 104 { 105 const char *value; 106 107 if (!nvlist_exists_string(nvl, key)) 108 return def; 109 110 value = nvlist_get_string(nvl, key); 111 if (strcasecmp(value, "true") == 0 || 112 strcasecmp(value, "1") == 0) 113 return true; 114 if (strcasecmp(value, "false") == 0 || 115 strcasecmp(value, "0") == 0) 116 return false; 117 118 log_warnx("Invalid value \"%s\" for boolean option %s", value, key); 119 return def; 120 } 121 122 static uint64_t 123 parse_number(const nvlist_t *nvl, const char *key, uint64_t def, uint64_t minv, 124 uint64_t maxv) 125 { 126 const char *value; 127 int64_t val; 128 129 if (!nvlist_exists_string(nvl, key)) 130 return def; 131 132 value = nvlist_get_string(nvl, key); 133 if (expand_number(value, &val) == 0 && val >= 0 && 134 (uint64_t)val >= minv && (uint64_t)val <= maxv) 135 return (uint64_t)val; 136 137 log_warnx("Invalid value \"%s\" for numeric option %s", value, key); 138 return def; 139 } 140 141 bool 142 nvmf_portal::prepare() 143 { 144 memset(&p_aparams, 0, sizeof(p_aparams)); 145 146 /* Options shared between discovery and I/O associations. */ 147 freebsd::nvlist_up nvl = portal_group()->options(); 148 p_aparams.tcp.header_digests = parse_bool(nvl.get(), "HDGST", false); 149 p_aparams.tcp.data_digests = parse_bool(nvl.get(), "DDGST", false); 150 uint64_t value = parse_number(nvl.get(), "MAXH2CDATA", 151 DEFAULT_MAXH2CDATA, 4096, UINT32_MAX); 152 if (value % 4 != 0) { 153 log_warnx("Invalid value \"%ju\" for option MAXH2CDATA", 154 (uintmax_t)value); 155 value = DEFAULT_MAXH2CDATA; 156 } 157 p_aparams.tcp.maxh2cdata = value; 158 159 switch (protocol()) { 160 case portal_protocol::NVME_TCP: 161 p_aparams.sq_flow_control = parse_bool(nvl.get(), "SQFC", 162 false); 163 p_aparams.dynamic_controller_model = true; 164 p_aparams.max_admin_qsize = parse_number(nvl.get(), 165 "max_admin_qsize", NVME_MAX_ADMIN_ENTRIES, 166 NVME_MIN_ADMIN_ENTRIES, NVME_MAX_ADMIN_ENTRIES); 167 p_aparams.max_io_qsize = parse_number(nvl.get(), "max_io_qsize", 168 NVMF_MAX_IO_ENTRIES, NVME_MIN_IO_ENTRIES, 169 NVME_MAX_IO_ENTRIES); 170 p_aparams.tcp.pda = 0; 171 break; 172 case portal_protocol::NVME_DISCOVERY_TCP: 173 p_aparams.sq_flow_control = false; 174 p_aparams.dynamic_controller_model = true; 175 p_aparams.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES; 176 p_aparams.tcp.pda = 0; 177 break; 178 default: 179 __assert_unreachable(); 180 } 181 182 p_association.reset(nvmf_allocate_association(NVMF_TRTYPE_TCP, true, 183 &p_aparams)); 184 if (!p_association) { 185 log_warn("Failed to create NVMe controller association"); 186 return false; 187 } 188 189 return true; 190 } 191 192 portal_group_up 193 nvmf_make_transport_group(struct conf *conf, std::string_view name) 194 { 195 return std::make_unique<nvmf_transport_group>(conf, name); 196 } 197 198 target_up 199 nvmf_make_controller(struct conf *conf, std::string_view name) 200 { 201 return std::make_unique<nvmf_controller>(conf, name); 202 } 203 204 void 205 nvmf_transport_group::allocate_tag() 206 { 207 set_tag(++last_port_id); 208 } 209 210 bool 211 nvmf_transport_group::add_portal(const char *value, portal_protocol protocol) 212 { 213 freebsd::addrinfo_up ai; 214 215 switch (protocol) { 216 case portal_protocol::NVME_TCP: 217 ai = parse_addr_port(value, "4420"); 218 break; 219 case portal_protocol::NVME_DISCOVERY_TCP: 220 ai = parse_addr_port(value, "8009"); 221 break; 222 default: 223 log_warnx("unsupported transport protocol for %s", value); 224 return false; 225 } 226 227 if (!ai) { 228 log_warnx("invalid listen address %s", value); 229 return false; 230 } 231 232 /* 233 * XXX: getaddrinfo(3) may return multiple addresses; we should turn 234 * those into multiple portals. 235 */ 236 237 portal_up portal; 238 if (protocol == portal_protocol::NVME_DISCOVERY_TCP) { 239 portal = std::make_unique<nvmf_discovery_portal>(this, value, 240 protocol, std::move(ai)); 241 } else { 242 portal = std::make_unique<nvmf_io_portal>(this, value, 243 protocol, std::move(ai)); 244 need_tcp_transport = true; 245 } 246 247 pg_portals.emplace_back(std::move(portal)); 248 return true; 249 } 250 251 void 252 nvmf_transport_group::add_default_portals() 253 { 254 add_portal("0.0.0.0", portal_protocol::NVME_DISCOVERY_TCP); 255 add_portal("[::]", portal_protocol::NVME_DISCOVERY_TCP); 256 add_portal("0.0.0.0", portal_protocol::NVME_TCP); 257 add_portal("[::]", portal_protocol::NVME_TCP); 258 } 259 260 bool 261 nvmf_transport_group::set_filter(const char *str) 262 { 263 enum discovery_filter filter; 264 265 if (strcmp(str, "none") == 0) { 266 filter = discovery_filter::NONE; 267 } else if (strcmp(str, "address") == 0) { 268 filter = discovery_filter::PORTAL; 269 } else if (strcmp(str, "address-name") == 0) { 270 filter = discovery_filter::PORTAL_NAME; 271 } else { 272 log_warnx("invalid discovery-filter \"%s\" for transport-group " 273 "\"%s\"; valid values are \"none\", \"address\", " 274 "and \"address-name\"", 275 str, name()); 276 return false; 277 } 278 279 if (pg_discovery_filter != discovery_filter::UNKNOWN && 280 pg_discovery_filter != filter) { 281 log_warnx("cannot set discovery-filter to \"%s\" for " 282 "transport-group \"%s\"; already has a different " 283 "value", str, name()); 284 return false; 285 } 286 287 pg_discovery_filter = filter; 288 return true; 289 } 290 291 port_up 292 nvmf_transport_group::create_port(struct target *target, auth_group_sp ag) 293 { 294 return std::make_unique<nvmf_port>(target, this, ag); 295 } 296 297 port_up 298 nvmf_transport_group::create_port(struct target *target, uint32_t ctl_port) 299 { 300 return std::make_unique<nvmf_port>(target, this, ctl_port); 301 } 302 303 void 304 nvmf_port::load_kernel_modules() 305 { 306 int saved_errno; 307 308 if (modules_loaded) 309 return; 310 311 saved_errno = errno; 312 if (modfind("nvmft") == -1 && kldload("nvmft") == -1) 313 log_warn("couldn't load nvmft"); 314 315 if (need_tcp_transport) { 316 if (modfind("nvmf/tcp") == -1 && kldload("nvmf_tcp") == -1) 317 log_warn("couldn't load nvmf_tcp"); 318 } 319 320 errno = saved_errno; 321 modules_loaded = true; 322 } 323 324 bool 325 nvmf_port::kernel_create_port() 326 { 327 struct portal_group *pg = p_portal_group; 328 struct target *targ = p_target; 329 330 load_kernel_modules(); 331 332 freebsd::nvlist_up nvl = pg->options(); 333 nvlist_add_string(nvl.get(), "subnqn", targ->name()); 334 nvlist_add_string(nvl.get(), "ctld_transport_group_name", 335 pg->name()); 336 nvlist_add_stringf(nvl.get(), "portid", "%u", pg->tag()); 337 338 return ctl_create_port("nvmf", nvl.get(), &p_ctl_port); 339 } 340 341 bool 342 nvmf_port::kernel_remove_port() 343 { 344 freebsd::nvlist_up nvl(nvlist_create(0)); 345 nvlist_add_string(nvl.get(), "subnqn", p_target->name()); 346 347 return ctl_remove_port("nvmf", nvl.get()); 348 } 349 350 bool 351 nvmf_controller::add_host_nqn(std::string_view name) 352 { 353 if (!use_private_auth("host-nqn")) 354 return false; 355 return t_auth_group->add_host_nqn(name); 356 } 357 358 bool 359 nvmf_controller::add_host_address(const char *addr) 360 { 361 if (!use_private_auth("host-address")) 362 return false; 363 return t_auth_group->add_host_address(addr); 364 } 365 366 bool 367 nvmf_controller::add_namespace(u_int id, const char *lun_name) 368 { 369 if (id == 0) { 370 log_warnx("namespace ID cannot be 0 for %s", label()); 371 return false; 372 } 373 374 std::string lun_label = "namespace ID " + std::to_string(id - 1); 375 return target::add_lun(id, lun_label.c_str(), lun_name); 376 } 377 378 bool 379 nvmf_controller::add_portal_group(const char *pg_name, const char *ag_name) 380 { 381 struct portal_group *pg; 382 auth_group_sp ag; 383 384 pg = t_conf->find_transport_group(pg_name); 385 if (pg == NULL) { 386 log_warnx("unknown transport-group \"%s\" for %s", pg_name, 387 label()); 388 return false; 389 } 390 391 if (ag_name != NULL) { 392 ag = t_conf->find_auth_group(ag_name); 393 if (ag == NULL) { 394 log_warnx("unknown auth-group \"%s\" for %s", ag_name, 395 label()); 396 return false; 397 } 398 } 399 400 if (!t_conf->add_port(this, pg, std::move(ag))) { 401 log_warnx("can't link transport-group \"%s\" to %s", pg_name, 402 label()); 403 return false; 404 } 405 return true; 406 } 407 408 struct lun * 409 nvmf_controller::start_namespace(u_int id) 410 { 411 if (id == 0) { 412 log_warnx("namespace ID cannot be 0 for %s", label()); 413 return nullptr; 414 } 415 416 std::string lun_label = "namespace ID " + std::to_string(id - 1); 417 std::string lun_name = freebsd::stringf("%s,nsid,%u", name(), id); 418 return target::start_lun(id, lun_label.c_str(), lun_name.c_str()); 419 } 420 421 struct portal_group * 422 nvmf_controller::default_portal_group() 423 { 424 return t_conf->find_transport_group("default"); 425 } 426 427 void 428 nvmf_io_portal::handle_connection(freebsd::fd_up fd, const char *host __unused, 429 const struct sockaddr *client_sa __unused) 430 { 431 struct nvmf_qpair_params qparams; 432 memset(&qparams, 0, sizeof(qparams)); 433 qparams.tcp.fd = fd; 434 435 struct nvmf_capsule *nc = NULL; 436 struct nvmf_fabric_connect_data data; 437 nvmf_qpair_up qp(nvmf_accept(association(), &qparams, &nc, &data)); 438 if (!qp) { 439 log_warnx("Failed to create NVMe I/O qpair: %s", 440 nvmf_association_error(association())); 441 return; 442 } 443 nvmf_capsule_up nc_guard(nc); 444 const struct nvmf_fabric_connect_cmd *cmd = 445 (const struct nvmf_fabric_connect_cmd *)nvmf_capsule_sqe(nc); 446 447 struct ctl_nvmf req; 448 memset(&req, 0, sizeof(req)); 449 req.type = CTL_NVMF_HANDOFF; 450 int error = nvmf_handoff_controller_qpair(qp.get(), cmd, &data, 451 &req.data.handoff); 452 if (error != 0) { 453 log_warnc(error, 454 "Failed to prepare NVMe I/O qpair for handoff"); 455 return; 456 } 457 458 if (ioctl(ctl_fd, CTL_NVMF, &req) != 0) 459 log_warn("ioctl(CTL_NVMF/CTL_NVMF_HANDOFF)"); 460 if (req.status == CTL_NVMF_ERROR) 461 log_warnx("Failed to handoff NVMF connection: %s", 462 req.error_str); 463 else if (req.status != CTL_NVMF_OK) 464 log_warnx("Failed to handoff NVMF connection with status %d", 465 req.status); 466 } 467