1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2025 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 #include <sys/linker.h> 10 #include <sys/module.h> 11 #include <sys/time.h> 12 #include <assert.h> 13 #include <ctype.h> 14 #include <errno.h> 15 #include <libiscsiutil.h> 16 #include <libnvmf.h> 17 #include <libutil.h> 18 #include <limits.h> 19 #include <stdbool.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <cam/ctl/ctl.h> 25 #include <cam/ctl/ctl_io.h> 26 #include <cam/ctl/ctl_ioctl.h> 27 28 #include <memory> 29 30 #include "ctld.hh" 31 #include "nvmf.hh" 32 33 #define DEFAULT_MAXH2CDATA (256 * 1024) 34 35 struct nvmf_io_portal final : public nvmf_portal { 36 nvmf_io_portal(struct portal_group *pg, const char *listen, 37 portal_protocol protocol, freebsd::addrinfo_up ai) : 38 nvmf_portal(pg, listen, protocol, std::move(ai)) {} 39 40 void handle_connection(freebsd::fd_up fd, const char *host, 41 const struct sockaddr *client_sa) override; 42 }; 43 44 struct nvmf_transport_group final : public portal_group { 45 nvmf_transport_group(struct conf *conf, std::string_view name) : 46 portal_group(conf, name) {} 47 48 const char *keyword() const override 49 { return "transport-group"; } 50 51 void allocate_tag() override; 52 bool add_portal(const char *value, portal_protocol protocol) 53 override; 54 void add_default_portals() override; 55 bool set_filter(const char *str) override; 56 57 virtual port_up create_port(struct target *target, auth_group_sp ag) 58 override; 59 virtual port_up create_port(struct target *target, uint32_t ctl_port) 60 override; 61 62 private: 63 static uint16_t last_port_id; 64 }; 65 66 struct nvmf_port final : public portal_group_port { 67 nvmf_port(struct target *target, struct portal_group *pg, 68 auth_group_sp ag) : 69 portal_group_port(target, pg, ag) {} 70 nvmf_port(struct target *target, struct portal_group *pg, 71 uint32_t ctl_port) : 72 portal_group_port(target, pg, ctl_port) {} 73 74 bool kernel_create_port() override; 75 bool kernel_remove_port() override; 76 77 private: 78 static bool modules_loaded; 79 static void load_kernel_modules(); 80 }; 81 82 struct nvmf_controller final : public target { 83 nvmf_controller(struct conf *conf, std::string_view name) : 84 target(conf, "controller", name) {} 85 86 bool add_host_nqn(std::string_view name) override; 87 bool add_host_address(const char *addr) override; 88 bool add_namespace(u_int id, const char *lun_name) override; 89 bool add_portal_group(const char *pg_name, const char *ag_name) 90 override; 91 struct lun *start_namespace(u_int id) override; 92 93 protected: 94 struct portal_group *default_portal_group() override; 95 }; 96 97 uint16_t nvmf_transport_group::last_port_id = 0; 98 bool nvmf_port::modules_loaded = false; 99 100 static bool need_tcp_transport = false; 101 102 static bool 103 parse_bool(const nvlist_t *nvl, const char *key, bool def) 104 { 105 const char *value; 106 107 if (!nvlist_exists_string(nvl, key)) 108 return def; 109 110 value = nvlist_get_string(nvl, key); 111 if (strcasecmp(value, "true") == 0 || 112 strcasecmp(value, "1") == 0) 113 return true; 114 if (strcasecmp(value, "false") == 0 || 115 strcasecmp(value, "0") == 0) 116 return false; 117 118 log_warnx("Invalid value \"%s\" for boolean option %s", value, key); 119 return def; 120 } 121 122 static uint64_t 123 parse_number(const nvlist_t *nvl, const char *key, uint64_t def, uint64_t minv, 124 uint64_t maxv) 125 { 126 const char *value; 127 int64_t val; 128 129 if (!nvlist_exists_string(nvl, key)) 130 return def; 131 132 value = nvlist_get_string(nvl, key); 133 if (expand_number(value, &val) == 0 && val >= 0 && 134 (uint64_t)val >= minv && (uint64_t)val <= maxv) 135 return (uint64_t)val; 136 137 log_warnx("Invalid value \"%s\" for numeric option %s", value, key); 138 return def; 139 } 140 141 bool 142 nvmf_portal::prepare() 143 { 144 memset(&p_aparams, 0, sizeof(p_aparams)); 145 146 /* Options shared between discovery and I/O associations. */ 147 freebsd::nvlist_up nvl = portal_group()->options(); 148 p_aparams.tcp.header_digests = parse_bool(nvl.get(), "HDGST", false); 149 p_aparams.tcp.data_digests = parse_bool(nvl.get(), "DDGST", false); 150 uint64_t value = parse_number(nvl.get(), "MAXH2CDATA", 151 DEFAULT_MAXH2CDATA, 4096, UINT32_MAX); 152 if (value % 4 != 0) { 153 log_warnx("Invalid value \"%ju\" for option MAXH2CDATA", 154 (uintmax_t)value); 155 value = DEFAULT_MAXH2CDATA; 156 } 157 p_aparams.tcp.maxh2cdata = value; 158 159 switch (protocol()) { 160 case portal_protocol::NVME_TCP: 161 p_aparams.sq_flow_control = parse_bool(nvl.get(), "SQFC", 162 false); 163 p_aparams.dynamic_controller_model = true; 164 p_aparams.max_admin_qsize = parse_number(nvl.get(), 165 "max_admin_qsize", NVME_MAX_ADMIN_ENTRIES, 166 NVME_MIN_ADMIN_ENTRIES, NVME_MAX_ADMIN_ENTRIES); 167 p_aparams.max_io_qsize = parse_number(nvl.get(), "max_io_qsize", 168 NVME_MAX_IO_ENTRIES, NVME_MIN_IO_ENTRIES, 169 NVME_MAX_IO_ENTRIES); 170 p_aparams.tcp.pda = 0; 171 break; 172 case portal_protocol::NVME_DISCOVERY_TCP: 173 p_aparams.sq_flow_control = false; 174 p_aparams.dynamic_controller_model = true; 175 p_aparams.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES; 176 p_aparams.tcp.pda = 0; 177 break; 178 default: 179 __assert_unreachable(); 180 } 181 182 p_association.reset(nvmf_allocate_association(NVMF_TRTYPE_TCP, true, 183 &p_aparams)); 184 if (!p_association) { 185 log_warn("Failed to create NVMe controller association"); 186 return false; 187 } 188 189 return true; 190 } 191 192 portal_group_up 193 nvmf_make_transport_group(struct conf *conf, std::string_view name) 194 { 195 return std::make_unique<nvmf_transport_group>(conf, name); 196 } 197 198 target_up 199 nvmf_make_controller(struct conf *conf, std::string_view name) 200 { 201 return std::make_unique<nvmf_controller>(conf, name); 202 } 203 204 void 205 nvmf_transport_group::allocate_tag() 206 { 207 set_tag(++last_port_id); 208 } 209 210 bool 211 nvmf_transport_group::add_portal(const char *value, portal_protocol protocol) 212 { 213 freebsd::addrinfo_up ai; 214 215 switch (protocol) { 216 case portal_protocol::NVME_TCP: 217 ai = parse_addr_port(value, "4420"); 218 break; 219 case portal_protocol::NVME_DISCOVERY_TCP: 220 ai = parse_addr_port(value, "8009"); 221 break; 222 default: 223 log_warnx("unsupported transport protocol for %s", value); 224 return false; 225 } 226 227 if (!ai) { 228 log_warnx("invalid listen address %s", value); 229 return false; 230 } 231 232 /* 233 * XXX: getaddrinfo(3) may return multiple addresses; we should turn 234 * those into multiple portals. 235 */ 236 237 portal_up portal; 238 if (protocol == portal_protocol::NVME_DISCOVERY_TCP) { 239 portal = std::make_unique<nvmf_discovery_portal>(this, value, 240 protocol, std::move(ai)); 241 } else { 242 portal = std::make_unique<nvmf_io_portal>(this, value, 243 protocol, std::move(ai)); 244 need_tcp_transport = true; 245 } 246 247 pg_portals.emplace_back(std::move(portal)); 248 return true; 249 } 250 251 void 252 nvmf_transport_group::add_default_portals() 253 { 254 add_portal("0.0.0.0", portal_protocol::NVME_DISCOVERY_TCP); 255 add_portal("[::]", portal_protocol::NVME_DISCOVERY_TCP); 256 add_portal("0.0.0.0", portal_protocol::NVME_TCP); 257 add_portal("[::]", portal_protocol::NVME_TCP); 258 } 259 260 bool 261 nvmf_transport_group::set_filter(const char *str) 262 { 263 enum discovery_filter filter; 264 265 if (strcmp(str, "none") == 0) { 266 filter = discovery_filter::NONE; 267 } else if (strcmp(str, "address") == 0) { 268 filter = discovery_filter::PORTAL; 269 } else if (strcmp(str, "address-name") == 0) { 270 filter = discovery_filter::PORTAL_NAME; 271 } else { 272 log_warnx("invalid discovery-filter \"%s\" for transport-group " 273 "\"%s\"; valid values are \"none\", \"address\", " 274 "and \"address-name\"", 275 str, name()); 276 return false; 277 } 278 279 if (pg_discovery_filter != discovery_filter::UNKNOWN && 280 pg_discovery_filter != filter) { 281 log_warnx("cannot set discovery-filter to \"%s\" for " 282 "transport-group \"%s\"; already has a different " 283 "value", str, name()); 284 return false; 285 } 286 287 pg_discovery_filter = filter; 288 return true; 289 } 290 291 port_up 292 nvmf_transport_group::create_port(struct target *target, auth_group_sp ag) 293 { 294 return std::make_unique<nvmf_port>(target, this, ag); 295 } 296 297 port_up 298 nvmf_transport_group::create_port(struct target *target, uint32_t ctl_port) 299 { 300 return std::make_unique<nvmf_port>(target, this, ctl_port); 301 } 302 303 void 304 nvmf_port::load_kernel_modules() 305 { 306 int saved_errno; 307 308 if (modules_loaded) 309 return; 310 311 saved_errno = errno; 312 if (modfind("nvmft") == -1 && kldload("nvmft") == -1) 313 log_warn("couldn't load nvmft"); 314 315 if (need_tcp_transport) { 316 if (modfind("nvmf/tcp") == -1 && kldload("nvmf_tcp") == -1) 317 log_warn("couldn't load nvmf_tcp"); 318 } 319 320 errno = saved_errno; 321 modules_loaded = true; 322 } 323 324 bool 325 nvmf_port::kernel_create_port() 326 { 327 struct portal_group *pg = p_portal_group; 328 struct target *targ = p_target; 329 330 load_kernel_modules(); 331 332 freebsd::nvlist_up nvl = pg->options(); 333 nvlist_add_string(nvl.get(), "subnqn", targ->name()); 334 nvlist_add_string(nvl.get(), "ctld_transport_group_name", 335 pg->name()); 336 nvlist_add_stringf(nvl.get(), "portid", "%u", pg->tag()); 337 if (!nvlist_exists_string(nvl.get(), "max_io_qsize")) 338 nvlist_add_stringf(nvl.get(), "max_io_qsize", "%u", 339 NVME_MAX_IO_ENTRIES); 340 341 return ctl_create_port("nvmf", nvl.get(), &p_ctl_port); 342 } 343 344 bool 345 nvmf_port::kernel_remove_port() 346 { 347 freebsd::nvlist_up nvl(nvlist_create(0)); 348 nvlist_add_string(nvl.get(), "subnqn", p_target->name()); 349 350 return ctl_remove_port("nvmf", nvl.get()); 351 } 352 353 bool 354 nvmf_controller::add_host_nqn(std::string_view name) 355 { 356 if (!use_private_auth("host-nqn")) 357 return false; 358 return t_auth_group->add_host_nqn(name); 359 } 360 361 bool 362 nvmf_controller::add_host_address(const char *addr) 363 { 364 if (!use_private_auth("host-address")) 365 return false; 366 return t_auth_group->add_host_address(addr); 367 } 368 369 bool 370 nvmf_controller::add_namespace(u_int id, const char *lun_name) 371 { 372 if (id == 0) { 373 log_warnx("namespace ID cannot be 0 for %s", label()); 374 return false; 375 } 376 377 std::string lun_label = "namespace ID " + std::to_string(id - 1); 378 return target::add_lun(id, lun_label.c_str(), lun_name); 379 } 380 381 bool 382 nvmf_controller::add_portal_group(const char *pg_name, const char *ag_name) 383 { 384 struct portal_group *pg; 385 auth_group_sp ag; 386 387 pg = t_conf->find_transport_group(pg_name); 388 if (pg == NULL) { 389 log_warnx("unknown transport-group \"%s\" for %s", pg_name, 390 label()); 391 return false; 392 } 393 394 if (ag_name != NULL) { 395 ag = t_conf->find_auth_group(ag_name); 396 if (ag == NULL) { 397 log_warnx("unknown auth-group \"%s\" for %s", ag_name, 398 label()); 399 return false; 400 } 401 } 402 403 if (!t_conf->add_port(this, pg, std::move(ag))) { 404 log_warnx("can't link transport-group \"%s\" to %s", pg_name, 405 label()); 406 return false; 407 } 408 return true; 409 } 410 411 struct lun * 412 nvmf_controller::start_namespace(u_int id) 413 { 414 if (id == 0) { 415 log_warnx("namespace ID cannot be 0 for %s", label()); 416 return nullptr; 417 } 418 419 std::string lun_label = "namespace ID " + std::to_string(id - 1); 420 std::string lun_name = freebsd::stringf("%s,nsid,%u", name(), id); 421 return target::start_lun(id, lun_label.c_str(), lun_name.c_str()); 422 } 423 424 struct portal_group * 425 nvmf_controller::default_portal_group() 426 { 427 return t_conf->find_transport_group("default"); 428 } 429 430 void 431 nvmf_io_portal::handle_connection(freebsd::fd_up fd, const char *host __unused, 432 const struct sockaddr *client_sa __unused) 433 { 434 struct nvmf_qpair_params qparams; 435 memset(&qparams, 0, sizeof(qparams)); 436 qparams.tcp.fd = fd; 437 438 struct nvmf_capsule *nc = NULL; 439 struct nvmf_fabric_connect_data data; 440 nvmf_qpair_up qp(nvmf_accept(association(), &qparams, &nc, &data)); 441 if (!qp) { 442 log_warnx("Failed to create NVMe I/O qpair: %s", 443 nvmf_association_error(association())); 444 return; 445 } 446 nvmf_capsule_up nc_guard(nc); 447 const struct nvmf_fabric_connect_cmd *cmd = 448 (const struct nvmf_fabric_connect_cmd *)nvmf_capsule_sqe(nc); 449 450 struct ctl_nvmf req; 451 memset(&req, 0, sizeof(req)); 452 req.type = CTL_NVMF_HANDOFF; 453 int error = nvmf_handoff_controller_qpair(qp.get(), cmd, &data, 454 &req.data.handoff); 455 if (error != 0) { 456 log_warnc(error, 457 "Failed to prepare NVMe I/O qpair for handoff"); 458 return; 459 } 460 461 if (ioctl(ctl_fd, CTL_NVMF, &req) != 0) 462 log_warn("ioctl(CTL_NVMF/CTL_NVMF_HANDOFF)"); 463 if (req.status == CTL_NVMF_ERROR) 464 log_warnx("Failed to handoff NVMF connection: %s", 465 req.error_str); 466 else if (req.status != CTL_NVMF_OK) 467 log_warnx("Failed to handoff NVMF connection with status %d", 468 req.status); 469 } 470