1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2025 Chelsio Communications, Inc. 5 * Written by: John Baldwin <jhb@FreeBSD.org> 6 */ 7 8 #include <sys/param.h> 9 #include <sys/linker.h> 10 #include <sys/module.h> 11 #include <sys/time.h> 12 #include <assert.h> 13 #include <ctype.h> 14 #include <errno.h> 15 #include <libiscsiutil.h> 16 #include <libnvmf.h> 17 #include <libutil.h> 18 #include <limits.h> 19 #include <stdbool.h> 20 #include <stdio.h> 21 #include <stdlib.h> 22 #include <string.h> 23 #include <unistd.h> 24 #include <cam/ctl/ctl.h> 25 #include <cam/ctl/ctl_io.h> 26 #include <cam/ctl/ctl_ioctl.h> 27 28 #include <memory> 29 30 #include "ctld.hh" 31 #include "nvmf.hh" 32 33 #define DEFAULT_MAXH2CDATA (256 * 1024) 34 35 struct nvmf_io_portal final : public nvmf_portal { 36 nvmf_io_portal(struct portal_group *pg, const char *listen, 37 portal_protocol protocol, freebsd::addrinfo_up ai, 38 const struct nvmf_association_params &aparams, 39 nvmf_association_up na) : 40 nvmf_portal(pg, listen, protocol, std::move(ai), aparams, 41 std::move(na)) {} 42 43 void handle_connection(freebsd::fd_up fd, const char *host, 44 const struct sockaddr *client_sa) override; 45 }; 46 47 struct nvmf_transport_group final : public portal_group { 48 nvmf_transport_group(struct conf *conf, std::string_view name) : 49 portal_group(conf, name) {} 50 51 const char *keyword() const override 52 { return "transport-group"; } 53 54 void allocate_tag() override; 55 bool add_portal(const char *value, portal_protocol protocol) 56 override; 57 void add_default_portals() override; 58 bool set_filter(const char *str) override; 59 60 virtual port_up create_port(struct target *target, auth_group_sp ag) 61 override; 62 virtual port_up create_port(struct target *target, uint32_t ctl_port) 63 override; 64 65 private: 66 struct nvmf_association_params init_aparams(portal_protocol protocol); 67 68 static uint16_t last_port_id; 69 }; 70 71 struct nvmf_port final : public portal_group_port { 72 nvmf_port(struct target *target, struct portal_group *pg, 73 auth_group_sp ag) : 74 portal_group_port(target, pg, ag) {} 75 nvmf_port(struct target *target, struct portal_group *pg, 76 uint32_t ctl_port) : 77 portal_group_port(target, pg, ctl_port) {} 78 79 bool kernel_create_port() override; 80 bool kernel_remove_port() override; 81 82 private: 83 static bool modules_loaded; 84 static void load_kernel_modules(); 85 }; 86 87 struct nvmf_controller final : public target { 88 nvmf_controller(struct conf *conf, std::string_view name) : 89 target(conf, "controller", name) {} 90 91 bool add_host_nqn(std::string_view name) override; 92 bool add_host_address(const char *addr) override; 93 bool add_namespace(u_int id, const char *lun_name) override; 94 bool add_portal_group(const char *pg_name, const char *ag_name) 95 override; 96 struct lun *start_namespace(u_int id) override; 97 98 protected: 99 struct portal_group *default_portal_group() override; 100 }; 101 102 uint16_t nvmf_transport_group::last_port_id = 0; 103 bool nvmf_port::modules_loaded = false; 104 105 static bool need_tcp_transport = false; 106 107 static bool 108 parse_bool(const nvlist_t *nvl, const char *key, bool def) 109 { 110 const char *value; 111 112 if (!nvlist_exists_string(nvl, key)) 113 return def; 114 115 value = nvlist_get_string(nvl, key); 116 if (strcasecmp(value, "true") == 0 || 117 strcasecmp(value, "1") == 0) 118 return true; 119 if (strcasecmp(value, "false") == 0 || 120 strcasecmp(value, "0") == 0) 121 return false; 122 123 log_warnx("Invalid value \"%s\" for boolean option %s", value, key); 124 return def; 125 } 126 127 static uint64_t 128 parse_number(const nvlist_t *nvl, const char *key, uint64_t def, uint64_t minv, 129 uint64_t maxv) 130 { 131 const char *value; 132 int64_t val; 133 134 if (!nvlist_exists_string(nvl, key)) 135 return def; 136 137 value = nvlist_get_string(nvl, key); 138 if (expand_number(value, &val) == 0 && val >= 0 && 139 (uint64_t)val >= minv && (uint64_t)val <= maxv) 140 return (uint64_t)val; 141 142 log_warnx("Invalid value \"%s\" for numeric option %s", value, key); 143 return def; 144 } 145 146 struct nvmf_association_params 147 nvmf_transport_group::init_aparams(portal_protocol protocol) 148 { 149 struct nvmf_association_params params; 150 memset(¶ms, 0, sizeof(params)); 151 152 /* Options shared between discovery and I/O associations. */ 153 const nvlist_t *nvl = pg_options.get(); 154 params.tcp.header_digests = parse_bool(nvl, "HDGST", false); 155 params.tcp.data_digests = parse_bool(nvl, "DDGST", false); 156 uint64_t value = parse_number(nvl, "MAXH2CDATA", DEFAULT_MAXH2CDATA, 157 4096, UINT32_MAX); 158 if (value % 4 != 0) { 159 log_warnx("Invalid value \"%ju\" for option MAXH2CDATA", 160 (uintmax_t)value); 161 value = DEFAULT_MAXH2CDATA; 162 } 163 params.tcp.maxh2cdata = value; 164 165 switch (protocol) { 166 case portal_protocol::NVME_TCP: 167 params.sq_flow_control = parse_bool(nvl, "SQFC", false); 168 params.dynamic_controller_model = true; 169 params.max_admin_qsize = parse_number(nvl, "max_admin_qsize", 170 NVME_MAX_ADMIN_ENTRIES, NVME_MIN_ADMIN_ENTRIES, 171 NVME_MAX_ADMIN_ENTRIES); 172 params.max_io_qsize = parse_number(nvl, "max_io_qsize", 173 NVME_MAX_IO_ENTRIES, NVME_MIN_IO_ENTRIES, 174 NVME_MAX_IO_ENTRIES); 175 params.tcp.pda = 0; 176 break; 177 case portal_protocol::NVME_DISCOVERY_TCP: 178 params.sq_flow_control = false; 179 params.dynamic_controller_model = true; 180 params.max_admin_qsize = NVME_MAX_ADMIN_ENTRIES; 181 params.tcp.pda = 0; 182 break; 183 default: 184 __assert_unreachable(); 185 } 186 187 return params; 188 } 189 190 portal_group_up 191 nvmf_make_transport_group(struct conf *conf, std::string_view name) 192 { 193 return std::make_unique<nvmf_transport_group>(conf, name); 194 } 195 196 target_up 197 nvmf_make_controller(struct conf *conf, std::string_view name) 198 { 199 return std::make_unique<nvmf_controller>(conf, name); 200 } 201 202 void 203 nvmf_transport_group::allocate_tag() 204 { 205 set_tag(++last_port_id); 206 } 207 208 bool 209 nvmf_transport_group::add_portal(const char *value, portal_protocol protocol) 210 { 211 freebsd::addrinfo_up ai; 212 enum nvmf_trtype trtype; 213 214 switch (protocol) { 215 case portal_protocol::NVME_TCP: 216 trtype = NVMF_TRTYPE_TCP; 217 ai = parse_addr_port(value, "4420"); 218 break; 219 case portal_protocol::NVME_DISCOVERY_TCP: 220 trtype = NVMF_TRTYPE_TCP; 221 ai = parse_addr_port(value, "8009"); 222 break; 223 default: 224 log_warnx("unsupported transport protocol for %s", value); 225 return false; 226 } 227 228 if (!ai) { 229 log_warnx("invalid listen address %s", value); 230 return false; 231 } 232 233 struct nvmf_association_params aparams = init_aparams(protocol); 234 nvmf_association_up association(nvmf_allocate_association(trtype, true, 235 &aparams)); 236 if (!association) { 237 log_warn("Failed to create NVMe controller association"); 238 return false; 239 } 240 241 /* 242 * XXX: getaddrinfo(3) may return multiple addresses; we should turn 243 * those into multiple portals. 244 */ 245 246 portal_up portal; 247 if (protocol == portal_protocol::NVME_DISCOVERY_TCP) { 248 portal = std::make_unique<nvmf_discovery_portal>(this, value, 249 protocol, std::move(ai), aparams, std::move(association)); 250 } else { 251 portal = std::make_unique<nvmf_io_portal>(this, value, 252 protocol, std::move(ai), aparams, std::move(association)); 253 need_tcp_transport = true; 254 } 255 256 pg_portals.emplace_back(std::move(portal)); 257 return true; 258 } 259 260 void 261 nvmf_transport_group::add_default_portals() 262 { 263 add_portal("0.0.0.0", portal_protocol::NVME_DISCOVERY_TCP); 264 add_portal("[::]", portal_protocol::NVME_DISCOVERY_TCP); 265 add_portal("0.0.0.0", portal_protocol::NVME_TCP); 266 add_portal("[::]", portal_protocol::NVME_TCP); 267 } 268 269 bool 270 nvmf_transport_group::set_filter(const char *str) 271 { 272 enum discovery_filter filter; 273 274 if (strcmp(str, "none") == 0) { 275 filter = discovery_filter::NONE; 276 } else if (strcmp(str, "address") == 0) { 277 filter = discovery_filter::PORTAL; 278 } else if (strcmp(str, "address-name") == 0) { 279 filter = discovery_filter::PORTAL_NAME; 280 } else { 281 log_warnx("invalid discovery-filter \"%s\" for transport-group " 282 "\"%s\"; valid values are \"none\", \"address\", " 283 "and \"address-name\"", 284 str, name()); 285 return false; 286 } 287 288 if (pg_discovery_filter != discovery_filter::UNKNOWN && 289 pg_discovery_filter != filter) { 290 log_warnx("cannot set discovery-filter to \"%s\" for " 291 "transport-group \"%s\"; already has a different " 292 "value", str, name()); 293 return false; 294 } 295 296 pg_discovery_filter = filter; 297 return true; 298 } 299 300 port_up 301 nvmf_transport_group::create_port(struct target *target, auth_group_sp ag) 302 { 303 return std::make_unique<nvmf_port>(target, this, ag); 304 } 305 306 port_up 307 nvmf_transport_group::create_port(struct target *target, uint32_t ctl_port) 308 { 309 return std::make_unique<nvmf_port>(target, this, ctl_port); 310 } 311 312 void 313 nvmf_port::load_kernel_modules() 314 { 315 int saved_errno; 316 317 if (modules_loaded) 318 return; 319 320 saved_errno = errno; 321 if (modfind("nvmft") == -1 && kldload("nvmft") == -1) 322 log_warn("couldn't load nvmft"); 323 324 if (need_tcp_transport) { 325 if (modfind("nvmf/tcp") == -1 && kldload("nvmf_tcp") == -1) 326 log_warn("couldn't load nvmf_tcp"); 327 } 328 329 errno = saved_errno; 330 modules_loaded = true; 331 } 332 333 bool 334 nvmf_port::kernel_create_port() 335 { 336 struct portal_group *pg = p_portal_group; 337 struct target *targ = p_target; 338 339 load_kernel_modules(); 340 341 freebsd::nvlist_up nvl = pg->options(); 342 nvlist_add_string(nvl.get(), "subnqn", targ->name()); 343 nvlist_add_string(nvl.get(), "ctld_transport_group_name", 344 pg->name()); 345 nvlist_add_stringf(nvl.get(), "portid", "%u", pg->tag()); 346 if (!nvlist_exists_string(nvl.get(), "max_io_qsize")) 347 nvlist_add_stringf(nvl.get(), "max_io_qsize", "%u", 348 NVME_MAX_IO_ENTRIES); 349 350 return ctl_create_port("nvmf", nvl.get(), &p_ctl_port); 351 } 352 353 bool 354 nvmf_port::kernel_remove_port() 355 { 356 freebsd::nvlist_up nvl(nvlist_create(0)); 357 nvlist_add_string(nvl.get(), "subnqn", p_target->name()); 358 359 return ctl_remove_port("nvmf", nvl.get()); 360 } 361 362 bool 363 nvmf_controller::add_host_nqn(std::string_view name) 364 { 365 if (!use_private_auth("host-nqn")) 366 return false; 367 return t_auth_group->add_host_nqn(name); 368 } 369 370 bool 371 nvmf_controller::add_host_address(const char *addr) 372 { 373 if (!use_private_auth("host-address")) 374 return false; 375 return t_auth_group->add_host_address(addr); 376 } 377 378 bool 379 nvmf_controller::add_namespace(u_int id, const char *lun_name) 380 { 381 if (id == 0) { 382 log_warnx("namespace ID cannot be 0 for %s", label()); 383 return false; 384 } 385 386 std::string lun_label = "namespace ID " + std::to_string(id - 1); 387 return target::add_lun(id, lun_label.c_str(), lun_name); 388 } 389 390 bool 391 nvmf_controller::add_portal_group(const char *pg_name, const char *ag_name) 392 { 393 struct portal_group *pg; 394 auth_group_sp ag; 395 396 pg = t_conf->find_transport_group(pg_name); 397 if (pg == NULL) { 398 log_warnx("unknown transport-group \"%s\" for %s", pg_name, 399 label()); 400 return false; 401 } 402 403 if (ag_name != NULL) { 404 ag = t_conf->find_auth_group(ag_name); 405 if (ag == NULL) { 406 log_warnx("unknown auth-group \"%s\" for %s", ag_name, 407 label()); 408 return false; 409 } 410 } 411 412 if (!t_conf->add_port(this, pg, std::move(ag))) { 413 log_warnx("can't link transport-group \"%s\" to %s", pg_name, 414 label()); 415 return false; 416 } 417 return true; 418 } 419 420 struct lun * 421 nvmf_controller::start_namespace(u_int id) 422 { 423 if (id == 0) { 424 log_warnx("namespace ID cannot be 0 for %s", label()); 425 return nullptr; 426 } 427 428 std::string lun_label = "namespace ID " + std::to_string(id - 1); 429 std::string lun_name = freebsd::stringf("%s,nsid,%u", name(), id); 430 return target::start_lun(id, lun_label.c_str(), lun_name.c_str()); 431 } 432 433 struct portal_group * 434 nvmf_controller::default_portal_group() 435 { 436 return t_conf->find_transport_group("default"); 437 } 438 439 void 440 nvmf_io_portal::handle_connection(freebsd::fd_up fd, const char *host __unused, 441 const struct sockaddr *client_sa __unused) 442 { 443 struct nvmf_qpair_params qparams; 444 memset(&qparams, 0, sizeof(qparams)); 445 qparams.tcp.fd = fd; 446 447 struct nvmf_capsule *nc = NULL; 448 struct nvmf_fabric_connect_data data; 449 nvmf_qpair_up qp(nvmf_accept(association(), &qparams, &nc, &data)); 450 if (!qp) { 451 log_warnx("Failed to create NVMe I/O qpair: %s", 452 nvmf_association_error(association())); 453 return; 454 } 455 nvmf_capsule_up nc_guard(nc); 456 const struct nvmf_fabric_connect_cmd *cmd = 457 (const struct nvmf_fabric_connect_cmd *)nvmf_capsule_sqe(nc); 458 459 struct ctl_nvmf req; 460 memset(&req, 0, sizeof(req)); 461 req.type = CTL_NVMF_HANDOFF; 462 int error = nvmf_handoff_controller_qpair(qp.get(), cmd, &data, 463 &req.data.handoff); 464 if (error != 0) { 465 log_warnc(error, 466 "Failed to prepare NVMe I/O qpair for handoff"); 467 return; 468 } 469 470 if (ioctl(ctl_fd, CTL_NVMF, &req) != 0) 471 log_warn("ioctl(CTL_NVMF/CTL_NVMF_HANDOFF)"); 472 if (req.status == CTL_NVMF_ERROR) 473 log_warnx("Failed to handoff NVMF connection: %s", 474 req.error_str); 475 else if (req.status != CTL_NVMF_OK) 476 log_warnx("Failed to handoff NVMF connection with status %d", 477 req.status); 478 } 479