1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics Persist Reservation. 4 * Copyright (c) 2024 Guixin Liu, Alibaba Group. 5 * All rights reserved. 6 */ 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 #include <linux/unaligned.h> 9 #include "nvmet.h" 10 11 #define NVMET_PR_NOTIFY_MASK_ALL \ 12 (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \ 13 1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \ 14 1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED) 15 16 static inline bool nvmet_pr_parse_ignore_key(u32 cdw10) 17 { 18 /* Ignore existing key, bit 03. */ 19 return (cdw10 >> 3) & 1; 20 } 21 22 static inline struct nvmet_ns *nvmet_pr_to_ns(struct nvmet_pr *pr) 23 { 24 return container_of(pr, struct nvmet_ns, pr); 25 } 26 27 static struct nvmet_pr_registrant * 28 nvmet_pr_find_registrant(struct nvmet_pr *pr, uuid_t *hostid) 29 { 30 struct nvmet_pr_registrant *reg; 31 32 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 33 if (uuid_equal(®->hostid, hostid)) 34 return reg; 35 } 36 return NULL; 37 } 38 39 u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask) 40 { 41 u32 nsid = le32_to_cpu(req->cmd->common.nsid); 42 struct nvmet_ctrl *ctrl = req->sq->ctrl; 43 struct nvmet_ns *ns; 44 unsigned long idx; 45 u16 status; 46 47 if (mask & ~(NVMET_PR_NOTIFY_MASK_ALL)) { 48 req->error_loc = offsetof(struct nvme_common_command, cdw11); 49 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 50 } 51 52 if (nsid != U32_MAX) { 53 status = nvmet_req_find_ns(req); 54 if (status) 55 return status; 56 if (!req->ns->pr.enable) 57 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 58 59 WRITE_ONCE(req->ns->pr.notify_mask, mask); 60 goto success; 61 } 62 63 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { 64 if (ns->pr.enable) 65 WRITE_ONCE(ns->pr.notify_mask, mask); 66 } 67 68 success: 69 nvmet_set_result(req, mask); 70 return NVME_SC_SUCCESS; 71 } 72 73 u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req) 74 { 75 u16 status; 76 77 status = nvmet_req_find_ns(req); 78 if (status) 79 return status; 80 81 if (!req->ns->pr.enable) 82 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 83 84 nvmet_set_result(req, READ_ONCE(req->ns->pr.notify_mask)); 85 return status; 86 } 87 88 void nvmet_execute_get_log_page_resv(struct nvmet_req *req) 89 { 90 struct nvmet_pr_log_mgr *log_mgr = &req->sq->ctrl->pr_log_mgr; 91 struct nvme_pr_log next_log = {0}; 92 struct nvme_pr_log log = {0}; 93 u16 status = NVME_SC_SUCCESS; 94 u64 lost_count; 95 u64 cur_count; 96 u64 next_count; 97 98 mutex_lock(&log_mgr->lock); 99 if (!kfifo_get(&log_mgr->log_queue, &log)) 100 goto out; 101 102 /* 103 * We can't get the last in kfifo. 104 * Utilize the current count and the count from the next log to 105 * calculate the number of lost logs, while also addressing cases 106 * of overflow. If there is no subsequent log, the number of lost 107 * logs is equal to the lost_count within the nvmet_pr_log_mgr. 108 */ 109 cur_count = le64_to_cpu(log.count); 110 if (kfifo_peek(&log_mgr->log_queue, &next_log)) { 111 next_count = le64_to_cpu(next_log.count); 112 if (next_count > cur_count) 113 lost_count = next_count - cur_count - 1; 114 else 115 lost_count = U64_MAX - cur_count + next_count - 1; 116 } else { 117 lost_count = log_mgr->lost_count; 118 } 119 120 log.count = cpu_to_le64((cur_count + lost_count) == 0 ? 121 1 : (cur_count + lost_count)); 122 log_mgr->lost_count -= lost_count; 123 124 log.nr_pages = kfifo_len(&log_mgr->log_queue); 125 126 out: 127 status = nvmet_copy_to_sgl(req, 0, &log, sizeof(log)); 128 mutex_unlock(&log_mgr->lock); 129 nvmet_req_complete(req, status); 130 } 131 132 static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type, 133 u32 nsid) 134 { 135 struct nvmet_pr_log_mgr *log_mgr = &ctrl->pr_log_mgr; 136 struct nvme_pr_log log = {0}; 137 138 mutex_lock(&log_mgr->lock); 139 log_mgr->counter++; 140 if (log_mgr->counter == 0) 141 log_mgr->counter = 1; 142 143 log.count = cpu_to_le64(log_mgr->counter); 144 log.type = log_type; 145 log.nsid = cpu_to_le32(nsid); 146 147 if (!kfifo_put(&log_mgr->log_queue, log)) { 148 pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%u\n", 149 ctrl->cntlid, log_type, nsid); 150 log_mgr->lost_count++; 151 } 152 153 mutex_unlock(&log_mgr->lock); 154 } 155 156 static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid) 157 { 158 struct nvmet_ns *ns = nvmet_pr_to_ns(pr); 159 struct nvmet_subsys *subsys = ns->subsys; 160 struct nvmet_ctrl *ctrl; 161 162 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_RELEASED, &pr->notify_mask)) 163 return; 164 165 mutex_lock(&subsys->lock); 166 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 167 if (!uuid_equal(&ctrl->hostid, hostid) && 168 nvmet_pr_find_registrant(pr, &ctrl->hostid)) { 169 nvmet_pr_add_resv_log(ctrl, 170 NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid); 171 nvmet_add_async_event(ctrl, NVME_AER_CSS, 172 NVME_AEN_RESV_LOG_PAGE_AVAILABLE, 173 NVME_LOG_RESERVATION); 174 } 175 } 176 mutex_unlock(&subsys->lock); 177 } 178 179 static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid, 180 u8 log_type) 181 { 182 struct nvmet_ns *ns = nvmet_pr_to_ns(pr); 183 struct nvmet_subsys *subsys = ns->subsys; 184 struct nvmet_ctrl *ctrl; 185 186 mutex_lock(&subsys->lock); 187 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 188 if (uuid_equal(hostid, &ctrl->hostid)) { 189 nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid); 190 nvmet_add_async_event(ctrl, NVME_AER_CSS, 191 NVME_AEN_RESV_LOG_PAGE_AVAILABLE, 192 NVME_LOG_RESERVATION); 193 } 194 } 195 mutex_unlock(&subsys->lock); 196 } 197 198 static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid) 199 { 200 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_PREEMPTED, &pr->notify_mask)) 201 return; 202 203 nvmet_pr_send_event_to_host(pr, hostid, 204 NVME_PR_LOG_RESERVATION_PREEMPTED); 205 } 206 207 static void nvmet_pr_registration_preempted(struct nvmet_pr *pr, 208 uuid_t *hostid) 209 { 210 if (test_bit(NVME_PR_NOTIFY_BIT_REG_PREEMPTED, &pr->notify_mask)) 211 return; 212 213 nvmet_pr_send_event_to_host(pr, hostid, 214 NVME_PR_LOG_REGISTRATION_PREEMPTED); 215 } 216 217 static inline void nvmet_pr_set_new_holder(struct nvmet_pr *pr, u8 new_rtype, 218 struct nvmet_pr_registrant *reg) 219 { 220 reg->rtype = new_rtype; 221 rcu_assign_pointer(pr->holder, reg); 222 } 223 224 static u16 nvmet_pr_register(struct nvmet_req *req, 225 struct nvmet_pr_register_data *d) 226 { 227 struct nvmet_ctrl *ctrl = req->sq->ctrl; 228 struct nvmet_pr_registrant *new, *reg; 229 struct nvmet_pr *pr = &req->ns->pr; 230 u16 status = NVME_SC_SUCCESS; 231 u64 nrkey = le64_to_cpu(d->nrkey); 232 233 new = kmalloc_obj(*new); 234 if (!new) 235 return NVME_SC_INTERNAL; 236 237 down(&pr->pr_sem); 238 reg = nvmet_pr_find_registrant(pr, &ctrl->hostid); 239 if (reg) { 240 if (reg->rkey != nrkey) 241 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 242 kfree(new); 243 goto out; 244 } 245 246 memset(new, 0, sizeof(*new)); 247 INIT_LIST_HEAD(&new->entry); 248 new->rkey = nrkey; 249 uuid_copy(&new->hostid, &ctrl->hostid); 250 list_add_tail_rcu(&new->entry, &pr->registrant_list); 251 252 out: 253 up(&pr->pr_sem); 254 return status; 255 } 256 257 static void nvmet_pr_unregister_one(struct nvmet_pr *pr, 258 struct nvmet_pr_registrant *reg) 259 { 260 struct nvmet_pr_registrant *first_reg; 261 struct nvmet_pr_registrant *holder; 262 u8 original_rtype; 263 264 list_del_rcu(®->entry); 265 266 holder = rcu_dereference_protected(pr->holder, 1); 267 if (reg != holder) 268 goto out; 269 270 original_rtype = holder->rtype; 271 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS || 272 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) { 273 first_reg = list_first_or_null_rcu(&pr->registrant_list, 274 struct nvmet_pr_registrant, entry); 275 if (first_reg) 276 first_reg->rtype = original_rtype; 277 rcu_assign_pointer(pr->holder, first_reg); 278 } else { 279 rcu_assign_pointer(pr->holder, NULL); 280 281 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_REG_ONLY || 282 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY) 283 nvmet_pr_resv_released(pr, ®->hostid); 284 } 285 out: 286 kfree_rcu(reg, rcu); 287 } 288 289 static u16 nvmet_pr_unregister(struct nvmet_req *req, 290 struct nvmet_pr_register_data *d, 291 bool ignore_key) 292 { 293 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 294 struct nvmet_ctrl *ctrl = req->sq->ctrl; 295 struct nvmet_pr *pr = &req->ns->pr; 296 struct nvmet_pr_registrant *reg; 297 298 down(&pr->pr_sem); 299 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 300 if (uuid_equal(®->hostid, &ctrl->hostid)) { 301 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) { 302 status = NVME_SC_SUCCESS; 303 nvmet_pr_unregister_one(pr, reg); 304 } 305 break; 306 } 307 } 308 up(&pr->pr_sem); 309 310 return status; 311 } 312 313 static void nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant *reg, 314 void *attr) 315 { 316 reg->rkey = *(u64 *)attr; 317 } 318 319 static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr, 320 struct nvmet_pr_registrant *reg, 321 void (*change_attr)(struct nvmet_pr_registrant *reg, 322 void *attr), 323 void *attr) 324 { 325 struct nvmet_pr_registrant *holder; 326 struct nvmet_pr_registrant *new; 327 328 holder = rcu_dereference_protected(pr->holder, 1); 329 if (reg != holder) { 330 change_attr(reg, attr); 331 return NVME_SC_SUCCESS; 332 } 333 334 new = kmalloc_obj(*new, GFP_ATOMIC); 335 if (!new) 336 return NVME_SC_INTERNAL; 337 338 new->rkey = holder->rkey; 339 new->rtype = holder->rtype; 340 uuid_copy(&new->hostid, &holder->hostid); 341 INIT_LIST_HEAD(&new->entry); 342 343 change_attr(new, attr); 344 list_replace_rcu(&holder->entry, &new->entry); 345 rcu_assign_pointer(pr->holder, new); 346 kfree_rcu(holder, rcu); 347 348 return NVME_SC_SUCCESS; 349 } 350 351 static u16 nvmet_pr_replace(struct nvmet_req *req, 352 struct nvmet_pr_register_data *d, 353 bool ignore_key) 354 { 355 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 356 struct nvmet_ctrl *ctrl = req->sq->ctrl; 357 struct nvmet_pr *pr = &req->ns->pr; 358 struct nvmet_pr_registrant *reg, *new = NULL; 359 u64 nrkey = le64_to_cpu(d->nrkey); 360 361 if (ignore_key && nrkey) { 362 new = kzalloc_obj(*new); 363 if (!new) 364 return NVME_SC_INTERNAL; 365 } 366 367 down(&pr->pr_sem); 368 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 369 if (uuid_equal(®->hostid, &ctrl->hostid)) { 370 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) 371 status = nvmet_pr_update_reg_attr(pr, reg, 372 nvmet_pr_update_reg_rkey, 373 &nrkey); 374 goto free_data; 375 } 376 } 377 378 if (ignore_key) { 379 if (!nrkey) { 380 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 381 goto free_data; 382 } 383 INIT_LIST_HEAD(&new->entry); 384 new->rkey = nrkey; 385 uuid_copy(&new->hostid, &ctrl->hostid); 386 list_add_tail_rcu(&new->entry, &pr->registrant_list); 387 status = NVME_SC_SUCCESS; 388 goto out; 389 } 390 391 free_data: 392 kfree(new); 393 out: 394 up(&pr->pr_sem); 395 return status; 396 } 397 398 static void nvmet_execute_pr_register(struct nvmet_req *req) 399 { 400 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 401 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10); 402 struct nvmet_pr_register_data *d; 403 u8 reg_act = cdw10 & 0x07; /* Reservation Register Action, bit 02:00 */ 404 u16 status; 405 406 d = kmalloc_obj(*d); 407 if (!d) { 408 status = NVME_SC_INTERNAL; 409 goto out; 410 } 411 412 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d)); 413 if (status) 414 goto free_data; 415 416 switch (reg_act) { 417 case NVME_PR_REGISTER_ACT_REG: 418 status = nvmet_pr_register(req, d); 419 break; 420 case NVME_PR_REGISTER_ACT_UNREG: 421 status = nvmet_pr_unregister(req, d, ignore_key); 422 break; 423 case NVME_PR_REGISTER_ACT_REPLACE: 424 status = nvmet_pr_replace(req, d, ignore_key); 425 break; 426 default: 427 req->error_loc = offsetof(struct nvme_common_command, cdw10); 428 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 429 break; 430 } 431 free_data: 432 kfree(d); 433 out: 434 if (!status) 435 atomic_inc(&req->ns->pr.generation); 436 nvmet_req_complete(req, status); 437 } 438 439 static u16 nvmet_pr_acquire(struct nvmet_req *req, 440 struct nvmet_pr_registrant *reg, 441 u8 rtype) 442 { 443 struct nvmet_pr *pr = &req->ns->pr; 444 struct nvmet_pr_registrant *holder; 445 446 holder = rcu_dereference_protected(pr->holder, 1); 447 if (holder && reg != holder) 448 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 449 if (holder && reg == holder) { 450 if (holder->rtype == rtype) 451 return NVME_SC_SUCCESS; 452 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 453 } 454 455 nvmet_pr_set_new_holder(pr, rtype, reg); 456 return NVME_SC_SUCCESS; 457 } 458 459 static void nvmet_pr_confirm_ns_pc_ref(struct percpu_ref *ref) 460 { 461 struct nvmet_pr_per_ctrl_ref *pc_ref = 462 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref); 463 464 complete(&pc_ref->confirm_done); 465 } 466 467 static void nvmet_pr_set_ctrl_to_abort(struct nvmet_req *req, uuid_t *hostid) 468 { 469 struct nvmet_pr_per_ctrl_ref *pc_ref; 470 struct nvmet_ns *ns = req->ns; 471 unsigned long idx; 472 473 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 474 if (uuid_equal(&pc_ref->hostid, hostid)) { 475 percpu_ref_kill_and_confirm(&pc_ref->ref, 476 nvmet_pr_confirm_ns_pc_ref); 477 wait_for_completion(&pc_ref->confirm_done); 478 } 479 } 480 } 481 482 static u16 nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req *req, u64 prkey, 483 uuid_t *send_hostid, 484 bool abort) 485 { 486 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 487 struct nvmet_pr_registrant *reg, *tmp; 488 struct nvmet_pr *pr = &req->ns->pr; 489 uuid_t hostid; 490 491 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 492 if (reg->rkey == prkey) { 493 status = NVME_SC_SUCCESS; 494 uuid_copy(&hostid, ®->hostid); 495 if (abort) 496 nvmet_pr_set_ctrl_to_abort(req, &hostid); 497 nvmet_pr_unregister_one(pr, reg); 498 if (!uuid_equal(&hostid, send_hostid)) 499 nvmet_pr_registration_preempted(pr, &hostid); 500 } 501 } 502 return status; 503 } 504 505 static void nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req *req, 506 u64 prkey, 507 uuid_t *send_hostid, 508 bool abort) 509 { 510 struct nvmet_pr_registrant *reg, *tmp; 511 struct nvmet_pr *pr = &req->ns->pr; 512 uuid_t hostid; 513 514 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 515 if (reg->rkey == prkey && 516 !uuid_equal(®->hostid, send_hostid)) { 517 uuid_copy(&hostid, ®->hostid); 518 if (abort) 519 nvmet_pr_set_ctrl_to_abort(req, &hostid); 520 nvmet_pr_unregister_one(pr, reg); 521 nvmet_pr_registration_preempted(pr, &hostid); 522 } 523 } 524 } 525 526 static void nvmet_pr_unreg_all_others(struct nvmet_req *req, 527 uuid_t *send_hostid, 528 bool abort) 529 { 530 struct nvmet_pr_registrant *reg, *tmp; 531 struct nvmet_pr *pr = &req->ns->pr; 532 uuid_t hostid; 533 534 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 535 if (!uuid_equal(®->hostid, send_hostid)) { 536 uuid_copy(&hostid, ®->hostid); 537 if (abort) 538 nvmet_pr_set_ctrl_to_abort(req, &hostid); 539 nvmet_pr_unregister_one(pr, reg); 540 nvmet_pr_registration_preempted(pr, &hostid); 541 } 542 } 543 } 544 545 static void nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant *reg, 546 void *attr) 547 { 548 u8 new_rtype = *(u8 *)attr; 549 550 reg->rtype = new_rtype; 551 } 552 553 static u16 nvmet_pr_preempt(struct nvmet_req *req, 554 struct nvmet_pr_registrant *reg, 555 u8 rtype, 556 struct nvmet_pr_acquire_data *d, 557 bool abort) 558 { 559 struct nvmet_ctrl *ctrl = req->sq->ctrl; 560 struct nvmet_pr *pr = &req->ns->pr; 561 struct nvmet_pr_registrant *holder; 562 enum nvme_pr_type original_rtype; 563 u64 prkey = le64_to_cpu(d->prkey); 564 u16 status; 565 566 holder = rcu_dereference_protected(pr->holder, 1); 567 if (!holder) 568 return nvmet_pr_unreg_all_host_by_prkey(req, prkey, 569 &ctrl->hostid, abort); 570 571 original_rtype = holder->rtype; 572 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS || 573 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) { 574 if (!prkey) { 575 /* 576 * To prevent possible access from other hosts, and 577 * avoid terminate the holder, set the new holder 578 * first before unregistering. 579 */ 580 nvmet_pr_set_new_holder(pr, rtype, reg); 581 nvmet_pr_unreg_all_others(req, &ctrl->hostid, abort); 582 return NVME_SC_SUCCESS; 583 } 584 return nvmet_pr_unreg_all_host_by_prkey(req, prkey, 585 &ctrl->hostid, abort); 586 } 587 588 if (holder == reg) { 589 status = nvmet_pr_update_reg_attr(pr, holder, 590 nvmet_pr_update_holder_rtype, &rtype); 591 if (!status && original_rtype != rtype) 592 nvmet_pr_resv_released(pr, ®->hostid); 593 return status; 594 } 595 596 if (prkey == holder->rkey) { 597 /* 598 * Same as before, set the new holder first. 599 */ 600 nvmet_pr_set_new_holder(pr, rtype, reg); 601 nvmet_pr_unreg_all_others_by_prkey(req, prkey, &ctrl->hostid, 602 abort); 603 if (original_rtype != rtype) 604 nvmet_pr_resv_released(pr, ®->hostid); 605 return NVME_SC_SUCCESS; 606 } 607 608 if (prkey) 609 return nvmet_pr_unreg_all_host_by_prkey(req, prkey, 610 &ctrl->hostid, abort); 611 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 612 } 613 614 static void nvmet_pr_do_abort(struct work_struct *w) 615 { 616 struct nvmet_req *req = container_of(w, struct nvmet_req, r.abort_work); 617 struct nvmet_pr_per_ctrl_ref *pc_ref; 618 struct nvmet_ns *ns = req->ns; 619 unsigned long idx; 620 621 /* 622 * The target does not support abort, just wait per-controller ref to 0. 623 */ 624 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 625 if (percpu_ref_is_dying(&pc_ref->ref)) { 626 wait_for_completion(&pc_ref->free_done); 627 reinit_completion(&pc_ref->confirm_done); 628 reinit_completion(&pc_ref->free_done); 629 percpu_ref_resurrect(&pc_ref->ref); 630 } 631 } 632 633 up(&ns->pr.pr_sem); 634 nvmet_req_complete(req, NVME_SC_SUCCESS); 635 } 636 637 static u16 __nvmet_execute_pr_acquire(struct nvmet_req *req, 638 struct nvmet_pr_registrant *reg, 639 u8 acquire_act, 640 u8 rtype, 641 struct nvmet_pr_acquire_data *d) 642 { 643 u16 status; 644 645 switch (acquire_act) { 646 case NVME_PR_ACQUIRE_ACT_ACQUIRE: 647 status = nvmet_pr_acquire(req, reg, rtype); 648 goto out; 649 case NVME_PR_ACQUIRE_ACT_PREEMPT: 650 status = nvmet_pr_preempt(req, reg, rtype, d, false); 651 goto inc_gen; 652 case NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT: 653 status = nvmet_pr_preempt(req, reg, rtype, d, true); 654 goto inc_gen; 655 default: 656 req->error_loc = offsetof(struct nvme_common_command, cdw10); 657 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 658 goto out; 659 } 660 inc_gen: 661 if (!status) 662 atomic_inc(&req->ns->pr.generation); 663 out: 664 return status; 665 } 666 667 static void nvmet_execute_pr_acquire(struct nvmet_req *req) 668 { 669 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 670 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10); 671 /* Reservation type, bit 15:08 */ 672 u8 rtype = (u8)((cdw10 >> 8) & 0xff); 673 /* Reservation acquire action, bit 02:00 */ 674 u8 acquire_act = cdw10 & 0x07; 675 struct nvmet_ctrl *ctrl = req->sq->ctrl; 676 struct nvmet_pr_acquire_data *d = NULL; 677 struct nvmet_pr *pr = &req->ns->pr; 678 struct nvmet_pr_registrant *reg; 679 u16 status = NVME_SC_SUCCESS; 680 681 if (ignore_key || 682 rtype < NVME_PR_WRITE_EXCLUSIVE || 683 rtype > NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) { 684 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 685 goto out; 686 } 687 688 d = kmalloc_obj(*d); 689 if (!d) { 690 status = NVME_SC_INTERNAL; 691 goto out; 692 } 693 694 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d)); 695 if (status) 696 goto free_data; 697 698 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 699 down(&pr->pr_sem); 700 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 701 if (uuid_equal(®->hostid, &ctrl->hostid) && 702 reg->rkey == le64_to_cpu(d->crkey)) { 703 status = __nvmet_execute_pr_acquire(req, reg, 704 acquire_act, rtype, d); 705 break; 706 } 707 } 708 709 if (!status && acquire_act == NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT) { 710 kfree(d); 711 INIT_WORK(&req->r.abort_work, nvmet_pr_do_abort); 712 queue_work(nvmet_wq, &req->r.abort_work); 713 return; 714 } 715 716 up(&pr->pr_sem); 717 718 free_data: 719 kfree(d); 720 out: 721 nvmet_req_complete(req, status); 722 } 723 724 static u16 nvmet_pr_release(struct nvmet_req *req, 725 struct nvmet_pr_registrant *reg, 726 u8 rtype) 727 { 728 struct nvmet_pr *pr = &req->ns->pr; 729 struct nvmet_pr_registrant *holder; 730 u8 original_rtype; 731 732 holder = rcu_dereference_protected(pr->holder, 1); 733 if (!holder || reg != holder) 734 return NVME_SC_SUCCESS; 735 736 original_rtype = holder->rtype; 737 if (original_rtype != rtype) 738 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 739 740 rcu_assign_pointer(pr->holder, NULL); 741 742 if (original_rtype != NVME_PR_WRITE_EXCLUSIVE && 743 original_rtype != NVME_PR_EXCLUSIVE_ACCESS) 744 nvmet_pr_resv_released(pr, ®->hostid); 745 746 return NVME_SC_SUCCESS; 747 } 748 749 static void nvmet_pr_clear(struct nvmet_req *req) 750 { 751 struct nvmet_pr_registrant *reg, *tmp; 752 struct nvmet_pr *pr = &req->ns->pr; 753 754 rcu_assign_pointer(pr->holder, NULL); 755 756 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 757 list_del_rcu(®->entry); 758 if (!uuid_equal(&req->sq->ctrl->hostid, ®->hostid)) 759 nvmet_pr_resv_preempted(pr, ®->hostid); 760 kfree_rcu(reg, rcu); 761 } 762 763 atomic_inc(&pr->generation); 764 } 765 766 static u16 __nvmet_execute_pr_release(struct nvmet_req *req, 767 struct nvmet_pr_registrant *reg, 768 u8 release_act, u8 rtype) 769 { 770 switch (release_act) { 771 case NVME_PR_RELEASE_ACT_RELEASE: 772 return nvmet_pr_release(req, reg, rtype); 773 case NVME_PR_RELEASE_ACT_CLEAR: 774 nvmet_pr_clear(req); 775 return NVME_SC_SUCCESS; 776 default: 777 req->error_loc = offsetof(struct nvme_common_command, cdw10); 778 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 779 } 780 } 781 782 static void nvmet_execute_pr_release(struct nvmet_req *req) 783 { 784 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 785 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10); 786 u8 rtype = (u8)((cdw10 >> 8) & 0xff); /* Reservation type, bit 15:08 */ 787 u8 release_act = cdw10 & 0x07; /* Reservation release action, bit 02:00 */ 788 struct nvmet_ctrl *ctrl = req->sq->ctrl; 789 struct nvmet_pr *pr = &req->ns->pr; 790 struct nvmet_pr_release_data *d; 791 struct nvmet_pr_registrant *reg; 792 u16 status; 793 794 if (ignore_key) { 795 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 796 goto out; 797 } 798 799 d = kmalloc_obj(*d); 800 if (!d) { 801 status = NVME_SC_INTERNAL; 802 goto out; 803 } 804 805 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d)); 806 if (status) 807 goto free_data; 808 809 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 810 down(&pr->pr_sem); 811 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 812 if (uuid_equal(®->hostid, &ctrl->hostid) && 813 reg->rkey == le64_to_cpu(d->crkey)) { 814 status = __nvmet_execute_pr_release(req, reg, 815 release_act, rtype); 816 break; 817 } 818 } 819 up(&pr->pr_sem); 820 free_data: 821 kfree(d); 822 out: 823 nvmet_req_complete(req, status); 824 } 825 826 static void nvmet_execute_pr_report(struct nvmet_req *req) 827 { 828 u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11); 829 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 830 u32 num_bytes = 4 * (cdw10 + 1); /* cdw10 is number of dwords */ 831 u8 eds = cdw11 & 1; /* Extended data structure, bit 00 */ 832 struct nvme_registered_ctrl_ext *ctrl_eds; 833 struct nvme_reservation_status_ext *data; 834 struct nvmet_pr *pr = &req->ns->pr; 835 struct nvmet_pr_registrant *holder; 836 struct nvmet_pr_registrant *reg; 837 u16 num_ctrls = 0; 838 u16 status; 839 u8 rtype; 840 841 /* nvmet hostid(uuid_t) is 128 bit. */ 842 if (!eds) { 843 req->error_loc = offsetof(struct nvme_common_command, cdw11); 844 status = NVME_SC_HOST_ID_INCONSIST | NVME_STATUS_DNR; 845 goto out; 846 } 847 848 if (num_bytes < sizeof(struct nvme_reservation_status_ext)) { 849 req->error_loc = offsetof(struct nvme_common_command, cdw10); 850 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 851 goto out; 852 } 853 854 data = kzalloc(num_bytes, GFP_KERNEL); 855 if (!data) { 856 status = NVME_SC_INTERNAL; 857 goto out; 858 } 859 data->gen = cpu_to_le32(atomic_read(&pr->generation)); 860 data->ptpls = 0; 861 ctrl_eds = data->regctl_eds; 862 863 rcu_read_lock(); 864 holder = rcu_dereference(pr->holder); 865 rtype = holder ? holder->rtype : 0; 866 data->rtype = rtype; 867 868 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 869 num_ctrls++; 870 /* 871 * continue to get the number of all registrans. 872 */ 873 if (((void *)ctrl_eds + sizeof(*ctrl_eds)) > 874 ((void *)data + num_bytes)) 875 continue; 876 /* 877 * Dynamic controller, set cntlid to 0xffff. 878 */ 879 ctrl_eds->cntlid = cpu_to_le16(NVME_CNTLID_DYNAMIC); 880 if (rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS || 881 rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) 882 ctrl_eds->rcsts = 1; 883 if (reg == holder) 884 ctrl_eds->rcsts = 1; 885 uuid_copy((uuid_t *)&ctrl_eds->hostid, ®->hostid); 886 ctrl_eds->rkey = cpu_to_le64(reg->rkey); 887 ctrl_eds++; 888 } 889 rcu_read_unlock(); 890 891 put_unaligned_le16(num_ctrls, data->regctl); 892 status = nvmet_copy_to_sgl(req, 0, data, num_bytes); 893 kfree(data); 894 out: 895 nvmet_req_complete(req, status); 896 } 897 898 u16 nvmet_parse_pr_cmd(struct nvmet_req *req) 899 { 900 struct nvme_command *cmd = req->cmd; 901 902 switch (cmd->common.opcode) { 903 case nvme_cmd_resv_register: 904 req->execute = nvmet_execute_pr_register; 905 break; 906 case nvme_cmd_resv_acquire: 907 req->execute = nvmet_execute_pr_acquire; 908 break; 909 case nvme_cmd_resv_release: 910 req->execute = nvmet_execute_pr_release; 911 break; 912 case nvme_cmd_resv_report: 913 req->execute = nvmet_execute_pr_report; 914 break; 915 default: 916 return 1; 917 } 918 return NVME_SC_SUCCESS; 919 } 920 921 static bool nvmet_is_req_write_cmd_group(struct nvmet_req *req) 922 { 923 u8 opcode = req->cmd->common.opcode; 924 925 if (req->sq->qid) { 926 switch (opcode) { 927 case nvme_cmd_flush: 928 case nvme_cmd_write: 929 case nvme_cmd_write_zeroes: 930 case nvme_cmd_dsm: 931 case nvme_cmd_zone_append: 932 case nvme_cmd_zone_mgmt_send: 933 return true; 934 default: 935 return false; 936 } 937 } 938 return false; 939 } 940 941 static bool nvmet_is_req_read_cmd_group(struct nvmet_req *req) 942 { 943 u8 opcode = req->cmd->common.opcode; 944 945 if (req->sq->qid) { 946 switch (opcode) { 947 case nvme_cmd_read: 948 case nvme_cmd_zone_mgmt_recv: 949 return true; 950 default: 951 return false; 952 } 953 } 954 return false; 955 } 956 957 u16 nvmet_pr_check_cmd_access(struct nvmet_req *req) 958 { 959 struct nvmet_ctrl *ctrl = req->sq->ctrl; 960 struct nvmet_pr_registrant *holder; 961 struct nvmet_ns *ns = req->ns; 962 struct nvmet_pr *pr = &ns->pr; 963 u16 status = NVME_SC_SUCCESS; 964 965 rcu_read_lock(); 966 holder = rcu_dereference(pr->holder); 967 if (!holder) 968 goto unlock; 969 if (uuid_equal(&ctrl->hostid, &holder->hostid)) 970 goto unlock; 971 972 /* 973 * The Reservation command group is checked in executing, 974 * allow it here. 975 */ 976 switch (holder->rtype) { 977 case NVME_PR_WRITE_EXCLUSIVE: 978 if (nvmet_is_req_write_cmd_group(req)) 979 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 980 break; 981 case NVME_PR_EXCLUSIVE_ACCESS: 982 if (nvmet_is_req_read_cmd_group(req) || 983 nvmet_is_req_write_cmd_group(req)) 984 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 985 break; 986 case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY: 987 case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS: 988 if ((nvmet_is_req_write_cmd_group(req)) && 989 !nvmet_pr_find_registrant(pr, &ctrl->hostid)) 990 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 991 break; 992 case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY: 993 case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS: 994 if ((nvmet_is_req_read_cmd_group(req) || 995 nvmet_is_req_write_cmd_group(req)) && 996 !nvmet_pr_find_registrant(pr, &ctrl->hostid)) 997 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 998 break; 999 default: 1000 pr_warn("the reservation type is set wrong, type:%d\n", 1001 holder->rtype); 1002 break; 1003 } 1004 1005 unlock: 1006 rcu_read_unlock(); 1007 if (status) 1008 req->error_loc = offsetof(struct nvme_common_command, opcode); 1009 return status; 1010 } 1011 1012 u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req) 1013 { 1014 struct nvmet_pr_per_ctrl_ref *pc_ref; 1015 1016 pc_ref = xa_load(&req->ns->pr_per_ctrl_refs, 1017 req->sq->ctrl->cntlid); 1018 if (unlikely(!percpu_ref_tryget_live(&pc_ref->ref))) 1019 return NVME_SC_INTERNAL; 1020 req->pc_ref = pc_ref; 1021 return NVME_SC_SUCCESS; 1022 } 1023 1024 static void nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref *ref) 1025 { 1026 struct nvmet_pr_per_ctrl_ref *pc_ref = 1027 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref); 1028 1029 complete(&pc_ref->free_done); 1030 } 1031 1032 static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns, 1033 unsigned long idx, 1034 uuid_t *hostid) 1035 { 1036 struct nvmet_pr_per_ctrl_ref *pc_ref; 1037 int ret; 1038 1039 pc_ref = kmalloc_obj(*pc_ref, GFP_ATOMIC); 1040 if (!pc_ref) 1041 return -ENOMEM; 1042 1043 ret = percpu_ref_init(&pc_ref->ref, nvmet_pr_ctrl_ns_all_cmds_done, 1044 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); 1045 if (ret) 1046 goto free; 1047 1048 init_completion(&pc_ref->free_done); 1049 init_completion(&pc_ref->confirm_done); 1050 uuid_copy(&pc_ref->hostid, hostid); 1051 1052 ret = xa_insert(&ns->pr_per_ctrl_refs, idx, pc_ref, GFP_KERNEL); 1053 if (ret) 1054 goto exit; 1055 return ret; 1056 exit: 1057 percpu_ref_exit(&pc_ref->ref); 1058 free: 1059 kfree(pc_ref); 1060 return ret; 1061 } 1062 1063 int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl) 1064 { 1065 struct nvmet_subsys *subsys = ctrl->subsys; 1066 struct nvmet_pr_per_ctrl_ref *pc_ref; 1067 struct nvmet_ns *ns = NULL; 1068 unsigned long idx; 1069 int ret; 1070 1071 ctrl->pr_log_mgr.counter = 0; 1072 ctrl->pr_log_mgr.lost_count = 0; 1073 mutex_init(&ctrl->pr_log_mgr.lock); 1074 INIT_KFIFO(ctrl->pr_log_mgr.log_queue); 1075 1076 /* 1077 * Here we are under subsys lock, if an ns not in subsys->namespaces, 1078 * we can make sure that ns is not enabled, and not call 1079 * nvmet_pr_init_ns(), see more details in nvmet_ns_enable(). 1080 * So just check ns->pr.enable. 1081 */ 1082 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) { 1083 if (ns->pr.enable) { 1084 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid, 1085 &ctrl->hostid); 1086 if (ret) 1087 goto free_per_ctrl_refs; 1088 } 1089 } 1090 return 0; 1091 1092 free_per_ctrl_refs: 1093 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) { 1094 if (ns->pr.enable) { 1095 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid); 1096 if (pc_ref) 1097 percpu_ref_exit(&pc_ref->ref); 1098 kfree(pc_ref); 1099 } 1100 } 1101 return ret; 1102 } 1103 1104 void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl) 1105 { 1106 struct nvmet_pr_per_ctrl_ref *pc_ref; 1107 struct nvmet_ns *ns; 1108 unsigned long idx; 1109 1110 kfifo_free(&ctrl->pr_log_mgr.log_queue); 1111 mutex_destroy(&ctrl->pr_log_mgr.lock); 1112 1113 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { 1114 if (ns->pr.enable) { 1115 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid); 1116 if (pc_ref) 1117 percpu_ref_exit(&pc_ref->ref); 1118 kfree(pc_ref); 1119 } 1120 } 1121 } 1122 1123 int nvmet_pr_init_ns(struct nvmet_ns *ns) 1124 { 1125 struct nvmet_subsys *subsys = ns->subsys; 1126 struct nvmet_pr_per_ctrl_ref *pc_ref; 1127 struct nvmet_ctrl *ctrl = NULL; 1128 unsigned long idx; 1129 int ret; 1130 1131 ns->pr.holder = NULL; 1132 atomic_set(&ns->pr.generation, 0); 1133 sema_init(&ns->pr.pr_sem, 1); 1134 INIT_LIST_HEAD(&ns->pr.registrant_list); 1135 ns->pr.notify_mask = 0; 1136 1137 xa_init(&ns->pr_per_ctrl_refs); 1138 1139 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 1140 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid, 1141 &ctrl->hostid); 1142 if (ret) 1143 goto free_per_ctrl_refs; 1144 } 1145 return 0; 1146 1147 free_per_ctrl_refs: 1148 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 1149 xa_erase(&ns->pr_per_ctrl_refs, idx); 1150 percpu_ref_exit(&pc_ref->ref); 1151 kfree(pc_ref); 1152 } 1153 return ret; 1154 } 1155 1156 void nvmet_pr_exit_ns(struct nvmet_ns *ns) 1157 { 1158 struct nvmet_pr_registrant *reg, *tmp; 1159 struct nvmet_pr_per_ctrl_ref *pc_ref; 1160 struct nvmet_pr *pr = &ns->pr; 1161 unsigned long idx; 1162 1163 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 1164 list_del(®->entry); 1165 kfree(reg); 1166 } 1167 1168 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 1169 /* 1170 * No command on ns here, we can safely free pc_ref. 1171 */ 1172 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, idx); 1173 percpu_ref_exit(&pc_ref->ref); 1174 kfree(pc_ref); 1175 } 1176 1177 xa_destroy(&ns->pr_per_ctrl_refs); 1178 } 1179