1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics Persist Reservation. 4 * Copyright (c) 2024 Guixin Liu, Alibaba Group. 5 * All rights reserved. 6 */ 7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 #include <linux/unaligned.h> 9 #include "nvmet.h" 10 11 #define NVMET_PR_NOTIFY_MASK_ALL \ 12 (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \ 13 1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \ 14 1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED) 15 16 static inline bool nvmet_pr_parse_ignore_key(u32 cdw10) 17 { 18 /* Ignore existing key, bit 03. */ 19 return (cdw10 >> 3) & 1; 20 } 21 22 static inline struct nvmet_ns *nvmet_pr_to_ns(struct nvmet_pr *pr) 23 { 24 return container_of(pr, struct nvmet_ns, pr); 25 } 26 27 static struct nvmet_pr_registrant * 28 nvmet_pr_find_registrant(struct nvmet_pr *pr, uuid_t *hostid) 29 { 30 struct nvmet_pr_registrant *reg; 31 32 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 33 if (uuid_equal(®->hostid, hostid)) 34 return reg; 35 } 36 return NULL; 37 } 38 39 u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask) 40 { 41 u32 nsid = le32_to_cpu(req->cmd->common.nsid); 42 struct nvmet_ctrl *ctrl = req->sq->ctrl; 43 struct nvmet_ns *ns; 44 unsigned long idx; 45 u16 status; 46 47 if (mask & ~(NVMET_PR_NOTIFY_MASK_ALL)) { 48 req->error_loc = offsetof(struct nvme_common_command, cdw11); 49 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 50 } 51 52 if (nsid != U32_MAX) { 53 status = nvmet_req_find_ns(req); 54 if (status == (NVME_SC_INVALID_NS | NVME_STATUS_DNR)) 55 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 56 if (status) 57 return status; 58 if (!req->ns->pr.enable) 59 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 60 61 WRITE_ONCE(req->ns->pr.notify_mask, mask); 62 goto success; 63 } 64 65 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { 66 if (ns->pr.enable) 67 WRITE_ONCE(ns->pr.notify_mask, mask); 68 } 69 70 success: 71 nvmet_set_result(req, mask); 72 return NVME_SC_SUCCESS; 73 } 74 75 u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req) 76 { 77 u16 status; 78 79 status = nvmet_req_find_ns(req); 80 if (status == (NVME_SC_INVALID_NS | NVME_STATUS_DNR)) 81 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 82 if (status) 83 return status; 84 85 if (!req->ns->pr.enable) 86 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 87 88 nvmet_set_result(req, READ_ONCE(req->ns->pr.notify_mask)); 89 return status; 90 } 91 92 void nvmet_execute_get_log_page_resv(struct nvmet_req *req) 93 { 94 struct nvmet_pr_log_mgr *log_mgr = &req->sq->ctrl->pr_log_mgr; 95 struct nvme_pr_log next_log = {0}; 96 struct nvme_pr_log log = {0}; 97 u16 status = NVME_SC_SUCCESS; 98 u64 lost_count; 99 u64 cur_count; 100 u64 next_count; 101 102 mutex_lock(&log_mgr->lock); 103 if (!kfifo_get(&log_mgr->log_queue, &log)) 104 goto out; 105 106 /* 107 * We can't get the last in kfifo. 108 * Utilize the current count and the count from the next log to 109 * calculate the number of lost logs, while also addressing cases 110 * of overflow. If there is no subsequent log, the number of lost 111 * logs is equal to the lost_count within the nvmet_pr_log_mgr. 112 */ 113 cur_count = le64_to_cpu(log.count); 114 if (kfifo_peek(&log_mgr->log_queue, &next_log)) { 115 next_count = le64_to_cpu(next_log.count); 116 if (next_count > cur_count) 117 lost_count = next_count - cur_count - 1; 118 else 119 lost_count = U64_MAX - cur_count + next_count - 1; 120 } else { 121 lost_count = log_mgr->lost_count; 122 } 123 124 log.count = cpu_to_le64((cur_count + lost_count) == 0 ? 125 1 : (cur_count + lost_count)); 126 log_mgr->lost_count -= lost_count; 127 128 log.nr_pages = kfifo_len(&log_mgr->log_queue); 129 130 out: 131 status = nvmet_copy_to_sgl(req, 0, &log, sizeof(log)); 132 mutex_unlock(&log_mgr->lock); 133 nvmet_req_complete(req, status); 134 } 135 136 static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type, 137 u32 nsid) 138 { 139 struct nvmet_pr_log_mgr *log_mgr = &ctrl->pr_log_mgr; 140 struct nvme_pr_log log = {0}; 141 142 mutex_lock(&log_mgr->lock); 143 log_mgr->counter++; 144 if (log_mgr->counter == 0) 145 log_mgr->counter = 1; 146 147 log.count = cpu_to_le64(log_mgr->counter); 148 log.type = log_type; 149 log.nsid = cpu_to_le32(nsid); 150 151 if (!kfifo_put(&log_mgr->log_queue, log)) { 152 pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%u\n", 153 ctrl->cntlid, log_type, nsid); 154 log_mgr->lost_count++; 155 } 156 157 mutex_unlock(&log_mgr->lock); 158 } 159 160 static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid) 161 { 162 struct nvmet_ns *ns = nvmet_pr_to_ns(pr); 163 struct nvmet_subsys *subsys = ns->subsys; 164 struct nvmet_ctrl *ctrl; 165 166 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_RELEASED, &pr->notify_mask)) 167 return; 168 169 mutex_lock(&subsys->lock); 170 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 171 if (!uuid_equal(&ctrl->hostid, hostid) && 172 nvmet_pr_find_registrant(pr, &ctrl->hostid)) { 173 nvmet_pr_add_resv_log(ctrl, 174 NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid); 175 nvmet_add_async_event(ctrl, NVME_AER_CSS, 176 NVME_AEN_RESV_LOG_PAGE_AVAILABLE, 177 NVME_LOG_RESERVATION); 178 } 179 } 180 mutex_unlock(&subsys->lock); 181 } 182 183 static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid, 184 u8 log_type) 185 { 186 struct nvmet_ns *ns = nvmet_pr_to_ns(pr); 187 struct nvmet_subsys *subsys = ns->subsys; 188 struct nvmet_ctrl *ctrl; 189 190 mutex_lock(&subsys->lock); 191 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 192 if (uuid_equal(hostid, &ctrl->hostid)) { 193 nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid); 194 nvmet_add_async_event(ctrl, NVME_AER_CSS, 195 NVME_AEN_RESV_LOG_PAGE_AVAILABLE, 196 NVME_LOG_RESERVATION); 197 } 198 } 199 mutex_unlock(&subsys->lock); 200 } 201 202 static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid) 203 { 204 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_PREEMPTED, &pr->notify_mask)) 205 return; 206 207 nvmet_pr_send_event_to_host(pr, hostid, 208 NVME_PR_LOG_RESERVATION_PREEMPTED); 209 } 210 211 static void nvmet_pr_registration_preempted(struct nvmet_pr *pr, 212 uuid_t *hostid) 213 { 214 if (test_bit(NVME_PR_NOTIFY_BIT_REG_PREEMPTED, &pr->notify_mask)) 215 return; 216 217 nvmet_pr_send_event_to_host(pr, hostid, 218 NVME_PR_LOG_REGISTRATION_PREEMPTED); 219 } 220 221 static inline void nvmet_pr_set_new_holder(struct nvmet_pr *pr, u8 new_rtype, 222 struct nvmet_pr_registrant *reg) 223 { 224 reg->rtype = new_rtype; 225 rcu_assign_pointer(pr->holder, reg); 226 } 227 228 static u16 nvmet_pr_register(struct nvmet_req *req, 229 struct nvmet_pr_register_data *d) 230 { 231 struct nvmet_ctrl *ctrl = req->sq->ctrl; 232 struct nvmet_pr_registrant *new, *reg; 233 struct nvmet_pr *pr = &req->ns->pr; 234 u16 status = NVME_SC_SUCCESS; 235 u64 nrkey = le64_to_cpu(d->nrkey); 236 237 new = kmalloc_obj(*new); 238 if (!new) 239 return NVME_SC_INTERNAL; 240 241 down(&pr->pr_sem); 242 reg = nvmet_pr_find_registrant(pr, &ctrl->hostid); 243 if (reg) { 244 if (reg->rkey != nrkey) 245 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 246 kfree(new); 247 goto out; 248 } 249 250 memset(new, 0, sizeof(*new)); 251 INIT_LIST_HEAD(&new->entry); 252 new->rkey = nrkey; 253 uuid_copy(&new->hostid, &ctrl->hostid); 254 list_add_tail_rcu(&new->entry, &pr->registrant_list); 255 256 out: 257 up(&pr->pr_sem); 258 return status; 259 } 260 261 static void nvmet_pr_unregister_one(struct nvmet_pr *pr, 262 struct nvmet_pr_registrant *reg) 263 { 264 struct nvmet_pr_registrant *first_reg; 265 struct nvmet_pr_registrant *holder; 266 u8 original_rtype; 267 268 list_del_rcu(®->entry); 269 270 holder = rcu_dereference_protected(pr->holder, 1); 271 if (reg != holder) 272 goto out; 273 274 original_rtype = holder->rtype; 275 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS || 276 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) { 277 first_reg = list_first_or_null_rcu(&pr->registrant_list, 278 struct nvmet_pr_registrant, entry); 279 if (first_reg) 280 first_reg->rtype = original_rtype; 281 rcu_assign_pointer(pr->holder, first_reg); 282 } else { 283 rcu_assign_pointer(pr->holder, NULL); 284 285 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_REG_ONLY || 286 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY) 287 nvmet_pr_resv_released(pr, ®->hostid); 288 } 289 out: 290 kfree_rcu(reg, rcu); 291 } 292 293 static u16 nvmet_pr_unregister(struct nvmet_req *req, 294 struct nvmet_pr_register_data *d, 295 bool ignore_key) 296 { 297 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 298 struct nvmet_ctrl *ctrl = req->sq->ctrl; 299 struct nvmet_pr *pr = &req->ns->pr; 300 struct nvmet_pr_registrant *reg; 301 302 down(&pr->pr_sem); 303 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 304 if (uuid_equal(®->hostid, &ctrl->hostid)) { 305 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) { 306 status = NVME_SC_SUCCESS; 307 nvmet_pr_unregister_one(pr, reg); 308 } 309 break; 310 } 311 } 312 up(&pr->pr_sem); 313 314 return status; 315 } 316 317 static void nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant *reg, 318 void *attr) 319 { 320 reg->rkey = *(u64 *)attr; 321 } 322 323 static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr, 324 struct nvmet_pr_registrant *reg, 325 void (*change_attr)(struct nvmet_pr_registrant *reg, 326 void *attr), 327 void *attr) 328 { 329 struct nvmet_pr_registrant *holder; 330 struct nvmet_pr_registrant *new; 331 332 holder = rcu_dereference_protected(pr->holder, 1); 333 if (reg != holder) { 334 change_attr(reg, attr); 335 return NVME_SC_SUCCESS; 336 } 337 338 new = kmalloc_obj(*new, GFP_ATOMIC); 339 if (!new) 340 return NVME_SC_INTERNAL; 341 342 new->rkey = holder->rkey; 343 new->rtype = holder->rtype; 344 uuid_copy(&new->hostid, &holder->hostid); 345 INIT_LIST_HEAD(&new->entry); 346 347 change_attr(new, attr); 348 list_replace_rcu(&holder->entry, &new->entry); 349 rcu_assign_pointer(pr->holder, new); 350 kfree_rcu(holder, rcu); 351 352 return NVME_SC_SUCCESS; 353 } 354 355 static u16 nvmet_pr_replace(struct nvmet_req *req, 356 struct nvmet_pr_register_data *d, 357 bool ignore_key) 358 { 359 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 360 struct nvmet_ctrl *ctrl = req->sq->ctrl; 361 struct nvmet_pr *pr = &req->ns->pr; 362 struct nvmet_pr_registrant *reg, *new = NULL; 363 u64 nrkey = le64_to_cpu(d->nrkey); 364 365 if (ignore_key && nrkey) { 366 new = kzalloc_obj(*new); 367 if (!new) 368 return NVME_SC_INTERNAL; 369 } 370 371 down(&pr->pr_sem); 372 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 373 if (uuid_equal(®->hostid, &ctrl->hostid)) { 374 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) 375 status = nvmet_pr_update_reg_attr(pr, reg, 376 nvmet_pr_update_reg_rkey, 377 &nrkey); 378 goto free_data; 379 } 380 } 381 382 if (ignore_key) { 383 if (!nrkey) { 384 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 385 goto free_data; 386 } 387 INIT_LIST_HEAD(&new->entry); 388 new->rkey = nrkey; 389 uuid_copy(&new->hostid, &ctrl->hostid); 390 list_add_tail_rcu(&new->entry, &pr->registrant_list); 391 status = NVME_SC_SUCCESS; 392 goto out; 393 } 394 395 free_data: 396 kfree(new); 397 out: 398 up(&pr->pr_sem); 399 return status; 400 } 401 402 static void nvmet_execute_pr_register(struct nvmet_req *req) 403 { 404 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 405 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10); 406 struct nvmet_pr_register_data *d; 407 u8 reg_act = cdw10 & 0x07; /* Reservation Register Action, bit 02:00 */ 408 u16 status; 409 410 d = kmalloc_obj(*d); 411 if (!d) { 412 status = NVME_SC_INTERNAL; 413 goto out; 414 } 415 416 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d)); 417 if (status) 418 goto free_data; 419 420 switch (reg_act) { 421 case NVME_PR_REGISTER_ACT_REG: 422 status = nvmet_pr_register(req, d); 423 break; 424 case NVME_PR_REGISTER_ACT_UNREG: 425 status = nvmet_pr_unregister(req, d, ignore_key); 426 break; 427 case NVME_PR_REGISTER_ACT_REPLACE: 428 status = nvmet_pr_replace(req, d, ignore_key); 429 break; 430 default: 431 req->error_loc = offsetof(struct nvme_common_command, cdw10); 432 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 433 break; 434 } 435 free_data: 436 kfree(d); 437 out: 438 if (!status) 439 atomic_inc(&req->ns->pr.generation); 440 nvmet_req_complete(req, status); 441 } 442 443 static u16 nvmet_pr_acquire(struct nvmet_req *req, 444 struct nvmet_pr_registrant *reg, 445 u8 rtype) 446 { 447 struct nvmet_pr *pr = &req->ns->pr; 448 struct nvmet_pr_registrant *holder; 449 450 holder = rcu_dereference_protected(pr->holder, 1); 451 if (holder && reg != holder) 452 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 453 if (holder && reg == holder) { 454 if (holder->rtype == rtype) 455 return NVME_SC_SUCCESS; 456 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 457 } 458 459 nvmet_pr_set_new_holder(pr, rtype, reg); 460 return NVME_SC_SUCCESS; 461 } 462 463 static void nvmet_pr_confirm_ns_pc_ref(struct percpu_ref *ref) 464 { 465 struct nvmet_pr_per_ctrl_ref *pc_ref = 466 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref); 467 468 complete(&pc_ref->confirm_done); 469 } 470 471 static void nvmet_pr_set_ctrl_to_abort(struct nvmet_req *req, uuid_t *hostid) 472 { 473 struct nvmet_pr_per_ctrl_ref *pc_ref; 474 struct nvmet_ns *ns = req->ns; 475 unsigned long idx; 476 477 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 478 if (uuid_equal(&pc_ref->hostid, hostid)) { 479 percpu_ref_kill_and_confirm(&pc_ref->ref, 480 nvmet_pr_confirm_ns_pc_ref); 481 wait_for_completion(&pc_ref->confirm_done); 482 } 483 } 484 } 485 486 static u16 nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req *req, u64 prkey, 487 uuid_t *send_hostid, 488 bool abort) 489 { 490 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 491 struct nvmet_pr_registrant *reg, *tmp; 492 struct nvmet_pr *pr = &req->ns->pr; 493 uuid_t hostid; 494 495 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 496 if (reg->rkey == prkey) { 497 status = NVME_SC_SUCCESS; 498 uuid_copy(&hostid, ®->hostid); 499 if (abort) 500 nvmet_pr_set_ctrl_to_abort(req, &hostid); 501 nvmet_pr_unregister_one(pr, reg); 502 if (!uuid_equal(&hostid, send_hostid)) 503 nvmet_pr_registration_preempted(pr, &hostid); 504 } 505 } 506 return status; 507 } 508 509 static void nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req *req, 510 u64 prkey, 511 uuid_t *send_hostid, 512 bool abort) 513 { 514 struct nvmet_pr_registrant *reg, *tmp; 515 struct nvmet_pr *pr = &req->ns->pr; 516 uuid_t hostid; 517 518 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 519 if (reg->rkey == prkey && 520 !uuid_equal(®->hostid, send_hostid)) { 521 uuid_copy(&hostid, ®->hostid); 522 if (abort) 523 nvmet_pr_set_ctrl_to_abort(req, &hostid); 524 nvmet_pr_unregister_one(pr, reg); 525 nvmet_pr_registration_preempted(pr, &hostid); 526 } 527 } 528 } 529 530 static void nvmet_pr_unreg_all_others(struct nvmet_req *req, 531 uuid_t *send_hostid, 532 bool abort) 533 { 534 struct nvmet_pr_registrant *reg, *tmp; 535 struct nvmet_pr *pr = &req->ns->pr; 536 uuid_t hostid; 537 538 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 539 if (!uuid_equal(®->hostid, send_hostid)) { 540 uuid_copy(&hostid, ®->hostid); 541 if (abort) 542 nvmet_pr_set_ctrl_to_abort(req, &hostid); 543 nvmet_pr_unregister_one(pr, reg); 544 nvmet_pr_registration_preempted(pr, &hostid); 545 } 546 } 547 } 548 549 static void nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant *reg, 550 void *attr) 551 { 552 u8 new_rtype = *(u8 *)attr; 553 554 reg->rtype = new_rtype; 555 } 556 557 static u16 nvmet_pr_preempt(struct nvmet_req *req, 558 struct nvmet_pr_registrant *reg, 559 u8 rtype, 560 struct nvmet_pr_acquire_data *d, 561 bool abort) 562 { 563 struct nvmet_ctrl *ctrl = req->sq->ctrl; 564 struct nvmet_pr *pr = &req->ns->pr; 565 struct nvmet_pr_registrant *holder; 566 enum nvme_pr_type original_rtype; 567 u64 prkey = le64_to_cpu(d->prkey); 568 u16 status; 569 570 holder = rcu_dereference_protected(pr->holder, 1); 571 if (!holder) 572 return nvmet_pr_unreg_all_host_by_prkey(req, prkey, 573 &ctrl->hostid, abort); 574 575 original_rtype = holder->rtype; 576 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS || 577 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) { 578 if (!prkey) { 579 /* 580 * To prevent possible access from other hosts, and 581 * avoid terminate the holder, set the new holder 582 * first before unregistering. 583 */ 584 nvmet_pr_set_new_holder(pr, rtype, reg); 585 nvmet_pr_unreg_all_others(req, &ctrl->hostid, abort); 586 return NVME_SC_SUCCESS; 587 } 588 return nvmet_pr_unreg_all_host_by_prkey(req, prkey, 589 &ctrl->hostid, abort); 590 } 591 592 if (holder == reg) { 593 status = nvmet_pr_update_reg_attr(pr, holder, 594 nvmet_pr_update_holder_rtype, &rtype); 595 if (!status && original_rtype != rtype) 596 nvmet_pr_resv_released(pr, ®->hostid); 597 return status; 598 } 599 600 if (prkey == holder->rkey) { 601 /* 602 * Same as before, set the new holder first. 603 */ 604 nvmet_pr_set_new_holder(pr, rtype, reg); 605 nvmet_pr_unreg_all_others_by_prkey(req, prkey, &ctrl->hostid, 606 abort); 607 if (original_rtype != rtype) 608 nvmet_pr_resv_released(pr, ®->hostid); 609 return NVME_SC_SUCCESS; 610 } 611 612 if (prkey) 613 return nvmet_pr_unreg_all_host_by_prkey(req, prkey, 614 &ctrl->hostid, abort); 615 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 616 } 617 618 static void nvmet_pr_do_abort(struct work_struct *w) 619 { 620 struct nvmet_req *req = container_of(w, struct nvmet_req, r.abort_work); 621 struct nvmet_pr_per_ctrl_ref *pc_ref; 622 struct nvmet_ns *ns = req->ns; 623 unsigned long idx; 624 625 /* 626 * The target does not support abort, just wait per-controller ref to 0. 627 */ 628 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 629 if (percpu_ref_is_dying(&pc_ref->ref)) { 630 wait_for_completion(&pc_ref->free_done); 631 reinit_completion(&pc_ref->confirm_done); 632 reinit_completion(&pc_ref->free_done); 633 percpu_ref_resurrect(&pc_ref->ref); 634 } 635 } 636 637 up(&ns->pr.pr_sem); 638 nvmet_req_complete(req, NVME_SC_SUCCESS); 639 } 640 641 static u16 __nvmet_execute_pr_acquire(struct nvmet_req *req, 642 struct nvmet_pr_registrant *reg, 643 u8 acquire_act, 644 u8 rtype, 645 struct nvmet_pr_acquire_data *d) 646 { 647 u16 status; 648 649 switch (acquire_act) { 650 case NVME_PR_ACQUIRE_ACT_ACQUIRE: 651 status = nvmet_pr_acquire(req, reg, rtype); 652 goto out; 653 case NVME_PR_ACQUIRE_ACT_PREEMPT: 654 status = nvmet_pr_preempt(req, reg, rtype, d, false); 655 goto inc_gen; 656 case NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT: 657 status = nvmet_pr_preempt(req, reg, rtype, d, true); 658 goto inc_gen; 659 default: 660 req->error_loc = offsetof(struct nvme_common_command, cdw10); 661 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 662 goto out; 663 } 664 inc_gen: 665 if (!status) 666 atomic_inc(&req->ns->pr.generation); 667 out: 668 return status; 669 } 670 671 static void nvmet_execute_pr_acquire(struct nvmet_req *req) 672 { 673 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 674 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10); 675 /* Reservation type, bit 15:08 */ 676 u8 rtype = (u8)((cdw10 >> 8) & 0xff); 677 /* Reservation acquire action, bit 02:00 */ 678 u8 acquire_act = cdw10 & 0x07; 679 struct nvmet_ctrl *ctrl = req->sq->ctrl; 680 struct nvmet_pr_acquire_data *d = NULL; 681 struct nvmet_pr *pr = &req->ns->pr; 682 struct nvmet_pr_registrant *reg; 683 u16 status = NVME_SC_SUCCESS; 684 685 if (ignore_key || 686 rtype < NVME_PR_WRITE_EXCLUSIVE || 687 rtype > NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) { 688 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 689 goto out; 690 } 691 692 d = kmalloc_obj(*d); 693 if (!d) { 694 status = NVME_SC_INTERNAL; 695 goto out; 696 } 697 698 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d)); 699 if (status) 700 goto free_data; 701 702 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 703 down(&pr->pr_sem); 704 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 705 if (uuid_equal(®->hostid, &ctrl->hostid) && 706 reg->rkey == le64_to_cpu(d->crkey)) { 707 status = __nvmet_execute_pr_acquire(req, reg, 708 acquire_act, rtype, d); 709 break; 710 } 711 } 712 713 if (!status && acquire_act == NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT) { 714 kfree(d); 715 INIT_WORK(&req->r.abort_work, nvmet_pr_do_abort); 716 queue_work(nvmet_wq, &req->r.abort_work); 717 return; 718 } 719 720 up(&pr->pr_sem); 721 722 free_data: 723 kfree(d); 724 out: 725 nvmet_req_complete(req, status); 726 } 727 728 static u16 nvmet_pr_release(struct nvmet_req *req, 729 struct nvmet_pr_registrant *reg, 730 u8 rtype) 731 { 732 struct nvmet_pr *pr = &req->ns->pr; 733 struct nvmet_pr_registrant *holder; 734 u8 original_rtype; 735 736 holder = rcu_dereference_protected(pr->holder, 1); 737 if (!holder || reg != holder) 738 return NVME_SC_SUCCESS; 739 740 original_rtype = holder->rtype; 741 if (original_rtype != rtype) 742 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 743 744 rcu_assign_pointer(pr->holder, NULL); 745 746 if (original_rtype != NVME_PR_WRITE_EXCLUSIVE && 747 original_rtype != NVME_PR_EXCLUSIVE_ACCESS) 748 nvmet_pr_resv_released(pr, ®->hostid); 749 750 return NVME_SC_SUCCESS; 751 } 752 753 static void nvmet_pr_clear(struct nvmet_req *req) 754 { 755 struct nvmet_pr_registrant *reg, *tmp; 756 struct nvmet_pr *pr = &req->ns->pr; 757 758 rcu_assign_pointer(pr->holder, NULL); 759 760 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 761 list_del_rcu(®->entry); 762 if (!uuid_equal(&req->sq->ctrl->hostid, ®->hostid)) 763 nvmet_pr_resv_preempted(pr, ®->hostid); 764 kfree_rcu(reg, rcu); 765 } 766 767 atomic_inc(&pr->generation); 768 } 769 770 static u16 __nvmet_execute_pr_release(struct nvmet_req *req, 771 struct nvmet_pr_registrant *reg, 772 u8 release_act, u8 rtype) 773 { 774 switch (release_act) { 775 case NVME_PR_RELEASE_ACT_RELEASE: 776 return nvmet_pr_release(req, reg, rtype); 777 case NVME_PR_RELEASE_ACT_CLEAR: 778 nvmet_pr_clear(req); 779 return NVME_SC_SUCCESS; 780 default: 781 req->error_loc = offsetof(struct nvme_common_command, cdw10); 782 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR; 783 } 784 } 785 786 static void nvmet_execute_pr_release(struct nvmet_req *req) 787 { 788 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 789 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10); 790 u8 rtype = (u8)((cdw10 >> 8) & 0xff); /* Reservation type, bit 15:08 */ 791 u8 release_act = cdw10 & 0x07; /* Reservation release action, bit 02:00 */ 792 struct nvmet_ctrl *ctrl = req->sq->ctrl; 793 struct nvmet_pr *pr = &req->ns->pr; 794 struct nvmet_pr_release_data *d; 795 struct nvmet_pr_registrant *reg; 796 u16 status; 797 798 if (ignore_key) { 799 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 800 goto out; 801 } 802 803 d = kmalloc_obj(*d); 804 if (!d) { 805 status = NVME_SC_INTERNAL; 806 goto out; 807 } 808 809 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d)); 810 if (status) 811 goto free_data; 812 813 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 814 down(&pr->pr_sem); 815 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 816 if (uuid_equal(®->hostid, &ctrl->hostid) && 817 reg->rkey == le64_to_cpu(d->crkey)) { 818 status = __nvmet_execute_pr_release(req, reg, 819 release_act, rtype); 820 break; 821 } 822 } 823 up(&pr->pr_sem); 824 free_data: 825 kfree(d); 826 out: 827 nvmet_req_complete(req, status); 828 } 829 830 static void nvmet_execute_pr_report(struct nvmet_req *req) 831 { 832 u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11); 833 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10); 834 u32 num_bytes = 4 * (cdw10 + 1); /* cdw10 is number of dwords */ 835 u8 eds = cdw11 & 1; /* Extended data structure, bit 00 */ 836 struct nvme_registered_ctrl_ext *ctrl_eds; 837 struct nvme_reservation_status_ext *data; 838 struct nvmet_pr *pr = &req->ns->pr; 839 struct nvmet_pr_registrant *holder; 840 struct nvmet_pr_registrant *reg; 841 u16 num_ctrls = 0; 842 u16 status; 843 u8 rtype; 844 845 /* nvmet hostid(uuid_t) is 128 bit. */ 846 if (!eds) { 847 req->error_loc = offsetof(struct nvme_common_command, cdw11); 848 status = NVME_SC_HOST_ID_INCONSIST | NVME_STATUS_DNR; 849 goto out; 850 } 851 852 if (num_bytes < sizeof(struct nvme_reservation_status_ext)) { 853 req->error_loc = offsetof(struct nvme_common_command, cdw10); 854 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR; 855 goto out; 856 } 857 858 data = kzalloc(num_bytes, GFP_KERNEL); 859 if (!data) { 860 status = NVME_SC_INTERNAL; 861 goto out; 862 } 863 data->gen = cpu_to_le32(atomic_read(&pr->generation)); 864 data->ptpls = 0; 865 ctrl_eds = data->regctl_eds; 866 867 rcu_read_lock(); 868 holder = rcu_dereference(pr->holder); 869 rtype = holder ? holder->rtype : 0; 870 data->rtype = rtype; 871 872 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) { 873 num_ctrls++; 874 /* 875 * continue to get the number of all registrans. 876 */ 877 if (((void *)ctrl_eds + sizeof(*ctrl_eds)) > 878 ((void *)data + num_bytes)) 879 continue; 880 /* 881 * Dynamic controller, set cntlid to 0xffff. 882 */ 883 ctrl_eds->cntlid = cpu_to_le16(NVME_CNTLID_DYNAMIC); 884 if (rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS || 885 rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) 886 ctrl_eds->rcsts = 1; 887 if (reg == holder) 888 ctrl_eds->rcsts = 1; 889 uuid_copy((uuid_t *)&ctrl_eds->hostid, ®->hostid); 890 ctrl_eds->rkey = cpu_to_le64(reg->rkey); 891 ctrl_eds++; 892 } 893 rcu_read_unlock(); 894 895 put_unaligned_le16(num_ctrls, data->regctl); 896 status = nvmet_copy_to_sgl(req, 0, data, num_bytes); 897 kfree(data); 898 out: 899 nvmet_req_complete(req, status); 900 } 901 902 u16 nvmet_parse_pr_cmd(struct nvmet_req *req) 903 { 904 struct nvme_command *cmd = req->cmd; 905 906 switch (cmd->common.opcode) { 907 case nvme_cmd_resv_register: 908 req->execute = nvmet_execute_pr_register; 909 break; 910 case nvme_cmd_resv_acquire: 911 req->execute = nvmet_execute_pr_acquire; 912 break; 913 case nvme_cmd_resv_release: 914 req->execute = nvmet_execute_pr_release; 915 break; 916 case nvme_cmd_resv_report: 917 req->execute = nvmet_execute_pr_report; 918 break; 919 default: 920 return 1; 921 } 922 return NVME_SC_SUCCESS; 923 } 924 925 static bool nvmet_is_req_write_cmd_group(struct nvmet_req *req) 926 { 927 u8 opcode = req->cmd->common.opcode; 928 929 if (req->sq->qid) { 930 switch (opcode) { 931 case nvme_cmd_flush: 932 case nvme_cmd_write: 933 case nvme_cmd_write_zeroes: 934 case nvme_cmd_dsm: 935 case nvme_cmd_zone_append: 936 case nvme_cmd_zone_mgmt_send: 937 return true; 938 default: 939 return false; 940 } 941 } 942 return false; 943 } 944 945 static bool nvmet_is_req_read_cmd_group(struct nvmet_req *req) 946 { 947 u8 opcode = req->cmd->common.opcode; 948 949 if (req->sq->qid) { 950 switch (opcode) { 951 case nvme_cmd_read: 952 case nvme_cmd_zone_mgmt_recv: 953 return true; 954 default: 955 return false; 956 } 957 } 958 return false; 959 } 960 961 u16 nvmet_pr_check_cmd_access(struct nvmet_req *req) 962 { 963 struct nvmet_ctrl *ctrl = req->sq->ctrl; 964 struct nvmet_pr_registrant *holder; 965 struct nvmet_ns *ns = req->ns; 966 struct nvmet_pr *pr = &ns->pr; 967 u16 status = NVME_SC_SUCCESS; 968 969 rcu_read_lock(); 970 holder = rcu_dereference(pr->holder); 971 if (!holder) 972 goto unlock; 973 if (uuid_equal(&ctrl->hostid, &holder->hostid)) 974 goto unlock; 975 976 /* 977 * The Reservation command group is checked in executing, 978 * allow it here. 979 */ 980 switch (holder->rtype) { 981 case NVME_PR_WRITE_EXCLUSIVE: 982 if (nvmet_is_req_write_cmd_group(req)) 983 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 984 break; 985 case NVME_PR_EXCLUSIVE_ACCESS: 986 if (nvmet_is_req_read_cmd_group(req) || 987 nvmet_is_req_write_cmd_group(req)) 988 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 989 break; 990 case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY: 991 case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS: 992 if ((nvmet_is_req_write_cmd_group(req)) && 993 !nvmet_pr_find_registrant(pr, &ctrl->hostid)) 994 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 995 break; 996 case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY: 997 case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS: 998 if ((nvmet_is_req_read_cmd_group(req) || 999 nvmet_is_req_write_cmd_group(req)) && 1000 !nvmet_pr_find_registrant(pr, &ctrl->hostid)) 1001 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR; 1002 break; 1003 default: 1004 pr_warn("the reservation type is set wrong, type:%d\n", 1005 holder->rtype); 1006 break; 1007 } 1008 1009 unlock: 1010 rcu_read_unlock(); 1011 if (status) 1012 req->error_loc = offsetof(struct nvme_common_command, opcode); 1013 return status; 1014 } 1015 1016 u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req) 1017 { 1018 struct nvmet_pr_per_ctrl_ref *pc_ref; 1019 1020 pc_ref = xa_load(&req->ns->pr_per_ctrl_refs, 1021 req->sq->ctrl->cntlid); 1022 if (unlikely(!percpu_ref_tryget_live(&pc_ref->ref))) 1023 return NVME_SC_INTERNAL; 1024 req->pc_ref = pc_ref; 1025 return NVME_SC_SUCCESS; 1026 } 1027 1028 static void nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref *ref) 1029 { 1030 struct nvmet_pr_per_ctrl_ref *pc_ref = 1031 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref); 1032 1033 complete(&pc_ref->free_done); 1034 } 1035 1036 static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns, 1037 unsigned long idx, 1038 uuid_t *hostid) 1039 { 1040 struct nvmet_pr_per_ctrl_ref *pc_ref; 1041 int ret; 1042 1043 pc_ref = kmalloc_obj(*pc_ref, GFP_ATOMIC); 1044 if (!pc_ref) 1045 return -ENOMEM; 1046 1047 ret = percpu_ref_init(&pc_ref->ref, nvmet_pr_ctrl_ns_all_cmds_done, 1048 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL); 1049 if (ret) 1050 goto free; 1051 1052 init_completion(&pc_ref->free_done); 1053 init_completion(&pc_ref->confirm_done); 1054 uuid_copy(&pc_ref->hostid, hostid); 1055 1056 ret = xa_insert(&ns->pr_per_ctrl_refs, idx, pc_ref, GFP_KERNEL); 1057 if (ret) 1058 goto exit; 1059 return ret; 1060 exit: 1061 percpu_ref_exit(&pc_ref->ref); 1062 free: 1063 kfree(pc_ref); 1064 return ret; 1065 } 1066 1067 int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl) 1068 { 1069 struct nvmet_subsys *subsys = ctrl->subsys; 1070 struct nvmet_pr_per_ctrl_ref *pc_ref; 1071 struct nvmet_ns *ns = NULL; 1072 unsigned long idx; 1073 int ret; 1074 1075 ctrl->pr_log_mgr.counter = 0; 1076 ctrl->pr_log_mgr.lost_count = 0; 1077 mutex_init(&ctrl->pr_log_mgr.lock); 1078 INIT_KFIFO(ctrl->pr_log_mgr.log_queue); 1079 1080 /* 1081 * Here we are under subsys lock, if an ns not in subsys->namespaces, 1082 * we can make sure that ns is not enabled, and not call 1083 * nvmet_pr_init_ns(), see more details in nvmet_ns_enable(). 1084 * So just check ns->pr.enable. 1085 */ 1086 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) { 1087 if (ns->pr.enable) { 1088 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid, 1089 &ctrl->hostid); 1090 if (ret) 1091 goto free_per_ctrl_refs; 1092 } 1093 } 1094 return 0; 1095 1096 free_per_ctrl_refs: 1097 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) { 1098 if (ns->pr.enable) { 1099 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid); 1100 if (pc_ref) 1101 percpu_ref_exit(&pc_ref->ref); 1102 kfree(pc_ref); 1103 } 1104 } 1105 return ret; 1106 } 1107 1108 void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl) 1109 { 1110 struct nvmet_pr_per_ctrl_ref *pc_ref; 1111 struct nvmet_ns *ns; 1112 unsigned long idx; 1113 1114 kfifo_free(&ctrl->pr_log_mgr.log_queue); 1115 mutex_destroy(&ctrl->pr_log_mgr.lock); 1116 1117 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) { 1118 if (ns->pr.enable) { 1119 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid); 1120 if (pc_ref) 1121 percpu_ref_exit(&pc_ref->ref); 1122 kfree(pc_ref); 1123 } 1124 } 1125 } 1126 1127 int nvmet_pr_init_ns(struct nvmet_ns *ns) 1128 { 1129 struct nvmet_subsys *subsys = ns->subsys; 1130 struct nvmet_pr_per_ctrl_ref *pc_ref; 1131 struct nvmet_ctrl *ctrl = NULL; 1132 unsigned long idx; 1133 int ret; 1134 1135 ns->pr.holder = NULL; 1136 atomic_set(&ns->pr.generation, 0); 1137 sema_init(&ns->pr.pr_sem, 1); 1138 INIT_LIST_HEAD(&ns->pr.registrant_list); 1139 ns->pr.notify_mask = 0; 1140 1141 xa_init(&ns->pr_per_ctrl_refs); 1142 1143 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 1144 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid, 1145 &ctrl->hostid); 1146 if (ret) 1147 goto free_per_ctrl_refs; 1148 } 1149 return 0; 1150 1151 free_per_ctrl_refs: 1152 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 1153 xa_erase(&ns->pr_per_ctrl_refs, idx); 1154 percpu_ref_exit(&pc_ref->ref); 1155 kfree(pc_ref); 1156 } 1157 return ret; 1158 } 1159 1160 void nvmet_pr_exit_ns(struct nvmet_ns *ns) 1161 { 1162 struct nvmet_pr_registrant *reg, *tmp; 1163 struct nvmet_pr_per_ctrl_ref *pc_ref; 1164 struct nvmet_pr *pr = &ns->pr; 1165 unsigned long idx; 1166 1167 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) { 1168 list_del(®->entry); 1169 kfree(reg); 1170 } 1171 1172 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) { 1173 /* 1174 * No command on ns here, we can safely free pc_ref. 1175 */ 1176 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, idx); 1177 percpu_ref_exit(&pc_ref->ref); 1178 kfree(pc_ref); 1179 } 1180 1181 xa_destroy(&ns->pr_per_ctrl_refs); 1182 } 1183