1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVMe over Fabrics Persist Reservation.
4 * Copyright (c) 2024 Guixin Liu, Alibaba Group.
5 * All rights reserved.
6 */
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/unaligned.h>
9 #include "nvmet.h"
10
11 #define NVMET_PR_NOTIFY_MASK_ALL \
12 (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \
13 1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \
14 1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED)
15
nvmet_pr_parse_ignore_key(u32 cdw10)16 static inline bool nvmet_pr_parse_ignore_key(u32 cdw10)
17 {
18 /* Ignore existing key, bit 03. */
19 return (cdw10 >> 3) & 1;
20 }
21
nvmet_pr_to_ns(struct nvmet_pr * pr)22 static inline struct nvmet_ns *nvmet_pr_to_ns(struct nvmet_pr *pr)
23 {
24 return container_of(pr, struct nvmet_ns, pr);
25 }
26
27 static struct nvmet_pr_registrant *
nvmet_pr_find_registrant(struct nvmet_pr * pr,uuid_t * hostid)28 nvmet_pr_find_registrant(struct nvmet_pr *pr, uuid_t *hostid)
29 {
30 struct nvmet_pr_registrant *reg;
31
32 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
33 if (uuid_equal(®->hostid, hostid))
34 return reg;
35 }
36 return NULL;
37 }
38
nvmet_set_feat_resv_notif_mask(struct nvmet_req * req,u32 mask)39 u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask)
40 {
41 u32 nsid = le32_to_cpu(req->cmd->common.nsid);
42 struct nvmet_ctrl *ctrl = req->sq->ctrl;
43 struct nvmet_ns *ns;
44 unsigned long idx;
45 u16 status;
46
47 if (mask & ~(NVMET_PR_NOTIFY_MASK_ALL)) {
48 req->error_loc = offsetof(struct nvme_common_command, cdw11);
49 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
50 }
51
52 if (nsid != U32_MAX) {
53 status = nvmet_req_find_ns(req);
54 if (status)
55 return status;
56 if (!req->ns->pr.enable)
57 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
58
59 WRITE_ONCE(req->ns->pr.notify_mask, mask);
60 goto success;
61 }
62
63 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
64 if (ns->pr.enable)
65 WRITE_ONCE(ns->pr.notify_mask, mask);
66 }
67
68 success:
69 nvmet_set_result(req, mask);
70 return NVME_SC_SUCCESS;
71 }
72
nvmet_get_feat_resv_notif_mask(struct nvmet_req * req)73 u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req)
74 {
75 u16 status;
76
77 status = nvmet_req_find_ns(req);
78 if (status)
79 return status;
80
81 if (!req->ns->pr.enable)
82 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
83
84 nvmet_set_result(req, READ_ONCE(req->ns->pr.notify_mask));
85 return status;
86 }
87
nvmet_execute_get_log_page_resv(struct nvmet_req * req)88 void nvmet_execute_get_log_page_resv(struct nvmet_req *req)
89 {
90 struct nvmet_pr_log_mgr *log_mgr = &req->sq->ctrl->pr_log_mgr;
91 struct nvme_pr_log next_log = {0};
92 struct nvme_pr_log log = {0};
93 u16 status = NVME_SC_SUCCESS;
94 u64 lost_count;
95 u64 cur_count;
96 u64 next_count;
97
98 mutex_lock(&log_mgr->lock);
99 if (!kfifo_get(&log_mgr->log_queue, &log))
100 goto out;
101
102 /*
103 * We can't get the last in kfifo.
104 * Utilize the current count and the count from the next log to
105 * calculate the number of lost logs, while also addressing cases
106 * of overflow. If there is no subsequent log, the number of lost
107 * logs is equal to the lost_count within the nvmet_pr_log_mgr.
108 */
109 cur_count = le64_to_cpu(log.count);
110 if (kfifo_peek(&log_mgr->log_queue, &next_log)) {
111 next_count = le64_to_cpu(next_log.count);
112 if (next_count > cur_count)
113 lost_count = next_count - cur_count - 1;
114 else
115 lost_count = U64_MAX - cur_count + next_count - 1;
116 } else {
117 lost_count = log_mgr->lost_count;
118 }
119
120 log.count = cpu_to_le64((cur_count + lost_count) == 0 ?
121 1 : (cur_count + lost_count));
122 log_mgr->lost_count -= lost_count;
123
124 log.nr_pages = kfifo_len(&log_mgr->log_queue);
125
126 out:
127 status = nvmet_copy_to_sgl(req, 0, &log, sizeof(log));
128 mutex_unlock(&log_mgr->lock);
129 nvmet_req_complete(req, status);
130 }
131
nvmet_pr_add_resv_log(struct nvmet_ctrl * ctrl,u8 log_type,u32 nsid)132 static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type,
133 u32 nsid)
134 {
135 struct nvmet_pr_log_mgr *log_mgr = &ctrl->pr_log_mgr;
136 struct nvme_pr_log log = {0};
137
138 mutex_lock(&log_mgr->lock);
139 log_mgr->counter++;
140 if (log_mgr->counter == 0)
141 log_mgr->counter = 1;
142
143 log.count = cpu_to_le64(log_mgr->counter);
144 log.type = log_type;
145 log.nsid = cpu_to_le32(nsid);
146
147 if (!kfifo_put(&log_mgr->log_queue, log)) {
148 pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%d\n",
149 ctrl->cntlid, log_type, nsid);
150 log_mgr->lost_count++;
151 }
152
153 mutex_unlock(&log_mgr->lock);
154 }
155
nvmet_pr_resv_released(struct nvmet_pr * pr,uuid_t * hostid)156 static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid)
157 {
158 struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
159 struct nvmet_subsys *subsys = ns->subsys;
160 struct nvmet_ctrl *ctrl;
161
162 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_RELEASED, &pr->notify_mask))
163 return;
164
165 mutex_lock(&subsys->lock);
166 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
167 if (!uuid_equal(&ctrl->hostid, hostid) &&
168 nvmet_pr_find_registrant(pr, &ctrl->hostid)) {
169 nvmet_pr_add_resv_log(ctrl,
170 NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid);
171 nvmet_add_async_event(ctrl, NVME_AER_CSS,
172 NVME_AEN_RESV_LOG_PAGE_AVAILABLE,
173 NVME_LOG_RESERVATION);
174 }
175 }
176 mutex_unlock(&subsys->lock);
177 }
178
nvmet_pr_send_event_to_host(struct nvmet_pr * pr,uuid_t * hostid,u8 log_type)179 static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid,
180 u8 log_type)
181 {
182 struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
183 struct nvmet_subsys *subsys = ns->subsys;
184 struct nvmet_ctrl *ctrl;
185
186 mutex_lock(&subsys->lock);
187 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
188 if (uuid_equal(hostid, &ctrl->hostid)) {
189 nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid);
190 nvmet_add_async_event(ctrl, NVME_AER_CSS,
191 NVME_AEN_RESV_LOG_PAGE_AVAILABLE,
192 NVME_LOG_RESERVATION);
193 }
194 }
195 mutex_unlock(&subsys->lock);
196 }
197
nvmet_pr_resv_preempted(struct nvmet_pr * pr,uuid_t * hostid)198 static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid)
199 {
200 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_PREEMPTED, &pr->notify_mask))
201 return;
202
203 nvmet_pr_send_event_to_host(pr, hostid,
204 NVME_PR_LOG_RESERVATION_PREEMPTED);
205 }
206
nvmet_pr_registration_preempted(struct nvmet_pr * pr,uuid_t * hostid)207 static void nvmet_pr_registration_preempted(struct nvmet_pr *pr,
208 uuid_t *hostid)
209 {
210 if (test_bit(NVME_PR_NOTIFY_BIT_REG_PREEMPTED, &pr->notify_mask))
211 return;
212
213 nvmet_pr_send_event_to_host(pr, hostid,
214 NVME_PR_LOG_REGISTRATION_PREEMPTED);
215 }
216
nvmet_pr_set_new_holder(struct nvmet_pr * pr,u8 new_rtype,struct nvmet_pr_registrant * reg)217 static inline void nvmet_pr_set_new_holder(struct nvmet_pr *pr, u8 new_rtype,
218 struct nvmet_pr_registrant *reg)
219 {
220 reg->rtype = new_rtype;
221 rcu_assign_pointer(pr->holder, reg);
222 }
223
nvmet_pr_register(struct nvmet_req * req,struct nvmet_pr_register_data * d)224 static u16 nvmet_pr_register(struct nvmet_req *req,
225 struct nvmet_pr_register_data *d)
226 {
227 struct nvmet_ctrl *ctrl = req->sq->ctrl;
228 struct nvmet_pr_registrant *new, *reg;
229 struct nvmet_pr *pr = &req->ns->pr;
230 u16 status = NVME_SC_SUCCESS;
231 u64 nrkey = le64_to_cpu(d->nrkey);
232
233 new = kmalloc_obj(*new);
234 if (!new)
235 return NVME_SC_INTERNAL;
236
237 down(&pr->pr_sem);
238 reg = nvmet_pr_find_registrant(pr, &ctrl->hostid);
239 if (reg) {
240 if (reg->rkey != nrkey)
241 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
242 kfree(new);
243 goto out;
244 }
245
246 memset(new, 0, sizeof(*new));
247 INIT_LIST_HEAD(&new->entry);
248 new->rkey = nrkey;
249 uuid_copy(&new->hostid, &ctrl->hostid);
250 list_add_tail_rcu(&new->entry, &pr->registrant_list);
251
252 out:
253 up(&pr->pr_sem);
254 return status;
255 }
256
nvmet_pr_unregister_one(struct nvmet_pr * pr,struct nvmet_pr_registrant * reg)257 static void nvmet_pr_unregister_one(struct nvmet_pr *pr,
258 struct nvmet_pr_registrant *reg)
259 {
260 struct nvmet_pr_registrant *first_reg;
261 struct nvmet_pr_registrant *holder;
262 u8 original_rtype;
263
264 list_del_rcu(®->entry);
265
266 holder = rcu_dereference_protected(pr->holder, 1);
267 if (reg != holder)
268 goto out;
269
270 original_rtype = holder->rtype;
271 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
272 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
273 first_reg = list_first_or_null_rcu(&pr->registrant_list,
274 struct nvmet_pr_registrant, entry);
275 if (first_reg)
276 first_reg->rtype = original_rtype;
277 rcu_assign_pointer(pr->holder, first_reg);
278 } else {
279 rcu_assign_pointer(pr->holder, NULL);
280
281 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_REG_ONLY ||
282 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY)
283 nvmet_pr_resv_released(pr, ®->hostid);
284 }
285 out:
286 kfree_rcu(reg, rcu);
287 }
288
nvmet_pr_unregister(struct nvmet_req * req,struct nvmet_pr_register_data * d,bool ignore_key)289 static u16 nvmet_pr_unregister(struct nvmet_req *req,
290 struct nvmet_pr_register_data *d,
291 bool ignore_key)
292 {
293 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
294 struct nvmet_ctrl *ctrl = req->sq->ctrl;
295 struct nvmet_pr *pr = &req->ns->pr;
296 struct nvmet_pr_registrant *reg;
297
298 down(&pr->pr_sem);
299 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
300 if (uuid_equal(®->hostid, &ctrl->hostid)) {
301 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) {
302 status = NVME_SC_SUCCESS;
303 nvmet_pr_unregister_one(pr, reg);
304 }
305 break;
306 }
307 }
308 up(&pr->pr_sem);
309
310 return status;
311 }
312
nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant * reg,void * attr)313 static void nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant *reg,
314 void *attr)
315 {
316 reg->rkey = *(u64 *)attr;
317 }
318
nvmet_pr_update_reg_attr(struct nvmet_pr * pr,struct nvmet_pr_registrant * reg,void (* change_attr)(struct nvmet_pr_registrant * reg,void * attr),void * attr)319 static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr,
320 struct nvmet_pr_registrant *reg,
321 void (*change_attr)(struct nvmet_pr_registrant *reg,
322 void *attr),
323 void *attr)
324 {
325 struct nvmet_pr_registrant *holder;
326 struct nvmet_pr_registrant *new;
327
328 holder = rcu_dereference_protected(pr->holder, 1);
329 if (reg != holder) {
330 change_attr(reg, attr);
331 return NVME_SC_SUCCESS;
332 }
333
334 new = kmalloc_obj(*new, GFP_ATOMIC);
335 if (!new)
336 return NVME_SC_INTERNAL;
337
338 new->rkey = holder->rkey;
339 new->rtype = holder->rtype;
340 uuid_copy(&new->hostid, &holder->hostid);
341 INIT_LIST_HEAD(&new->entry);
342
343 change_attr(new, attr);
344 list_replace_rcu(&holder->entry, &new->entry);
345 rcu_assign_pointer(pr->holder, new);
346 kfree_rcu(holder, rcu);
347
348 return NVME_SC_SUCCESS;
349 }
350
nvmet_pr_replace(struct nvmet_req * req,struct nvmet_pr_register_data * d,bool ignore_key)351 static u16 nvmet_pr_replace(struct nvmet_req *req,
352 struct nvmet_pr_register_data *d,
353 bool ignore_key)
354 {
355 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
356 struct nvmet_ctrl *ctrl = req->sq->ctrl;
357 struct nvmet_pr *pr = &req->ns->pr;
358 struct nvmet_pr_registrant *reg, *new = NULL;
359 u64 nrkey = le64_to_cpu(d->nrkey);
360
361 if (ignore_key && nrkey) {
362 new = kzalloc_obj(*new);
363 if (!new)
364 return NVME_SC_INTERNAL;
365 }
366
367 down(&pr->pr_sem);
368 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
369 if (uuid_equal(®->hostid, &ctrl->hostid)) {
370 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey))
371 status = nvmet_pr_update_reg_attr(pr, reg,
372 nvmet_pr_update_reg_rkey,
373 &nrkey);
374 goto free_data;
375 }
376 }
377
378 if (ignore_key) {
379 if (!nrkey) {
380 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
381 goto free_data;
382 }
383 INIT_LIST_HEAD(&new->entry);
384 new->rkey = nrkey;
385 uuid_copy(&new->hostid, &ctrl->hostid);
386 list_add_tail_rcu(&new->entry, &pr->registrant_list);
387 status = NVME_SC_SUCCESS;
388 goto out;
389 }
390
391 free_data:
392 kfree(new);
393 out:
394 up(&pr->pr_sem);
395 return status;
396 }
397
nvmet_execute_pr_register(struct nvmet_req * req)398 static void nvmet_execute_pr_register(struct nvmet_req *req)
399 {
400 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
401 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
402 struct nvmet_pr_register_data *d;
403 u8 reg_act = cdw10 & 0x07; /* Reservation Register Action, bit 02:00 */
404 u16 status;
405
406 d = kmalloc_obj(*d);
407 if (!d) {
408 status = NVME_SC_INTERNAL;
409 goto out;
410 }
411
412 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
413 if (status)
414 goto free_data;
415
416 switch (reg_act) {
417 case NVME_PR_REGISTER_ACT_REG:
418 status = nvmet_pr_register(req, d);
419 break;
420 case NVME_PR_REGISTER_ACT_UNREG:
421 status = nvmet_pr_unregister(req, d, ignore_key);
422 break;
423 case NVME_PR_REGISTER_ACT_REPLACE:
424 status = nvmet_pr_replace(req, d, ignore_key);
425 break;
426 default:
427 req->error_loc = offsetof(struct nvme_common_command, cdw10);
428 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
429 break;
430 }
431 free_data:
432 kfree(d);
433 out:
434 if (!status)
435 atomic_inc(&req->ns->pr.generation);
436 nvmet_req_complete(req, status);
437 }
438
nvmet_pr_acquire(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 rtype)439 static u16 nvmet_pr_acquire(struct nvmet_req *req,
440 struct nvmet_pr_registrant *reg,
441 u8 rtype)
442 {
443 struct nvmet_pr *pr = &req->ns->pr;
444 struct nvmet_pr_registrant *holder;
445
446 holder = rcu_dereference_protected(pr->holder, 1);
447 if (holder && reg != holder)
448 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
449 if (holder && reg == holder) {
450 if (holder->rtype == rtype)
451 return NVME_SC_SUCCESS;
452 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
453 }
454
455 nvmet_pr_set_new_holder(pr, rtype, reg);
456 return NVME_SC_SUCCESS;
457 }
458
nvmet_pr_confirm_ns_pc_ref(struct percpu_ref * ref)459 static void nvmet_pr_confirm_ns_pc_ref(struct percpu_ref *ref)
460 {
461 struct nvmet_pr_per_ctrl_ref *pc_ref =
462 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);
463
464 complete(&pc_ref->confirm_done);
465 }
466
nvmet_pr_set_ctrl_to_abort(struct nvmet_req * req,uuid_t * hostid)467 static void nvmet_pr_set_ctrl_to_abort(struct nvmet_req *req, uuid_t *hostid)
468 {
469 struct nvmet_pr_per_ctrl_ref *pc_ref;
470 struct nvmet_ns *ns = req->ns;
471 unsigned long idx;
472
473 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
474 if (uuid_equal(&pc_ref->hostid, hostid)) {
475 percpu_ref_kill_and_confirm(&pc_ref->ref,
476 nvmet_pr_confirm_ns_pc_ref);
477 wait_for_completion(&pc_ref->confirm_done);
478 }
479 }
480 }
481
nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req * req,u64 prkey,uuid_t * send_hostid,bool abort)482 static u16 nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req *req, u64 prkey,
483 uuid_t *send_hostid,
484 bool abort)
485 {
486 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
487 struct nvmet_pr_registrant *reg, *tmp;
488 struct nvmet_pr *pr = &req->ns->pr;
489 uuid_t hostid;
490
491 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
492 if (reg->rkey == prkey) {
493 status = NVME_SC_SUCCESS;
494 uuid_copy(&hostid, ®->hostid);
495 if (abort)
496 nvmet_pr_set_ctrl_to_abort(req, &hostid);
497 nvmet_pr_unregister_one(pr, reg);
498 if (!uuid_equal(&hostid, send_hostid))
499 nvmet_pr_registration_preempted(pr, &hostid);
500 }
501 }
502 return status;
503 }
504
nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req * req,u64 prkey,uuid_t * send_hostid,bool abort)505 static void nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req *req,
506 u64 prkey,
507 uuid_t *send_hostid,
508 bool abort)
509 {
510 struct nvmet_pr_registrant *reg, *tmp;
511 struct nvmet_pr *pr = &req->ns->pr;
512 uuid_t hostid;
513
514 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
515 if (reg->rkey == prkey &&
516 !uuid_equal(®->hostid, send_hostid)) {
517 uuid_copy(&hostid, ®->hostid);
518 if (abort)
519 nvmet_pr_set_ctrl_to_abort(req, &hostid);
520 nvmet_pr_unregister_one(pr, reg);
521 nvmet_pr_registration_preempted(pr, &hostid);
522 }
523 }
524 }
525
nvmet_pr_unreg_all_others(struct nvmet_req * req,uuid_t * send_hostid,bool abort)526 static void nvmet_pr_unreg_all_others(struct nvmet_req *req,
527 uuid_t *send_hostid,
528 bool abort)
529 {
530 struct nvmet_pr_registrant *reg, *tmp;
531 struct nvmet_pr *pr = &req->ns->pr;
532 uuid_t hostid;
533
534 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
535 if (!uuid_equal(®->hostid, send_hostid)) {
536 uuid_copy(&hostid, ®->hostid);
537 if (abort)
538 nvmet_pr_set_ctrl_to_abort(req, &hostid);
539 nvmet_pr_unregister_one(pr, reg);
540 nvmet_pr_registration_preempted(pr, &hostid);
541 }
542 }
543 }
544
nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant * reg,void * attr)545 static void nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant *reg,
546 void *attr)
547 {
548 u8 new_rtype = *(u8 *)attr;
549
550 reg->rtype = new_rtype;
551 }
552
nvmet_pr_preempt(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 rtype,struct nvmet_pr_acquire_data * d,bool abort)553 static u16 nvmet_pr_preempt(struct nvmet_req *req,
554 struct nvmet_pr_registrant *reg,
555 u8 rtype,
556 struct nvmet_pr_acquire_data *d,
557 bool abort)
558 {
559 struct nvmet_ctrl *ctrl = req->sq->ctrl;
560 struct nvmet_pr *pr = &req->ns->pr;
561 struct nvmet_pr_registrant *holder;
562 enum nvme_pr_type original_rtype;
563 u64 prkey = le64_to_cpu(d->prkey);
564 u16 status;
565
566 holder = rcu_dereference_protected(pr->holder, 1);
567 if (!holder)
568 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
569 &ctrl->hostid, abort);
570
571 original_rtype = holder->rtype;
572 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
573 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
574 if (!prkey) {
575 /*
576 * To prevent possible access from other hosts, and
577 * avoid terminate the holder, set the new holder
578 * first before unregistering.
579 */
580 nvmet_pr_set_new_holder(pr, rtype, reg);
581 nvmet_pr_unreg_all_others(req, &ctrl->hostid, abort);
582 return NVME_SC_SUCCESS;
583 }
584 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
585 &ctrl->hostid, abort);
586 }
587
588 if (holder == reg) {
589 status = nvmet_pr_update_reg_attr(pr, holder,
590 nvmet_pr_update_holder_rtype, &rtype);
591 if (!status && original_rtype != rtype)
592 nvmet_pr_resv_released(pr, ®->hostid);
593 return status;
594 }
595
596 if (prkey == holder->rkey) {
597 /*
598 * Same as before, set the new holder first.
599 */
600 nvmet_pr_set_new_holder(pr, rtype, reg);
601 nvmet_pr_unreg_all_others_by_prkey(req, prkey, &ctrl->hostid,
602 abort);
603 if (original_rtype != rtype)
604 nvmet_pr_resv_released(pr, ®->hostid);
605 return NVME_SC_SUCCESS;
606 }
607
608 if (prkey)
609 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
610 &ctrl->hostid, abort);
611 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
612 }
613
nvmet_pr_do_abort(struct work_struct * w)614 static void nvmet_pr_do_abort(struct work_struct *w)
615 {
616 struct nvmet_req *req = container_of(w, struct nvmet_req, r.abort_work);
617 struct nvmet_pr_per_ctrl_ref *pc_ref;
618 struct nvmet_ns *ns = req->ns;
619 unsigned long idx;
620
621 /*
622 * The target does not support abort, just wait per-controller ref to 0.
623 */
624 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
625 if (percpu_ref_is_dying(&pc_ref->ref)) {
626 wait_for_completion(&pc_ref->free_done);
627 reinit_completion(&pc_ref->confirm_done);
628 reinit_completion(&pc_ref->free_done);
629 percpu_ref_resurrect(&pc_ref->ref);
630 }
631 }
632
633 up(&ns->pr.pr_sem);
634 nvmet_req_complete(req, NVME_SC_SUCCESS);
635 }
636
__nvmet_execute_pr_acquire(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 acquire_act,u8 rtype,struct nvmet_pr_acquire_data * d)637 static u16 __nvmet_execute_pr_acquire(struct nvmet_req *req,
638 struct nvmet_pr_registrant *reg,
639 u8 acquire_act,
640 u8 rtype,
641 struct nvmet_pr_acquire_data *d)
642 {
643 u16 status;
644
645 switch (acquire_act) {
646 case NVME_PR_ACQUIRE_ACT_ACQUIRE:
647 status = nvmet_pr_acquire(req, reg, rtype);
648 goto out;
649 case NVME_PR_ACQUIRE_ACT_PREEMPT:
650 status = nvmet_pr_preempt(req, reg, rtype, d, false);
651 goto inc_gen;
652 case NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT:
653 status = nvmet_pr_preempt(req, reg, rtype, d, true);
654 goto inc_gen;
655 default:
656 req->error_loc = offsetof(struct nvme_common_command, cdw10);
657 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
658 goto out;
659 }
660 inc_gen:
661 if (!status)
662 atomic_inc(&req->ns->pr.generation);
663 out:
664 return status;
665 }
666
nvmet_execute_pr_acquire(struct nvmet_req * req)667 static void nvmet_execute_pr_acquire(struct nvmet_req *req)
668 {
669 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
670 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
671 /* Reservation type, bit 15:08 */
672 u8 rtype = (u8)((cdw10 >> 8) & 0xff);
673 /* Reservation acquire action, bit 02:00 */
674 u8 acquire_act = cdw10 & 0x07;
675 struct nvmet_ctrl *ctrl = req->sq->ctrl;
676 struct nvmet_pr_acquire_data *d = NULL;
677 struct nvmet_pr *pr = &req->ns->pr;
678 struct nvmet_pr_registrant *reg;
679 u16 status = NVME_SC_SUCCESS;
680
681 if (ignore_key ||
682 rtype < NVME_PR_WRITE_EXCLUSIVE ||
683 rtype > NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
684 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
685 goto out;
686 }
687
688 d = kmalloc_obj(*d);
689 if (!d) {
690 status = NVME_SC_INTERNAL;
691 goto out;
692 }
693
694 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
695 if (status)
696 goto free_data;
697
698 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
699 down(&pr->pr_sem);
700 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
701 if (uuid_equal(®->hostid, &ctrl->hostid) &&
702 reg->rkey == le64_to_cpu(d->crkey)) {
703 status = __nvmet_execute_pr_acquire(req, reg,
704 acquire_act, rtype, d);
705 break;
706 }
707 }
708
709 if (!status && acquire_act == NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT) {
710 kfree(d);
711 INIT_WORK(&req->r.abort_work, nvmet_pr_do_abort);
712 queue_work(nvmet_wq, &req->r.abort_work);
713 return;
714 }
715
716 up(&pr->pr_sem);
717
718 free_data:
719 kfree(d);
720 out:
721 nvmet_req_complete(req, status);
722 }
723
nvmet_pr_release(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 rtype)724 static u16 nvmet_pr_release(struct nvmet_req *req,
725 struct nvmet_pr_registrant *reg,
726 u8 rtype)
727 {
728 struct nvmet_pr *pr = &req->ns->pr;
729 struct nvmet_pr_registrant *holder;
730 u8 original_rtype;
731
732 holder = rcu_dereference_protected(pr->holder, 1);
733 if (!holder || reg != holder)
734 return NVME_SC_SUCCESS;
735
736 original_rtype = holder->rtype;
737 if (original_rtype != rtype)
738 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
739
740 rcu_assign_pointer(pr->holder, NULL);
741
742 if (original_rtype != NVME_PR_WRITE_EXCLUSIVE &&
743 original_rtype != NVME_PR_EXCLUSIVE_ACCESS)
744 nvmet_pr_resv_released(pr, ®->hostid);
745
746 return NVME_SC_SUCCESS;
747 }
748
nvmet_pr_clear(struct nvmet_req * req)749 static void nvmet_pr_clear(struct nvmet_req *req)
750 {
751 struct nvmet_pr_registrant *reg, *tmp;
752 struct nvmet_pr *pr = &req->ns->pr;
753
754 rcu_assign_pointer(pr->holder, NULL);
755
756 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
757 list_del_rcu(®->entry);
758 if (!uuid_equal(&req->sq->ctrl->hostid, ®->hostid))
759 nvmet_pr_resv_preempted(pr, ®->hostid);
760 kfree_rcu(reg, rcu);
761 }
762
763 atomic_inc(&pr->generation);
764 }
765
__nvmet_execute_pr_release(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 release_act,u8 rtype)766 static u16 __nvmet_execute_pr_release(struct nvmet_req *req,
767 struct nvmet_pr_registrant *reg,
768 u8 release_act, u8 rtype)
769 {
770 switch (release_act) {
771 case NVME_PR_RELEASE_ACT_RELEASE:
772 return nvmet_pr_release(req, reg, rtype);
773 case NVME_PR_RELEASE_ACT_CLEAR:
774 nvmet_pr_clear(req);
775 return NVME_SC_SUCCESS;
776 default:
777 req->error_loc = offsetof(struct nvme_common_command, cdw10);
778 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
779 }
780 }
781
nvmet_execute_pr_release(struct nvmet_req * req)782 static void nvmet_execute_pr_release(struct nvmet_req *req)
783 {
784 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
785 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
786 u8 rtype = (u8)((cdw10 >> 8) & 0xff); /* Reservation type, bit 15:08 */
787 u8 release_act = cdw10 & 0x07; /* Reservation release action, bit 02:00 */
788 struct nvmet_ctrl *ctrl = req->sq->ctrl;
789 struct nvmet_pr *pr = &req->ns->pr;
790 struct nvmet_pr_release_data *d;
791 struct nvmet_pr_registrant *reg;
792 u16 status;
793
794 if (ignore_key) {
795 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
796 goto out;
797 }
798
799 d = kmalloc_obj(*d);
800 if (!d) {
801 status = NVME_SC_INTERNAL;
802 goto out;
803 }
804
805 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
806 if (status)
807 goto free_data;
808
809 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
810 down(&pr->pr_sem);
811 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
812 if (uuid_equal(®->hostid, &ctrl->hostid) &&
813 reg->rkey == le64_to_cpu(d->crkey)) {
814 status = __nvmet_execute_pr_release(req, reg,
815 release_act, rtype);
816 break;
817 }
818 }
819 up(&pr->pr_sem);
820 free_data:
821 kfree(d);
822 out:
823 nvmet_req_complete(req, status);
824 }
825
nvmet_execute_pr_report(struct nvmet_req * req)826 static void nvmet_execute_pr_report(struct nvmet_req *req)
827 {
828 u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11);
829 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
830 u32 num_bytes = 4 * (cdw10 + 1); /* cdw10 is number of dwords */
831 u8 eds = cdw11 & 1; /* Extended data structure, bit 00 */
832 struct nvme_registered_ctrl_ext *ctrl_eds;
833 struct nvme_reservation_status_ext *data;
834 struct nvmet_pr *pr = &req->ns->pr;
835 struct nvmet_pr_registrant *holder;
836 struct nvmet_pr_registrant *reg;
837 u16 num_ctrls = 0;
838 u16 status;
839 u8 rtype;
840
841 /* nvmet hostid(uuid_t) is 128 bit. */
842 if (!eds) {
843 req->error_loc = offsetof(struct nvme_common_command, cdw11);
844 status = NVME_SC_HOST_ID_INCONSIST | NVME_STATUS_DNR;
845 goto out;
846 }
847
848 if (num_bytes < sizeof(struct nvme_reservation_status_ext)) {
849 req->error_loc = offsetof(struct nvme_common_command, cdw10);
850 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
851 goto out;
852 }
853
854 data = kzalloc(num_bytes, GFP_KERNEL);
855 if (!data) {
856 status = NVME_SC_INTERNAL;
857 goto out;
858 }
859 data->gen = cpu_to_le32(atomic_read(&pr->generation));
860 data->ptpls = 0;
861 ctrl_eds = data->regctl_eds;
862
863 rcu_read_lock();
864 holder = rcu_dereference(pr->holder);
865 rtype = holder ? holder->rtype : 0;
866 data->rtype = rtype;
867
868 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
869 num_ctrls++;
870 /*
871 * continue to get the number of all registrans.
872 */
873 if (((void *)ctrl_eds + sizeof(*ctrl_eds)) >
874 ((void *)data + num_bytes))
875 continue;
876 /*
877 * Dynamic controller, set cntlid to 0xffff.
878 */
879 ctrl_eds->cntlid = cpu_to_le16(NVME_CNTLID_DYNAMIC);
880 if (rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
881 rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS)
882 ctrl_eds->rcsts = 1;
883 if (reg == holder)
884 ctrl_eds->rcsts = 1;
885 uuid_copy((uuid_t *)&ctrl_eds->hostid, ®->hostid);
886 ctrl_eds->rkey = cpu_to_le64(reg->rkey);
887 ctrl_eds++;
888 }
889 rcu_read_unlock();
890
891 put_unaligned_le16(num_ctrls, data->regctl);
892 status = nvmet_copy_to_sgl(req, 0, data, num_bytes);
893 kfree(data);
894 out:
895 nvmet_req_complete(req, status);
896 }
897
nvmet_parse_pr_cmd(struct nvmet_req * req)898 u16 nvmet_parse_pr_cmd(struct nvmet_req *req)
899 {
900 struct nvme_command *cmd = req->cmd;
901
902 switch (cmd->common.opcode) {
903 case nvme_cmd_resv_register:
904 req->execute = nvmet_execute_pr_register;
905 break;
906 case nvme_cmd_resv_acquire:
907 req->execute = nvmet_execute_pr_acquire;
908 break;
909 case nvme_cmd_resv_release:
910 req->execute = nvmet_execute_pr_release;
911 break;
912 case nvme_cmd_resv_report:
913 req->execute = nvmet_execute_pr_report;
914 break;
915 default:
916 return 1;
917 }
918 return NVME_SC_SUCCESS;
919 }
920
nvmet_is_req_write_cmd_group(struct nvmet_req * req)921 static bool nvmet_is_req_write_cmd_group(struct nvmet_req *req)
922 {
923 u8 opcode = req->cmd->common.opcode;
924
925 if (req->sq->qid) {
926 switch (opcode) {
927 case nvme_cmd_flush:
928 case nvme_cmd_write:
929 case nvme_cmd_write_zeroes:
930 case nvme_cmd_dsm:
931 case nvme_cmd_zone_append:
932 case nvme_cmd_zone_mgmt_send:
933 return true;
934 default:
935 return false;
936 }
937 }
938 return false;
939 }
940
nvmet_is_req_read_cmd_group(struct nvmet_req * req)941 static bool nvmet_is_req_read_cmd_group(struct nvmet_req *req)
942 {
943 u8 opcode = req->cmd->common.opcode;
944
945 if (req->sq->qid) {
946 switch (opcode) {
947 case nvme_cmd_read:
948 case nvme_cmd_zone_mgmt_recv:
949 return true;
950 default:
951 return false;
952 }
953 }
954 return false;
955 }
956
nvmet_pr_check_cmd_access(struct nvmet_req * req)957 u16 nvmet_pr_check_cmd_access(struct nvmet_req *req)
958 {
959 struct nvmet_ctrl *ctrl = req->sq->ctrl;
960 struct nvmet_pr_registrant *holder;
961 struct nvmet_ns *ns = req->ns;
962 struct nvmet_pr *pr = &ns->pr;
963 u16 status = NVME_SC_SUCCESS;
964
965 rcu_read_lock();
966 holder = rcu_dereference(pr->holder);
967 if (!holder)
968 goto unlock;
969 if (uuid_equal(&ctrl->hostid, &holder->hostid))
970 goto unlock;
971
972 /*
973 * The Reservation command group is checked in executing,
974 * allow it here.
975 */
976 switch (holder->rtype) {
977 case NVME_PR_WRITE_EXCLUSIVE:
978 if (nvmet_is_req_write_cmd_group(req))
979 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
980 break;
981 case NVME_PR_EXCLUSIVE_ACCESS:
982 if (nvmet_is_req_read_cmd_group(req) ||
983 nvmet_is_req_write_cmd_group(req))
984 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
985 break;
986 case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY:
987 case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS:
988 if ((nvmet_is_req_write_cmd_group(req)) &&
989 !nvmet_pr_find_registrant(pr, &ctrl->hostid))
990 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
991 break;
992 case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY:
993 case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS:
994 if ((nvmet_is_req_read_cmd_group(req) ||
995 nvmet_is_req_write_cmd_group(req)) &&
996 !nvmet_pr_find_registrant(pr, &ctrl->hostid))
997 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
998 break;
999 default:
1000 pr_warn("the reservation type is set wrong, type:%d\n",
1001 holder->rtype);
1002 break;
1003 }
1004
1005 unlock:
1006 rcu_read_unlock();
1007 if (status)
1008 req->error_loc = offsetof(struct nvme_common_command, opcode);
1009 return status;
1010 }
1011
nvmet_pr_get_ns_pc_ref(struct nvmet_req * req)1012 u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req)
1013 {
1014 struct nvmet_pr_per_ctrl_ref *pc_ref;
1015
1016 pc_ref = xa_load(&req->ns->pr_per_ctrl_refs,
1017 req->sq->ctrl->cntlid);
1018 if (unlikely(!percpu_ref_tryget_live(&pc_ref->ref)))
1019 return NVME_SC_INTERNAL;
1020 req->pc_ref = pc_ref;
1021 return NVME_SC_SUCCESS;
1022 }
1023
nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref * ref)1024 static void nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref *ref)
1025 {
1026 struct nvmet_pr_per_ctrl_ref *pc_ref =
1027 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);
1028
1029 complete(&pc_ref->free_done);
1030 }
1031
nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns * ns,unsigned long idx,uuid_t * hostid)1032 static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns,
1033 unsigned long idx,
1034 uuid_t *hostid)
1035 {
1036 struct nvmet_pr_per_ctrl_ref *pc_ref;
1037 int ret;
1038
1039 pc_ref = kmalloc_obj(*pc_ref, GFP_ATOMIC);
1040 if (!pc_ref)
1041 return -ENOMEM;
1042
1043 ret = percpu_ref_init(&pc_ref->ref, nvmet_pr_ctrl_ns_all_cmds_done,
1044 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
1045 if (ret)
1046 goto free;
1047
1048 init_completion(&pc_ref->free_done);
1049 init_completion(&pc_ref->confirm_done);
1050 uuid_copy(&pc_ref->hostid, hostid);
1051
1052 ret = xa_insert(&ns->pr_per_ctrl_refs, idx, pc_ref, GFP_KERNEL);
1053 if (ret)
1054 goto exit;
1055 return ret;
1056 exit:
1057 percpu_ref_exit(&pc_ref->ref);
1058 free:
1059 kfree(pc_ref);
1060 return ret;
1061 }
1062
nvmet_ctrl_init_pr(struct nvmet_ctrl * ctrl)1063 int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl)
1064 {
1065 struct nvmet_subsys *subsys = ctrl->subsys;
1066 struct nvmet_pr_per_ctrl_ref *pc_ref;
1067 struct nvmet_ns *ns = NULL;
1068 unsigned long idx;
1069 int ret;
1070
1071 ctrl->pr_log_mgr.counter = 0;
1072 ctrl->pr_log_mgr.lost_count = 0;
1073 mutex_init(&ctrl->pr_log_mgr.lock);
1074 INIT_KFIFO(ctrl->pr_log_mgr.log_queue);
1075
1076 /*
1077 * Here we are under subsys lock, if an ns not in subsys->namespaces,
1078 * we can make sure that ns is not enabled, and not call
1079 * nvmet_pr_init_ns(), see more details in nvmet_ns_enable().
1080 * So just check ns->pr.enable.
1081 */
1082 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
1083 if (ns->pr.enable) {
1084 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
1085 &ctrl->hostid);
1086 if (ret)
1087 goto free_per_ctrl_refs;
1088 }
1089 }
1090 return 0;
1091
1092 free_per_ctrl_refs:
1093 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
1094 if (ns->pr.enable) {
1095 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
1096 if (pc_ref)
1097 percpu_ref_exit(&pc_ref->ref);
1098 kfree(pc_ref);
1099 }
1100 }
1101 return ret;
1102 }
1103
nvmet_ctrl_destroy_pr(struct nvmet_ctrl * ctrl)1104 void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl)
1105 {
1106 struct nvmet_pr_per_ctrl_ref *pc_ref;
1107 struct nvmet_ns *ns;
1108 unsigned long idx;
1109
1110 kfifo_free(&ctrl->pr_log_mgr.log_queue);
1111 mutex_destroy(&ctrl->pr_log_mgr.lock);
1112
1113 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
1114 if (ns->pr.enable) {
1115 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
1116 if (pc_ref)
1117 percpu_ref_exit(&pc_ref->ref);
1118 kfree(pc_ref);
1119 }
1120 }
1121 }
1122
nvmet_pr_init_ns(struct nvmet_ns * ns)1123 int nvmet_pr_init_ns(struct nvmet_ns *ns)
1124 {
1125 struct nvmet_subsys *subsys = ns->subsys;
1126 struct nvmet_pr_per_ctrl_ref *pc_ref;
1127 struct nvmet_ctrl *ctrl = NULL;
1128 unsigned long idx;
1129 int ret;
1130
1131 ns->pr.holder = NULL;
1132 atomic_set(&ns->pr.generation, 0);
1133 sema_init(&ns->pr.pr_sem, 1);
1134 INIT_LIST_HEAD(&ns->pr.registrant_list);
1135 ns->pr.notify_mask = 0;
1136
1137 xa_init(&ns->pr_per_ctrl_refs);
1138
1139 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
1140 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
1141 &ctrl->hostid);
1142 if (ret)
1143 goto free_per_ctrl_refs;
1144 }
1145 return 0;
1146
1147 free_per_ctrl_refs:
1148 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
1149 xa_erase(&ns->pr_per_ctrl_refs, idx);
1150 percpu_ref_exit(&pc_ref->ref);
1151 kfree(pc_ref);
1152 }
1153 return ret;
1154 }
1155
nvmet_pr_exit_ns(struct nvmet_ns * ns)1156 void nvmet_pr_exit_ns(struct nvmet_ns *ns)
1157 {
1158 struct nvmet_pr_registrant *reg, *tmp;
1159 struct nvmet_pr_per_ctrl_ref *pc_ref;
1160 struct nvmet_pr *pr = &ns->pr;
1161 unsigned long idx;
1162
1163 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
1164 list_del(®->entry);
1165 kfree(reg);
1166 }
1167
1168 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
1169 /*
1170 * No command on ns here, we can safely free pc_ref.
1171 */
1172 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, idx);
1173 percpu_ref_exit(&pc_ref->ref);
1174 kfree(pc_ref);
1175 }
1176
1177 xa_destroy(&ns->pr_per_ctrl_refs);
1178 }
1179