1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * NVMe over Fabrics Persist Reservation.
4 * Copyright (c) 2024 Guixin Liu, Alibaba Group.
5 * All rights reserved.
6 */
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/unaligned.h>
9 #include "nvmet.h"
10
11 #define NVMET_PR_NOTIFY_MASK_ALL \
12 (1 << NVME_PR_NOTIFY_BIT_REG_PREEMPTED | \
13 1 << NVME_PR_NOTIFY_BIT_RESV_RELEASED | \
14 1 << NVME_PR_NOTIFY_BIT_RESV_PREEMPTED)
15
nvmet_pr_parse_ignore_key(u32 cdw10)16 static inline bool nvmet_pr_parse_ignore_key(u32 cdw10)
17 {
18 /* Ignore existing key, bit 03. */
19 return (cdw10 >> 3) & 1;
20 }
21
nvmet_pr_to_ns(struct nvmet_pr * pr)22 static inline struct nvmet_ns *nvmet_pr_to_ns(struct nvmet_pr *pr)
23 {
24 return container_of(pr, struct nvmet_ns, pr);
25 }
26
27 static struct nvmet_pr_registrant *
nvmet_pr_find_registrant(struct nvmet_pr * pr,uuid_t * hostid)28 nvmet_pr_find_registrant(struct nvmet_pr *pr, uuid_t *hostid)
29 {
30 struct nvmet_pr_registrant *reg;
31
32 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
33 if (uuid_equal(®->hostid, hostid))
34 return reg;
35 }
36 return NULL;
37 }
38
nvmet_set_feat_resv_notif_mask(struct nvmet_req * req,u32 mask)39 u16 nvmet_set_feat_resv_notif_mask(struct nvmet_req *req, u32 mask)
40 {
41 u32 nsid = le32_to_cpu(req->cmd->common.nsid);
42 struct nvmet_ctrl *ctrl = req->sq->ctrl;
43 struct nvmet_ns *ns;
44 unsigned long idx;
45 u16 status;
46
47 if (mask & ~(NVMET_PR_NOTIFY_MASK_ALL)) {
48 req->error_loc = offsetof(struct nvme_common_command, cdw11);
49 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
50 }
51
52 if (nsid != U32_MAX) {
53 status = nvmet_req_find_ns(req);
54 if (status == (NVME_SC_INVALID_NS | NVME_STATUS_DNR))
55 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
56 if (status)
57 return status;
58 if (!req->ns->pr.enable)
59 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
60
61 WRITE_ONCE(req->ns->pr.notify_mask, mask);
62 goto success;
63 }
64
65 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
66 if (ns->pr.enable)
67 WRITE_ONCE(ns->pr.notify_mask, mask);
68 }
69
70 success:
71 nvmet_set_result(req, mask);
72 return NVME_SC_SUCCESS;
73 }
74
nvmet_get_feat_resv_notif_mask(struct nvmet_req * req)75 u16 nvmet_get_feat_resv_notif_mask(struct nvmet_req *req)
76 {
77 u16 status;
78
79 status = nvmet_req_find_ns(req);
80 if (status == (NVME_SC_INVALID_NS | NVME_STATUS_DNR))
81 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
82 if (status)
83 return status;
84
85 if (!req->ns->pr.enable)
86 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
87
88 nvmet_set_result(req, READ_ONCE(req->ns->pr.notify_mask));
89 return status;
90 }
91
nvmet_execute_get_log_page_resv(struct nvmet_req * req)92 void nvmet_execute_get_log_page_resv(struct nvmet_req *req)
93 {
94 struct nvmet_pr_log_mgr *log_mgr = &req->sq->ctrl->pr_log_mgr;
95 struct nvme_pr_log next_log = {0};
96 struct nvme_pr_log log = {0};
97 u16 status = NVME_SC_SUCCESS;
98 u64 lost_count;
99 u64 cur_count;
100 u64 next_count;
101
102 mutex_lock(&log_mgr->lock);
103 if (!kfifo_get(&log_mgr->log_queue, &log))
104 goto out;
105
106 /*
107 * We can't get the last in kfifo.
108 * Utilize the current count and the count from the next log to
109 * calculate the number of lost logs, while also addressing cases
110 * of overflow. If there is no subsequent log, the number of lost
111 * logs is equal to the lost_count within the nvmet_pr_log_mgr.
112 */
113 cur_count = le64_to_cpu(log.count);
114 if (kfifo_peek(&log_mgr->log_queue, &next_log)) {
115 next_count = le64_to_cpu(next_log.count);
116 if (next_count > cur_count)
117 lost_count = next_count - cur_count - 1;
118 else
119 lost_count = U64_MAX - cur_count + next_count - 1;
120 } else {
121 lost_count = log_mgr->lost_count;
122 }
123
124 log.count = cpu_to_le64((cur_count + lost_count) == 0 ?
125 1 : (cur_count + lost_count));
126 log_mgr->lost_count -= lost_count;
127
128 log.nr_pages = kfifo_len(&log_mgr->log_queue);
129
130 out:
131 status = nvmet_copy_to_sgl(req, 0, &log, sizeof(log));
132 mutex_unlock(&log_mgr->lock);
133 nvmet_req_complete(req, status);
134 }
135
nvmet_pr_add_resv_log(struct nvmet_ctrl * ctrl,u8 log_type,u32 nsid)136 static void nvmet_pr_add_resv_log(struct nvmet_ctrl *ctrl, u8 log_type,
137 u32 nsid)
138 {
139 struct nvmet_pr_log_mgr *log_mgr = &ctrl->pr_log_mgr;
140 struct nvme_pr_log log = {0};
141
142 mutex_lock(&log_mgr->lock);
143 log_mgr->counter++;
144 if (log_mgr->counter == 0)
145 log_mgr->counter = 1;
146
147 log.count = cpu_to_le64(log_mgr->counter);
148 log.type = log_type;
149 log.nsid = cpu_to_le32(nsid);
150
151 if (!kfifo_put(&log_mgr->log_queue, log)) {
152 pr_info("a reservation log lost, cntlid:%d, log_type:%d, nsid:%u\n",
153 ctrl->cntlid, log_type, nsid);
154 log_mgr->lost_count++;
155 }
156
157 mutex_unlock(&log_mgr->lock);
158 }
159
nvmet_pr_resv_released(struct nvmet_pr * pr,uuid_t * hostid)160 static void nvmet_pr_resv_released(struct nvmet_pr *pr, uuid_t *hostid)
161 {
162 struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
163 struct nvmet_subsys *subsys = ns->subsys;
164 struct nvmet_ctrl *ctrl;
165
166 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_RELEASED, &pr->notify_mask))
167 return;
168
169 mutex_lock(&subsys->lock);
170 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
171 if (!uuid_equal(&ctrl->hostid, hostid) &&
172 nvmet_pr_find_registrant(pr, &ctrl->hostid)) {
173 nvmet_pr_add_resv_log(ctrl,
174 NVME_PR_LOG_RESERVATION_RELEASED, ns->nsid);
175 nvmet_add_async_event(ctrl, NVME_AER_CSS,
176 NVME_AEN_RESV_LOG_PAGE_AVAILABLE,
177 NVME_LOG_RESERVATION);
178 }
179 }
180 mutex_unlock(&subsys->lock);
181 }
182
nvmet_pr_send_event_to_host(struct nvmet_pr * pr,uuid_t * hostid,u8 log_type)183 static void nvmet_pr_send_event_to_host(struct nvmet_pr *pr, uuid_t *hostid,
184 u8 log_type)
185 {
186 struct nvmet_ns *ns = nvmet_pr_to_ns(pr);
187 struct nvmet_subsys *subsys = ns->subsys;
188 struct nvmet_ctrl *ctrl;
189
190 mutex_lock(&subsys->lock);
191 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
192 if (uuid_equal(hostid, &ctrl->hostid)) {
193 nvmet_pr_add_resv_log(ctrl, log_type, ns->nsid);
194 nvmet_add_async_event(ctrl, NVME_AER_CSS,
195 NVME_AEN_RESV_LOG_PAGE_AVAILABLE,
196 NVME_LOG_RESERVATION);
197 }
198 }
199 mutex_unlock(&subsys->lock);
200 }
201
nvmet_pr_resv_preempted(struct nvmet_pr * pr,uuid_t * hostid)202 static void nvmet_pr_resv_preempted(struct nvmet_pr *pr, uuid_t *hostid)
203 {
204 if (test_bit(NVME_PR_NOTIFY_BIT_RESV_PREEMPTED, &pr->notify_mask))
205 return;
206
207 nvmet_pr_send_event_to_host(pr, hostid,
208 NVME_PR_LOG_RESERVATION_PREEMPTED);
209 }
210
nvmet_pr_registration_preempted(struct nvmet_pr * pr,uuid_t * hostid)211 static void nvmet_pr_registration_preempted(struct nvmet_pr *pr,
212 uuid_t *hostid)
213 {
214 if (test_bit(NVME_PR_NOTIFY_BIT_REG_PREEMPTED, &pr->notify_mask))
215 return;
216
217 nvmet_pr_send_event_to_host(pr, hostid,
218 NVME_PR_LOG_REGISTRATION_PREEMPTED);
219 }
220
nvmet_pr_set_new_holder(struct nvmet_pr * pr,u8 new_rtype,struct nvmet_pr_registrant * reg)221 static inline void nvmet_pr_set_new_holder(struct nvmet_pr *pr, u8 new_rtype,
222 struct nvmet_pr_registrant *reg)
223 {
224 reg->rtype = new_rtype;
225 rcu_assign_pointer(pr->holder, reg);
226 }
227
nvmet_pr_register(struct nvmet_req * req,struct nvmet_pr_register_data * d)228 static u16 nvmet_pr_register(struct nvmet_req *req,
229 struct nvmet_pr_register_data *d)
230 {
231 struct nvmet_ctrl *ctrl = req->sq->ctrl;
232 struct nvmet_pr_registrant *new, *reg;
233 struct nvmet_pr *pr = &req->ns->pr;
234 u16 status = NVME_SC_SUCCESS;
235 u64 nrkey = le64_to_cpu(d->nrkey);
236
237 new = kmalloc_obj(*new);
238 if (!new)
239 return NVME_SC_INTERNAL;
240
241 down(&pr->pr_sem);
242 reg = nvmet_pr_find_registrant(pr, &ctrl->hostid);
243 if (reg) {
244 if (reg->rkey != nrkey)
245 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
246 kfree(new);
247 goto out;
248 }
249
250 memset(new, 0, sizeof(*new));
251 INIT_LIST_HEAD(&new->entry);
252 new->rkey = nrkey;
253 uuid_copy(&new->hostid, &ctrl->hostid);
254 list_add_tail_rcu(&new->entry, &pr->registrant_list);
255
256 out:
257 up(&pr->pr_sem);
258 return status;
259 }
260
nvmet_pr_unregister_one(struct nvmet_pr * pr,struct nvmet_pr_registrant * reg)261 static void nvmet_pr_unregister_one(struct nvmet_pr *pr,
262 struct nvmet_pr_registrant *reg)
263 {
264 struct nvmet_pr_registrant *first_reg;
265 struct nvmet_pr_registrant *holder;
266 u8 original_rtype;
267
268 list_del_rcu(®->entry);
269
270 holder = rcu_dereference_protected(pr->holder, 1);
271 if (reg != holder)
272 goto out;
273
274 original_rtype = holder->rtype;
275 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
276 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
277 first_reg = list_first_or_null_rcu(&pr->registrant_list,
278 struct nvmet_pr_registrant, entry);
279 if (first_reg)
280 first_reg->rtype = original_rtype;
281 rcu_assign_pointer(pr->holder, first_reg);
282 } else {
283 rcu_assign_pointer(pr->holder, NULL);
284
285 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_REG_ONLY ||
286 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY)
287 nvmet_pr_resv_released(pr, ®->hostid);
288 }
289 out:
290 kfree_rcu(reg, rcu);
291 }
292
nvmet_pr_unregister(struct nvmet_req * req,struct nvmet_pr_register_data * d,bool ignore_key)293 static u16 nvmet_pr_unregister(struct nvmet_req *req,
294 struct nvmet_pr_register_data *d,
295 bool ignore_key)
296 {
297 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
298 struct nvmet_ctrl *ctrl = req->sq->ctrl;
299 struct nvmet_pr *pr = &req->ns->pr;
300 struct nvmet_pr_registrant *reg;
301
302 down(&pr->pr_sem);
303 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
304 if (uuid_equal(®->hostid, &ctrl->hostid)) {
305 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey)) {
306 status = NVME_SC_SUCCESS;
307 nvmet_pr_unregister_one(pr, reg);
308 }
309 break;
310 }
311 }
312 up(&pr->pr_sem);
313
314 return status;
315 }
316
nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant * reg,void * attr)317 static void nvmet_pr_update_reg_rkey(struct nvmet_pr_registrant *reg,
318 void *attr)
319 {
320 reg->rkey = *(u64 *)attr;
321 }
322
nvmet_pr_update_reg_attr(struct nvmet_pr * pr,struct nvmet_pr_registrant * reg,void (* change_attr)(struct nvmet_pr_registrant * reg,void * attr),void * attr)323 static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr,
324 struct nvmet_pr_registrant *reg,
325 void (*change_attr)(struct nvmet_pr_registrant *reg,
326 void *attr),
327 void *attr)
328 {
329 struct nvmet_pr_registrant *holder;
330 struct nvmet_pr_registrant *new;
331
332 holder = rcu_dereference_protected(pr->holder, 1);
333 if (reg != holder) {
334 change_attr(reg, attr);
335 return NVME_SC_SUCCESS;
336 }
337
338 new = kmalloc_obj(*new, GFP_ATOMIC);
339 if (!new)
340 return NVME_SC_INTERNAL;
341
342 new->rkey = holder->rkey;
343 new->rtype = holder->rtype;
344 uuid_copy(&new->hostid, &holder->hostid);
345 INIT_LIST_HEAD(&new->entry);
346
347 change_attr(new, attr);
348 list_replace_rcu(&holder->entry, &new->entry);
349 rcu_assign_pointer(pr->holder, new);
350 kfree_rcu(holder, rcu);
351
352 return NVME_SC_SUCCESS;
353 }
354
nvmet_pr_replace(struct nvmet_req * req,struct nvmet_pr_register_data * d,bool ignore_key)355 static u16 nvmet_pr_replace(struct nvmet_req *req,
356 struct nvmet_pr_register_data *d,
357 bool ignore_key)
358 {
359 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
360 struct nvmet_ctrl *ctrl = req->sq->ctrl;
361 struct nvmet_pr *pr = &req->ns->pr;
362 struct nvmet_pr_registrant *reg, *new = NULL;
363 u64 nrkey = le64_to_cpu(d->nrkey);
364
365 if (ignore_key && nrkey) {
366 new = kzalloc_obj(*new);
367 if (!new)
368 return NVME_SC_INTERNAL;
369 }
370
371 down(&pr->pr_sem);
372 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
373 if (uuid_equal(®->hostid, &ctrl->hostid)) {
374 if (ignore_key || reg->rkey == le64_to_cpu(d->crkey))
375 status = nvmet_pr_update_reg_attr(pr, reg,
376 nvmet_pr_update_reg_rkey,
377 &nrkey);
378 goto free_data;
379 }
380 }
381
382 if (ignore_key) {
383 if (!nrkey) {
384 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
385 goto free_data;
386 }
387 INIT_LIST_HEAD(&new->entry);
388 new->rkey = nrkey;
389 uuid_copy(&new->hostid, &ctrl->hostid);
390 list_add_tail_rcu(&new->entry, &pr->registrant_list);
391 status = NVME_SC_SUCCESS;
392 goto out;
393 }
394
395 free_data:
396 kfree(new);
397 out:
398 up(&pr->pr_sem);
399 return status;
400 }
401
nvmet_execute_pr_register(struct nvmet_req * req)402 static void nvmet_execute_pr_register(struct nvmet_req *req)
403 {
404 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
405 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
406 struct nvmet_pr_register_data *d;
407 u8 reg_act = cdw10 & 0x07; /* Reservation Register Action, bit 02:00 */
408 u16 status;
409
410 d = kmalloc_obj(*d);
411 if (!d) {
412 status = NVME_SC_INTERNAL;
413 goto out;
414 }
415
416 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
417 if (status)
418 goto free_data;
419
420 switch (reg_act) {
421 case NVME_PR_REGISTER_ACT_REG:
422 status = nvmet_pr_register(req, d);
423 break;
424 case NVME_PR_REGISTER_ACT_UNREG:
425 status = nvmet_pr_unregister(req, d, ignore_key);
426 break;
427 case NVME_PR_REGISTER_ACT_REPLACE:
428 status = nvmet_pr_replace(req, d, ignore_key);
429 break;
430 default:
431 req->error_loc = offsetof(struct nvme_common_command, cdw10);
432 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
433 break;
434 }
435 free_data:
436 kfree(d);
437 out:
438 if (!status)
439 atomic_inc(&req->ns->pr.generation);
440 nvmet_req_complete(req, status);
441 }
442
nvmet_pr_acquire(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 rtype)443 static u16 nvmet_pr_acquire(struct nvmet_req *req,
444 struct nvmet_pr_registrant *reg,
445 u8 rtype)
446 {
447 struct nvmet_pr *pr = &req->ns->pr;
448 struct nvmet_pr_registrant *holder;
449
450 holder = rcu_dereference_protected(pr->holder, 1);
451 if (holder && reg != holder)
452 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
453 if (holder && reg == holder) {
454 if (holder->rtype == rtype)
455 return NVME_SC_SUCCESS;
456 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
457 }
458
459 nvmet_pr_set_new_holder(pr, rtype, reg);
460 return NVME_SC_SUCCESS;
461 }
462
nvmet_pr_confirm_ns_pc_ref(struct percpu_ref * ref)463 static void nvmet_pr_confirm_ns_pc_ref(struct percpu_ref *ref)
464 {
465 struct nvmet_pr_per_ctrl_ref *pc_ref =
466 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);
467
468 complete(&pc_ref->confirm_done);
469 }
470
nvmet_pr_set_ctrl_to_abort(struct nvmet_req * req,uuid_t * hostid)471 static void nvmet_pr_set_ctrl_to_abort(struct nvmet_req *req, uuid_t *hostid)
472 {
473 struct nvmet_pr_per_ctrl_ref *pc_ref;
474 struct nvmet_ns *ns = req->ns;
475 unsigned long idx;
476
477 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
478 if (uuid_equal(&pc_ref->hostid, hostid)) {
479 percpu_ref_kill_and_confirm(&pc_ref->ref,
480 nvmet_pr_confirm_ns_pc_ref);
481 wait_for_completion(&pc_ref->confirm_done);
482 }
483 }
484 }
485
nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req * req,u64 prkey,uuid_t * send_hostid,bool abort)486 static u16 nvmet_pr_unreg_all_host_by_prkey(struct nvmet_req *req, u64 prkey,
487 uuid_t *send_hostid,
488 bool abort)
489 {
490 u16 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
491 struct nvmet_pr_registrant *reg, *tmp;
492 struct nvmet_pr *pr = &req->ns->pr;
493 uuid_t hostid;
494
495 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
496 if (reg->rkey == prkey) {
497 status = NVME_SC_SUCCESS;
498 uuid_copy(&hostid, ®->hostid);
499 if (abort)
500 nvmet_pr_set_ctrl_to_abort(req, &hostid);
501 nvmet_pr_unregister_one(pr, reg);
502 if (!uuid_equal(&hostid, send_hostid))
503 nvmet_pr_registration_preempted(pr, &hostid);
504 }
505 }
506 return status;
507 }
508
nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req * req,u64 prkey,uuid_t * send_hostid,bool abort)509 static void nvmet_pr_unreg_all_others_by_prkey(struct nvmet_req *req,
510 u64 prkey,
511 uuid_t *send_hostid,
512 bool abort)
513 {
514 struct nvmet_pr_registrant *reg, *tmp;
515 struct nvmet_pr *pr = &req->ns->pr;
516 uuid_t hostid;
517
518 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
519 if (reg->rkey == prkey &&
520 !uuid_equal(®->hostid, send_hostid)) {
521 uuid_copy(&hostid, ®->hostid);
522 if (abort)
523 nvmet_pr_set_ctrl_to_abort(req, &hostid);
524 nvmet_pr_unregister_one(pr, reg);
525 nvmet_pr_registration_preempted(pr, &hostid);
526 }
527 }
528 }
529
nvmet_pr_unreg_all_others(struct nvmet_req * req,uuid_t * send_hostid,bool abort)530 static void nvmet_pr_unreg_all_others(struct nvmet_req *req,
531 uuid_t *send_hostid,
532 bool abort)
533 {
534 struct nvmet_pr_registrant *reg, *tmp;
535 struct nvmet_pr *pr = &req->ns->pr;
536 uuid_t hostid;
537
538 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
539 if (!uuid_equal(®->hostid, send_hostid)) {
540 uuid_copy(&hostid, ®->hostid);
541 if (abort)
542 nvmet_pr_set_ctrl_to_abort(req, &hostid);
543 nvmet_pr_unregister_one(pr, reg);
544 nvmet_pr_registration_preempted(pr, &hostid);
545 }
546 }
547 }
548
nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant * reg,void * attr)549 static void nvmet_pr_update_holder_rtype(struct nvmet_pr_registrant *reg,
550 void *attr)
551 {
552 u8 new_rtype = *(u8 *)attr;
553
554 reg->rtype = new_rtype;
555 }
556
nvmet_pr_preempt(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 rtype,struct nvmet_pr_acquire_data * d,bool abort)557 static u16 nvmet_pr_preempt(struct nvmet_req *req,
558 struct nvmet_pr_registrant *reg,
559 u8 rtype,
560 struct nvmet_pr_acquire_data *d,
561 bool abort)
562 {
563 struct nvmet_ctrl *ctrl = req->sq->ctrl;
564 struct nvmet_pr *pr = &req->ns->pr;
565 struct nvmet_pr_registrant *holder;
566 enum nvme_pr_type original_rtype;
567 u64 prkey = le64_to_cpu(d->prkey);
568 u16 status;
569
570 holder = rcu_dereference_protected(pr->holder, 1);
571 if (!holder)
572 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
573 &ctrl->hostid, abort);
574
575 original_rtype = holder->rtype;
576 if (original_rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
577 original_rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
578 if (!prkey) {
579 /*
580 * To prevent possible access from other hosts, and
581 * avoid terminate the holder, set the new holder
582 * first before unregistering.
583 */
584 nvmet_pr_set_new_holder(pr, rtype, reg);
585 nvmet_pr_unreg_all_others(req, &ctrl->hostid, abort);
586 return NVME_SC_SUCCESS;
587 }
588 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
589 &ctrl->hostid, abort);
590 }
591
592 if (holder == reg) {
593 status = nvmet_pr_update_reg_attr(pr, holder,
594 nvmet_pr_update_holder_rtype, &rtype);
595 if (!status && original_rtype != rtype)
596 nvmet_pr_resv_released(pr, ®->hostid);
597 return status;
598 }
599
600 if (prkey == holder->rkey) {
601 /*
602 * Same as before, set the new holder first.
603 */
604 nvmet_pr_set_new_holder(pr, rtype, reg);
605 nvmet_pr_unreg_all_others_by_prkey(req, prkey, &ctrl->hostid,
606 abort);
607 if (original_rtype != rtype)
608 nvmet_pr_resv_released(pr, ®->hostid);
609 return NVME_SC_SUCCESS;
610 }
611
612 if (prkey)
613 return nvmet_pr_unreg_all_host_by_prkey(req, prkey,
614 &ctrl->hostid, abort);
615 return NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
616 }
617
nvmet_pr_do_abort(struct work_struct * w)618 static void nvmet_pr_do_abort(struct work_struct *w)
619 {
620 struct nvmet_req *req = container_of(w, struct nvmet_req, r.abort_work);
621 struct nvmet_pr_per_ctrl_ref *pc_ref;
622 struct nvmet_ns *ns = req->ns;
623 unsigned long idx;
624
625 /*
626 * The target does not support abort, just wait per-controller ref to 0.
627 */
628 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
629 if (percpu_ref_is_dying(&pc_ref->ref)) {
630 wait_for_completion(&pc_ref->free_done);
631 reinit_completion(&pc_ref->confirm_done);
632 reinit_completion(&pc_ref->free_done);
633 percpu_ref_resurrect(&pc_ref->ref);
634 }
635 }
636
637 up(&ns->pr.pr_sem);
638 nvmet_req_complete(req, NVME_SC_SUCCESS);
639 }
640
__nvmet_execute_pr_acquire(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 acquire_act,u8 rtype,struct nvmet_pr_acquire_data * d)641 static u16 __nvmet_execute_pr_acquire(struct nvmet_req *req,
642 struct nvmet_pr_registrant *reg,
643 u8 acquire_act,
644 u8 rtype,
645 struct nvmet_pr_acquire_data *d)
646 {
647 u16 status;
648
649 switch (acquire_act) {
650 case NVME_PR_ACQUIRE_ACT_ACQUIRE:
651 status = nvmet_pr_acquire(req, reg, rtype);
652 goto out;
653 case NVME_PR_ACQUIRE_ACT_PREEMPT:
654 status = nvmet_pr_preempt(req, reg, rtype, d, false);
655 goto inc_gen;
656 case NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT:
657 status = nvmet_pr_preempt(req, reg, rtype, d, true);
658 goto inc_gen;
659 default:
660 req->error_loc = offsetof(struct nvme_common_command, cdw10);
661 status = NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
662 goto out;
663 }
664 inc_gen:
665 if (!status)
666 atomic_inc(&req->ns->pr.generation);
667 out:
668 return status;
669 }
670
nvmet_execute_pr_acquire(struct nvmet_req * req)671 static void nvmet_execute_pr_acquire(struct nvmet_req *req)
672 {
673 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
674 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
675 /* Reservation type, bit 15:08 */
676 u8 rtype = (u8)((cdw10 >> 8) & 0xff);
677 /* Reservation acquire action, bit 02:00 */
678 u8 acquire_act = cdw10 & 0x07;
679 struct nvmet_ctrl *ctrl = req->sq->ctrl;
680 struct nvmet_pr_acquire_data *d = NULL;
681 struct nvmet_pr *pr = &req->ns->pr;
682 struct nvmet_pr_registrant *reg;
683 u16 status = NVME_SC_SUCCESS;
684
685 if (ignore_key ||
686 rtype < NVME_PR_WRITE_EXCLUSIVE ||
687 rtype > NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS) {
688 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
689 goto out;
690 }
691
692 d = kmalloc_obj(*d);
693 if (!d) {
694 status = NVME_SC_INTERNAL;
695 goto out;
696 }
697
698 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
699 if (status)
700 goto free_data;
701
702 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
703 down(&pr->pr_sem);
704 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
705 if (uuid_equal(®->hostid, &ctrl->hostid) &&
706 reg->rkey == le64_to_cpu(d->crkey)) {
707 status = __nvmet_execute_pr_acquire(req, reg,
708 acquire_act, rtype, d);
709 break;
710 }
711 }
712
713 if (!status && acquire_act == NVME_PR_ACQUIRE_ACT_PREEMPT_AND_ABORT) {
714 kfree(d);
715 INIT_WORK(&req->r.abort_work, nvmet_pr_do_abort);
716 queue_work(nvmet_wq, &req->r.abort_work);
717 return;
718 }
719
720 up(&pr->pr_sem);
721
722 free_data:
723 kfree(d);
724 out:
725 nvmet_req_complete(req, status);
726 }
727
nvmet_pr_release(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 rtype)728 static u16 nvmet_pr_release(struct nvmet_req *req,
729 struct nvmet_pr_registrant *reg,
730 u8 rtype)
731 {
732 struct nvmet_pr *pr = &req->ns->pr;
733 struct nvmet_pr_registrant *holder;
734 u8 original_rtype;
735
736 holder = rcu_dereference_protected(pr->holder, 1);
737 if (!holder || reg != holder)
738 return NVME_SC_SUCCESS;
739
740 original_rtype = holder->rtype;
741 if (original_rtype != rtype)
742 return NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
743
744 rcu_assign_pointer(pr->holder, NULL);
745
746 if (original_rtype != NVME_PR_WRITE_EXCLUSIVE &&
747 original_rtype != NVME_PR_EXCLUSIVE_ACCESS)
748 nvmet_pr_resv_released(pr, ®->hostid);
749
750 return NVME_SC_SUCCESS;
751 }
752
nvmet_pr_clear(struct nvmet_req * req)753 static void nvmet_pr_clear(struct nvmet_req *req)
754 {
755 struct nvmet_pr_registrant *reg, *tmp;
756 struct nvmet_pr *pr = &req->ns->pr;
757
758 rcu_assign_pointer(pr->holder, NULL);
759
760 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
761 list_del_rcu(®->entry);
762 if (!uuid_equal(&req->sq->ctrl->hostid, ®->hostid))
763 nvmet_pr_resv_preempted(pr, ®->hostid);
764 kfree_rcu(reg, rcu);
765 }
766
767 atomic_inc(&pr->generation);
768 }
769
__nvmet_execute_pr_release(struct nvmet_req * req,struct nvmet_pr_registrant * reg,u8 release_act,u8 rtype)770 static u16 __nvmet_execute_pr_release(struct nvmet_req *req,
771 struct nvmet_pr_registrant *reg,
772 u8 release_act, u8 rtype)
773 {
774 switch (release_act) {
775 case NVME_PR_RELEASE_ACT_RELEASE:
776 return nvmet_pr_release(req, reg, rtype);
777 case NVME_PR_RELEASE_ACT_CLEAR:
778 nvmet_pr_clear(req);
779 return NVME_SC_SUCCESS;
780 default:
781 req->error_loc = offsetof(struct nvme_common_command, cdw10);
782 return NVME_SC_INVALID_OPCODE | NVME_STATUS_DNR;
783 }
784 }
785
nvmet_execute_pr_release(struct nvmet_req * req)786 static void nvmet_execute_pr_release(struct nvmet_req *req)
787 {
788 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
789 bool ignore_key = nvmet_pr_parse_ignore_key(cdw10);
790 u8 rtype = (u8)((cdw10 >> 8) & 0xff); /* Reservation type, bit 15:08 */
791 u8 release_act = cdw10 & 0x07; /* Reservation release action, bit 02:00 */
792 struct nvmet_ctrl *ctrl = req->sq->ctrl;
793 struct nvmet_pr *pr = &req->ns->pr;
794 struct nvmet_pr_release_data *d;
795 struct nvmet_pr_registrant *reg;
796 u16 status;
797
798 if (ignore_key) {
799 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
800 goto out;
801 }
802
803 d = kmalloc_obj(*d);
804 if (!d) {
805 status = NVME_SC_INTERNAL;
806 goto out;
807 }
808
809 status = nvmet_copy_from_sgl(req, 0, d, sizeof(*d));
810 if (status)
811 goto free_data;
812
813 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
814 down(&pr->pr_sem);
815 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
816 if (uuid_equal(®->hostid, &ctrl->hostid) &&
817 reg->rkey == le64_to_cpu(d->crkey)) {
818 status = __nvmet_execute_pr_release(req, reg,
819 release_act, rtype);
820 break;
821 }
822 }
823 up(&pr->pr_sem);
824 free_data:
825 kfree(d);
826 out:
827 nvmet_req_complete(req, status);
828 }
829
nvmet_execute_pr_report(struct nvmet_req * req)830 static void nvmet_execute_pr_report(struct nvmet_req *req)
831 {
832 u32 cdw11 = le32_to_cpu(req->cmd->common.cdw11);
833 u32 cdw10 = le32_to_cpu(req->cmd->common.cdw10);
834 u32 num_bytes = 4 * (cdw10 + 1); /* cdw10 is number of dwords */
835 u8 eds = cdw11 & 1; /* Extended data structure, bit 00 */
836 struct nvme_registered_ctrl_ext *ctrl_eds;
837 struct nvme_reservation_status_ext *data;
838 struct nvmet_pr *pr = &req->ns->pr;
839 struct nvmet_pr_registrant *holder;
840 struct nvmet_pr_registrant *reg;
841 u16 num_ctrls = 0;
842 u16 status;
843 u8 rtype;
844
845 /* nvmet hostid(uuid_t) is 128 bit. */
846 if (!eds) {
847 req->error_loc = offsetof(struct nvme_common_command, cdw11);
848 status = NVME_SC_HOST_ID_INCONSIST | NVME_STATUS_DNR;
849 goto out;
850 }
851
852 if (num_bytes < sizeof(struct nvme_reservation_status_ext)) {
853 req->error_loc = offsetof(struct nvme_common_command, cdw10);
854 status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
855 goto out;
856 }
857
858 data = kzalloc(num_bytes, GFP_KERNEL);
859 if (!data) {
860 status = NVME_SC_INTERNAL;
861 goto out;
862 }
863 data->gen = cpu_to_le32(atomic_read(&pr->generation));
864 data->ptpls = 0;
865 ctrl_eds = data->regctl_eds;
866
867 rcu_read_lock();
868 holder = rcu_dereference(pr->holder);
869 rtype = holder ? holder->rtype : 0;
870 data->rtype = rtype;
871
872 list_for_each_entry_rcu(reg, &pr->registrant_list, entry) {
873 num_ctrls++;
874 /*
875 * continue to get the number of all registrans.
876 */
877 if (((void *)ctrl_eds + sizeof(*ctrl_eds)) >
878 ((void *)data + num_bytes))
879 continue;
880 /*
881 * Dynamic controller, set cntlid to 0xffff.
882 */
883 ctrl_eds->cntlid = cpu_to_le16(NVME_CNTLID_DYNAMIC);
884 if (rtype == NVME_PR_WRITE_EXCLUSIVE_ALL_REGS ||
885 rtype == NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS)
886 ctrl_eds->rcsts = 1;
887 if (reg == holder)
888 ctrl_eds->rcsts = 1;
889 uuid_copy((uuid_t *)&ctrl_eds->hostid, ®->hostid);
890 ctrl_eds->rkey = cpu_to_le64(reg->rkey);
891 ctrl_eds++;
892 }
893 rcu_read_unlock();
894
895 put_unaligned_le16(num_ctrls, data->regctl);
896 status = nvmet_copy_to_sgl(req, 0, data, num_bytes);
897 kfree(data);
898 out:
899 nvmet_req_complete(req, status);
900 }
901
nvmet_parse_pr_cmd(struct nvmet_req * req)902 u16 nvmet_parse_pr_cmd(struct nvmet_req *req)
903 {
904 struct nvme_command *cmd = req->cmd;
905
906 switch (cmd->common.opcode) {
907 case nvme_cmd_resv_register:
908 req->execute = nvmet_execute_pr_register;
909 break;
910 case nvme_cmd_resv_acquire:
911 req->execute = nvmet_execute_pr_acquire;
912 break;
913 case nvme_cmd_resv_release:
914 req->execute = nvmet_execute_pr_release;
915 break;
916 case nvme_cmd_resv_report:
917 req->execute = nvmet_execute_pr_report;
918 break;
919 default:
920 return 1;
921 }
922 return NVME_SC_SUCCESS;
923 }
924
nvmet_is_req_write_cmd_group(struct nvmet_req * req)925 static bool nvmet_is_req_write_cmd_group(struct nvmet_req *req)
926 {
927 u8 opcode = req->cmd->common.opcode;
928
929 if (req->sq->qid) {
930 switch (opcode) {
931 case nvme_cmd_flush:
932 case nvme_cmd_write:
933 case nvme_cmd_write_zeroes:
934 case nvme_cmd_dsm:
935 case nvme_cmd_zone_append:
936 case nvme_cmd_zone_mgmt_send:
937 return true;
938 default:
939 return false;
940 }
941 }
942 return false;
943 }
944
nvmet_is_req_read_cmd_group(struct nvmet_req * req)945 static bool nvmet_is_req_read_cmd_group(struct nvmet_req *req)
946 {
947 u8 opcode = req->cmd->common.opcode;
948
949 if (req->sq->qid) {
950 switch (opcode) {
951 case nvme_cmd_read:
952 case nvme_cmd_zone_mgmt_recv:
953 return true;
954 default:
955 return false;
956 }
957 }
958 return false;
959 }
960
nvmet_pr_check_cmd_access(struct nvmet_req * req)961 u16 nvmet_pr_check_cmd_access(struct nvmet_req *req)
962 {
963 struct nvmet_ctrl *ctrl = req->sq->ctrl;
964 struct nvmet_pr_registrant *holder;
965 struct nvmet_ns *ns = req->ns;
966 struct nvmet_pr *pr = &ns->pr;
967 u16 status = NVME_SC_SUCCESS;
968
969 rcu_read_lock();
970 holder = rcu_dereference(pr->holder);
971 if (!holder)
972 goto unlock;
973 if (uuid_equal(&ctrl->hostid, &holder->hostid))
974 goto unlock;
975
976 /*
977 * The Reservation command group is checked in executing,
978 * allow it here.
979 */
980 switch (holder->rtype) {
981 case NVME_PR_WRITE_EXCLUSIVE:
982 if (nvmet_is_req_write_cmd_group(req))
983 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
984 break;
985 case NVME_PR_EXCLUSIVE_ACCESS:
986 if (nvmet_is_req_read_cmd_group(req) ||
987 nvmet_is_req_write_cmd_group(req))
988 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
989 break;
990 case NVME_PR_WRITE_EXCLUSIVE_REG_ONLY:
991 case NVME_PR_WRITE_EXCLUSIVE_ALL_REGS:
992 if ((nvmet_is_req_write_cmd_group(req)) &&
993 !nvmet_pr_find_registrant(pr, &ctrl->hostid))
994 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
995 break;
996 case NVME_PR_EXCLUSIVE_ACCESS_REG_ONLY:
997 case NVME_PR_EXCLUSIVE_ACCESS_ALL_REGS:
998 if ((nvmet_is_req_read_cmd_group(req) ||
999 nvmet_is_req_write_cmd_group(req)) &&
1000 !nvmet_pr_find_registrant(pr, &ctrl->hostid))
1001 status = NVME_SC_RESERVATION_CONFLICT | NVME_STATUS_DNR;
1002 break;
1003 default:
1004 pr_warn("the reservation type is set wrong, type:%d\n",
1005 holder->rtype);
1006 break;
1007 }
1008
1009 unlock:
1010 rcu_read_unlock();
1011 if (status)
1012 req->error_loc = offsetof(struct nvme_common_command, opcode);
1013 return status;
1014 }
1015
nvmet_pr_get_ns_pc_ref(struct nvmet_req * req)1016 u16 nvmet_pr_get_ns_pc_ref(struct nvmet_req *req)
1017 {
1018 struct nvmet_pr_per_ctrl_ref *pc_ref;
1019
1020 pc_ref = xa_load(&req->ns->pr_per_ctrl_refs,
1021 req->sq->ctrl->cntlid);
1022 if (unlikely(!percpu_ref_tryget_live(&pc_ref->ref)))
1023 return NVME_SC_INTERNAL;
1024 req->pc_ref = pc_ref;
1025 return NVME_SC_SUCCESS;
1026 }
1027
nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref * ref)1028 static void nvmet_pr_ctrl_ns_all_cmds_done(struct percpu_ref *ref)
1029 {
1030 struct nvmet_pr_per_ctrl_ref *pc_ref =
1031 container_of(ref, struct nvmet_pr_per_ctrl_ref, ref);
1032
1033 complete(&pc_ref->free_done);
1034 }
1035
nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns * ns,unsigned long idx,uuid_t * hostid)1036 static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns,
1037 unsigned long idx,
1038 uuid_t *hostid)
1039 {
1040 struct nvmet_pr_per_ctrl_ref *pc_ref;
1041 int ret;
1042
1043 pc_ref = kmalloc_obj(*pc_ref, GFP_ATOMIC);
1044 if (!pc_ref)
1045 return -ENOMEM;
1046
1047 ret = percpu_ref_init(&pc_ref->ref, nvmet_pr_ctrl_ns_all_cmds_done,
1048 PERCPU_REF_ALLOW_REINIT, GFP_KERNEL);
1049 if (ret)
1050 goto free;
1051
1052 init_completion(&pc_ref->free_done);
1053 init_completion(&pc_ref->confirm_done);
1054 uuid_copy(&pc_ref->hostid, hostid);
1055
1056 ret = xa_insert(&ns->pr_per_ctrl_refs, idx, pc_ref, GFP_KERNEL);
1057 if (ret)
1058 goto exit;
1059 return ret;
1060 exit:
1061 percpu_ref_exit(&pc_ref->ref);
1062 free:
1063 kfree(pc_ref);
1064 return ret;
1065 }
1066
nvmet_ctrl_init_pr(struct nvmet_ctrl * ctrl)1067 int nvmet_ctrl_init_pr(struct nvmet_ctrl *ctrl)
1068 {
1069 struct nvmet_subsys *subsys = ctrl->subsys;
1070 struct nvmet_pr_per_ctrl_ref *pc_ref;
1071 struct nvmet_ns *ns = NULL;
1072 unsigned long idx;
1073 int ret;
1074
1075 ctrl->pr_log_mgr.counter = 0;
1076 ctrl->pr_log_mgr.lost_count = 0;
1077 mutex_init(&ctrl->pr_log_mgr.lock);
1078 INIT_KFIFO(ctrl->pr_log_mgr.log_queue);
1079
1080 /*
1081 * Here we are under subsys lock, if an ns not in subsys->namespaces,
1082 * we can make sure that ns is not enabled, and not call
1083 * nvmet_pr_init_ns(), see more details in nvmet_ns_enable().
1084 * So just check ns->pr.enable.
1085 */
1086 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
1087 if (ns->pr.enable) {
1088 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
1089 &ctrl->hostid);
1090 if (ret)
1091 goto free_per_ctrl_refs;
1092 }
1093 }
1094 return 0;
1095
1096 free_per_ctrl_refs:
1097 nvmet_for_each_enabled_ns(&subsys->namespaces, idx, ns) {
1098 if (ns->pr.enable) {
1099 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
1100 if (pc_ref)
1101 percpu_ref_exit(&pc_ref->ref);
1102 kfree(pc_ref);
1103 }
1104 }
1105 return ret;
1106 }
1107
nvmet_ctrl_destroy_pr(struct nvmet_ctrl * ctrl)1108 void nvmet_ctrl_destroy_pr(struct nvmet_ctrl *ctrl)
1109 {
1110 struct nvmet_pr_per_ctrl_ref *pc_ref;
1111 struct nvmet_ns *ns;
1112 unsigned long idx;
1113
1114 kfifo_free(&ctrl->pr_log_mgr.log_queue);
1115 mutex_destroy(&ctrl->pr_log_mgr.lock);
1116
1117 nvmet_for_each_enabled_ns(&ctrl->subsys->namespaces, idx, ns) {
1118 if (ns->pr.enable) {
1119 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, ctrl->cntlid);
1120 if (pc_ref)
1121 percpu_ref_exit(&pc_ref->ref);
1122 kfree(pc_ref);
1123 }
1124 }
1125 }
1126
nvmet_pr_init_ns(struct nvmet_ns * ns)1127 int nvmet_pr_init_ns(struct nvmet_ns *ns)
1128 {
1129 struct nvmet_subsys *subsys = ns->subsys;
1130 struct nvmet_pr_per_ctrl_ref *pc_ref;
1131 struct nvmet_ctrl *ctrl = NULL;
1132 unsigned long idx;
1133 int ret;
1134
1135 ns->pr.holder = NULL;
1136 atomic_set(&ns->pr.generation, 0);
1137 sema_init(&ns->pr.pr_sem, 1);
1138 INIT_LIST_HEAD(&ns->pr.registrant_list);
1139 ns->pr.notify_mask = 0;
1140
1141 xa_init(&ns->pr_per_ctrl_refs);
1142
1143 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
1144 ret = nvmet_pr_alloc_and_insert_pc_ref(ns, ctrl->cntlid,
1145 &ctrl->hostid);
1146 if (ret)
1147 goto free_per_ctrl_refs;
1148 }
1149 return 0;
1150
1151 free_per_ctrl_refs:
1152 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
1153 xa_erase(&ns->pr_per_ctrl_refs, idx);
1154 percpu_ref_exit(&pc_ref->ref);
1155 kfree(pc_ref);
1156 }
1157 return ret;
1158 }
1159
nvmet_pr_exit_ns(struct nvmet_ns * ns)1160 void nvmet_pr_exit_ns(struct nvmet_ns *ns)
1161 {
1162 struct nvmet_pr_registrant *reg, *tmp;
1163 struct nvmet_pr_per_ctrl_ref *pc_ref;
1164 struct nvmet_pr *pr = &ns->pr;
1165 unsigned long idx;
1166
1167 list_for_each_entry_safe(reg, tmp, &pr->registrant_list, entry) {
1168 list_del(®->entry);
1169 kfree(reg);
1170 }
1171
1172 xa_for_each(&ns->pr_per_ctrl_refs, idx, pc_ref) {
1173 /*
1174 * No command on ns here, we can safely free pc_ref.
1175 */
1176 pc_ref = xa_erase(&ns->pr_per_ctrl_refs, idx);
1177 percpu_ref_exit(&pc_ref->ref);
1178 kfree(pc_ref);
1179 }
1180
1181 xa_destroy(&ns->pr_per_ctrl_refs);
1182 }
1183