1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * Copyright (c) 2024 Oracle. All rights reserved.
4 */
5
6 /* #include <linux/module.h>
7 #include <linux/slab.h> */
8 #include <linux/xarray.h>
9 #include <linux/types.h>
10 #include <linux/kref.h>
11 #include <linux/completion.h>
12
13 #include <linux/sunrpc/svc_rdma.h>
14 #include <linux/sunrpc/rdma_rn.h>
15
16 #include "xprt_rdma.h"
17 #include <trace/events/rpcrdma.h>
18
19 /* Per-ib_device private data for rpcrdma */
20 struct rpcrdma_device {
21 struct kref rd_kref;
22 unsigned long rd_flags;
23 struct ib_device *rd_device;
24 struct xarray rd_xa;
25 struct completion rd_done;
26 };
27
28 #define RPCRDMA_RD_F_REMOVING (0)
29
30 static struct ib_client rpcrdma_ib_client;
31
32 /*
33 * Listeners have no associated device, so we never register them.
34 * Note that ib_get_client_data() does not check if @device is
35 * NULL for us.
36 */
rpcrdma_get_client_data(struct ib_device * device)37 static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device)
38 {
39 if (!device)
40 return NULL;
41 return ib_get_client_data(device, &rpcrdma_ib_client);
42 }
43
44 /**
45 * rpcrdma_rn_register - register to get device removal notifications
46 * @device: device to monitor
47 * @rn: notification object that wishes to be notified
48 * @done: callback to notify caller of device removal
49 *
50 * Returns zero on success. The callback in rn_done is guaranteed
51 * to be invoked when the device is removed, unless this notification
52 * is unregistered first.
53 *
54 * On failure, a negative errno is returned. rn->rn_done is left
55 * NULL on every failure path (it is armed before xa_alloc but
56 * cleared again if xa_alloc fails), so the @rn may safely be
57 * passed to rpcrdma_rn_unregister() without a separate
58 * registered/unregistered flag in the caller.
59 */
rpcrdma_rn_register(struct ib_device * device,struct rpcrdma_notification * rn,void (* done)(struct rpcrdma_notification * rn))60 int rpcrdma_rn_register(struct ib_device *device,
61 struct rpcrdma_notification *rn,
62 void (*done)(struct rpcrdma_notification *rn))
63 {
64 struct rpcrdma_device *rd = rpcrdma_get_client_data(device);
65
66 if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags))
67 return -ENETUNREACH;
68
69 /*
70 * Arm rn_done before xa_alloc() publishes @rn: once @rn is
71 * visible in rd_xa, a concurrent rpcrdma_remove_one() can
72 * call rn->rn_done(), so the pointer must already be set.
73 *
74 * Restore NULL if xa_alloc() fails. rn_done doubles as the
75 * registration sentinel for rpcrdma_rn_unregister(); a stale
76 * value would unregister an @rn that was never inserted.
77 */
78 rn->rn_done = done;
79 if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) {
80 rn->rn_done = NULL;
81 return -ENOMEM;
82 }
83 kref_get(&rd->rd_kref);
84 trace_rpcrdma_client_register(device, rn);
85 return 0;
86 }
87
rpcrdma_rn_release(struct kref * kref)88 static void rpcrdma_rn_release(struct kref *kref)
89 {
90 struct rpcrdma_device *rd = container_of(kref, struct rpcrdma_device,
91 rd_kref);
92
93 trace_rpcrdma_client_completion(rd->rd_device);
94 complete(&rd->rd_done);
95 }
96
97 /**
98 * rpcrdma_rn_unregister - stop device removal notifications
99 * @device: monitored device
100 * @rn: notification object that no longer wishes to be notified
101 *
102 * It is safe to call this on an @rn whose registration never
103 * completed or failed; rn_done == NULL is treated as
104 * never-registered and the call is a no-op.
105 */
rpcrdma_rn_unregister(struct ib_device * device,struct rpcrdma_notification * rn)106 void rpcrdma_rn_unregister(struct ib_device *device,
107 struct rpcrdma_notification *rn)
108 {
109 struct rpcrdma_device *rd = rpcrdma_get_client_data(device);
110
111 if (!rd)
112 return;
113
114 /*
115 * rn_done is the registration sentinel: rpcrdma_rn_register
116 * leaves it NULL on every failure path, clearing it again if
117 * xa_alloc fails, so a non-NULL rn_done marks a completed
118 * registration. A NULL rn_done means this notification was
119 * never registered (or its registration failed) or has
120 * already been unregistered, and the call is a no-op.
121 * Without this guard, rn_index == 0 from a kzalloc'd
122 * parent would erase another caller's slot 0 and underflow
123 * rd_kref.
124 */
125 if (!rn->rn_done)
126 return;
127 rn->rn_done = NULL;
128
129 trace_rpcrdma_client_unregister(device, rn);
130 xa_erase(&rd->rd_xa, rn->rn_index);
131 kref_put(&rd->rd_kref, rpcrdma_rn_release);
132 }
133
134 /**
135 * rpcrdma_add_one - ib_client device insertion callback
136 * @device: device about to be inserted
137 *
138 * Returns zero on success. xprtrdma private data has been allocated
139 * for this device. On failure, a negative errno is returned.
140 */
rpcrdma_add_one(struct ib_device * device)141 static int rpcrdma_add_one(struct ib_device *device)
142 {
143 struct rpcrdma_device *rd;
144
145 rd = kzalloc_obj(*rd);
146 if (!rd)
147 return -ENOMEM;
148
149 kref_init(&rd->rd_kref);
150 xa_init_flags(&rd->rd_xa, XA_FLAGS_ALLOC);
151 rd->rd_device = device;
152 init_completion(&rd->rd_done);
153 ib_set_client_data(device, &rpcrdma_ib_client, rd);
154
155 trace_rpcrdma_client_add_one(device);
156 return 0;
157 }
158
159 /**
160 * rpcrdma_remove_one - ib_client device removal callback
161 * @device: device about to be removed
162 * @client_data: this module's private per-device data
163 *
164 * Upon return, all transports associated with @device have divested
165 * themselves from IB hardware resources.
166 */
rpcrdma_remove_one(struct ib_device * device,void * client_data)167 static void rpcrdma_remove_one(struct ib_device *device,
168 void *client_data)
169 {
170 struct rpcrdma_device *rd = client_data;
171 struct rpcrdma_notification *rn;
172 unsigned long index;
173
174 trace_rpcrdma_client_remove_one(device);
175
176 set_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags);
177 xa_for_each(&rd->rd_xa, index, rn)
178 rn->rn_done(rn);
179
180 /*
181 * Wait only if there are still outstanding notification
182 * registrants for this device.
183 */
184 if (!refcount_dec_and_test(&rd->rd_kref.refcount)) {
185 trace_rpcrdma_client_wait_on(device);
186 wait_for_completion(&rd->rd_done);
187 }
188
189 trace_rpcrdma_client_remove_one_done(device);
190 xa_destroy(&rd->rd_xa);
191 kfree(rd);
192 }
193
194 static struct ib_client rpcrdma_ib_client = {
195 .name = "rpcrdma",
196 .add = rpcrdma_add_one,
197 .remove = rpcrdma_remove_one,
198 };
199
200 /**
201 * rpcrdma_ib_client_unregister - unregister ib_client for xprtrdma
202 *
203 * cel: watch for orphaned rpcrdma_device objects on module unload
204 */
rpcrdma_ib_client_unregister(void)205 void rpcrdma_ib_client_unregister(void)
206 {
207 ib_unregister_client(&rpcrdma_ib_client);
208 }
209
210 /**
211 * rpcrdma_ib_client_register - register ib_client for rpcrdma
212 *
213 * Returns zero on success, or a negative errno.
214 */
rpcrdma_ib_client_register(void)215 int rpcrdma_ib_client_register(void)
216 {
217 return ib_register_client(&rpcrdma_ib_client);
218 }
219