1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /*
3 * Copyright (c) 2024 Oracle. All rights reserved.
4 */
5
6 /* #include <linux/module.h>
7 #include <linux/slab.h> */
8 #include <linux/xarray.h>
9 #include <linux/types.h>
10 #include <linux/kref.h>
11 #include <linux/completion.h>
12
13 #include <linux/sunrpc/svc_rdma.h>
14 #include <linux/sunrpc/rdma_rn.h>
15
16 #include "xprt_rdma.h"
17 #include <trace/events/rpcrdma.h>
18
19 /* Per-ib_device private data for rpcrdma */
20 struct rpcrdma_device {
21 struct kref rd_kref;
22 unsigned long rd_flags;
23 struct ib_device *rd_device;
24 struct xarray rd_xa;
25 struct completion rd_done;
26 };
27
28 #define RPCRDMA_RD_F_REMOVING (0)
29
30 static struct ib_client rpcrdma_ib_client;
31
32 /*
33 * Listeners have no associated device, so we never register them.
34 * Note that ib_get_client_data() does not check if @device is
35 * NULL for us.
36 */
rpcrdma_get_client_data(struct ib_device * device)37 static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device)
38 {
39 if (!device)
40 return NULL;
41 return ib_get_client_data(device, &rpcrdma_ib_client);
42 }
43
44 /**
45 * rpcrdma_rn_register - register to get device removal notifications
46 * @device: device to monitor
47 * @rn: notification object that wishes to be notified
48 * @done: callback to notify caller of device removal
49 *
50 * Returns zero on success. The callback in rn_done is guaranteed
51 * to be invoked when the device is removed, unless this notification
52 * is unregistered first.
53 *
54 * On failure, a negative errno is returned.
55 */
rpcrdma_rn_register(struct ib_device * device,struct rpcrdma_notification * rn,void (* done)(struct rpcrdma_notification * rn))56 int rpcrdma_rn_register(struct ib_device *device,
57 struct rpcrdma_notification *rn,
58 void (*done)(struct rpcrdma_notification *rn))
59 {
60 struct rpcrdma_device *rd = rpcrdma_get_client_data(device);
61
62 if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags))
63 return -ENETUNREACH;
64
65 if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0)
66 return -ENOMEM;
67 kref_get(&rd->rd_kref);
68 rn->rn_done = done;
69 trace_rpcrdma_client_register(device, rn);
70 return 0;
71 }
72
rpcrdma_rn_release(struct kref * kref)73 static void rpcrdma_rn_release(struct kref *kref)
74 {
75 struct rpcrdma_device *rd = container_of(kref, struct rpcrdma_device,
76 rd_kref);
77
78 trace_rpcrdma_client_completion(rd->rd_device);
79 complete(&rd->rd_done);
80 }
81
82 /**
83 * rpcrdma_rn_unregister - stop device removal notifications
84 * @device: monitored device
85 * @rn: notification object that no longer wishes to be notified
86 */
rpcrdma_rn_unregister(struct ib_device * device,struct rpcrdma_notification * rn)87 void rpcrdma_rn_unregister(struct ib_device *device,
88 struct rpcrdma_notification *rn)
89 {
90 struct rpcrdma_device *rd = rpcrdma_get_client_data(device);
91
92 if (!rd)
93 return;
94
95 trace_rpcrdma_client_unregister(device, rn);
96 xa_erase(&rd->rd_xa, rn->rn_index);
97 kref_put(&rd->rd_kref, rpcrdma_rn_release);
98 }
99
100 /**
101 * rpcrdma_add_one - ib_client device insertion callback
102 * @device: device about to be inserted
103 *
104 * Returns zero on success. xprtrdma private data has been allocated
105 * for this device. On failure, a negative errno is returned.
106 */
rpcrdma_add_one(struct ib_device * device)107 static int rpcrdma_add_one(struct ib_device *device)
108 {
109 struct rpcrdma_device *rd;
110
111 rd = kzalloc(sizeof(*rd), GFP_KERNEL);
112 if (!rd)
113 return -ENOMEM;
114
115 kref_init(&rd->rd_kref);
116 xa_init_flags(&rd->rd_xa, XA_FLAGS_ALLOC);
117 rd->rd_device = device;
118 init_completion(&rd->rd_done);
119 ib_set_client_data(device, &rpcrdma_ib_client, rd);
120
121 trace_rpcrdma_client_add_one(device);
122 return 0;
123 }
124
125 /**
126 * rpcrdma_remove_one - ib_client device removal callback
127 * @device: device about to be removed
128 * @client_data: this module's private per-device data
129 *
130 * Upon return, all transports associated with @device have divested
131 * themselves from IB hardware resources.
132 */
rpcrdma_remove_one(struct ib_device * device,void * client_data)133 static void rpcrdma_remove_one(struct ib_device *device,
134 void *client_data)
135 {
136 struct rpcrdma_device *rd = client_data;
137 struct rpcrdma_notification *rn;
138 unsigned long index;
139
140 trace_rpcrdma_client_remove_one(device);
141
142 set_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags);
143 xa_for_each(&rd->rd_xa, index, rn)
144 rn->rn_done(rn);
145
146 /*
147 * Wait only if there are still outstanding notification
148 * registrants for this device.
149 */
150 if (!refcount_dec_and_test(&rd->rd_kref.refcount)) {
151 trace_rpcrdma_client_wait_on(device);
152 wait_for_completion(&rd->rd_done);
153 }
154
155 trace_rpcrdma_client_remove_one_done(device);
156 xa_destroy(&rd->rd_xa);
157 kfree(rd);
158 }
159
160 static struct ib_client rpcrdma_ib_client = {
161 .name = "rpcrdma",
162 .add = rpcrdma_add_one,
163 .remove = rpcrdma_remove_one,
164 };
165
166 /**
167 * rpcrdma_ib_client_unregister - unregister ib_client for xprtrdma
168 *
169 * cel: watch for orphaned rpcrdma_device objects on module unload
170 */
rpcrdma_ib_client_unregister(void)171 void rpcrdma_ib_client_unregister(void)
172 {
173 ib_unregister_client(&rpcrdma_ib_client);
174 }
175
176 /**
177 * rpcrdma_ib_client_register - register ib_client for rpcrdma
178 *
179 * Returns zero on success, or a negative errno.
180 */
rpcrdma_ib_client_register(void)181 int rpcrdma_ib_client_register(void)
182 {
183 return ib_register_client(&rpcrdma_ib_client);
184 }
185