1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* 3 * Copyright (c) 2024 Oracle. All rights reserved. 4 */ 5 6 /* #include <linux/module.h> 7 #include <linux/slab.h> */ 8 #include <linux/xarray.h> 9 #include <linux/types.h> 10 #include <linux/kref.h> 11 #include <linux/completion.h> 12 13 #include <linux/sunrpc/svc_rdma.h> 14 #include <linux/sunrpc/rdma_rn.h> 15 16 #include "xprt_rdma.h" 17 #include <trace/events/rpcrdma.h> 18 19 /* Per-ib_device private data for rpcrdma */ 20 struct rpcrdma_device { 21 struct kref rd_kref; 22 unsigned long rd_flags; 23 struct ib_device *rd_device; 24 struct xarray rd_xa; 25 struct completion rd_done; 26 }; 27 28 #define RPCRDMA_RD_F_REMOVING (0) 29 30 static struct ib_client rpcrdma_ib_client; 31 32 /* 33 * Listeners have no associated device, so we never register them. 34 * Note that ib_get_client_data() does not check if @device is 35 * NULL for us. 36 */ 37 static struct rpcrdma_device *rpcrdma_get_client_data(struct ib_device *device) 38 { 39 if (!device) 40 return NULL; 41 return ib_get_client_data(device, &rpcrdma_ib_client); 42 } 43 44 /** 45 * rpcrdma_rn_register - register to get device removal notifications 46 * @device: device to monitor 47 * @rn: notification object that wishes to be notified 48 * @done: callback to notify caller of device removal 49 * 50 * Returns zero on success. The callback in rn_done is guaranteed 51 * to be invoked when the device is removed, unless this notification 52 * is unregistered first. 53 * 54 * On failure, a negative errno is returned. 55 */ 56 int rpcrdma_rn_register(struct ib_device *device, 57 struct rpcrdma_notification *rn, 58 void (*done)(struct rpcrdma_notification *rn)) 59 { 60 struct rpcrdma_device *rd = rpcrdma_get_client_data(device); 61 62 if (!rd || test_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags)) 63 return -ENETUNREACH; 64 65 if (xa_alloc(&rd->rd_xa, &rn->rn_index, rn, xa_limit_32b, GFP_KERNEL) < 0) 66 return -ENOMEM; 67 kref_get(&rd->rd_kref); 68 rn->rn_done = done; 69 trace_rpcrdma_client_register(device, rn); 70 return 0; 71 } 72 73 static void rpcrdma_rn_release(struct kref *kref) 74 { 75 struct rpcrdma_device *rd = container_of(kref, struct rpcrdma_device, 76 rd_kref); 77 78 trace_rpcrdma_client_completion(rd->rd_device); 79 complete(&rd->rd_done); 80 } 81 82 /** 83 * rpcrdma_rn_unregister - stop device removal notifications 84 * @device: monitored device 85 * @rn: notification object that no longer wishes to be notified 86 */ 87 void rpcrdma_rn_unregister(struct ib_device *device, 88 struct rpcrdma_notification *rn) 89 { 90 struct rpcrdma_device *rd = rpcrdma_get_client_data(device); 91 92 if (!rd) 93 return; 94 95 trace_rpcrdma_client_unregister(device, rn); 96 xa_erase(&rd->rd_xa, rn->rn_index); 97 kref_put(&rd->rd_kref, rpcrdma_rn_release); 98 } 99 100 /** 101 * rpcrdma_add_one - ib_client device insertion callback 102 * @device: device about to be inserted 103 * 104 * Returns zero on success. xprtrdma private data has been allocated 105 * for this device. On failure, a negative errno is returned. 106 */ 107 static int rpcrdma_add_one(struct ib_device *device) 108 { 109 struct rpcrdma_device *rd; 110 111 rd = kzalloc(sizeof(*rd), GFP_KERNEL); 112 if (!rd) 113 return -ENOMEM; 114 115 kref_init(&rd->rd_kref); 116 xa_init_flags(&rd->rd_xa, XA_FLAGS_ALLOC); 117 rd->rd_device = device; 118 init_completion(&rd->rd_done); 119 ib_set_client_data(device, &rpcrdma_ib_client, rd); 120 121 trace_rpcrdma_client_add_one(device); 122 return 0; 123 } 124 125 /** 126 * rpcrdma_remove_one - ib_client device removal callback 127 * @device: device about to be removed 128 * @client_data: this module's private per-device data 129 * 130 * Upon return, all transports associated with @device have divested 131 * themselves from IB hardware resources. 132 */ 133 static void rpcrdma_remove_one(struct ib_device *device, 134 void *client_data) 135 { 136 struct rpcrdma_device *rd = client_data; 137 struct rpcrdma_notification *rn; 138 unsigned long index; 139 140 trace_rpcrdma_client_remove_one(device); 141 142 set_bit(RPCRDMA_RD_F_REMOVING, &rd->rd_flags); 143 xa_for_each(&rd->rd_xa, index, rn) 144 rn->rn_done(rn); 145 146 /* 147 * Wait only if there are still outstanding notification 148 * registrants for this device. 149 */ 150 if (!refcount_dec_and_test(&rd->rd_kref.refcount)) { 151 trace_rpcrdma_client_wait_on(device); 152 wait_for_completion(&rd->rd_done); 153 } 154 155 trace_rpcrdma_client_remove_one_done(device); 156 kfree(rd); 157 } 158 159 static struct ib_client rpcrdma_ib_client = { 160 .name = "rpcrdma", 161 .add = rpcrdma_add_one, 162 .remove = rpcrdma_remove_one, 163 }; 164 165 /** 166 * rpcrdma_ib_client_unregister - unregister ib_client for xprtrdma 167 * 168 * cel: watch for orphaned rpcrdma_device objects on module unload 169 */ 170 void rpcrdma_ib_client_unregister(void) 171 { 172 ib_unregister_client(&rpcrdma_ib_client); 173 } 174 175 /** 176 * rpcrdma_ib_client_register - register ib_client for rpcrdma 177 * 178 * Returns zero on success, or a negative errno. 179 */ 180 int rpcrdma_ib_client_register(void) 181 { 182 return ib_register_client(&rpcrdma_ib_client); 183 } 184