1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2024 Mike Snitzer <snitzer@hammerspace.com> 4 * Copyright (C) 2024 NeilBrown <neilb@suse.de> 5 */ 6 7 #include <linux/module.h> 8 #include <linux/list.h> 9 #include <linux/nfslocalio.h> 10 #include <linux/nfs3.h> 11 #include <linux/nfs4.h> 12 #include <linux/nfs_fs.h> 13 #include <net/netns/generic.h> 14 15 #include "localio_trace.h" 16 17 MODULE_LICENSE("GPL"); 18 MODULE_DESCRIPTION("NFS localio protocol bypass support"); 19 20 static DEFINE_SPINLOCK(nfs_uuids_lock); 21 22 /* 23 * Global list of nfs_uuid_t instances 24 * that is protected by nfs_uuids_lock. 25 */ 26 static LIST_HEAD(nfs_uuids); 27 28 /* 29 * Lock ordering: 30 * 1: nfs_uuid->lock 31 * 2: nfs_uuids_lock 32 * 3: nfs_uuid->list_lock (aka nn->local_clients_lock) 33 * 34 * May skip locks in select cases, but never hold multiple 35 * locks out of order. 36 */ 37 38 void nfs_uuid_init(nfs_uuid_t *nfs_uuid) 39 { 40 RCU_INIT_POINTER(nfs_uuid->net, NULL); 41 nfs_uuid->dom = NULL; 42 nfs_uuid->list_lock = NULL; 43 INIT_LIST_HEAD(&nfs_uuid->list); 44 INIT_LIST_HEAD(&nfs_uuid->files); 45 spin_lock_init(&nfs_uuid->lock); 46 nfs_uuid->nfs3_localio_probe_count = 0; 47 } 48 EXPORT_SYMBOL_GPL(nfs_uuid_init); 49 50 bool nfs_uuid_begin(nfs_uuid_t *nfs_uuid) 51 { 52 spin_lock(&nfs_uuid->lock); 53 if (rcu_access_pointer(nfs_uuid->net)) { 54 /* This nfs_uuid is already in use */ 55 spin_unlock(&nfs_uuid->lock); 56 return false; 57 } 58 59 spin_lock(&nfs_uuids_lock); 60 if (!list_empty(&nfs_uuid->list)) { 61 /* This nfs_uuid is already in use */ 62 spin_unlock(&nfs_uuids_lock); 63 spin_unlock(&nfs_uuid->lock); 64 return false; 65 } 66 list_add_tail(&nfs_uuid->list, &nfs_uuids); 67 spin_unlock(&nfs_uuids_lock); 68 69 uuid_gen(&nfs_uuid->uuid); 70 spin_unlock(&nfs_uuid->lock); 71 72 return true; 73 } 74 EXPORT_SYMBOL_GPL(nfs_uuid_begin); 75 76 void nfs_uuid_end(nfs_uuid_t *nfs_uuid) 77 { 78 if (!rcu_access_pointer(nfs_uuid->net)) { 79 spin_lock(&nfs_uuid->lock); 80 if (!rcu_access_pointer(nfs_uuid->net)) { 81 /* Not local, remove from nfs_uuids */ 82 spin_lock(&nfs_uuids_lock); 83 list_del_init(&nfs_uuid->list); 84 spin_unlock(&nfs_uuids_lock); 85 } 86 spin_unlock(&nfs_uuid->lock); 87 } 88 } 89 EXPORT_SYMBOL_GPL(nfs_uuid_end); 90 91 static nfs_uuid_t * nfs_uuid_lookup_locked(const uuid_t *uuid) 92 { 93 nfs_uuid_t *nfs_uuid; 94 95 list_for_each_entry(nfs_uuid, &nfs_uuids, list) 96 if (uuid_equal(&nfs_uuid->uuid, uuid)) 97 return nfs_uuid; 98 99 return NULL; 100 } 101 102 static struct module *nfsd_mod; 103 104 void nfs_uuid_is_local(const uuid_t *uuid, struct list_head *list, 105 spinlock_t *list_lock, struct net *net, 106 struct auth_domain *dom, struct module *mod) 107 { 108 nfs_uuid_t *nfs_uuid; 109 110 spin_lock(&nfs_uuids_lock); 111 nfs_uuid = nfs_uuid_lookup_locked(uuid); 112 if (!nfs_uuid) { 113 spin_unlock(&nfs_uuids_lock); 114 return; 115 } 116 117 /* 118 * We don't hold a ref on the net, but instead put 119 * ourselves on @list (nn->local_clients) so the net 120 * pointer can be invalidated. 121 */ 122 spin_lock(list_lock); /* list_lock is nn->local_clients_lock */ 123 list_move(&nfs_uuid->list, list); 124 spin_unlock(list_lock); 125 126 spin_unlock(&nfs_uuids_lock); 127 /* Once nfs_uuid is parented to @list, avoid global nfs_uuids_lock */ 128 spin_lock(&nfs_uuid->lock); 129 130 __module_get(mod); 131 nfsd_mod = mod; 132 133 nfs_uuid->list_lock = list_lock; 134 kref_get(&dom->ref); 135 nfs_uuid->dom = dom; 136 rcu_assign_pointer(nfs_uuid->net, net); 137 spin_unlock(&nfs_uuid->lock); 138 } 139 EXPORT_SYMBOL_GPL(nfs_uuid_is_local); 140 141 void nfs_localio_enable_client(struct nfs_client *clp) 142 { 143 /* nfs_uuid_is_local() does the actual enablement */ 144 trace_nfs_localio_enable_client(clp); 145 } 146 EXPORT_SYMBOL_GPL(nfs_localio_enable_client); 147 148 /* 149 * Cleanup the nfs_uuid_t embedded in an nfs_client. 150 * This is the long-form of nfs_uuid_init(). 151 */ 152 static bool nfs_uuid_put(nfs_uuid_t *nfs_uuid) 153 { 154 LIST_HEAD(local_files); 155 struct nfs_file_localio *nfl, *tmp; 156 157 spin_lock(&nfs_uuid->lock); 158 if (unlikely(!rcu_access_pointer(nfs_uuid->net))) { 159 spin_unlock(&nfs_uuid->lock); 160 return false; 161 } 162 RCU_INIT_POINTER(nfs_uuid->net, NULL); 163 164 if (nfs_uuid->dom) { 165 auth_domain_put(nfs_uuid->dom); 166 nfs_uuid->dom = NULL; 167 } 168 169 list_splice_init(&nfs_uuid->files, &local_files); 170 spin_unlock(&nfs_uuid->lock); 171 172 /* Walk list of files and ensure their last references dropped */ 173 list_for_each_entry_safe(nfl, tmp, &local_files, list) { 174 nfs_close_local_fh(nfl); 175 cond_resched(); 176 } 177 178 spin_lock(&nfs_uuid->lock); 179 BUG_ON(!list_empty(&nfs_uuid->files)); 180 181 /* Remove client from nn->local_clients */ 182 if (nfs_uuid->list_lock) { 183 spin_lock(nfs_uuid->list_lock); 184 BUG_ON(list_empty(&nfs_uuid->list)); 185 list_del_init(&nfs_uuid->list); 186 spin_unlock(nfs_uuid->list_lock); 187 nfs_uuid->list_lock = NULL; 188 } 189 190 module_put(nfsd_mod); 191 spin_unlock(&nfs_uuid->lock); 192 193 return true; 194 } 195 196 void nfs_localio_disable_client(struct nfs_client *clp) 197 { 198 if (nfs_uuid_put(&clp->cl_uuid)) 199 trace_nfs_localio_disable_client(clp); 200 } 201 EXPORT_SYMBOL_GPL(nfs_localio_disable_client); 202 203 void nfs_localio_invalidate_clients(struct list_head *nn_local_clients, 204 spinlock_t *nn_local_clients_lock) 205 { 206 LIST_HEAD(local_clients); 207 nfs_uuid_t *nfs_uuid, *tmp; 208 struct nfs_client *clp; 209 210 spin_lock(nn_local_clients_lock); 211 list_splice_init(nn_local_clients, &local_clients); 212 spin_unlock(nn_local_clients_lock); 213 list_for_each_entry_safe(nfs_uuid, tmp, &local_clients, list) { 214 if (WARN_ON(nfs_uuid->list_lock != nn_local_clients_lock)) 215 break; 216 clp = container_of(nfs_uuid, struct nfs_client, cl_uuid); 217 nfs_localio_disable_client(clp); 218 } 219 } 220 EXPORT_SYMBOL_GPL(nfs_localio_invalidate_clients); 221 222 static void nfs_uuid_add_file(nfs_uuid_t *nfs_uuid, struct nfs_file_localio *nfl) 223 { 224 /* Add nfl to nfs_uuid->files if it isn't already */ 225 spin_lock(&nfs_uuid->lock); 226 if (list_empty(&nfl->list)) { 227 rcu_assign_pointer(nfl->nfs_uuid, nfs_uuid); 228 list_add_tail(&nfl->list, &nfs_uuid->files); 229 } 230 spin_unlock(&nfs_uuid->lock); 231 } 232 233 /* 234 * Caller is responsible for calling nfsd_net_put and 235 * nfsd_file_put (via nfs_to_nfsd_file_put_local). 236 */ 237 struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, 238 struct rpc_clnt *rpc_clnt, const struct cred *cred, 239 const struct nfs_fh *nfs_fh, struct nfs_file_localio *nfl, 240 const fmode_t fmode) 241 { 242 struct net *net; 243 struct nfsd_file *localio; 244 245 /* 246 * Not running in nfsd context, so must safely get reference on nfsd_serv. 247 * But the server may already be shutting down, if so disallow new localio. 248 * uuid->net is NOT a counted reference, but rcu_read_lock() ensures that 249 * if uuid->net is not NULL, then calling nfsd_net_try_get() is safe 250 * and if it succeeds we will have an implied reference to the net. 251 * 252 * Otherwise NFS may not have ref on NFSD and therefore cannot safely 253 * make 'nfs_to' calls. 254 */ 255 rcu_read_lock(); 256 net = rcu_dereference(uuid->net); 257 if (!net || !nfs_to->nfsd_net_try_get(net)) { 258 rcu_read_unlock(); 259 return ERR_PTR(-ENXIO); 260 } 261 rcu_read_unlock(); 262 /* We have an implied reference to net thanks to nfsd_net_try_get */ 263 localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, 264 cred, nfs_fh, fmode); 265 if (IS_ERR(localio)) 266 nfs_to_nfsd_net_put(net); 267 else 268 nfs_uuid_add_file(uuid, nfl); 269 270 return localio; 271 } 272 EXPORT_SYMBOL_GPL(nfs_open_local_fh); 273 274 void nfs_close_local_fh(struct nfs_file_localio *nfl) 275 { 276 struct nfsd_file *ro_nf = NULL; 277 struct nfsd_file *rw_nf = NULL; 278 nfs_uuid_t *nfs_uuid; 279 280 rcu_read_lock(); 281 nfs_uuid = rcu_dereference(nfl->nfs_uuid); 282 if (!nfs_uuid) { 283 /* regular (non-LOCALIO) NFS will hammer this */ 284 rcu_read_unlock(); 285 return; 286 } 287 288 ro_nf = rcu_access_pointer(nfl->ro_file); 289 rw_nf = rcu_access_pointer(nfl->rw_file); 290 if (ro_nf || rw_nf) { 291 spin_lock(&nfs_uuid->lock); 292 if (ro_nf) 293 ro_nf = rcu_dereference_protected(xchg(&nfl->ro_file, NULL), 1); 294 if (rw_nf) 295 rw_nf = rcu_dereference_protected(xchg(&nfl->rw_file, NULL), 1); 296 297 /* Remove nfl from nfs_uuid->files list */ 298 RCU_INIT_POINTER(nfl->nfs_uuid, NULL); 299 list_del_init(&nfl->list); 300 spin_unlock(&nfs_uuid->lock); 301 rcu_read_unlock(); 302 303 if (ro_nf) 304 nfs_to_nfsd_file_put_local(ro_nf); 305 if (rw_nf) 306 nfs_to_nfsd_file_put_local(rw_nf); 307 return; 308 } 309 rcu_read_unlock(); 310 } 311 EXPORT_SYMBOL_GPL(nfs_close_local_fh); 312 313 /* 314 * The NFS LOCALIO code needs to call into NFSD using various symbols, 315 * but cannot be statically linked, because that will make the NFS 316 * module always depend on the NFSD module. 317 * 318 * 'nfs_to' provides NFS access to NFSD functions needed for LOCALIO, 319 * its lifetime is tightly coupled to the NFSD module and will always 320 * be available to NFS LOCALIO because any successful client<->server 321 * LOCALIO handshake results in a reference on the NFSD module (above), 322 * so NFS implicitly holds a reference to the NFSD module and its 323 * functions in the 'nfs_to' nfsd_localio_operations cannot disappear. 324 * 325 * If the last NFS client using LOCALIO disconnects (and its reference 326 * on NFSD dropped) then NFSD could be unloaded, resulting in 'nfs_to' 327 * functions being invalid pointers. But if NFSD isn't loaded then NFS 328 * will not be able to handshake with NFSD and will have no cause to 329 * try to call 'nfs_to' function pointers. If/when NFSD is reloaded it 330 * will reinitialize the 'nfs_to' function pointers and make LOCALIO 331 * possible. 332 */ 333 const struct nfsd_localio_operations *nfs_to; 334 EXPORT_SYMBOL_GPL(nfs_to); 335