1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Object lifetime handling and tracing. 3 * 4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/slab.h> 9 #include <linux/mempool.h> 10 #include "internal.h" 11 12 static void netfs_free_request(struct work_struct *work); 13 14 /* 15 * Allocate an I/O request and initialise it. 16 */ 17 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, 18 struct file *file, 19 loff_t start, size_t len, 20 enum netfs_io_origin origin) 21 { 22 static atomic_t debug_ids; 23 struct inode *inode = file ? file_inode(file) : mapping->host; 24 struct netfs_inode *ctx = netfs_inode(inode); 25 struct netfs_io_request *rreq; 26 mempool_t *mempool = ctx->ops->request_pool ?: &netfs_request_pool; 27 struct kmem_cache *cache = mempool->pool_data; 28 gfp_t gfp = GFP_KERNEL; 29 int ret; 30 31 /* Writeback is part of memory reclaim and must not fail due to ENOMEM. */ 32 if (origin == NETFS_WRITEBACK || origin == NETFS_WRITEBACK_SINGLE) { 33 gfp = GFP_NOFS; /* Allows use of mempools. */ 34 35 rreq = mempool_alloc(mempool, gfp); 36 } else { 37 rreq = mempool->alloc(gfp, mempool->pool_data); 38 if (!rreq) 39 return ERR_PTR(-ENOMEM); 40 } 41 42 memset(rreq, 0, kmem_cache_size(cache)); 43 INIT_WORK(&rreq->cleanup_work, netfs_free_request); 44 rreq->gfp = gfp; 45 rreq->start = start; 46 rreq->len = len; 47 rreq->origin = origin; 48 rreq->netfs_ops = ctx->ops; 49 rreq->mapping = mapping; 50 rreq->inode = inode; 51 rreq->i_size = i_size_read(inode); 52 rreq->debug_id = atomic_inc_return(&debug_ids); 53 rreq->wsize = INT_MAX; 54 rreq->io_streams[0].sreq_max_len = ULONG_MAX; 55 rreq->io_streams[0].sreq_max_segs = 0; 56 spin_lock_init(&rreq->lock); 57 INIT_LIST_HEAD(&rreq->io_streams[0].subrequests); 58 INIT_LIST_HEAD(&rreq->io_streams[1].subrequests); 59 init_waitqueue_head(&rreq->waitq); 60 refcount_set(&rreq->ref, 2); 61 62 if (origin == NETFS_READAHEAD || 63 origin == NETFS_READPAGE || 64 origin == NETFS_READ_GAPS || 65 origin == NETFS_READ_SINGLE || 66 origin == NETFS_READ_FOR_WRITE || 67 origin == NETFS_UNBUFFERED_READ || 68 origin == NETFS_DIO_READ) { 69 INIT_WORK(&rreq->work, netfs_read_collection_worker); 70 rreq->io_streams[0].avail = true; 71 } else { 72 INIT_WORK(&rreq->work, netfs_write_collection_worker); 73 } 74 75 __set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags); 76 if (rreq->netfs_ops->init_request) { 77 ret = rreq->netfs_ops->init_request(rreq, file); 78 if (ret < 0) { 79 mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); 80 return ERR_PTR(ret); 81 } 82 } 83 84 atomic_inc(&ctx->io_count); 85 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), netfs_rreq_trace_new); 86 netfs_proc_add_rreq(rreq); 87 netfs_stat(&netfs_n_rh_rreq); 88 return rreq; 89 } 90 91 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) 92 { 93 int r; 94 95 __refcount_inc(&rreq->ref, &r); 96 trace_netfs_rreq_ref(rreq->debug_id, r + 1, what); 97 } 98 99 void netfs_clear_subrequests(struct netfs_io_request *rreq) 100 { 101 struct netfs_io_subrequest *subreq; 102 struct netfs_io_stream *stream; 103 int s; 104 105 for (s = 0; s < ARRAY_SIZE(rreq->io_streams); s++) { 106 stream = &rreq->io_streams[s]; 107 while (!list_empty(&stream->subrequests)) { 108 subreq = list_first_entry(&stream->subrequests, 109 struct netfs_io_subrequest, rreq_link); 110 list_del(&subreq->rreq_link); 111 netfs_put_subrequest(subreq, netfs_sreq_trace_put_clear); 112 } 113 } 114 } 115 116 static void netfs_free_request_rcu(struct rcu_head *rcu) 117 { 118 struct netfs_io_request *rreq = container_of(rcu, struct netfs_io_request, rcu); 119 120 mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); 121 netfs_stat_d(&netfs_n_rh_rreq); 122 } 123 124 static void netfs_deinit_request(struct netfs_io_request *rreq) 125 { 126 struct netfs_inode *ictx = netfs_inode(rreq->inode); 127 unsigned int i; 128 129 trace_netfs_rreq(rreq, netfs_rreq_trace_free); 130 131 /* Cancel/flush the result collection worker. That does not carry a 132 * ref of its own, so we must wait for it somewhere. 133 */ 134 cancel_work_sync(&rreq->work); 135 136 netfs_proc_del_rreq(rreq); 137 netfs_clear_subrequests(rreq); 138 if (rreq->netfs_ops->free_request) 139 rreq->netfs_ops->free_request(rreq); 140 if (rreq->cache_resources.ops) 141 rreq->cache_resources.ops->end_operation(&rreq->cache_resources); 142 if (rreq->direct_bv) { 143 for (i = 0; i < rreq->direct_bv_count; i++) { 144 if (rreq->direct_bv[i].bv_page) { 145 if (rreq->direct_bv_unpin) 146 unpin_user_page(rreq->direct_bv[i].bv_page); 147 } 148 } 149 kvfree(rreq->direct_bv); 150 } 151 rolling_buffer_clear(&rreq->buffer); 152 153 if (atomic_dec_and_test(&ictx->io_count)) 154 wake_up_var(&ictx->io_count); 155 } 156 157 static void netfs_free_request(struct work_struct *work) 158 { 159 struct netfs_io_request *rreq = 160 container_of(work, struct netfs_io_request, cleanup_work); 161 162 netfs_deinit_request(rreq); 163 call_rcu(&rreq->rcu, netfs_free_request_rcu); 164 } 165 166 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) 167 { 168 unsigned int debug_id; 169 bool dead; 170 int r; 171 172 if (rreq) { 173 debug_id = rreq->debug_id; 174 dead = __refcount_dec_and_test(&rreq->ref, &r); 175 trace_netfs_rreq_ref(debug_id, r - 1, what); 176 if (dead) 177 WARN_ON(!queue_work(system_dfl_wq, &rreq->cleanup_work)); 178 } 179 } 180 181 /* 182 * Free a request (synchronously) that was just allocated but has 183 * failed before it could be submitted. 184 */ 185 void netfs_put_failed_request(struct netfs_io_request *rreq) 186 { 187 int r = refcount_read(&rreq->ref); 188 189 /* new requests have two references (see 190 * netfs_alloc_request(), and this function is only allowed on 191 * new request objects 192 */ 193 WARN_ON_ONCE(r != 2); 194 195 trace_netfs_rreq_ref(rreq->debug_id, r, netfs_rreq_trace_put_failed); 196 netfs_free_request(&rreq->cleanup_work); 197 } 198 199 /* 200 * Allocate and partially initialise an I/O request structure. 201 */ 202 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq) 203 { 204 struct netfs_io_subrequest *subreq; 205 mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool; 206 struct kmem_cache *cache = mempool->pool_data; 207 208 if (rreq->gfp == GFP_KERNEL) 209 subreq = mempool->alloc(rreq->gfp, mempool->pool_data); 210 else 211 subreq = mempool_alloc(mempool, rreq->gfp); 212 if (!subreq) 213 return NULL; 214 215 memset(subreq, 0, kmem_cache_size(cache)); 216 INIT_WORK(&subreq->work, NULL); 217 INIT_LIST_HEAD(&subreq->rreq_link); 218 refcount_set(&subreq->ref, 2); 219 subreq->rreq = rreq; 220 subreq->debug_index = atomic_inc_return(&rreq->subreq_counter); 221 netfs_get_request(rreq, netfs_rreq_trace_get_subreq); 222 netfs_stat(&netfs_n_rh_sreq); 223 return subreq; 224 } 225 226 void netfs_get_subrequest(struct netfs_io_subrequest *subreq, 227 enum netfs_sreq_ref_trace what) 228 { 229 int r; 230 231 __refcount_inc(&subreq->ref, &r); 232 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1, 233 what); 234 } 235 236 static void netfs_free_subrequest(struct netfs_io_subrequest *subreq) 237 { 238 struct netfs_io_request *rreq = subreq->rreq; 239 240 trace_netfs_sreq(subreq, netfs_sreq_trace_free); 241 if (rreq->netfs_ops->free_subrequest) 242 rreq->netfs_ops->free_subrequest(subreq); 243 mempool_free(subreq, rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool); 244 netfs_stat_d(&netfs_n_rh_sreq); 245 netfs_put_request(rreq, netfs_rreq_trace_put_subreq); 246 } 247 248 void netfs_put_subrequest(struct netfs_io_subrequest *subreq, 249 enum netfs_sreq_ref_trace what) 250 { 251 unsigned int debug_index = subreq->debug_index; 252 unsigned int debug_id = subreq->rreq->debug_id; 253 bool dead; 254 int r; 255 256 dead = __refcount_dec_and_test(&subreq->ref, &r); 257 trace_netfs_sreq_ref(debug_id, debug_index, r - 1, what); 258 if (dead) 259 netfs_free_subrequest(subreq); 260 } 261