1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Object lifetime handling and tracing. 3 * 4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/slab.h> 9 #include <linux/mempool.h> 10 #include "internal.h" 11 12 static void netfs_free_request(struct work_struct *work); 13 14 /* 15 * Allocate an I/O request and initialise it. 16 */ 17 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, 18 struct file *file, 19 uoff_t start, size_t len, 20 enum netfs_io_origin origin) 21 { 22 static atomic_t debug_ids; 23 struct inode *inode = file ? file_inode(file) : mapping->host; 24 struct netfs_inode *ctx = netfs_inode(inode); 25 struct netfs_io_request *rreq; 26 mempool_t *mempool = ctx->ops->request_pool ?: &netfs_request_pool; 27 struct kmem_cache *cache = mempool->pool_data; 28 gfp_t gfp = GFP_KERNEL; 29 int ret; 30 31 /* Writeback is part of memory reclaim and must not fail due to ENOMEM. */ 32 if (origin == NETFS_WRITEBACK || origin == NETFS_WRITEBACK_SINGLE) { 33 gfp = GFP_NOFS; /* Allows use of mempools. */ 34 35 rreq = mempool_alloc(mempool, gfp); 36 } else { 37 rreq = mempool_alloc_noreserve(mempool, gfp); 38 if (!rreq) 39 return ERR_PTR(-ENOMEM); 40 } 41 42 memset(rreq, 0, kmem_cache_size(cache)); 43 INIT_WORK(&rreq->cleanup_work, netfs_free_request); 44 rreq->gfp = gfp; 45 rreq->start = start; 46 rreq->collected_to = start; 47 rreq->cache_coll_to = start; 48 rreq->cleaned_to = start; 49 rreq->len = len; 50 rreq->progress_at = 0; 51 rreq->origin = origin; 52 rreq->netfs_ops = ctx->ops; 53 rreq->mapping = mapping; 54 rreq->inode = inode; 55 rreq->i_size = i_size_read(inode); 56 rreq->debug_id = atomic_inc_return(&debug_ids); 57 rreq->wsize = INT_MAX; 58 rreq->io_streams[0].sreq_max_len = ULONG_MAX; 59 rreq->io_streams[0].sreq_max_segs = 0; 60 spin_lock_init(&rreq->lock); 61 init_waitqueue_head(&rreq->waitq); 62 refcount_set(&rreq->ref, 2); 63 64 for (int s = 0; s < NR_IO_STREAMS; s++) { 65 struct netfs_io_stream *stream = &rreq->io_streams[s]; 66 67 INIT_LIST_HEAD(&stream->subrequests); 68 stream->collected_to = rreq->start; 69 } 70 71 if (origin == NETFS_READAHEAD || 72 origin == NETFS_READPAGE || 73 origin == NETFS_READ_GAPS || 74 origin == NETFS_READ_SINGLE || 75 origin == NETFS_READ_FOR_WRITE || 76 origin == NETFS_UNBUFFERED_READ || 77 origin == NETFS_DIO_READ) { 78 INIT_WORK(&rreq->work, netfs_read_collection_worker); 79 rreq->io_streams[0].avail = true; 80 } else { 81 INIT_WORK(&rreq->work, netfs_write_collection_worker); 82 } 83 84 __set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags); 85 if (rreq->netfs_ops->init_request) { 86 ret = rreq->netfs_ops->init_request(rreq, file); 87 if (ret < 0) { 88 mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); 89 return ERR_PTR(ret); 90 } 91 } 92 93 atomic_inc(&ctx->io_count); 94 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), netfs_rreq_trace_new); 95 netfs_proc_add_rreq(rreq); 96 netfs_stat(&netfs_n_rh_rreq); 97 return rreq; 98 } 99 100 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) 101 { 102 int r; 103 104 __refcount_inc(&rreq->ref, &r); 105 trace_netfs_rreq_ref(rreq->debug_id, r + 1, what); 106 } 107 108 void netfs_clear_subrequests(struct netfs_io_request *rreq) 109 { 110 struct netfs_io_subrequest *subreq; 111 struct netfs_io_stream *stream; 112 int s; 113 114 for (s = 0; s < ARRAY_SIZE(rreq->io_streams); s++) { 115 stream = &rreq->io_streams[s]; 116 while (!list_empty(&stream->subrequests)) { 117 subreq = list_first_entry(&stream->subrequests, 118 struct netfs_io_subrequest, rreq_link); 119 list_del(&subreq->rreq_link); 120 netfs_put_subrequest(subreq, netfs_sreq_trace_put_clear); 121 } 122 } 123 } 124 125 static void netfs_free_request_rcu(struct rcu_head *rcu) 126 { 127 struct netfs_io_request *rreq = container_of(rcu, struct netfs_io_request, rcu); 128 129 mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); 130 netfs_stat_d(&netfs_n_rh_rreq); 131 } 132 133 static void netfs_deinit_request(struct netfs_io_request *rreq) 134 { 135 struct netfs_inode *ictx = netfs_inode(rreq->inode); 136 unsigned int i; 137 138 trace_netfs_rreq(rreq, netfs_rreq_trace_free); 139 140 /* Cancel/flush the result collection worker. That does not carry a 141 * ref of its own, so we must wait for it somewhere. 142 */ 143 cancel_work_sync(&rreq->work); 144 145 netfs_proc_del_rreq(rreq); 146 netfs_clear_subrequests(rreq); 147 if (rreq->netfs_ops->free_request) 148 rreq->netfs_ops->free_request(rreq); 149 if (rreq->cache_resources.ops) 150 rreq->cache_resources.ops->end_operation(&rreq->cache_resources); 151 if (rreq->direct_bv) { 152 for (i = 0; i < rreq->direct_bv_count; i++) { 153 if (rreq->direct_bv[i].bv_page) { 154 if (rreq->direct_bv_unpin) 155 unpin_user_page(rreq->direct_bv[i].bv_page); 156 } 157 } 158 kvfree(rreq->direct_bv); 159 } 160 rolling_buffer_clear(&rreq->buffer); 161 162 if (atomic_dec_and_test(&ictx->io_count)) 163 wake_up_var(&ictx->io_count); 164 } 165 166 static void netfs_free_request(struct work_struct *work) 167 { 168 struct netfs_io_request *rreq = 169 container_of(work, struct netfs_io_request, cleanup_work); 170 171 netfs_deinit_request(rreq); 172 call_rcu(&rreq->rcu, netfs_free_request_rcu); 173 } 174 175 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) 176 { 177 unsigned int debug_id; 178 bool dead; 179 int r; 180 181 if (rreq) { 182 debug_id = rreq->debug_id; 183 dead = __refcount_dec_and_test(&rreq->ref, &r); 184 trace_netfs_rreq_ref(debug_id, r - 1, what); 185 if (dead) 186 WARN_ON(!queue_work(system_dfl_wq, &rreq->cleanup_work)); 187 } 188 } 189 190 /* 191 * Free a request (synchronously) that was just allocated but has 192 * failed before it could be submitted. 193 */ 194 void netfs_put_failed_request(struct netfs_io_request *rreq) 195 { 196 int r = refcount_read(&rreq->ref); 197 198 /* new requests have two references (see 199 * netfs_alloc_request(), and this function is only allowed on 200 * new request objects 201 */ 202 WARN_ON_ONCE(r != 2); 203 204 trace_netfs_rreq_ref(rreq->debug_id, r, netfs_rreq_trace_put_failed); 205 netfs_free_request(&rreq->cleanup_work); 206 } 207 208 /* 209 * Allocate and partially initialise an I/O request structure. 210 */ 211 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq, 212 enum netfs_io_source source) 213 { 214 struct netfs_io_subrequest *subreq; 215 mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool; 216 struct kmem_cache *cache = mempool->pool_data; 217 218 if (rreq->gfp == GFP_KERNEL) 219 subreq = mempool_alloc_noreserve(mempool, rreq->gfp); 220 else 221 subreq = mempool_alloc(mempool, rreq->gfp); 222 if (!subreq) 223 return NULL; 224 225 memset(subreq, 0, kmem_cache_size(cache)); 226 INIT_WORK(&subreq->work, NULL); 227 INIT_LIST_HEAD(&subreq->rreq_link); 228 refcount_set(&subreq->ref, 2); 229 subreq->source = source; 230 subreq->rreq = rreq; 231 subreq->debug_index = atomic_inc_return(&rreq->subreq_counter); 232 netfs_get_request(rreq, netfs_rreq_trace_get_subreq); 233 netfs_stat(&netfs_n_rh_sreq); 234 return subreq; 235 } 236 237 void netfs_get_subrequest(struct netfs_io_subrequest *subreq, 238 enum netfs_sreq_ref_trace what) 239 { 240 int r; 241 242 __refcount_inc(&subreq->ref, &r); 243 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1, 244 what); 245 } 246 247 static void netfs_free_subrequest(struct netfs_io_subrequest *subreq) 248 { 249 struct netfs_io_request *rreq = subreq->rreq; 250 251 trace_netfs_sreq(subreq, netfs_sreq_trace_free); 252 if (rreq->netfs_ops->free_subrequest) 253 rreq->netfs_ops->free_subrequest(subreq); 254 mempool_free(subreq, rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool); 255 netfs_stat_d(&netfs_n_rh_sreq); 256 netfs_put_request(rreq, netfs_rreq_trace_put_subreq); 257 } 258 259 void netfs_put_subrequest(struct netfs_io_subrequest *subreq, 260 enum netfs_sreq_ref_trace what) 261 { 262 unsigned int debug_index = subreq->debug_index; 263 unsigned int debug_id = subreq->rreq->debug_id; 264 bool dead; 265 int r; 266 267 dead = __refcount_dec_and_test(&subreq->ref, &r); 268 trace_netfs_sreq_ref(debug_id, debug_index, r - 1, what); 269 if (dead) 270 netfs_free_subrequest(subreq); 271 } 272