1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Object lifetime handling and tracing. 3 * 4 * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/slab.h> 9 #include <linux/mempool.h> 10 #include "internal.h" 11 12 static void netfs_free_request(struct work_struct *work); 13 14 /* 15 * Allocate an I/O request and initialise it. 16 */ 17 struct netfs_io_request *netfs_alloc_request(struct address_space *mapping, 18 struct file *file, 19 loff_t start, size_t len, 20 enum netfs_io_origin origin) 21 { 22 static atomic_t debug_ids; 23 struct inode *inode = file ? file_inode(file) : mapping->host; 24 struct netfs_inode *ctx = netfs_inode(inode); 25 struct netfs_io_request *rreq; 26 mempool_t *mempool = ctx->ops->request_pool ?: &netfs_request_pool; 27 struct kmem_cache *cache = mempool->pool_data; 28 gfp_t gfp = GFP_KERNEL; 29 int ret; 30 31 /* Writeback is part of memory reclaim and must not fail due to ENOMEM. */ 32 if (origin == NETFS_WRITEBACK || origin == NETFS_WRITEBACK_SINGLE) { 33 gfp = GFP_NOFS; /* Allows use of mempools. */ 34 35 rreq = mempool_alloc(mempool, gfp); 36 } else { 37 rreq = mempool->alloc(gfp, mempool->pool_data); 38 if (!rreq) 39 return ERR_PTR(-ENOMEM); 40 } 41 42 memset(rreq, 0, kmem_cache_size(cache)); 43 INIT_WORK(&rreq->cleanup_work, netfs_free_request); 44 rreq->gfp = gfp; 45 rreq->start = start; 46 rreq->collected_to = start; 47 rreq->cleaned_to = start; 48 rreq->len = len; 49 rreq->progress_at = 0; 50 rreq->origin = origin; 51 rreq->netfs_ops = ctx->ops; 52 rreq->mapping = mapping; 53 rreq->inode = inode; 54 rreq->i_size = i_size_read(inode); 55 rreq->debug_id = atomic_inc_return(&debug_ids); 56 rreq->wsize = INT_MAX; 57 rreq->io_streams[0].sreq_max_len = ULONG_MAX; 58 rreq->io_streams[0].sreq_max_segs = 0; 59 spin_lock_init(&rreq->lock); 60 init_waitqueue_head(&rreq->waitq); 61 refcount_set(&rreq->ref, 2); 62 63 for (int s = 0; s < NR_IO_STREAMS; s++) { 64 struct netfs_io_stream *stream = &rreq->io_streams[s]; 65 66 INIT_LIST_HEAD(&stream->subrequests); 67 stream->collected_to = rreq->start; 68 } 69 70 if (origin == NETFS_READAHEAD || 71 origin == NETFS_READPAGE || 72 origin == NETFS_READ_GAPS || 73 origin == NETFS_READ_SINGLE || 74 origin == NETFS_READ_FOR_WRITE || 75 origin == NETFS_UNBUFFERED_READ || 76 origin == NETFS_DIO_READ) { 77 INIT_WORK(&rreq->work, netfs_read_collection_worker); 78 rreq->io_streams[0].avail = true; 79 } else { 80 INIT_WORK(&rreq->work, netfs_write_collection_worker); 81 } 82 83 __set_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags); 84 if (rreq->netfs_ops->init_request) { 85 ret = rreq->netfs_ops->init_request(rreq, file); 86 if (ret < 0) { 87 mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); 88 return ERR_PTR(ret); 89 } 90 } 91 92 atomic_inc(&ctx->io_count); 93 trace_netfs_rreq_ref(rreq->debug_id, refcount_read(&rreq->ref), netfs_rreq_trace_new); 94 netfs_proc_add_rreq(rreq); 95 netfs_stat(&netfs_n_rh_rreq); 96 return rreq; 97 } 98 99 void netfs_get_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) 100 { 101 int r; 102 103 __refcount_inc(&rreq->ref, &r); 104 trace_netfs_rreq_ref(rreq->debug_id, r + 1, what); 105 } 106 107 void netfs_clear_subrequests(struct netfs_io_request *rreq) 108 { 109 struct netfs_io_subrequest *subreq; 110 struct netfs_io_stream *stream; 111 int s; 112 113 for (s = 0; s < ARRAY_SIZE(rreq->io_streams); s++) { 114 stream = &rreq->io_streams[s]; 115 while (!list_empty(&stream->subrequests)) { 116 subreq = list_first_entry(&stream->subrequests, 117 struct netfs_io_subrequest, rreq_link); 118 list_del(&subreq->rreq_link); 119 netfs_put_subrequest(subreq, netfs_sreq_trace_put_clear); 120 } 121 } 122 } 123 124 static void netfs_free_request_rcu(struct rcu_head *rcu) 125 { 126 struct netfs_io_request *rreq = container_of(rcu, struct netfs_io_request, rcu); 127 128 mempool_free(rreq, rreq->netfs_ops->request_pool ?: &netfs_request_pool); 129 netfs_stat_d(&netfs_n_rh_rreq); 130 } 131 132 static void netfs_deinit_request(struct netfs_io_request *rreq) 133 { 134 struct netfs_inode *ictx = netfs_inode(rreq->inode); 135 unsigned int i; 136 137 trace_netfs_rreq(rreq, netfs_rreq_trace_free); 138 139 /* Cancel/flush the result collection worker. That does not carry a 140 * ref of its own, so we must wait for it somewhere. 141 */ 142 cancel_work_sync(&rreq->work); 143 144 netfs_proc_del_rreq(rreq); 145 netfs_clear_subrequests(rreq); 146 if (rreq->netfs_ops->free_request) 147 rreq->netfs_ops->free_request(rreq); 148 if (rreq->cache_resources.ops) 149 rreq->cache_resources.ops->end_operation(&rreq->cache_resources); 150 if (rreq->direct_bv) { 151 for (i = 0; i < rreq->direct_bv_count; i++) { 152 if (rreq->direct_bv[i].bv_page) { 153 if (rreq->direct_bv_unpin) 154 unpin_user_page(rreq->direct_bv[i].bv_page); 155 } 156 } 157 kvfree(rreq->direct_bv); 158 } 159 rolling_buffer_clear(&rreq->buffer); 160 161 if (atomic_dec_and_test(&ictx->io_count)) 162 wake_up_var(&ictx->io_count); 163 } 164 165 static void netfs_free_request(struct work_struct *work) 166 { 167 struct netfs_io_request *rreq = 168 container_of(work, struct netfs_io_request, cleanup_work); 169 170 netfs_deinit_request(rreq); 171 call_rcu(&rreq->rcu, netfs_free_request_rcu); 172 } 173 174 void netfs_put_request(struct netfs_io_request *rreq, enum netfs_rreq_ref_trace what) 175 { 176 unsigned int debug_id; 177 bool dead; 178 int r; 179 180 if (rreq) { 181 debug_id = rreq->debug_id; 182 dead = __refcount_dec_and_test(&rreq->ref, &r); 183 trace_netfs_rreq_ref(debug_id, r - 1, what); 184 if (dead) 185 WARN_ON(!queue_work(system_dfl_wq, &rreq->cleanup_work)); 186 } 187 } 188 189 /* 190 * Free a request (synchronously) that was just allocated but has 191 * failed before it could be submitted. 192 */ 193 void netfs_put_failed_request(struct netfs_io_request *rreq) 194 { 195 int r = refcount_read(&rreq->ref); 196 197 /* new requests have two references (see 198 * netfs_alloc_request(), and this function is only allowed on 199 * new request objects 200 */ 201 WARN_ON_ONCE(r != 2); 202 203 trace_netfs_rreq_ref(rreq->debug_id, r, netfs_rreq_trace_put_failed); 204 netfs_free_request(&rreq->cleanup_work); 205 } 206 207 /* 208 * Allocate and partially initialise an I/O request structure. 209 */ 210 struct netfs_io_subrequest *netfs_alloc_subrequest(struct netfs_io_request *rreq) 211 { 212 struct netfs_io_subrequest *subreq; 213 mempool_t *mempool = rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool; 214 struct kmem_cache *cache = mempool->pool_data; 215 216 if (rreq->gfp == GFP_KERNEL) 217 subreq = mempool->alloc(rreq->gfp, mempool->pool_data); 218 else 219 subreq = mempool_alloc(mempool, rreq->gfp); 220 if (!subreq) 221 return NULL; 222 223 memset(subreq, 0, kmem_cache_size(cache)); 224 INIT_WORK(&subreq->work, NULL); 225 INIT_LIST_HEAD(&subreq->rreq_link); 226 refcount_set(&subreq->ref, 2); 227 subreq->rreq = rreq; 228 subreq->debug_index = atomic_inc_return(&rreq->subreq_counter); 229 netfs_get_request(rreq, netfs_rreq_trace_get_subreq); 230 netfs_stat(&netfs_n_rh_sreq); 231 return subreq; 232 } 233 234 void netfs_get_subrequest(struct netfs_io_subrequest *subreq, 235 enum netfs_sreq_ref_trace what) 236 { 237 int r; 238 239 __refcount_inc(&subreq->ref, &r); 240 trace_netfs_sreq_ref(subreq->rreq->debug_id, subreq->debug_index, r + 1, 241 what); 242 } 243 244 static void netfs_free_subrequest(struct netfs_io_subrequest *subreq) 245 { 246 struct netfs_io_request *rreq = subreq->rreq; 247 248 trace_netfs_sreq(subreq, netfs_sreq_trace_free); 249 if (rreq->netfs_ops->free_subrequest) 250 rreq->netfs_ops->free_subrequest(subreq); 251 mempool_free(subreq, rreq->netfs_ops->subrequest_pool ?: &netfs_subrequest_pool); 252 netfs_stat_d(&netfs_n_rh_sreq); 253 netfs_put_request(rreq, netfs_rreq_trace_put_subreq); 254 } 255 256 void netfs_put_subrequest(struct netfs_io_subrequest *subreq, 257 enum netfs_sreq_ref_trace what) 258 { 259 unsigned int debug_index = subreq->debug_index; 260 unsigned int debug_id = subreq->rreq->debug_id; 261 bool dead; 262 int r; 263 264 dead = __refcount_dec_and_test(&subreq->ref, &r); 265 trace_netfs_sreq_ref(debug_id, debug_index, r - 1, what); 266 if (dead) 267 netfs_free_subrequest(subreq); 268 } 269