1c9de560dSAlex Tomas /* 2c9de560dSAlex Tomas * Copyright (c) 2003-2006, Cluster File Systems, Inc, info@clusterfs.com 3c9de560dSAlex Tomas * Written by Alex Tomas <alex@clusterfs.com> 4c9de560dSAlex Tomas * 5c9de560dSAlex Tomas * This program is free software; you can redistribute it and/or modify 6c9de560dSAlex Tomas * it under the terms of the GNU General Public License version 2 as 7c9de560dSAlex Tomas * published by the Free Software Foundation. 8c9de560dSAlex Tomas * 9c9de560dSAlex Tomas * This program is distributed in the hope that it will be useful, 10c9de560dSAlex Tomas * but WITHOUT ANY WARRANTY; without even the implied warranty of 11c9de560dSAlex Tomas * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12c9de560dSAlex Tomas * GNU General Public License for more details. 13c9de560dSAlex Tomas * 14c9de560dSAlex Tomas * You should have received a copy of the GNU General Public Licens 15c9de560dSAlex Tomas * along with this program; if not, write to the Free Software 16c9de560dSAlex Tomas * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111- 17c9de560dSAlex Tomas */ 18c9de560dSAlex Tomas 19c9de560dSAlex Tomas 20c9de560dSAlex Tomas /* 21c9de560dSAlex Tomas * mballoc.c contains the multiblocks allocation routines 22c9de560dSAlex Tomas */ 23c9de560dSAlex Tomas 248f6e39a7SMingming Cao #include "mballoc.h" 256ba495e9STheodore Ts'o #include <linux/debugfs.h> 265a0e3ad6STejun Heo #include <linux/slab.h> 279bffad1eSTheodore Ts'o #include <trace/events/ext4.h> 289bffad1eSTheodore Ts'o 29c9de560dSAlex Tomas /* 30c9de560dSAlex Tomas * MUSTDO: 31c9de560dSAlex Tomas * - test ext4_ext_search_left() and ext4_ext_search_right() 32c9de560dSAlex Tomas * - search for metadata in few groups 33c9de560dSAlex Tomas * 34c9de560dSAlex Tomas * TODO v4: 35c9de560dSAlex Tomas * - normalization should take into account whether file is still open 36c9de560dSAlex Tomas * - discard preallocations if no free space left (policy?) 37c9de560dSAlex Tomas * - don't normalize tails 38c9de560dSAlex Tomas * - quota 39c9de560dSAlex Tomas * - reservation for superuser 40c9de560dSAlex Tomas * 41c9de560dSAlex Tomas * TODO v3: 42c9de560dSAlex Tomas * - bitmap read-ahead (proposed by Oleg Drokin aka green) 43c9de560dSAlex Tomas * - track min/max extents in each group for better group selection 44c9de560dSAlex Tomas * - mb_mark_used() may allocate chunk right after splitting buddy 45c9de560dSAlex Tomas * - tree of groups sorted by number of free blocks 46c9de560dSAlex Tomas * - error handling 47c9de560dSAlex Tomas */ 48c9de560dSAlex Tomas 49c9de560dSAlex Tomas /* 50c9de560dSAlex Tomas * The allocation request involve request for multiple number of blocks 51c9de560dSAlex Tomas * near to the goal(block) value specified. 52c9de560dSAlex Tomas * 53b713a5ecSTheodore Ts'o * During initialization phase of the allocator we decide to use the 54b713a5ecSTheodore Ts'o * group preallocation or inode preallocation depending on the size of 55b713a5ecSTheodore Ts'o * the file. The size of the file could be the resulting file size we 56b713a5ecSTheodore Ts'o * would have after allocation, or the current file size, which ever 57b713a5ecSTheodore Ts'o * is larger. If the size is less than sbi->s_mb_stream_request we 58b713a5ecSTheodore Ts'o * select to use the group preallocation. The default value of 59b713a5ecSTheodore Ts'o * s_mb_stream_request is 16 blocks. This can also be tuned via 60b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_stream_req. The value is represented in 61b713a5ecSTheodore Ts'o * terms of number of blocks. 62c9de560dSAlex Tomas * 63c9de560dSAlex Tomas * The main motivation for having small file use group preallocation is to 64b713a5ecSTheodore Ts'o * ensure that we have small files closer together on the disk. 65c9de560dSAlex Tomas * 66b713a5ecSTheodore Ts'o * First stage the allocator looks at the inode prealloc list, 67b713a5ecSTheodore Ts'o * ext4_inode_info->i_prealloc_list, which contains list of prealloc 68b713a5ecSTheodore Ts'o * spaces for this particular inode. The inode prealloc space is 69b713a5ecSTheodore Ts'o * represented as: 70c9de560dSAlex Tomas * 71c9de560dSAlex Tomas * pa_lstart -> the logical start block for this prealloc space 72c9de560dSAlex Tomas * pa_pstart -> the physical start block for this prealloc space 73*53accfa9STheodore Ts'o * pa_len -> length for this prealloc space (in clusters) 74*53accfa9STheodore Ts'o * pa_free -> free space available in this prealloc space (in clusters) 75c9de560dSAlex Tomas * 76c9de560dSAlex Tomas * The inode preallocation space is used looking at the _logical_ start 77c9de560dSAlex Tomas * block. If only the logical file block falls within the range of prealloc 78caaf7a29STao Ma * space we will consume the particular prealloc space. This makes sure that 79caaf7a29STao Ma * we have contiguous physical blocks representing the file blocks 80c9de560dSAlex Tomas * 81c9de560dSAlex Tomas * The important thing to be noted in case of inode prealloc space is that 82c9de560dSAlex Tomas * we don't modify the values associated to inode prealloc space except 83c9de560dSAlex Tomas * pa_free. 84c9de560dSAlex Tomas * 85c9de560dSAlex Tomas * If we are not able to find blocks in the inode prealloc space and if we 86c9de560dSAlex Tomas * have the group allocation flag set then we look at the locality group 87caaf7a29STao Ma * prealloc space. These are per CPU prealloc list represented as 88c9de560dSAlex Tomas * 89c9de560dSAlex Tomas * ext4_sb_info.s_locality_groups[smp_processor_id()] 90c9de560dSAlex Tomas * 91c9de560dSAlex Tomas * The reason for having a per cpu locality group is to reduce the contention 92c9de560dSAlex Tomas * between CPUs. It is possible to get scheduled at this point. 93c9de560dSAlex Tomas * 94c9de560dSAlex Tomas * The locality group prealloc space is used looking at whether we have 9525985edcSLucas De Marchi * enough free space (pa_free) within the prealloc space. 96c9de560dSAlex Tomas * 97c9de560dSAlex Tomas * If we can't allocate blocks via inode prealloc or/and locality group 98c9de560dSAlex Tomas * prealloc then we look at the buddy cache. The buddy cache is represented 99c9de560dSAlex Tomas * by ext4_sb_info.s_buddy_cache (struct inode) whose file offset gets 100c9de560dSAlex Tomas * mapped to the buddy and bitmap information regarding different 101c9de560dSAlex Tomas * groups. The buddy information is attached to buddy cache inode so that 102c9de560dSAlex Tomas * we can access them through the page cache. The information regarding 103c9de560dSAlex Tomas * each group is loaded via ext4_mb_load_buddy. The information involve 104c9de560dSAlex Tomas * block bitmap and buddy information. The information are stored in the 105c9de560dSAlex Tomas * inode as: 106c9de560dSAlex Tomas * 107c9de560dSAlex Tomas * { page } 108c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 109c9de560dSAlex Tomas * 110c9de560dSAlex Tomas * 111c9de560dSAlex Tomas * one block each for bitmap and buddy information. So for each group we 112c9de560dSAlex Tomas * take up 2 blocks. A page can contain blocks_per_page (PAGE_CACHE_SIZE / 113c9de560dSAlex Tomas * blocksize) blocks. So it can have information regarding groups_per_page 114c9de560dSAlex Tomas * which is blocks_per_page/2 115c9de560dSAlex Tomas * 116c9de560dSAlex Tomas * The buddy cache inode is not stored on disk. The inode is thrown 117c9de560dSAlex Tomas * away when the filesystem is unmounted. 118c9de560dSAlex Tomas * 119c9de560dSAlex Tomas * We look for count number of blocks in the buddy cache. If we were able 120c9de560dSAlex Tomas * to locate that many free blocks we return with additional information 121c9de560dSAlex Tomas * regarding rest of the contiguous physical block available 122c9de560dSAlex Tomas * 123c9de560dSAlex Tomas * Before allocating blocks via buddy cache we normalize the request 124c9de560dSAlex Tomas * blocks. This ensure we ask for more blocks that we needed. The extra 125c9de560dSAlex Tomas * blocks that we get after allocation is added to the respective prealloc 126c9de560dSAlex Tomas * list. In case of inode preallocation we follow a list of heuristics 127c9de560dSAlex Tomas * based on file size. This can be found in ext4_mb_normalize_request. If 128c9de560dSAlex Tomas * we are doing a group prealloc we try to normalize the request to 129b713a5ecSTheodore Ts'o * sbi->s_mb_group_prealloc. Default value of s_mb_group_prealloc is 130c9de560dSAlex Tomas * 512 blocks. This can be tuned via 131d7a1fee1SDan Ehrenberg * /sys/fs/ext4/<partition>/mb_group_prealloc. The value is represented in 132c9de560dSAlex Tomas * terms of number of blocks. If we have mounted the file system with -O 133c9de560dSAlex Tomas * stripe=<value> option the group prealloc request is normalized to the 134d7a1fee1SDan Ehrenberg * the smallest multiple of the stripe value (sbi->s_stripe) which is 135d7a1fee1SDan Ehrenberg * greater than the default mb_group_prealloc. 136c9de560dSAlex Tomas * 137d7a1fee1SDan Ehrenberg * The regular allocator (using the buddy cache) supports a few tunables. 138c9de560dSAlex Tomas * 139b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_min_to_scan 140b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_max_to_scan 141b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req 142c9de560dSAlex Tomas * 143b713a5ecSTheodore Ts'o * The regular allocator uses buddy scan only if the request len is power of 144c9de560dSAlex Tomas * 2 blocks and the order of allocation is >= sbi->s_mb_order2_reqs. The 145c9de560dSAlex Tomas * value of s_mb_order2_reqs can be tuned via 146b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_order2_req. If the request len is equal to 147af901ca1SAndré Goddard Rosa * stripe size (sbi->s_stripe), we try to search for contiguous block in 148b713a5ecSTheodore Ts'o * stripe size. This should result in better allocation on RAID setups. If 149b713a5ecSTheodore Ts'o * not, we search in the specific group using bitmap for best extents. The 150b713a5ecSTheodore Ts'o * tunable min_to_scan and max_to_scan control the behaviour here. 151c9de560dSAlex Tomas * min_to_scan indicate how long the mballoc __must__ look for a best 152b713a5ecSTheodore Ts'o * extent and max_to_scan indicates how long the mballoc __can__ look for a 153c9de560dSAlex Tomas * best extent in the found extents. Searching for the blocks starts with 154c9de560dSAlex Tomas * the group specified as the goal value in allocation context via 155c9de560dSAlex Tomas * ac_g_ex. Each group is first checked based on the criteria whether it 156caaf7a29STao Ma * can be used for allocation. ext4_mb_good_group explains how the groups are 157c9de560dSAlex Tomas * checked. 158c9de560dSAlex Tomas * 159c9de560dSAlex Tomas * Both the prealloc space are getting populated as above. So for the first 160c9de560dSAlex Tomas * request we will hit the buddy cache which will result in this prealloc 161c9de560dSAlex Tomas * space getting filled. The prealloc space is then later used for the 162c9de560dSAlex Tomas * subsequent request. 163c9de560dSAlex Tomas */ 164c9de560dSAlex Tomas 165c9de560dSAlex Tomas /* 166c9de560dSAlex Tomas * mballoc operates on the following data: 167c9de560dSAlex Tomas * - on-disk bitmap 168c9de560dSAlex Tomas * - in-core buddy (actually includes buddy and bitmap) 169c9de560dSAlex Tomas * - preallocation descriptors (PAs) 170c9de560dSAlex Tomas * 171c9de560dSAlex Tomas * there are two types of preallocations: 172c9de560dSAlex Tomas * - inode 173c9de560dSAlex Tomas * assiged to specific inode and can be used for this inode only. 174c9de560dSAlex Tomas * it describes part of inode's space preallocated to specific 175c9de560dSAlex Tomas * physical blocks. any block from that preallocated can be used 176c9de560dSAlex Tomas * independent. the descriptor just tracks number of blocks left 177c9de560dSAlex Tomas * unused. so, before taking some block from descriptor, one must 178c9de560dSAlex Tomas * make sure corresponded logical block isn't allocated yet. this 179c9de560dSAlex Tomas * also means that freeing any block within descriptor's range 180c9de560dSAlex Tomas * must discard all preallocated blocks. 181c9de560dSAlex Tomas * - locality group 182c9de560dSAlex Tomas * assigned to specific locality group which does not translate to 183c9de560dSAlex Tomas * permanent set of inodes: inode can join and leave group. space 184c9de560dSAlex Tomas * from this type of preallocation can be used for any inode. thus 185c9de560dSAlex Tomas * it's consumed from the beginning to the end. 186c9de560dSAlex Tomas * 187c9de560dSAlex Tomas * relation between them can be expressed as: 188c9de560dSAlex Tomas * in-core buddy = on-disk bitmap + preallocation descriptors 189c9de560dSAlex Tomas * 190c9de560dSAlex Tomas * this mean blocks mballoc considers used are: 191c9de560dSAlex Tomas * - allocated blocks (persistent) 192c9de560dSAlex Tomas * - preallocated blocks (non-persistent) 193c9de560dSAlex Tomas * 194c9de560dSAlex Tomas * consistency in mballoc world means that at any time a block is either 195c9de560dSAlex Tomas * free or used in ALL structures. notice: "any time" should not be read 196c9de560dSAlex Tomas * literally -- time is discrete and delimited by locks. 197c9de560dSAlex Tomas * 198c9de560dSAlex Tomas * to keep it simple, we don't use block numbers, instead we count number of 199c9de560dSAlex Tomas * blocks: how many blocks marked used/free in on-disk bitmap, buddy and PA. 200c9de560dSAlex Tomas * 201c9de560dSAlex Tomas * all operations can be expressed as: 202c9de560dSAlex Tomas * - init buddy: buddy = on-disk + PAs 203c9de560dSAlex Tomas * - new PA: buddy += N; PA = N 204c9de560dSAlex Tomas * - use inode PA: on-disk += N; PA -= N 205c9de560dSAlex Tomas * - discard inode PA buddy -= on-disk - PA; PA = 0 206c9de560dSAlex Tomas * - use locality group PA on-disk += N; PA -= N 207c9de560dSAlex Tomas * - discard locality group PA buddy -= PA; PA = 0 208c9de560dSAlex Tomas * note: 'buddy -= on-disk - PA' is used to show that on-disk bitmap 209c9de560dSAlex Tomas * is used in real operation because we can't know actual used 210c9de560dSAlex Tomas * bits from PA, only from on-disk bitmap 211c9de560dSAlex Tomas * 212c9de560dSAlex Tomas * if we follow this strict logic, then all operations above should be atomic. 213c9de560dSAlex Tomas * given some of them can block, we'd have to use something like semaphores 214c9de560dSAlex Tomas * killing performance on high-end SMP hardware. let's try to relax it using 215c9de560dSAlex Tomas * the following knowledge: 216c9de560dSAlex Tomas * 1) if buddy is referenced, it's already initialized 217c9de560dSAlex Tomas * 2) while block is used in buddy and the buddy is referenced, 218c9de560dSAlex Tomas * nobody can re-allocate that block 219c9de560dSAlex Tomas * 3) we work on bitmaps and '+' actually means 'set bits'. if on-disk has 220c9de560dSAlex Tomas * bit set and PA claims same block, it's OK. IOW, one can set bit in 221c9de560dSAlex Tomas * on-disk bitmap if buddy has same bit set or/and PA covers corresponded 222c9de560dSAlex Tomas * block 223c9de560dSAlex Tomas * 224c9de560dSAlex Tomas * so, now we're building a concurrency table: 225c9de560dSAlex Tomas * - init buddy vs. 226c9de560dSAlex Tomas * - new PA 227c9de560dSAlex Tomas * blocks for PA are allocated in the buddy, buddy must be referenced 228c9de560dSAlex Tomas * until PA is linked to allocation group to avoid concurrent buddy init 229c9de560dSAlex Tomas * - use inode PA 230c9de560dSAlex Tomas * we need to make sure that either on-disk bitmap or PA has uptodate data 231c9de560dSAlex Tomas * given (3) we care that PA-=N operation doesn't interfere with init 232c9de560dSAlex Tomas * - discard inode PA 233c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 234c9de560dSAlex Tomas * - use locality group PA 235c9de560dSAlex Tomas * again PA-=N must be serialized with init 236c9de560dSAlex Tomas * - discard locality group PA 237c9de560dSAlex Tomas * the simplest way would be to have buddy initialized by the discard 238c9de560dSAlex Tomas * - new PA vs. 239c9de560dSAlex Tomas * - use inode PA 240c9de560dSAlex Tomas * i_data_sem serializes them 241c9de560dSAlex Tomas * - discard inode PA 242c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 243c9de560dSAlex Tomas * - use locality group PA 244c9de560dSAlex Tomas * some mutex should serialize them 245c9de560dSAlex Tomas * - discard locality group PA 246c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 247c9de560dSAlex Tomas * - use inode PA 248c9de560dSAlex Tomas * - use inode PA 249c9de560dSAlex Tomas * i_data_sem or another mutex should serializes them 250c9de560dSAlex Tomas * - discard inode PA 251c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 252c9de560dSAlex Tomas * - use locality group PA 253c9de560dSAlex Tomas * nothing wrong here -- they're different PAs covering different blocks 254c9de560dSAlex Tomas * - discard locality group PA 255c9de560dSAlex Tomas * discard process must wait until PA isn't used by another process 256c9de560dSAlex Tomas * 257c9de560dSAlex Tomas * now we're ready to make few consequences: 258c9de560dSAlex Tomas * - PA is referenced and while it is no discard is possible 259c9de560dSAlex Tomas * - PA is referenced until block isn't marked in on-disk bitmap 260c9de560dSAlex Tomas * - PA changes only after on-disk bitmap 261c9de560dSAlex Tomas * - discard must not compete with init. either init is done before 262c9de560dSAlex Tomas * any discard or they're serialized somehow 263c9de560dSAlex Tomas * - buddy init as sum of on-disk bitmap and PAs is done atomically 264c9de560dSAlex Tomas * 265c9de560dSAlex Tomas * a special case when we've used PA to emptiness. no need to modify buddy 266c9de560dSAlex Tomas * in this case, but we should care about concurrent init 267c9de560dSAlex Tomas * 268c9de560dSAlex Tomas */ 269c9de560dSAlex Tomas 270c9de560dSAlex Tomas /* 271c9de560dSAlex Tomas * Logic in few words: 272c9de560dSAlex Tomas * 273c9de560dSAlex Tomas * - allocation: 274c9de560dSAlex Tomas * load group 275c9de560dSAlex Tomas * find blocks 276c9de560dSAlex Tomas * mark bits in on-disk bitmap 277c9de560dSAlex Tomas * release group 278c9de560dSAlex Tomas * 279c9de560dSAlex Tomas * - use preallocation: 280c9de560dSAlex Tomas * find proper PA (per-inode or group) 281c9de560dSAlex Tomas * load group 282c9de560dSAlex Tomas * mark bits in on-disk bitmap 283c9de560dSAlex Tomas * release group 284c9de560dSAlex Tomas * release PA 285c9de560dSAlex Tomas * 286c9de560dSAlex Tomas * - free: 287c9de560dSAlex Tomas * load group 288c9de560dSAlex Tomas * mark bits in on-disk bitmap 289c9de560dSAlex Tomas * release group 290c9de560dSAlex Tomas * 291c9de560dSAlex Tomas * - discard preallocations in group: 292c9de560dSAlex Tomas * mark PAs deleted 293c9de560dSAlex Tomas * move them onto local list 294c9de560dSAlex Tomas * load on-disk bitmap 295c9de560dSAlex Tomas * load group 296c9de560dSAlex Tomas * remove PA from object (inode or locality group) 297c9de560dSAlex Tomas * mark free blocks in-core 298c9de560dSAlex Tomas * 299c9de560dSAlex Tomas * - discard inode's preallocations: 300c9de560dSAlex Tomas */ 301c9de560dSAlex Tomas 302c9de560dSAlex Tomas /* 303c9de560dSAlex Tomas * Locking rules 304c9de560dSAlex Tomas * 305c9de560dSAlex Tomas * Locks: 306c9de560dSAlex Tomas * - bitlock on a group (group) 307c9de560dSAlex Tomas * - object (inode/locality) (object) 308c9de560dSAlex Tomas * - per-pa lock (pa) 309c9de560dSAlex Tomas * 310c9de560dSAlex Tomas * Paths: 311c9de560dSAlex Tomas * - new pa 312c9de560dSAlex Tomas * object 313c9de560dSAlex Tomas * group 314c9de560dSAlex Tomas * 315c9de560dSAlex Tomas * - find and use pa: 316c9de560dSAlex Tomas * pa 317c9de560dSAlex Tomas * 318c9de560dSAlex Tomas * - release consumed pa: 319c9de560dSAlex Tomas * pa 320c9de560dSAlex Tomas * group 321c9de560dSAlex Tomas * object 322c9de560dSAlex Tomas * 323c9de560dSAlex Tomas * - generate in-core bitmap: 324c9de560dSAlex Tomas * group 325c9de560dSAlex Tomas * pa 326c9de560dSAlex Tomas * 327c9de560dSAlex Tomas * - discard all for given object (inode, locality group): 328c9de560dSAlex Tomas * object 329c9de560dSAlex Tomas * pa 330c9de560dSAlex Tomas * group 331c9de560dSAlex Tomas * 332c9de560dSAlex Tomas * - discard all for given group: 333c9de560dSAlex Tomas * group 334c9de560dSAlex Tomas * pa 335c9de560dSAlex Tomas * group 336c9de560dSAlex Tomas * object 337c9de560dSAlex Tomas * 338c9de560dSAlex Tomas */ 339c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_pspace_cachep; 340c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_ac_cachep; 341c3a326a6SAneesh Kumar K.V static struct kmem_cache *ext4_free_ext_cachep; 342fb1813f4SCurt Wohlgemuth 343fb1813f4SCurt Wohlgemuth /* We create slab caches for groupinfo data structures based on the 344fb1813f4SCurt Wohlgemuth * superblock block size. There will be one per mounted filesystem for 345fb1813f4SCurt Wohlgemuth * each unique s_blocksize_bits */ 3462892c15dSEric Sandeen #define NR_GRPINFO_CACHES 8 347fb1813f4SCurt Wohlgemuth static struct kmem_cache *ext4_groupinfo_caches[NR_GRPINFO_CACHES]; 348fb1813f4SCurt Wohlgemuth 3492892c15dSEric Sandeen static const char *ext4_groupinfo_slab_names[NR_GRPINFO_CACHES] = { 3502892c15dSEric Sandeen "ext4_groupinfo_1k", "ext4_groupinfo_2k", "ext4_groupinfo_4k", 3512892c15dSEric Sandeen "ext4_groupinfo_8k", "ext4_groupinfo_16k", "ext4_groupinfo_32k", 3522892c15dSEric Sandeen "ext4_groupinfo_64k", "ext4_groupinfo_128k" 3532892c15dSEric Sandeen }; 3542892c15dSEric Sandeen 355c3a326a6SAneesh Kumar K.V static void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 356c3a326a6SAneesh Kumar K.V ext4_group_t group); 3577a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 3587a2fcbf7SAneesh Kumar K.V ext4_group_t group); 359c3a326a6SAneesh Kumar K.V static void release_blocks_on_commit(journal_t *journal, transaction_t *txn); 360c3a326a6SAneesh Kumar K.V 361ffad0a44SAneesh Kumar K.V static inline void *mb_correct_addr_and_bit(int *bit, void *addr) 362ffad0a44SAneesh Kumar K.V { 363c9de560dSAlex Tomas #if BITS_PER_LONG == 64 364ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 7UL) << 3; 365ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~7UL); 366c9de560dSAlex Tomas #elif BITS_PER_LONG == 32 367ffad0a44SAneesh Kumar K.V *bit += ((unsigned long) addr & 3UL) << 3; 368ffad0a44SAneesh Kumar K.V addr = (void *) ((unsigned long) addr & ~3UL); 369c9de560dSAlex Tomas #else 370c9de560dSAlex Tomas #error "how many bits you are?!" 371c9de560dSAlex Tomas #endif 372ffad0a44SAneesh Kumar K.V return addr; 373ffad0a44SAneesh Kumar K.V } 374c9de560dSAlex Tomas 375c9de560dSAlex Tomas static inline int mb_test_bit(int bit, void *addr) 376c9de560dSAlex Tomas { 377c9de560dSAlex Tomas /* 378c9de560dSAlex Tomas * ext4_test_bit on architecture like powerpc 379c9de560dSAlex Tomas * needs unsigned long aligned address 380c9de560dSAlex Tomas */ 381ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 382c9de560dSAlex Tomas return ext4_test_bit(bit, addr); 383c9de560dSAlex Tomas } 384c9de560dSAlex Tomas 385c9de560dSAlex Tomas static inline void mb_set_bit(int bit, void *addr) 386c9de560dSAlex Tomas { 387ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 388c9de560dSAlex Tomas ext4_set_bit(bit, addr); 389c9de560dSAlex Tomas } 390c9de560dSAlex Tomas 391c9de560dSAlex Tomas static inline void mb_clear_bit(int bit, void *addr) 392c9de560dSAlex Tomas { 393ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&bit, addr); 394c9de560dSAlex Tomas ext4_clear_bit(bit, addr); 395c9de560dSAlex Tomas } 396c9de560dSAlex Tomas 397ffad0a44SAneesh Kumar K.V static inline int mb_find_next_zero_bit(void *addr, int max, int start) 398ffad0a44SAneesh Kumar K.V { 399e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 400ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 401e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 402ffad0a44SAneesh Kumar K.V start += fix; 403ffad0a44SAneesh Kumar K.V 404e7dfb246SAneesh Kumar K.V ret = ext4_find_next_zero_bit(addr, tmpmax, start) - fix; 405e7dfb246SAneesh Kumar K.V if (ret > max) 406e7dfb246SAneesh Kumar K.V return max; 407e7dfb246SAneesh Kumar K.V return ret; 408ffad0a44SAneesh Kumar K.V } 409ffad0a44SAneesh Kumar K.V 410ffad0a44SAneesh Kumar K.V static inline int mb_find_next_bit(void *addr, int max, int start) 411ffad0a44SAneesh Kumar K.V { 412e7dfb246SAneesh Kumar K.V int fix = 0, ret, tmpmax; 413ffad0a44SAneesh Kumar K.V addr = mb_correct_addr_and_bit(&fix, addr); 414e7dfb246SAneesh Kumar K.V tmpmax = max + fix; 415ffad0a44SAneesh Kumar K.V start += fix; 416ffad0a44SAneesh Kumar K.V 417e7dfb246SAneesh Kumar K.V ret = ext4_find_next_bit(addr, tmpmax, start) - fix; 418e7dfb246SAneesh Kumar K.V if (ret > max) 419e7dfb246SAneesh Kumar K.V return max; 420e7dfb246SAneesh Kumar K.V return ret; 421ffad0a44SAneesh Kumar K.V } 422ffad0a44SAneesh Kumar K.V 423c9de560dSAlex Tomas static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max) 424c9de560dSAlex Tomas { 425c9de560dSAlex Tomas char *bb; 426c9de560dSAlex Tomas 427c9de560dSAlex Tomas BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b)); 428c9de560dSAlex Tomas BUG_ON(max == NULL); 429c9de560dSAlex Tomas 430c9de560dSAlex Tomas if (order > e4b->bd_blkbits + 1) { 431c9de560dSAlex Tomas *max = 0; 432c9de560dSAlex Tomas return NULL; 433c9de560dSAlex Tomas } 434c9de560dSAlex Tomas 435c9de560dSAlex Tomas /* at order 0 we see each particular block */ 43684b775a3SColy Li if (order == 0) { 437c9de560dSAlex Tomas *max = 1 << (e4b->bd_blkbits + 3); 438c9de560dSAlex Tomas return EXT4_MB_BITMAP(e4b); 43984b775a3SColy Li } 440c9de560dSAlex Tomas 441c9de560dSAlex Tomas bb = EXT4_MB_BUDDY(e4b) + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order]; 442c9de560dSAlex Tomas *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order]; 443c9de560dSAlex Tomas 444c9de560dSAlex Tomas return bb; 445c9de560dSAlex Tomas } 446c9de560dSAlex Tomas 447c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 448c9de560dSAlex Tomas static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b, 449c9de560dSAlex Tomas int first, int count) 450c9de560dSAlex Tomas { 451c9de560dSAlex Tomas int i; 452c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 453c9de560dSAlex Tomas 454c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 455c9de560dSAlex Tomas return; 456bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 457c9de560dSAlex Tomas for (i = 0; i < count; i++) { 458c9de560dSAlex Tomas if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) { 459c9de560dSAlex Tomas ext4_fsblk_t blocknr; 4605661bd68SAkinobu Mita 4615661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 462*53accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), first + i); 4635d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 464e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 465e29136f8STheodore Ts'o blocknr, 466e29136f8STheodore Ts'o "freeing block already freed " 467e29136f8STheodore Ts'o "(bit %u)", 468e29136f8STheodore Ts'o first + i); 469c9de560dSAlex Tomas } 470c9de560dSAlex Tomas mb_clear_bit(first + i, e4b->bd_info->bb_bitmap); 471c9de560dSAlex Tomas } 472c9de560dSAlex Tomas } 473c9de560dSAlex Tomas 474c9de560dSAlex Tomas static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count) 475c9de560dSAlex Tomas { 476c9de560dSAlex Tomas int i; 477c9de560dSAlex Tomas 478c9de560dSAlex Tomas if (unlikely(e4b->bd_info->bb_bitmap == NULL)) 479c9de560dSAlex Tomas return; 480bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 481c9de560dSAlex Tomas for (i = 0; i < count; i++) { 482c9de560dSAlex Tomas BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap)); 483c9de560dSAlex Tomas mb_set_bit(first + i, e4b->bd_info->bb_bitmap); 484c9de560dSAlex Tomas } 485c9de560dSAlex Tomas } 486c9de560dSAlex Tomas 487c9de560dSAlex Tomas static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 488c9de560dSAlex Tomas { 489c9de560dSAlex Tomas if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) { 490c9de560dSAlex Tomas unsigned char *b1, *b2; 491c9de560dSAlex Tomas int i; 492c9de560dSAlex Tomas b1 = (unsigned char *) e4b->bd_info->bb_bitmap; 493c9de560dSAlex Tomas b2 = (unsigned char *) bitmap; 494c9de560dSAlex Tomas for (i = 0; i < e4b->bd_sb->s_blocksize; i++) { 495c9de560dSAlex Tomas if (b1[i] != b2[i]) { 4969d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_ERR, 4979d8b9ec4STheodore Ts'o "corruption in group %u " 4984776004fSTheodore Ts'o "at byte %u(%u): %x in copy != %x " 4999d8b9ec4STheodore Ts'o "on disk/prealloc", 500c9de560dSAlex Tomas e4b->bd_group, i, i * 8, b1[i], b2[i]); 501c9de560dSAlex Tomas BUG(); 502c9de560dSAlex Tomas } 503c9de560dSAlex Tomas } 504c9de560dSAlex Tomas } 505c9de560dSAlex Tomas } 506c9de560dSAlex Tomas 507c9de560dSAlex Tomas #else 508c9de560dSAlex Tomas static inline void mb_free_blocks_double(struct inode *inode, 509c9de560dSAlex Tomas struct ext4_buddy *e4b, int first, int count) 510c9de560dSAlex Tomas { 511c9de560dSAlex Tomas return; 512c9de560dSAlex Tomas } 513c9de560dSAlex Tomas static inline void mb_mark_used_double(struct ext4_buddy *e4b, 514c9de560dSAlex Tomas int first, int count) 515c9de560dSAlex Tomas { 516c9de560dSAlex Tomas return; 517c9de560dSAlex Tomas } 518c9de560dSAlex Tomas static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap) 519c9de560dSAlex Tomas { 520c9de560dSAlex Tomas return; 521c9de560dSAlex Tomas } 522c9de560dSAlex Tomas #endif 523c9de560dSAlex Tomas 524c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 525c9de560dSAlex Tomas 526c9de560dSAlex Tomas #define MB_CHECK_ASSERT(assert) \ 527c9de560dSAlex Tomas do { \ 528c9de560dSAlex Tomas if (!(assert)) { \ 529c9de560dSAlex Tomas printk(KERN_EMERG \ 530c9de560dSAlex Tomas "Assertion failure in %s() at %s:%d: \"%s\"\n", \ 531c9de560dSAlex Tomas function, file, line, # assert); \ 532c9de560dSAlex Tomas BUG(); \ 533c9de560dSAlex Tomas } \ 534c9de560dSAlex Tomas } while (0) 535c9de560dSAlex Tomas 536c9de560dSAlex Tomas static int __mb_check_buddy(struct ext4_buddy *e4b, char *file, 537c9de560dSAlex Tomas const char *function, int line) 538c9de560dSAlex Tomas { 539c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 540c9de560dSAlex Tomas int order = e4b->bd_blkbits + 1; 541c9de560dSAlex Tomas int max; 542c9de560dSAlex Tomas int max2; 543c9de560dSAlex Tomas int i; 544c9de560dSAlex Tomas int j; 545c9de560dSAlex Tomas int k; 546c9de560dSAlex Tomas int count; 547c9de560dSAlex Tomas struct ext4_group_info *grp; 548c9de560dSAlex Tomas int fragments = 0; 549c9de560dSAlex Tomas int fstart; 550c9de560dSAlex Tomas struct list_head *cur; 551c9de560dSAlex Tomas void *buddy; 552c9de560dSAlex Tomas void *buddy2; 553c9de560dSAlex Tomas 554c9de560dSAlex Tomas { 555c9de560dSAlex Tomas static int mb_check_counter; 556c9de560dSAlex Tomas if (mb_check_counter++ % 100 != 0) 557c9de560dSAlex Tomas return 0; 558c9de560dSAlex Tomas } 559c9de560dSAlex Tomas 560c9de560dSAlex Tomas while (order > 1) { 561c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 562c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy); 563c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order - 1, &max2); 564c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy2); 565c9de560dSAlex Tomas MB_CHECK_ASSERT(buddy != buddy2); 566c9de560dSAlex Tomas MB_CHECK_ASSERT(max * 2 == max2); 567c9de560dSAlex Tomas 568c9de560dSAlex Tomas count = 0; 569c9de560dSAlex Tomas for (i = 0; i < max; i++) { 570c9de560dSAlex Tomas 571c9de560dSAlex Tomas if (mb_test_bit(i, buddy)) { 572c9de560dSAlex Tomas /* only single bit in buddy2 may be 1 */ 573c9de560dSAlex Tomas if (!mb_test_bit(i << 1, buddy2)) { 574c9de560dSAlex Tomas MB_CHECK_ASSERT( 575c9de560dSAlex Tomas mb_test_bit((i<<1)+1, buddy2)); 576c9de560dSAlex Tomas } else if (!mb_test_bit((i << 1) + 1, buddy2)) { 577c9de560dSAlex Tomas MB_CHECK_ASSERT( 578c9de560dSAlex Tomas mb_test_bit(i << 1, buddy2)); 579c9de560dSAlex Tomas } 580c9de560dSAlex Tomas continue; 581c9de560dSAlex Tomas } 582c9de560dSAlex Tomas 583c9de560dSAlex Tomas /* both bits in buddy2 must be 0 */ 584c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(i << 1, buddy2)); 585c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit((i << 1) + 1, buddy2)); 586c9de560dSAlex Tomas 587c9de560dSAlex Tomas for (j = 0; j < (1 << order); j++) { 588c9de560dSAlex Tomas k = (i * (1 << order)) + j; 589c9de560dSAlex Tomas MB_CHECK_ASSERT( 590c9de560dSAlex Tomas !mb_test_bit(k, EXT4_MB_BITMAP(e4b))); 591c9de560dSAlex Tomas } 592c9de560dSAlex Tomas count++; 593c9de560dSAlex Tomas } 594c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count); 595c9de560dSAlex Tomas order--; 596c9de560dSAlex Tomas } 597c9de560dSAlex Tomas 598c9de560dSAlex Tomas fstart = -1; 599c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, 0, &max); 600c9de560dSAlex Tomas for (i = 0; i < max; i++) { 601c9de560dSAlex Tomas if (!mb_test_bit(i, buddy)) { 602c9de560dSAlex Tomas MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free); 603c9de560dSAlex Tomas if (fstart == -1) { 604c9de560dSAlex Tomas fragments++; 605c9de560dSAlex Tomas fstart = i; 606c9de560dSAlex Tomas } 607c9de560dSAlex Tomas continue; 608c9de560dSAlex Tomas } 609c9de560dSAlex Tomas fstart = -1; 610c9de560dSAlex Tomas /* check used bits only */ 611c9de560dSAlex Tomas for (j = 0; j < e4b->bd_blkbits + 1; j++) { 612c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, j, &max2); 613c9de560dSAlex Tomas k = i >> j; 614c9de560dSAlex Tomas MB_CHECK_ASSERT(k < max2); 615c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k, buddy2)); 616c9de560dSAlex Tomas } 617c9de560dSAlex Tomas } 618c9de560dSAlex Tomas MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info)); 619c9de560dSAlex Tomas MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments); 620c9de560dSAlex Tomas 621c9de560dSAlex Tomas grp = ext4_get_group_info(sb, e4b->bd_group); 622c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 623c9de560dSAlex Tomas ext4_group_t groupnr; 624c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 62560bd63d1SSolofo Ramangalahy pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 62660bd63d1SSolofo Ramangalahy ext4_get_group_no_and_offset(sb, pa->pa_pstart, &groupnr, &k); 627c9de560dSAlex Tomas MB_CHECK_ASSERT(groupnr == e4b->bd_group); 62860bd63d1SSolofo Ramangalahy for (i = 0; i < pa->pa_len; i++) 629c9de560dSAlex Tomas MB_CHECK_ASSERT(mb_test_bit(k + i, buddy)); 630c9de560dSAlex Tomas } 631c9de560dSAlex Tomas return 0; 632c9de560dSAlex Tomas } 633c9de560dSAlex Tomas #undef MB_CHECK_ASSERT 634c9de560dSAlex Tomas #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \ 63546e665e9SHarvey Harrison __FILE__, __func__, __LINE__) 636c9de560dSAlex Tomas #else 637c9de560dSAlex Tomas #define mb_check_buddy(e4b) 638c9de560dSAlex Tomas #endif 639c9de560dSAlex Tomas 6407c786059SColy Li /* 6417c786059SColy Li * Divide blocks started from @first with length @len into 6427c786059SColy Li * smaller chunks with power of 2 blocks. 6437c786059SColy Li * Clear the bits in bitmap which the blocks of the chunk(s) covered, 6447c786059SColy Li * then increase bb_counters[] for corresponded chunk size. 6457c786059SColy Li */ 646c9de560dSAlex Tomas static void ext4_mb_mark_free_simple(struct super_block *sb, 647a36b4498SEric Sandeen void *buddy, ext4_grpblk_t first, ext4_grpblk_t len, 648c9de560dSAlex Tomas struct ext4_group_info *grp) 649c9de560dSAlex Tomas { 650c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 651a36b4498SEric Sandeen ext4_grpblk_t min; 652a36b4498SEric Sandeen ext4_grpblk_t max; 653a36b4498SEric Sandeen ext4_grpblk_t chunk; 654c9de560dSAlex Tomas unsigned short border; 655c9de560dSAlex Tomas 6567137d7a4STheodore Ts'o BUG_ON(len > EXT4_CLUSTERS_PER_GROUP(sb)); 657c9de560dSAlex Tomas 658c9de560dSAlex Tomas border = 2 << sb->s_blocksize_bits; 659c9de560dSAlex Tomas 660c9de560dSAlex Tomas while (len > 0) { 661c9de560dSAlex Tomas /* find how many blocks can be covered since this position */ 662c9de560dSAlex Tomas max = ffs(first | border) - 1; 663c9de560dSAlex Tomas 664c9de560dSAlex Tomas /* find how many blocks of power 2 we need to mark */ 665c9de560dSAlex Tomas min = fls(len) - 1; 666c9de560dSAlex Tomas 667c9de560dSAlex Tomas if (max < min) 668c9de560dSAlex Tomas min = max; 669c9de560dSAlex Tomas chunk = 1 << min; 670c9de560dSAlex Tomas 671c9de560dSAlex Tomas /* mark multiblock chunks only */ 672c9de560dSAlex Tomas grp->bb_counters[min]++; 673c9de560dSAlex Tomas if (min > 0) 674c9de560dSAlex Tomas mb_clear_bit(first >> min, 675c9de560dSAlex Tomas buddy + sbi->s_mb_offsets[min]); 676c9de560dSAlex Tomas 677c9de560dSAlex Tomas len -= chunk; 678c9de560dSAlex Tomas first += chunk; 679c9de560dSAlex Tomas } 680c9de560dSAlex Tomas } 681c9de560dSAlex Tomas 6828a57d9d6SCurt Wohlgemuth /* 6838a57d9d6SCurt Wohlgemuth * Cache the order of the largest free extent we have available in this block 6848a57d9d6SCurt Wohlgemuth * group. 6858a57d9d6SCurt Wohlgemuth */ 6868a57d9d6SCurt Wohlgemuth static void 6878a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(struct super_block *sb, struct ext4_group_info *grp) 6888a57d9d6SCurt Wohlgemuth { 6898a57d9d6SCurt Wohlgemuth int i; 6908a57d9d6SCurt Wohlgemuth int bits; 6918a57d9d6SCurt Wohlgemuth 6928a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = -1; /* uninit */ 6938a57d9d6SCurt Wohlgemuth 6948a57d9d6SCurt Wohlgemuth bits = sb->s_blocksize_bits + 1; 6958a57d9d6SCurt Wohlgemuth for (i = bits; i >= 0; i--) { 6968a57d9d6SCurt Wohlgemuth if (grp->bb_counters[i] > 0) { 6978a57d9d6SCurt Wohlgemuth grp->bb_largest_free_order = i; 6988a57d9d6SCurt Wohlgemuth break; 6998a57d9d6SCurt Wohlgemuth } 7008a57d9d6SCurt Wohlgemuth } 7018a57d9d6SCurt Wohlgemuth } 7028a57d9d6SCurt Wohlgemuth 703089ceeccSEric Sandeen static noinline_for_stack 704089ceeccSEric Sandeen void ext4_mb_generate_buddy(struct super_block *sb, 705c9de560dSAlex Tomas void *buddy, void *bitmap, ext4_group_t group) 706c9de560dSAlex Tomas { 707c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 7087137d7a4STheodore Ts'o ext4_grpblk_t max = EXT4_CLUSTERS_PER_GROUP(sb); 709a36b4498SEric Sandeen ext4_grpblk_t i = 0; 710a36b4498SEric Sandeen ext4_grpblk_t first; 711a36b4498SEric Sandeen ext4_grpblk_t len; 712c9de560dSAlex Tomas unsigned free = 0; 713c9de560dSAlex Tomas unsigned fragments = 0; 714c9de560dSAlex Tomas unsigned long long period = get_cycles(); 715c9de560dSAlex Tomas 716c9de560dSAlex Tomas /* initialize buddy from bitmap which is aggregation 717c9de560dSAlex Tomas * of on-disk bitmap and preallocations */ 718ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, 0); 719c9de560dSAlex Tomas grp->bb_first_free = i; 720c9de560dSAlex Tomas while (i < max) { 721c9de560dSAlex Tomas fragments++; 722c9de560dSAlex Tomas first = i; 723ffad0a44SAneesh Kumar K.V i = mb_find_next_bit(bitmap, max, i); 724c9de560dSAlex Tomas len = i - first; 725c9de560dSAlex Tomas free += len; 726c9de560dSAlex Tomas if (len > 1) 727c9de560dSAlex Tomas ext4_mb_mark_free_simple(sb, buddy, first, len, grp); 728c9de560dSAlex Tomas else 729c9de560dSAlex Tomas grp->bb_counters[0]++; 730c9de560dSAlex Tomas if (i < max) 731ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, max, i); 732c9de560dSAlex Tomas } 733c9de560dSAlex Tomas grp->bb_fragments = fragments; 734c9de560dSAlex Tomas 735c9de560dSAlex Tomas if (free != grp->bb_free) { 736e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, 737*53accfa9STheodore Ts'o "%u clusters in bitmap, %u in gd", 738e29136f8STheodore Ts'o free, grp->bb_free); 739e56eb659SAneesh Kumar K.V /* 740e56eb659SAneesh Kumar K.V * If we intent to continue, we consider group descritor 741e56eb659SAneesh Kumar K.V * corrupt and update bb_free using bitmap value 742e56eb659SAneesh Kumar K.V */ 743c9de560dSAlex Tomas grp->bb_free = free; 744c9de560dSAlex Tomas } 7458a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, grp); 746c9de560dSAlex Tomas 747c9de560dSAlex Tomas clear_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, &(grp->bb_state)); 748c9de560dSAlex Tomas 749c9de560dSAlex Tomas period = get_cycles() - period; 750c9de560dSAlex Tomas spin_lock(&EXT4_SB(sb)->s_bal_lock); 751c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_buddies_generated++; 752c9de560dSAlex Tomas EXT4_SB(sb)->s_mb_generation_time += period; 753c9de560dSAlex Tomas spin_unlock(&EXT4_SB(sb)->s_bal_lock); 754c9de560dSAlex Tomas } 755c9de560dSAlex Tomas 756c9de560dSAlex Tomas /* The buddy information is attached the buddy cache inode 757c9de560dSAlex Tomas * for convenience. The information regarding each group 758c9de560dSAlex Tomas * is loaded via ext4_mb_load_buddy. The information involve 759c9de560dSAlex Tomas * block bitmap and buddy information. The information are 760c9de560dSAlex Tomas * stored in the inode as 761c9de560dSAlex Tomas * 762c9de560dSAlex Tomas * { page } 763c3a326a6SAneesh Kumar K.V * [ group 0 bitmap][ group 0 buddy] [group 1][ group 1]... 764c9de560dSAlex Tomas * 765c9de560dSAlex Tomas * 766c9de560dSAlex Tomas * one block each for bitmap and buddy information. 767c9de560dSAlex Tomas * So for each group we take up 2 blocks. A page can 768c9de560dSAlex Tomas * contain blocks_per_page (PAGE_CACHE_SIZE / blocksize) blocks. 769c9de560dSAlex Tomas * So it can have information regarding groups_per_page which 770c9de560dSAlex Tomas * is blocks_per_page/2 7718a57d9d6SCurt Wohlgemuth * 7728a57d9d6SCurt Wohlgemuth * Locking note: This routine takes the block group lock of all groups 7738a57d9d6SCurt Wohlgemuth * for this page; do not hold this lock when calling this routine! 774c9de560dSAlex Tomas */ 775c9de560dSAlex Tomas 776c9de560dSAlex Tomas static int ext4_mb_init_cache(struct page *page, char *incore) 777c9de560dSAlex Tomas { 7788df9675fSTheodore Ts'o ext4_group_t ngroups; 779c9de560dSAlex Tomas int blocksize; 780c9de560dSAlex Tomas int blocks_per_page; 781c9de560dSAlex Tomas int groups_per_page; 782c9de560dSAlex Tomas int err = 0; 783c9de560dSAlex Tomas int i; 784c9de560dSAlex Tomas ext4_group_t first_group; 785c9de560dSAlex Tomas int first_block; 786c9de560dSAlex Tomas struct super_block *sb; 787c9de560dSAlex Tomas struct buffer_head *bhs; 788c9de560dSAlex Tomas struct buffer_head **bh; 789c9de560dSAlex Tomas struct inode *inode; 790c9de560dSAlex Tomas char *data; 791c9de560dSAlex Tomas char *bitmap; 7929b8b7d35SAmir Goldstein struct ext4_group_info *grinfo; 793c9de560dSAlex Tomas 7946ba495e9STheodore Ts'o mb_debug(1, "init page %lu\n", page->index); 795c9de560dSAlex Tomas 796c9de560dSAlex Tomas inode = page->mapping->host; 797c9de560dSAlex Tomas sb = inode->i_sb; 7988df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 799c9de560dSAlex Tomas blocksize = 1 << inode->i_blkbits; 800c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / blocksize; 801c9de560dSAlex Tomas 802c9de560dSAlex Tomas groups_per_page = blocks_per_page >> 1; 803c9de560dSAlex Tomas if (groups_per_page == 0) 804c9de560dSAlex Tomas groups_per_page = 1; 805c9de560dSAlex Tomas 806c9de560dSAlex Tomas /* allocate buffer_heads to read bitmaps */ 807c9de560dSAlex Tomas if (groups_per_page > 1) { 808c9de560dSAlex Tomas err = -ENOMEM; 809c9de560dSAlex Tomas i = sizeof(struct buffer_head *) * groups_per_page; 810c9de560dSAlex Tomas bh = kzalloc(i, GFP_NOFS); 811c9de560dSAlex Tomas if (bh == NULL) 812c9de560dSAlex Tomas goto out; 813c9de560dSAlex Tomas } else 814c9de560dSAlex Tomas bh = &bhs; 815c9de560dSAlex Tomas 816c9de560dSAlex Tomas first_group = page->index * blocks_per_page / 2; 817c9de560dSAlex Tomas 818c9de560dSAlex Tomas /* read all groups the page covers into the cache */ 819c9de560dSAlex Tomas for (i = 0; i < groups_per_page; i++) { 820c9de560dSAlex Tomas struct ext4_group_desc *desc; 821c9de560dSAlex Tomas 8228df9675fSTheodore Ts'o if (first_group + i >= ngroups) 823c9de560dSAlex Tomas break; 824c9de560dSAlex Tomas 8259b8b7d35SAmir Goldstein grinfo = ext4_get_group_info(sb, first_group + i); 8269b8b7d35SAmir Goldstein /* 8279b8b7d35SAmir Goldstein * If page is uptodate then we came here after online resize 8289b8b7d35SAmir Goldstein * which added some new uninitialized group info structs, so 8299b8b7d35SAmir Goldstein * we must skip all initialized uptodate buddies on the page, 8309b8b7d35SAmir Goldstein * which may be currently in use by an allocating task. 8319b8b7d35SAmir Goldstein */ 8329b8b7d35SAmir Goldstein if (PageUptodate(page) && !EXT4_MB_GRP_NEED_INIT(grinfo)) { 8339b8b7d35SAmir Goldstein bh[i] = NULL; 8349b8b7d35SAmir Goldstein continue; 8359b8b7d35SAmir Goldstein } 8369b8b7d35SAmir Goldstein 837c9de560dSAlex Tomas err = -EIO; 838c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, first_group + i, NULL); 839c9de560dSAlex Tomas if (desc == NULL) 840c9de560dSAlex Tomas goto out; 841c9de560dSAlex Tomas 842c9de560dSAlex Tomas err = -ENOMEM; 843c9de560dSAlex Tomas bh[i] = sb_getblk(sb, ext4_block_bitmap(sb, desc)); 844c9de560dSAlex Tomas if (bh[i] == NULL) 845c9de560dSAlex Tomas goto out; 846c9de560dSAlex Tomas 8472ccb5fb9SAneesh Kumar K.V if (bitmap_uptodate(bh[i])) 848c9de560dSAlex Tomas continue; 849c9de560dSAlex Tomas 850c806e68fSFrederic Bohe lock_buffer(bh[i]); 8512ccb5fb9SAneesh Kumar K.V if (bitmap_uptodate(bh[i])) { 8522ccb5fb9SAneesh Kumar K.V unlock_buffer(bh[i]); 8532ccb5fb9SAneesh Kumar K.V continue; 8542ccb5fb9SAneesh Kumar K.V } 855955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, first_group + i); 856c9de560dSAlex Tomas if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 857c9de560dSAlex Tomas ext4_init_block_bitmap(sb, bh[i], 858c9de560dSAlex Tomas first_group + i, desc); 8592ccb5fb9SAneesh Kumar K.V set_bitmap_uptodate(bh[i]); 860c9de560dSAlex Tomas set_buffer_uptodate(bh[i]); 861955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, first_group + i); 8623300bedaSAneesh Kumar K.V unlock_buffer(bh[i]); 863c9de560dSAlex Tomas continue; 864c9de560dSAlex Tomas } 865955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, first_group + i); 8662ccb5fb9SAneesh Kumar K.V if (buffer_uptodate(bh[i])) { 8672ccb5fb9SAneesh Kumar K.V /* 8682ccb5fb9SAneesh Kumar K.V * if not uninit if bh is uptodate, 8692ccb5fb9SAneesh Kumar K.V * bitmap is also uptodate 8702ccb5fb9SAneesh Kumar K.V */ 8712ccb5fb9SAneesh Kumar K.V set_bitmap_uptodate(bh[i]); 8722ccb5fb9SAneesh Kumar K.V unlock_buffer(bh[i]); 8732ccb5fb9SAneesh Kumar K.V continue; 8742ccb5fb9SAneesh Kumar K.V } 875c9de560dSAlex Tomas get_bh(bh[i]); 8762ccb5fb9SAneesh Kumar K.V /* 8772ccb5fb9SAneesh Kumar K.V * submit the buffer_head for read. We can 8782ccb5fb9SAneesh Kumar K.V * safely mark the bitmap as uptodate now. 8792ccb5fb9SAneesh Kumar K.V * We do it here so the bitmap uptodate bit 8802ccb5fb9SAneesh Kumar K.V * get set with buffer lock held. 8812ccb5fb9SAneesh Kumar K.V */ 8822ccb5fb9SAneesh Kumar K.V set_bitmap_uptodate(bh[i]); 883c9de560dSAlex Tomas bh[i]->b_end_io = end_buffer_read_sync; 884c9de560dSAlex Tomas submit_bh(READ, bh[i]); 8856ba495e9STheodore Ts'o mb_debug(1, "read bitmap for group %u\n", first_group + i); 886c9de560dSAlex Tomas } 887c9de560dSAlex Tomas 888c9de560dSAlex Tomas /* wait for I/O completion */ 8899b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 8909b8b7d35SAmir Goldstein if (bh[i]) 891c9de560dSAlex Tomas wait_on_buffer(bh[i]); 892c9de560dSAlex Tomas 893c9de560dSAlex Tomas err = -EIO; 8949b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 8959b8b7d35SAmir Goldstein if (bh[i] && !buffer_uptodate(bh[i])) 896c9de560dSAlex Tomas goto out; 897c9de560dSAlex Tomas 89831b481dcSMingming Cao err = 0; 899c9de560dSAlex Tomas first_block = page->index * blocks_per_page; 900c9de560dSAlex Tomas for (i = 0; i < blocks_per_page; i++) { 901c9de560dSAlex Tomas int group; 902c9de560dSAlex Tomas 903c9de560dSAlex Tomas group = (first_block + i) >> 1; 9048df9675fSTheodore Ts'o if (group >= ngroups) 905c9de560dSAlex Tomas break; 906c9de560dSAlex Tomas 9079b8b7d35SAmir Goldstein if (!bh[group - first_group]) 9089b8b7d35SAmir Goldstein /* skip initialized uptodate buddy */ 9099b8b7d35SAmir Goldstein continue; 9109b8b7d35SAmir Goldstein 911c9de560dSAlex Tomas /* 912c9de560dSAlex Tomas * data carry information regarding this 913c9de560dSAlex Tomas * particular group in the format specified 914c9de560dSAlex Tomas * above 915c9de560dSAlex Tomas * 916c9de560dSAlex Tomas */ 917c9de560dSAlex Tomas data = page_address(page) + (i * blocksize); 918c9de560dSAlex Tomas bitmap = bh[group - first_group]->b_data; 919c9de560dSAlex Tomas 920c9de560dSAlex Tomas /* 921c9de560dSAlex Tomas * We place the buddy block and bitmap block 922c9de560dSAlex Tomas * close together 923c9de560dSAlex Tomas */ 924c9de560dSAlex Tomas if ((first_block + i) & 1) { 925c9de560dSAlex Tomas /* this is block of buddy */ 926c9de560dSAlex Tomas BUG_ON(incore == NULL); 9276ba495e9STheodore Ts'o mb_debug(1, "put buddy for group %u in page %lu/%x\n", 928c9de560dSAlex Tomas group, page->index, i * blocksize); 929f307333eSTheodore Ts'o trace_ext4_mb_buddy_bitmap_load(sb, group); 930c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, group); 931c9de560dSAlex Tomas grinfo->bb_fragments = 0; 932c9de560dSAlex Tomas memset(grinfo->bb_counters, 0, 9331927805eSEric Sandeen sizeof(*grinfo->bb_counters) * 9341927805eSEric Sandeen (sb->s_blocksize_bits+2)); 935c9de560dSAlex Tomas /* 936c9de560dSAlex Tomas * incore got set to the group block bitmap below 937c9de560dSAlex Tomas */ 9387a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, group); 9399b8b7d35SAmir Goldstein /* init the buddy */ 9409b8b7d35SAmir Goldstein memset(data, 0xff, blocksize); 941c9de560dSAlex Tomas ext4_mb_generate_buddy(sb, data, incore, group); 9427a2fcbf7SAneesh Kumar K.V ext4_unlock_group(sb, group); 943c9de560dSAlex Tomas incore = NULL; 944c9de560dSAlex Tomas } else { 945c9de560dSAlex Tomas /* this is block of bitmap */ 946c9de560dSAlex Tomas BUG_ON(incore != NULL); 9476ba495e9STheodore Ts'o mb_debug(1, "put bitmap for group %u in page %lu/%x\n", 948c9de560dSAlex Tomas group, page->index, i * blocksize); 949f307333eSTheodore Ts'o trace_ext4_mb_bitmap_load(sb, group); 950c9de560dSAlex Tomas 951c9de560dSAlex Tomas /* see comments in ext4_mb_put_pa() */ 952c9de560dSAlex Tomas ext4_lock_group(sb, group); 953c9de560dSAlex Tomas memcpy(data, bitmap, blocksize); 954c9de560dSAlex Tomas 955c9de560dSAlex Tomas /* mark all preallocated blks used in in-core bitmap */ 956c9de560dSAlex Tomas ext4_mb_generate_from_pa(sb, data, group); 9577a2fcbf7SAneesh Kumar K.V ext4_mb_generate_from_freelist(sb, data, group); 958c9de560dSAlex Tomas ext4_unlock_group(sb, group); 959c9de560dSAlex Tomas 960c9de560dSAlex Tomas /* set incore so that the buddy information can be 961c9de560dSAlex Tomas * generated using this 962c9de560dSAlex Tomas */ 963c9de560dSAlex Tomas incore = data; 964c9de560dSAlex Tomas } 965c9de560dSAlex Tomas } 966c9de560dSAlex Tomas SetPageUptodate(page); 967c9de560dSAlex Tomas 968c9de560dSAlex Tomas out: 969c9de560dSAlex Tomas if (bh) { 9709b8b7d35SAmir Goldstein for (i = 0; i < groups_per_page; i++) 971c9de560dSAlex Tomas brelse(bh[i]); 972c9de560dSAlex Tomas if (bh != &bhs) 973c9de560dSAlex Tomas kfree(bh); 974c9de560dSAlex Tomas } 975c9de560dSAlex Tomas return err; 976c9de560dSAlex Tomas } 977c9de560dSAlex Tomas 9788a57d9d6SCurt Wohlgemuth /* 9792de8807bSAmir Goldstein * Lock the buddy and bitmap pages. This make sure other parallel init_group 9802de8807bSAmir Goldstein * on the same buddy page doesn't happen whild holding the buddy page lock. 9812de8807bSAmir Goldstein * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap 9822de8807bSAmir Goldstein * are on the same page e4b->bd_buddy_page is NULL and return value is 0. 983eee4adc7SEric Sandeen */ 9842de8807bSAmir Goldstein static int ext4_mb_get_buddy_page_lock(struct super_block *sb, 9852de8807bSAmir Goldstein ext4_group_t group, struct ext4_buddy *e4b) 986eee4adc7SEric Sandeen { 9872de8807bSAmir Goldstein struct inode *inode = EXT4_SB(sb)->s_buddy_cache; 9882de8807bSAmir Goldstein int block, pnum, poff; 989eee4adc7SEric Sandeen int blocks_per_page; 9902de8807bSAmir Goldstein struct page *page; 9912de8807bSAmir Goldstein 9922de8807bSAmir Goldstein e4b->bd_buddy_page = NULL; 9932de8807bSAmir Goldstein e4b->bd_bitmap_page = NULL; 994eee4adc7SEric Sandeen 995eee4adc7SEric Sandeen blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 996eee4adc7SEric Sandeen /* 997eee4adc7SEric Sandeen * the buddy cache inode stores the block bitmap 998eee4adc7SEric Sandeen * and buddy information in consecutive blocks. 999eee4adc7SEric Sandeen * So for each group we need two blocks. 1000eee4adc7SEric Sandeen */ 1001eee4adc7SEric Sandeen block = group * 2; 1002eee4adc7SEric Sandeen pnum = block / blocks_per_page; 10032de8807bSAmir Goldstein poff = block % blocks_per_page; 10042de8807bSAmir Goldstein page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 10052de8807bSAmir Goldstein if (!page) 10062de8807bSAmir Goldstein return -EIO; 10072de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 10082de8807bSAmir Goldstein e4b->bd_bitmap_page = page; 10092de8807bSAmir Goldstein e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1010eee4adc7SEric Sandeen 10112de8807bSAmir Goldstein if (blocks_per_page >= 2) { 10122de8807bSAmir Goldstein /* buddy and bitmap are on the same page */ 10132de8807bSAmir Goldstein return 0; 1014eee4adc7SEric Sandeen } 1015eee4adc7SEric Sandeen 10162de8807bSAmir Goldstein block++; 1017eee4adc7SEric Sandeen pnum = block / blocks_per_page; 10182de8807bSAmir Goldstein poff = block % blocks_per_page; 10192de8807bSAmir Goldstein page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 10202de8807bSAmir Goldstein if (!page) 10212de8807bSAmir Goldstein return -EIO; 10222de8807bSAmir Goldstein BUG_ON(page->mapping != inode->i_mapping); 10232de8807bSAmir Goldstein e4b->bd_buddy_page = page; 10242de8807bSAmir Goldstein return 0; 1025eee4adc7SEric Sandeen } 1026eee4adc7SEric Sandeen 10272de8807bSAmir Goldstein static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b) 10282de8807bSAmir Goldstein { 10292de8807bSAmir Goldstein if (e4b->bd_bitmap_page) { 10302de8807bSAmir Goldstein unlock_page(e4b->bd_bitmap_page); 10312de8807bSAmir Goldstein page_cache_release(e4b->bd_bitmap_page); 10322de8807bSAmir Goldstein } 10332de8807bSAmir Goldstein if (e4b->bd_buddy_page) { 10342de8807bSAmir Goldstein unlock_page(e4b->bd_buddy_page); 10352de8807bSAmir Goldstein page_cache_release(e4b->bd_buddy_page); 10362de8807bSAmir Goldstein } 1037eee4adc7SEric Sandeen } 1038eee4adc7SEric Sandeen 1039eee4adc7SEric Sandeen /* 10408a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 10418a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 10428a57d9d6SCurt Wohlgemuth * calling this routine! 10438a57d9d6SCurt Wohlgemuth */ 1044b6a758ecSAneesh Kumar K.V static noinline_for_stack 1045b6a758ecSAneesh Kumar K.V int ext4_mb_init_group(struct super_block *sb, ext4_group_t group) 1046b6a758ecSAneesh Kumar K.V { 1047b6a758ecSAneesh Kumar K.V 1048b6a758ecSAneesh Kumar K.V struct ext4_group_info *this_grp; 10492de8807bSAmir Goldstein struct ext4_buddy e4b; 10502de8807bSAmir Goldstein struct page *page; 10512de8807bSAmir Goldstein int ret = 0; 1052b6a758ecSAneesh Kumar K.V 1053b6a758ecSAneesh Kumar K.V mb_debug(1, "init group %u\n", group); 1054b6a758ecSAneesh Kumar K.V this_grp = ext4_get_group_info(sb, group); 1055b6a758ecSAneesh Kumar K.V /* 105608c3a813SAneesh Kumar K.V * This ensures that we don't reinit the buddy cache 105708c3a813SAneesh Kumar K.V * page which map to the group from which we are already 105808c3a813SAneesh Kumar K.V * allocating. If we are looking at the buddy cache we would 105908c3a813SAneesh Kumar K.V * have taken a reference using ext4_mb_load_buddy and that 10602de8807bSAmir Goldstein * would have pinned buddy page to page cache. 1061b6a758ecSAneesh Kumar K.V */ 10622de8807bSAmir Goldstein ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b); 10632de8807bSAmir Goldstein if (ret || !EXT4_MB_GRP_NEED_INIT(this_grp)) { 1064b6a758ecSAneesh Kumar K.V /* 1065b6a758ecSAneesh Kumar K.V * somebody initialized the group 1066b6a758ecSAneesh Kumar K.V * return without doing anything 1067b6a758ecSAneesh Kumar K.V */ 1068b6a758ecSAneesh Kumar K.V goto err; 1069b6a758ecSAneesh Kumar K.V } 10702de8807bSAmir Goldstein 10712de8807bSAmir Goldstein page = e4b.bd_bitmap_page; 1072b6a758ecSAneesh Kumar K.V ret = ext4_mb_init_cache(page, NULL); 10732de8807bSAmir Goldstein if (ret) 1074b6a758ecSAneesh Kumar K.V goto err; 10752de8807bSAmir Goldstein if (!PageUptodate(page)) { 1076b6a758ecSAneesh Kumar K.V ret = -EIO; 1077b6a758ecSAneesh Kumar K.V goto err; 1078b6a758ecSAneesh Kumar K.V } 1079b6a758ecSAneesh Kumar K.V mark_page_accessed(page); 1080b6a758ecSAneesh Kumar K.V 10812de8807bSAmir Goldstein if (e4b.bd_buddy_page == NULL) { 1082b6a758ecSAneesh Kumar K.V /* 1083b6a758ecSAneesh Kumar K.V * If both the bitmap and buddy are in 1084b6a758ecSAneesh Kumar K.V * the same page we don't need to force 1085b6a758ecSAneesh Kumar K.V * init the buddy 1086b6a758ecSAneesh Kumar K.V */ 10872de8807bSAmir Goldstein ret = 0; 1088b6a758ecSAneesh Kumar K.V goto err; 1089b6a758ecSAneesh Kumar K.V } 10902de8807bSAmir Goldstein /* init buddy cache */ 10912de8807bSAmir Goldstein page = e4b.bd_buddy_page; 10922de8807bSAmir Goldstein ret = ext4_mb_init_cache(page, e4b.bd_bitmap); 10932de8807bSAmir Goldstein if (ret) 10942de8807bSAmir Goldstein goto err; 10952de8807bSAmir Goldstein if (!PageUptodate(page)) { 1096b6a758ecSAneesh Kumar K.V ret = -EIO; 1097b6a758ecSAneesh Kumar K.V goto err; 1098b6a758ecSAneesh Kumar K.V } 1099b6a758ecSAneesh Kumar K.V mark_page_accessed(page); 1100b6a758ecSAneesh Kumar K.V err: 11012de8807bSAmir Goldstein ext4_mb_put_buddy_page_lock(&e4b); 1102b6a758ecSAneesh Kumar K.V return ret; 1103b6a758ecSAneesh Kumar K.V } 1104b6a758ecSAneesh Kumar K.V 11058a57d9d6SCurt Wohlgemuth /* 11068a57d9d6SCurt Wohlgemuth * Locking note: This routine calls ext4_mb_init_cache(), which takes the 11078a57d9d6SCurt Wohlgemuth * block group lock of all groups for this page; do not hold the BG lock when 11088a57d9d6SCurt Wohlgemuth * calling this routine! 11098a57d9d6SCurt Wohlgemuth */ 11104ddfef7bSEric Sandeen static noinline_for_stack int 11114ddfef7bSEric Sandeen ext4_mb_load_buddy(struct super_block *sb, ext4_group_t group, 1112c9de560dSAlex Tomas struct ext4_buddy *e4b) 1113c9de560dSAlex Tomas { 1114c9de560dSAlex Tomas int blocks_per_page; 1115c9de560dSAlex Tomas int block; 1116c9de560dSAlex Tomas int pnum; 1117c9de560dSAlex Tomas int poff; 1118c9de560dSAlex Tomas struct page *page; 1119fdf6c7a7SShen Feng int ret; 1120920313a7SAneesh Kumar K.V struct ext4_group_info *grp; 1121920313a7SAneesh Kumar K.V struct ext4_sb_info *sbi = EXT4_SB(sb); 1122920313a7SAneesh Kumar K.V struct inode *inode = sbi->s_buddy_cache; 1123c9de560dSAlex Tomas 11246ba495e9STheodore Ts'o mb_debug(1, "load group %u\n", group); 1125c9de560dSAlex Tomas 1126c9de560dSAlex Tomas blocks_per_page = PAGE_CACHE_SIZE / sb->s_blocksize; 1127920313a7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 1128c9de560dSAlex Tomas 1129c9de560dSAlex Tomas e4b->bd_blkbits = sb->s_blocksize_bits; 1130529da704STao Ma e4b->bd_info = grp; 1131c9de560dSAlex Tomas e4b->bd_sb = sb; 1132c9de560dSAlex Tomas e4b->bd_group = group; 1133c9de560dSAlex Tomas e4b->bd_buddy_page = NULL; 1134c9de560dSAlex Tomas e4b->bd_bitmap_page = NULL; 1135c9de560dSAlex Tomas 1136f41c0750SAneesh Kumar K.V if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 1137f41c0750SAneesh Kumar K.V /* 1138f41c0750SAneesh Kumar K.V * we need full data about the group 1139f41c0750SAneesh Kumar K.V * to make a good selection 1140f41c0750SAneesh Kumar K.V */ 1141f41c0750SAneesh Kumar K.V ret = ext4_mb_init_group(sb, group); 1142f41c0750SAneesh Kumar K.V if (ret) 1143f41c0750SAneesh Kumar K.V return ret; 1144f41c0750SAneesh Kumar K.V } 1145f41c0750SAneesh Kumar K.V 1146c9de560dSAlex Tomas /* 1147c9de560dSAlex Tomas * the buddy cache inode stores the block bitmap 1148c9de560dSAlex Tomas * and buddy information in consecutive blocks. 1149c9de560dSAlex Tomas * So for each group we need two blocks. 1150c9de560dSAlex Tomas */ 1151c9de560dSAlex Tomas block = group * 2; 1152c9de560dSAlex Tomas pnum = block / blocks_per_page; 1153c9de560dSAlex Tomas poff = block % blocks_per_page; 1154c9de560dSAlex Tomas 1155c9de560dSAlex Tomas /* we could use find_or_create_page(), but it locks page 1156c9de560dSAlex Tomas * what we'd like to avoid in fast path ... */ 1157c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1158c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1159c9de560dSAlex Tomas if (page) 1160920313a7SAneesh Kumar K.V /* 1161920313a7SAneesh Kumar K.V * drop the page reference and try 1162920313a7SAneesh Kumar K.V * to get the page with lock. If we 1163920313a7SAneesh Kumar K.V * are not uptodate that implies 1164920313a7SAneesh Kumar K.V * somebody just created the page but 1165920313a7SAneesh Kumar K.V * is yet to initialize the same. So 1166920313a7SAneesh Kumar K.V * wait for it to initialize. 1167920313a7SAneesh Kumar K.V */ 1168c9de560dSAlex Tomas page_cache_release(page); 1169c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1170c9de560dSAlex Tomas if (page) { 1171c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1172c9de560dSAlex Tomas if (!PageUptodate(page)) { 1173fdf6c7a7SShen Feng ret = ext4_mb_init_cache(page, NULL); 1174fdf6c7a7SShen Feng if (ret) { 1175fdf6c7a7SShen Feng unlock_page(page); 1176fdf6c7a7SShen Feng goto err; 1177fdf6c7a7SShen Feng } 1178c9de560dSAlex Tomas mb_cmp_bitmaps(e4b, page_address(page) + 1179c9de560dSAlex Tomas (poff * sb->s_blocksize)); 1180c9de560dSAlex Tomas } 1181c9de560dSAlex Tomas unlock_page(page); 1182c9de560dSAlex Tomas } 1183c9de560dSAlex Tomas } 1184fdf6c7a7SShen Feng if (page == NULL || !PageUptodate(page)) { 1185fdf6c7a7SShen Feng ret = -EIO; 1186c9de560dSAlex Tomas goto err; 1187fdf6c7a7SShen Feng } 1188c9de560dSAlex Tomas e4b->bd_bitmap_page = page; 1189c9de560dSAlex Tomas e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize); 1190c9de560dSAlex Tomas mark_page_accessed(page); 1191c9de560dSAlex Tomas 1192c9de560dSAlex Tomas block++; 1193c9de560dSAlex Tomas pnum = block / blocks_per_page; 1194c9de560dSAlex Tomas poff = block % blocks_per_page; 1195c9de560dSAlex Tomas 1196c9de560dSAlex Tomas page = find_get_page(inode->i_mapping, pnum); 1197c9de560dSAlex Tomas if (page == NULL || !PageUptodate(page)) { 1198c9de560dSAlex Tomas if (page) 1199c9de560dSAlex Tomas page_cache_release(page); 1200c9de560dSAlex Tomas page = find_or_create_page(inode->i_mapping, pnum, GFP_NOFS); 1201c9de560dSAlex Tomas if (page) { 1202c9de560dSAlex Tomas BUG_ON(page->mapping != inode->i_mapping); 1203fdf6c7a7SShen Feng if (!PageUptodate(page)) { 1204fdf6c7a7SShen Feng ret = ext4_mb_init_cache(page, e4b->bd_bitmap); 1205fdf6c7a7SShen Feng if (ret) { 1206fdf6c7a7SShen Feng unlock_page(page); 1207fdf6c7a7SShen Feng goto err; 1208fdf6c7a7SShen Feng } 1209fdf6c7a7SShen Feng } 1210c9de560dSAlex Tomas unlock_page(page); 1211c9de560dSAlex Tomas } 1212c9de560dSAlex Tomas } 1213fdf6c7a7SShen Feng if (page == NULL || !PageUptodate(page)) { 1214fdf6c7a7SShen Feng ret = -EIO; 1215c9de560dSAlex Tomas goto err; 1216fdf6c7a7SShen Feng } 1217c9de560dSAlex Tomas e4b->bd_buddy_page = page; 1218c9de560dSAlex Tomas e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize); 1219c9de560dSAlex Tomas mark_page_accessed(page); 1220c9de560dSAlex Tomas 1221c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 1222c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 1223c9de560dSAlex Tomas 1224c9de560dSAlex Tomas return 0; 1225c9de560dSAlex Tomas 1226c9de560dSAlex Tomas err: 122726626f11SYang Ruirui if (page) 122826626f11SYang Ruirui page_cache_release(page); 1229c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1230c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1231c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1232c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1233c9de560dSAlex Tomas e4b->bd_buddy = NULL; 1234c9de560dSAlex Tomas e4b->bd_bitmap = NULL; 1235fdf6c7a7SShen Feng return ret; 1236c9de560dSAlex Tomas } 1237c9de560dSAlex Tomas 1238e39e07fdSJing Zhang static void ext4_mb_unload_buddy(struct ext4_buddy *e4b) 1239c9de560dSAlex Tomas { 1240c9de560dSAlex Tomas if (e4b->bd_bitmap_page) 1241c9de560dSAlex Tomas page_cache_release(e4b->bd_bitmap_page); 1242c9de560dSAlex Tomas if (e4b->bd_buddy_page) 1243c9de560dSAlex Tomas page_cache_release(e4b->bd_buddy_page); 1244c9de560dSAlex Tomas } 1245c9de560dSAlex Tomas 1246c9de560dSAlex Tomas 1247c9de560dSAlex Tomas static int mb_find_order_for_block(struct ext4_buddy *e4b, int block) 1248c9de560dSAlex Tomas { 1249c9de560dSAlex Tomas int order = 1; 1250c9de560dSAlex Tomas void *bb; 1251c9de560dSAlex Tomas 1252c9de560dSAlex Tomas BUG_ON(EXT4_MB_BITMAP(e4b) == EXT4_MB_BUDDY(e4b)); 1253c9de560dSAlex Tomas BUG_ON(block >= (1 << (e4b->bd_blkbits + 3))); 1254c9de560dSAlex Tomas 1255c9de560dSAlex Tomas bb = EXT4_MB_BUDDY(e4b); 1256c9de560dSAlex Tomas while (order <= e4b->bd_blkbits + 1) { 1257c9de560dSAlex Tomas block = block >> 1; 1258c9de560dSAlex Tomas if (!mb_test_bit(block, bb)) { 1259c9de560dSAlex Tomas /* this block is part of buddy of order 'order' */ 1260c9de560dSAlex Tomas return order; 1261c9de560dSAlex Tomas } 1262c9de560dSAlex Tomas bb += 1 << (e4b->bd_blkbits - order); 1263c9de560dSAlex Tomas order++; 1264c9de560dSAlex Tomas } 1265c9de560dSAlex Tomas return 0; 1266c9de560dSAlex Tomas } 1267c9de560dSAlex Tomas 1268955ce5f5SAneesh Kumar K.V static void mb_clear_bits(void *bm, int cur, int len) 1269c9de560dSAlex Tomas { 1270c9de560dSAlex Tomas __u32 *addr; 1271c9de560dSAlex Tomas 1272c9de560dSAlex Tomas len = cur + len; 1273c9de560dSAlex Tomas while (cur < len) { 1274c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1275c9de560dSAlex Tomas /* fast path: clear whole word at once */ 1276c9de560dSAlex Tomas addr = bm + (cur >> 3); 1277c9de560dSAlex Tomas *addr = 0; 1278c9de560dSAlex Tomas cur += 32; 1279c9de560dSAlex Tomas continue; 1280c9de560dSAlex Tomas } 1281e8134b27SAneesh Kumar K.V mb_clear_bit(cur, bm); 1282c9de560dSAlex Tomas cur++; 1283c9de560dSAlex Tomas } 1284c9de560dSAlex Tomas } 1285c9de560dSAlex Tomas 1286c3e94d1dSYongqiang Yang void ext4_set_bits(void *bm, int cur, int len) 1287c9de560dSAlex Tomas { 1288c9de560dSAlex Tomas __u32 *addr; 1289c9de560dSAlex Tomas 1290c9de560dSAlex Tomas len = cur + len; 1291c9de560dSAlex Tomas while (cur < len) { 1292c9de560dSAlex Tomas if ((cur & 31) == 0 && (len - cur) >= 32) { 1293c9de560dSAlex Tomas /* fast path: set whole word at once */ 1294c9de560dSAlex Tomas addr = bm + (cur >> 3); 1295c9de560dSAlex Tomas *addr = 0xffffffff; 1296c9de560dSAlex Tomas cur += 32; 1297c9de560dSAlex Tomas continue; 1298c9de560dSAlex Tomas } 1299e8134b27SAneesh Kumar K.V mb_set_bit(cur, bm); 1300c9de560dSAlex Tomas cur++; 1301c9de560dSAlex Tomas } 1302c9de560dSAlex Tomas } 1303c9de560dSAlex Tomas 13047e5a8cddSShen Feng static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b, 1305c9de560dSAlex Tomas int first, int count) 1306c9de560dSAlex Tomas { 1307c9de560dSAlex Tomas int block = 0; 1308c9de560dSAlex Tomas int max = 0; 1309c9de560dSAlex Tomas int order; 1310c9de560dSAlex Tomas void *buddy; 1311c9de560dSAlex Tomas void *buddy2; 1312c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 1313c9de560dSAlex Tomas 1314c9de560dSAlex Tomas BUG_ON(first + count > (sb->s_blocksize << 3)); 1315bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group)); 1316c9de560dSAlex Tomas mb_check_buddy(e4b); 1317c9de560dSAlex Tomas mb_free_blocks_double(inode, e4b, first, count); 1318c9de560dSAlex Tomas 1319c9de560dSAlex Tomas e4b->bd_info->bb_free += count; 1320c9de560dSAlex Tomas if (first < e4b->bd_info->bb_first_free) 1321c9de560dSAlex Tomas e4b->bd_info->bb_first_free = first; 1322c9de560dSAlex Tomas 1323c9de560dSAlex Tomas /* let's maintain fragments counter */ 1324c9de560dSAlex Tomas if (first != 0) 1325c9de560dSAlex Tomas block = !mb_test_bit(first - 1, EXT4_MB_BITMAP(e4b)); 1326c9de560dSAlex Tomas if (first + count < EXT4_SB(sb)->s_mb_maxs[0]) 1327c9de560dSAlex Tomas max = !mb_test_bit(first + count, EXT4_MB_BITMAP(e4b)); 1328c9de560dSAlex Tomas if (block && max) 1329c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1330c9de560dSAlex Tomas else if (!block && !max) 1331c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1332c9de560dSAlex Tomas 1333c9de560dSAlex Tomas /* let's maintain buddy itself */ 1334c9de560dSAlex Tomas while (count-- > 0) { 1335c9de560dSAlex Tomas block = first++; 1336c9de560dSAlex Tomas order = 0; 1337c9de560dSAlex Tomas 1338c9de560dSAlex Tomas if (!mb_test_bit(block, EXT4_MB_BITMAP(e4b))) { 1339c9de560dSAlex Tomas ext4_fsblk_t blocknr; 13405661bd68SAkinobu Mita 13415661bd68SAkinobu Mita blocknr = ext4_group_first_block_no(sb, e4b->bd_group); 1342*53accfa9STheodore Ts'o blocknr += EXT4_C2B(EXT4_SB(sb), block); 13435d1b1b3fSAneesh Kumar K.V ext4_grp_locked_error(sb, e4b->bd_group, 1344e29136f8STheodore Ts'o inode ? inode->i_ino : 0, 1345e29136f8STheodore Ts'o blocknr, 1346e29136f8STheodore Ts'o "freeing already freed block " 1347e29136f8STheodore Ts'o "(bit %u)", block); 1348c9de560dSAlex Tomas } 1349c9de560dSAlex Tomas mb_clear_bit(block, EXT4_MB_BITMAP(e4b)); 1350c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]++; 1351c9de560dSAlex Tomas 1352c9de560dSAlex Tomas /* start of the buddy */ 1353c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 1354c9de560dSAlex Tomas 1355c9de560dSAlex Tomas do { 1356c9de560dSAlex Tomas block &= ~1UL; 1357c9de560dSAlex Tomas if (mb_test_bit(block, buddy) || 1358c9de560dSAlex Tomas mb_test_bit(block + 1, buddy)) 1359c9de560dSAlex Tomas break; 1360c9de560dSAlex Tomas 1361c9de560dSAlex Tomas /* both the buddies are free, try to coalesce them */ 1362c9de560dSAlex Tomas buddy2 = mb_find_buddy(e4b, order + 1, &max); 1363c9de560dSAlex Tomas 1364c9de560dSAlex Tomas if (!buddy2) 1365c9de560dSAlex Tomas break; 1366c9de560dSAlex Tomas 1367c9de560dSAlex Tomas if (order > 0) { 1368c9de560dSAlex Tomas /* for special purposes, we don't set 1369c9de560dSAlex Tomas * free bits in bitmap */ 1370c9de560dSAlex Tomas mb_set_bit(block, buddy); 1371c9de560dSAlex Tomas mb_set_bit(block + 1, buddy); 1372c9de560dSAlex Tomas } 1373c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]--; 1374c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]--; 1375c9de560dSAlex Tomas 1376c9de560dSAlex Tomas block = block >> 1; 1377c9de560dSAlex Tomas order++; 1378c9de560dSAlex Tomas e4b->bd_info->bb_counters[order]++; 1379c9de560dSAlex Tomas 1380c9de560dSAlex Tomas mb_clear_bit(block, buddy2); 1381c9de560dSAlex Tomas buddy = buddy2; 1382c9de560dSAlex Tomas } while (1); 1383c9de560dSAlex Tomas } 13848a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(sb, e4b->bd_info); 1385c9de560dSAlex Tomas mb_check_buddy(e4b); 1386c9de560dSAlex Tomas } 1387c9de560dSAlex Tomas 1388c9de560dSAlex Tomas static int mb_find_extent(struct ext4_buddy *e4b, int order, int block, 1389c9de560dSAlex Tomas int needed, struct ext4_free_extent *ex) 1390c9de560dSAlex Tomas { 1391c9de560dSAlex Tomas int next = block; 1392c9de560dSAlex Tomas int max; 1393c9de560dSAlex Tomas int ord; 1394c9de560dSAlex Tomas void *buddy; 1395c9de560dSAlex Tomas 1396bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1397c9de560dSAlex Tomas BUG_ON(ex == NULL); 1398c9de560dSAlex Tomas 1399c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, order, &max); 1400c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1401c9de560dSAlex Tomas BUG_ON(block >= max); 1402c9de560dSAlex Tomas if (mb_test_bit(block, buddy)) { 1403c9de560dSAlex Tomas ex->fe_len = 0; 1404c9de560dSAlex Tomas ex->fe_start = 0; 1405c9de560dSAlex Tomas ex->fe_group = 0; 1406c9de560dSAlex Tomas return 0; 1407c9de560dSAlex Tomas } 1408c9de560dSAlex Tomas 1409c9de560dSAlex Tomas /* FIXME dorp order completely ? */ 1410c9de560dSAlex Tomas if (likely(order == 0)) { 1411c9de560dSAlex Tomas /* find actual order */ 1412c9de560dSAlex Tomas order = mb_find_order_for_block(e4b, block); 1413c9de560dSAlex Tomas block = block >> order; 1414c9de560dSAlex Tomas } 1415c9de560dSAlex Tomas 1416c9de560dSAlex Tomas ex->fe_len = 1 << order; 1417c9de560dSAlex Tomas ex->fe_start = block << order; 1418c9de560dSAlex Tomas ex->fe_group = e4b->bd_group; 1419c9de560dSAlex Tomas 1420c9de560dSAlex Tomas /* calc difference from given start */ 1421c9de560dSAlex Tomas next = next - ex->fe_start; 1422c9de560dSAlex Tomas ex->fe_len -= next; 1423c9de560dSAlex Tomas ex->fe_start += next; 1424c9de560dSAlex Tomas 1425c9de560dSAlex Tomas while (needed > ex->fe_len && 1426c9de560dSAlex Tomas (buddy = mb_find_buddy(e4b, order, &max))) { 1427c9de560dSAlex Tomas 1428c9de560dSAlex Tomas if (block + 1 >= max) 1429c9de560dSAlex Tomas break; 1430c9de560dSAlex Tomas 1431c9de560dSAlex Tomas next = (block + 1) * (1 << order); 1432c9de560dSAlex Tomas if (mb_test_bit(next, EXT4_MB_BITMAP(e4b))) 1433c9de560dSAlex Tomas break; 1434c9de560dSAlex Tomas 1435c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, next); 1436c9de560dSAlex Tomas 1437c9de560dSAlex Tomas order = ord; 1438c9de560dSAlex Tomas block = next >> order; 1439c9de560dSAlex Tomas ex->fe_len += 1 << order; 1440c9de560dSAlex Tomas } 1441c9de560dSAlex Tomas 1442c9de560dSAlex Tomas BUG_ON(ex->fe_start + ex->fe_len > (1 << (e4b->bd_blkbits + 3))); 1443c9de560dSAlex Tomas return ex->fe_len; 1444c9de560dSAlex Tomas } 1445c9de560dSAlex Tomas 1446c9de560dSAlex Tomas static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex) 1447c9de560dSAlex Tomas { 1448c9de560dSAlex Tomas int ord; 1449c9de560dSAlex Tomas int mlen = 0; 1450c9de560dSAlex Tomas int max = 0; 1451c9de560dSAlex Tomas int cur; 1452c9de560dSAlex Tomas int start = ex->fe_start; 1453c9de560dSAlex Tomas int len = ex->fe_len; 1454c9de560dSAlex Tomas unsigned ret = 0; 1455c9de560dSAlex Tomas int len0 = len; 1456c9de560dSAlex Tomas void *buddy; 1457c9de560dSAlex Tomas 1458c9de560dSAlex Tomas BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3)); 1459c9de560dSAlex Tomas BUG_ON(e4b->bd_group != ex->fe_group); 1460bc8e6740SVincent Minet assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group)); 1461c9de560dSAlex Tomas mb_check_buddy(e4b); 1462c9de560dSAlex Tomas mb_mark_used_double(e4b, start, len); 1463c9de560dSAlex Tomas 1464c9de560dSAlex Tomas e4b->bd_info->bb_free -= len; 1465c9de560dSAlex Tomas if (e4b->bd_info->bb_first_free == start) 1466c9de560dSAlex Tomas e4b->bd_info->bb_first_free += len; 1467c9de560dSAlex Tomas 1468c9de560dSAlex Tomas /* let's maintain fragments counter */ 1469c9de560dSAlex Tomas if (start != 0) 1470c9de560dSAlex Tomas mlen = !mb_test_bit(start - 1, EXT4_MB_BITMAP(e4b)); 1471c9de560dSAlex Tomas if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0]) 1472c9de560dSAlex Tomas max = !mb_test_bit(start + len, EXT4_MB_BITMAP(e4b)); 1473c9de560dSAlex Tomas if (mlen && max) 1474c9de560dSAlex Tomas e4b->bd_info->bb_fragments++; 1475c9de560dSAlex Tomas else if (!mlen && !max) 1476c9de560dSAlex Tomas e4b->bd_info->bb_fragments--; 1477c9de560dSAlex Tomas 1478c9de560dSAlex Tomas /* let's maintain buddy itself */ 1479c9de560dSAlex Tomas while (len) { 1480c9de560dSAlex Tomas ord = mb_find_order_for_block(e4b, start); 1481c9de560dSAlex Tomas 1482c9de560dSAlex Tomas if (((start >> ord) << ord) == start && len >= (1 << ord)) { 1483c9de560dSAlex Tomas /* the whole chunk may be allocated at once! */ 1484c9de560dSAlex Tomas mlen = 1 << ord; 1485c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1486c9de560dSAlex Tomas BUG_ON((start >> ord) >= max); 1487c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1488c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1489c9de560dSAlex Tomas start += mlen; 1490c9de560dSAlex Tomas len -= mlen; 1491c9de560dSAlex Tomas BUG_ON(len < 0); 1492c9de560dSAlex Tomas continue; 1493c9de560dSAlex Tomas } 1494c9de560dSAlex Tomas 1495c9de560dSAlex Tomas /* store for history */ 1496c9de560dSAlex Tomas if (ret == 0) 1497c9de560dSAlex Tomas ret = len | (ord << 16); 1498c9de560dSAlex Tomas 1499c9de560dSAlex Tomas /* we have to split large buddy */ 1500c9de560dSAlex Tomas BUG_ON(ord <= 0); 1501c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1502c9de560dSAlex Tomas mb_set_bit(start >> ord, buddy); 1503c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]--; 1504c9de560dSAlex Tomas 1505c9de560dSAlex Tomas ord--; 1506c9de560dSAlex Tomas cur = (start >> ord) & ~1U; 1507c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, ord, &max); 1508c9de560dSAlex Tomas mb_clear_bit(cur, buddy); 1509c9de560dSAlex Tomas mb_clear_bit(cur + 1, buddy); 1510c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1511c9de560dSAlex Tomas e4b->bd_info->bb_counters[ord]++; 1512c9de560dSAlex Tomas } 15138a57d9d6SCurt Wohlgemuth mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info); 1514c9de560dSAlex Tomas 1515c3e94d1dSYongqiang Yang ext4_set_bits(EXT4_MB_BITMAP(e4b), ex->fe_start, len0); 1516c9de560dSAlex Tomas mb_check_buddy(e4b); 1517c9de560dSAlex Tomas 1518c9de560dSAlex Tomas return ret; 1519c9de560dSAlex Tomas } 1520c9de560dSAlex Tomas 1521c9de560dSAlex Tomas /* 1522c9de560dSAlex Tomas * Must be called under group lock! 1523c9de560dSAlex Tomas */ 1524c9de560dSAlex Tomas static void ext4_mb_use_best_found(struct ext4_allocation_context *ac, 1525c9de560dSAlex Tomas struct ext4_buddy *e4b) 1526c9de560dSAlex Tomas { 1527c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1528c9de560dSAlex Tomas int ret; 1529c9de560dSAlex Tomas 1530c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group); 1531c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1532c9de560dSAlex Tomas 1533c9de560dSAlex Tomas ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len); 1534c9de560dSAlex Tomas ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical; 1535c9de560dSAlex Tomas ret = mb_mark_used(e4b, &ac->ac_b_ex); 1536c9de560dSAlex Tomas 1537c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 1538c9de560dSAlex Tomas * allocated blocks for history */ 1539c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 1540c9de560dSAlex Tomas 1541c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 1542c9de560dSAlex Tomas ac->ac_tail = ret & 0xffff; 1543c9de560dSAlex Tomas ac->ac_buddy = ret >> 16; 1544c9de560dSAlex Tomas 1545c3a326a6SAneesh Kumar K.V /* 1546c3a326a6SAneesh Kumar K.V * take the page reference. We want the page to be pinned 1547c3a326a6SAneesh Kumar K.V * so that we don't get a ext4_mb_init_cache_call for this 1548c3a326a6SAneesh Kumar K.V * group until we update the bitmap. That would mean we 1549c3a326a6SAneesh Kumar K.V * double allocate blocks. The reference is dropped 1550c3a326a6SAneesh Kumar K.V * in ext4_mb_release_context 1551c3a326a6SAneesh Kumar K.V */ 1552c9de560dSAlex Tomas ac->ac_bitmap_page = e4b->bd_bitmap_page; 1553c9de560dSAlex Tomas get_page(ac->ac_bitmap_page); 1554c9de560dSAlex Tomas ac->ac_buddy_page = e4b->bd_buddy_page; 1555c9de560dSAlex Tomas get_page(ac->ac_buddy_page); 1556c9de560dSAlex Tomas /* store last allocated for subsequent stream allocation */ 15574ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 1558c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 1559c9de560dSAlex Tomas sbi->s_mb_last_group = ac->ac_f_ex.fe_group; 1560c9de560dSAlex Tomas sbi->s_mb_last_start = ac->ac_f_ex.fe_start; 1561c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 1562c9de560dSAlex Tomas } 1563c9de560dSAlex Tomas } 1564c9de560dSAlex Tomas 1565c9de560dSAlex Tomas /* 1566c9de560dSAlex Tomas * regular allocator, for general purposes allocation 1567c9de560dSAlex Tomas */ 1568c9de560dSAlex Tomas 1569c9de560dSAlex Tomas static void ext4_mb_check_limits(struct ext4_allocation_context *ac, 1570c9de560dSAlex Tomas struct ext4_buddy *e4b, 1571c9de560dSAlex Tomas int finish_group) 1572c9de560dSAlex Tomas { 1573c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1574c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1575c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1576c9de560dSAlex Tomas struct ext4_free_extent ex; 1577c9de560dSAlex Tomas int max; 1578c9de560dSAlex Tomas 1579032115fcSAneesh Kumar K.V if (ac->ac_status == AC_STATUS_FOUND) 1580032115fcSAneesh Kumar K.V return; 1581c9de560dSAlex Tomas /* 1582c9de560dSAlex Tomas * We don't want to scan for a whole year 1583c9de560dSAlex Tomas */ 1584c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan && 1585c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1586c9de560dSAlex Tomas ac->ac_status = AC_STATUS_BREAK; 1587c9de560dSAlex Tomas return; 1588c9de560dSAlex Tomas } 1589c9de560dSAlex Tomas 1590c9de560dSAlex Tomas /* 1591c9de560dSAlex Tomas * Haven't found good chunk so far, let's continue 1592c9de560dSAlex Tomas */ 1593c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) 1594c9de560dSAlex Tomas return; 1595c9de560dSAlex Tomas 1596c9de560dSAlex Tomas if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan) 1597c9de560dSAlex Tomas && bex->fe_group == e4b->bd_group) { 1598c9de560dSAlex Tomas /* recheck chunk's availability - we don't know 1599c9de560dSAlex Tomas * when it was found (within this lock-unlock 1600c9de560dSAlex Tomas * period or not) */ 1601c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, bex->fe_start, gex->fe_len, &ex); 1602c9de560dSAlex Tomas if (max >= gex->fe_len) { 1603c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1604c9de560dSAlex Tomas return; 1605c9de560dSAlex Tomas } 1606c9de560dSAlex Tomas } 1607c9de560dSAlex Tomas } 1608c9de560dSAlex Tomas 1609c9de560dSAlex Tomas /* 1610c9de560dSAlex Tomas * The routine checks whether found extent is good enough. If it is, 1611c9de560dSAlex Tomas * then the extent gets marked used and flag is set to the context 1612c9de560dSAlex Tomas * to stop scanning. Otherwise, the extent is compared with the 1613c9de560dSAlex Tomas * previous found extent and if new one is better, then it's stored 1614c9de560dSAlex Tomas * in the context. Later, the best found extent will be used, if 1615c9de560dSAlex Tomas * mballoc can't find good enough extent. 1616c9de560dSAlex Tomas * 1617c9de560dSAlex Tomas * FIXME: real allocation policy is to be designed yet! 1618c9de560dSAlex Tomas */ 1619c9de560dSAlex Tomas static void ext4_mb_measure_extent(struct ext4_allocation_context *ac, 1620c9de560dSAlex Tomas struct ext4_free_extent *ex, 1621c9de560dSAlex Tomas struct ext4_buddy *e4b) 1622c9de560dSAlex Tomas { 1623c9de560dSAlex Tomas struct ext4_free_extent *bex = &ac->ac_b_ex; 1624c9de560dSAlex Tomas struct ext4_free_extent *gex = &ac->ac_g_ex; 1625c9de560dSAlex Tomas 1626c9de560dSAlex Tomas BUG_ON(ex->fe_len <= 0); 16277137d7a4STheodore Ts'o BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 16287137d7a4STheodore Ts'o BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 1629c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_CONTINUE); 1630c9de560dSAlex Tomas 1631c9de560dSAlex Tomas ac->ac_found++; 1632c9de560dSAlex Tomas 1633c9de560dSAlex Tomas /* 1634c9de560dSAlex Tomas * The special case - take what you catch first 1635c9de560dSAlex Tomas */ 1636c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 1637c9de560dSAlex Tomas *bex = *ex; 1638c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1639c9de560dSAlex Tomas return; 1640c9de560dSAlex Tomas } 1641c9de560dSAlex Tomas 1642c9de560dSAlex Tomas /* 1643c9de560dSAlex Tomas * Let's check whether the chuck is good enough 1644c9de560dSAlex Tomas */ 1645c9de560dSAlex Tomas if (ex->fe_len == gex->fe_len) { 1646c9de560dSAlex Tomas *bex = *ex; 1647c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1648c9de560dSAlex Tomas return; 1649c9de560dSAlex Tomas } 1650c9de560dSAlex Tomas 1651c9de560dSAlex Tomas /* 1652c9de560dSAlex Tomas * If this is first found extent, just store it in the context 1653c9de560dSAlex Tomas */ 1654c9de560dSAlex Tomas if (bex->fe_len == 0) { 1655c9de560dSAlex Tomas *bex = *ex; 1656c9de560dSAlex Tomas return; 1657c9de560dSAlex Tomas } 1658c9de560dSAlex Tomas 1659c9de560dSAlex Tomas /* 1660c9de560dSAlex Tomas * If new found extent is better, store it in the context 1661c9de560dSAlex Tomas */ 1662c9de560dSAlex Tomas if (bex->fe_len < gex->fe_len) { 1663c9de560dSAlex Tomas /* if the request isn't satisfied, any found extent 1664c9de560dSAlex Tomas * larger than previous best one is better */ 1665c9de560dSAlex Tomas if (ex->fe_len > bex->fe_len) 1666c9de560dSAlex Tomas *bex = *ex; 1667c9de560dSAlex Tomas } else if (ex->fe_len > gex->fe_len) { 1668c9de560dSAlex Tomas /* if the request is satisfied, then we try to find 1669c9de560dSAlex Tomas * an extent that still satisfy the request, but is 1670c9de560dSAlex Tomas * smaller than previous one */ 1671c9de560dSAlex Tomas if (ex->fe_len < bex->fe_len) 1672c9de560dSAlex Tomas *bex = *ex; 1673c9de560dSAlex Tomas } 1674c9de560dSAlex Tomas 1675c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 0); 1676c9de560dSAlex Tomas } 1677c9de560dSAlex Tomas 1678089ceeccSEric Sandeen static noinline_for_stack 1679089ceeccSEric Sandeen int ext4_mb_try_best_found(struct ext4_allocation_context *ac, 1680c9de560dSAlex Tomas struct ext4_buddy *e4b) 1681c9de560dSAlex Tomas { 1682c9de560dSAlex Tomas struct ext4_free_extent ex = ac->ac_b_ex; 1683c9de560dSAlex Tomas ext4_group_t group = ex.fe_group; 1684c9de560dSAlex Tomas int max; 1685c9de560dSAlex Tomas int err; 1686c9de560dSAlex Tomas 1687c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1688c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1689c9de560dSAlex Tomas if (err) 1690c9de560dSAlex Tomas return err; 1691c9de560dSAlex Tomas 1692c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 1693c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, ex.fe_start, ex.fe_len, &ex); 1694c9de560dSAlex Tomas 1695c9de560dSAlex Tomas if (max > 0) { 1696c9de560dSAlex Tomas ac->ac_b_ex = ex; 1697c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1698c9de560dSAlex Tomas } 1699c9de560dSAlex Tomas 1700c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1701e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1702c9de560dSAlex Tomas 1703c9de560dSAlex Tomas return 0; 1704c9de560dSAlex Tomas } 1705c9de560dSAlex Tomas 1706089ceeccSEric Sandeen static noinline_for_stack 1707089ceeccSEric Sandeen int ext4_mb_find_by_goal(struct ext4_allocation_context *ac, 1708c9de560dSAlex Tomas struct ext4_buddy *e4b) 1709c9de560dSAlex Tomas { 1710c9de560dSAlex Tomas ext4_group_t group = ac->ac_g_ex.fe_group; 1711c9de560dSAlex Tomas int max; 1712c9de560dSAlex Tomas int err; 1713c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 1714c9de560dSAlex Tomas struct ext4_free_extent ex; 1715c9de560dSAlex Tomas 1716c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_TRY_GOAL)) 1717c9de560dSAlex Tomas return 0; 1718c9de560dSAlex Tomas 1719c9de560dSAlex Tomas err = ext4_mb_load_buddy(ac->ac_sb, group, e4b); 1720c9de560dSAlex Tomas if (err) 1721c9de560dSAlex Tomas return err; 1722c9de560dSAlex Tomas 1723c9de560dSAlex Tomas ext4_lock_group(ac->ac_sb, group); 1724c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, ac->ac_g_ex.fe_start, 1725c9de560dSAlex Tomas ac->ac_g_ex.fe_len, &ex); 1726c9de560dSAlex Tomas 1727c9de560dSAlex Tomas if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) { 1728c9de560dSAlex Tomas ext4_fsblk_t start; 1729c9de560dSAlex Tomas 17305661bd68SAkinobu Mita start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) + 17315661bd68SAkinobu Mita ex.fe_start; 1732c9de560dSAlex Tomas /* use do_div to get remainder (would be 64-bit modulo) */ 1733c9de560dSAlex Tomas if (do_div(start, sbi->s_stripe) == 0) { 1734c9de560dSAlex Tomas ac->ac_found++; 1735c9de560dSAlex Tomas ac->ac_b_ex = ex; 1736c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1737c9de560dSAlex Tomas } 1738c9de560dSAlex Tomas } else if (max >= ac->ac_g_ex.fe_len) { 1739c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1740c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1741c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1742c9de560dSAlex Tomas ac->ac_found++; 1743c9de560dSAlex Tomas ac->ac_b_ex = ex; 1744c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1745c9de560dSAlex Tomas } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) { 1746c9de560dSAlex Tomas /* Sometimes, caller may want to merge even small 1747c9de560dSAlex Tomas * number of blocks to an existing extent */ 1748c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 1749c9de560dSAlex Tomas BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group); 1750c9de560dSAlex Tomas BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start); 1751c9de560dSAlex Tomas ac->ac_found++; 1752c9de560dSAlex Tomas ac->ac_b_ex = ex; 1753c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1754c9de560dSAlex Tomas } 1755c9de560dSAlex Tomas ext4_unlock_group(ac->ac_sb, group); 1756e39e07fdSJing Zhang ext4_mb_unload_buddy(e4b); 1757c9de560dSAlex Tomas 1758c9de560dSAlex Tomas return 0; 1759c9de560dSAlex Tomas } 1760c9de560dSAlex Tomas 1761c9de560dSAlex Tomas /* 1762c9de560dSAlex Tomas * The routine scans buddy structures (not bitmap!) from given order 1763c9de560dSAlex Tomas * to max order and tries to find big enough chunk to satisfy the req 1764c9de560dSAlex Tomas */ 1765089ceeccSEric Sandeen static noinline_for_stack 1766089ceeccSEric Sandeen void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac, 1767c9de560dSAlex Tomas struct ext4_buddy *e4b) 1768c9de560dSAlex Tomas { 1769c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1770c9de560dSAlex Tomas struct ext4_group_info *grp = e4b->bd_info; 1771c9de560dSAlex Tomas void *buddy; 1772c9de560dSAlex Tomas int i; 1773c9de560dSAlex Tomas int k; 1774c9de560dSAlex Tomas int max; 1775c9de560dSAlex Tomas 1776c9de560dSAlex Tomas BUG_ON(ac->ac_2order <= 0); 1777c9de560dSAlex Tomas for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) { 1778c9de560dSAlex Tomas if (grp->bb_counters[i] == 0) 1779c9de560dSAlex Tomas continue; 1780c9de560dSAlex Tomas 1781c9de560dSAlex Tomas buddy = mb_find_buddy(e4b, i, &max); 1782c9de560dSAlex Tomas BUG_ON(buddy == NULL); 1783c9de560dSAlex Tomas 1784ffad0a44SAneesh Kumar K.V k = mb_find_next_zero_bit(buddy, max, 0); 1785c9de560dSAlex Tomas BUG_ON(k >= max); 1786c9de560dSAlex Tomas 1787c9de560dSAlex Tomas ac->ac_found++; 1788c9de560dSAlex Tomas 1789c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 1 << i; 1790c9de560dSAlex Tomas ac->ac_b_ex.fe_start = k << i; 1791c9de560dSAlex Tomas ac->ac_b_ex.fe_group = e4b->bd_group; 1792c9de560dSAlex Tomas 1793c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1794c9de560dSAlex Tomas 1795c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len != ac->ac_g_ex.fe_len); 1796c9de560dSAlex Tomas 1797c9de560dSAlex Tomas if (EXT4_SB(sb)->s_mb_stats) 1798c9de560dSAlex Tomas atomic_inc(&EXT4_SB(sb)->s_bal_2orders); 1799c9de560dSAlex Tomas 1800c9de560dSAlex Tomas break; 1801c9de560dSAlex Tomas } 1802c9de560dSAlex Tomas } 1803c9de560dSAlex Tomas 1804c9de560dSAlex Tomas /* 1805c9de560dSAlex Tomas * The routine scans the group and measures all found extents. 1806c9de560dSAlex Tomas * In order to optimize scanning, caller must pass number of 1807c9de560dSAlex Tomas * free blocks in the group, so the routine can know upper limit. 1808c9de560dSAlex Tomas */ 1809089ceeccSEric Sandeen static noinline_for_stack 1810089ceeccSEric Sandeen void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac, 1811c9de560dSAlex Tomas struct ext4_buddy *e4b) 1812c9de560dSAlex Tomas { 1813c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1814c9de560dSAlex Tomas void *bitmap = EXT4_MB_BITMAP(e4b); 1815c9de560dSAlex Tomas struct ext4_free_extent ex; 1816c9de560dSAlex Tomas int i; 1817c9de560dSAlex Tomas int free; 1818c9de560dSAlex Tomas 1819c9de560dSAlex Tomas free = e4b->bd_info->bb_free; 1820c9de560dSAlex Tomas BUG_ON(free <= 0); 1821c9de560dSAlex Tomas 1822c9de560dSAlex Tomas i = e4b->bd_info->bb_first_free; 1823c9de560dSAlex Tomas 1824c9de560dSAlex Tomas while (free && ac->ac_status == AC_STATUS_CONTINUE) { 1825ffad0a44SAneesh Kumar K.V i = mb_find_next_zero_bit(bitmap, 18267137d7a4STheodore Ts'o EXT4_CLUSTERS_PER_GROUP(sb), i); 18277137d7a4STheodore Ts'o if (i >= EXT4_CLUSTERS_PER_GROUP(sb)) { 182826346ff6SAneesh Kumar K.V /* 1829e56eb659SAneesh Kumar K.V * IF we have corrupt bitmap, we won't find any 183026346ff6SAneesh Kumar K.V * free blocks even though group info says we 183126346ff6SAneesh Kumar K.V * we have free blocks 183226346ff6SAneesh Kumar K.V */ 1833e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 1834*53accfa9STheodore Ts'o "%d free clusters as per " 1835fde4d95aSTheodore Ts'o "group info. But bitmap says 0", 183626346ff6SAneesh Kumar K.V free); 1837c9de560dSAlex Tomas break; 1838c9de560dSAlex Tomas } 1839c9de560dSAlex Tomas 1840c9de560dSAlex Tomas mb_find_extent(e4b, 0, i, ac->ac_g_ex.fe_len, &ex); 1841c9de560dSAlex Tomas BUG_ON(ex.fe_len <= 0); 184226346ff6SAneesh Kumar K.V if (free < ex.fe_len) { 1843e29136f8STheodore Ts'o ext4_grp_locked_error(sb, e4b->bd_group, 0, 0, 1844*53accfa9STheodore Ts'o "%d free clusters as per " 1845fde4d95aSTheodore Ts'o "group info. But got %d blocks", 184626346ff6SAneesh Kumar K.V free, ex.fe_len); 1847e56eb659SAneesh Kumar K.V /* 1848e56eb659SAneesh Kumar K.V * The number of free blocks differs. This mostly 1849e56eb659SAneesh Kumar K.V * indicate that the bitmap is corrupt. So exit 1850e56eb659SAneesh Kumar K.V * without claiming the space. 1851e56eb659SAneesh Kumar K.V */ 1852e56eb659SAneesh Kumar K.V break; 185326346ff6SAneesh Kumar K.V } 1854c9de560dSAlex Tomas 1855c9de560dSAlex Tomas ext4_mb_measure_extent(ac, &ex, e4b); 1856c9de560dSAlex Tomas 1857c9de560dSAlex Tomas i += ex.fe_len; 1858c9de560dSAlex Tomas free -= ex.fe_len; 1859c9de560dSAlex Tomas } 1860c9de560dSAlex Tomas 1861c9de560dSAlex Tomas ext4_mb_check_limits(ac, e4b, 1); 1862c9de560dSAlex Tomas } 1863c9de560dSAlex Tomas 1864c9de560dSAlex Tomas /* 1865c9de560dSAlex Tomas * This is a special case for storages like raid5 1866506bf2d8SEric Sandeen * we try to find stripe-aligned chunks for stripe-size-multiple requests 1867c9de560dSAlex Tomas */ 1868089ceeccSEric Sandeen static noinline_for_stack 1869089ceeccSEric Sandeen void ext4_mb_scan_aligned(struct ext4_allocation_context *ac, 1870c9de560dSAlex Tomas struct ext4_buddy *e4b) 1871c9de560dSAlex Tomas { 1872c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 1873c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 1874c9de560dSAlex Tomas void *bitmap = EXT4_MB_BITMAP(e4b); 1875c9de560dSAlex Tomas struct ext4_free_extent ex; 1876c9de560dSAlex Tomas ext4_fsblk_t first_group_block; 1877c9de560dSAlex Tomas ext4_fsblk_t a; 1878c9de560dSAlex Tomas ext4_grpblk_t i; 1879c9de560dSAlex Tomas int max; 1880c9de560dSAlex Tomas 1881c9de560dSAlex Tomas BUG_ON(sbi->s_stripe == 0); 1882c9de560dSAlex Tomas 1883c9de560dSAlex Tomas /* find first stripe-aligned block in group */ 18845661bd68SAkinobu Mita first_group_block = ext4_group_first_block_no(sb, e4b->bd_group); 18855661bd68SAkinobu Mita 1886c9de560dSAlex Tomas a = first_group_block + sbi->s_stripe - 1; 1887c9de560dSAlex Tomas do_div(a, sbi->s_stripe); 1888c9de560dSAlex Tomas i = (a * sbi->s_stripe) - first_group_block; 1889c9de560dSAlex Tomas 18907137d7a4STheodore Ts'o while (i < EXT4_CLUSTERS_PER_GROUP(sb)) { 1891c9de560dSAlex Tomas if (!mb_test_bit(i, bitmap)) { 1892c9de560dSAlex Tomas max = mb_find_extent(e4b, 0, i, sbi->s_stripe, &ex); 1893c9de560dSAlex Tomas if (max >= sbi->s_stripe) { 1894c9de560dSAlex Tomas ac->ac_found++; 1895c9de560dSAlex Tomas ac->ac_b_ex = ex; 1896c9de560dSAlex Tomas ext4_mb_use_best_found(ac, e4b); 1897c9de560dSAlex Tomas break; 1898c9de560dSAlex Tomas } 1899c9de560dSAlex Tomas } 1900c9de560dSAlex Tomas i += sbi->s_stripe; 1901c9de560dSAlex Tomas } 1902c9de560dSAlex Tomas } 1903c9de560dSAlex Tomas 19048a57d9d6SCurt Wohlgemuth /* This is now called BEFORE we load the buddy bitmap. */ 1905c9de560dSAlex Tomas static int ext4_mb_good_group(struct ext4_allocation_context *ac, 1906c9de560dSAlex Tomas ext4_group_t group, int cr) 1907c9de560dSAlex Tomas { 1908c9de560dSAlex Tomas unsigned free, fragments; 1909a4912123STheodore Ts'o int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb)); 1910c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group); 1911c9de560dSAlex Tomas 1912c9de560dSAlex Tomas BUG_ON(cr < 0 || cr >= 4); 19138a57d9d6SCurt Wohlgemuth 19148a57d9d6SCurt Wohlgemuth /* We only do this if the grp has never been initialized */ 19158a57d9d6SCurt Wohlgemuth if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 19168a57d9d6SCurt Wohlgemuth int ret = ext4_mb_init_group(ac->ac_sb, group); 19178a57d9d6SCurt Wohlgemuth if (ret) 19188a57d9d6SCurt Wohlgemuth return 0; 19198a57d9d6SCurt Wohlgemuth } 1920c9de560dSAlex Tomas 1921c9de560dSAlex Tomas free = grp->bb_free; 1922c9de560dSAlex Tomas fragments = grp->bb_fragments; 1923c9de560dSAlex Tomas if (free == 0) 1924c9de560dSAlex Tomas return 0; 1925c9de560dSAlex Tomas if (fragments == 0) 1926c9de560dSAlex Tomas return 0; 1927c9de560dSAlex Tomas 1928c9de560dSAlex Tomas switch (cr) { 1929c9de560dSAlex Tomas case 0: 1930c9de560dSAlex Tomas BUG_ON(ac->ac_2order == 0); 1931c9de560dSAlex Tomas 19328a57d9d6SCurt Wohlgemuth if (grp->bb_largest_free_order < ac->ac_2order) 19338a57d9d6SCurt Wohlgemuth return 0; 19348a57d9d6SCurt Wohlgemuth 1935a4912123STheodore Ts'o /* Avoid using the first bg of a flexgroup for data files */ 1936a4912123STheodore Ts'o if ((ac->ac_flags & EXT4_MB_HINT_DATA) && 1937a4912123STheodore Ts'o (flex_size >= EXT4_FLEX_SIZE_DIR_ALLOC_SCHEME) && 1938a4912123STheodore Ts'o ((group % flex_size) == 0)) 1939a4912123STheodore Ts'o return 0; 1940a4912123STheodore Ts'o 1941c9de560dSAlex Tomas return 1; 1942c9de560dSAlex Tomas case 1: 1943c9de560dSAlex Tomas if ((free / fragments) >= ac->ac_g_ex.fe_len) 1944c9de560dSAlex Tomas return 1; 1945c9de560dSAlex Tomas break; 1946c9de560dSAlex Tomas case 2: 1947c9de560dSAlex Tomas if (free >= ac->ac_g_ex.fe_len) 1948c9de560dSAlex Tomas return 1; 1949c9de560dSAlex Tomas break; 1950c9de560dSAlex Tomas case 3: 1951c9de560dSAlex Tomas return 1; 1952c9de560dSAlex Tomas default: 1953c9de560dSAlex Tomas BUG(); 1954c9de560dSAlex Tomas } 1955c9de560dSAlex Tomas 1956c9de560dSAlex Tomas return 0; 1957c9de560dSAlex Tomas } 1958c9de560dSAlex Tomas 19594ddfef7bSEric Sandeen static noinline_for_stack int 19604ddfef7bSEric Sandeen ext4_mb_regular_allocator(struct ext4_allocation_context *ac) 1961c9de560dSAlex Tomas { 19628df9675fSTheodore Ts'o ext4_group_t ngroups, group, i; 1963c9de560dSAlex Tomas int cr; 1964c9de560dSAlex Tomas int err = 0; 1965c9de560dSAlex Tomas struct ext4_sb_info *sbi; 1966c9de560dSAlex Tomas struct super_block *sb; 1967c9de560dSAlex Tomas struct ext4_buddy e4b; 1968c9de560dSAlex Tomas 1969c9de560dSAlex Tomas sb = ac->ac_sb; 1970c9de560dSAlex Tomas sbi = EXT4_SB(sb); 19718df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 1972fb0a387dSEric Sandeen /* non-extent files are limited to low blocks/groups */ 197312e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))) 1974fb0a387dSEric Sandeen ngroups = sbi->s_blockfile_groups; 1975fb0a387dSEric Sandeen 1976c9de560dSAlex Tomas BUG_ON(ac->ac_status == AC_STATUS_FOUND); 1977c9de560dSAlex Tomas 1978c9de560dSAlex Tomas /* first, try the goal */ 1979c9de560dSAlex Tomas err = ext4_mb_find_by_goal(ac, &e4b); 1980c9de560dSAlex Tomas if (err || ac->ac_status == AC_STATUS_FOUND) 1981c9de560dSAlex Tomas goto out; 1982c9de560dSAlex Tomas 1983c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 1984c9de560dSAlex Tomas goto out; 1985c9de560dSAlex Tomas 1986c9de560dSAlex Tomas /* 1987c9de560dSAlex Tomas * ac->ac2_order is set only if the fe_len is a power of 2 1988c9de560dSAlex Tomas * if ac2_order is set we also set criteria to 0 so that we 1989c9de560dSAlex Tomas * try exact allocation using buddy. 1990c9de560dSAlex Tomas */ 1991c9de560dSAlex Tomas i = fls(ac->ac_g_ex.fe_len); 1992c9de560dSAlex Tomas ac->ac_2order = 0; 1993c9de560dSAlex Tomas /* 1994c9de560dSAlex Tomas * We search using buddy data only if the order of the request 1995c9de560dSAlex Tomas * is greater than equal to the sbi_s_mb_order2_reqs 1996b713a5ecSTheodore Ts'o * You can tune it via /sys/fs/ext4/<partition>/mb_order2_req 1997c9de560dSAlex Tomas */ 1998c9de560dSAlex Tomas if (i >= sbi->s_mb_order2_reqs) { 1999c9de560dSAlex Tomas /* 2000c9de560dSAlex Tomas * This should tell if fe_len is exactly power of 2 2001c9de560dSAlex Tomas */ 2002c9de560dSAlex Tomas if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0) 2003c9de560dSAlex Tomas ac->ac_2order = i - 1; 2004c9de560dSAlex Tomas } 2005c9de560dSAlex Tomas 20064ba74d00STheodore Ts'o /* if stream allocation is enabled, use global goal */ 20074ba74d00STheodore Ts'o if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) { 2008c9de560dSAlex Tomas /* TBD: may be hot point */ 2009c9de560dSAlex Tomas spin_lock(&sbi->s_md_lock); 2010c9de560dSAlex Tomas ac->ac_g_ex.fe_group = sbi->s_mb_last_group; 2011c9de560dSAlex Tomas ac->ac_g_ex.fe_start = sbi->s_mb_last_start; 2012c9de560dSAlex Tomas spin_unlock(&sbi->s_md_lock); 2013c9de560dSAlex Tomas } 20144ba74d00STheodore Ts'o 2015c9de560dSAlex Tomas /* Let's just scan groups to find more-less suitable blocks */ 2016c9de560dSAlex Tomas cr = ac->ac_2order ? 0 : 1; 2017c9de560dSAlex Tomas /* 2018c9de560dSAlex Tomas * cr == 0 try to get exact allocation, 2019c9de560dSAlex Tomas * cr == 3 try to get anything 2020c9de560dSAlex Tomas */ 2021c9de560dSAlex Tomas repeat: 2022c9de560dSAlex Tomas for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) { 2023c9de560dSAlex Tomas ac->ac_criteria = cr; 2024ed8f9c75SAneesh Kumar K.V /* 2025ed8f9c75SAneesh Kumar K.V * searching for the right group start 2026ed8f9c75SAneesh Kumar K.V * from the goal value specified 2027ed8f9c75SAneesh Kumar K.V */ 2028ed8f9c75SAneesh Kumar K.V group = ac->ac_g_ex.fe_group; 2029ed8f9c75SAneesh Kumar K.V 20308df9675fSTheodore Ts'o for (i = 0; i < ngroups; group++, i++) { 20318df9675fSTheodore Ts'o if (group == ngroups) 2032c9de560dSAlex Tomas group = 0; 2033c9de560dSAlex Tomas 20348a57d9d6SCurt Wohlgemuth /* This now checks without needing the buddy page */ 20358a57d9d6SCurt Wohlgemuth if (!ext4_mb_good_group(ac, group, cr)) 2036c9de560dSAlex Tomas continue; 2037c9de560dSAlex Tomas 2038c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2039c9de560dSAlex Tomas if (err) 2040c9de560dSAlex Tomas goto out; 2041c9de560dSAlex Tomas 2042c9de560dSAlex Tomas ext4_lock_group(sb, group); 20438a57d9d6SCurt Wohlgemuth 20448a57d9d6SCurt Wohlgemuth /* 20458a57d9d6SCurt Wohlgemuth * We need to check again after locking the 20468a57d9d6SCurt Wohlgemuth * block group 20478a57d9d6SCurt Wohlgemuth */ 2048c9de560dSAlex Tomas if (!ext4_mb_good_group(ac, group, cr)) { 2049c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2050e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2051c9de560dSAlex Tomas continue; 2052c9de560dSAlex Tomas } 2053c9de560dSAlex Tomas 2054c9de560dSAlex Tomas ac->ac_groups_scanned++; 205575507efbSTheodore Ts'o if (cr == 0) 2056c9de560dSAlex Tomas ext4_mb_simple_scan_group(ac, &e4b); 2057506bf2d8SEric Sandeen else if (cr == 1 && sbi->s_stripe && 2058506bf2d8SEric Sandeen !(ac->ac_g_ex.fe_len % sbi->s_stripe)) 2059c9de560dSAlex Tomas ext4_mb_scan_aligned(ac, &e4b); 2060c9de560dSAlex Tomas else 2061c9de560dSAlex Tomas ext4_mb_complex_scan_group(ac, &e4b); 2062c9de560dSAlex Tomas 2063c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2064e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2065c9de560dSAlex Tomas 2066c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_CONTINUE) 2067c9de560dSAlex Tomas break; 2068c9de560dSAlex Tomas } 2069c9de560dSAlex Tomas } 2070c9de560dSAlex Tomas 2071c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND && 2072c9de560dSAlex Tomas !(ac->ac_flags & EXT4_MB_HINT_FIRST)) { 2073c9de560dSAlex Tomas /* 2074c9de560dSAlex Tomas * We've been searching too long. Let's try to allocate 2075c9de560dSAlex Tomas * the best chunk we've found so far 2076c9de560dSAlex Tomas */ 2077c9de560dSAlex Tomas 2078c9de560dSAlex Tomas ext4_mb_try_best_found(ac, &e4b); 2079c9de560dSAlex Tomas if (ac->ac_status != AC_STATUS_FOUND) { 2080c9de560dSAlex Tomas /* 2081c9de560dSAlex Tomas * Someone more lucky has already allocated it. 2082c9de560dSAlex Tomas * The only thing we can do is just take first 2083c9de560dSAlex Tomas * found block(s) 2084c9de560dSAlex Tomas printk(KERN_DEBUG "EXT4-fs: someone won our chunk\n"); 2085c9de560dSAlex Tomas */ 2086c9de560dSAlex Tomas ac->ac_b_ex.fe_group = 0; 2087c9de560dSAlex Tomas ac->ac_b_ex.fe_start = 0; 2088c9de560dSAlex Tomas ac->ac_b_ex.fe_len = 0; 2089c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 2090c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_FIRST; 2091c9de560dSAlex Tomas cr = 3; 2092c9de560dSAlex Tomas atomic_inc(&sbi->s_mb_lost_chunks); 2093c9de560dSAlex Tomas goto repeat; 2094c9de560dSAlex Tomas } 2095c9de560dSAlex Tomas } 2096c9de560dSAlex Tomas out: 2097c9de560dSAlex Tomas return err; 2098c9de560dSAlex Tomas } 2099c9de560dSAlex Tomas 2100c9de560dSAlex Tomas static void *ext4_mb_seq_groups_start(struct seq_file *seq, loff_t *pos) 2101c9de560dSAlex Tomas { 2102c9de560dSAlex Tomas struct super_block *sb = seq->private; 2103c9de560dSAlex Tomas ext4_group_t group; 2104c9de560dSAlex Tomas 21058df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2106c9de560dSAlex Tomas return NULL; 2107c9de560dSAlex Tomas group = *pos + 1; 2108a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2109c9de560dSAlex Tomas } 2110c9de560dSAlex Tomas 2111c9de560dSAlex Tomas static void *ext4_mb_seq_groups_next(struct seq_file *seq, void *v, loff_t *pos) 2112c9de560dSAlex Tomas { 2113c9de560dSAlex Tomas struct super_block *sb = seq->private; 2114c9de560dSAlex Tomas ext4_group_t group; 2115c9de560dSAlex Tomas 2116c9de560dSAlex Tomas ++*pos; 21178df9675fSTheodore Ts'o if (*pos < 0 || *pos >= ext4_get_groups_count(sb)) 2118c9de560dSAlex Tomas return NULL; 2119c9de560dSAlex Tomas group = *pos + 1; 2120a9df9a49STheodore Ts'o return (void *) ((unsigned long) group); 2121c9de560dSAlex Tomas } 2122c9de560dSAlex Tomas 2123c9de560dSAlex Tomas static int ext4_mb_seq_groups_show(struct seq_file *seq, void *v) 2124c9de560dSAlex Tomas { 2125c9de560dSAlex Tomas struct super_block *sb = seq->private; 2126a9df9a49STheodore Ts'o ext4_group_t group = (ext4_group_t) ((unsigned long) v); 2127c9de560dSAlex Tomas int i; 2128c9de560dSAlex Tomas int err; 2129c9de560dSAlex Tomas struct ext4_buddy e4b; 2130c9de560dSAlex Tomas struct sg { 2131c9de560dSAlex Tomas struct ext4_group_info info; 2132a36b4498SEric Sandeen ext4_grpblk_t counters[16]; 2133c9de560dSAlex Tomas } sg; 2134c9de560dSAlex Tomas 2135c9de560dSAlex Tomas group--; 2136c9de560dSAlex Tomas if (group == 0) 2137c9de560dSAlex Tomas seq_printf(seq, "#%-5s: %-5s %-5s %-5s " 2138c9de560dSAlex Tomas "[ %-5s %-5s %-5s %-5s %-5s %-5s %-5s " 2139c9de560dSAlex Tomas "%-5s %-5s %-5s %-5s %-5s %-5s %-5s ]\n", 2140c9de560dSAlex Tomas "group", "free", "frags", "first", 2141c9de560dSAlex Tomas "2^0", "2^1", "2^2", "2^3", "2^4", "2^5", "2^6", 2142c9de560dSAlex Tomas "2^7", "2^8", "2^9", "2^10", "2^11", "2^12", "2^13"); 2143c9de560dSAlex Tomas 2144c9de560dSAlex Tomas i = (sb->s_blocksize_bits + 2) * sizeof(sg.info.bb_counters[0]) + 2145c9de560dSAlex Tomas sizeof(struct ext4_group_info); 2146c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 2147c9de560dSAlex Tomas if (err) { 2148a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: I/O error\n", group); 2149c9de560dSAlex Tomas return 0; 2150c9de560dSAlex Tomas } 2151c9de560dSAlex Tomas ext4_lock_group(sb, group); 2152c9de560dSAlex Tomas memcpy(&sg, ext4_get_group_info(sb, group), i); 2153c9de560dSAlex Tomas ext4_unlock_group(sb, group); 2154e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 2155c9de560dSAlex Tomas 2156a9df9a49STheodore Ts'o seq_printf(seq, "#%-5u: %-5u %-5u %-5u [", group, sg.info.bb_free, 2157c9de560dSAlex Tomas sg.info.bb_fragments, sg.info.bb_first_free); 2158c9de560dSAlex Tomas for (i = 0; i <= 13; i++) 2159c9de560dSAlex Tomas seq_printf(seq, " %-5u", i <= sb->s_blocksize_bits + 1 ? 2160c9de560dSAlex Tomas sg.info.bb_counters[i] : 0); 2161c9de560dSAlex Tomas seq_printf(seq, " ]\n"); 2162c9de560dSAlex Tomas 2163c9de560dSAlex Tomas return 0; 2164c9de560dSAlex Tomas } 2165c9de560dSAlex Tomas 2166c9de560dSAlex Tomas static void ext4_mb_seq_groups_stop(struct seq_file *seq, void *v) 2167c9de560dSAlex Tomas { 2168c9de560dSAlex Tomas } 2169c9de560dSAlex Tomas 21707f1346a9STobias Klauser static const struct seq_operations ext4_mb_seq_groups_ops = { 2171c9de560dSAlex Tomas .start = ext4_mb_seq_groups_start, 2172c9de560dSAlex Tomas .next = ext4_mb_seq_groups_next, 2173c9de560dSAlex Tomas .stop = ext4_mb_seq_groups_stop, 2174c9de560dSAlex Tomas .show = ext4_mb_seq_groups_show, 2175c9de560dSAlex Tomas }; 2176c9de560dSAlex Tomas 2177c9de560dSAlex Tomas static int ext4_mb_seq_groups_open(struct inode *inode, struct file *file) 2178c9de560dSAlex Tomas { 2179c9de560dSAlex Tomas struct super_block *sb = PDE(inode)->data; 2180c9de560dSAlex Tomas int rc; 2181c9de560dSAlex Tomas 2182c9de560dSAlex Tomas rc = seq_open(file, &ext4_mb_seq_groups_ops); 2183c9de560dSAlex Tomas if (rc == 0) { 2184a271fe85SJoe Perches struct seq_file *m = file->private_data; 2185c9de560dSAlex Tomas m->private = sb; 2186c9de560dSAlex Tomas } 2187c9de560dSAlex Tomas return rc; 2188c9de560dSAlex Tomas 2189c9de560dSAlex Tomas } 2190c9de560dSAlex Tomas 21917f1346a9STobias Klauser static const struct file_operations ext4_mb_seq_groups_fops = { 2192c9de560dSAlex Tomas .owner = THIS_MODULE, 2193c9de560dSAlex Tomas .open = ext4_mb_seq_groups_open, 2194c9de560dSAlex Tomas .read = seq_read, 2195c9de560dSAlex Tomas .llseek = seq_lseek, 2196c9de560dSAlex Tomas .release = seq_release, 2197c9de560dSAlex Tomas }; 2198c9de560dSAlex Tomas 2199fb1813f4SCurt Wohlgemuth static struct kmem_cache *get_groupinfo_cache(int blocksize_bits) 2200fb1813f4SCurt Wohlgemuth { 2201fb1813f4SCurt Wohlgemuth int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 2202fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = ext4_groupinfo_caches[cache_index]; 2203fb1813f4SCurt Wohlgemuth 2204fb1813f4SCurt Wohlgemuth BUG_ON(!cachep); 2205fb1813f4SCurt Wohlgemuth return cachep; 2206fb1813f4SCurt Wohlgemuth } 22075f21b0e6SFrederic Bohe 22085f21b0e6SFrederic Bohe /* Create and initialize ext4_group_info data for the given group. */ 2209920313a7SAneesh Kumar K.V int ext4_mb_add_groupinfo(struct super_block *sb, ext4_group_t group, 22105f21b0e6SFrederic Bohe struct ext4_group_desc *desc) 22115f21b0e6SFrederic Bohe { 2212fb1813f4SCurt Wohlgemuth int i; 22135f21b0e6SFrederic Bohe int metalen = 0; 22145f21b0e6SFrederic Bohe struct ext4_sb_info *sbi = EXT4_SB(sb); 22155f21b0e6SFrederic Bohe struct ext4_group_info **meta_group_info; 2216fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 22175f21b0e6SFrederic Bohe 22185f21b0e6SFrederic Bohe /* 22195f21b0e6SFrederic Bohe * First check if this group is the first of a reserved block. 22205f21b0e6SFrederic Bohe * If it's true, we have to allocate a new table of pointers 22215f21b0e6SFrederic Bohe * to ext4_group_info structures 22225f21b0e6SFrederic Bohe */ 22235f21b0e6SFrederic Bohe if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 22245f21b0e6SFrederic Bohe metalen = sizeof(*meta_group_info) << 22255f21b0e6SFrederic Bohe EXT4_DESC_PER_BLOCK_BITS(sb); 22265f21b0e6SFrederic Bohe meta_group_info = kmalloc(metalen, GFP_KERNEL); 22275f21b0e6SFrederic Bohe if (meta_group_info == NULL) { 22289d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "EXT4-fs: can't allocate mem " 22299d8b9ec4STheodore Ts'o "for a buddy group"); 22305f21b0e6SFrederic Bohe goto exit_meta_group_info; 22315f21b0e6SFrederic Bohe } 22325f21b0e6SFrederic Bohe sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = 22335f21b0e6SFrederic Bohe meta_group_info; 22345f21b0e6SFrederic Bohe } 22355f21b0e6SFrederic Bohe 22365f21b0e6SFrederic Bohe meta_group_info = 22375f21b0e6SFrederic Bohe sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]; 22385f21b0e6SFrederic Bohe i = group & (EXT4_DESC_PER_BLOCK(sb) - 1); 22395f21b0e6SFrederic Bohe 2240fb1813f4SCurt Wohlgemuth meta_group_info[i] = kmem_cache_alloc(cachep, GFP_KERNEL); 22415f21b0e6SFrederic Bohe if (meta_group_info[i] == NULL) { 22429d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "EXT4-fs: can't allocate buddy mem"); 22435f21b0e6SFrederic Bohe goto exit_group_info; 22445f21b0e6SFrederic Bohe } 2245fb1813f4SCurt Wohlgemuth memset(meta_group_info[i], 0, kmem_cache_size(cachep)); 22465f21b0e6SFrederic Bohe set_bit(EXT4_GROUP_INFO_NEED_INIT_BIT, 22475f21b0e6SFrederic Bohe &(meta_group_info[i]->bb_state)); 22485f21b0e6SFrederic Bohe 22495f21b0e6SFrederic Bohe /* 22505f21b0e6SFrederic Bohe * initialize bb_free to be able to skip 22515f21b0e6SFrederic Bohe * empty groups without initialization 22525f21b0e6SFrederic Bohe */ 22535f21b0e6SFrederic Bohe if (desc->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 22545f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 22555f21b0e6SFrederic Bohe ext4_free_blocks_after_init(sb, group, desc); 22565f21b0e6SFrederic Bohe } else { 22575f21b0e6SFrederic Bohe meta_group_info[i]->bb_free = 2258560671a0SAneesh Kumar K.V ext4_free_blks_count(sb, desc); 22595f21b0e6SFrederic Bohe } 22605f21b0e6SFrederic Bohe 22615f21b0e6SFrederic Bohe INIT_LIST_HEAD(&meta_group_info[i]->bb_prealloc_list); 2262920313a7SAneesh Kumar K.V init_rwsem(&meta_group_info[i]->alloc_sem); 226364e290ecSVenkatesh Pallipadi meta_group_info[i]->bb_free_root = RB_ROOT; 22648a57d9d6SCurt Wohlgemuth meta_group_info[i]->bb_largest_free_order = -1; /* uninit */ 22655f21b0e6SFrederic Bohe 22665f21b0e6SFrederic Bohe #ifdef DOUBLE_CHECK 22675f21b0e6SFrederic Bohe { 22685f21b0e6SFrederic Bohe struct buffer_head *bh; 22695f21b0e6SFrederic Bohe meta_group_info[i]->bb_bitmap = 22705f21b0e6SFrederic Bohe kmalloc(sb->s_blocksize, GFP_KERNEL); 22715f21b0e6SFrederic Bohe BUG_ON(meta_group_info[i]->bb_bitmap == NULL); 22725f21b0e6SFrederic Bohe bh = ext4_read_block_bitmap(sb, group); 22735f21b0e6SFrederic Bohe BUG_ON(bh == NULL); 22745f21b0e6SFrederic Bohe memcpy(meta_group_info[i]->bb_bitmap, bh->b_data, 22755f21b0e6SFrederic Bohe sb->s_blocksize); 22765f21b0e6SFrederic Bohe put_bh(bh); 22775f21b0e6SFrederic Bohe } 22785f21b0e6SFrederic Bohe #endif 22795f21b0e6SFrederic Bohe 22805f21b0e6SFrederic Bohe return 0; 22815f21b0e6SFrederic Bohe 22825f21b0e6SFrederic Bohe exit_group_info: 22835f21b0e6SFrederic Bohe /* If a meta_group_info table has been allocated, release it now */ 2284caaf7a29STao Ma if (group % EXT4_DESC_PER_BLOCK(sb) == 0) { 22855f21b0e6SFrederic Bohe kfree(sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)]); 2286caaf7a29STao Ma sbi->s_group_info[group >> EXT4_DESC_PER_BLOCK_BITS(sb)] = NULL; 2287caaf7a29STao Ma } 22885f21b0e6SFrederic Bohe exit_meta_group_info: 22895f21b0e6SFrederic Bohe return -ENOMEM; 22905f21b0e6SFrederic Bohe } /* ext4_mb_add_groupinfo */ 22915f21b0e6SFrederic Bohe 2292c9de560dSAlex Tomas static int ext4_mb_init_backend(struct super_block *sb) 2293c9de560dSAlex Tomas { 22948df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2295c9de560dSAlex Tomas ext4_group_t i; 2296c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 22975f21b0e6SFrederic Bohe struct ext4_super_block *es = sbi->s_es; 22985f21b0e6SFrederic Bohe int num_meta_group_infos; 22995f21b0e6SFrederic Bohe int num_meta_group_infos_max; 23005f21b0e6SFrederic Bohe int array_size; 23015f21b0e6SFrederic Bohe struct ext4_group_desc *desc; 2302fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep; 2303c9de560dSAlex Tomas 23045f21b0e6SFrederic Bohe /* This is the number of blocks used by GDT */ 23058df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + EXT4_DESC_PER_BLOCK(sb) - 23065f21b0e6SFrederic Bohe 1) >> EXT4_DESC_PER_BLOCK_BITS(sb); 23075f21b0e6SFrederic Bohe 23085f21b0e6SFrederic Bohe /* 23095f21b0e6SFrederic Bohe * This is the total number of blocks used by GDT including 23105f21b0e6SFrederic Bohe * the number of reserved blocks for GDT. 23115f21b0e6SFrederic Bohe * The s_group_info array is allocated with this value 23125f21b0e6SFrederic Bohe * to allow a clean online resize without a complex 23135f21b0e6SFrederic Bohe * manipulation of pointer. 23145f21b0e6SFrederic Bohe * The drawback is the unused memory when no resize 23155f21b0e6SFrederic Bohe * occurs but it's very low in terms of pages 23165f21b0e6SFrederic Bohe * (see comments below) 23175f21b0e6SFrederic Bohe * Need to handle this properly when META_BG resizing is allowed 23185f21b0e6SFrederic Bohe */ 23195f21b0e6SFrederic Bohe num_meta_group_infos_max = num_meta_group_infos + 23205f21b0e6SFrederic Bohe le16_to_cpu(es->s_reserved_gdt_blocks); 23215f21b0e6SFrederic Bohe 23225f21b0e6SFrederic Bohe /* 23235f21b0e6SFrederic Bohe * array_size is the size of s_group_info array. We round it 23245f21b0e6SFrederic Bohe * to the next power of two because this approximation is done 23255f21b0e6SFrederic Bohe * internally by kmalloc so we can have some more memory 23265f21b0e6SFrederic Bohe * for free here (e.g. may be used for META_BG resize). 23275f21b0e6SFrederic Bohe */ 23285f21b0e6SFrederic Bohe array_size = 1; 23295f21b0e6SFrederic Bohe while (array_size < sizeof(*sbi->s_group_info) * 23305f21b0e6SFrederic Bohe num_meta_group_infos_max) 23315f21b0e6SFrederic Bohe array_size = array_size << 1; 2332c9de560dSAlex Tomas /* An 8TB filesystem with 64-bit pointers requires a 4096 byte 2333c9de560dSAlex Tomas * kmalloc. A 128kb malloc should suffice for a 256TB filesystem. 2334c9de560dSAlex Tomas * So a two level scheme suffices for now. */ 2335f18a5f21STheodore Ts'o sbi->s_group_info = ext4_kvzalloc(array_size, GFP_KERNEL); 2336c9de560dSAlex Tomas if (sbi->s_group_info == NULL) { 23379d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't allocate buddy meta group"); 2338c9de560dSAlex Tomas return -ENOMEM; 2339c9de560dSAlex Tomas } 2340c9de560dSAlex Tomas sbi->s_buddy_cache = new_inode(sb); 2341c9de560dSAlex Tomas if (sbi->s_buddy_cache == NULL) { 23429d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't get new inode"); 2343c9de560dSAlex Tomas goto err_freesgi; 2344c9de560dSAlex Tomas } 234548e6061bSYu Jian /* To avoid potentially colliding with an valid on-disk inode number, 234648e6061bSYu Jian * use EXT4_BAD_INO for the buddy cache inode number. This inode is 234748e6061bSYu Jian * not in the inode hash, so it should never be found by iget(), but 234848e6061bSYu Jian * this will avoid confusion if it ever shows up during debugging. */ 234948e6061bSYu Jian sbi->s_buddy_cache->i_ino = EXT4_BAD_INO; 2350c9de560dSAlex Tomas EXT4_I(sbi->s_buddy_cache)->i_disksize = 0; 23518df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 2352c9de560dSAlex Tomas desc = ext4_get_group_desc(sb, i, NULL); 2353c9de560dSAlex Tomas if (desc == NULL) { 23549d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, "can't read descriptor %u", i); 2355c9de560dSAlex Tomas goto err_freebuddy; 2356c9de560dSAlex Tomas } 23575f21b0e6SFrederic Bohe if (ext4_mb_add_groupinfo(sb, i, desc) != 0) 23585f21b0e6SFrederic Bohe goto err_freebuddy; 2359c9de560dSAlex Tomas } 2360c9de560dSAlex Tomas 2361c9de560dSAlex Tomas return 0; 2362c9de560dSAlex Tomas 2363c9de560dSAlex Tomas err_freebuddy: 2364fb1813f4SCurt Wohlgemuth cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2365f1fa3342SRoel Kluin while (i-- > 0) 2366fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, ext4_get_group_info(sb, i)); 2367c9de560dSAlex Tomas i = num_meta_group_infos; 2368f1fa3342SRoel Kluin while (i-- > 0) 2369c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2370c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2371c9de560dSAlex Tomas err_freesgi: 2372f18a5f21STheodore Ts'o ext4_kvfree(sbi->s_group_info); 2373c9de560dSAlex Tomas return -ENOMEM; 2374c9de560dSAlex Tomas } 2375c9de560dSAlex Tomas 23762892c15dSEric Sandeen static void ext4_groupinfo_destroy_slabs(void) 23772892c15dSEric Sandeen { 23782892c15dSEric Sandeen int i; 23792892c15dSEric Sandeen 23802892c15dSEric Sandeen for (i = 0; i < NR_GRPINFO_CACHES; i++) { 23812892c15dSEric Sandeen if (ext4_groupinfo_caches[i]) 23822892c15dSEric Sandeen kmem_cache_destroy(ext4_groupinfo_caches[i]); 23832892c15dSEric Sandeen ext4_groupinfo_caches[i] = NULL; 23842892c15dSEric Sandeen } 23852892c15dSEric Sandeen } 23862892c15dSEric Sandeen 23872892c15dSEric Sandeen static int ext4_groupinfo_create_slab(size_t size) 23882892c15dSEric Sandeen { 23892892c15dSEric Sandeen static DEFINE_MUTEX(ext4_grpinfo_slab_create_mutex); 23902892c15dSEric Sandeen int slab_size; 23912892c15dSEric Sandeen int blocksize_bits = order_base_2(size); 23922892c15dSEric Sandeen int cache_index = blocksize_bits - EXT4_MIN_BLOCK_LOG_SIZE; 23932892c15dSEric Sandeen struct kmem_cache *cachep; 23942892c15dSEric Sandeen 23952892c15dSEric Sandeen if (cache_index >= NR_GRPINFO_CACHES) 23962892c15dSEric Sandeen return -EINVAL; 23972892c15dSEric Sandeen 23982892c15dSEric Sandeen if (unlikely(cache_index < 0)) 23992892c15dSEric Sandeen cache_index = 0; 24002892c15dSEric Sandeen 24012892c15dSEric Sandeen mutex_lock(&ext4_grpinfo_slab_create_mutex); 24022892c15dSEric Sandeen if (ext4_groupinfo_caches[cache_index]) { 24032892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 24042892c15dSEric Sandeen return 0; /* Already created */ 24052892c15dSEric Sandeen } 24062892c15dSEric Sandeen 24072892c15dSEric Sandeen slab_size = offsetof(struct ext4_group_info, 24082892c15dSEric Sandeen bb_counters[blocksize_bits + 2]); 24092892c15dSEric Sandeen 24102892c15dSEric Sandeen cachep = kmem_cache_create(ext4_groupinfo_slab_names[cache_index], 24112892c15dSEric Sandeen slab_size, 0, SLAB_RECLAIM_ACCOUNT, 24122892c15dSEric Sandeen NULL); 24132892c15dSEric Sandeen 2414823ba01fSTao Ma ext4_groupinfo_caches[cache_index] = cachep; 2415823ba01fSTao Ma 24162892c15dSEric Sandeen mutex_unlock(&ext4_grpinfo_slab_create_mutex); 24172892c15dSEric Sandeen if (!cachep) { 24189d8b9ec4STheodore Ts'o printk(KERN_EMERG 24199d8b9ec4STheodore Ts'o "EXT4-fs: no memory for groupinfo slab cache\n"); 24202892c15dSEric Sandeen return -ENOMEM; 24212892c15dSEric Sandeen } 24222892c15dSEric Sandeen 24232892c15dSEric Sandeen return 0; 24242892c15dSEric Sandeen } 24252892c15dSEric Sandeen 2426c9de560dSAlex Tomas int ext4_mb_init(struct super_block *sb, int needs_recovery) 2427c9de560dSAlex Tomas { 2428c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 24296be2ded1SAneesh Kumar K.V unsigned i, j; 2430c9de560dSAlex Tomas unsigned offset; 2431c9de560dSAlex Tomas unsigned max; 243274767c5aSShen Feng int ret; 2433c9de560dSAlex Tomas 24341927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_offsets); 2435c9de560dSAlex Tomas 2436c9de560dSAlex Tomas sbi->s_mb_offsets = kmalloc(i, GFP_KERNEL); 2437c9de560dSAlex Tomas if (sbi->s_mb_offsets == NULL) { 2438fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2439fb1813f4SCurt Wohlgemuth goto out; 2440c9de560dSAlex Tomas } 2441ff7ef329SYasunori Goto 24421927805eSEric Sandeen i = (sb->s_blocksize_bits + 2) * sizeof(*sbi->s_mb_maxs); 2443c9de560dSAlex Tomas sbi->s_mb_maxs = kmalloc(i, GFP_KERNEL); 2444c9de560dSAlex Tomas if (sbi->s_mb_maxs == NULL) { 2445fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2446fb1813f4SCurt Wohlgemuth goto out; 2447fb1813f4SCurt Wohlgemuth } 2448fb1813f4SCurt Wohlgemuth 24492892c15dSEric Sandeen ret = ext4_groupinfo_create_slab(sb->s_blocksize); 24502892c15dSEric Sandeen if (ret < 0) 2451fb1813f4SCurt Wohlgemuth goto out; 2452c9de560dSAlex Tomas 2453c9de560dSAlex Tomas /* order 0 is regular bitmap */ 2454c9de560dSAlex Tomas sbi->s_mb_maxs[0] = sb->s_blocksize << 3; 2455c9de560dSAlex Tomas sbi->s_mb_offsets[0] = 0; 2456c9de560dSAlex Tomas 2457c9de560dSAlex Tomas i = 1; 2458c9de560dSAlex Tomas offset = 0; 2459c9de560dSAlex Tomas max = sb->s_blocksize << 2; 2460c9de560dSAlex Tomas do { 2461c9de560dSAlex Tomas sbi->s_mb_offsets[i] = offset; 2462c9de560dSAlex Tomas sbi->s_mb_maxs[i] = max; 2463c9de560dSAlex Tomas offset += 1 << (sb->s_blocksize_bits - i); 2464c9de560dSAlex Tomas max = max >> 1; 2465c9de560dSAlex Tomas i++; 2466c9de560dSAlex Tomas } while (i <= sb->s_blocksize_bits + 1); 2467c9de560dSAlex Tomas 2468c9de560dSAlex Tomas spin_lock_init(&sbi->s_md_lock); 2469c9de560dSAlex Tomas spin_lock_init(&sbi->s_bal_lock); 2470c9de560dSAlex Tomas 2471c9de560dSAlex Tomas sbi->s_mb_max_to_scan = MB_DEFAULT_MAX_TO_SCAN; 2472c9de560dSAlex Tomas sbi->s_mb_min_to_scan = MB_DEFAULT_MIN_TO_SCAN; 2473c9de560dSAlex Tomas sbi->s_mb_stats = MB_DEFAULT_STATS; 2474c9de560dSAlex Tomas sbi->s_mb_stream_request = MB_DEFAULT_STREAM_THRESHOLD; 2475c9de560dSAlex Tomas sbi->s_mb_order2_reqs = MB_DEFAULT_ORDER2_REQS; 2476c9de560dSAlex Tomas sbi->s_mb_group_prealloc = MB_DEFAULT_GROUP_PREALLOC; 2477d7a1fee1SDan Ehrenberg /* 2478d7a1fee1SDan Ehrenberg * If there is a s_stripe > 1, then we set the s_mb_group_prealloc 2479d7a1fee1SDan Ehrenberg * to the lowest multiple of s_stripe which is bigger than 2480d7a1fee1SDan Ehrenberg * the s_mb_group_prealloc as determined above. We want 2481d7a1fee1SDan Ehrenberg * the preallocation size to be an exact multiple of the 2482d7a1fee1SDan Ehrenberg * RAID stripe size so that preallocations don't fragment 2483d7a1fee1SDan Ehrenberg * the stripes. 2484d7a1fee1SDan Ehrenberg */ 2485d7a1fee1SDan Ehrenberg if (sbi->s_stripe > 1) { 2486d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc = roundup( 2487d7a1fee1SDan Ehrenberg sbi->s_mb_group_prealloc, sbi->s_stripe); 2488d7a1fee1SDan Ehrenberg } 2489c9de560dSAlex Tomas 2490730c213cSEric Sandeen sbi->s_locality_groups = alloc_percpu(struct ext4_locality_group); 2491c9de560dSAlex Tomas if (sbi->s_locality_groups == NULL) { 2492fb1813f4SCurt Wohlgemuth ret = -ENOMEM; 2493fb1813f4SCurt Wohlgemuth goto out; 2494c9de560dSAlex Tomas } 2495730c213cSEric Sandeen for_each_possible_cpu(i) { 2496c9de560dSAlex Tomas struct ext4_locality_group *lg; 2497730c213cSEric Sandeen lg = per_cpu_ptr(sbi->s_locality_groups, i); 2498c9de560dSAlex Tomas mutex_init(&lg->lg_mutex); 24996be2ded1SAneesh Kumar K.V for (j = 0; j < PREALLOC_TB_SIZE; j++) 25006be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&lg->lg_prealloc_list[j]); 2501c9de560dSAlex Tomas spin_lock_init(&lg->lg_prealloc_lock); 2502c9de560dSAlex Tomas } 2503c9de560dSAlex Tomas 250479a77c5aSYu Jian /* init file for buddy data */ 250579a77c5aSYu Jian ret = ext4_mb_init_backend(sb); 250679a77c5aSYu Jian if (ret != 0) { 250779a77c5aSYu Jian goto out; 250879a77c5aSYu Jian } 250979a77c5aSYu Jian 2510296c355cSTheodore Ts'o if (sbi->s_proc) 2511296c355cSTheodore Ts'o proc_create_data("mb_groups", S_IRUGO, sbi->s_proc, 2512296c355cSTheodore Ts'o &ext4_mb_seq_groups_fops, sb); 2513c9de560dSAlex Tomas 25140390131bSFrank Mayhar if (sbi->s_journal) 25153e624fc7STheodore Ts'o sbi->s_journal->j_commit_callback = release_blocks_on_commit; 2516fb1813f4SCurt Wohlgemuth out: 2517fb1813f4SCurt Wohlgemuth if (ret) { 2518fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_offsets); 2519fb1813f4SCurt Wohlgemuth kfree(sbi->s_mb_maxs); 2520fb1813f4SCurt Wohlgemuth } 2521fb1813f4SCurt Wohlgemuth return ret; 2522c9de560dSAlex Tomas } 2523c9de560dSAlex Tomas 2524955ce5f5SAneesh Kumar K.V /* need to called with the ext4 group lock held */ 2525c9de560dSAlex Tomas static void ext4_mb_cleanup_pa(struct ext4_group_info *grp) 2526c9de560dSAlex Tomas { 2527c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 2528c9de560dSAlex Tomas struct list_head *cur, *tmp; 2529c9de560dSAlex Tomas int count = 0; 2530c9de560dSAlex Tomas 2531c9de560dSAlex Tomas list_for_each_safe(cur, tmp, &grp->bb_prealloc_list) { 2532c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 2533c9de560dSAlex Tomas list_del(&pa->pa_group_list); 2534c9de560dSAlex Tomas count++; 2535688f05a0SAneesh Kumar K.V kmem_cache_free(ext4_pspace_cachep, pa); 2536c9de560dSAlex Tomas } 2537c9de560dSAlex Tomas if (count) 25386ba495e9STheodore Ts'o mb_debug(1, "mballoc: %u PAs left\n", count); 2539c9de560dSAlex Tomas 2540c9de560dSAlex Tomas } 2541c9de560dSAlex Tomas 2542c9de560dSAlex Tomas int ext4_mb_release(struct super_block *sb) 2543c9de560dSAlex Tomas { 25448df9675fSTheodore Ts'o ext4_group_t ngroups = ext4_get_groups_count(sb); 2545c9de560dSAlex Tomas ext4_group_t i; 2546c9de560dSAlex Tomas int num_meta_group_infos; 2547c9de560dSAlex Tomas struct ext4_group_info *grinfo; 2548c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 2549fb1813f4SCurt Wohlgemuth struct kmem_cache *cachep = get_groupinfo_cache(sb->s_blocksize_bits); 2550c9de560dSAlex Tomas 2551c9de560dSAlex Tomas if (sbi->s_group_info) { 25528df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 2553c9de560dSAlex Tomas grinfo = ext4_get_group_info(sb, i); 2554c9de560dSAlex Tomas #ifdef DOUBLE_CHECK 2555c9de560dSAlex Tomas kfree(grinfo->bb_bitmap); 2556c9de560dSAlex Tomas #endif 2557c9de560dSAlex Tomas ext4_lock_group(sb, i); 2558c9de560dSAlex Tomas ext4_mb_cleanup_pa(grinfo); 2559c9de560dSAlex Tomas ext4_unlock_group(sb, i); 2560fb1813f4SCurt Wohlgemuth kmem_cache_free(cachep, grinfo); 2561c9de560dSAlex Tomas } 25628df9675fSTheodore Ts'o num_meta_group_infos = (ngroups + 2563c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK(sb) - 1) >> 2564c9de560dSAlex Tomas EXT4_DESC_PER_BLOCK_BITS(sb); 2565c9de560dSAlex Tomas for (i = 0; i < num_meta_group_infos; i++) 2566c9de560dSAlex Tomas kfree(sbi->s_group_info[i]); 2567f18a5f21STheodore Ts'o ext4_kvfree(sbi->s_group_info); 2568c9de560dSAlex Tomas } 2569c9de560dSAlex Tomas kfree(sbi->s_mb_offsets); 2570c9de560dSAlex Tomas kfree(sbi->s_mb_maxs); 2571c9de560dSAlex Tomas if (sbi->s_buddy_cache) 2572c9de560dSAlex Tomas iput(sbi->s_buddy_cache); 2573c9de560dSAlex Tomas if (sbi->s_mb_stats) { 25749d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 25759d8b9ec4STheodore Ts'o "mballoc: %u blocks %u reqs (%u success)", 2576c9de560dSAlex Tomas atomic_read(&sbi->s_bal_allocated), 2577c9de560dSAlex Tomas atomic_read(&sbi->s_bal_reqs), 2578c9de560dSAlex Tomas atomic_read(&sbi->s_bal_success)); 25799d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 25809d8b9ec4STheodore Ts'o "mballoc: %u extents scanned, %u goal hits, " 25819d8b9ec4STheodore Ts'o "%u 2^N hits, %u breaks, %u lost", 2582c9de560dSAlex Tomas atomic_read(&sbi->s_bal_ex_scanned), 2583c9de560dSAlex Tomas atomic_read(&sbi->s_bal_goals), 2584c9de560dSAlex Tomas atomic_read(&sbi->s_bal_2orders), 2585c9de560dSAlex Tomas atomic_read(&sbi->s_bal_breaks), 2586c9de560dSAlex Tomas atomic_read(&sbi->s_mb_lost_chunks)); 25879d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 25889d8b9ec4STheodore Ts'o "mballoc: %lu generated and it took %Lu", 2589ced156e4STao Ma sbi->s_mb_buddies_generated, 2590c9de560dSAlex Tomas sbi->s_mb_generation_time); 25919d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_INFO, 25929d8b9ec4STheodore Ts'o "mballoc: %u preallocated, %u discarded", 2593c9de560dSAlex Tomas atomic_read(&sbi->s_mb_preallocated), 2594c9de560dSAlex Tomas atomic_read(&sbi->s_mb_discarded)); 2595c9de560dSAlex Tomas } 2596c9de560dSAlex Tomas 2597730c213cSEric Sandeen free_percpu(sbi->s_locality_groups); 2598296c355cSTheodore Ts'o if (sbi->s_proc) 2599296c355cSTheodore Ts'o remove_proc_entry("mb_groups", sbi->s_proc); 2600c9de560dSAlex Tomas 2601c9de560dSAlex Tomas return 0; 2602c9de560dSAlex Tomas } 2603c9de560dSAlex Tomas 260477ca6cdfSLukas Czerner static inline int ext4_issue_discard(struct super_block *sb, 26055c521830SJiaying Zhang ext4_group_t block_group, ext4_grpblk_t block, int count) 26065c521830SJiaying Zhang { 26075c521830SJiaying Zhang ext4_fsblk_t discard_block; 26085c521830SJiaying Zhang 26095c521830SJiaying Zhang discard_block = block + ext4_group_first_block_no(sb, block_group); 26105c521830SJiaying Zhang trace_ext4_discard_blocks(sb, 26115c521830SJiaying Zhang (unsigned long long) discard_block, count); 261293259636SLukas Czerner return sb_issue_discard(sb, discard_block, count, GFP_NOFS, 0); 26135c521830SJiaying Zhang } 26145c521830SJiaying Zhang 26153e624fc7STheodore Ts'o /* 26163e624fc7STheodore Ts'o * This function is called by the jbd2 layer once the commit has finished, 26173e624fc7STheodore Ts'o * so we know we can free the blocks that were released with that commit. 26183e624fc7STheodore Ts'o */ 26193e624fc7STheodore Ts'o static void release_blocks_on_commit(journal_t *journal, transaction_t *txn) 2620c9de560dSAlex Tomas { 26213e624fc7STheodore Ts'o struct super_block *sb = journal->j_private; 2622c9de560dSAlex Tomas struct ext4_buddy e4b; 2623c894058dSAneesh Kumar K.V struct ext4_group_info *db; 2624d9f34504STheodore Ts'o int err, count = 0, count2 = 0; 2625c894058dSAneesh Kumar K.V struct ext4_free_data *entry; 26263e624fc7STheodore Ts'o struct list_head *l, *ltmp; 2627c9de560dSAlex Tomas 26283e624fc7STheodore Ts'o list_for_each_safe(l, ltmp, &txn->t_private_list) { 26293e624fc7STheodore Ts'o entry = list_entry(l, struct ext4_free_data, list); 2630c9de560dSAlex Tomas 26316ba495e9STheodore Ts'o mb_debug(1, "gonna free %u blocks in group %u (0x%p):", 2632c894058dSAneesh Kumar K.V entry->count, entry->group, entry); 2633c9de560dSAlex Tomas 2634d9f34504STheodore Ts'o if (test_opt(sb, DISCARD)) 2635d9f34504STheodore Ts'o ext4_issue_discard(sb, entry->group, 26365c521830SJiaying Zhang entry->start_blk, entry->count); 2637b90f6870STheodore Ts'o 2638c894058dSAneesh Kumar K.V err = ext4_mb_load_buddy(sb, entry->group, &e4b); 2639c9de560dSAlex Tomas /* we expect to find existing buddy because it's pinned */ 2640c9de560dSAlex Tomas BUG_ON(err != 0); 2641c9de560dSAlex Tomas 2642c894058dSAneesh Kumar K.V db = e4b.bd_info; 2643c9de560dSAlex Tomas /* there are blocks to put in buddy to make them really free */ 2644c894058dSAneesh Kumar K.V count += entry->count; 2645c9de560dSAlex Tomas count2++; 2646c894058dSAneesh Kumar K.V ext4_lock_group(sb, entry->group); 2647c894058dSAneesh Kumar K.V /* Take it out of per group rb tree */ 2648c894058dSAneesh Kumar K.V rb_erase(&entry->node, &(db->bb_free_root)); 2649c894058dSAneesh Kumar K.V mb_free_blocks(NULL, &e4b, entry->start_blk, entry->count); 2650c9de560dSAlex Tomas 26513d56b8d2STao Ma /* 26523d56b8d2STao Ma * Clear the trimmed flag for the group so that the next 26533d56b8d2STao Ma * ext4_trim_fs can trim it. 26543d56b8d2STao Ma * If the volume is mounted with -o discard, online discard 26553d56b8d2STao Ma * is supported and the free blocks will be trimmed online. 26563d56b8d2STao Ma */ 26573d56b8d2STao Ma if (!test_opt(sb, DISCARD)) 26583d56b8d2STao Ma EXT4_MB_GRP_CLEAR_TRIMMED(db); 26593d56b8d2STao Ma 2660c894058dSAneesh Kumar K.V if (!db->bb_free_root.rb_node) { 2661c894058dSAneesh Kumar K.V /* No more items in the per group rb tree 2662c894058dSAneesh Kumar K.V * balance refcounts from ext4_mb_free_metadata() 2663c894058dSAneesh Kumar K.V */ 2664c9de560dSAlex Tomas page_cache_release(e4b.bd_buddy_page); 2665c9de560dSAlex Tomas page_cache_release(e4b.bd_bitmap_page); 2666c894058dSAneesh Kumar K.V } 2667c894058dSAneesh Kumar K.V ext4_unlock_group(sb, entry->group); 2668c894058dSAneesh Kumar K.V kmem_cache_free(ext4_free_ext_cachep, entry); 2669e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 26703e624fc7STheodore Ts'o } 2671c9de560dSAlex Tomas 26726ba495e9STheodore Ts'o mb_debug(1, "freed %u blocks in %u structures\n", count, count2); 2673c9de560dSAlex Tomas } 2674c9de560dSAlex Tomas 26756ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 26766ba495e9STheodore Ts'o u8 mb_enable_debug __read_mostly; 26776ba495e9STheodore Ts'o 26786ba495e9STheodore Ts'o static struct dentry *debugfs_dir; 26796ba495e9STheodore Ts'o static struct dentry *debugfs_debug; 26806ba495e9STheodore Ts'o 26816ba495e9STheodore Ts'o static void __init ext4_create_debugfs_entry(void) 26826ba495e9STheodore Ts'o { 26836ba495e9STheodore Ts'o debugfs_dir = debugfs_create_dir("ext4", NULL); 26846ba495e9STheodore Ts'o if (debugfs_dir) 26856ba495e9STheodore Ts'o debugfs_debug = debugfs_create_u8("mballoc-debug", 26866ba495e9STheodore Ts'o S_IRUGO | S_IWUSR, 26876ba495e9STheodore Ts'o debugfs_dir, 26886ba495e9STheodore Ts'o &mb_enable_debug); 26896ba495e9STheodore Ts'o } 26906ba495e9STheodore Ts'o 26916ba495e9STheodore Ts'o static void ext4_remove_debugfs_entry(void) 26926ba495e9STheodore Ts'o { 26936ba495e9STheodore Ts'o debugfs_remove(debugfs_debug); 26946ba495e9STheodore Ts'o debugfs_remove(debugfs_dir); 26956ba495e9STheodore Ts'o } 26966ba495e9STheodore Ts'o 26976ba495e9STheodore Ts'o #else 26986ba495e9STheodore Ts'o 26996ba495e9STheodore Ts'o static void __init ext4_create_debugfs_entry(void) 27006ba495e9STheodore Ts'o { 27016ba495e9STheodore Ts'o } 27026ba495e9STheodore Ts'o 27036ba495e9STheodore Ts'o static void ext4_remove_debugfs_entry(void) 27046ba495e9STheodore Ts'o { 27056ba495e9STheodore Ts'o } 27066ba495e9STheodore Ts'o 27076ba495e9STheodore Ts'o #endif 27086ba495e9STheodore Ts'o 27095dabfc78STheodore Ts'o int __init ext4_init_mballoc(void) 2710c9de560dSAlex Tomas { 271116828088STheodore Ts'o ext4_pspace_cachep = KMEM_CACHE(ext4_prealloc_space, 271216828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2713c9de560dSAlex Tomas if (ext4_pspace_cachep == NULL) 2714c9de560dSAlex Tomas return -ENOMEM; 2715c9de560dSAlex Tomas 271616828088STheodore Ts'o ext4_ac_cachep = KMEM_CACHE(ext4_allocation_context, 271716828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2718256bdb49SEric Sandeen if (ext4_ac_cachep == NULL) { 2719256bdb49SEric Sandeen kmem_cache_destroy(ext4_pspace_cachep); 2720256bdb49SEric Sandeen return -ENOMEM; 2721256bdb49SEric Sandeen } 2722c894058dSAneesh Kumar K.V 272316828088STheodore Ts'o ext4_free_ext_cachep = KMEM_CACHE(ext4_free_data, 272416828088STheodore Ts'o SLAB_RECLAIM_ACCOUNT); 2725c894058dSAneesh Kumar K.V if (ext4_free_ext_cachep == NULL) { 2726c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_pspace_cachep); 2727c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_ac_cachep); 2728c894058dSAneesh Kumar K.V return -ENOMEM; 2729c894058dSAneesh Kumar K.V } 27306ba495e9STheodore Ts'o ext4_create_debugfs_entry(); 2731c9de560dSAlex Tomas return 0; 2732c9de560dSAlex Tomas } 2733c9de560dSAlex Tomas 27345dabfc78STheodore Ts'o void ext4_exit_mballoc(void) 2735c9de560dSAlex Tomas { 27363e03f9caSJesper Dangaard Brouer /* 27373e03f9caSJesper Dangaard Brouer * Wait for completion of call_rcu()'s on ext4_pspace_cachep 27383e03f9caSJesper Dangaard Brouer * before destroying the slab cache. 27393e03f9caSJesper Dangaard Brouer */ 27403e03f9caSJesper Dangaard Brouer rcu_barrier(); 2741c9de560dSAlex Tomas kmem_cache_destroy(ext4_pspace_cachep); 2742256bdb49SEric Sandeen kmem_cache_destroy(ext4_ac_cachep); 2743c894058dSAneesh Kumar K.V kmem_cache_destroy(ext4_free_ext_cachep); 27442892c15dSEric Sandeen ext4_groupinfo_destroy_slabs(); 27456ba495e9STheodore Ts'o ext4_remove_debugfs_entry(); 2746c9de560dSAlex Tomas } 2747c9de560dSAlex Tomas 2748c9de560dSAlex Tomas 2749c9de560dSAlex Tomas /* 275073b2c716SUwe Kleine-König * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps 2751c9de560dSAlex Tomas * Returns 0 if success or error code 2752c9de560dSAlex Tomas */ 27534ddfef7bSEric Sandeen static noinline_for_stack int 27544ddfef7bSEric Sandeen ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac, 2755*53accfa9STheodore Ts'o handle_t *handle, unsigned int reserv_clstrs) 2756c9de560dSAlex Tomas { 2757c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 2758c9de560dSAlex Tomas struct ext4_group_desc *gdp; 2759c9de560dSAlex Tomas struct buffer_head *gdp_bh; 2760c9de560dSAlex Tomas struct ext4_sb_info *sbi; 2761c9de560dSAlex Tomas struct super_block *sb; 2762c9de560dSAlex Tomas ext4_fsblk_t block; 2763519deca0SAneesh Kumar K.V int err, len; 2764c9de560dSAlex Tomas 2765c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 2766c9de560dSAlex Tomas BUG_ON(ac->ac_b_ex.fe_len <= 0); 2767c9de560dSAlex Tomas 2768c9de560dSAlex Tomas sb = ac->ac_sb; 2769c9de560dSAlex Tomas sbi = EXT4_SB(sb); 2770c9de560dSAlex Tomas 2771c9de560dSAlex Tomas err = -EIO; 2772574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group); 2773c9de560dSAlex Tomas if (!bitmap_bh) 2774c9de560dSAlex Tomas goto out_err; 2775c9de560dSAlex Tomas 2776c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 2777c9de560dSAlex Tomas if (err) 2778c9de560dSAlex Tomas goto out_err; 2779c9de560dSAlex Tomas 2780c9de560dSAlex Tomas err = -EIO; 2781c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh); 2782c9de560dSAlex Tomas if (!gdp) 2783c9de560dSAlex Tomas goto out_err; 2784c9de560dSAlex Tomas 2785a9df9a49STheodore Ts'o ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group, 27869fd9784cSThadeu Lima de Souza Cascardo ext4_free_blks_count(sb, gdp)); 278703cddb80SAneesh Kumar K.V 2788c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gdp_bh); 2789c9de560dSAlex Tomas if (err) 2790c9de560dSAlex Tomas goto out_err; 2791c9de560dSAlex Tomas 2792bda00de7SAkinobu Mita block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 2793c9de560dSAlex Tomas 2794*53accfa9STheodore Ts'o len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 27956fd058f7STheodore Ts'o if (!ext4_data_block_valid(sbi, block, len)) { 279612062dddSEric Sandeen ext4_error(sb, "Allocating blocks %llu-%llu which overlap " 27976fd058f7STheodore Ts'o "fs metadata\n", block, block+len); 2798519deca0SAneesh Kumar K.V /* File system mounted not to panic on error 2799519deca0SAneesh Kumar K.V * Fix the bitmap and repeat the block allocation 2800519deca0SAneesh Kumar K.V * We leak some of the blocks here. 2801519deca0SAneesh Kumar K.V */ 2802955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 2803c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 2804519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len); 2805955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 28060390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 2807519deca0SAneesh Kumar K.V if (!err) 2808519deca0SAneesh Kumar K.V err = -EAGAIN; 2809519deca0SAneesh Kumar K.V goto out_err; 2810c9de560dSAlex Tomas } 2811955ce5f5SAneesh Kumar K.V 2812955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, ac->ac_b_ex.fe_group); 2813c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 2814c9de560dSAlex Tomas { 2815c9de560dSAlex Tomas int i; 2816c9de560dSAlex Tomas for (i = 0; i < ac->ac_b_ex.fe_len; i++) { 2817c9de560dSAlex Tomas BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i, 2818c9de560dSAlex Tomas bitmap_bh->b_data)); 2819c9de560dSAlex Tomas } 2820c9de560dSAlex Tomas } 2821c9de560dSAlex Tomas #endif 2822c3e94d1dSYongqiang Yang ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start, 2823c3e94d1dSYongqiang Yang ac->ac_b_ex.fe_len); 2824c9de560dSAlex Tomas if (gdp->bg_flags & cpu_to_le16(EXT4_BG_BLOCK_UNINIT)) { 2825c9de560dSAlex Tomas gdp->bg_flags &= cpu_to_le16(~EXT4_BG_BLOCK_UNINIT); 2826560671a0SAneesh Kumar K.V ext4_free_blks_set(sb, gdp, 2827560671a0SAneesh Kumar K.V ext4_free_blocks_after_init(sb, 2828560671a0SAneesh Kumar K.V ac->ac_b_ex.fe_group, gdp)); 2829c9de560dSAlex Tomas } 2830560671a0SAneesh Kumar K.V len = ext4_free_blks_count(sb, gdp) - ac->ac_b_ex.fe_len; 2831560671a0SAneesh Kumar K.V ext4_free_blks_set(sb, gdp, len); 2832c9de560dSAlex Tomas gdp->bg_checksum = ext4_group_desc_csum(sbi, ac->ac_b_ex.fe_group, gdp); 2833955ce5f5SAneesh Kumar K.V 2834955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 28356bc6e63fSAneesh Kumar K.V percpu_counter_sub(&sbi->s_freeblocks_counter, ac->ac_b_ex.fe_len); 2836d2a17637SMingming Cao /* 28376bc6e63fSAneesh Kumar K.V * Now reduce the dirty block count also. Should not go negative 2838d2a17637SMingming Cao */ 28396bc6e63fSAneesh Kumar K.V if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED)) 28406bc6e63fSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 2841*53accfa9STheodore Ts'o percpu_counter_sub(&sbi->s_dirtyblocks_counter, reserv_clstrs); 2842c9de560dSAlex Tomas 2843772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 2844772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, 2845772cb7c8SJose R. Santos ac->ac_b_ex.fe_group); 28469f24e420STheodore Ts'o atomic_sub(ac->ac_b_ex.fe_len, 28479f24e420STheodore Ts'o &sbi->s_flex_groups[flex_group].free_blocks); 2848772cb7c8SJose R. Santos } 2849772cb7c8SJose R. Santos 28500390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 2851c9de560dSAlex Tomas if (err) 2852c9de560dSAlex Tomas goto out_err; 28530390131bSFrank Mayhar err = ext4_handle_dirty_metadata(handle, NULL, gdp_bh); 2854c9de560dSAlex Tomas 2855c9de560dSAlex Tomas out_err: 2856a0375156STheodore Ts'o ext4_mark_super_dirty(sb); 285742a10addSAneesh Kumar K.V brelse(bitmap_bh); 2858c9de560dSAlex Tomas return err; 2859c9de560dSAlex Tomas } 2860c9de560dSAlex Tomas 2861c9de560dSAlex Tomas /* 2862c9de560dSAlex Tomas * here we normalize request for locality group 2863d7a1fee1SDan Ehrenberg * Group request are normalized to s_mb_group_prealloc, which goes to 2864d7a1fee1SDan Ehrenberg * s_strip if we set the same via mount option. 2865d7a1fee1SDan Ehrenberg * s_mb_group_prealloc can be configured via 2866b713a5ecSTheodore Ts'o * /sys/fs/ext4/<partition>/mb_group_prealloc 2867c9de560dSAlex Tomas * 2868c9de560dSAlex Tomas * XXX: should we try to preallocate more than the group has now? 2869c9de560dSAlex Tomas */ 2870c9de560dSAlex Tomas static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac) 2871c9de560dSAlex Tomas { 2872c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 2873c9de560dSAlex Tomas struct ext4_locality_group *lg = ac->ac_lg; 2874c9de560dSAlex Tomas 2875c9de560dSAlex Tomas BUG_ON(lg == NULL); 2876c9de560dSAlex Tomas ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc; 28776ba495e9STheodore Ts'o mb_debug(1, "#%u: goal %u blocks for locality group\n", 2878c9de560dSAlex Tomas current->pid, ac->ac_g_ex.fe_len); 2879c9de560dSAlex Tomas } 2880c9de560dSAlex Tomas 2881c9de560dSAlex Tomas /* 2882c9de560dSAlex Tomas * Normalization means making request better in terms of 2883c9de560dSAlex Tomas * size and alignment 2884c9de560dSAlex Tomas */ 28854ddfef7bSEric Sandeen static noinline_for_stack void 28864ddfef7bSEric Sandeen ext4_mb_normalize_request(struct ext4_allocation_context *ac, 2887c9de560dSAlex Tomas struct ext4_allocation_request *ar) 2888c9de560dSAlex Tomas { 2889*53accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 2890c9de560dSAlex Tomas int bsbits, max; 2891c9de560dSAlex Tomas ext4_lblk_t end; 2892c9de560dSAlex Tomas loff_t size, orig_size, start_off; 28935a0790c2SAndi Kleen ext4_lblk_t start; 2894c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 28959a0762c5SAneesh Kumar K.V struct ext4_prealloc_space *pa; 2896c9de560dSAlex Tomas 2897c9de560dSAlex Tomas /* do normalize only data requests, metadata requests 2898c9de560dSAlex Tomas do not need preallocation */ 2899c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 2900c9de560dSAlex Tomas return; 2901c9de560dSAlex Tomas 2902c9de560dSAlex Tomas /* sometime caller may want exact blocks */ 2903c9de560dSAlex Tomas if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 2904c9de560dSAlex Tomas return; 2905c9de560dSAlex Tomas 2906c9de560dSAlex Tomas /* caller may indicate that preallocation isn't 2907c9de560dSAlex Tomas * required (it's a tail, for example) */ 2908c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC) 2909c9de560dSAlex Tomas return; 2910c9de560dSAlex Tomas 2911c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) { 2912c9de560dSAlex Tomas ext4_mb_normalize_group_request(ac); 2913c9de560dSAlex Tomas return ; 2914c9de560dSAlex Tomas } 2915c9de560dSAlex Tomas 2916c9de560dSAlex Tomas bsbits = ac->ac_sb->s_blocksize_bits; 2917c9de560dSAlex Tomas 2918c9de560dSAlex Tomas /* first, let's learn actual file size 2919c9de560dSAlex Tomas * given current request is allocated */ 2920*53accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 2921c9de560dSAlex Tomas size = size << bsbits; 2922c9de560dSAlex Tomas if (size < i_size_read(ac->ac_inode)) 2923c9de560dSAlex Tomas size = i_size_read(ac->ac_inode); 29245a0790c2SAndi Kleen orig_size = size; 2925c9de560dSAlex Tomas 29261930479cSValerie Clement /* max size of free chunks */ 29271930479cSValerie Clement max = 2 << bsbits; 2928c9de560dSAlex Tomas 29291930479cSValerie Clement #define NRL_CHECK_SIZE(req, size, max, chunk_size) \ 29301930479cSValerie Clement (req <= (size) || max <= (chunk_size)) 2931c9de560dSAlex Tomas 2932c9de560dSAlex Tomas /* first, try to predict filesize */ 2933c9de560dSAlex Tomas /* XXX: should this table be tunable? */ 2934c9de560dSAlex Tomas start_off = 0; 2935c9de560dSAlex Tomas if (size <= 16 * 1024) { 2936c9de560dSAlex Tomas size = 16 * 1024; 2937c9de560dSAlex Tomas } else if (size <= 32 * 1024) { 2938c9de560dSAlex Tomas size = 32 * 1024; 2939c9de560dSAlex Tomas } else if (size <= 64 * 1024) { 2940c9de560dSAlex Tomas size = 64 * 1024; 2941c9de560dSAlex Tomas } else if (size <= 128 * 1024) { 2942c9de560dSAlex Tomas size = 128 * 1024; 2943c9de560dSAlex Tomas } else if (size <= 256 * 1024) { 2944c9de560dSAlex Tomas size = 256 * 1024; 2945c9de560dSAlex Tomas } else if (size <= 512 * 1024) { 2946c9de560dSAlex Tomas size = 512 * 1024; 2947c9de560dSAlex Tomas } else if (size <= 1024 * 1024) { 2948c9de560dSAlex Tomas size = 1024 * 1024; 29491930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 4 * 1024 * 1024, max, 2 * 1024)) { 2950c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 29511930479cSValerie Clement (21 - bsbits)) << 21; 29521930479cSValerie Clement size = 2 * 1024 * 1024; 29531930479cSValerie Clement } else if (NRL_CHECK_SIZE(size, 8 * 1024 * 1024, max, 4 * 1024)) { 2954c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 2955c9de560dSAlex Tomas (22 - bsbits)) << 22; 2956c9de560dSAlex Tomas size = 4 * 1024 * 1024; 2957c9de560dSAlex Tomas } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len, 29581930479cSValerie Clement (8<<20)>>bsbits, max, 8 * 1024)) { 2959c9de560dSAlex Tomas start_off = ((loff_t)ac->ac_o_ex.fe_logical >> 2960c9de560dSAlex Tomas (23 - bsbits)) << 23; 2961c9de560dSAlex Tomas size = 8 * 1024 * 1024; 2962c9de560dSAlex Tomas } else { 2963c9de560dSAlex Tomas start_off = (loff_t)ac->ac_o_ex.fe_logical << bsbits; 2964c9de560dSAlex Tomas size = ac->ac_o_ex.fe_len << bsbits; 2965c9de560dSAlex Tomas } 29665a0790c2SAndi Kleen size = size >> bsbits; 29675a0790c2SAndi Kleen start = start_off >> bsbits; 2968c9de560dSAlex Tomas 2969c9de560dSAlex Tomas /* don't cover already allocated blocks in selected range */ 2970c9de560dSAlex Tomas if (ar->pleft && start <= ar->lleft) { 2971c9de560dSAlex Tomas size -= ar->lleft + 1 - start; 2972c9de560dSAlex Tomas start = ar->lleft + 1; 2973c9de560dSAlex Tomas } 2974c9de560dSAlex Tomas if (ar->pright && start + size - 1 >= ar->lright) 2975c9de560dSAlex Tomas size -= start + size - ar->lright; 2976c9de560dSAlex Tomas 2977c9de560dSAlex Tomas end = start + size; 2978c9de560dSAlex Tomas 2979c9de560dSAlex Tomas /* check we don't cross already preallocated blocks */ 2980c9de560dSAlex Tomas rcu_read_lock(); 29819a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 2982498e5f24STheodore Ts'o ext4_lblk_t pa_end; 2983c9de560dSAlex Tomas 2984c9de560dSAlex Tomas if (pa->pa_deleted) 2985c9de560dSAlex Tomas continue; 2986c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 2987c9de560dSAlex Tomas if (pa->pa_deleted) { 2988c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 2989c9de560dSAlex Tomas continue; 2990c9de560dSAlex Tomas } 2991c9de560dSAlex Tomas 2992*53accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 2993*53accfa9STheodore Ts'o pa->pa_len); 2994c9de560dSAlex Tomas 2995c9de560dSAlex Tomas /* PA must not overlap original request */ 2996c9de560dSAlex Tomas BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end || 2997c9de560dSAlex Tomas ac->ac_o_ex.fe_logical < pa->pa_lstart)); 2998c9de560dSAlex Tomas 299938877f4eSEric Sandeen /* skip PAs this normalized request doesn't overlap with */ 300038877f4eSEric Sandeen if (pa->pa_lstart >= end || pa_end <= start) { 3001c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3002c9de560dSAlex Tomas continue; 3003c9de560dSAlex Tomas } 3004c9de560dSAlex Tomas BUG_ON(pa->pa_lstart <= start && pa_end >= end); 3005c9de560dSAlex Tomas 300638877f4eSEric Sandeen /* adjust start or end to be adjacent to this pa */ 3007c9de560dSAlex Tomas if (pa_end <= ac->ac_o_ex.fe_logical) { 3008c9de560dSAlex Tomas BUG_ON(pa_end < start); 3009c9de560dSAlex Tomas start = pa_end; 301038877f4eSEric Sandeen } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) { 3011c9de560dSAlex Tomas BUG_ON(pa->pa_lstart > end); 3012c9de560dSAlex Tomas end = pa->pa_lstart; 3013c9de560dSAlex Tomas } 3014c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3015c9de560dSAlex Tomas } 3016c9de560dSAlex Tomas rcu_read_unlock(); 3017c9de560dSAlex Tomas size = end - start; 3018c9de560dSAlex Tomas 3019c9de560dSAlex Tomas /* XXX: extra loop to check we really don't overlap preallocations */ 3020c9de560dSAlex Tomas rcu_read_lock(); 30219a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3022498e5f24STheodore Ts'o ext4_lblk_t pa_end; 3023*53accfa9STheodore Ts'o 3024c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3025c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 3026*53accfa9STheodore Ts'o pa_end = pa->pa_lstart + EXT4_C2B(EXT4_SB(ac->ac_sb), 3027*53accfa9STheodore Ts'o pa->pa_len); 3028c9de560dSAlex Tomas BUG_ON(!(start >= pa_end || end <= pa->pa_lstart)); 3029c9de560dSAlex Tomas } 3030c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3031c9de560dSAlex Tomas } 3032c9de560dSAlex Tomas rcu_read_unlock(); 3033c9de560dSAlex Tomas 3034c9de560dSAlex Tomas if (start + size <= ac->ac_o_ex.fe_logical && 3035c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical) { 30369d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, 30379d8b9ec4STheodore Ts'o "start %lu, size %lu, fe_logical %lu", 3038c9de560dSAlex Tomas (unsigned long) start, (unsigned long) size, 3039c9de560dSAlex Tomas (unsigned long) ac->ac_o_ex.fe_logical); 3040c9de560dSAlex Tomas } 3041c9de560dSAlex Tomas BUG_ON(start + size <= ac->ac_o_ex.fe_logical && 3042c9de560dSAlex Tomas start > ac->ac_o_ex.fe_logical); 30437137d7a4STheodore Ts'o BUG_ON(size <= 0 || size > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb)); 3044c9de560dSAlex Tomas 3045c9de560dSAlex Tomas /* now prepare goal request */ 3046c9de560dSAlex Tomas 3047c9de560dSAlex Tomas /* XXX: is it better to align blocks WRT to logical 3048c9de560dSAlex Tomas * placement or satisfy big request as is */ 3049c9de560dSAlex Tomas ac->ac_g_ex.fe_logical = start; 3050*53accfa9STheodore Ts'o ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size); 3051c9de560dSAlex Tomas 3052c9de560dSAlex Tomas /* define goal start in order to merge */ 3053c9de560dSAlex Tomas if (ar->pright && (ar->lright == (start + size))) { 3054c9de560dSAlex Tomas /* merge to the right */ 3055c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size, 3056c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3057c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3058c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3059c9de560dSAlex Tomas } 3060c9de560dSAlex Tomas if (ar->pleft && (ar->lleft + 1 == start)) { 3061c9de560dSAlex Tomas /* merge to the left */ 3062c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1, 3063c9de560dSAlex Tomas &ac->ac_f_ex.fe_group, 3064c9de560dSAlex Tomas &ac->ac_f_ex.fe_start); 3065c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL; 3066c9de560dSAlex Tomas } 3067c9de560dSAlex Tomas 30686ba495e9STheodore Ts'o mb_debug(1, "goal: %u(was %u) blocks at %u\n", (unsigned) size, 3069c9de560dSAlex Tomas (unsigned) orig_size, (unsigned) start); 3070c9de560dSAlex Tomas } 3071c9de560dSAlex Tomas 3072c9de560dSAlex Tomas static void ext4_mb_collect_stats(struct ext4_allocation_context *ac) 3073c9de560dSAlex Tomas { 3074c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3075c9de560dSAlex Tomas 3076c9de560dSAlex Tomas if (sbi->s_mb_stats && ac->ac_g_ex.fe_len > 1) { 3077c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_reqs); 3078c9de560dSAlex Tomas atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated); 3079291dae47SCurt Wohlgemuth if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len) 3080c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_success); 3081c9de560dSAlex Tomas atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned); 3082c9de560dSAlex Tomas if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start && 3083c9de560dSAlex Tomas ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group) 3084c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_goals); 3085c9de560dSAlex Tomas if (ac->ac_found > sbi->s_mb_max_to_scan) 3086c9de560dSAlex Tomas atomic_inc(&sbi->s_bal_breaks); 3087c9de560dSAlex Tomas } 3088c9de560dSAlex Tomas 3089296c355cSTheodore Ts'o if (ac->ac_op == EXT4_MB_HISTORY_ALLOC) 3090296c355cSTheodore Ts'o trace_ext4_mballoc_alloc(ac); 3091296c355cSTheodore Ts'o else 3092296c355cSTheodore Ts'o trace_ext4_mballoc_prealloc(ac); 3093c9de560dSAlex Tomas } 3094c9de560dSAlex Tomas 3095c9de560dSAlex Tomas /* 3096b844167eSCurt Wohlgemuth * Called on failure; free up any blocks from the inode PA for this 3097b844167eSCurt Wohlgemuth * context. We don't need this for MB_GROUP_PA because we only change 3098b844167eSCurt Wohlgemuth * pa_free in ext4_mb_release_context(), but on failure, we've already 3099b844167eSCurt Wohlgemuth * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed. 3100b844167eSCurt Wohlgemuth */ 3101b844167eSCurt Wohlgemuth static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac) 3102b844167eSCurt Wohlgemuth { 3103b844167eSCurt Wohlgemuth struct ext4_prealloc_space *pa = ac->ac_pa; 3104b844167eSCurt Wohlgemuth int len; 3105b844167eSCurt Wohlgemuth 3106b844167eSCurt Wohlgemuth if (pa && pa->pa_type == MB_INODE_PA) { 3107b844167eSCurt Wohlgemuth len = ac->ac_b_ex.fe_len; 3108b844167eSCurt Wohlgemuth pa->pa_free += len; 3109b844167eSCurt Wohlgemuth } 3110b844167eSCurt Wohlgemuth 3111b844167eSCurt Wohlgemuth } 3112b844167eSCurt Wohlgemuth 3113b844167eSCurt Wohlgemuth /* 3114c9de560dSAlex Tomas * use blocks preallocated to inode 3115c9de560dSAlex Tomas */ 3116c9de560dSAlex Tomas static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac, 3117c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3118c9de560dSAlex Tomas { 3119*53accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3120c9de560dSAlex Tomas ext4_fsblk_t start; 3121c9de560dSAlex Tomas ext4_fsblk_t end; 3122c9de560dSAlex Tomas int len; 3123c9de560dSAlex Tomas 3124c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3125c9de560dSAlex Tomas start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart); 3126*53accfa9STheodore Ts'o end = min(pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len), 3127*53accfa9STheodore Ts'o start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len)); 3128*53accfa9STheodore Ts'o len = EXT4_NUM_B2C(sbi, end - start); 3129c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group, 3130c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3131c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3132c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3133c9de560dSAlex Tomas ac->ac_pa = pa; 3134c9de560dSAlex Tomas 3135c9de560dSAlex Tomas BUG_ON(start < pa->pa_pstart); 3136*53accfa9STheodore Ts'o BUG_ON(end > pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len)); 3137c9de560dSAlex Tomas BUG_ON(pa->pa_free < len); 3138c9de560dSAlex Tomas pa->pa_free -= len; 3139c9de560dSAlex Tomas 31406ba495e9STheodore Ts'o mb_debug(1, "use %llu/%u from inode pa %p\n", start, len, pa); 3141c9de560dSAlex Tomas } 3142c9de560dSAlex Tomas 3143c9de560dSAlex Tomas /* 3144c9de560dSAlex Tomas * use blocks preallocated to locality group 3145c9de560dSAlex Tomas */ 3146c9de560dSAlex Tomas static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac, 3147c9de560dSAlex Tomas struct ext4_prealloc_space *pa) 3148c9de560dSAlex Tomas { 314903cddb80SAneesh Kumar K.V unsigned int len = ac->ac_o_ex.fe_len; 31506be2ded1SAneesh Kumar K.V 3151c9de560dSAlex Tomas ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart, 3152c9de560dSAlex Tomas &ac->ac_b_ex.fe_group, 3153c9de560dSAlex Tomas &ac->ac_b_ex.fe_start); 3154c9de560dSAlex Tomas ac->ac_b_ex.fe_len = len; 3155c9de560dSAlex Tomas ac->ac_status = AC_STATUS_FOUND; 3156c9de560dSAlex Tomas ac->ac_pa = pa; 3157c9de560dSAlex Tomas 3158c9de560dSAlex Tomas /* we don't correct pa_pstart or pa_plen here to avoid 315926346ff6SAneesh Kumar K.V * possible race when the group is being loaded concurrently 3160c9de560dSAlex Tomas * instead we correct pa later, after blocks are marked 316126346ff6SAneesh Kumar K.V * in on-disk bitmap -- see ext4_mb_release_context() 316226346ff6SAneesh Kumar K.V * Other CPUs are prevented from allocating from this pa by lg_mutex 3163c9de560dSAlex Tomas */ 31646ba495e9STheodore Ts'o mb_debug(1, "use %u/%u from group pa %p\n", pa->pa_lstart-len, len, pa); 3165c9de560dSAlex Tomas } 3166c9de560dSAlex Tomas 3167c9de560dSAlex Tomas /* 31685e745b04SAneesh Kumar K.V * Return the prealloc space that have minimal distance 31695e745b04SAneesh Kumar K.V * from the goal block. @cpa is the prealloc 31705e745b04SAneesh Kumar K.V * space that is having currently known minimal distance 31715e745b04SAneesh Kumar K.V * from the goal block. 31725e745b04SAneesh Kumar K.V */ 31735e745b04SAneesh Kumar K.V static struct ext4_prealloc_space * 31745e745b04SAneesh Kumar K.V ext4_mb_check_group_pa(ext4_fsblk_t goal_block, 31755e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, 31765e745b04SAneesh Kumar K.V struct ext4_prealloc_space *cpa) 31775e745b04SAneesh Kumar K.V { 31785e745b04SAneesh Kumar K.V ext4_fsblk_t cur_distance, new_distance; 31795e745b04SAneesh Kumar K.V 31805e745b04SAneesh Kumar K.V if (cpa == NULL) { 31815e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 31825e745b04SAneesh Kumar K.V return pa; 31835e745b04SAneesh Kumar K.V } 31845e745b04SAneesh Kumar K.V cur_distance = abs(goal_block - cpa->pa_pstart); 31855e745b04SAneesh Kumar K.V new_distance = abs(goal_block - pa->pa_pstart); 31865e745b04SAneesh Kumar K.V 31875a54b2f1SColy Li if (cur_distance <= new_distance) 31885e745b04SAneesh Kumar K.V return cpa; 31895e745b04SAneesh Kumar K.V 31905e745b04SAneesh Kumar K.V /* drop the previous reference */ 31915e745b04SAneesh Kumar K.V atomic_dec(&cpa->pa_count); 31925e745b04SAneesh Kumar K.V atomic_inc(&pa->pa_count); 31935e745b04SAneesh Kumar K.V return pa; 31945e745b04SAneesh Kumar K.V } 31955e745b04SAneesh Kumar K.V 31965e745b04SAneesh Kumar K.V /* 3197c9de560dSAlex Tomas * search goal blocks in preallocated space 3198c9de560dSAlex Tomas */ 31994ddfef7bSEric Sandeen static noinline_for_stack int 32004ddfef7bSEric Sandeen ext4_mb_use_preallocated(struct ext4_allocation_context *ac) 3201c9de560dSAlex Tomas { 3202*53accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 32036be2ded1SAneesh Kumar K.V int order, i; 3204c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(ac->ac_inode); 3205c9de560dSAlex Tomas struct ext4_locality_group *lg; 32065e745b04SAneesh Kumar K.V struct ext4_prealloc_space *pa, *cpa = NULL; 32075e745b04SAneesh Kumar K.V ext4_fsblk_t goal_block; 3208c9de560dSAlex Tomas 3209c9de560dSAlex Tomas /* only data can be preallocated */ 3210c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3211c9de560dSAlex Tomas return 0; 3212c9de560dSAlex Tomas 3213c9de560dSAlex Tomas /* first, try per-file preallocation */ 3214c9de560dSAlex Tomas rcu_read_lock(); 32159a0762c5SAneesh Kumar K.V list_for_each_entry_rcu(pa, &ei->i_prealloc_list, pa_inode_list) { 3216c9de560dSAlex Tomas 3217c9de560dSAlex Tomas /* all fields in this condition don't change, 3218c9de560dSAlex Tomas * so we can skip locking for them */ 3219c9de560dSAlex Tomas if (ac->ac_o_ex.fe_logical < pa->pa_lstart || 3220*53accfa9STheodore Ts'o ac->ac_o_ex.fe_logical >= (pa->pa_lstart + 3221*53accfa9STheodore Ts'o EXT4_C2B(sbi, pa->pa_len))) 3222c9de560dSAlex Tomas continue; 3223c9de560dSAlex Tomas 3224fb0a387dSEric Sandeen /* non-extent files can't have physical blocks past 2^32 */ 322512e9b892SDmitry Monakhov if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) && 3226*53accfa9STheodore Ts'o (pa->pa_pstart + EXT4_C2B(sbi, pa->pa_len) > 3227*53accfa9STheodore Ts'o EXT4_MAX_BLOCK_FILE_PHYS)) 3228fb0a387dSEric Sandeen continue; 3229fb0a387dSEric Sandeen 3230c9de560dSAlex Tomas /* found preallocated blocks, use them */ 3231c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3232c9de560dSAlex Tomas if (pa->pa_deleted == 0 && pa->pa_free) { 3233c9de560dSAlex Tomas atomic_inc(&pa->pa_count); 3234c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3235c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3236c9de560dSAlex Tomas ac->ac_criteria = 10; 3237c9de560dSAlex Tomas rcu_read_unlock(); 3238c9de560dSAlex Tomas return 1; 3239c9de560dSAlex Tomas } 3240c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3241c9de560dSAlex Tomas } 3242c9de560dSAlex Tomas rcu_read_unlock(); 3243c9de560dSAlex Tomas 3244c9de560dSAlex Tomas /* can we use group allocation? */ 3245c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)) 3246c9de560dSAlex Tomas return 0; 3247c9de560dSAlex Tomas 3248c9de560dSAlex Tomas /* inode may have no locality group for some reason */ 3249c9de560dSAlex Tomas lg = ac->ac_lg; 3250c9de560dSAlex Tomas if (lg == NULL) 3251c9de560dSAlex Tomas return 0; 32526be2ded1SAneesh Kumar K.V order = fls(ac->ac_o_ex.fe_len) - 1; 32536be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 32546be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 32556be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 3256c9de560dSAlex Tomas 3257bda00de7SAkinobu Mita goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex); 32585e745b04SAneesh Kumar K.V /* 32595e745b04SAneesh Kumar K.V * search for the prealloc space that is having 32605e745b04SAneesh Kumar K.V * minimal distance from the goal block. 32615e745b04SAneesh Kumar K.V */ 32626be2ded1SAneesh Kumar K.V for (i = order; i < PREALLOC_TB_SIZE; i++) { 3263c9de560dSAlex Tomas rcu_read_lock(); 32646be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[i], 32656be2ded1SAneesh Kumar K.V pa_inode_list) { 3266c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 32676be2ded1SAneesh Kumar K.V if (pa->pa_deleted == 0 && 32686be2ded1SAneesh Kumar K.V pa->pa_free >= ac->ac_o_ex.fe_len) { 32695e745b04SAneesh Kumar K.V 32705e745b04SAneesh Kumar K.V cpa = ext4_mb_check_group_pa(goal_block, 32715e745b04SAneesh Kumar K.V pa, cpa); 32725e745b04SAneesh Kumar K.V } 3273c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 32745e745b04SAneesh Kumar K.V } 32755e745b04SAneesh Kumar K.V rcu_read_unlock(); 32765e745b04SAneesh Kumar K.V } 32775e745b04SAneesh Kumar K.V if (cpa) { 32785e745b04SAneesh Kumar K.V ext4_mb_use_group_pa(ac, cpa); 3279c9de560dSAlex Tomas ac->ac_criteria = 20; 3280c9de560dSAlex Tomas return 1; 3281c9de560dSAlex Tomas } 3282c9de560dSAlex Tomas return 0; 3283c9de560dSAlex Tomas } 3284c9de560dSAlex Tomas 3285c9de560dSAlex Tomas /* 32867a2fcbf7SAneesh Kumar K.V * the function goes through all block freed in the group 32877a2fcbf7SAneesh Kumar K.V * but not yet committed and marks them used in in-core bitmap. 32887a2fcbf7SAneesh Kumar K.V * buddy must be generated from this bitmap 3289955ce5f5SAneesh Kumar K.V * Need to be called with the ext4 group lock held 32907a2fcbf7SAneesh Kumar K.V */ 32917a2fcbf7SAneesh Kumar K.V static void ext4_mb_generate_from_freelist(struct super_block *sb, void *bitmap, 32927a2fcbf7SAneesh Kumar K.V ext4_group_t group) 32937a2fcbf7SAneesh Kumar K.V { 32947a2fcbf7SAneesh Kumar K.V struct rb_node *n; 32957a2fcbf7SAneesh Kumar K.V struct ext4_group_info *grp; 32967a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 32977a2fcbf7SAneesh Kumar K.V 32987a2fcbf7SAneesh Kumar K.V grp = ext4_get_group_info(sb, group); 32997a2fcbf7SAneesh Kumar K.V n = rb_first(&(grp->bb_free_root)); 33007a2fcbf7SAneesh Kumar K.V 33017a2fcbf7SAneesh Kumar K.V while (n) { 33027a2fcbf7SAneesh Kumar K.V entry = rb_entry(n, struct ext4_free_data, node); 3303c3e94d1dSYongqiang Yang ext4_set_bits(bitmap, entry->start_blk, entry->count); 33047a2fcbf7SAneesh Kumar K.V n = rb_next(n); 33057a2fcbf7SAneesh Kumar K.V } 33067a2fcbf7SAneesh Kumar K.V return; 33077a2fcbf7SAneesh Kumar K.V } 33087a2fcbf7SAneesh Kumar K.V 33097a2fcbf7SAneesh Kumar K.V /* 3310c9de560dSAlex Tomas * the function goes through all preallocation in this group and marks them 3311c9de560dSAlex Tomas * used in in-core bitmap. buddy must be generated from this bitmap 3312955ce5f5SAneesh Kumar K.V * Need to be called with ext4 group lock held 3313c9de560dSAlex Tomas */ 3314089ceeccSEric Sandeen static noinline_for_stack 3315089ceeccSEric Sandeen void ext4_mb_generate_from_pa(struct super_block *sb, void *bitmap, 3316c9de560dSAlex Tomas ext4_group_t group) 3317c9de560dSAlex Tomas { 3318c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3319c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3320c9de560dSAlex Tomas struct list_head *cur; 3321c9de560dSAlex Tomas ext4_group_t groupnr; 3322c9de560dSAlex Tomas ext4_grpblk_t start; 3323c9de560dSAlex Tomas int preallocated = 0; 3324c9de560dSAlex Tomas int count = 0; 3325c9de560dSAlex Tomas int len; 3326c9de560dSAlex Tomas 3327c9de560dSAlex Tomas /* all form of preallocation discards first load group, 3328c9de560dSAlex Tomas * so the only competing code is preallocation use. 3329c9de560dSAlex Tomas * we don't need any locking here 3330c9de560dSAlex Tomas * notice we do NOT ignore preallocations with pa_deleted 3331c9de560dSAlex Tomas * otherwise we could leave used blocks available for 3332c9de560dSAlex Tomas * allocation in buddy when concurrent ext4_mb_put_pa() 3333c9de560dSAlex Tomas * is dropping preallocation 3334c9de560dSAlex Tomas */ 3335c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3336c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, pa_group_list); 3337c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3338c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3339c9de560dSAlex Tomas &groupnr, &start); 3340c9de560dSAlex Tomas len = pa->pa_len; 3341c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3342c9de560dSAlex Tomas if (unlikely(len == 0)) 3343c9de560dSAlex Tomas continue; 3344c9de560dSAlex Tomas BUG_ON(groupnr != group); 3345c3e94d1dSYongqiang Yang ext4_set_bits(bitmap, start, len); 3346c9de560dSAlex Tomas preallocated += len; 3347c9de560dSAlex Tomas count++; 3348c9de560dSAlex Tomas } 33496ba495e9STheodore Ts'o mb_debug(1, "prellocated %u for group %u\n", preallocated, group); 3350c9de560dSAlex Tomas } 3351c9de560dSAlex Tomas 3352c9de560dSAlex Tomas static void ext4_mb_pa_callback(struct rcu_head *head) 3353c9de560dSAlex Tomas { 3354c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3355c9de560dSAlex Tomas pa = container_of(head, struct ext4_prealloc_space, u.pa_rcu); 3356c9de560dSAlex Tomas kmem_cache_free(ext4_pspace_cachep, pa); 3357c9de560dSAlex Tomas } 3358c9de560dSAlex Tomas 3359c9de560dSAlex Tomas /* 3360c9de560dSAlex Tomas * drops a reference to preallocated space descriptor 3361c9de560dSAlex Tomas * if this was the last reference and the space is consumed 3362c9de560dSAlex Tomas */ 3363c9de560dSAlex Tomas static void ext4_mb_put_pa(struct ext4_allocation_context *ac, 3364c9de560dSAlex Tomas struct super_block *sb, struct ext4_prealloc_space *pa) 3365c9de560dSAlex Tomas { 3366a9df9a49STheodore Ts'o ext4_group_t grp; 3367d33a1976SEric Sandeen ext4_fsblk_t grp_blk; 3368c9de560dSAlex Tomas 3369c9de560dSAlex Tomas if (!atomic_dec_and_test(&pa->pa_count) || pa->pa_free != 0) 3370c9de560dSAlex Tomas return; 3371c9de560dSAlex Tomas 3372c9de560dSAlex Tomas /* in this short window concurrent discard can set pa_deleted */ 3373c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3374c9de560dSAlex Tomas if (pa->pa_deleted == 1) { 3375c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3376c9de560dSAlex Tomas return; 3377c9de560dSAlex Tomas } 3378c9de560dSAlex Tomas 3379c9de560dSAlex Tomas pa->pa_deleted = 1; 3380c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3381c9de560dSAlex Tomas 3382d33a1976SEric Sandeen grp_blk = pa->pa_pstart; 3383cc0fb9adSAneesh Kumar K.V /* 3384cc0fb9adSAneesh Kumar K.V * If doing group-based preallocation, pa_pstart may be in the 3385cc0fb9adSAneesh Kumar K.V * next group when pa is used up 3386cc0fb9adSAneesh Kumar K.V */ 3387cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 3388d33a1976SEric Sandeen grp_blk--; 3389d33a1976SEric Sandeen 3390d33a1976SEric Sandeen ext4_get_group_no_and_offset(sb, grp_blk, &grp, NULL); 3391c9de560dSAlex Tomas 3392c9de560dSAlex Tomas /* 3393c9de560dSAlex Tomas * possible race: 3394c9de560dSAlex Tomas * 3395c9de560dSAlex Tomas * P1 (buddy init) P2 (regular allocation) 3396c9de560dSAlex Tomas * find block B in PA 3397c9de560dSAlex Tomas * copy on-disk bitmap to buddy 3398c9de560dSAlex Tomas * mark B in on-disk bitmap 3399c9de560dSAlex Tomas * drop PA from group 3400c9de560dSAlex Tomas * mark all PAs in buddy 3401c9de560dSAlex Tomas * 3402c9de560dSAlex Tomas * thus, P1 initializes buddy with B available. to prevent this 3403c9de560dSAlex Tomas * we make "copy" and "mark all PAs" atomic and serialize "drop PA" 3404c9de560dSAlex Tomas * against that pair 3405c9de560dSAlex Tomas */ 3406c9de560dSAlex Tomas ext4_lock_group(sb, grp); 3407c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3408c9de560dSAlex Tomas ext4_unlock_group(sb, grp); 3409c9de560dSAlex Tomas 3410c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3411c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3412c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3413c9de560dSAlex Tomas 3414c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3415c9de560dSAlex Tomas } 3416c9de560dSAlex Tomas 3417c9de560dSAlex Tomas /* 3418c9de560dSAlex Tomas * creates new preallocated space for given inode 3419c9de560dSAlex Tomas */ 34204ddfef7bSEric Sandeen static noinline_for_stack int 34214ddfef7bSEric Sandeen ext4_mb_new_inode_pa(struct ext4_allocation_context *ac) 3422c9de560dSAlex Tomas { 3423c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3424*53accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(sb); 3425c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3426c9de560dSAlex Tomas struct ext4_group_info *grp; 3427c9de560dSAlex Tomas struct ext4_inode_info *ei; 3428c9de560dSAlex Tomas 3429c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3430c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3431c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3432c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3433c9de560dSAlex Tomas 3434c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3435c9de560dSAlex Tomas if (pa == NULL) 3436c9de560dSAlex Tomas return -ENOMEM; 3437c9de560dSAlex Tomas 3438c9de560dSAlex Tomas if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) { 3439c9de560dSAlex Tomas int winl; 3440c9de560dSAlex Tomas int wins; 3441c9de560dSAlex Tomas int win; 3442c9de560dSAlex Tomas int offs; 3443c9de560dSAlex Tomas 3444c9de560dSAlex Tomas /* we can't allocate as much as normalizer wants. 3445c9de560dSAlex Tomas * so, found space must get proper lstart 3446c9de560dSAlex Tomas * to cover original request */ 3447c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical); 3448c9de560dSAlex Tomas BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len); 3449c9de560dSAlex Tomas 3450c9de560dSAlex Tomas /* we're limited by original request in that 3451c9de560dSAlex Tomas * logical block must be covered any way 3452c9de560dSAlex Tomas * winl is window we can move our chunk within */ 3453c9de560dSAlex Tomas winl = ac->ac_o_ex.fe_logical - ac->ac_g_ex.fe_logical; 3454c9de560dSAlex Tomas 3455c9de560dSAlex Tomas /* also, we should cover whole original request */ 3456*53accfa9STheodore Ts'o wins = EXT4_C2B(sbi, ac->ac_b_ex.fe_len - ac->ac_o_ex.fe_len); 3457c9de560dSAlex Tomas 3458c9de560dSAlex Tomas /* the smallest one defines real window */ 3459c9de560dSAlex Tomas win = min(winl, wins); 3460c9de560dSAlex Tomas 3461*53accfa9STheodore Ts'o offs = ac->ac_o_ex.fe_logical % 3462*53accfa9STheodore Ts'o EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 3463c9de560dSAlex Tomas if (offs && offs < win) 3464c9de560dSAlex Tomas win = offs; 3465c9de560dSAlex Tomas 3466*53accfa9STheodore Ts'o ac->ac_b_ex.fe_logical = ac->ac_o_ex.fe_logical - 3467*53accfa9STheodore Ts'o EXT4_B2C(sbi, win); 3468c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical); 3469c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len); 3470c9de560dSAlex Tomas } 3471c9de560dSAlex Tomas 3472c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3473c9de560dSAlex Tomas * allocated blocks for history */ 3474c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3475c9de560dSAlex Tomas 3476c9de560dSAlex Tomas pa->pa_lstart = ac->ac_b_ex.fe_logical; 3477c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3478c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3479c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3480c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3481c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 3482d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3483d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3484c9de560dSAlex Tomas pa->pa_deleted = 0; 3485cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_INODE_PA; 3486c9de560dSAlex Tomas 34876ba495e9STheodore Ts'o mb_debug(1, "new inode pa %p: %llu/%u for %u\n", pa, 3488c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 34899bffad1eSTheodore Ts'o trace_ext4_mb_new_inode_pa(ac, pa); 3490c9de560dSAlex Tomas 3491c9de560dSAlex Tomas ext4_mb_use_inode_pa(ac, pa); 3492*53accfa9STheodore Ts'o atomic_add(pa->pa_free, &sbi->s_mb_preallocated); 3493c9de560dSAlex Tomas 3494c9de560dSAlex Tomas ei = EXT4_I(ac->ac_inode); 3495c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3496c9de560dSAlex Tomas 3497c9de560dSAlex Tomas pa->pa_obj_lock = &ei->i_prealloc_lock; 3498c9de560dSAlex Tomas pa->pa_inode = ac->ac_inode; 3499c9de560dSAlex Tomas 3500c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3501c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3502c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3503c9de560dSAlex Tomas 3504c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3505c9de560dSAlex Tomas list_add_rcu(&pa->pa_inode_list, &ei->i_prealloc_list); 3506c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3507c9de560dSAlex Tomas 3508c9de560dSAlex Tomas return 0; 3509c9de560dSAlex Tomas } 3510c9de560dSAlex Tomas 3511c9de560dSAlex Tomas /* 3512c9de560dSAlex Tomas * creates new preallocated space for locality group inodes belongs to 3513c9de560dSAlex Tomas */ 35144ddfef7bSEric Sandeen static noinline_for_stack int 35154ddfef7bSEric Sandeen ext4_mb_new_group_pa(struct ext4_allocation_context *ac) 3516c9de560dSAlex Tomas { 3517c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 3518c9de560dSAlex Tomas struct ext4_locality_group *lg; 3519c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3520c9de560dSAlex Tomas struct ext4_group_info *grp; 3521c9de560dSAlex Tomas 3522c9de560dSAlex Tomas /* preallocate only when found space is larger then requested */ 3523c9de560dSAlex Tomas BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len); 3524c9de560dSAlex Tomas BUG_ON(ac->ac_status != AC_STATUS_FOUND); 3525c9de560dSAlex Tomas BUG_ON(!S_ISREG(ac->ac_inode->i_mode)); 3526c9de560dSAlex Tomas 3527c9de560dSAlex Tomas BUG_ON(ext4_pspace_cachep == NULL); 3528c9de560dSAlex Tomas pa = kmem_cache_alloc(ext4_pspace_cachep, GFP_NOFS); 3529c9de560dSAlex Tomas if (pa == NULL) 3530c9de560dSAlex Tomas return -ENOMEM; 3531c9de560dSAlex Tomas 3532c9de560dSAlex Tomas /* preallocation can change ac_b_ex, thus we store actually 3533c9de560dSAlex Tomas * allocated blocks for history */ 3534c9de560dSAlex Tomas ac->ac_f_ex = ac->ac_b_ex; 3535c9de560dSAlex Tomas 3536c9de560dSAlex Tomas pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 3537c9de560dSAlex Tomas pa->pa_lstart = pa->pa_pstart; 3538c9de560dSAlex Tomas pa->pa_len = ac->ac_b_ex.fe_len; 3539c9de560dSAlex Tomas pa->pa_free = pa->pa_len; 3540c9de560dSAlex Tomas atomic_set(&pa->pa_count, 1); 3541c9de560dSAlex Tomas spin_lock_init(&pa->pa_lock); 35426be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_inode_list); 3543d794bf8eSAneesh Kumar K.V INIT_LIST_HEAD(&pa->pa_group_list); 3544c9de560dSAlex Tomas pa->pa_deleted = 0; 3545cc0fb9adSAneesh Kumar K.V pa->pa_type = MB_GROUP_PA; 3546c9de560dSAlex Tomas 35476ba495e9STheodore Ts'o mb_debug(1, "new group pa %p: %llu/%u for %u\n", pa, 3548c9de560dSAlex Tomas pa->pa_pstart, pa->pa_len, pa->pa_lstart); 35499bffad1eSTheodore Ts'o trace_ext4_mb_new_group_pa(ac, pa); 3550c9de560dSAlex Tomas 3551c9de560dSAlex Tomas ext4_mb_use_group_pa(ac, pa); 3552c9de560dSAlex Tomas atomic_add(pa->pa_free, &EXT4_SB(sb)->s_mb_preallocated); 3553c9de560dSAlex Tomas 3554c9de560dSAlex Tomas grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group); 3555c9de560dSAlex Tomas lg = ac->ac_lg; 3556c9de560dSAlex Tomas BUG_ON(lg == NULL); 3557c9de560dSAlex Tomas 3558c9de560dSAlex Tomas pa->pa_obj_lock = &lg->lg_prealloc_lock; 3559c9de560dSAlex Tomas pa->pa_inode = NULL; 3560c9de560dSAlex Tomas 3561c9de560dSAlex Tomas ext4_lock_group(sb, ac->ac_b_ex.fe_group); 3562c9de560dSAlex Tomas list_add(&pa->pa_group_list, &grp->bb_prealloc_list); 3563c9de560dSAlex Tomas ext4_unlock_group(sb, ac->ac_b_ex.fe_group); 3564c9de560dSAlex Tomas 35656be2ded1SAneesh Kumar K.V /* 35666be2ded1SAneesh Kumar K.V * We will later add the new pa to the right bucket 35676be2ded1SAneesh Kumar K.V * after updating the pa_free in ext4_mb_release_context 35686be2ded1SAneesh Kumar K.V */ 3569c9de560dSAlex Tomas return 0; 3570c9de560dSAlex Tomas } 3571c9de560dSAlex Tomas 3572c9de560dSAlex Tomas static int ext4_mb_new_preallocation(struct ext4_allocation_context *ac) 3573c9de560dSAlex Tomas { 3574c9de560dSAlex Tomas int err; 3575c9de560dSAlex Tomas 3576c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 3577c9de560dSAlex Tomas err = ext4_mb_new_group_pa(ac); 3578c9de560dSAlex Tomas else 3579c9de560dSAlex Tomas err = ext4_mb_new_inode_pa(ac); 3580c9de560dSAlex Tomas return err; 3581c9de560dSAlex Tomas } 3582c9de560dSAlex Tomas 3583c9de560dSAlex Tomas /* 3584c9de560dSAlex Tomas * finds all unused blocks in on-disk bitmap, frees them in 3585c9de560dSAlex Tomas * in-core bitmap and buddy. 3586c9de560dSAlex Tomas * @pa must be unlinked from inode and group lists, so that 3587c9de560dSAlex Tomas * nobody else can find/use it. 3588c9de560dSAlex Tomas * the caller MUST hold group/inode locks. 3589c9de560dSAlex Tomas * TODO: optimize the case when there are no in-core structures yet 3590c9de560dSAlex Tomas */ 35914ddfef7bSEric Sandeen static noinline_for_stack int 35924ddfef7bSEric Sandeen ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh, 35933e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 3594c9de560dSAlex Tomas { 3595c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3596c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 3597498e5f24STheodore Ts'o unsigned int end; 3598498e5f24STheodore Ts'o unsigned int next; 3599c9de560dSAlex Tomas ext4_group_t group; 3600c9de560dSAlex Tomas ext4_grpblk_t bit; 3601ba80b101STheodore Ts'o unsigned long long grp_blk_start; 3602c9de560dSAlex Tomas int err = 0; 3603c9de560dSAlex Tomas int free = 0; 3604c9de560dSAlex Tomas 3605c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3606c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3607*53accfa9STheodore Ts'o grp_blk_start = pa->pa_pstart - EXT4_C2B(sbi, bit); 3608c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3609c9de560dSAlex Tomas end = bit + pa->pa_len; 3610c9de560dSAlex Tomas 3611c9de560dSAlex Tomas while (bit < end) { 3612ffad0a44SAneesh Kumar K.V bit = mb_find_next_zero_bit(bitmap_bh->b_data, end, bit); 3613c9de560dSAlex Tomas if (bit >= end) 3614c9de560dSAlex Tomas break; 3615ffad0a44SAneesh Kumar K.V next = mb_find_next_bit(bitmap_bh->b_data, end, bit); 36166ba495e9STheodore Ts'o mb_debug(1, " free preallocated %u/%u in group %u\n", 36175a0790c2SAndi Kleen (unsigned) ext4_group_first_block_no(sb, group) + bit, 36185a0790c2SAndi Kleen (unsigned) next - bit, (unsigned) group); 3619c9de560dSAlex Tomas free += next - bit; 3620c9de560dSAlex Tomas 36213e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, next - bit); 3622*53accfa9STheodore Ts'o trace_ext4_mb_release_inode_pa(pa, (grp_blk_start + 3623*53accfa9STheodore Ts'o EXT4_C2B(sbi, bit)), 3624a9c667f8SLukas Czerner next - bit); 3625c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, next - bit); 3626c9de560dSAlex Tomas bit = next + 1; 3627c9de560dSAlex Tomas } 3628c9de560dSAlex Tomas if (free != pa->pa_free) { 36299d8b9ec4STheodore Ts'o ext4_msg(e4b->bd_sb, KERN_CRIT, 36309d8b9ec4STheodore Ts'o "pa %p: logic %lu, phys. %lu, len %lu", 3631c9de560dSAlex Tomas pa, (unsigned long) pa->pa_lstart, 3632c9de560dSAlex Tomas (unsigned long) pa->pa_pstart, 3633c9de560dSAlex Tomas (unsigned long) pa->pa_len); 3634e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 0, "free %u, pa_free %u", 363526346ff6SAneesh Kumar K.V free, pa->pa_free); 3636e56eb659SAneesh Kumar K.V /* 3637e56eb659SAneesh Kumar K.V * pa is already deleted so we use the value obtained 3638e56eb659SAneesh Kumar K.V * from the bitmap and continue. 3639e56eb659SAneesh Kumar K.V */ 3640c9de560dSAlex Tomas } 3641c9de560dSAlex Tomas atomic_add(free, &sbi->s_mb_discarded); 3642c9de560dSAlex Tomas 3643c9de560dSAlex Tomas return err; 3644c9de560dSAlex Tomas } 3645c9de560dSAlex Tomas 36464ddfef7bSEric Sandeen static noinline_for_stack int 36474ddfef7bSEric Sandeen ext4_mb_release_group_pa(struct ext4_buddy *e4b, 36483e1e5f50SEric Sandeen struct ext4_prealloc_space *pa) 3649c9de560dSAlex Tomas { 3650c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 3651c9de560dSAlex Tomas ext4_group_t group; 3652c9de560dSAlex Tomas ext4_grpblk_t bit; 3653c9de560dSAlex Tomas 3654a9c667f8SLukas Czerner trace_ext4_mb_release_group_pa(pa); 3655c9de560dSAlex Tomas BUG_ON(pa->pa_deleted == 0); 3656c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, &bit); 3657c9de560dSAlex Tomas BUG_ON(group != e4b->bd_group && pa->pa_len != 0); 3658c9de560dSAlex Tomas mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len); 3659c9de560dSAlex Tomas atomic_add(pa->pa_len, &EXT4_SB(sb)->s_mb_discarded); 36603e1e5f50SEric Sandeen trace_ext4_mballoc_discard(sb, NULL, group, bit, pa->pa_len); 3661c9de560dSAlex Tomas 3662c9de560dSAlex Tomas return 0; 3663c9de560dSAlex Tomas } 3664c9de560dSAlex Tomas 3665c9de560dSAlex Tomas /* 3666c9de560dSAlex Tomas * releases all preallocations in given group 3667c9de560dSAlex Tomas * 3668c9de560dSAlex Tomas * first, we need to decide discard policy: 3669c9de560dSAlex Tomas * - when do we discard 3670c9de560dSAlex Tomas * 1) ENOSPC 3671c9de560dSAlex Tomas * - how many do we discard 3672c9de560dSAlex Tomas * 1) how many requested 3673c9de560dSAlex Tomas */ 36744ddfef7bSEric Sandeen static noinline_for_stack int 36754ddfef7bSEric Sandeen ext4_mb_discard_group_preallocations(struct super_block *sb, 3676c9de560dSAlex Tomas ext4_group_t group, int needed) 3677c9de560dSAlex Tomas { 3678c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, group); 3679c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3680c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3681c9de560dSAlex Tomas struct list_head list; 3682c9de560dSAlex Tomas struct ext4_buddy e4b; 3683c9de560dSAlex Tomas int err; 3684c9de560dSAlex Tomas int busy = 0; 3685c9de560dSAlex Tomas int free = 0; 3686c9de560dSAlex Tomas 36876ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for group %u\n", group); 3688c9de560dSAlex Tomas 3689c9de560dSAlex Tomas if (list_empty(&grp->bb_prealloc_list)) 3690c9de560dSAlex Tomas return 0; 3691c9de560dSAlex Tomas 3692574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 3693c9de560dSAlex Tomas if (bitmap_bh == NULL) { 369412062dddSEric Sandeen ext4_error(sb, "Error reading block bitmap for %u", group); 3695ce89f46cSAneesh Kumar K.V return 0; 3696c9de560dSAlex Tomas } 3697c9de560dSAlex Tomas 3698c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3699ce89f46cSAneesh Kumar K.V if (err) { 370012062dddSEric Sandeen ext4_error(sb, "Error loading buddy information for %u", group); 3701ce89f46cSAneesh Kumar K.V put_bh(bitmap_bh); 3702ce89f46cSAneesh Kumar K.V return 0; 3703ce89f46cSAneesh Kumar K.V } 3704c9de560dSAlex Tomas 3705c9de560dSAlex Tomas if (needed == 0) 37067137d7a4STheodore Ts'o needed = EXT4_CLUSTERS_PER_GROUP(sb) + 1; 3707c9de560dSAlex Tomas 3708c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3709c9de560dSAlex Tomas repeat: 3710c9de560dSAlex Tomas ext4_lock_group(sb, group); 3711c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, 3712c9de560dSAlex Tomas &grp->bb_prealloc_list, pa_group_list) { 3713c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3714c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3715c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3716c9de560dSAlex Tomas busy = 1; 3717c9de560dSAlex Tomas continue; 3718c9de560dSAlex Tomas } 3719c9de560dSAlex Tomas if (pa->pa_deleted) { 3720c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3721c9de560dSAlex Tomas continue; 3722c9de560dSAlex Tomas } 3723c9de560dSAlex Tomas 3724c9de560dSAlex Tomas /* seems this one can be freed ... */ 3725c9de560dSAlex Tomas pa->pa_deleted = 1; 3726c9de560dSAlex Tomas 3727c9de560dSAlex Tomas /* we can trust pa_free ... */ 3728c9de560dSAlex Tomas free += pa->pa_free; 3729c9de560dSAlex Tomas 3730c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3731c9de560dSAlex Tomas 3732c9de560dSAlex Tomas list_del(&pa->pa_group_list); 3733c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3734c9de560dSAlex Tomas } 3735c9de560dSAlex Tomas 3736c9de560dSAlex Tomas /* if we still need more blocks and some PAs were used, try again */ 3737c9de560dSAlex Tomas if (free < needed && busy) { 3738c9de560dSAlex Tomas busy = 0; 3739c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3740c9de560dSAlex Tomas /* 3741c9de560dSAlex Tomas * Yield the CPU here so that we don't get soft lockup 3742c9de560dSAlex Tomas * in non preempt case. 3743c9de560dSAlex Tomas */ 3744c9de560dSAlex Tomas yield(); 3745c9de560dSAlex Tomas goto repeat; 3746c9de560dSAlex Tomas } 3747c9de560dSAlex Tomas 3748c9de560dSAlex Tomas /* found anything to free? */ 3749c9de560dSAlex Tomas if (list_empty(&list)) { 3750c9de560dSAlex Tomas BUG_ON(free != 0); 3751c9de560dSAlex Tomas goto out; 3752c9de560dSAlex Tomas } 3753c9de560dSAlex Tomas 3754c9de560dSAlex Tomas /* now free all selected PAs */ 3755c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3756c9de560dSAlex Tomas 3757c9de560dSAlex Tomas /* remove from object (inode or locality group) */ 3758c9de560dSAlex Tomas spin_lock(pa->pa_obj_lock); 3759c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3760c9de560dSAlex Tomas spin_unlock(pa->pa_obj_lock); 3761c9de560dSAlex Tomas 3762cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) 37633e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 3764c9de560dSAlex Tomas else 37653e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 3766c9de560dSAlex Tomas 3767c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 3768c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3769c9de560dSAlex Tomas } 3770c9de560dSAlex Tomas 3771c9de560dSAlex Tomas out: 3772c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3773e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3774c9de560dSAlex Tomas put_bh(bitmap_bh); 3775c9de560dSAlex Tomas return free; 3776c9de560dSAlex Tomas } 3777c9de560dSAlex Tomas 3778c9de560dSAlex Tomas /* 3779c9de560dSAlex Tomas * releases all non-used preallocated blocks for given inode 3780c9de560dSAlex Tomas * 3781c9de560dSAlex Tomas * It's important to discard preallocations under i_data_sem 3782c9de560dSAlex Tomas * We don't want another block to be served from the prealloc 3783c9de560dSAlex Tomas * space when we are discarding the inode prealloc space. 3784c9de560dSAlex Tomas * 3785c9de560dSAlex Tomas * FIXME!! Make sure it is valid at all the call sites 3786c9de560dSAlex Tomas */ 3787c2ea3fdeSTheodore Ts'o void ext4_discard_preallocations(struct inode *inode) 3788c9de560dSAlex Tomas { 3789c9de560dSAlex Tomas struct ext4_inode_info *ei = EXT4_I(inode); 3790c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 3791c9de560dSAlex Tomas struct buffer_head *bitmap_bh = NULL; 3792c9de560dSAlex Tomas struct ext4_prealloc_space *pa, *tmp; 3793c9de560dSAlex Tomas ext4_group_t group = 0; 3794c9de560dSAlex Tomas struct list_head list; 3795c9de560dSAlex Tomas struct ext4_buddy e4b; 3796c9de560dSAlex Tomas int err; 3797c9de560dSAlex Tomas 3798c2ea3fdeSTheodore Ts'o if (!S_ISREG(inode->i_mode)) { 3799c9de560dSAlex Tomas /*BUG_ON(!list_empty(&ei->i_prealloc_list));*/ 3800c9de560dSAlex Tomas return; 3801c9de560dSAlex Tomas } 3802c9de560dSAlex Tomas 38036ba495e9STheodore Ts'o mb_debug(1, "discard preallocation for inode %lu\n", inode->i_ino); 38049bffad1eSTheodore Ts'o trace_ext4_discard_preallocations(inode); 3805c9de560dSAlex Tomas 3806c9de560dSAlex Tomas INIT_LIST_HEAD(&list); 3807c9de560dSAlex Tomas 3808c9de560dSAlex Tomas repeat: 3809c9de560dSAlex Tomas /* first, collect all pa's in the inode */ 3810c9de560dSAlex Tomas spin_lock(&ei->i_prealloc_lock); 3811c9de560dSAlex Tomas while (!list_empty(&ei->i_prealloc_list)) { 3812c9de560dSAlex Tomas pa = list_entry(ei->i_prealloc_list.next, 3813c9de560dSAlex Tomas struct ext4_prealloc_space, pa_inode_list); 3814c9de560dSAlex Tomas BUG_ON(pa->pa_obj_lock != &ei->i_prealloc_lock); 3815c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3816c9de560dSAlex Tomas if (atomic_read(&pa->pa_count)) { 3817c9de560dSAlex Tomas /* this shouldn't happen often - nobody should 3818c9de560dSAlex Tomas * use preallocation while we're discarding it */ 3819c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3820c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 38219d8b9ec4STheodore Ts'o ext4_msg(sb, KERN_ERR, 38229d8b9ec4STheodore Ts'o "uh-oh! used pa while discarding"); 3823c9de560dSAlex Tomas WARN_ON(1); 3824c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 3825c9de560dSAlex Tomas goto repeat; 3826c9de560dSAlex Tomas 3827c9de560dSAlex Tomas } 3828c9de560dSAlex Tomas if (pa->pa_deleted == 0) { 3829c9de560dSAlex Tomas pa->pa_deleted = 1; 3830c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3831c9de560dSAlex Tomas list_del_rcu(&pa->pa_inode_list); 3832c9de560dSAlex Tomas list_add(&pa->u.pa_tmp_list, &list); 3833c9de560dSAlex Tomas continue; 3834c9de560dSAlex Tomas } 3835c9de560dSAlex Tomas 3836c9de560dSAlex Tomas /* someone is deleting pa right now */ 3837c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 3838c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 3839c9de560dSAlex Tomas 3840c9de560dSAlex Tomas /* we have to wait here because pa_deleted 3841c9de560dSAlex Tomas * doesn't mean pa is already unlinked from 3842c9de560dSAlex Tomas * the list. as we might be called from 3843c9de560dSAlex Tomas * ->clear_inode() the inode will get freed 3844c9de560dSAlex Tomas * and concurrent thread which is unlinking 3845c9de560dSAlex Tomas * pa from inode's list may access already 3846c9de560dSAlex Tomas * freed memory, bad-bad-bad */ 3847c9de560dSAlex Tomas 3848c9de560dSAlex Tomas /* XXX: if this happens too often, we can 3849c9de560dSAlex Tomas * add a flag to force wait only in case 3850c9de560dSAlex Tomas * of ->clear_inode(), but not in case of 3851c9de560dSAlex Tomas * regular truncate */ 3852c9de560dSAlex Tomas schedule_timeout_uninterruptible(HZ); 3853c9de560dSAlex Tomas goto repeat; 3854c9de560dSAlex Tomas } 3855c9de560dSAlex Tomas spin_unlock(&ei->i_prealloc_lock); 3856c9de560dSAlex Tomas 3857c9de560dSAlex Tomas list_for_each_entry_safe(pa, tmp, &list, u.pa_tmp_list) { 3858cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_INODE_PA); 3859c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL); 3860c9de560dSAlex Tomas 3861c9de560dSAlex Tomas err = ext4_mb_load_buddy(sb, group, &e4b); 3862ce89f46cSAneesh Kumar K.V if (err) { 386312062dddSEric Sandeen ext4_error(sb, "Error loading buddy information for %u", 386412062dddSEric Sandeen group); 3865ce89f46cSAneesh Kumar K.V continue; 3866ce89f46cSAneesh Kumar K.V } 3867c9de560dSAlex Tomas 3868574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, group); 3869c9de560dSAlex Tomas if (bitmap_bh == NULL) { 387012062dddSEric Sandeen ext4_error(sb, "Error reading block bitmap for %u", 387112062dddSEric Sandeen group); 3872e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3873ce89f46cSAneesh Kumar K.V continue; 3874c9de560dSAlex Tomas } 3875c9de560dSAlex Tomas 3876c9de560dSAlex Tomas ext4_lock_group(sb, group); 3877c9de560dSAlex Tomas list_del(&pa->pa_group_list); 38783e1e5f50SEric Sandeen ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa); 3879c9de560dSAlex Tomas ext4_unlock_group(sb, group); 3880c9de560dSAlex Tomas 3881e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 3882c9de560dSAlex Tomas put_bh(bitmap_bh); 3883c9de560dSAlex Tomas 3884c9de560dSAlex Tomas list_del(&pa->u.pa_tmp_list); 3885c9de560dSAlex Tomas call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 3886c9de560dSAlex Tomas } 3887c9de560dSAlex Tomas } 3888c9de560dSAlex Tomas 38896ba495e9STheodore Ts'o #ifdef CONFIG_EXT4_DEBUG 3890c9de560dSAlex Tomas static void ext4_mb_show_ac(struct ext4_allocation_context *ac) 3891c9de560dSAlex Tomas { 3892c9de560dSAlex Tomas struct super_block *sb = ac->ac_sb; 38938df9675fSTheodore Ts'o ext4_group_t ngroups, i; 3894c9de560dSAlex Tomas 38954dd89fc6STheodore Ts'o if (!mb_enable_debug || 38964dd89fc6STheodore Ts'o (EXT4_SB(sb)->s_mount_flags & EXT4_MF_FS_ABORTED)) 3897e3570639SEric Sandeen return; 3898e3570639SEric Sandeen 38999d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, "EXT4-fs: Can't allocate:" 39009d8b9ec4STheodore Ts'o " Allocation context details:"); 39019d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, "EXT4-fs: status %d flags %d", 3902c9de560dSAlex Tomas ac->ac_status, ac->ac_flags); 39039d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, "EXT4-fs: orig %lu/%lu/%lu@%lu, " 39049d8b9ec4STheodore Ts'o "goal %lu/%lu/%lu@%lu, " 39059d8b9ec4STheodore Ts'o "best %lu/%lu/%lu@%lu cr %d", 3906c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_group, 3907c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_start, 3908c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_len, 3909c9de560dSAlex Tomas (unsigned long)ac->ac_o_ex.fe_logical, 3910c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_group, 3911c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_start, 3912c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_len, 3913c9de560dSAlex Tomas (unsigned long)ac->ac_g_ex.fe_logical, 3914c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_group, 3915c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_start, 3916c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_len, 3917c9de560dSAlex Tomas (unsigned long)ac->ac_b_ex.fe_logical, 3918c9de560dSAlex Tomas (int)ac->ac_criteria); 39199d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, "EXT4-fs: %lu scanned, %d found", 39209d8b9ec4STheodore Ts'o ac->ac_ex_scanned, ac->ac_found); 39219d8b9ec4STheodore Ts'o ext4_msg(ac->ac_sb, KERN_ERR, "EXT4-fs: groups: "); 39228df9675fSTheodore Ts'o ngroups = ext4_get_groups_count(sb); 39238df9675fSTheodore Ts'o for (i = 0; i < ngroups; i++) { 3924c9de560dSAlex Tomas struct ext4_group_info *grp = ext4_get_group_info(sb, i); 3925c9de560dSAlex Tomas struct ext4_prealloc_space *pa; 3926c9de560dSAlex Tomas ext4_grpblk_t start; 3927c9de560dSAlex Tomas struct list_head *cur; 3928c9de560dSAlex Tomas ext4_lock_group(sb, i); 3929c9de560dSAlex Tomas list_for_each(cur, &grp->bb_prealloc_list) { 3930c9de560dSAlex Tomas pa = list_entry(cur, struct ext4_prealloc_space, 3931c9de560dSAlex Tomas pa_group_list); 3932c9de560dSAlex Tomas spin_lock(&pa->pa_lock); 3933c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, pa->pa_pstart, 3934c9de560dSAlex Tomas NULL, &start); 3935c9de560dSAlex Tomas spin_unlock(&pa->pa_lock); 39361c718505SAkira Fujita printk(KERN_ERR "PA:%u:%d:%u \n", i, 3937c9de560dSAlex Tomas start, pa->pa_len); 3938c9de560dSAlex Tomas } 393960bd63d1SSolofo Ramangalahy ext4_unlock_group(sb, i); 3940c9de560dSAlex Tomas 3941c9de560dSAlex Tomas if (grp->bb_free == 0) 3942c9de560dSAlex Tomas continue; 39431c718505SAkira Fujita printk(KERN_ERR "%u: %d/%d \n", 3944c9de560dSAlex Tomas i, grp->bb_free, grp->bb_fragments); 3945c9de560dSAlex Tomas } 3946c9de560dSAlex Tomas printk(KERN_ERR "\n"); 3947c9de560dSAlex Tomas } 3948c9de560dSAlex Tomas #else 3949c9de560dSAlex Tomas static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac) 3950c9de560dSAlex Tomas { 3951c9de560dSAlex Tomas return; 3952c9de560dSAlex Tomas } 3953c9de560dSAlex Tomas #endif 3954c9de560dSAlex Tomas 3955c9de560dSAlex Tomas /* 3956c9de560dSAlex Tomas * We use locality group preallocation for small size file. The size of the 3957c9de560dSAlex Tomas * file is determined by the current size or the resulting size after 3958c9de560dSAlex Tomas * allocation which ever is larger 3959c9de560dSAlex Tomas * 3960b713a5ecSTheodore Ts'o * One can tune this size via /sys/fs/ext4/<partition>/mb_stream_req 3961c9de560dSAlex Tomas */ 3962c9de560dSAlex Tomas static void ext4_mb_group_or_file(struct ext4_allocation_context *ac) 3963c9de560dSAlex Tomas { 3964c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 3965c9de560dSAlex Tomas int bsbits = ac->ac_sb->s_blocksize_bits; 3966c9de560dSAlex Tomas loff_t size, isize; 3967c9de560dSAlex Tomas 3968c9de560dSAlex Tomas if (!(ac->ac_flags & EXT4_MB_HINT_DATA)) 3969c9de560dSAlex Tomas return; 3970c9de560dSAlex Tomas 39714ba74d00STheodore Ts'o if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)) 39724ba74d00STheodore Ts'o return; 39734ba74d00STheodore Ts'o 3974*53accfa9STheodore Ts'o size = ac->ac_o_ex.fe_logical + EXT4_C2B(sbi, ac->ac_o_ex.fe_len); 397550797481STheodore Ts'o isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1) 397650797481STheodore Ts'o >> bsbits; 3977c9de560dSAlex Tomas 397850797481STheodore Ts'o if ((size == isize) && 397950797481STheodore Ts'o !ext4_fs_is_busy(sbi) && 398050797481STheodore Ts'o (atomic_read(&ac->ac_inode->i_writecount) == 0)) { 398150797481STheodore Ts'o ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC; 398250797481STheodore Ts'o return; 398350797481STheodore Ts'o } 398450797481STheodore Ts'o 3985c9de560dSAlex Tomas /* don't use group allocation for large files */ 398671780577STheodore Ts'o size = max(size, isize); 3987cc483f10STao Ma if (size > sbi->s_mb_stream_request) { 39884ba74d00STheodore Ts'o ac->ac_flags |= EXT4_MB_STREAM_ALLOC; 3989c9de560dSAlex Tomas return; 39904ba74d00STheodore Ts'o } 3991c9de560dSAlex Tomas 3992c9de560dSAlex Tomas BUG_ON(ac->ac_lg != NULL); 3993c9de560dSAlex Tomas /* 3994c9de560dSAlex Tomas * locality group prealloc space are per cpu. The reason for having 3995c9de560dSAlex Tomas * per cpu locality group is to reduce the contention between block 3996c9de560dSAlex Tomas * request from multiple CPUs. 3997c9de560dSAlex Tomas */ 3998ca0c9584SChristoph Lameter ac->ac_lg = __this_cpu_ptr(sbi->s_locality_groups); 3999c9de560dSAlex Tomas 4000c9de560dSAlex Tomas /* we're going to use group allocation */ 4001c9de560dSAlex Tomas ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC; 4002c9de560dSAlex Tomas 4003c9de560dSAlex Tomas /* serialize all allocations in the group */ 4004c9de560dSAlex Tomas mutex_lock(&ac->ac_lg->lg_mutex); 4005c9de560dSAlex Tomas } 4006c9de560dSAlex Tomas 40074ddfef7bSEric Sandeen static noinline_for_stack int 40084ddfef7bSEric Sandeen ext4_mb_initialize_context(struct ext4_allocation_context *ac, 4009c9de560dSAlex Tomas struct ext4_allocation_request *ar) 4010c9de560dSAlex Tomas { 4011c9de560dSAlex Tomas struct super_block *sb = ar->inode->i_sb; 4012c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4013c9de560dSAlex Tomas struct ext4_super_block *es = sbi->s_es; 4014c9de560dSAlex Tomas ext4_group_t group; 4015498e5f24STheodore Ts'o unsigned int len; 4016498e5f24STheodore Ts'o ext4_fsblk_t goal; 4017c9de560dSAlex Tomas ext4_grpblk_t block; 4018c9de560dSAlex Tomas 4019c9de560dSAlex Tomas /* we can't allocate > group size */ 4020c9de560dSAlex Tomas len = ar->len; 4021c9de560dSAlex Tomas 4022c9de560dSAlex Tomas /* just a dirty hack to filter too big requests */ 40237137d7a4STheodore Ts'o if (len >= EXT4_CLUSTERS_PER_GROUP(sb) - 10) 40247137d7a4STheodore Ts'o len = EXT4_CLUSTERS_PER_GROUP(sb) - 10; 4025c9de560dSAlex Tomas 4026c9de560dSAlex Tomas /* start searching from the goal */ 4027c9de560dSAlex Tomas goal = ar->goal; 4028c9de560dSAlex Tomas if (goal < le32_to_cpu(es->s_first_data_block) || 4029c9de560dSAlex Tomas goal >= ext4_blocks_count(es)) 4030c9de560dSAlex Tomas goal = le32_to_cpu(es->s_first_data_block); 4031c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, goal, &group, &block); 4032c9de560dSAlex Tomas 4033c9de560dSAlex Tomas /* set up allocation goals */ 4034833576b3STheodore Ts'o memset(ac, 0, sizeof(struct ext4_allocation_context)); 4035*53accfa9STheodore Ts'o ac->ac_b_ex.fe_logical = ar->logical & ~(sbi->s_cluster_ratio - 1); 4036c9de560dSAlex Tomas ac->ac_status = AC_STATUS_CONTINUE; 4037c9de560dSAlex Tomas ac->ac_sb = sb; 4038c9de560dSAlex Tomas ac->ac_inode = ar->inode; 4039*53accfa9STheodore Ts'o ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical; 4040c9de560dSAlex Tomas ac->ac_o_ex.fe_group = group; 4041c9de560dSAlex Tomas ac->ac_o_ex.fe_start = block; 4042c9de560dSAlex Tomas ac->ac_o_ex.fe_len = len; 4043*53accfa9STheodore Ts'o ac->ac_g_ex = ac->ac_o_ex; 4044c9de560dSAlex Tomas ac->ac_flags = ar->flags; 4045c9de560dSAlex Tomas 4046c9de560dSAlex Tomas /* we have to define context: we'll we work with a file or 4047c9de560dSAlex Tomas * locality group. this is a policy, actually */ 4048c9de560dSAlex Tomas ext4_mb_group_or_file(ac); 4049c9de560dSAlex Tomas 40506ba495e9STheodore Ts'o mb_debug(1, "init ac: %u blocks @ %u, goal %u, flags %x, 2^%d, " 4051c9de560dSAlex Tomas "left: %u/%u, right %u/%u to %swritable\n", 4052c9de560dSAlex Tomas (unsigned) ar->len, (unsigned) ar->logical, 4053c9de560dSAlex Tomas (unsigned) ar->goal, ac->ac_flags, ac->ac_2order, 4054c9de560dSAlex Tomas (unsigned) ar->lleft, (unsigned) ar->pleft, 4055c9de560dSAlex Tomas (unsigned) ar->lright, (unsigned) ar->pright, 4056c9de560dSAlex Tomas atomic_read(&ar->inode->i_writecount) ? "" : "non-"); 4057c9de560dSAlex Tomas return 0; 4058c9de560dSAlex Tomas 4059c9de560dSAlex Tomas } 4060c9de560dSAlex Tomas 40616be2ded1SAneesh Kumar K.V static noinline_for_stack void 40626be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(struct super_block *sb, 40636be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg, 40646be2ded1SAneesh Kumar K.V int order, int total_entries) 40656be2ded1SAneesh Kumar K.V { 40666be2ded1SAneesh Kumar K.V ext4_group_t group = 0; 40676be2ded1SAneesh Kumar K.V struct ext4_buddy e4b; 40686be2ded1SAneesh Kumar K.V struct list_head discard_list; 40696be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa, *tmp; 40706be2ded1SAneesh Kumar K.V 40716ba495e9STheodore Ts'o mb_debug(1, "discard locality group preallocation\n"); 40726be2ded1SAneesh Kumar K.V 40736be2ded1SAneesh Kumar K.V INIT_LIST_HEAD(&discard_list); 40746be2ded1SAneesh Kumar K.V 40756be2ded1SAneesh Kumar K.V spin_lock(&lg->lg_prealloc_lock); 40766be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(pa, &lg->lg_prealloc_list[order], 40776be2ded1SAneesh Kumar K.V pa_inode_list) { 40786be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 40796be2ded1SAneesh Kumar K.V if (atomic_read(&pa->pa_count)) { 40806be2ded1SAneesh Kumar K.V /* 40816be2ded1SAneesh Kumar K.V * This is the pa that we just used 40826be2ded1SAneesh Kumar K.V * for block allocation. So don't 40836be2ded1SAneesh Kumar K.V * free that 40846be2ded1SAneesh Kumar K.V */ 40856be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 40866be2ded1SAneesh Kumar K.V continue; 40876be2ded1SAneesh Kumar K.V } 40886be2ded1SAneesh Kumar K.V if (pa->pa_deleted) { 40896be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 40906be2ded1SAneesh Kumar K.V continue; 40916be2ded1SAneesh Kumar K.V } 40926be2ded1SAneesh Kumar K.V /* only lg prealloc space */ 4093cc0fb9adSAneesh Kumar K.V BUG_ON(pa->pa_type != MB_GROUP_PA); 40946be2ded1SAneesh Kumar K.V 40956be2ded1SAneesh Kumar K.V /* seems this one can be freed ... */ 40966be2ded1SAneesh Kumar K.V pa->pa_deleted = 1; 40976be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 40986be2ded1SAneesh Kumar K.V 40996be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 41006be2ded1SAneesh Kumar K.V list_add(&pa->u.pa_tmp_list, &discard_list); 41016be2ded1SAneesh Kumar K.V 41026be2ded1SAneesh Kumar K.V total_entries--; 41036be2ded1SAneesh Kumar K.V if (total_entries <= 5) { 41046be2ded1SAneesh Kumar K.V /* 41056be2ded1SAneesh Kumar K.V * we want to keep only 5 entries 41066be2ded1SAneesh Kumar K.V * allowing it to grow to 8. This 41076be2ded1SAneesh Kumar K.V * mak sure we don't call discard 41086be2ded1SAneesh Kumar K.V * soon for this list. 41096be2ded1SAneesh Kumar K.V */ 41106be2ded1SAneesh Kumar K.V break; 41116be2ded1SAneesh Kumar K.V } 41126be2ded1SAneesh Kumar K.V } 41136be2ded1SAneesh Kumar K.V spin_unlock(&lg->lg_prealloc_lock); 41146be2ded1SAneesh Kumar K.V 41156be2ded1SAneesh Kumar K.V list_for_each_entry_safe(pa, tmp, &discard_list, u.pa_tmp_list) { 41166be2ded1SAneesh Kumar K.V 41176be2ded1SAneesh Kumar K.V ext4_get_group_no_and_offset(sb, pa->pa_pstart, &group, NULL); 41186be2ded1SAneesh Kumar K.V if (ext4_mb_load_buddy(sb, group, &e4b)) { 411912062dddSEric Sandeen ext4_error(sb, "Error loading buddy information for %u", 412012062dddSEric Sandeen group); 41216be2ded1SAneesh Kumar K.V continue; 41226be2ded1SAneesh Kumar K.V } 41236be2ded1SAneesh Kumar K.V ext4_lock_group(sb, group); 41246be2ded1SAneesh Kumar K.V list_del(&pa->pa_group_list); 41253e1e5f50SEric Sandeen ext4_mb_release_group_pa(&e4b, pa); 41266be2ded1SAneesh Kumar K.V ext4_unlock_group(sb, group); 41276be2ded1SAneesh Kumar K.V 4128e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 41296be2ded1SAneesh Kumar K.V list_del(&pa->u.pa_tmp_list); 41306be2ded1SAneesh Kumar K.V call_rcu(&(pa)->u.pa_rcu, ext4_mb_pa_callback); 41316be2ded1SAneesh Kumar K.V } 41326be2ded1SAneesh Kumar K.V } 41336be2ded1SAneesh Kumar K.V 41346be2ded1SAneesh Kumar K.V /* 41356be2ded1SAneesh Kumar K.V * We have incremented pa_count. So it cannot be freed at this 41366be2ded1SAneesh Kumar K.V * point. Also we hold lg_mutex. So no parallel allocation is 41376be2ded1SAneesh Kumar K.V * possible from this lg. That means pa_free cannot be updated. 41386be2ded1SAneesh Kumar K.V * 41396be2ded1SAneesh Kumar K.V * A parallel ext4_mb_discard_group_preallocations is possible. 41406be2ded1SAneesh Kumar K.V * which can cause the lg_prealloc_list to be updated. 41416be2ded1SAneesh Kumar K.V */ 41426be2ded1SAneesh Kumar K.V 41436be2ded1SAneesh Kumar K.V static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac) 41446be2ded1SAneesh Kumar K.V { 41456be2ded1SAneesh Kumar K.V int order, added = 0, lg_prealloc_count = 1; 41466be2ded1SAneesh Kumar K.V struct super_block *sb = ac->ac_sb; 41476be2ded1SAneesh Kumar K.V struct ext4_locality_group *lg = ac->ac_lg; 41486be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa; 41496be2ded1SAneesh Kumar K.V 41506be2ded1SAneesh Kumar K.V order = fls(pa->pa_free) - 1; 41516be2ded1SAneesh Kumar K.V if (order > PREALLOC_TB_SIZE - 1) 41526be2ded1SAneesh Kumar K.V /* The max size of hash table is PREALLOC_TB_SIZE */ 41536be2ded1SAneesh Kumar K.V order = PREALLOC_TB_SIZE - 1; 41546be2ded1SAneesh Kumar K.V /* Add the prealloc space to lg */ 41556be2ded1SAneesh Kumar K.V rcu_read_lock(); 41566be2ded1SAneesh Kumar K.V list_for_each_entry_rcu(tmp_pa, &lg->lg_prealloc_list[order], 41576be2ded1SAneesh Kumar K.V pa_inode_list) { 41586be2ded1SAneesh Kumar K.V spin_lock(&tmp_pa->pa_lock); 41596be2ded1SAneesh Kumar K.V if (tmp_pa->pa_deleted) { 4160e7c9e3e9STheodore Ts'o spin_unlock(&tmp_pa->pa_lock); 41616be2ded1SAneesh Kumar K.V continue; 41626be2ded1SAneesh Kumar K.V } 41636be2ded1SAneesh Kumar K.V if (!added && pa->pa_free < tmp_pa->pa_free) { 41646be2ded1SAneesh Kumar K.V /* Add to the tail of the previous entry */ 41656be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 41666be2ded1SAneesh Kumar K.V &tmp_pa->pa_inode_list); 41676be2ded1SAneesh Kumar K.V added = 1; 41686be2ded1SAneesh Kumar K.V /* 41696be2ded1SAneesh Kumar K.V * we want to count the total 41706be2ded1SAneesh Kumar K.V * number of entries in the list 41716be2ded1SAneesh Kumar K.V */ 41726be2ded1SAneesh Kumar K.V } 41736be2ded1SAneesh Kumar K.V spin_unlock(&tmp_pa->pa_lock); 41746be2ded1SAneesh Kumar K.V lg_prealloc_count++; 41756be2ded1SAneesh Kumar K.V } 41766be2ded1SAneesh Kumar K.V if (!added) 41776be2ded1SAneesh Kumar K.V list_add_tail_rcu(&pa->pa_inode_list, 41786be2ded1SAneesh Kumar K.V &lg->lg_prealloc_list[order]); 41796be2ded1SAneesh Kumar K.V rcu_read_unlock(); 41806be2ded1SAneesh Kumar K.V 41816be2ded1SAneesh Kumar K.V /* Now trim the list to be not more than 8 elements */ 41826be2ded1SAneesh Kumar K.V if (lg_prealloc_count > 8) { 41836be2ded1SAneesh Kumar K.V ext4_mb_discard_lg_preallocations(sb, lg, 41846be2ded1SAneesh Kumar K.V order, lg_prealloc_count); 41856be2ded1SAneesh Kumar K.V return; 41866be2ded1SAneesh Kumar K.V } 41876be2ded1SAneesh Kumar K.V return ; 41886be2ded1SAneesh Kumar K.V } 41896be2ded1SAneesh Kumar K.V 4190c9de560dSAlex Tomas /* 4191c9de560dSAlex Tomas * release all resource we used in allocation 4192c9de560dSAlex Tomas */ 4193c9de560dSAlex Tomas static int ext4_mb_release_context(struct ext4_allocation_context *ac) 4194c9de560dSAlex Tomas { 4195*53accfa9STheodore Ts'o struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb); 41966be2ded1SAneesh Kumar K.V struct ext4_prealloc_space *pa = ac->ac_pa; 41976be2ded1SAneesh Kumar K.V if (pa) { 4198cc0fb9adSAneesh Kumar K.V if (pa->pa_type == MB_GROUP_PA) { 4199c9de560dSAlex Tomas /* see comment in ext4_mb_use_group_pa() */ 42006be2ded1SAneesh Kumar K.V spin_lock(&pa->pa_lock); 4201*53accfa9STheodore Ts'o pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 4202*53accfa9STheodore Ts'o pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len); 42036be2ded1SAneesh Kumar K.V pa->pa_free -= ac->ac_b_ex.fe_len; 42046be2ded1SAneesh Kumar K.V pa->pa_len -= ac->ac_b_ex.fe_len; 42056be2ded1SAneesh Kumar K.V spin_unlock(&pa->pa_lock); 4206ba443916SAneesh Kumar K.V } 4207ba443916SAneesh Kumar K.V } 4208ba443916SAneesh Kumar K.V if (pa) { 42096be2ded1SAneesh Kumar K.V /* 42106be2ded1SAneesh Kumar K.V * We want to add the pa to the right bucket. 42116be2ded1SAneesh Kumar K.V * Remove it from the list and while adding 42126be2ded1SAneesh Kumar K.V * make sure the list to which we are adding 421344183d42SAmir Goldstein * doesn't grow big. 42146be2ded1SAneesh Kumar K.V */ 4215cc0fb9adSAneesh Kumar K.V if ((pa->pa_type == MB_GROUP_PA) && likely(pa->pa_free)) { 42166be2ded1SAneesh Kumar K.V spin_lock(pa->pa_obj_lock); 42176be2ded1SAneesh Kumar K.V list_del_rcu(&pa->pa_inode_list); 42186be2ded1SAneesh Kumar K.V spin_unlock(pa->pa_obj_lock); 42196be2ded1SAneesh Kumar K.V ext4_mb_add_n_trim(ac); 4220c9de560dSAlex Tomas } 42216be2ded1SAneesh Kumar K.V ext4_mb_put_pa(ac, ac->ac_sb, pa); 4222c9de560dSAlex Tomas } 4223c9de560dSAlex Tomas if (ac->ac_bitmap_page) 4224c9de560dSAlex Tomas page_cache_release(ac->ac_bitmap_page); 4225c9de560dSAlex Tomas if (ac->ac_buddy_page) 4226c9de560dSAlex Tomas page_cache_release(ac->ac_buddy_page); 4227c9de560dSAlex Tomas if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) 4228c9de560dSAlex Tomas mutex_unlock(&ac->ac_lg->lg_mutex); 4229c9de560dSAlex Tomas ext4_mb_collect_stats(ac); 4230c9de560dSAlex Tomas return 0; 4231c9de560dSAlex Tomas } 4232c9de560dSAlex Tomas 4233c9de560dSAlex Tomas static int ext4_mb_discard_preallocations(struct super_block *sb, int needed) 4234c9de560dSAlex Tomas { 42358df9675fSTheodore Ts'o ext4_group_t i, ngroups = ext4_get_groups_count(sb); 4236c9de560dSAlex Tomas int ret; 4237c9de560dSAlex Tomas int freed = 0; 4238c9de560dSAlex Tomas 42399bffad1eSTheodore Ts'o trace_ext4_mb_discard_preallocations(sb, needed); 42408df9675fSTheodore Ts'o for (i = 0; i < ngroups && needed > 0; i++) { 4241c9de560dSAlex Tomas ret = ext4_mb_discard_group_preallocations(sb, i, needed); 4242c9de560dSAlex Tomas freed += ret; 4243c9de560dSAlex Tomas needed -= ret; 4244c9de560dSAlex Tomas } 4245c9de560dSAlex Tomas 4246c9de560dSAlex Tomas return freed; 4247c9de560dSAlex Tomas } 4248c9de560dSAlex Tomas 4249c9de560dSAlex Tomas /* 4250c9de560dSAlex Tomas * Main entry point into mballoc to allocate blocks 4251c9de560dSAlex Tomas * it tries to use preallocation first, then falls back 4252c9de560dSAlex Tomas * to usual allocation 4253c9de560dSAlex Tomas */ 4254c9de560dSAlex Tomas ext4_fsblk_t ext4_mb_new_blocks(handle_t *handle, 4255c9de560dSAlex Tomas struct ext4_allocation_request *ar, int *errp) 4256c9de560dSAlex Tomas { 42576bc6e63fSAneesh Kumar K.V int freed; 4258256bdb49SEric Sandeen struct ext4_allocation_context *ac = NULL; 4259c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4260c9de560dSAlex Tomas struct super_block *sb; 4261c9de560dSAlex Tomas ext4_fsblk_t block = 0; 426260e58e0fSMingming Cao unsigned int inquota = 0; 4263*53accfa9STheodore Ts'o unsigned int reserv_clstrs = 0; 4264c9de560dSAlex Tomas 4265c9de560dSAlex Tomas sb = ar->inode->i_sb; 4266c9de560dSAlex Tomas sbi = EXT4_SB(sb); 4267c9de560dSAlex Tomas 42689bffad1eSTheodore Ts'o trace_ext4_request_blocks(ar); 4269ba80b101STheodore Ts'o 4270d2a17637SMingming Cao /* 427160e58e0fSMingming Cao * For delayed allocation, we could skip the ENOSPC and 427260e58e0fSMingming Cao * EDQUOT check, as blocks and quotas have been already 427360e58e0fSMingming Cao * reserved when data being copied into pagecache. 427460e58e0fSMingming Cao */ 4275f2321097STheodore Ts'o if (ext4_test_inode_state(ar->inode, EXT4_STATE_DELALLOC_RESERVED)) 427660e58e0fSMingming Cao ar->flags |= EXT4_MB_DELALLOC_RESERVED; 427760e58e0fSMingming Cao else { 427860e58e0fSMingming Cao /* Without delayed allocation we need to verify 427960e58e0fSMingming Cao * there is enough free blocks to do block allocation 428060e58e0fSMingming Cao * and verify allocation doesn't exceed the quota limits. 4281d2a17637SMingming Cao */ 428255f020dbSAllison Henderson while (ar->len && 428355f020dbSAllison Henderson ext4_claim_free_blocks(sbi, ar->len, ar->flags)) { 428455f020dbSAllison Henderson 4285030ba6bcSAneesh Kumar K.V /* let others to free the space */ 4286030ba6bcSAneesh Kumar K.V yield(); 4287030ba6bcSAneesh Kumar K.V ar->len = ar->len >> 1; 4288030ba6bcSAneesh Kumar K.V } 4289030ba6bcSAneesh Kumar K.V if (!ar->len) { 429007031431SMingming Cao *errp = -ENOSPC; 429107031431SMingming Cao return 0; 429207031431SMingming Cao } 4293*53accfa9STheodore Ts'o reserv_clstrs = ar->len; 429455f020dbSAllison Henderson if (ar->flags & EXT4_MB_USE_ROOT_BLOCKS) { 4295*53accfa9STheodore Ts'o dquot_alloc_block_nofail(ar->inode, 4296*53accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len)); 429755f020dbSAllison Henderson } else { 429855f020dbSAllison Henderson while (ar->len && 4299*53accfa9STheodore Ts'o dquot_alloc_block(ar->inode, 4300*53accfa9STheodore Ts'o EXT4_C2B(sbi, ar->len))) { 430155f020dbSAllison Henderson 4302c9de560dSAlex Tomas ar->flags |= EXT4_MB_HINT_NOPREALLOC; 4303c9de560dSAlex Tomas ar->len--; 4304c9de560dSAlex Tomas } 430555f020dbSAllison Henderson } 430660e58e0fSMingming Cao inquota = ar->len; 4307c9de560dSAlex Tomas if (ar->len == 0) { 4308c9de560dSAlex Tomas *errp = -EDQUOT; 43096c7a120aSAditya Kali goto out; 4310c9de560dSAlex Tomas } 431160e58e0fSMingming Cao } 4312d2a17637SMingming Cao 4313256bdb49SEric Sandeen ac = kmem_cache_alloc(ext4_ac_cachep, GFP_NOFS); 4314833576b3STheodore Ts'o if (!ac) { 4315363d4251SShen Feng ar->len = 0; 4316256bdb49SEric Sandeen *errp = -ENOMEM; 43176c7a120aSAditya Kali goto out; 4318256bdb49SEric Sandeen } 4319256bdb49SEric Sandeen 4320256bdb49SEric Sandeen *errp = ext4_mb_initialize_context(ac, ar); 4321c9de560dSAlex Tomas if (*errp) { 4322c9de560dSAlex Tomas ar->len = 0; 43236c7a120aSAditya Kali goto out; 4324c9de560dSAlex Tomas } 4325c9de560dSAlex Tomas 4326256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_PREALLOC; 4327256bdb49SEric Sandeen if (!ext4_mb_use_preallocated(ac)) { 4328256bdb49SEric Sandeen ac->ac_op = EXT4_MB_HISTORY_ALLOC; 4329256bdb49SEric Sandeen ext4_mb_normalize_request(ac, ar); 4330c9de560dSAlex Tomas repeat: 4331c9de560dSAlex Tomas /* allocate space in core */ 43326c7a120aSAditya Kali *errp = ext4_mb_regular_allocator(ac); 43336c7a120aSAditya Kali if (*errp) 43346c7a120aSAditya Kali goto errout; 4335c9de560dSAlex Tomas 4336c9de560dSAlex Tomas /* as we've just preallocated more space than 4337c9de560dSAlex Tomas * user requested orinally, we store allocated 4338c9de560dSAlex Tomas * space in a special descriptor */ 4339256bdb49SEric Sandeen if (ac->ac_status == AC_STATUS_FOUND && 4340256bdb49SEric Sandeen ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len) 4341256bdb49SEric Sandeen ext4_mb_new_preallocation(ac); 4342c9de560dSAlex Tomas } 4343256bdb49SEric Sandeen if (likely(ac->ac_status == AC_STATUS_FOUND)) { 4344*53accfa9STheodore Ts'o *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs); 4345519deca0SAneesh Kumar K.V if (*errp == -EAGAIN) { 43468556e8f3SAneesh Kumar K.V /* 43478556e8f3SAneesh Kumar K.V * drop the reference that we took 43488556e8f3SAneesh Kumar K.V * in ext4_mb_use_best_found 43498556e8f3SAneesh Kumar K.V */ 43508556e8f3SAneesh Kumar K.V ext4_mb_release_context(ac); 4351519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_group = 0; 4352519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_start = 0; 4353519deca0SAneesh Kumar K.V ac->ac_b_ex.fe_len = 0; 4354519deca0SAneesh Kumar K.V ac->ac_status = AC_STATUS_CONTINUE; 4355519deca0SAneesh Kumar K.V goto repeat; 43566c7a120aSAditya Kali } else if (*errp) 43576c7a120aSAditya Kali errout: 4358b844167eSCurt Wohlgemuth ext4_discard_allocated_blocks(ac); 43596c7a120aSAditya Kali else { 4360256bdb49SEric Sandeen block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex); 4361256bdb49SEric Sandeen ar->len = ac->ac_b_ex.fe_len; 4362519deca0SAneesh Kumar K.V } 4363c9de560dSAlex Tomas } else { 4364256bdb49SEric Sandeen freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len); 4365c9de560dSAlex Tomas if (freed) 4366c9de560dSAlex Tomas goto repeat; 4367c9de560dSAlex Tomas *errp = -ENOSPC; 43686c7a120aSAditya Kali } 43696c7a120aSAditya Kali 43706c7a120aSAditya Kali if (*errp) { 4371256bdb49SEric Sandeen ac->ac_b_ex.fe_len = 0; 4372c9de560dSAlex Tomas ar->len = 0; 4373256bdb49SEric Sandeen ext4_mb_show_ac(ac); 4374c9de560dSAlex Tomas } 4375256bdb49SEric Sandeen ext4_mb_release_context(ac); 43766c7a120aSAditya Kali out: 43776c7a120aSAditya Kali if (ac) 4378363d4251SShen Feng kmem_cache_free(ext4_ac_cachep, ac); 437960e58e0fSMingming Cao if (inquota && ar->len < inquota) 4380*53accfa9STheodore Ts'o dquot_free_block(ar->inode, EXT4_C2B(sbi, inquota - ar->len)); 43810087d9fbSAneesh Kumar K.V if (!ar->len) { 4382f2321097STheodore Ts'o if (!ext4_test_inode_state(ar->inode, 4383f2321097STheodore Ts'o EXT4_STATE_DELALLOC_RESERVED)) 43840087d9fbSAneesh Kumar K.V /* release all the reserved blocks if non delalloc */ 43850087d9fbSAneesh Kumar K.V percpu_counter_sub(&sbi->s_dirtyblocks_counter, 4386*53accfa9STheodore Ts'o reserv_clstrs); 43870087d9fbSAneesh Kumar K.V } 4388c9de560dSAlex Tomas 43899bffad1eSTheodore Ts'o trace_ext4_allocate_blocks(ar, (unsigned long long)block); 4390ba80b101STheodore Ts'o 4391c9de560dSAlex Tomas return block; 4392c9de560dSAlex Tomas } 4393c9de560dSAlex Tomas 4394c894058dSAneesh Kumar K.V /* 4395c894058dSAneesh Kumar K.V * We can merge two free data extents only if the physical blocks 4396c894058dSAneesh Kumar K.V * are contiguous, AND the extents were freed by the same transaction, 4397c894058dSAneesh Kumar K.V * AND the blocks are associated with the same group. 4398c894058dSAneesh Kumar K.V */ 4399c894058dSAneesh Kumar K.V static int can_merge(struct ext4_free_data *entry1, 4400c894058dSAneesh Kumar K.V struct ext4_free_data *entry2) 4401c894058dSAneesh Kumar K.V { 4402c894058dSAneesh Kumar K.V if ((entry1->t_tid == entry2->t_tid) && 4403c894058dSAneesh Kumar K.V (entry1->group == entry2->group) && 4404c894058dSAneesh Kumar K.V ((entry1->start_blk + entry1->count) == entry2->start_blk)) 4405c894058dSAneesh Kumar K.V return 1; 4406c894058dSAneesh Kumar K.V return 0; 4407c894058dSAneesh Kumar K.V } 4408c894058dSAneesh Kumar K.V 44094ddfef7bSEric Sandeen static noinline_for_stack int 44104ddfef7bSEric Sandeen ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b, 44117a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry) 4412c9de560dSAlex Tomas { 4413e29136f8STheodore Ts'o ext4_group_t group = e4b->bd_group; 44147a2fcbf7SAneesh Kumar K.V ext4_grpblk_t block; 44157a2fcbf7SAneesh Kumar K.V struct ext4_free_data *entry; 4416c9de560dSAlex Tomas struct ext4_group_info *db = e4b->bd_info; 4417c9de560dSAlex Tomas struct super_block *sb = e4b->bd_sb; 4418c9de560dSAlex Tomas struct ext4_sb_info *sbi = EXT4_SB(sb); 4419c894058dSAneesh Kumar K.V struct rb_node **n = &db->bb_free_root.rb_node, *node; 4420c894058dSAneesh Kumar K.V struct rb_node *parent = NULL, *new_node; 4421c894058dSAneesh Kumar K.V 44220390131bSFrank Mayhar BUG_ON(!ext4_handle_valid(handle)); 4423c9de560dSAlex Tomas BUG_ON(e4b->bd_bitmap_page == NULL); 4424c9de560dSAlex Tomas BUG_ON(e4b->bd_buddy_page == NULL); 4425c9de560dSAlex Tomas 4426c894058dSAneesh Kumar K.V new_node = &new_entry->node; 44277a2fcbf7SAneesh Kumar K.V block = new_entry->start_blk; 4428c9de560dSAlex Tomas 4429c894058dSAneesh Kumar K.V if (!*n) { 4430c894058dSAneesh Kumar K.V /* first free block exent. We need to 4431c894058dSAneesh Kumar K.V protect buddy cache from being freed, 4432c9de560dSAlex Tomas * otherwise we'll refresh it from 4433c9de560dSAlex Tomas * on-disk bitmap and lose not-yet-available 4434c9de560dSAlex Tomas * blocks */ 4435c9de560dSAlex Tomas page_cache_get(e4b->bd_buddy_page); 4436c9de560dSAlex Tomas page_cache_get(e4b->bd_bitmap_page); 4437c894058dSAneesh Kumar K.V } 4438c894058dSAneesh Kumar K.V while (*n) { 4439c894058dSAneesh Kumar K.V parent = *n; 4440c894058dSAneesh Kumar K.V entry = rb_entry(parent, struct ext4_free_data, node); 4441c894058dSAneesh Kumar K.V if (block < entry->start_blk) 4442c894058dSAneesh Kumar K.V n = &(*n)->rb_left; 4443c894058dSAneesh Kumar K.V else if (block >= (entry->start_blk + entry->count)) 4444c894058dSAneesh Kumar K.V n = &(*n)->rb_right; 4445c894058dSAneesh Kumar K.V else { 4446e29136f8STheodore Ts'o ext4_grp_locked_error(sb, group, 0, 4447e29136f8STheodore Ts'o ext4_group_first_block_no(sb, group) + block, 4448e29136f8STheodore Ts'o "Block already on to-be-freed list"); 4449c894058dSAneesh Kumar K.V return 0; 4450c9de560dSAlex Tomas } 4451c9de560dSAlex Tomas } 4452c9de560dSAlex Tomas 4453c894058dSAneesh Kumar K.V rb_link_node(new_node, parent, n); 4454c894058dSAneesh Kumar K.V rb_insert_color(new_node, &db->bb_free_root); 4455c894058dSAneesh Kumar K.V 4456c894058dSAneesh Kumar K.V /* Now try to see the extent can be merged to left and right */ 4457c894058dSAneesh Kumar K.V node = rb_prev(new_node); 4458c894058dSAneesh Kumar K.V if (node) { 4459c894058dSAneesh Kumar K.V entry = rb_entry(node, struct ext4_free_data, node); 4460c894058dSAneesh Kumar K.V if (can_merge(entry, new_entry)) { 4461c894058dSAneesh Kumar K.V new_entry->start_blk = entry->start_blk; 4462c894058dSAneesh Kumar K.V new_entry->count += entry->count; 4463c894058dSAneesh Kumar K.V rb_erase(node, &(db->bb_free_root)); 4464c894058dSAneesh Kumar K.V spin_lock(&sbi->s_md_lock); 4465c894058dSAneesh Kumar K.V list_del(&entry->list); 4466c894058dSAneesh Kumar K.V spin_unlock(&sbi->s_md_lock); 4467c894058dSAneesh Kumar K.V kmem_cache_free(ext4_free_ext_cachep, entry); 4468c9de560dSAlex Tomas } 4469c9de560dSAlex Tomas } 4470c894058dSAneesh Kumar K.V 4471c894058dSAneesh Kumar K.V node = rb_next(new_node); 4472c894058dSAneesh Kumar K.V if (node) { 4473c894058dSAneesh Kumar K.V entry = rb_entry(node, struct ext4_free_data, node); 4474c894058dSAneesh Kumar K.V if (can_merge(new_entry, entry)) { 4475c894058dSAneesh Kumar K.V new_entry->count += entry->count; 4476c894058dSAneesh Kumar K.V rb_erase(node, &(db->bb_free_root)); 4477c894058dSAneesh Kumar K.V spin_lock(&sbi->s_md_lock); 4478c894058dSAneesh Kumar K.V list_del(&entry->list); 4479c894058dSAneesh Kumar K.V spin_unlock(&sbi->s_md_lock); 4480c894058dSAneesh Kumar K.V kmem_cache_free(ext4_free_ext_cachep, entry); 4481c894058dSAneesh Kumar K.V } 4482c894058dSAneesh Kumar K.V } 44833e624fc7STheodore Ts'o /* Add the extent to transaction's private list */ 4484c894058dSAneesh Kumar K.V spin_lock(&sbi->s_md_lock); 44853e624fc7STheodore Ts'o list_add(&new_entry->list, &handle->h_transaction->t_private_list); 4486c894058dSAneesh Kumar K.V spin_unlock(&sbi->s_md_lock); 4487c9de560dSAlex Tomas return 0; 4488c9de560dSAlex Tomas } 4489c9de560dSAlex Tomas 449044338711STheodore Ts'o /** 449144338711STheodore Ts'o * ext4_free_blocks() -- Free given blocks and update quota 449244338711STheodore Ts'o * @handle: handle for this transaction 449344338711STheodore Ts'o * @inode: inode 449444338711STheodore Ts'o * @block: start physical block to free 449544338711STheodore Ts'o * @count: number of blocks to count 44965def1360SYongqiang Yang * @flags: flags used by ext4_free_blocks 4497c9de560dSAlex Tomas */ 449844338711STheodore Ts'o void ext4_free_blocks(handle_t *handle, struct inode *inode, 4499e6362609STheodore Ts'o struct buffer_head *bh, ext4_fsblk_t block, 4500e6362609STheodore Ts'o unsigned long count, int flags) 4501c9de560dSAlex Tomas { 450226346ff6SAneesh Kumar K.V struct buffer_head *bitmap_bh = NULL; 4503c9de560dSAlex Tomas struct super_block *sb = inode->i_sb; 4504c9de560dSAlex Tomas struct ext4_group_desc *gdp; 450544338711STheodore Ts'o unsigned long freed = 0; 4506498e5f24STheodore Ts'o unsigned int overflow; 4507c9de560dSAlex Tomas ext4_grpblk_t bit; 4508c9de560dSAlex Tomas struct buffer_head *gd_bh; 4509c9de560dSAlex Tomas ext4_group_t block_group; 4510c9de560dSAlex Tomas struct ext4_sb_info *sbi; 4511c9de560dSAlex Tomas struct ext4_buddy e4b; 4512c9de560dSAlex Tomas int err = 0; 4513c9de560dSAlex Tomas int ret; 4514c9de560dSAlex Tomas 4515e6362609STheodore Ts'o if (bh) { 4516e6362609STheodore Ts'o if (block) 4517e6362609STheodore Ts'o BUG_ON(block != bh->b_blocknr); 4518e6362609STheodore Ts'o else 4519e6362609STheodore Ts'o block = bh->b_blocknr; 4520e6362609STheodore Ts'o } 4521c9de560dSAlex Tomas 4522c9de560dSAlex Tomas sbi = EXT4_SB(sb); 45231f2acb60STheodore Ts'o if (!(flags & EXT4_FREE_BLOCKS_VALIDATED) && 45241f2acb60STheodore Ts'o !ext4_data_block_valid(sbi, block, count)) { 452512062dddSEric Sandeen ext4_error(sb, "Freeing blocks not in datazone - " 45260610b6e9STheodore Ts'o "block = %llu, count = %lu", block, count); 4527c9de560dSAlex Tomas goto error_return; 4528c9de560dSAlex Tomas } 4529c9de560dSAlex Tomas 45300610b6e9STheodore Ts'o ext4_debug("freeing block %llu\n", block); 4531e6362609STheodore Ts'o trace_ext4_free_blocks(inode, block, count, flags); 4532e6362609STheodore Ts'o 4533e6362609STheodore Ts'o if (flags & EXT4_FREE_BLOCKS_FORGET) { 4534e6362609STheodore Ts'o struct buffer_head *tbh = bh; 4535e6362609STheodore Ts'o int i; 4536e6362609STheodore Ts'o 4537e6362609STheodore Ts'o BUG_ON(bh && (count > 1)); 4538e6362609STheodore Ts'o 4539e6362609STheodore Ts'o for (i = 0; i < count; i++) { 4540e6362609STheodore Ts'o if (!bh) 4541e6362609STheodore Ts'o tbh = sb_find_get_block(inode->i_sb, 4542e6362609STheodore Ts'o block + i); 454387783690SNamhyung Kim if (unlikely(!tbh)) 454487783690SNamhyung Kim continue; 4545e6362609STheodore Ts'o ext4_forget(handle, flags & EXT4_FREE_BLOCKS_METADATA, 4546e6362609STheodore Ts'o inode, tbh, block + i); 4547e6362609STheodore Ts'o } 4548e6362609STheodore Ts'o } 4549e6362609STheodore Ts'o 4550e6362609STheodore Ts'o /* 4551e6362609STheodore Ts'o * We need to make sure we don't reuse the freed block until 4552e6362609STheodore Ts'o * after the transaction is committed, which we can do by 4553e6362609STheodore Ts'o * treating the block as metadata, below. We make an 4554e6362609STheodore Ts'o * exception if the inode is to be written in writeback mode 4555e6362609STheodore Ts'o * since writeback mode has weak data consistency guarantees. 4556e6362609STheodore Ts'o */ 4557e6362609STheodore Ts'o if (!ext4_should_writeback_data(inode)) 4558e6362609STheodore Ts'o flags |= EXT4_FREE_BLOCKS_METADATA; 4559c9de560dSAlex Tomas 4560c9de560dSAlex Tomas do_more: 4561c9de560dSAlex Tomas overflow = 0; 4562c9de560dSAlex Tomas ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 4563c9de560dSAlex Tomas 4564c9de560dSAlex Tomas /* 4565c9de560dSAlex Tomas * Check to see if we are freeing blocks across a group 4566c9de560dSAlex Tomas * boundary. 4567c9de560dSAlex Tomas */ 45687137d7a4STheodore Ts'o if (bit + count > EXT4_CLUSTERS_PER_GROUP(sb)) { 45697137d7a4STheodore Ts'o overflow = bit + count - EXT4_CLUSTERS_PER_GROUP(sb); 4570c9de560dSAlex Tomas count -= overflow; 4571c9de560dSAlex Tomas } 4572574ca174STheodore Ts'o bitmap_bh = ext4_read_block_bitmap(sb, block_group); 4573ce89f46cSAneesh Kumar K.V if (!bitmap_bh) { 4574ce89f46cSAneesh Kumar K.V err = -EIO; 4575c9de560dSAlex Tomas goto error_return; 4576ce89f46cSAneesh Kumar K.V } 4577c9de560dSAlex Tomas gdp = ext4_get_group_desc(sb, block_group, &gd_bh); 4578ce89f46cSAneesh Kumar K.V if (!gdp) { 4579ce89f46cSAneesh Kumar K.V err = -EIO; 4580c9de560dSAlex Tomas goto error_return; 4581ce89f46cSAneesh Kumar K.V } 4582c9de560dSAlex Tomas 4583c9de560dSAlex Tomas if (in_range(ext4_block_bitmap(sb, gdp), block, count) || 4584c9de560dSAlex Tomas in_range(ext4_inode_bitmap(sb, gdp), block, count) || 4585c9de560dSAlex Tomas in_range(block, ext4_inode_table(sb, gdp), 4586c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group) || 4587c9de560dSAlex Tomas in_range(block + count - 1, ext4_inode_table(sb, gdp), 4588c9de560dSAlex Tomas EXT4_SB(sb)->s_itb_per_group)) { 4589c9de560dSAlex Tomas 459012062dddSEric Sandeen ext4_error(sb, "Freeing blocks in system zone - " 45910610b6e9STheodore Ts'o "Block = %llu, count = %lu", block, count); 4592519deca0SAneesh Kumar K.V /* err = 0. ext4_std_error should be a no op */ 4593519deca0SAneesh Kumar K.V goto error_return; 4594c9de560dSAlex Tomas } 4595c9de560dSAlex Tomas 4596c9de560dSAlex Tomas BUFFER_TRACE(bitmap_bh, "getting write access"); 4597c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, bitmap_bh); 4598c9de560dSAlex Tomas if (err) 4599c9de560dSAlex Tomas goto error_return; 4600c9de560dSAlex Tomas 4601c9de560dSAlex Tomas /* 4602c9de560dSAlex Tomas * We are about to modify some metadata. Call the journal APIs 4603c9de560dSAlex Tomas * to unshare ->b_data if a currently-committing transaction is 4604c9de560dSAlex Tomas * using it 4605c9de560dSAlex Tomas */ 4606c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "get_write_access"); 4607c9de560dSAlex Tomas err = ext4_journal_get_write_access(handle, gd_bh); 4608c9de560dSAlex Tomas if (err) 4609c9de560dSAlex Tomas goto error_return; 4610c9de560dSAlex Tomas #ifdef AGGRESSIVE_CHECK 4611c9de560dSAlex Tomas { 4612c9de560dSAlex Tomas int i; 4613c9de560dSAlex Tomas for (i = 0; i < count; i++) 4614c9de560dSAlex Tomas BUG_ON(!mb_test_bit(bit + i, bitmap_bh->b_data)); 4615c9de560dSAlex Tomas } 4616c9de560dSAlex Tomas #endif 46173e1e5f50SEric Sandeen trace_ext4_mballoc_free(sb, inode, block_group, bit, count); 4618c9de560dSAlex Tomas 4619920313a7SAneesh Kumar K.V err = ext4_mb_load_buddy(sb, block_group, &e4b); 4620920313a7SAneesh Kumar K.V if (err) 4621920313a7SAneesh Kumar K.V goto error_return; 4622e6362609STheodore Ts'o 4623e6362609STheodore Ts'o if ((flags & EXT4_FREE_BLOCKS_METADATA) && ext4_handle_valid(handle)) { 46247a2fcbf7SAneesh Kumar K.V struct ext4_free_data *new_entry; 46257a2fcbf7SAneesh Kumar K.V /* 46267a2fcbf7SAneesh Kumar K.V * blocks being freed are metadata. these blocks shouldn't 46277a2fcbf7SAneesh Kumar K.V * be used until this transaction is committed 46287a2fcbf7SAneesh Kumar K.V */ 46297a2fcbf7SAneesh Kumar K.V new_entry = kmem_cache_alloc(ext4_free_ext_cachep, GFP_NOFS); 4630b72143abSTheodore Ts'o if (!new_entry) { 4631b72143abSTheodore Ts'o err = -ENOMEM; 4632b72143abSTheodore Ts'o goto error_return; 4633b72143abSTheodore Ts'o } 46347a2fcbf7SAneesh Kumar K.V new_entry->start_blk = bit; 46357a2fcbf7SAneesh Kumar K.V new_entry->group = block_group; 46367a2fcbf7SAneesh Kumar K.V new_entry->count = count; 46377a2fcbf7SAneesh Kumar K.V new_entry->t_tid = handle->h_transaction->t_tid; 4638955ce5f5SAneesh Kumar K.V 46397a2fcbf7SAneesh Kumar K.V ext4_lock_group(sb, block_group); 4640955ce5f5SAneesh Kumar K.V mb_clear_bits(bitmap_bh->b_data, bit, count); 46417a2fcbf7SAneesh Kumar K.V ext4_mb_free_metadata(handle, &e4b, new_entry); 4642c9de560dSAlex Tomas } else { 46437a2fcbf7SAneesh Kumar K.V /* need to update group_info->bb_free and bitmap 46447a2fcbf7SAneesh Kumar K.V * with group lock held. generate_buddy look at 46457a2fcbf7SAneesh Kumar K.V * them with group lock_held 46467a2fcbf7SAneesh Kumar K.V */ 4647955ce5f5SAneesh Kumar K.V ext4_lock_group(sb, block_group); 4648955ce5f5SAneesh Kumar K.V mb_clear_bits(bitmap_bh->b_data, bit, count); 46497e5a8cddSShen Feng mb_free_blocks(inode, &e4b, bit, count); 4650c9de560dSAlex Tomas } 4651c9de560dSAlex Tomas 4652560671a0SAneesh Kumar K.V ret = ext4_free_blks_count(sb, gdp) + count; 4653560671a0SAneesh Kumar K.V ext4_free_blks_set(sb, gdp, ret); 4654c9de560dSAlex Tomas gdp->bg_checksum = ext4_group_desc_csum(sbi, block_group, gdp); 4655955ce5f5SAneesh Kumar K.V ext4_unlock_group(sb, block_group); 4656c9de560dSAlex Tomas percpu_counter_add(&sbi->s_freeblocks_counter, count); 4657c9de560dSAlex Tomas 4658772cb7c8SJose R. Santos if (sbi->s_log_groups_per_flex) { 4659772cb7c8SJose R. Santos ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 46609f24e420STheodore Ts'o atomic_add(count, &sbi->s_flex_groups[flex_group].free_blocks); 4661772cb7c8SJose R. Santos } 4662772cb7c8SJose R. Santos 4663e39e07fdSJing Zhang ext4_mb_unload_buddy(&e4b); 4664c9de560dSAlex Tomas 466544338711STheodore Ts'o freed += count; 4666c9de560dSAlex Tomas 46677a2fcbf7SAneesh Kumar K.V /* We dirtied the bitmap block */ 46687a2fcbf7SAneesh Kumar K.V BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 46697a2fcbf7SAneesh Kumar K.V err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 46707a2fcbf7SAneesh Kumar K.V 4671c9de560dSAlex Tomas /* And the group descriptor block */ 4672c9de560dSAlex Tomas BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 46730390131bSFrank Mayhar ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 4674c9de560dSAlex Tomas if (!err) 4675c9de560dSAlex Tomas err = ret; 4676c9de560dSAlex Tomas 4677c9de560dSAlex Tomas if (overflow && !err) { 4678c9de560dSAlex Tomas block += count; 4679c9de560dSAlex Tomas count = overflow; 4680c9de560dSAlex Tomas put_bh(bitmap_bh); 4681c9de560dSAlex Tomas goto do_more; 4682c9de560dSAlex Tomas } 4683a0375156STheodore Ts'o ext4_mark_super_dirty(sb); 4684c9de560dSAlex Tomas error_return: 46857132de74SMaxim Patlasov if (freed && !(flags & EXT4_FREE_BLOCKS_NO_QUOT_UPDATE)) 46865dd4056dSChristoph Hellwig dquot_free_block(inode, freed); 4687c9de560dSAlex Tomas brelse(bitmap_bh); 4688c9de560dSAlex Tomas ext4_std_error(sb, err); 4689c9de560dSAlex Tomas return; 4690c9de560dSAlex Tomas } 46917360d173SLukas Czerner 46927360d173SLukas Czerner /** 46930529155eSYongqiang Yang * ext4_group_add_blocks() -- Add given blocks to an existing group 46942846e820SAmir Goldstein * @handle: handle to this transaction 46952846e820SAmir Goldstein * @sb: super block 46962846e820SAmir Goldstein * @block: start physcial block to add to the block group 46972846e820SAmir Goldstein * @count: number of blocks to free 46982846e820SAmir Goldstein * 4699e73a347bSAmir Goldstein * This marks the blocks as free in the bitmap and buddy. 47002846e820SAmir Goldstein */ 4701cc7365dfSYongqiang Yang int ext4_group_add_blocks(handle_t *handle, struct super_block *sb, 47022846e820SAmir Goldstein ext4_fsblk_t block, unsigned long count) 47032846e820SAmir Goldstein { 47042846e820SAmir Goldstein struct buffer_head *bitmap_bh = NULL; 47052846e820SAmir Goldstein struct buffer_head *gd_bh; 47062846e820SAmir Goldstein ext4_group_t block_group; 47072846e820SAmir Goldstein ext4_grpblk_t bit; 47082846e820SAmir Goldstein unsigned int i; 47092846e820SAmir Goldstein struct ext4_group_desc *desc; 47102846e820SAmir Goldstein struct ext4_sb_info *sbi = EXT4_SB(sb); 4711e73a347bSAmir Goldstein struct ext4_buddy e4b; 47122846e820SAmir Goldstein int err = 0, ret, blk_free_count; 47132846e820SAmir Goldstein ext4_grpblk_t blocks_freed; 47142846e820SAmir Goldstein 47152846e820SAmir Goldstein ext4_debug("Adding block(s) %llu-%llu\n", block, block + count - 1); 47162846e820SAmir Goldstein 47174740b830SYongqiang Yang if (count == 0) 47184740b830SYongqiang Yang return 0; 47194740b830SYongqiang Yang 47202846e820SAmir Goldstein ext4_get_group_no_and_offset(sb, block, &block_group, &bit); 47212846e820SAmir Goldstein /* 47222846e820SAmir Goldstein * Check to see if we are freeing blocks across a group 47232846e820SAmir Goldstein * boundary. 47242846e820SAmir Goldstein */ 4725cc7365dfSYongqiang Yang if (bit + count > EXT4_BLOCKS_PER_GROUP(sb)) { 4726cc7365dfSYongqiang Yang ext4_warning(sb, "too much blocks added to group %u\n", 4727cc7365dfSYongqiang Yang block_group); 4728cc7365dfSYongqiang Yang err = -EINVAL; 47292846e820SAmir Goldstein goto error_return; 4730cc7365dfSYongqiang Yang } 47312cd05cc3STheodore Ts'o 47322846e820SAmir Goldstein bitmap_bh = ext4_read_block_bitmap(sb, block_group); 4733cc7365dfSYongqiang Yang if (!bitmap_bh) { 4734cc7365dfSYongqiang Yang err = -EIO; 47352846e820SAmir Goldstein goto error_return; 4736cc7365dfSYongqiang Yang } 4737cc7365dfSYongqiang Yang 47382846e820SAmir Goldstein desc = ext4_get_group_desc(sb, block_group, &gd_bh); 4739cc7365dfSYongqiang Yang if (!desc) { 4740cc7365dfSYongqiang Yang err = -EIO; 47412846e820SAmir Goldstein goto error_return; 4742cc7365dfSYongqiang Yang } 47432846e820SAmir Goldstein 47442846e820SAmir Goldstein if (in_range(ext4_block_bitmap(sb, desc), block, count) || 47452846e820SAmir Goldstein in_range(ext4_inode_bitmap(sb, desc), block, count) || 47462846e820SAmir Goldstein in_range(block, ext4_inode_table(sb, desc), sbi->s_itb_per_group) || 47472846e820SAmir Goldstein in_range(block + count - 1, ext4_inode_table(sb, desc), 47482846e820SAmir Goldstein sbi->s_itb_per_group)) { 47492846e820SAmir Goldstein ext4_error(sb, "Adding blocks in system zones - " 47502846e820SAmir Goldstein "Block = %llu, count = %lu", 47512846e820SAmir Goldstein block, count); 4752cc7365dfSYongqiang Yang err = -EINVAL; 47532846e820SAmir Goldstein goto error_return; 47542846e820SAmir Goldstein } 47552846e820SAmir Goldstein 47562cd05cc3STheodore Ts'o BUFFER_TRACE(bitmap_bh, "getting write access"); 47572cd05cc3STheodore Ts'o err = ext4_journal_get_write_access(handle, bitmap_bh); 47582846e820SAmir Goldstein if (err) 47592846e820SAmir Goldstein goto error_return; 47602846e820SAmir Goldstein 47612846e820SAmir Goldstein /* 47622846e820SAmir Goldstein * We are about to modify some metadata. Call the journal APIs 47632846e820SAmir Goldstein * to unshare ->b_data if a currently-committing transaction is 47642846e820SAmir Goldstein * using it 47652846e820SAmir Goldstein */ 47662846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "get_write_access"); 47672846e820SAmir Goldstein err = ext4_journal_get_write_access(handle, gd_bh); 47682846e820SAmir Goldstein if (err) 47692846e820SAmir Goldstein goto error_return; 4770e73a347bSAmir Goldstein 47712846e820SAmir Goldstein for (i = 0, blocks_freed = 0; i < count; i++) { 47722846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "clear bit"); 4773e73a347bSAmir Goldstein if (!mb_test_bit(bit + i, bitmap_bh->b_data)) { 47742846e820SAmir Goldstein ext4_error(sb, "bit already cleared for block %llu", 47752846e820SAmir Goldstein (ext4_fsblk_t)(block + i)); 47762846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "bit already cleared"); 47772846e820SAmir Goldstein } else { 47782846e820SAmir Goldstein blocks_freed++; 47792846e820SAmir Goldstein } 47802846e820SAmir Goldstein } 4781e73a347bSAmir Goldstein 4782e73a347bSAmir Goldstein err = ext4_mb_load_buddy(sb, block_group, &e4b); 4783e73a347bSAmir Goldstein if (err) 4784e73a347bSAmir Goldstein goto error_return; 4785e73a347bSAmir Goldstein 4786e73a347bSAmir Goldstein /* 4787e73a347bSAmir Goldstein * need to update group_info->bb_free and bitmap 4788e73a347bSAmir Goldstein * with group lock held. generate_buddy look at 4789e73a347bSAmir Goldstein * them with group lock_held 4790e73a347bSAmir Goldstein */ 47912846e820SAmir Goldstein ext4_lock_group(sb, block_group); 4792e73a347bSAmir Goldstein mb_clear_bits(bitmap_bh->b_data, bit, count); 4793e73a347bSAmir Goldstein mb_free_blocks(NULL, &e4b, bit, count); 47942846e820SAmir Goldstein blk_free_count = blocks_freed + ext4_free_blks_count(sb, desc); 47952846e820SAmir Goldstein ext4_free_blks_set(sb, desc, blk_free_count); 47962846e820SAmir Goldstein desc->bg_checksum = ext4_group_desc_csum(sbi, block_group, desc); 47972846e820SAmir Goldstein ext4_unlock_group(sb, block_group); 47982846e820SAmir Goldstein percpu_counter_add(&sbi->s_freeblocks_counter, blocks_freed); 47992846e820SAmir Goldstein 48002846e820SAmir Goldstein if (sbi->s_log_groups_per_flex) { 48012846e820SAmir Goldstein ext4_group_t flex_group = ext4_flex_group(sbi, block_group); 48022846e820SAmir Goldstein atomic_add(blocks_freed, 48032846e820SAmir Goldstein &sbi->s_flex_groups[flex_group].free_blocks); 48042846e820SAmir Goldstein } 4805e73a347bSAmir Goldstein 4806e73a347bSAmir Goldstein ext4_mb_unload_buddy(&e4b); 48072846e820SAmir Goldstein 48082846e820SAmir Goldstein /* We dirtied the bitmap block */ 48092846e820SAmir Goldstein BUFFER_TRACE(bitmap_bh, "dirtied bitmap block"); 48102846e820SAmir Goldstein err = ext4_handle_dirty_metadata(handle, NULL, bitmap_bh); 48112846e820SAmir Goldstein 48122846e820SAmir Goldstein /* And the group descriptor block */ 48132846e820SAmir Goldstein BUFFER_TRACE(gd_bh, "dirtied group descriptor block"); 48142846e820SAmir Goldstein ret = ext4_handle_dirty_metadata(handle, NULL, gd_bh); 48152846e820SAmir Goldstein if (!err) 48162846e820SAmir Goldstein err = ret; 48172846e820SAmir Goldstein 48182846e820SAmir Goldstein error_return: 48192846e820SAmir Goldstein brelse(bitmap_bh); 48202846e820SAmir Goldstein ext4_std_error(sb, err); 4821cc7365dfSYongqiang Yang return err; 48222846e820SAmir Goldstein } 48232846e820SAmir Goldstein 48242846e820SAmir Goldstein /** 48257360d173SLukas Czerner * ext4_trim_extent -- function to TRIM one single free extent in the group 48267360d173SLukas Czerner * @sb: super block for the file system 48277360d173SLukas Czerner * @start: starting block of the free extent in the alloc. group 48287360d173SLukas Czerner * @count: number of blocks to TRIM 48297360d173SLukas Czerner * @group: alloc. group we are working with 48307360d173SLukas Czerner * @e4b: ext4 buddy for the group 48317360d173SLukas Czerner * 48327360d173SLukas Czerner * Trim "count" blocks starting at "start" in the "group". To assure that no 48337360d173SLukas Czerner * one will allocate those blocks, mark it as used in buddy bitmap. This must 48347360d173SLukas Czerner * be called with under the group lock. 48357360d173SLukas Czerner */ 4836d9f34504STheodore Ts'o static void ext4_trim_extent(struct super_block *sb, int start, int count, 48377360d173SLukas Czerner ext4_group_t group, struct ext4_buddy *e4b) 48387360d173SLukas Czerner { 48397360d173SLukas Czerner struct ext4_free_extent ex; 48407360d173SLukas Czerner 4841b3d4c2b1STao Ma trace_ext4_trim_extent(sb, group, start, count); 4842b3d4c2b1STao Ma 48437360d173SLukas Czerner assert_spin_locked(ext4_group_lock_ptr(sb, group)); 48447360d173SLukas Czerner 48457360d173SLukas Czerner ex.fe_start = start; 48467360d173SLukas Czerner ex.fe_group = group; 48477360d173SLukas Czerner ex.fe_len = count; 48487360d173SLukas Czerner 48497360d173SLukas Czerner /* 48507360d173SLukas Czerner * Mark blocks used, so no one can reuse them while 48517360d173SLukas Czerner * being trimmed. 48527360d173SLukas Czerner */ 48537360d173SLukas Czerner mb_mark_used(e4b, &ex); 48547360d173SLukas Czerner ext4_unlock_group(sb, group); 4855d9f34504STheodore Ts'o ext4_issue_discard(sb, group, start, count); 48567360d173SLukas Czerner ext4_lock_group(sb, group); 48577360d173SLukas Czerner mb_free_blocks(NULL, e4b, start, ex.fe_len); 48587360d173SLukas Czerner } 48597360d173SLukas Czerner 48607360d173SLukas Czerner /** 48617360d173SLukas Czerner * ext4_trim_all_free -- function to trim all free space in alloc. group 48627360d173SLukas Czerner * @sb: super block for file system 486322612283STao Ma * @group: group to be trimmed 48647360d173SLukas Czerner * @start: first group block to examine 48657360d173SLukas Czerner * @max: last group block to examine 48667360d173SLukas Czerner * @minblocks: minimum extent block count 48677360d173SLukas Czerner * 48687360d173SLukas Czerner * ext4_trim_all_free walks through group's buddy bitmap searching for free 48697360d173SLukas Czerner * extents. When the free block is found, ext4_trim_extent is called to TRIM 48707360d173SLukas Czerner * the extent. 48717360d173SLukas Czerner * 48727360d173SLukas Czerner * 48737360d173SLukas Czerner * ext4_trim_all_free walks through group's block bitmap searching for free 48747360d173SLukas Czerner * extents. When the free extent is found, mark it as used in group buddy 48757360d173SLukas Czerner * bitmap. Then issue a TRIM command on this extent and free the extent in 48767360d173SLukas Czerner * the group buddy bitmap. This is done until whole group is scanned. 48777360d173SLukas Czerner */ 48780b75a840SLukas Czerner static ext4_grpblk_t 487978944086SLukas Czerner ext4_trim_all_free(struct super_block *sb, ext4_group_t group, 488078944086SLukas Czerner ext4_grpblk_t start, ext4_grpblk_t max, 488178944086SLukas Czerner ext4_grpblk_t minblocks) 48827360d173SLukas Czerner { 48837360d173SLukas Czerner void *bitmap; 4884169ddc3eSTao Ma ext4_grpblk_t next, count = 0, free_count = 0; 488578944086SLukas Czerner struct ext4_buddy e4b; 488678944086SLukas Czerner int ret; 48877360d173SLukas Czerner 4888b3d4c2b1STao Ma trace_ext4_trim_all_free(sb, group, start, max); 4889b3d4c2b1STao Ma 489078944086SLukas Czerner ret = ext4_mb_load_buddy(sb, group, &e4b); 489178944086SLukas Czerner if (ret) { 489278944086SLukas Czerner ext4_error(sb, "Error in loading buddy " 489378944086SLukas Czerner "information for %u", group); 489478944086SLukas Czerner return ret; 489578944086SLukas Czerner } 489678944086SLukas Czerner bitmap = e4b.bd_bitmap; 489728739eeaSLukas Czerner 489828739eeaSLukas Czerner ext4_lock_group(sb, group); 48993d56b8d2STao Ma if (EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) && 49003d56b8d2STao Ma minblocks >= atomic_read(&EXT4_SB(sb)->s_last_trim_minblks)) 49013d56b8d2STao Ma goto out; 49023d56b8d2STao Ma 490378944086SLukas Czerner start = (e4b.bd_info->bb_first_free > start) ? 490478944086SLukas Czerner e4b.bd_info->bb_first_free : start; 49057360d173SLukas Czerner 49067360d173SLukas Czerner while (start < max) { 49077360d173SLukas Czerner start = mb_find_next_zero_bit(bitmap, max, start); 49087360d173SLukas Czerner if (start >= max) 49097360d173SLukas Czerner break; 49107360d173SLukas Czerner next = mb_find_next_bit(bitmap, max, start); 49117360d173SLukas Czerner 49127360d173SLukas Czerner if ((next - start) >= minblocks) { 4913d9f34504STheodore Ts'o ext4_trim_extent(sb, start, 491478944086SLukas Czerner next - start, group, &e4b); 49157360d173SLukas Czerner count += next - start; 49167360d173SLukas Czerner } 4917169ddc3eSTao Ma free_count += next - start; 49187360d173SLukas Czerner start = next + 1; 49197360d173SLukas Czerner 49207360d173SLukas Czerner if (fatal_signal_pending(current)) { 49217360d173SLukas Czerner count = -ERESTARTSYS; 49227360d173SLukas Czerner break; 49237360d173SLukas Czerner } 49247360d173SLukas Czerner 49257360d173SLukas Czerner if (need_resched()) { 49267360d173SLukas Czerner ext4_unlock_group(sb, group); 49277360d173SLukas Czerner cond_resched(); 49287360d173SLukas Czerner ext4_lock_group(sb, group); 49297360d173SLukas Czerner } 49307360d173SLukas Czerner 4931169ddc3eSTao Ma if ((e4b.bd_info->bb_free - free_count) < minblocks) 49327360d173SLukas Czerner break; 49337360d173SLukas Czerner } 49343d56b8d2STao Ma 49353d56b8d2STao Ma if (!ret) 49363d56b8d2STao Ma EXT4_MB_GRP_SET_TRIMMED(e4b.bd_info); 49373d56b8d2STao Ma out: 49387360d173SLukas Czerner ext4_unlock_group(sb, group); 493978944086SLukas Czerner ext4_mb_unload_buddy(&e4b); 49407360d173SLukas Czerner 49417360d173SLukas Czerner ext4_debug("trimmed %d blocks in the group %d\n", 49427360d173SLukas Czerner count, group); 49437360d173SLukas Czerner 49447360d173SLukas Czerner return count; 49457360d173SLukas Czerner } 49467360d173SLukas Czerner 49477360d173SLukas Czerner /** 49487360d173SLukas Czerner * ext4_trim_fs() -- trim ioctl handle function 49497360d173SLukas Czerner * @sb: superblock for filesystem 49507360d173SLukas Czerner * @range: fstrim_range structure 49517360d173SLukas Czerner * 49527360d173SLukas Czerner * start: First Byte to trim 49537360d173SLukas Czerner * len: number of Bytes to trim from start 49547360d173SLukas Czerner * minlen: minimum extent length in Bytes 49557360d173SLukas Czerner * ext4_trim_fs goes through all allocation groups containing Bytes from 49567360d173SLukas Czerner * start to start+len. For each such a group ext4_trim_all_free function 49577360d173SLukas Czerner * is invoked to trim all free space. 49587360d173SLukas Czerner */ 49597360d173SLukas Czerner int ext4_trim_fs(struct super_block *sb, struct fstrim_range *range) 49607360d173SLukas Czerner { 496178944086SLukas Czerner struct ext4_group_info *grp; 49627360d173SLukas Czerner ext4_group_t first_group, last_group; 49637360d173SLukas Czerner ext4_group_t group, ngroups = ext4_get_groups_count(sb); 49647137d7a4STheodore Ts'o ext4_grpblk_t cnt = 0, first_cluster, last_cluster; 496578944086SLukas Czerner uint64_t start, len, minlen, trimmed = 0; 49660f0a25bfSJan Kara ext4_fsblk_t first_data_blk = 49670f0a25bfSJan Kara le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block); 49687360d173SLukas Czerner int ret = 0; 49697360d173SLukas Czerner 49707360d173SLukas Czerner start = range->start >> sb->s_blocksize_bits; 49717360d173SLukas Czerner len = range->len >> sb->s_blocksize_bits; 49727360d173SLukas Czerner minlen = range->minlen >> sb->s_blocksize_bits; 49737360d173SLukas Czerner 49747137d7a4STheodore Ts'o if (unlikely(minlen > EXT4_CLUSTERS_PER_GROUP(sb))) 49757360d173SLukas Czerner return -EINVAL; 497622f10457STao Ma if (start + len <= first_data_blk) 497722f10457STao Ma goto out; 49780f0a25bfSJan Kara if (start < first_data_blk) { 49790f0a25bfSJan Kara len -= first_data_blk - start; 49800f0a25bfSJan Kara start = first_data_blk; 49810f0a25bfSJan Kara } 49827360d173SLukas Czerner 49837360d173SLukas Czerner /* Determine first and last group to examine based on start and len */ 49847360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) start, 49857137d7a4STheodore Ts'o &first_group, &first_cluster); 49867360d173SLukas Czerner ext4_get_group_no_and_offset(sb, (ext4_fsblk_t) (start + len), 49877137d7a4STheodore Ts'o &last_group, &last_cluster); 49887360d173SLukas Czerner last_group = (last_group > ngroups - 1) ? ngroups - 1 : last_group; 49897137d7a4STheodore Ts'o last_cluster = EXT4_CLUSTERS_PER_GROUP(sb); 49907360d173SLukas Czerner 49917360d173SLukas Czerner if (first_group > last_group) 49927360d173SLukas Czerner return -EINVAL; 49937360d173SLukas Czerner 49947360d173SLukas Czerner for (group = first_group; group <= last_group; group++) { 499578944086SLukas Czerner grp = ext4_get_group_info(sb, group); 499678944086SLukas Czerner /* We only do this if the grp has never been initialized */ 499778944086SLukas Czerner if (unlikely(EXT4_MB_GRP_NEED_INIT(grp))) { 499878944086SLukas Czerner ret = ext4_mb_init_group(sb, group); 499978944086SLukas Czerner if (ret) 50007360d173SLukas Czerner break; 50017360d173SLukas Czerner } 50027360d173SLukas Czerner 50030ba08517STao Ma /* 50040ba08517STao Ma * For all the groups except the last one, last block will 50050ba08517STao Ma * always be EXT4_BLOCKS_PER_GROUP(sb), so we only need to 50060ba08517STao Ma * change it for the last group in which case start + 50070ba08517STao Ma * len < EXT4_BLOCKS_PER_GROUP(sb). 50080ba08517STao Ma */ 50097137d7a4STheodore Ts'o if (first_cluster + len < EXT4_CLUSTERS_PER_GROUP(sb)) 50107137d7a4STheodore Ts'o last_cluster = first_cluster + len; 50117137d7a4STheodore Ts'o len -= last_cluster - first_cluster; 50127360d173SLukas Czerner 501378944086SLukas Czerner if (grp->bb_free >= minlen) { 50147137d7a4STheodore Ts'o cnt = ext4_trim_all_free(sb, group, first_cluster, 50157137d7a4STheodore Ts'o last_cluster, minlen); 50167360d173SLukas Czerner if (cnt < 0) { 50177360d173SLukas Czerner ret = cnt; 50187360d173SLukas Czerner break; 50197360d173SLukas Czerner } 50207360d173SLukas Czerner } 50217360d173SLukas Czerner trimmed += cnt; 50227137d7a4STheodore Ts'o first_cluster = 0; 50237360d173SLukas Czerner } 50247360d173SLukas Czerner range->len = trimmed * sb->s_blocksize; 50257360d173SLukas Czerner 50263d56b8d2STao Ma if (!ret) 50273d56b8d2STao Ma atomic_set(&EXT4_SB(sb)->s_last_trim_minblks, minlen); 50283d56b8d2STao Ma 502922f10457STao Ma out: 50307360d173SLukas Czerner return ret; 50317360d173SLukas Czerner } 5032